marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-06-30 00:14:18 +02:00
parent 3e3559aa8f
commit c794fcaccf
152 changed files with 9485 additions and 9583 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
backend/textkit.py
View File

@@ -1,19 +1,19 @@
"""Reine Text-Helfer: Titel-Normalisierung, Listen-Parser, Chunk-Aufteilung.
"""Pure text helpers: title normalization, list parsers, chunk splitting.
Kein Zustand, keine IO — überall gefahrlos importierbar.
No state, no IO — safe to import anywhere.
"""
import re
import unicodedata
_CATEGORIES = ("KERN", "WICHTIG", "REST") # nur noch für den Altformat-Reader
_CATEGORIES = ("KERN", "WICHTIG", "REST") # only for the legacy-format reader now
def _norm_titel(s: str) -> str:
"""Normalisiert einen Titel für den Schlüssel-Vergleich.
def _norm_title(s: str) -> str:
"""Normalize a title for key comparison.
NFKC + casefold fangen Unicode-Varianten; Anführungszeichen, Markdown-
Emphasis und Dash-Varianten kommen aus KI-Output in allen Spielarten.
NFKC + casefold catch Unicode variants; quotes, markdown emphasis
and dash variants come out of AI output in every shape.
"""
s = unicodedata.normalize("NFKC", s)
s = re.sub(r"[`'\"<>„“”‚’«»*_]", "", s)
@@ -22,49 +22,49 @@ def _norm_titel(s: str) -> str:
return s.casefold()
def _titel(entry: str) -> str:
def _title(entry: str) -> str:
return entry.split(" — ")[0].strip() or entry
def _eindeutige_titel(entries: dict[int, str]) -> dict[int, str]:
"""Macht Titel eindeutig (Suffix " (2)", " (3)" …), damit sie als Schlüssel taugen."""
def _unique_title(entries: dict[int, str]) -> dict[int, str]:
"""Make titles unique (suffix " (2)", " (3)" …) so they work as keys."""
seen: dict[str, int] = {}
out: dict[int, str] = {}
for num, text in entries.items():
titel = _titel(text)
key = _norm_titel(titel)
title = _title(text)
key = _norm_title(title)
seen[key] = seen.get(key, 0) + 1
if seen[key] > 1:
rest = text.split(" — ", 1)
text = f"{titel} ({seen[key]})" + (f" — {rest[1]}" if len(rest) == 2 else "")
# zweiter Durchlauf nicht nötig: Suffixe kollidieren praktisch nicht
text = f"{title} ({seen[key]})" + (f" — {rest[1]}" if len(rest) == 2 else "")
# a second pass isn't needed: suffixes practically never collide
out[num] = text
return out
def _titel_index(entries: dict[int, str]) -> dict[str, int]:
return {_norm_titel(_titel(text)): num for num, text in entries.items()}
def _title_index(entries: dict[int, str]) -> dict[str, int]:
return {_norm_title(_title(text)): num for num, text in entries.items()}
def _titel_aufloesen(idx: dict[str, int], t: str) -> int | None:
"""Titel → Nummer; toleriert mitgeschleppte Beschreibungen ("Titel — …")."""
def _resolve_title(idx: dict[str, int], t: str) -> int | None:
"""Title → number; tolerates trailing descriptions ("Title — …")."""
if not isinstance(t, str):
return None
return idx.get(_norm_titel(t)) or idx.get(_norm_titel(_titel(t)))
return idx.get(_norm_title(t)) or idx.get(_norm_title(_title(t)))
def _norm_dash(s: str) -> str:
"""Space-umgebene Dash-Varianten (en/em/figure/bar/hyphen) → einheitlicher Trenner ' — '.
Manche Modelle (v.a. nicht-westliche) setzen statt des Em-Dashs einen En-Dash „–"; ohne
Normalisierung scheitert der ` — `-Split komplett und der ganze Eintrag wird zum Titel.
ASCII-Bindestrich „-" bleibt unangetastet (sonst zerlegt es Formeln wie „n - 1")."""
"""Space-surrounded dash variants (en/em/figure/bar/hyphen) → uniform separator ' — '.
Some models (especially non-western ones) use an en-dash "–" instead of the em-dash; without
normalization the ` — ` split fails entirely and the whole entry becomes the title.
The ASCII hyphen "-" is left untouched (otherwise it would split formulas like "n - 1")."""
return re.sub(r"\s+[‒–—―‐]\s+", " — ", s)
def _parse_auswahl(text: str) -> dict[int, str]:
"""Parst eine Baustein-Liste: `N. Titel — Kurzbeschreibung` pro Zeile."""
def _parse_selection(text: str) -> dict[int, str]:
"""Parse a block list: `N. Title — short description` per line."""
entries: dict[int, str] = {}
last = None
for line in text.splitlines():
@@ -77,8 +77,8 @@ def _parse_auswahl(text: str) -> dict[int, str]:
return entries
def _parse_kategorien(text: str) -> dict[str, list[str]]:
"""Altformat-Reader: finale Baustein-Datei mit ## KERN/WICHTIG/REST-Abschnitten."""
def _parse_categories(text: str) -> dict[str, list[str]]:
"""Legacy-format reader: final block file with ## KERN/WICHTIG/REST sections."""
cats: dict[str, list[str]] = {}
current = None
for line in text.splitlines():
@@ -94,40 +94,40 @@ def _parse_kategorien(text: str) -> dict[str, list[str]]:
return cats
def _lade_bausteine(text: str) -> dict[int, str]:
"""Lädt die finale Baustein-Datei — sortierte Liste (neu) oder Kategorien (Altformat)."""
def _load_blocks(text: str) -> dict[int, str]:
"""Load the final block file — sorted list (new) or categories (legacy format)."""
if re.search(r"^#+\s*KERN\b", text, re.IGNORECASE | re.MULTILINE):
cats = _parse_kategorien(text)
cats = _parse_categories(text)
texts = [t for cat in _CATEGORIES for t in cats.get(cat, [])]
return {i: t for i, t in enumerate(texts, 1)}
return _parse_auswahl(text)
return _parse_selection(text)
_FRAGMENT_KAPITEL_RE = re.compile(r"<!--\s*kapitel\s*:\s*(.*?)\s*-->", re.IGNORECASE)
_FRAGMENT_SECTION_RE = re.compile(r"<!--\s*section\s*:\s*(.*?)\s*-->", re.IGNORECASE)
_FRAGMENT_SUB_RE = re.compile(r"<!--\s*sub\s*:\s*(.*?)\s*-->", re.IGNORECASE)
_FRAGMENT_BAUSTEIN_RE = re.compile(r"<!--\s*baustein\s*:\s*(.*?)\s*-->", re.IGNORECASE)
# Zwei Lese-Schichten je Section: kompakt (Merksätze) + ausführlich (Erklärung).
_FRAGMENT_KOMPAKT_RE = re.compile(r"<!--\s*kompakt\s*-->", re.IGNORECASE)
_FRAGMENT_BAUSTEIN_RE = re.compile(r"<!--\s*block\s*:\s*(.*?)\s*-->", re.IGNORECASE)
# Two reading layers per section: compact (key sentences) + detailed (explanation).
_FRAGMENT_KOMPAKT_RE = re.compile(r"<!--\s*compact\s*-->", re.IGNORECASE)
_FRAGMENT_AUSF_RE = re.compile(r"<!--\s*ausf(?:ü|ue)hrlich\s*-->", re.IGNORECASE)
# Lernpfad-Stufen + Rand; alte Schwierigkeits-Werte abwärtskompatibel akzeptiert.
_STUFEN = ("anfaenger", "fortgeschritten", "experte", "rand", "einfach", "mittel", "schwer")
# Learning-path levels + peripheral; old difficulty values accepted for backward compatibility.
_STUFEN = ("beginner", "advanced", "expert", "peripheral", "easy", "medium", "hard")
def _parse_fragment(text: str) -> list[dict]:
"""Parst eine Writer-Datei → [{kapitel, titel, md, kompakt, anker, anker_kompakt, subs}].
"""Parse a writer file → [{kapitel, title, md, compact, anker, anker_compact, subs}].
Zwei Lese-Schichten je Section über `<!-- kompakt -->` / `<!-- ausführlich -->`. Innerhalb
beider markieren `<!-- sub: stufe | titel -->`-Marker je Subbaustein einen Block; gleicher
Sub-Titel in beiden Schichten wird gemergt → `sec["subs"] = [{stufe, titel, md, kompakt}]`.
Text VOR dem ersten Sub-Marker ist der Anker (Einordnung) → `anker`/`anker_kompakt`.
`md`/`kompakt` bleiben die VOLLE Fassung (Anker + alle Subs) — rückwärtskompatibel.
Two reading layers per section via `<!-- compact -->` / `<!-- ausführlich -->`. Within
both, `<!-- sub: level | title -->` markers mark a block per subblock; the same
sub title in both layers is merged → `sec["subs"] = [{level, title, md, compact}]`.
Text BEFORE the first sub marker is the anchor (framing) → `anker`/`anker_compact`.
`md`/`compact` stay the FULL version (anchor + all subs) — backward compatible.
"""
sections: list[dict] = []
kapitel = None
current = None
cur_sub = None
cur_layer = "md" # Default: alles ohne Schicht-Marker ist die ausführliche Fassung
cur_layer = "md" # default: anything without a layer marker is the detailed version
for line in text.splitlines():
s = line.strip()
m = _FRAGMENT_KAPITEL_RE.match(s)
@@ -138,14 +138,14 @@ def _parse_fragment(text: str) -> list[dict]:
continue
m = _FRAGMENT_SECTION_RE.match(s)
if m:
current = {"kapitel": kapitel, "titel": m.group(1), "md": [], "kompakt": [],
"anker_md": [], "anker_kompakt": [], "_submap": {}, "_suborder": []}
current = {"chapters": kapitel, "title": m.group(1), "md": [], "compact": [],
"anker_md": [], "anker_compact": [], "_submap": {}, "_suborder": []}
cur_sub = None
cur_layer = "md"
sections.append(current)
continue
if current is not None and _FRAGMENT_KOMPAKT_RE.match(s):
cur_layer = "kompakt"
cur_layer = "compact"
cur_sub = None
continue
if current is not None and _FRAGMENT_AUSF_RE.match(s):
@@ -154,17 +154,17 @@ def _parse_fragment(text: str) -> list[dict]:
continue
m = _FRAGMENT_SUB_RE.match(s)
if m and current is not None:
teil = m.group(1).split("|", 1)
stufe = teil[0].strip().casefold()
titel = teil[1].strip() if len(teil) == 2 else ""
key = titel.casefold() or f"_pos{len(current['_suborder'])}"
parts = m.group(1).split("|", 1)
level = parts[0].strip().casefold()
title = parts[1].strip() if len(parts) == 2 else ""
key = title.casefold() or f"_pos{len(current['_suborder'])}"
cur_sub = current["_submap"].get(key)
if cur_sub is None:
cur_sub = {"stufe": stufe if stufe in _STUFEN else "anfaenger", "titel": titel, "md": [], "kompakt": []}
cur_sub = {"level": level if level in _STUFEN else "beginner", "title": title, "md": [], "compact": []}
current["_submap"][key] = cur_sub
current["_suborder"].append(key)
elif stufe in _STUFEN:
cur_sub["stufe"] = stufe
elif level in _STUFEN:
cur_sub["level"] = level
continue
if current is not None:
current[cur_layer].append(line)
@@ -178,24 +178,24 @@ def _parse_fragment(text: str) -> list[dict]:
for key in sec["_suborder"]:
sub = sec["_submap"][key]
sub["md"] = "\n".join(sub["md"]).strip()
sub["kompakt"] = "\n".join(sub["kompakt"]).strip()
if sub["md"] or sub["kompakt"]:
sub["compact"] = "\n".join(sub["compact"]).strip()
if sub["md"] or sub["compact"]:
subs.append(sub)
out.append({
"kapitel": sec["kapitel"], "titel": sec["titel"],
"chapters": sec["chapters"], "title": sec["title"],
"md": "\n".join(sec["md"]).strip(),
"kompakt": "\n".join(sec["kompakt"]).strip(),
"anker": "\n".join(sec["anker_md"]).strip(),
"anker_kompakt": "\n".join(sec["anker_kompakt"]).strip(),
"compact": "\n".join(sec["compact"]).strip(),
"anchor": "\n".join(sec["anker_md"]).strip(),
"anker_compact": "\n".join(sec["anker_compact"]).strip(),
"subs": subs,
})
return out
def _parse_subbausteine(text: str) -> dict[str, list[str]]:
"""Parst eine Subbaustein-Datei → {Baustein-Titel: [Subbaustein, …]} in Reihenfolge.
def _parse_subblocks(text: str) -> dict[str, list[str]]:
"""Parse a subblock file → {block title: [subblock, …]} in order.
Format: `<!-- baustein: Titel -->` gefolgt von Listenzeilen `- Subbaustein`.
Format: `<!-- block: Title -->` followed by list lines `- Subblock`.
"""
out: dict[str, list[str]] = {}
current = None
@@ -215,7 +215,7 @@ def _parse_subbausteine(text: str) -> dict[str, list[str]]:
def _split_chunks(chapters: list[dict], n: int) -> list[list[dict]]:
"""Teilt Kapitel in bis zu n zusammenhängende Chunks, balanciert nach Section-Anzahl."""
"""Split chapters into up to n contiguous chunks, balanced by section count."""
n = max(1, min(n, len(chapters)))
chunks: list[list[dict]] = []
current: list[dict] = []