239 lines
9.0 KiB
Python
239 lines
9.0 KiB
Python
"""Pure text helpers: title normalization, list parsers, chunk splitting.
|
||
|
||
No state, no IO — safe to import anywhere.
|
||
"""
|
||
|
||
import re
|
||
import unicodedata
|
||
|
||
_CATEGORIES = ("KERN", "WICHTIG", "REST") # only for the legacy-format reader now
|
||
|
||
|
||
def _norm_title(s: str) -> str:
|
||
"""Normalize a title for key comparison.
|
||
|
||
NFKC + casefold catch Unicode variants; quotes, markdown emphasis
|
||
and dash variants come out of AI output in every shape.
|
||
"""
|
||
s = unicodedata.normalize("NFKC", s)
|
||
s = re.sub(r"[`'\"<>„“”‚’«»*_]", "", s)
|
||
s = re.sub(r"[–—‐]", "-", s)
|
||
s = re.sub(r"\s+", " ", s).strip().strip(".:;").strip()
|
||
return s.casefold()
|
||
|
||
|
||
def _title(entry: str) -> str:
|
||
return entry.split(" — ")[0].strip() or entry
|
||
|
||
|
||
def _unique_title(entries: dict[int, str]) -> dict[int, str]:
|
||
"""Make titles unique (suffix " (2)", " (3)" …) so they work as keys."""
|
||
seen: dict[str, int] = {}
|
||
out: dict[int, str] = {}
|
||
for num, text in entries.items():
|
||
title = _title(text)
|
||
key = _norm_title(title)
|
||
seen[key] = seen.get(key, 0) + 1
|
||
if seen[key] > 1:
|
||
rest = text.split(" — ", 1)
|
||
text = f"{title} ({seen[key]})" + (f" — {rest[1]}" if len(rest) == 2 else "")
|
||
# a second pass isn't needed: suffixes practically never collide
|
||
out[num] = text
|
||
return out
|
||
|
||
|
||
|
||
|
||
def _title_index(entries: dict[int, str]) -> dict[str, int]:
|
||
return {_norm_title(_title(text)): num for num, text in entries.items()}
|
||
|
||
|
||
def _resolve_title(idx: dict[str, int], t: str) -> int | None:
|
||
"""Title → number; tolerates trailing descriptions ("Title — …")."""
|
||
if not isinstance(t, str):
|
||
return None
|
||
return idx.get(_norm_title(t)) or idx.get(_norm_title(_title(t)))
|
||
|
||
|
||
def _norm_dash(s: str) -> str:
|
||
"""Space-surrounded dash variants (en/em/figure/bar/hyphen) → uniform separator ' — '.
|
||
Some models (especially non-western ones) use an en-dash "–" instead of the em-dash; without
|
||
normalization the ` — ` split fails entirely and the whole entry becomes the title.
|
||
The ASCII hyphen "-" is left untouched (otherwise it would split formulas like "n - 1")."""
|
||
return re.sub(r"\s+[‒–—―‐]\s+", " — ", s)
|
||
|
||
|
||
def _parse_selection(text: str) -> dict[int, str]:
|
||
"""Parse a block list: `N. Title — short description` per line."""
|
||
entries: dict[int, str] = {}
|
||
last = None
|
||
for line in text.splitlines():
|
||
m = re.match(r"\s*(\d+)[.)]\s+(.*\S)", line)
|
||
if m:
|
||
last = int(m.group(1))
|
||
entries[last] = _norm_dash(m.group(2))
|
||
elif last is not None and line.strip():
|
||
entries[last] += " " + _norm_dash(line.strip())
|
||
return entries
|
||
|
||
|
||
def _parse_categories(text: str) -> dict[str, list[str]]:
|
||
"""Legacy-format reader: final block file with ## KERN/WICHTIG/REST sections."""
|
||
cats: dict[str, list[str]] = {}
|
||
current = None
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
m = re.match(r"#+\s*(KERN|WICHTIG|REST)\b", s, re.IGNORECASE)
|
||
if m:
|
||
current = m.group(1).upper()
|
||
cats.setdefault(current, [])
|
||
continue
|
||
m = re.match(r"(\d+)[.)]\s+(.*\S)", s)
|
||
if m and current:
|
||
cats[current].append(m.group(2))
|
||
return cats
|
||
|
||
|
||
def _load_blocks(text: str) -> dict[int, str]:
|
||
"""Load the final block file — sorted list (new) or categories (legacy format)."""
|
||
if re.search(r"^#+\s*KERN\b", text, re.IGNORECASE | re.MULTILINE):
|
||
cats = _parse_categories(text)
|
||
texts = [t for cat in _CATEGORIES for t in cats.get(cat, [])]
|
||
return {i: t for i, t in enumerate(texts, 1)}
|
||
return _parse_selection(text)
|
||
|
||
|
||
_FRAGMENT_KAPITEL_RE = re.compile(r"<!--\s*kapitel\s*:\s*(.*?)\s*-->", re.IGNORECASE)
|
||
_FRAGMENT_SECTION_RE = re.compile(r"<!--\s*section\s*:\s*(.*?)\s*-->", re.IGNORECASE)
|
||
_FRAGMENT_SUB_RE = re.compile(r"<!--\s*sub\s*:\s*(.*?)\s*-->", re.IGNORECASE)
|
||
_FRAGMENT_BAUSTEIN_RE = re.compile(r"<!--\s*block\s*:\s*(.*?)\s*-->", re.IGNORECASE)
|
||
# Two reading layers per section: compact (key sentences) + detailed (explanation).
|
||
_FRAGMENT_KOMPAKT_RE = re.compile(r"<!--\s*compact\s*-->", re.IGNORECASE)
|
||
_FRAGMENT_AUSF_RE = re.compile(r"<!--\s*ausf(?:ü|ue)hrlich\s*-->", re.IGNORECASE)
|
||
# Learning-path levels + peripheral; old difficulty values accepted for backward compatibility.
|
||
_STUFEN = ("beginner", "advanced", "expert", "peripheral", "easy", "medium", "hard")
|
||
|
||
|
||
def _parse_fragment(text: str) -> list[dict]:
|
||
"""Parse a writer file → [{kapitel, title, md, compact, anker, anker_compact, subs}].
|
||
|
||
Two reading layers per section via `<!-- compact -->` / `<!-- ausführlich -->`. Within
|
||
both, `<!-- sub: level | title -->` markers mark a block per subblock; the same
|
||
sub title in both layers is merged → `sec["subs"] = [{level, title, md, compact}]`.
|
||
Text BEFORE the first sub marker is the anchor (framing) → `anker`/`anker_compact`.
|
||
`md`/`compact` stay the FULL version (anchor + all subs) — backward compatible.
|
||
"""
|
||
sections: list[dict] = []
|
||
kapitel = None
|
||
current = None
|
||
cur_sub = None
|
||
cur_layer = "md" # default: anything without a layer marker is the detailed version
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
m = _FRAGMENT_KAPITEL_RE.match(s)
|
||
if m:
|
||
kapitel = m.group(1)
|
||
current = None
|
||
cur_sub = None
|
||
continue
|
||
m = _FRAGMENT_SECTION_RE.match(s)
|
||
if m:
|
||
current = {"chapters": kapitel, "title": m.group(1), "md": [], "compact": [],
|
||
"anker_md": [], "anker_compact": [], "_submap": {}, "_suborder": []}
|
||
cur_sub = None
|
||
cur_layer = "md"
|
||
sections.append(current)
|
||
continue
|
||
if current is not None and _FRAGMENT_KOMPAKT_RE.match(s):
|
||
cur_layer = "compact"
|
||
cur_sub = None
|
||
continue
|
||
if current is not None and _FRAGMENT_AUSF_RE.match(s):
|
||
cur_layer = "md"
|
||
cur_sub = None
|
||
continue
|
||
m = _FRAGMENT_SUB_RE.match(s)
|
||
if m and current is not None:
|
||
parts = m.group(1).split("|", 1)
|
||
level = parts[0].strip().casefold()
|
||
title = parts[1].strip() if len(parts) == 2 else ""
|
||
key = title.casefold() or f"_pos{len(current['_suborder'])}"
|
||
cur_sub = current["_submap"].get(key)
|
||
if cur_sub is None:
|
||
cur_sub = {"level": level if level in _STUFEN else "beginner", "title": title, "md": [], "compact": []}
|
||
current["_submap"][key] = cur_sub
|
||
current["_suborder"].append(key)
|
||
elif level in _STUFEN:
|
||
cur_sub["level"] = level
|
||
continue
|
||
if current is not None:
|
||
current[cur_layer].append(line)
|
||
if cur_sub is not None:
|
||
cur_sub[cur_layer].append(line)
|
||
else:
|
||
current["anker_" + cur_layer].append(line)
|
||
out: list[dict] = []
|
||
for sec in sections:
|
||
subs = []
|
||
for key in sec["_suborder"]:
|
||
sub = sec["_submap"][key]
|
||
sub["md"] = "\n".join(sub["md"]).strip()
|
||
sub["compact"] = "\n".join(sub["compact"]).strip()
|
||
if sub["md"] or sub["compact"]:
|
||
subs.append(sub)
|
||
out.append({
|
||
"chapters": sec["chapters"], "title": sec["title"],
|
||
"md": "\n".join(sec["md"]).strip(),
|
||
"compact": "\n".join(sec["compact"]).strip(),
|
||
"anchor": "\n".join(sec["anker_md"]).strip(),
|
||
"anker_compact": "\n".join(sec["anker_compact"]).strip(),
|
||
"subs": subs,
|
||
})
|
||
return out
|
||
|
||
|
||
def _parse_subblocks(text: str) -> dict[str, list[str]]:
|
||
"""Parse a subblock file → {block title: [subblock, …]} in order.
|
||
|
||
Format: `<!-- block: Title -->` followed by list lines `- Subblock`.
|
||
"""
|
||
out: dict[str, list[str]] = {}
|
||
current = None
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
m = _FRAGMENT_BAUSTEIN_RE.match(s)
|
||
if m:
|
||
current = m.group(1).strip()
|
||
out.setdefault(current, [])
|
||
continue
|
||
if current is None:
|
||
continue
|
||
m = re.match(r"[-*]\s+(.*\S)", s)
|
||
if m:
|
||
out[current].append(m.group(1).strip())
|
||
return {k: v for k, v in out.items() if v}
|
||
|
||
|
||
def _split_chunks(chapters: list[dict], n: int) -> list[list[dict]]:
|
||
"""Split chapters into up to n contiguous chunks, balanced by section count."""
|
||
n = max(1, min(n, len(chapters)))
|
||
chunks: list[list[dict]] = []
|
||
current: list[dict] = []
|
||
count = 0
|
||
remaining_total = sum(len(c["nums"]) for c in chapters)
|
||
remaining_chunks = n
|
||
for ch in chapters:
|
||
current.append(ch)
|
||
count += len(ch["nums"])
|
||
if remaining_chunks > 1 and count >= remaining_total / remaining_chunks:
|
||
chunks.append(current)
|
||
remaining_total -= count
|
||
remaining_chunks -= 1
|
||
current = []
|
||
count = 0
|
||
if current:
|
||
chunks.append(current)
|
||
return chunks
|
||
|
||
|