306 lines
12 KiB
Python
306 lines
12 KiB
Python
"""Pure text helpers: title normalization, list parsers, chunk splitting.
|
||
|
||
No state, no IO — safe to import anywhere.
|
||
"""
|
||
|
||
import json
|
||
import re
|
||
import unicodedata
|
||
|
||
_CATEGORIES = ("KERN", "WICHTIG", "REST") # only for the legacy-format reader now
|
||
|
||
|
||
def parse_facts(raw) -> dict:
|
||
"""subblocks.facts ist ein JSON-Blob aus LLM-Hand — leer/kaputt/kein dict → {}."""
|
||
try:
|
||
d = json.loads(raw) if raw else {}
|
||
except (ValueError, TypeError):
|
||
return {}
|
||
return d if isinstance(d, dict) else {}
|
||
|
||
|
||
def _norm_title(s: str) -> str:
|
||
"""Normalize a title for key comparison.
|
||
|
||
NFKC + casefold catch Unicode variants; quotes, markdown emphasis
|
||
and dash variants come out of AI output in every shape.
|
||
"""
|
||
s = unicodedata.normalize("NFKC", s)
|
||
s = re.sub(r"[`'\"<>„“”‚’«»*_]", "", s)
|
||
s = re.sub(r"[–—‐]", "-", s)
|
||
s = re.sub(r"\s+", " ", s).strip().strip(".:;").strip()
|
||
return s.casefold()
|
||
|
||
|
||
# Bracket-aware separator handling: a ' — ' (or a dash that _norm_dash would normalize) INSIDE
|
||
# ()[]{} is part of the title, not a title/description separator. Without this a spaced dash within
|
||
# a math title truncates it: 'Aε-Algorithmus (Güte 1+ε, Laufzeit O(n³)) — …' → 'Aε-Algorithmus (Güte…'.
|
||
_BRACKETS = {"(": ")", "[": "]", "{": "}"}
|
||
_CLOSERS = {v: k for k, v in _BRACKETS.items()}
|
||
|
||
|
||
def _unclosed(s: str) -> bool:
|
||
"""True if s has a dangling opening bracket ()[]{} — the symptom of a mid-bracket cut. Extra
|
||
closers (more ')' than '(') do NOT count; only an unclosed opener signals a truncated title."""
|
||
depth = 0
|
||
for c in s:
|
||
if c in _BRACKETS:
|
||
depth += 1
|
||
elif c in _CLOSERS and depth > 0:
|
||
depth -= 1
|
||
return depth > 0
|
||
|
||
|
||
def _split_top(s: str, sep: str) -> str:
|
||
"""First bracket-depth-0 segment of s split on sep; whole s if sep never occurs at depth 0."""
|
||
depth = 0
|
||
for i, c in enumerate(s):
|
||
if c in _BRACKETS:
|
||
depth += 1
|
||
elif c in _CLOSERS and depth > 0:
|
||
depth -= 1
|
||
elif depth == 0 and s.startswith(sep, i):
|
||
return s[:i]
|
||
return s
|
||
|
||
|
||
def _title(entry: str) -> str:
|
||
"""Title = text before the first TOP-LEVEL ' — '. A separator inside ()[]{} does not split
|
||
(keeps math titles intact); if the extracted title has an unclosed bracket (malformed source),
|
||
keep the whole entry rather than emit a truncated title."""
|
||
title = _split_top(entry, " — ")
|
||
if _unclosed(title):
|
||
return entry.strip() or entry
|
||
return title.strip() or entry
|
||
|
||
|
||
def clean_title(s: str) -> str:
|
||
"""Strip markdown noise from a DISPLAY title (norm keys use _norm_title).
|
||
Only clearly-markdown characters go: `**` pairs and backticks. Single `*`,
|
||
underscores and pipes stay — they are legitimate in math titles
|
||
(„2|prec, pi∈{1,2}|Cmax", „x_i", „P*")."""
|
||
s = (s or "").replace("**", "").replace("`", "")
|
||
return re.sub(r"\s+", " ", s).strip()
|
||
|
||
|
||
def _unique_title(entries: dict[int, str]) -> dict[int, str]:
|
||
"""Make titles unique (suffix " (2)", " (3)" …) so they work as keys."""
|
||
seen: dict[str, int] = {}
|
||
out: dict[int, str] = {}
|
||
for num, text in entries.items():
|
||
title = _title(text)
|
||
key = _norm_title(title)
|
||
seen[key] = seen.get(key, 0) + 1
|
||
if seen[key] > 1:
|
||
rest = text.split(" — ", 1)
|
||
text = f"{title} ({seen[key]})" + (f" — {rest[1]}" if len(rest) == 2 else "")
|
||
# a second pass isn't needed: suffixes practically never collide
|
||
out[num] = text
|
||
return out
|
||
|
||
|
||
|
||
|
||
def _title_index(entries: dict[int, str]) -> dict[str, int]:
|
||
return {_norm_title(_title(text)): num for num, text in entries.items()}
|
||
|
||
|
||
def _resolve_title(idx: dict[str, int], t: str) -> int | None:
|
||
"""Title → number; tolerates trailing descriptions ("Title — …")."""
|
||
if not isinstance(t, str):
|
||
return None
|
||
return idx.get(_norm_title(t)) or idx.get(_norm_title(_title(t)))
|
||
|
||
|
||
_DASH_SEP_RE = re.compile(r"\s*[‒–—―]\s+|\s+[‒–—―]\s*")
|
||
|
||
|
||
def _norm_dash(s: str) -> str:
|
||
"""Dash variants (en/em/figure/bar) with whitespace on AT LEAST ONE side → uniform separator ' — '.
|
||
Some models (especially non-western ones) use an en-dash "–" instead of the em-dash; without
|
||
normalization the ` — ` split fails entirely and the whole entry becomes the title. A one-sided
|
||
space ("Titel —Beschreibung" / "Titel— Beschreibung") also breaks the split and leaks the source
|
||
filename into the description — so a dash with a space on either side is repaired too. The ASCII
|
||
hyphen "-" is deliberately NOT in the class (would split "n - 1"/"3-SAT"); requiring ≥1 surrounding
|
||
space keeps glued compounds like "Backtracking—Verfahren" and number ranges like "12–15" untouched.
|
||
A qualifying dash INSIDE an open bracket ()[]{} is left untouched — it is part of the title, not a
|
||
separator (else 'Algorithmus (Güte 1+ε – O(n³)) — …' would split mid-parenthetical)."""
|
||
def _repl(m):
|
||
return m.group(0) if _unclosed(s[:m.start()]) else " — "
|
||
return _DASH_SEP_RE.sub(_repl, s)
|
||
|
||
|
||
def _parse_selection(text: str) -> dict[int, str]:
|
||
"""Parse a block list: `N. Title — short description` per line."""
|
||
entries: dict[int, str] = {}
|
||
last = None
|
||
for line in text.splitlines():
|
||
m = re.match(r"\s*(\d+)[.)]\s+(.*\S)", line)
|
||
if m:
|
||
last = int(m.group(1))
|
||
entries[last] = _norm_dash(m.group(2))
|
||
elif last is not None and line.strip():
|
||
entries[last] += " " + _norm_dash(line.strip())
|
||
return entries
|
||
|
||
|
||
def _parse_categories(text: str) -> dict[str, list[str]]:
|
||
"""Legacy-format reader: final block file with ## KERN/WICHTIG/REST sections."""
|
||
cats: dict[str, list[str]] = {}
|
||
current = None
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
m = re.match(r"#+\s*(KERN|WICHTIG|REST)\b", s, re.IGNORECASE)
|
||
if m:
|
||
current = m.group(1).upper()
|
||
cats.setdefault(current, [])
|
||
continue
|
||
m = re.match(r"(\d+)[.)]\s+(.*\S)", s)
|
||
if m and current:
|
||
cats[current].append(m.group(2))
|
||
return cats
|
||
|
||
|
||
def _load_blocks(text: str) -> dict[int, str]:
|
||
"""Load the final block file — sorted list (new) or categories (legacy format)."""
|
||
if re.search(r"^#+\s*KERN\b", text, re.IGNORECASE | re.MULTILINE):
|
||
cats = _parse_categories(text)
|
||
texts = [t for cat in _CATEGORIES for t in cats.get(cat, [])]
|
||
return {i: t for i, t in enumerate(texts, 1)}
|
||
return _parse_selection(text)
|
||
|
||
|
||
_FRAGMENT_KAPITEL_RE = re.compile(r"<!--\s*kapitel\s*:\s*(.*?)\s*-->", re.IGNORECASE)
|
||
_FRAGMENT_SECTION_RE = re.compile(r"<!--\s*section\s*:\s*(.*?)\s*-->", re.IGNORECASE)
|
||
_FRAGMENT_SUB_RE = re.compile(r"<!--\s*sub\s*:\s*(.*?)\s*-->", re.IGNORECASE)
|
||
_FRAGMENT_BAUSTEIN_RE = re.compile(r"<!--\s*block\s*:\s*(.*?)\s*-->", re.IGNORECASE)
|
||
# Two reading layers per section: compact (key sentences) + detailed (explanation).
|
||
_FRAGMENT_KOMPAKT_RE = re.compile(r"<!--\s*compact\s*-->", re.IGNORECASE)
|
||
_FRAGMENT_AUSF_RE = re.compile(r"<!--\s*ausf(?:ü|ue)hrlich\s*-->", re.IGNORECASE)
|
||
# Learning-path levels + peripheral; old difficulty values accepted for backward compatibility.
|
||
_STUFEN = ("beginner", "advanced", "expert", "peripheral", "easy", "medium", "hard")
|
||
|
||
|
||
def _parse_fragment(text: str) -> list[dict]:
|
||
"""Parse a writer file → [{kapitel, title, md, compact, anker, anker_compact, subs}].
|
||
|
||
Two reading layers per section via `<!-- compact -->` / `<!-- ausführlich -->`. Within
|
||
both, `<!-- sub: level | title -->` markers mark a block per subblock; the same
|
||
sub title in both layers is merged → `sec["subs"] = [{level, title, md, compact}]`.
|
||
Text BEFORE the first sub marker is the anchor (framing) → `anker`/`anker_compact`.
|
||
`md`/`compact` stay the FULL version (anchor + all subs) — backward compatible.
|
||
"""
|
||
sections: list[dict] = []
|
||
kapitel = None
|
||
current = None
|
||
cur_sub = None
|
||
cur_layer = "md" # default: anything without a layer marker is the detailed version
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
m = _FRAGMENT_KAPITEL_RE.match(s)
|
||
if m:
|
||
kapitel = m.group(1)
|
||
current = None
|
||
cur_sub = None
|
||
continue
|
||
m = _FRAGMENT_SECTION_RE.match(s)
|
||
if m:
|
||
current = {"chapters": kapitel, "title": m.group(1), "md": [], "compact": [],
|
||
"anker_md": [], "anker_compact": [], "_submap": {}, "_suborder": []}
|
||
cur_sub = None
|
||
cur_layer = "md"
|
||
sections.append(current)
|
||
continue
|
||
if current is not None and _FRAGMENT_KOMPAKT_RE.match(s):
|
||
cur_layer = "compact"
|
||
cur_sub = None
|
||
continue
|
||
if current is not None and _FRAGMENT_AUSF_RE.match(s):
|
||
cur_layer = "md"
|
||
cur_sub = None
|
||
continue
|
||
m = _FRAGMENT_SUB_RE.match(s)
|
||
if m and current is not None:
|
||
parts = m.group(1).split("|", 1)
|
||
level = parts[0].strip().casefold()
|
||
title = parts[1].strip() if len(parts) == 2 else ""
|
||
key = title.casefold() or f"_pos{len(current['_suborder'])}"
|
||
cur_sub = current["_submap"].get(key)
|
||
if cur_sub is None:
|
||
cur_sub = {"level": level if level in _STUFEN else "beginner", "title": title, "md": [], "compact": []}
|
||
current["_submap"][key] = cur_sub
|
||
current["_suborder"].append(key)
|
||
elif level in _STUFEN:
|
||
cur_sub["level"] = level
|
||
continue
|
||
if current is not None:
|
||
current[cur_layer].append(line)
|
||
if cur_sub is not None:
|
||
cur_sub[cur_layer].append(line)
|
||
else:
|
||
current["anker_" + cur_layer].append(line)
|
||
out: list[dict] = []
|
||
for sec in sections:
|
||
subs = []
|
||
for key in sec["_suborder"]:
|
||
sub = sec["_submap"][key]
|
||
sub["md"] = "\n".join(sub["md"]).strip()
|
||
sub["compact"] = "\n".join(sub["compact"]).strip()
|
||
if sub["md"] or sub["compact"]:
|
||
subs.append(sub)
|
||
out.append({
|
||
"chapters": sec["chapters"], "title": sec["title"],
|
||
"md": "\n".join(sec["md"]).strip(),
|
||
"compact": "\n".join(sec["compact"]).strip(),
|
||
"anchor": "\n".join(sec["anker_md"]).strip(),
|
||
"anker_compact": "\n".join(sec["anker_compact"]).strip(),
|
||
"subs": subs,
|
||
})
|
||
return out
|
||
|
||
|
||
def _parse_subblocks(text: str) -> dict[str, list[str]]:
|
||
"""Parse a subblock file → {block title: [subblock, …]} in order.
|
||
|
||
Format: `<!-- block: Title -->` followed by list lines `- Subblock`.
|
||
"""
|
||
out: dict[str, list[str]] = {}
|
||
current = None
|
||
for line in text.splitlines():
|
||
s = line.strip()
|
||
m = _FRAGMENT_BAUSTEIN_RE.match(s)
|
||
if m:
|
||
current = m.group(1).strip()
|
||
out.setdefault(current, [])
|
||
continue
|
||
if current is None:
|
||
continue
|
||
m = re.match(r"[-*]\s+(.*\S)", s)
|
||
if m:
|
||
out[current].append(m.group(1).strip())
|
||
return {k: v for k, v in out.items() if v}
|
||
|
||
|
||
def _split_chunks(chapters: list[dict], n: int) -> list[list[dict]]:
|
||
"""Split chapters into up to n contiguous chunks, balanced by section count."""
|
||
n = max(1, min(n, len(chapters)))
|
||
chunks: list[list[dict]] = []
|
||
current: list[dict] = []
|
||
count = 0
|
||
remaining_total = sum(len(c["nums"]) for c in chapters)
|
||
remaining_chunks = n
|
||
for ch in chapters:
|
||
current.append(ch)
|
||
count += len(ch["nums"])
|
||
if remaining_chunks > 1 and count >= remaining_total / remaining_chunks:
|
||
chunks.append(current)
|
||
remaining_total -= count
|
||
remaining_chunks -= 1
|
||
current = []
|
||
count = 0
|
||
if current:
|
||
chunks.append(current)
|
||
return chunks
|
||
|
||
|