diff --git a/backend/blocks.py b/backend/blocks.py index 0418831..21bbae0 100644 --- a/backend/blocks.py +++ b/backend/blocks.py @@ -18,12 +18,13 @@ import re import shutil import subprocess import time +import unicodedata from pathlib import Path import database as db import embedding from agents import kill_process, cancel_scope, clear_scope, run_agent -from config import CONSENSUS_GRACE, RESEARCH_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP +from config import CONSENSUS_GRACE, RESEARCH_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP, GROUP_RECONCILE_FLOOR, GROUP_MIN_COS_FLOOR from fsutil import atomic_write_text, atomic_write_json from jsonio import read_json_file as _json_file from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder @@ -55,6 +56,7 @@ SUBBLOCK_CAP = 900 # subblock find loop per chunk (15 min) CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (dedups everything); above that chunked + merge pass — fallback path only DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair (complete-link aggregates → no chaining) DEDUP_PAIRS_CHUNK = 40 # pairs per judge package (pairwise verification instead of a block mixer) +DEDUP_TITLE_AUTO = 0.95 # near-identical TITLE cosine ⇒ same entity → merge without the judge (recall net) FILTER_CHUNK = 35 # blocks to assess per judge in the degrade pass (full list as context) # Balance question-pattern chunks by sub load via LPT (makespan), not by block count. QUESTION_CHUNK_SUBS = 50 # target sum of relevant subs per chunk @@ -63,6 +65,7 @@ FACTS_CHUNK_SUBS = 25 # facts extraction: smaller chunks (facts are bulkier FACTS_CHECK_PANEL = 3 # judges per chunk in the facts check (majority objects) CONSOLIDATION_PANEL = 3 # mapping judges per chunk (panel → reconcile instead of a single judge) SUBBLOCK_PANEL = 3 # source judges in the subblock clarification (majority instead of a single judge) +FILTER_RECHECK_PANEL = 3 # judges in the survivor-recheck (rare-positive "fragment" recall; majority ≥2) log = logging.getLogger("creator.blocks") @@ -198,7 +201,7 @@ def _blocks_steps(topic: str) -> tuple: all packages run in parallel; the step remains until the last package is done. """ q = load_source(topic) - base = ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter") + base = ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung") rest = ( "Subblocks find", "Subblocks select", "Subblocks clarify", "Facts find", "Facts check", "Facts fix", @@ -228,7 +231,7 @@ def _report_p(set_p, topic: str, step: str): # Special steps (Source laden, Supplement) belong to the "Inventory" phase. PHASEN = ( ("Source", ("Source prep",)), - ("Inventory", ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Supplement")), + ("Inventory", ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung", "Supplement")), ("Subblocks", ("Subblocks find", "Subblocks select", "Subblocks clarify")), ("Facts", ("Facts find", "Facts check", "Facts fix")), ("Levels", ("Levels find", "Levels select", "Levels clarify")), @@ -290,7 +293,8 @@ def _all_slot_files(files: dict) -> list[Path]: + list(work_dir.glob("question-pattern-*")) + list(work_dir.glob("outline-*")) + list(work_dir.glob("artifact-*")) + list(work_dir.glob("research-*")) + list(work_dir.glob("consolidation-*")) + list(work_dir.glob("clarification*")) + list(work_dir.glob("dedup-*")) - + list(work_dir.glob("inventar-filter*"))) if work_dir.is_dir() else [] + + list(work_dir.glob("inventar-filter*")) + + list(work_dir.glob("gruppierung-*")) + list(work_dir.glob("inventar-gruppierung*"))) if work_dir.is_dir() else [] return [ *files["research"], files["research_mapping"], *(p for slots in files["selection"].values() for p in slots), @@ -317,10 +321,10 @@ async def _resume_step(topic: str) -> int: files = _blocks_files(topic) steps_all = _blocks_steps(topic) if not files["final"].exists(): - for step in ("Source prep", "Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter"): + for step in ("Source prep", "Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung"): if step in steps_all and await db.get_step_status(topic, step) != "done": return _step_idx(topic, step) - return _step_idx(topic, "Blocks-Filter") # statuses done but artefact gone → rewrite + return _step_idx(topic, "Blocks-Gruppierung") # statuses done but artefact gone → rewrite q = load_source(topic) if q["type"] == "projekt" and not files["ergaenzung"].exists(): return _step_idx(topic, "Supplement") @@ -501,6 +505,24 @@ async def _reset_from_step(topic: str, step_idx: int) -> None: if any(s.startswith("Subblock") for s in affected): files["sub_roh"].unlink(missing_ok=True); gd("subblock-*"); await db.delete_subblocks(topic) # --- Inventory (DB status cascades) --- + if "Blocks-Gruppierung" in affected and not ({"Consolidation", "Research"} & affected): + # Only status-flip resets (grouping/filter/dedup level): undo umbrella grouping — members back + # to consensus (with their original description), synthesized umbrellas discarded. On a + # Consolidation/Research reset the blocks are wiped below anyway, so skip there. Independent of + # the filter/dedup branches (a reset-from-Blocks-Filter needs BOTH grouping- and filter-undo). + d = _json_file(work_dir / "inventar-gruppierung.json") + for u in (d.get("umbrellas", []) if isinstance(d, dict) else []): + member_norms = set() + for m in u.get("mitglieder", []): + mn = m.get("title_norm") or _norm_title(m.get("title", "")) + if not mn: + continue + member_norms.add(mn) + await db.set_block_status(topic, mn, "consensus", description=m.get("description")) + un = _norm_title(u.get("umbrella", "")) + if un and un not in member_norms: # synthesized umbrella (not a reused member title) → drop it + await db.set_block_status(topic, un, "discarded") + gd("gruppierung-*"); gd("inventar-gruppierung*") if "Blocks-Filter" in affected and not ({"Clarification", "Consolidation", "Research", "Dedup"} & affected): # Only filter rebuilt: degraded blocks back to consensus. d = _json_file(work_dir / "inventar-filter.json") @@ -528,7 +550,7 @@ async def _reset_from_step(topic: str, step_idx: int) -> None: # blocks.md is the inventory aggregate — stale once any inventory sub-step is reset. Delete it so the # status/resume see the inventory as open from the reset step (the pipeline rewrites it; the DB step # statuses of the kept earlier steps let those skip). - if {"Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter"} & affected: + if {"Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung"} & affected: files["final"].unlink(missing_ok=True) @@ -1932,7 +1954,7 @@ async def _research_batch(ctx: GenContext, set_p, files: dict, q: dict, folder, await db.delete_blocks(topic) # coverage/content belongs to the triage — do NOT delete await db.set_step_status(topic, "Research", "running") - async def _ingest(reader_id: str, text: str) -> None: + async def _ingest(reader_id: str, text: str, source: str | None = None) -> None: seen_set = set() for record in _parse_selection(text).values(): title = _title(record) @@ -1942,8 +1964,16 @@ async def _research_batch(ctx: GenContext, set_p, files: dict, q: dict, folder, seen_set.add(norm) # one reader = one vote per concept split_parts = [t.strip() for t in record.split(" — ")] desc = split_parts[1] if len(split_parts) >= 2 else "" - source = [split_parts[2]] if len(split_parts) >= 3 and split_parts[2] else [] - await db.upsert_block(topic, norm, title, desc, source, reader=reader_id) + # Provenance out-of-band: in section mode the source filename is known deterministically + # (`source`), so the agent no longer appends it to the line — the filename must never be + # inlined into title/desc (that leaked "— alle_klausuren.txt" into descriptions and polluted + # title_norm, breaking dedup). Fall back to a segment-3 source only where the section file + # isn't 1:1 with the reader (thema/crawl). + if source: + sources = [source] + else: + sources = [split_parts[2]] if len(split_parts) >= 3 and split_parts[2] else [] + await db.upsert_block(topic, norm, title, desc, sources, reader=reader_id) pages = await db.list_content(topic) # pages marked as content by the triage if not pages and folder: @@ -1988,16 +2018,16 @@ async def _research_batch(ctx: GenContext, set_p, files: dict, q: dict, folder, set_p(f"Research ({len(eintraege)} sections)…", step=_step_idx(topic, "Research")) async def _read_section(ei: int, fn: str, section_text: str) -> None: - block = (f"ARBEITE AUSSCHLIESSLICH MIT DIESEM TEXTABSCHNITT (Source: {fn}). Lies ihn " - f"VOLLSTÄNDIG, überspringe nichts. Notiere `{fn}` als Source jedes Bausteins. " - f"Suche NICHT im Web — nur dieser Section zählt.\n\n-----\n{section_text}\n-----") + block = (f"ARBEITE AUSSCHLIESSLICH MIT DIESEM TEXTABSCHNITT. Lies ihn VOLLSTÄNDIG, " + f"überspringe nichts. Suche NICHT im Web — nur diese Section zählt." + f"\n\n-----\n{section_text}\n-----") paths = [work_dir / f"research-a{ei}-{i}.md" for i in range(1, RESEARCH_READERS + 1)] # reader file reuse: if all reader outputs are present and valid (resume / # re-run without research change), re-ingest instead of spawning agents again. existing = [(f"a{ei}-{i}", t) for i, p in enumerate(paths, 1) if (t := _file_payload(p))] if len(existing) == len(paths): for rid, text in existing: - await _ingest(rid, text) + await _ingest(rid, text, fn) # provenance injected deterministically from the section file return for p in paths: p.unlink(missing_ok=True) @@ -2014,7 +2044,7 @@ async def _research_batch(ctx: GenContext, set_p, files: dict, q: dict, folder, agent_texts = await _race(topic, f"Research section {ei}", slots, 2, _timeout("research", 1), provider, cancelled=is_cancelled, grace=RESEARCH_GRACE) for rid, text in (agent_texts or []): - await _ingest(rid, text) + await _ingest(rid, text, fn) # provenance injected deterministically from the section file await _gather_progress([_read_section(ei, fn, a) for ei, (fn, a) in enumerate(eintraege, 1)], len(eintraege), _report_p(set_p, topic, "Research")) @@ -2095,7 +2125,11 @@ def _grp_schema(data, ids: set[int]): _ASPECT_MARKER = ("∈ np", "∈np", " in np", "np-schwer", "np-vollständig", "verifizierer", "zertifikat", "ndtm", "nicht-determ", "lower bound", "untere schranke", - "bzgl", "als sprache") + "bzgl", "als sprache", "beweis", "güte", "austausch", + # generic bound/limit/runtime property stems (a "…-Grenze"/"…-Schranke"/"…-Laufzeit" + # is a property OF a concept, not the concept — MDM survivorship must never pick it as + # the representative, so it scores >0 here like the other aspect markers): + "grenze", "schranke", "laufzeit") def _aspect_marker(title: str) -> int: @@ -2380,13 +2414,30 @@ async def _clarify_inventory(ctx: GenContext, set_p, files: dict) -> bool: if await db.get_step_status(topic, "Clarification") == "done": return True set_p("Clarification running…", step=_step_idx(topic, "Clarification")) + work_dir = files["arbeit"] rest_rows = await db.list_blocks(topic, status="rest") + # C2 — deterministic pre-reject (precision gate, BEFORE the recall-biased panel): a single-reader + # `rest` item whose title is a mechanical exercise/notation artefact (Blatt/Aufgabe/Beispiel N, + # "(Variante)", "|·|", "Güte N") and is NOT a named statement/reduction is dropped without the panel. + # Reuses the filter's FP≈0, title-head-only helpers, gated by the statement keep-guard so real + # reductions/theorems survive. Deterministic → resume-identical; reset-safe (rest→discarded only). + pruned: list[dict] = [] + pre_rejected = [b for b in rest_rows + if (_is_artifact(b["title"]) or _FILTER_NOTATION.search(b["title"])) + and not _is_named_statement(b["title"], b["description"])] + if pre_rejected: + pr_norms = {b["title_norm"] for b in pre_rejected} + for b in pre_rejected: + await db.set_block_status(topic, b["title_norm"], "discarded") + pruned.append({"title": b["title"], "grund": "pre-reject"}) + rest_rows = [b for b in rest_rows if b["title_norm"] not in pr_norms] + _log(topic, f"Clarification: {len(pre_rejected)} single-reader artefact(s) pre-rejected (deterministic)") # Continuous gate (EDC "Define"): also check consensus blocks with reference/placeholder titles # ("Satz 7.18", "Korollar 6.18", "Bedingung (**)") — otherwise they bypass every exam. suspicious = [b for b in await db.list_blocks(topic, status="consensus") if _is_reference(b["title"])] + rest_norms = {b["title_norm"] for b in rest_rows} # single-reader origin → stricter quorum (C1) check_rows = rest_rows + suspicious if check_rows: - work_dir = files["arbeit"] paths = [work_dir / f"clarification-j{j}.json" for j in range(1, CONSOLIDATION_PANEL + 1)] # final=False: a judge with an accidentally non-empty `rest` must not fail entirely # (otherwise the panel collapses to 1 judge). Its `aufnehmen` counts; rest entries count as @@ -2436,9 +2487,14 @@ async def _clarify_inventory(ctx: GenContext, set_p, files: dict) -> bool: renames[_norm_title(str(old))][new] += 1 seen_norm = {b["title_norm"] for b in await db.list_blocks(topic, status="consensus")} for b in check_rows: - accept = votes.get(b["title_norm"], 0) * 2 >= len(outs) + v = votes.get(b["title_norm"], 0) + # C1 — origin-split quorum: a single-reader `rest` item needs UNANIMITY (a corroboration + # proxy — collapsing the ≥2-reader gate is what over-admitted ~99%); an already-consensus + # reference title keeps the majority rule (it only needs a rename/exam, not a harder bar). + accept = (v >= len(outs)) if b["title_norm"] in rest_norms else (v * 2 >= len(outs)) if not accept: await db.set_block_status(topic, b["title_norm"], "discarded") + pruned.append({"title": b["title"], "grund": "failed-quorum", "votes": v, "judges": len(outs)}) continue new_title = None if _is_reference(b["title"]) and (suggestions := renames.get(b["title_norm"])): @@ -2453,6 +2509,11 @@ async def _clarify_inventory(ctx: GenContext, set_p, files: dict) -> bool: await db.set_block_status(topic, b["title_norm"], "consensus", title=t, neu_norm=nn) else: await db.set_block_status(topic, b["title_norm"], "consensus") + # C4 — audit journal (swept by the existing gd("clarification*") on reset; not needed for + # correctness — clarification is rebuild-not-rollback — only to see what the gate pruned). + atomic_write_json(work_dir / "clarification-journal.json", + {"pre_rejected": len(pre_rejected), "quorum_pruned": len(pruned) - len(pre_rejected), + "pruned": pruned}, indent=1) await db.set_step_status(topic, "Clarification", "done") return True @@ -2494,6 +2555,58 @@ def _cliques(n: int, edge_list: list[tuple[int, int]]) -> list[list[int]]: return groups +# Canonical-name blocking (dedup recall): entity resolution's recall ceiling is set by candidate +# generation — a pair that never shares a candidate can never be merged. Normalize a title to a +# scaffolding-free, operator-class-normalized, order-independent key so surface variants of ONE entity +# collapse ("Offenes Problem P=NP?" ≡ "P vs NP" ≡ "P=NP"). Generic (no course terms): strip a small +# stoplist of catalogue/wrapper words, fold the relation operators into class tokens, sort content tokens. +_CANON_STOP = re.compile( + r'\b(?:offenes?|open|problem|frage|question|algorithmus|algorithm|verfahren|method|methode|' + r'satz|theorem|lemma|korollar|definition|def|das|der|die|the|ein|eine|einen|a|an|' + r'von|of|für|for|und|and|zum|zur|im)\b', re.I) + + +def _canonical_key(title: str) -> str: + """Order-independent canonical key of a title (scaffolding stripped, relation operators normalized). + Two titles with the same key denote the same entity with ~100% precision (ER blocking). Empty string + if nothing survives (never auto-merged).""" + s = unicodedata.normalize("NFKC", title).casefold() + s = re.sub(r'≟|\bversus\b|\bvs\.?\b|=', ' opeq ', s) # equality / "vs" → one token + s = re.sub(r'≤|⪯|→|⇒|⟹|\breduces?\s+to\b|\breduziert\b', ' opred ', s) # reduction → one token + s = _CANON_STOP.sub(' ', s) + s = re.sub(r'[^\w ]', ' ', s) # drop punctuation/symbols + return " ".join(sorted(t for t in s.split() if t)) + + +# Relation-triple individuation (dedup precision): a reduction/relation "A ≤ B" is identified by BOTH +# operands AND direction (RDF-triple identity / SKOS narrowMatch — a subset/restriction is NOT the same). +# So "SAT ≤ Clique" ≠ "3-SAT ≤ Clique" (source differs) and "A → B" ≠ "B → A" (direction). Two DIFFERENT +# relations must never merge, even if a judge or a high title-cosine says so. +_REL_STRIP = re.compile(r'^\s*(?:satz|lemma|korollar|theorem)\s*[\d.]*\s*:?\s*|^\s*reduktion(?:en)?\s*:?\s*', re.I) +_REL_OPERATOR = re.compile(r'[≤⪯≥⊆⊊→⇒⟹⇔←]|=>|<=|->') + + +def _relation_operands(title: str) -> tuple[str, str] | None: + """(canonical_source, canonical_target) of a relation/reduction title, else None (not a relation). + Operands canonicalized (lowercased, non-alphanumerics stripped) so spacing/hyphenation don't matter.""" + t = _REL_STRIP.sub('', title, count=1) + m = _REL_OPERATOR.search(t) + if not m: + return None + left = re.sub(r'[\W_]', '', t[:m.start()].casefold()) # keep unicode letters/digits (umlauts), drop the rest + right = re.sub(r'[\W_]', '', t[m.end():].casefold()) + if not left or not right: + return None + return (left, right) + + +def _relation_conflict(title_a: str, title_b: str) -> bool: + """True if BOTH titles are relations/reductions but denote DIFFERENT ones (operands or direction + differ) → they must NOT be merged. False if either is not a relation, or they are the same relation.""" + a, b = _relation_operands(title_a), _relation_operands(title_b) + return a is not None and b is not None and a != b + + async def _dedup_inventory(ctx: GenContext, set_p, files: dict) -> bool: """Final dedup pass over the finished consensus list: pairwise verification (entity resolution). Embedding yields candidate PAIRS (cosine ≥ DEDUP_PAAR_FLOOR), a judge @@ -2511,13 +2624,39 @@ async def _dedup_inventory(ctx: GenContext, set_p, files: dict) -> bool: consensus = await db.list_blocks(topic, status="consensus") if len(consensus) >= 2: import numpy as np - texts = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in consensus] - sims = await asyncio.to_thread(embedding.embed_sims, texts) + # Candidate cosine = MEAN of title-only and title+description similarity. Dedup is entity + # resolution: the TITLE identifies the concept, the description only elaborates. Title-duplicates + # with divergent descriptions ("SetCover" vs "SetCover-Problem": title-cos ~0.69 but title+desc + # ~0.54) fell below the 0.6 floor and were never checked. Averaging boosts them over the floor + # while keeping pairs that are unrelated in BOTH signals out (title-similar-only reduction + # families like "3-SAT ≤ X"/"3-SAT ≤ Y" stay below 0.6 via the divergent description). The + # per-pair judge still verifies each candidate, so recall rises without loosening precision. + texts_full = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in consensus] + texts_title = [b["title"] for b in consensus] + sims_full = await asyncio.to_thread(embedding.embed_sims, texts_full) + sims_title = await asyncio.to_thread(embedding.embed_sims, texts_title) + sims = (sims_full + sims_title) / 2 if (sims_full is not None and sims_title is not None) else sims_full if sims is not None: n = len(consensus) iu = np.triu_indices(n, k=1) cands = [(int(iu[0][m]), int(iu[1][m])) for m in np.where(sims[iu] >= DEDUP_PAIR_FLOOR)[0]] - _log(topic, f"Dedup: {len(cands)} candidate pairs (cosine ≥ {DEDUP_PAIR_FLOOR}) → pairwise filter") + # D3 — canonical-name blocking: raise recall by force-adding pairs whose scaffolding-stripped, + # operator-normalized key is EQUAL (embedding can miss them: "Offenes Problem P=NP?" vs "P vs + # NP" share no wording). The pairwise judge still verifies each; exact-key pairs additionally + # auto-merge below (ER-standard, ~100% precision). key_groups reused for the auto-merge. + key_groups: dict[str, list[int]] = {} + for i, b in enumerate(consensus): + k = _canonical_key(b["title"]) + if k: + key_groups.setdefault(k, []).append(i) + cand_set = set(cands) + for grp in key_groups.values(): + for x in range(len(grp)): + for y in range(x + 1, len(grp)): + cand_set.add((grp[x], grp[y])) + cands = sorted(cand_set) + _log(topic, f"Dedup: {len(cands)} candidate pairs (mean(title,title+desc) cosine ≥ {DEDUP_PAIR_FLOOR}, " + f"+ canonical-key) → pairwise filter") packages = [cands[i:i + DEDUP_PAIRS_CHUNK] for i in range(0, len(cands), DEDUP_PAIRS_CHUNK)] def pair_path(pi): return work_dir / f"dedup-paar-c{pi}.json" @@ -2545,18 +2684,51 @@ async def _dedup_inventory(ctx: GenContext, set_p, files: dict) -> bool: return False # Collect confirmed "ja" edges, then COMPLETE-LINK (greedy cliques) instead of single-link # union-find — prevents chaining (A=B + B=C does NOT merge A,C without a direct A=C). - edge_list, ja = [], 0 + edge_list, ja, rel_blocked = [], 0, 0 for pi, paare in enumerate(packages): verdict = _pairs_schema(_json_file(pair_path(pi))) or {} for j, (a, b) in enumerate(paare): if verdict.get(j + 1): + # D2 — relation-triple guard: never merge two DIFFERENT reductions/relations even + # if the judge said "ja" (a reduction is individuated by both operands + direction). + if _relation_conflict(consensus[a]["title"], consensus[b]["title"]): + rel_blocked += 1 + continue edge_list.append((a, b)) ja += 1 + # Autonomous recall net: near-identical TITLE cosine ⇒ same entity, merge without the judge + # (catches false-negatives like "Algorithmus ΔTSP1"/"ΔTSP1"). Duplicate edges are harmless + # (_cliques builds a set adjacency). These land in `groups` → dedup-runde-1.json → reversible. + auto = 0 + if sims_title is not None: + for a, b in cands: + if float(sims_title[a][b]) >= DEDUP_TITLE_AUTO and not _relation_conflict( + consensus[a]["title"], consensus[b]["title"]): + edge_list.append((a, b)) + auto += 1 + # D3 — exact-canonical-key auto-merge (equal key ⇒ same entity, ER ~100% precision). The + # relation guard still applies: a sorted key can collide on direction ("A→B"/"B→A") — the + # operand check blocks that, so only genuinely identical entities merge. + key_auto = 0 + for grp in key_groups.values(): + for x in range(len(grp)): + for y in range(x + 1, len(grp)): + a, b = grp[x], grp[y] + if not _relation_conflict(consensus[a]["title"], consensus[b]["title"]): + edge_list.append((a, b)) + key_auto += 1 groups = _cliques(n, edge_list) removed = 0 for idxs in groups: - # representative = main concept (fewest property markers), then shortest title. - rep = min(idxs, key=lambda k: (_aspect_marker(consensus[k]["title"]), len(consensus[k]["title"]), k)) + # D1 — survivorship / golden-record (MDM standard): the representative is the MOST + # INFORMATIVE record, NEVER the shortest. Cascade (higher wins): fewest property markers + # (the main concept, not a "…-Grenze"/"…-Schranke" fragment) → richest description (most + # complete) → more specific/longer title (never shortest) → stable low index. + rep = max(idxs, key=lambda k: ( + -_aspect_marker(consensus[k]["title"]), + len(consensus[k]["description"] or ""), + len(consensus[k]["title"]), + -k)) for k in idxs: if k != rep: await db.set_block_status(topic, consensus[k]["title_norm"], "discarded") @@ -2564,9 +2736,11 @@ async def _dedup_inventory(ctx: GenContext, set_p, files: dict) -> bool: from collections import Counter atomic_write_json(work_dir / "dedup-runde-1.json", {"vorher": n, "entfernt": removed, "paare_geprueft": len(cands), "paare_ja": ja, + "auto_titel": auto, "auto_key": key_auto, "relation_blockiert": rel_blocked, "clique_groessen": dict(sorted(Counter(len(g) for g in groups).items())), "groups": [[consensus[k]["title"] for k in g] for g in groups]}, indent=1) - _log(topic, f"Dedup (pairwise): {n} → {n - removed} (−{removed}); {ja}/{len(cands)} pairs confirmed") + _log(topic, f"Dedup (pairwise): {n} → {n - removed} (−{removed}); {ja}/{len(cands)} pairs confirmed, " + f"{auto} auto-title, {key_auto} auto-key, {rel_blocked} relation-conflicts blocked") await db.set_step_status(topic, "Dedup", "done") return True @@ -2590,11 +2764,46 @@ def _filter_schema(data) -> dict[int, int] | None: # Pure notation/symbols without a standalone concept — kept narrow (FP~0, checked against aak; # "KNF"/"MST"/"NP" do NOT match). These are discarded autonomously (need no parent). _FILTER_NOTATION = re.compile(r'^\s*\|.{1,6}\|\s*$|^Güte\s+\d+\s*$') +# Exercise-sheet / cross-reference artefacts — the SECOND gate for a judge `drop` verdict: a hard-drop +# (parentless removal) only fires when the judge lists the number in `drop` AND `_is_artifact(title)`. +# A judge-drop without a match degrades to "keep" (logged), never deleted (FP~0 discipline like +# _FILTER_NOTATION). Shape-matching on a NORMALIZED, title-side string (see _is_artifact): four branches — +# (1) lettered/Roman/numeric sub-claims "(Aussage i)"/"Aussage (a)"/"Teil (b)"/"Fall (2)" in ANY +# parenthesization; (2) sheet refs "Blatt 10"/"Aufgabe 3"/"Übung"; (3) worked-example/table/figure refs +# "Beispiel Tab. 7.1" (a NUMBER is required — guards polysemes "Hash-Tabelle"/"Bijektive Abbildung"); +# (4) parenthesized "(Variante)". Theorem words (Satz/Definition/Lemma) are deliberately NOT in the +# vocabulary, so "Satz 6.24 Cook/Levin — SAT ist NP-vollständig" is kept. Word boundaries guard prefixes +# (Aussagenlogik/Teilmenge/Blattknoten/Fallunterscheidung). +_FILTER_ARTIFACT = re.compile(r""" + \b(?:aussage|teil|behauptung|fall)\b[\s(]*(?:[ivx]{1,4}|[a-z]|\d{1,2})\)?(?![a-zäöüß]) + | \b(?:blatt|aufgabe|serie|hausaufgabe)\s*\d+(?:[.,]\d+)* + | \b(?:übungsblatt|übung|uebung)\b + | \b(?:beispiel|abbildung|abb|tabelle|tab|bild|grafik|diagramm|figur|skizze)\b\.?\s*\d+(?:[.,]\d+)* + | \(\s*(?:variante|variation|spezialfall|sonderfall)\s*\) +""", re.VERBOSE) + + +def _is_artifact(title: str) -> bool: + """True if the TITLE looks like exercise-sheet / cross-reference scaffolding (P1 hard-drop gate). + NFKC + casefold normalization (position/case/umlaut/Unicode-invariant: folds Ⅱ→ii, full-width, NBSP), + then match only the title side of the em-dash — a real concept whose *description* merely cites a + sheet ("… — vgl. Aufgabe 3") is never dropped. Normalize for matching only; never store the result.""" + norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip() + head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0] # spaced dash only → "3-SAT"/"np-schwer" unsplit + return bool(_FILTER_ARTIFACT.search(head)) # Property/runtime suspicion — marks lines for the judge's verdict (NO auto-drop, FP too high: # "NP-Schwere", reductions with "∈NP" are real blocks). Complements _aspekt_marker. _FILTER_PREDICATE = re.compile( r'ist NP-(vollständig|schwer)|NP-(Vollständigkeit|Schwere) von|ETH (Konsequenz|Lower Bound)' - r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP', re.I) + r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP' + # fragment families that the aspect substrings miss (all title/desc, advisory ⚠ only): + r'|\bSatz\s+\d|\bLemma\s+\d|\bKorollar\s+\d' # bare theorem/proof references (with number) + r'|^\s*(?:Remark|Bemerkung|Anmerkung|Note|Notiz|Beobachtung|Observation)\b' # EN+DE remark labels + r'|\bSatz\s*:|\bSatz\s+[A-Z]\b' # "Satz:" (colon, no number) / "Satz D*" letter label + r'|\bGegenbeispiel\b|\bWorst[- ]?Case\b|Schärfe\s+der\b' # proof-example / sharpness facets + r'|2\s*\^\s*[{(]?\s*[Ωωoο]\s*\(' # ETH exponential bound 2^Ω(…)/2^o(…) + r'|\d\s*[−–-]\s*1\s*/\s*m' # approximation-güte ratio "2 − 1/m" + r'|Variablenungleichung|[α-ωΑ-Ω][a-z]?-?Variablen', re.I) # proof variables (αu-Variablen …) def _filter_suspect(b: dict) -> bool: @@ -2602,6 +2811,99 @@ def _filter_suspect(b: dict) -> bool: return _aspect_marker(b["title"]) > 0 or bool(_FILTER_PREDICATE.search(f"{b['title']} {b['description'] or ''}")) +def _root(nr: int, fragments: dict[int, int]) -> tuple[int, bool]: + """Follow the fragment→parent chain to the first ancestor that is NOT itself a fragment. + Returns (root, cyclic). This walk IS the fixpoint (no separate iteration): a mid-tier + fragment whose own parent is also a fragment resolves to the top-level real block, so all + chain levels collapse in one pass (unlike the old single-level parent_set shield). + Cycle guard: on revisiting a node returns (node, True) → caller keeps both (no annihilation).""" + seen: set[int] = set() + cur = nr + while cur in fragments: + if cur in seen: + return cur, True + seen.add(cur) + cur = fragments[cur] + return cur, False + + +def _containment_parent(frag_norm: str, others: list[tuple[int, str]]) -> int | None: + """Deterministic parent-by-name-containment for a ⚠-flagged survivor: a fragment whose title NAMES + another block ("Lower Bound … für VERTEX COVER" → Vertex Cover; "List Scheduling Güte …" → List + Scheduling). `others` = (nr, title_norm) of the OTHER blocks. A parent is a block whose normalized + title occurs as a WHOLE-WORD span inside `frag_norm`, is SIGNIFICANT (≥2 tokens or ≥6 chars — never + "p"/"np"/"sat") and strictly shorter than the fragment. Returns the parent nr on EXACTLY ONE match, + else None (0 or ≥2 → leave to the judge). Whole-word + significance + exactly-one → near-FP-0.""" + hits = [] + for nr, ptitle in others: + if not ptitle or ptitle == frag_norm or len(ptitle) >= len(frag_norm): + continue + if len(ptitle) < 6 and ptitle.count(" ") < 1: # reject short single-token names (P/NP/SAT) + continue + if re.search(r'(? bool: + """True for the narrow, parent-less noise classes safe to hard-drop (subject to KEEP-guards).""" + norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip() + head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0] + return bool(_PARENTLESS_NOISE.search(head)) and not _PARENTLESS_KEEP.search(norm) + + +# F1 — statement-gate keep-guard (Knowledge-Component / OMDoc theory): a self-contained ASSERTION is its +# own learning unit, not a whole-part fragment. Protect two general classes from demotion: +# (a) a named reduction between two PROBLEMS ("3-SAT ≤ Clique", "Clique → Vertex Cover"), and +# (b) a LABELED or ATTRIBUTED theorem carrying its own biconditional/implication ("Satz 6.37: … ⇔ …"). +# NOT protected: a bare label ("Satz 7.18", "Remark 7.28"), unary status ("X ist NP-vollständig", "X ∈ NP"), +# a proof-size step ("Strikte Reduktion |A| = O(m)"), or a güte/bound facet — those carry neither a +# two-sided reduction operator NOR a ⇔/⇒ assertion. General: attribution is structural, no author whitelist. +_STMT_LABEL = re.compile(r'^\s*(?:satz|lemma|korollar|theorem|proposition|prop|folgerung)\b', re.I) +_STMT_ATTRIB = re.compile(r'\b(?:satz|lemma|theorem|korollar)\s+von\s+[A-ZÄÖÜ]') # "Satz von Cook/Levin" +_STMT_ASSERT = re.compile(r'⇔|⇒|⟺|⟹|\bgdw\.?\b|\bgenau dann\b', re.I) +_REDUCTION_ONLY_OP = re.compile(r'[≤⪯]|→|⇒|⟹|->') # genuine reduction operators (NOT plain "=") + + +def _is_reduction_statement(title: str) -> bool: + """True if the title is a reduction between two NAMED problems (both sides carry ≥3 letters and + neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP" and proof-size "|A| = O(m)".""" + t = _REL_STRIP.sub('', title, count=1) + m = _REDUCTION_ONLY_OP.search(t) + if not m: + return False + + def _named(s: str) -> bool: + return len(re.findall(r'[a-zäöüß]', s, re.I)) >= 3 and not re.match(r'\s*[Oo]\s*\(', s) + + return _named(t[:m.start()]) and _named(t[m.end():]) + + +def _is_named_statement(title: str, desc: str = "") -> bool: + """Statement-gate keep-guard: a named reduction (a) or a labeled/attributed theorem WITH its own + ⇔/⇒ assertion (b). Bare labels / unary status / proof-size steps return False (stay demotable).""" + if _is_reduction_statement(title): + return True + if (_STMT_LABEL.match(title) or _STMT_ATTRIB.search(title)) and _STMT_ASSERT.search(f"{title} {desc or ''}"): + return True + return False + + async def _filter_inventory(ctx: GenContext, set_p, files: dict) -> bool: """Degrade pass (granularity): separates real blocks from fragments (properties, proof gadgets, notation, runtime details). Each judge sees the FULL block list @@ -2657,29 +2959,474 @@ async def _filter_inventory(ctx: GenContext, set_p, files: dict) -> bool: if is_cancelled(): return False fragments: dict[int, int] = {} + drops: set[int] = set() for ci, numbers in enumerate(chunks): - verdict = _filter_schema(_json_file(filt_path(ci))) or {} + raw = _json_file(filt_path(ci)) + verdict = _filter_schema(raw) or {} nset = set(numbers) for nr, parent in verdict.items(): if 1 <= parent <= n and nr in nset: fragments[nr] = parent - # Chain protection: a block that is itself the parent of a fragment stays (its child needs the anchor). - parent_set = set(fragments.values()) - removed, debug = 0, [] - for nr, parent in fragments.items(): - if nr in parent_set: + # `drop` is read additively from the raw JSON (like the `rename` field in _clarify_inventory) + # so _filter_schema keeps returning dict[int,int] and its three call sites stay untouched. + for x in (raw.get("drop", []) if isinstance(raw, dict) else []): + try: + dnr = int(x) + except (ValueError, TypeError): + continue + if 1 <= dnr <= n and dnr in nset: + drops.add(dnr) + # Hard-drop double-gate (Fix B): honour a judge `drop` ONLY if the title also matches the artefact + # regex — a parentless drop is the one irreversible-by-design deletion, so a single judge FP must + # not delete a concept. A judge-drop without a match degrades to "keep" (logged), never deleted. + honored_drops = {nr for nr in drops if _is_artifact(consensus[nr - 1]["title"])} + refused = drops - honored_drops + if refused: + _log(topic, f"Blocks-Filter: {len(refused)} judge-drop(s) refused (no artefact match, kept): " + f"{[consensus[nr - 1]['title'] for nr in sorted(refused)][:6]}") + for nr in honored_drops: # a hard-drop wins over a demote (it is noise, not a fragment of X) + fragments.pop(nr, None) + # Deterministic parent-by-containment demote (recall net for the single judge): a ⚠-flagged survivor + # whose title NAMES another block is demoted to it without the judge (near-FP-0: whole-word + + # significant parent + exactly-one match). Feeds the same `fragments` journal → transitive + reversible. + norms = [(i, consensus[i - 1]["title_norm"]) for i in range(1, n + 1)] + cont = 0 + for i in range(1, n + 1): + if i in fragments or i in honored_drops or not _filter_suspect(consensus[i - 1]): continue - b = consensus[nr - 1] - await db.set_block_status(topic, b["title_norm"], "discarded") + parent = _containment_parent(consensus[i - 1]["title_norm"], [(nr, t) for nr, t in norms if nr != i]) + if parent is not None and parent != i and parent not in honored_drops: + fragments[i] = parent + cont += 1 + if cont: + _log(topic, f"Blocks-Filter: {cont} fragment(s) auto-demoted by title-containment") + # Narrow journaled parent-less hard-drop (F3): bare label refs (Remark 7.28 / Satz D*) and single-var + # notation (r = n + m) that have NO parent — routed through honored_drops so they land in the `fragments` + # journal (2822-style) and stay reversible, unlike the un-journaled _FILTER_NOTATION prepass. KEEP-guards + # inside _is_parentless_noise protect named theorems / class (in)equalities / definitions. + pl = 0 + for i in range(1, n + 1): + if i in fragments or i in honored_drops or not _is_parentless_noise(consensus[i - 1]["title"]): + continue + if _containment_parent(consensus[i - 1]["title_norm"], [(nr, t) for nr, t in norms if nr != i]) is None: + honored_drops.add(i) + pl += 1 + if pl: + _log(topic, f"Blocks-Filter: {pl} parent-less noise block(s) hard-dropped (Remark/Satz-label/notation)") + # F2 — survivor-recheck panel: the main filter is a SINGLE judge and "fragment" is a rare positive + # class (single-pass recall ~30-60%). A focused 3-judge panel over ONLY the still-⚠-surviving, + # un-demoted blocks (majority ≥2) recovers the judge-missable ones. Same {fragments, drop} schema → + # merged into fragments/honored_drops (→ journaled by the loop below, reversible). + survivors = [i for i in range(1, n + 1) + if i not in fragments and i not in honored_drops and _filter_suspect(consensus[i - 1])] + if survivors: + rchunks = [survivors[k:k + FILTER_CHUNK] for k in range(0, len(survivors), FILTER_CHUNK)] + + def rc_path(ci, j): return work_dir / f"inventar-filter-recheck-c{ci}-j{j}.json" + + async def _recheck(ci, nums): + block = "\n".join( + (f"{i}. {consensus[i - 1]['title']} — {consensus[i - 1]['description']}" + if consensus[i - 1]['description'] else f"{i}. {consensus[i - 1]['title']}") for i in nums) + pending = [j for j in range(1, FILTER_RECHECK_PANEL + 1) if _filter_schema(_json_file(rc_path(ci, j))) is None] + if is_cancelled(): + return + await asyncio.gather(*[ + run_agent(f"blocks-{topic}-filter-recheck-c{ci}-j{j}", + _prompt("Blocks-Filter-Recheck", topic=topic, survivors=block, list=full_list, out_path=rc_path(ci, j)), + _timeout("selection_mapping", len(nums)), provider=ctx.provider, role="judge", capabilities="files") + for j in pending], return_exceptions=True) + + await _gather_progress([_recheck(ci, nm) for ci, nm in enumerate(rchunks)], + len(rchunks), _report_p(set_p, topic, "Blocks-Filter")) + if is_cancelled(): + return False + rc = 0 + for ci, nums in enumerate(rchunks): + nset, dem, drp = set(nums), {}, {} + for j in range(1, FILTER_RECHECK_PANEL + 1): + raw = _json_file(rc_path(ci, j)) + v = _filter_schema(raw) + if v is None: + continue + for nr, parent in v.items(): + if nr in nset and 1 <= parent <= n and nr != parent: + dem.setdefault(nr, []).append(parent) + for x in (raw.get("drop", []) if isinstance(raw, dict) else []): + try: + dnr = int(x) + except (ValueError, TypeError): + continue + if dnr in nset: + drp[dnr] = drp.get(dnr, 0) + 1 + for nr in nums: # majority ≥2; drop only if the title is also artefact/parent-less noise + if nr in fragments or nr in honored_drops: + continue + if drp.get(nr, 0) >= 2 and (_is_artifact(consensus[nr - 1]["title"]) or _is_parentless_noise(consensus[nr - 1]["title"])): + honored_drops.add(nr); rc += 1 + elif len(dem.get(nr, [])) >= 2: + fragments[nr] = max(set(dem[nr]), key=dem[nr].count); rc += 1 + if rc: + _log(topic, f"Blocks-Filter: recheck panel demoted/dropped {rc} more survivor(s)") + # F1 — statement-gate keep-guard (final override): a named reduction between two problems, or a + # labeled/attributed theorem WITH its own ⇔/⇒ assertion, is a knowledge unit — rescue it from ANY + # producer (judge, containment, parentless, recheck). Runs last so it wins; bare labels / unary status + # / proof-size steps are untouched (they fail _is_named_statement) and stay demoted/dropped. + def _protected(nr): + return _is_named_statement(consensus[nr - 1]["title"], consensus[nr - 1]["description"]) + saved = [nr for nr in list(fragments) if _protected(nr)] + for nr in saved: + fragments.pop(nr, None) + saved_drops = {nr for nr in honored_drops if _protected(nr)} + honored_drops -= saved_drops + if saved or saved_drops: + _log(topic, f"Blocks-Filter: {len(saved) + len(saved_drops)} named statement(s)/reduction(s) " + f"protected from demotion (statement-gate)") + # Transitive resolution (Fix C) + drops (Fix B) in one pass. Every discard is recorded under the + # SAME `fragments` key of inventar-filter.json → the existing reset rollback (see _reset_from_step) + # restores it, so both fixes stay wiring-free and reversible. + removed, debug = 0, [] + for nr in fragments: + root, cyclic = _root(nr, fragments) + if cyclic or not (1 <= root <= n): # cycle / invalid root → keep (no loss) + continue + await db.set_block_status(topic, consensus[nr - 1]["title_norm"], "discarded") removed += 1 - debug.append({"fragment": b["title"], "eltern": consensus[parent - 1]["title"]}) + if root in honored_drops: # ancestor judged noise → the child is collateral noise too + debug.append({"fragment": consensus[nr - 1]["title"], "eltern": None, "grund": "drop-collateral"}) + else: + debug.append({"fragment": consensus[nr - 1]["title"], "eltern": consensus[root - 1]["title"]}) + for nr in sorted(honored_drops): # explicit parentless hard-drops + await db.set_block_status(topic, consensus[nr - 1]["title_norm"], "discarded") + removed += 1 + debug.append({"fragment": consensus[nr - 1]["title"], "eltern": None, "grund": "drop"}) atomic_write_json(work_dir / "inventar-filter.json", {"vorher": n, "degradiert": removed, "fragments": debug}, indent=1) - _log(topic, f"Blocks-Filter: {n} → {n - removed} (−{removed} fragments → subblocks)") + _log(topic, f"Blocks-Filter: {n} → {n - removed} (−{removed}: fragments→subblocks + {len(honored_drops)} hard-drops)") await db.set_step_status(topic, "Blocks-Filter", "done") return True +def _umbrella_schema(data, ids: set[int]): + """{"umbrellas":[{"title":str,"description":str,"members":[int,…]}, …]} + → [(title, description, [member ids])]. [] = valid (no umbrella); None ONLY on broken JSON + (so resume treats a valid-but-empty file as done, like _filter_schema's {} vs None). Each id + used at most once across all umbrellas (first wins); members filtered to `ids`; an umbrella + needs ≥2 surviving members; `description` required + non-empty — it MUST enumerate the children, + else the source-scoped subblock step can't re-derive them (no web on uni).""" + if not isinstance(data, dict) or not isinstance(data.get("umbrellas"), list): + return None + out, used = [], set() + for u in data["umbrellas"]: + if not isinstance(u, dict): + continue + title = str(u.get("title", "")).strip() + desc = str(u.get("description", "")).strip() + raw_members = u.get("members") + if not title or not desc or not isinstance(raw_members, list): + continue + members = [] + for x in raw_members: + try: + m = int(x) + except (ValueError, TypeError): + continue + if m in ids and m not in used: + used.add(m) + members.append(m) + if len(members) >= 2: + out.append((title, desc, members)) + return out + + +def _completion_schema(data, n_umbrellas: int, ids: set[int]): + """{"additions":[{"umbrella":int,"members":[int,…]}, …]} → [(umbrella_idx, [member ids])]. + [] = valid (nothing to absorb); None only on broken JSON. umbrella idx in range; members drawn + from `ids` (the still-standalone leftovers), de-duped within an addition.""" + if not isinstance(data, dict) or not isinstance(data.get("additions"), list): + return None + out = [] + for a in data["additions"]: + if not isinstance(a, dict): + continue + try: + k = int(a.get("umbrella")) + except (ValueError, TypeError): + continue + if not (0 <= k < n_umbrellas): + continue + mem = [] + for x in (a.get("members") or []): + try: + m = int(x) + except (ValueError, TypeError): + continue + if m in ids and m not in mem: + mem.append(m) + if mem: + out.append((k, mem)) + return out + + +# Deterministic backstop to the grouping judge's TEST 1 (type gate): an umbrella may bundle ONLY +# constituent sub-definitions of ONE definition. If a member title carries a standalone-unit signal +# (a named algorithm / problem / reduction / theorem), the umbrella is dissolved — those stay their own +# blocks. Kept narrow so real definition-parts (Konfiguration, Übergangsfunktion δ, Literale, Makespan, +# m Maschinen) never match; checked against the aak over-merge (member „Greedy-Algorithmus GA" hits). +# Suffix-anchored head nouns (German compounds are head-final: „Approximations+algorithmus" has NO word +# boundary before „algorithmus", so \bAlgorithmus\b misses it → the MAX-SAT over-merge). \w* absorbs the +# modifier; the head noun stays the discriminator. FP-safe: no real TM/KNF definition-part ends in these +# heads (Berechnung is deliberately NOT a head → „Akzeptierende Berechnung" stays a valid member). +_GROUP_STANDALONE = re.compile( + r'\w*algorithm(?:us|en)\b|\w*problem(?:e|s|en)?\b|\w*reduktion(?:en)?\b|\bscheduling\b|[≤⪯]' + r'|^\s*(?:Satz|Lemma|Korollar|Theorem|Bemerkung|Beobachtung)\s*\d' + # atomicity: a named COMPLEXITY CLASS / a "…-Vollständigkeit(completeness)" / a "…Transformation" is a + # self-contained concept (learning-object / atomic-KC), never a sub-definition — a bundle of ≥1 such + # member is siblings, not one model → dissolve (catches the P/NP/NP-Vollständigkeit over-merge that NO + # cosine floor separates). Head-final compounds (\w*klasse absorbs "Komplexitäts+klasse"); FP-safe — + # no real TM/KNF/TSP/Scheduling definition-part carries these heads. + r'|\w*vollständigkeit\b|\w*completeness\b|\w*transformation(?:en)?\b|\w*klasse[nr]?\b', re.I) + + +async def _group_inventory(ctx: GenContext, set_p, files: dict, dry_run: bool = False) -> bool: + """Umbrella grouping (granularity level 2, AFTER the filter): collapse sibling DEFINITIONS that + are components of ONE umbrella concept into a single block whose description enumerates the + children (→ the subblock step re-derives them from the source). Embedding builds coarse capped + candidate clusters (low sibling-floor → high recall); one judge per multi-cluster sees the FULL + surviving list as context (hybrid recall) and synthesizes umbrella title+description+members. + Skipped (like Dedup) when the flag is off or no embedding model. Idempotent: per-cluster judge + artefacts + a single tail DB pass. dry_run writes the decision artefact WITHOUT any DB mutation + (and does NOT mark the step done) — for safe tuning on a clone.""" + topic, is_cancelled = ctx.topic, ctx.is_cancelled + if not BLOCKS_GRUPPIERUNG_AKTIV: + await db.set_step_status(topic, "Blocks-Gruppierung", "done") + return True + if not dry_run and await db.get_step_status(topic, "Blocks-Gruppierung") == "done": + return True + if not (EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available)): + await db.set_step_status(topic, "Blocks-Gruppierung", "done") + return True + set_p("Blocks-Gruppierung…", step=_step_idx(topic, "Blocks-Gruppierung")) + work_dir = files["arbeit"] + consensus = await db.list_blocks(topic, status="consensus") + n = len(consensus) + if n < 3: # nothing meaningful to group + await db.set_step_status(topic, "Blocks-Gruppierung", "done") + return True + texts = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in consensus] + sims = await asyncio.to_thread(embedding.embed_sims, texts) + if sims is None: + await db.set_step_status(topic, "Blocks-Gruppierung", "done") + return True + clusters = await asyncio.to_thread(embedding.capped_blocks, sims, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP) + multi = [c for c in clusters if len(c) > 1] + + def _min_cos(idxs): # internal coherence (chains would be ~0.3 → over-merge signal) + if len(idxs) < 2: + return 1.0 + return round(min(float(sims[i][j]) for a, i in enumerate(idxs) for j in idxs[a + 1:]), 3) + + all_ids = set(range(1, n + 1)) + full_list = "\n".join(f"{i}. {texts[i - 1]}" for i in range(1, n + 1)) + + def grp_path(ci): return work_dir / f"gruppierung-block-c{ci}.json" + + async def _assess(ci, cluster): + p = grp_path(ci) + if _umbrella_schema(_json_file(p), all_ids) is not None: + return # resume: keep a valid file + p.unlink(missing_ok=True) + if is_cancelled(): + return + cand = "\n".join(f"{g + 1}. {texts[g]}" for g in cluster) # global numbers (1-based) + await run_single_slot( + ctx, f"Blocks-Gruppierung {ci}", + key=f"blocks-{topic}-gruppierung-block-c{ci}", + prompt=_prompt("Blocks-Gruppierung", topic=topic, candidates=cand, list=full_list, out_path=p), + role="judge", capabilities="files", + payload=lambda result, p=p: _umbrella_schema(_json_file(p), all_ids), + timeout=_timeout("research_mapping", len(cluster)), + ) + + # Top-down pass (recall): one judge sees ONLY the full list (no cluster seed) and proposes umbrellas + # across it — cures the single-linkage fragmentation where heterogeneous siblings (TM-model, KNF) + # never co-cluster and so are never seeded as one candidate. Its output flows through the SAME + # collection loop → type-gate → reconcile (which merges it with any cluster-proposed duplicate). + async def _assess_top(): + p = grp_path("TOP") + if _umbrella_schema(_json_file(p), all_ids) is not None: + return # resume + p.unlink(missing_ok=True) + if is_cancelled(): + return + cand = ("Scan the ENTIRE block list below and propose EVERY genuine umbrella you find — " + "do not restrict yourself to any subset.") + await run_single_slot( + ctx, "Blocks-Gruppierung top", + key=f"blocks-{topic}-gruppierung-block-cTOP", + prompt=_prompt("Blocks-Gruppierung", topic=topic, candidates=cand, list=full_list, out_path=p), + role="judge", capabilities="files", + payload=lambda result, p=p: _umbrella_schema(_json_file(p), all_ids), + timeout=_timeout("research_mapping", n), + ) + + await _gather_progress([_assess(ci, c) for ci, c in enumerate(multi)] + [_assess_top()], + len(multi) + 1, _report_p(set_p, topic, "Blocks-Gruppierung")) + if is_cancelled(): + return False + + # Collect umbrellas across clusters; GLOBAL first-wins dedup on members (a block can be pulled + # into only one umbrella). Resolve title collisions BEFORE any DB write. + used, seen_norm = set(), {b["title_norm"] for b in consensus} + chosen, skipped, plan = [], [], [] + sources = [grp_path("TOP")] + [grp_path(ci) for ci in range(len(multi))] # TOP first → wins used-ties + for src in sources: + for title, desc, members in (_umbrella_schema(_json_file(src), all_ids) or []): + # " — " (space-em/en-dash-space) is the reserved title/description separator; a title + # containing it would be truncated by _title() downstream ("P||Cmax — X" → "P||Cmax", + # colliding with the real "P||Cmax" block). Defuse it regardless of what the judge emits. + title = re.sub(r"\s+[—–]\s+", ": ", title).strip() + members = [m for m in members if m not in used] + if len(members) < 2: + continue + rows = [consensus[m - 1] for m in members] + # Type gate (deterministic backstop): a member that is a named algorithm/problem/reduction/ + # theorem is a standalone block, never a sub-definition → dissolve the umbrella, keep members. + if any(_GROUP_STANDALONE.search(r["title"]) for r in rows): + skipped.append({"umbrella": title, "grund": "type-gate", + "mitglieder": [r["title"] for r in rows]}) + continue + # No-structure backstop ONLY (member-vs-member cosine is the WRONG instrument for over-merge — + # meronymy ≠ similarity, empirically inverted; the atomicity type-guard above is the real + # precision floor). Rejects a literally structureless chain (below the random-pair baseline); + # the floor sits BELOW the legitimate heterogeneous minimum so it never kills a real model. + mc = _min_cos([m - 1 for m in members]) + if mc < GROUP_MIN_COS_FLOOR: + skipped.append({"umbrella": title, "grund": "min-cos", "min_cos": mc, + "mitglieder": [r["title"] for r in rows]}) + continue + unorm = _norm_title(title) + member_norms = {r["title_norm"] for r in rows} + if unorm not in member_norms and unorm in seen_norm: # collision with a NON-member block + skipped.append({"umbrella": title, "grund": "title-collision", + "mitglieder": [r["title"] for r in rows]}) + continue + used.update(members) + seen_norm.add(unorm) + chosen.append({"umbrella": title, "description": desc, "min_cos": mc, + "mitglieder": [{"title": r["title"], "description": r["description"], + "title_norm": r["title_norm"]} for r in rows]}) + plan.append((unorm, title, desc, rows)) + + # Reconcile pass: two independently-judged clusters can emit the SAME parent under different titles + # + disjoint members (e.g. two „Turingmaschine"-umbrellas) — neither member-id nor title dedup catches + # that. Merge umbrella pairs whose title+description cosine ≥ GROUP_RECONCILE_FLOOR: union members, + # concatenate the enumerating descriptions (every child stays named → subblock re-derivation intact), + # keep the title of the umbrella with the most members. Deterministic (embed_sims), no extra LLM. + if len(chosen) >= 2: + u_sims = await asyncio.to_thread( + embedding.embed_sims, [f"{c['umbrella']} — {c['description']}" for c in chosen]) + if u_sims is not None: + parent = list(range(len(chosen))) + for i in range(len(chosen)): + for j in range(i + 1, len(chosen)): + if float(u_sims[i][j]) >= GROUP_RECONCILE_FLOOR: + embedding._union(parent, i, j) + comp: dict[int, list[int]] = {} + for i in range(len(chosen)): + comp.setdefault(embedding._find(parent, i), []).append(i) + m_chosen, m_plan = [], [] + for grp in comp.values(): + if len(grp) == 1: + m_chosen.append(chosen[grp[0]]) + m_plan.append(plan[grp[0]]) + continue + seen_m, rows = set(), [] # union member rows (dedup by norm) + for gi in grp: + for r in plan[gi][3]: + if r["title_norm"] not in seen_m: + seen_m.add(r["title_norm"]) + rows.append(r) + rep = max(grp, key=lambda gi: len(plan[gi][3])) # richest umbrella keeps its title + title = plan[rep][1] + desc = " · ".join(chosen[gi]["description"] for gi in grp) + m_chosen.append({**chosen[rep], "umbrella": title, "description": desc, + "mitglieder": [{"title": r["title"], "description": r["description"], + "title_norm": r["title_norm"]} for r in rows], + "reconciled_from": [chosen[gi]["umbrella"] for gi in grp]}) + m_plan.append((_norm_title(title), title, desc, rows)) + if len(m_chosen) < len(chosen): + _log(topic, f"Blocks-Gruppierung: reconciled {len(chosen)} → {len(m_chosen)} umbrellas (merged same-parent duplicates)") + chosen, plan = m_chosen, m_plan + + # Membership-completion pass (G-B): reconcile only merges umbrella↔umbrella, so a partial umbrella + # (e.g. TM with 2 of ~6 parts) + leftover standalone siblings never unites. One judge — anchored on the + # chosen umbrellas (parent + enumerated parts) — decides which of the still-standalone blocks are ALSO + # constituent parts of each parent; a per-member type-guard veto keeps precision. Updates chosen+plan+ + # used in lockstep so the artefact stays reset-faithful. Empty/absent → no-op. + leftover = sorted(all_ids - used) + if chosen and leftover: + cp = work_dir / "gruppierung-completion.json" + add = _completion_schema(_json_file(cp), len(chosen), set(leftover)) + if add is None: + cp.unlink(missing_ok=True) + if not is_cancelled(): + anchors = "\n".join( + f"UMBRELLA {k}: {c['umbrella']} — {c['description']}\n bereits: " + + ", ".join(m["title"] for m in c["mitglieder"]) for k, c in enumerate(chosen)) + rest = "\n".join(f"{i}. {texts[i - 1]}" for i in leftover) + status, add = await run_single_slot( + ctx, "Blocks-Gruppierung completion", + key=f"blocks-{topic}-gruppierung-completion", + prompt=_prompt("Blocks-Gruppierung-Completion", topic=topic, umbrellas=anchors, rest=rest, out_path=cp), + role="judge", capabilities="files", + payload=lambda result, p=cp: _completion_schema(_json_file(p), len(chosen), set(leftover)), + timeout=_timeout("research_mapping", len(leftover))) + add = add if status == OK else [] + if is_cancelled(): + return False + norm2idx = {consensus[i - 1]["title_norm"]: i for i in range(1, n + 1)} + absorbed = 0 + for k, new_members in (add or []): + hit = False + for m in new_members: + if m in used or not (1 <= m <= n) or _GROUP_STANDALONE.search(consensus[m - 1]["title"]): + continue # type-guard veto: never absorb a named algorithm/problem/theorem + r = consensus[m - 1] + used.add(m) + chosen[k]["mitglieder"].append({"title": r["title"], "description": r["description"], "title_norm": r["title_norm"]}) + plan[k][3].append(r) + absorbed += 1 + hit = True + if hit: # refresh the diagnostic min_cos over the enlarged member set + idxs = [norm2idx[mm["title_norm"]] - 1 for mm in chosen[k]["mitglieder"] if mm["title_norm"] in norm2idx] + chosen[k]["min_cos"] = _min_cos(idxs) + if absorbed: + _log(topic, f"Blocks-Gruppierung: completion absorbed {absorbed} standalone part(s) into umbrellas") + + entfernt = sum(len(c["mitglieder"]) for c in chosen) - len(chosen) + atomic_write_json(work_dir / "inventar-gruppierung.json", + {"vorher": n, "nachher": n - entfernt, "umbrellas": chosen, "skipped": skipped}, indent=1) + _log(topic, f"Blocks-Gruppierung: {len(multi)} clusters → {len(chosen)} umbrellas " + f"(−{entfernt} blocks{', DRY-RUN (no DB write)' if dry_run else ''})") + if dry_run: + return True # decision written; no DB mutation, step NOT marked done + + # Single tail pass — all DB writes here (idempotent under the step gate). The umbrella row carries + # the enumerating description; members (except a title-reused anchor) go to discarded. + for unorm, title, desc, rows in plan: + await db.upsert_block(topic, unorm, title, desc) + # Force title+desc: upsert's ON CONFLICT keeps the OLD description, so an anchor-reuse umbrella + # (unorm == a member's norm) or a reset→rerun would otherwise keep a stale/narrow description and + # the enumerating child list — the subblock step's re-derivation anchor — would be lost. + await db.set_block_status(topic, unorm, "consensus", title=title, description=desc) + for r in rows: + if r["title_norm"] != unorm: + await db.set_block_status(topic, r["title_norm"], "discarded") + await db.set_step_status(topic, "Blocks-Gruppierung", "done") + return True + + # --- Outline (blocks artifact: chapter structure, only read by the guide) --- def _outline_complete(files: dict) -> bool: @@ -3073,7 +3820,7 @@ async def _reset_db_from_phase(topic: str, label: str) -> None: await db.delete_subblocks(topic) if idx <= 1: # Inventory: inventory + research steps — triage stays await db.delete_blocks(topic) - await db.delete_pipeline_state(topic, ["Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter"]) + await db.delete_pipeline_state(topic, ["Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung"]) if idx <= 0: # Source: redo triage (coverage/content + step) await db.delete_coverage(topic) await db.delete_pipeline_state(topic, ["Source prep"]) @@ -3162,6 +3909,9 @@ async def generate_blocks(topic: str, instructions: str = "", provider: str = DE if _past_limit("Blocks-Filter"): return if not await _stage(_filter_inventory(ctx, set_p, files)): return + if _past_limit("Blocks-Gruppierung"): return + if not await _stage(_group_inventory(ctx, set_p, files)): + return consensus_rows = await db.list_blocks(topic, status="consensus") entries = { i: (f"{b['title']} — {b['description']}" if b["description"] else b["title"]) diff --git a/backend/config.py b/backend/config.py index 54959f4..adaa0f3 100644 --- a/backend/config.py +++ b/backend/config.py @@ -42,6 +42,30 @@ EMBEDDING_BLOCK_CAP = 25 # max. titles per block (keep the LLM list short/s # without exception true duplicates). Conservative 0.90 so different aspects (∈NP ≠ NP-hard) stay separate. EMBEDDING_SUB_DUP = 0.90 +# Umbrella grouping (block granularity level 2, step "Blocks-Gruppierung", AFTER the filter): +# collapse sibling DEFINITIONS that are components of ONE umbrella concept (TM model: +# Konfiguration/Start-/Folge-/Stopkonfiguration/Berechnung/Alphabet/δ; KNF: Literale/Klauseln/ +# Variablen) into ONE block whose description ENUMERATES the children — so the later subblock step +# re-derives them from the source under the umbrella scope (demotion is re-derivation, not transfer). +# If the flag is off or no embedding model → step is silently skipped (like Dedup). +BLOCKS_GRUPPIERUNG_AKTIV = True +# Lower floor than consolidation (0.5, paraphrase-tuned) for higher sibling recall; the LLM judge is +# the precision gate. Smaller cap since a lower floor pulls in more nodes → keep the judge lists short. +EMBEDDING_SIBLING_FLOOR = 0.35 # heterogeneous facets of one model co-cluster weakly → low floor (recall) +EMBEDDING_SIBLING_CAP = 18 # a rich model (TM) can have many constituent parts +# Reconcile pass: two independently-judged clusters can emit the SAME parent concept under different +# titles (e.g. two „Turingmaschine"-umbrellas). Merge umbrella pairs whose title+description cosine is +# ≥ this (conservative → only true same-parent duplicates, never two distinct umbrellas). +GROUP_RECONCILE_FLOOR = 0.75 +# Over-merge backstop ONLY (no-structure floor). Research (meronymy ≠ similarity): parts of ONE model are +# legitimately DISSIMILAR (TM: Alphabet/Konfiguration/δ ~0.22), while distinct same-type concepts (P/NP/…) +# are SIMILAR (~0.85) — so member-vs-member cosine is the WRONG instrument for over-merge (empirically +# inverted: TM 0.218 < the P/NP bundle 0.227). The real precision floor is the ATOMICITY type-guard +# (_GROUP_STANDALONE: a member that is a named algorithm/problem/theorem/complexity-class dissolves the +# umbrella). This floor is demoted to a near-zero backstop that only rejects a literally structureless +# chain (random-pair baseline), set BELOW the legitimate heterogeneous minimum so it never kills a real model. +GROUP_MIN_COS_FLOOR = 0.15 + # Cap for concurrent CLI agent processes (across all generations). # Own lane for interactive calls (chat, elements) so they don't hang behind # running writers in the queue. diff --git a/backend/textkit.py b/backend/textkit.py index c8df3ae..af9bb09 100644 --- a/backend/textkit.py +++ b/backend/textkit.py @@ -56,11 +56,14 @@ def _resolve_title(idx: dict[str, int], t: str) -> int | None: def _norm_dash(s: str) -> str: - """Space-surrounded dash variants (en/em/figure/bar/hyphen) → uniform separator ' — '. + """Dash variants (en/em/figure/bar) with whitespace on AT LEAST ONE side → uniform separator ' — '. Some models (especially non-western ones) use an en-dash "–" instead of the em-dash; without - normalization the ` — ` split fails entirely and the whole entry becomes the title. - The ASCII hyphen "-" is left untouched (otherwise it would split formulas like "n - 1").""" - return re.sub(r"\s+[‒–—―‐]\s+", " — ", s) + normalization the ` — ` split fails entirely and the whole entry becomes the title. A one-sided + space ("Titel —Beschreibung" / "Titel— Beschreibung") also breaks the split and leaks the source + filename into the description — so a dash with a space on either side is repaired too. The ASCII + hyphen "-" is deliberately NOT in the class (would split "n - 1"/"3-SAT"); requiring ≥1 surrounding + space keeps glued compounds like "Backtracking—Verfahren" and number ranges like "12–15" untouched.""" + return re.sub(r"\s*[‒–—―]\s+|\s+[‒–—―]\s*", " — ", s) def _parse_selection(text: str) -> dict[int, str]: diff --git a/templates/Prompt/Blocks-Filter-Recheck.md b/templates/Prompt/Blocks-Filter-Recheck.md new file mode 100644 index 0000000..347b0e7 --- /dev/null +++ b/templates/Prompt/Blocks-Filter-Recheck.md @@ -0,0 +1,24 @@ +Topic "{topic}". A first pass kept the blocks below, but each was FLAGGED as a likely **fragment** — a property, proof step, remark, bound, or notation that belongs to another block, not its own learning unit. Re-judge each one carefully. This is a focused second opinion; a good learning list has roughly 75–82 top-level blocks, so real fragments here SHOULD be demoted or dropped — but never sacrifice a genuine concept to hit a number. + +RE-JUDGE THESE (by their number): +{survivors} + +FULL BLOCK LIST (context — to find a parent number): +{list} + +## Decide each survivor → one of three +- **demote (→ parent number):** it presupposes another block as its subject — a property/status („X ist NP-vollständig", „X ∈ NP"), a **lower bound** of X, a **bare theorem/remark** about X („Bemerkung: HK auch für gerichtete Graphen" → Hamiltonkreis; „Satz: F erfüllbar ⇔ … 3-dim Matching" → 3-dim Matching), an **approximation-güte facet** („Schärfe der 3/2 Rate" → Christofides), a **proof-example/gadget** („MST in Gegenbeispiel" → the approximation proof), a proof-variable. Put `{{"<nr>": <parent-nr>}}` in `fragments`. +- **drop:** pure exercise/reference scaffolding with NO real content and NO parent — a bare label („Remark 7.28", „Satz D*"), a one-off notation assignment („r = n + m"). Put its number in `drop`. +- **keep:** it IS a self-contained concept. Do NOT touch it. (Just omit it.) + +## KEEP-guards — these are real blocks, never demote/drop them +- A **named theorem WITH its own statement or an author**: „Satz 6.24 Cook/Levin — SAT ist NP-vollständig", „Satz von Immerman–Szelepcsényi". KEEP. +- A **complexity-class (in)equality / open question**: „P = NP?", „NL = coNL". KEEP. +- Anything headed „**Definition**", a **problem**, an **algorithm**, a **reduction** („3-SAT ≤ Clique"). KEEP. + +Judge by the CONTENT (after „—"), not the label. When unsure whether something is a fragment or a concept: if it has an obvious parent in the list → demote; otherwise → keep (never drop on doubt). + +Write ONLY the JSON file to: {out_path} + +Format (both keys; each may be empty): +{{"fragments": {{"3": 17, "9": 41}}, "drop": [12]}} diff --git a/templates/Prompt/Blocks-Filter.md b/templates/Prompt/Blocks-Filter.md index 046433b..39f3965 100644 --- a/templates/Prompt/Blocks-Filter.md +++ b/templates/Prompt/Blocks-Filter.md @@ -23,17 +23,30 @@ A block is self-contained: you can explain it WITHOUT presupposing another block ## What is a FRAGMENT (belongs to another block → demote)? Self-containment test: does the entry presuppose ANOTHER concept in the list as its subject? Then it is that concept's property/part, not its own block. - **Property/status** of a problem X (that is itself in the list): „X ist NP-vollständig", „X ∈ NP", „NP-Schwere von X", „Approximationsgüte von X". → parent = X. -- **Proof/reduction gadget**: „αEnde", „A-Komponente", „Dummy Items", „Schedule D*", „Knoten z", auxiliary variables. → parent = the theorem/reduction in whose proof it appears. -- **Pure notation/symbol**: „|x|", „Σ∗", „Güte 2". → parent = the defining definition. +- **Lower bound / ETH bound** of a problem X: „Lower Bound bzgl. Knoten für VERTEX COVER", „ETH untere Schranke HITTING SET (|U|)", „2^Ω(√|E|) …". → parent = X (the problem the bound is about). +- **Bare theorem / proof reference**: „Satz 6.12: P ⊆ NP", „Beweis Satz 6.16 (⇒)", „Beweis ⊃ von Satz 6.21", „Satz 7.20 (Sahni)", „Pm||Cmax NP-vollständig (Satz 7.23)" — a restated inclusion/membership or a bare „Satz N"/„Beweis …" is a proof detail. → parent = the problem/algorithm/class it is about (P/NP, Sahni's algorithm, Pm||Cmax …). +- **Proof/reduction gadget or variable**: „αEnde", „A-Komponente", „Dummy Items", „αu-Variablen", „Variablenungleichungen im ILP", „Austausch-Argument". → parent = the theorem/reduction in whose proof it appears. +- **Approximation-guarantee facet**: „Güte 2", „Güte 2 − 1/m", „Approximative Güte 2", „List Scheduling Güte (2 − 1/m)". → parent = the algorithm it bounds (List Scheduling, LPT, …). - **Runtime/size detail**: „O(|V|⁴) Verifizierer-Laufzeit", „|V'| = |V| bei Reduktion", „Reduktion in O(|E|)". → parent = the algorithm/reduction. +- **Parent named in the entry's OWN title:** if the title itself contains another block's name („Lower Bound … für **VERTEX COVER**", „**List Scheduling** Güte …", „**Pm||Cmax** NP-vollständig"), that named block IS the parent — demote to it. Do not keep such an entry just because you would scan the whole list; the parent is right there in the title. +- **Over-specific variant** of a base problem that is itself in the list: „Even-Knapsack", „Subset Sum Cardinality", „Partition (3·Summe)", „SAT3" are exercise-tweaked variants of „Rucksackproblem"/„Subset Sum"/„Partition"/„SAT". → parent = the base problem. (A genuinely different problem with its own theory stays its own block.) + +## What is an EXERCISE ARTEFACT (no concept at all → hard-drop)? +Rare, and applied cautiously. ONLY clear exercise-sheet / cross-reference scaffolding that is neither a learnable concept nor a fragment of one AND has no parent in the list. These forms all count, no matter where the marker sits: +- a lettered OR **Roman-numbered** sub-claim, in any parenthesization: „Aussage (a): …", „(Aussage i)", „(Aussage ii)", „NP-schwer ≠ P (Aussage ii)", „Teil (b)", „Fall (2)"; +- a bare sheet/task reference: „Blatt 10", „Aufgabe 3", „Übung 7.31"; +- a worked-example / table / figure reference: „Scheduling Beispiel Tab. 7.1", „Beispiel 3.2", „Abbildung 4.5"; +- a one-off framing with no standalone content. +Put its number in `drop`. NEVER drop anything that names a real problem/method/definition/theorem/reduction — if there is any doubt, keep it (or demote it as a fragment with a parent). A **named theorem WITH its own statement** („Satz 6.24 Cook/Levin — SAT ist NP-vollständig") is a real block, never an artefact. If it has a parent in the list, prefer demoting (fragment) over dropping. ## Rules - A fragment is demoted ONLY if its **parent block is in the list** (give its number). If you find no parent → keep it (don't list it). - The doubt concerns STANDALONE-NESS: if it's unclear whether an entry stands on its own → keep it. But a clear property/notation/proof part WITH a parent in the list IS a fragment — don't keep it out of caution. - A standalone **reduction between two problems** is a block, NOT a fragment („3-SAT ≤ Clique"). +- A **named theorem WITH its own relational statement** — a biconditional/implication/reduction between two named objects („Satz 6.37: 3-SAT ≤ 3-Färbung … ⇔ …") — is a block; keep it even if it references other blocks. Only a BARE label with no statement („Satz 7.18", „Remark 7.28"), a unary status („X ist NP-vollständig", „X ∈ NP"), or a güte/bound/proof-size facet is a fragment. - Judge by the CONTENT (after the „—"), not the title. Write ONLY the JSON file to: {out_path} -Format (only the fragment numbers from {from_n}–{to_n}, each with its parent number; `fragments` may be empty): -{{"fragments": {{"12": 5, "13": 5, "27": 19}}}} +Format — always include `fragments` (fragment number → parent number, may be empty); `drop` is the list of exercise-artefact numbers with NO parent (usually empty). Only numbers from {from_n}–{to_n}: +{{"fragments": {{"12": 5, "13": 5, "27": 19}}, "drop": [17]}} diff --git a/templates/Prompt/Blocks-Gruppierung-Completion.md b/templates/Prompt/Blocks-Gruppierung-Completion.md new file mode 100644 index 0000000..3c50356 --- /dev/null +++ b/templates/Prompt/Blocks-Gruppierung-Completion.md @@ -0,0 +1,18 @@ +Topic "{topic}". Umbrella blocks were formed, each bundling the constituent parts of ONE model/definition. Some parts were missed and are still listed as standalone blocks. Your job: for each umbrella, find which of the remaining standalone blocks are ALSO constituent parts of that same parent, so the model is complete. + +UMBRELLAS (parent — already-collected parts): +{umbrellas} + +REMAINING STANDALONE BLOCKS (numbered): +{rest} + +## Rule — attach a block to an umbrella only if BOTH hold +1. **Presupposition:** the block's definition **requires the umbrella's parent to already exist** — it makes no sense as a topic on its own without that model (e.g. „Alphabet Σ", „Übergangsfunktion δ", „Konfiguration", „Akzeptierende Berechnung" all presuppose the Turing-machine; „Literale", „Klausel", „Belegung" presuppose the KNF/logic definition). The parent must NOT presuppose the block (directional). +2. **Not standalone:** the block is a *definitional component / notation*, NOT itself a named **algorithm, problem, theorem, reduction, or complexity class** (those stay their own block — a downstream guard will reject them anyway). + +Do NOT attach a block merely because it shares a topic. When unsure → leave it standalone. Most remaining blocks will NOT be attached; a few genuine missed parts will. + +Write ONLY the JSON file to: {out_path} + +Format (`additions` may be empty; `umbrella` = the UMBRELLA index, `members` = standalone block numbers to attach): +{{"additions": [{{"umbrella": 0, "members": [8, 12, 34]}}]}} diff --git a/templates/Prompt/Blocks-Gruppierung.md b/templates/Prompt/Blocks-Gruppierung.md new file mode 100644 index 0000000..7959296 --- /dev/null +++ b/templates/Prompt/Blocks-Gruppierung.md @@ -0,0 +1,34 @@ +Topic "{topic}". A previous step produced a flat list of learning blocks that is TOO FINE-GRAINED — several blocks are **constituent sub-definitions / notation of ONE larger definition or model** and should become a single umbrella block. Find these groups. A good run finds several genuine umbrellas AND leaves most blocks standalone; judge each candidate on its merits. + +**Propose generously.** A deterministic guard downstream rejects any umbrella that swallows a named algorithm/problem/theorem, so a wrong-but-plausible merge is cheap — a missed umbrella is not. Do NOT withhold a merge merely because the members are lexically dissimilar (facets of one model routinely are) or because you are unsure of the parent's exact name. + +CANDIDATES (your starting point): +{candidates} + +FULL BLOCK LIST (you may pull in ANY numbers below that are constituents of the same definition): +{list} + +## Merge test — propose an umbrella when ALL THREE hold +1. **One parent.** The members are constituent parts/facets of ONE named parent model or definition — each member PRESUPPOSES that parent (you cannot introduce the member without first invoking the parent). TM-model parts (Konfiguration, Übergangsfunktion δ, Alphabet Σ, Akzeptierende Berechnung) presuppose „Turingmaschine"; KNF parts (Literale, Klauseln, Boolesche Variablen) presuppose „Konjunktive Normalform". +2. **Studied together.** A learner meets them together as one unit. +3. **No standalone unit among them.** No member is itself a named **algorithm, problem, theorem, reduction/transformation, or complexity class** („List Scheduling", „3-SAT", „Cook/Levin", „3-SAT ≤ Clique", „NP", „NP-Vollständigkeit", „Polynomielle Transformation"). Each of those is its own concept, so a group containing one is a set of SIBLINGS, not the decomposition of one model — keep them separate. + +*Note on test 1: „can this be defined at all?" is the WRONG question — Alphabet Σ and DTM CAN be stated in isolation, yet in THIS topic they are parts of the Turing-machine model and belong together. The question is whether the member PRESUPPOSES the shared parent, not whether a standalone sentence exists.* + +## Examples +DO NOT MERGE — distinct named units that merely share a topic: +- „Greedy-Algorithmus GA" + „ModifiedGreedy" + „Multiple-Choice-Knapsack" → two algorithms + a problem, each standalone (test 3 fails). Keep separate. + +MERGE — one definition decomposed (the canonical cases — end here so this is your default lens): +- „Alphabet Σ" + „NDTM" + „DTM" + „Akzeptierende Berechnung" + „Folgekonfiguration" → ONE umbrella **„Turingmaschine (Modell)"**. (The members are lexically very different from each other — that is EXPECTED for facets of one model and is NOT a reason to keep them apart.) +- „Klausel" + „Boolesche Variable" + „Erfüllende Belegung" + „KNF" → ONE umbrella **„Aussagenlogik & KNF"**. + +## Synthesize each umbrella +- `title`: the parent concept's name (e.g. „Turingmaschine (Modell)"). A real self-contained definition; must NOT contain „ — " (a reserved separator) — use „(…)" or „:". +- `description`: **name EVERY merged child explicitly** — the next step recovers the children as sub-points from the source. E.g. „Formales TM-Modell: Konfiguration, Übergangsfunktion δ, Alphabet Σ, Akzeptierende Berechnung, Folgekonfiguration." +- `members`: the block NUMBERS (from the full list) folded in. At least 2 per umbrella; each number appears in at most one umbrella. + +Write ONLY the JSON file to: {out_path} + +Format (`umbrellas` may be empty): +{{"umbrellas": [{{"title": "Turingmaschine (Modell)", "description": "Formales TM-Modell: Konfiguration, Übergangsfunktion δ, Alphabet Σ, Akzeptierende Berechnung, Folgekonfiguration.", "members": [1, 2, 5, 12, 34]}}]}} diff --git a/templates/Prompt/Blocks-Klaerung.md b/templates/Prompt/Blocks-Klaerung.md index 1a90d66..3cc4d95 100644 --- a/templates/Prompt/Blocks-Klaerung.md +++ b/templates/Prompt/Blocks-Klaerung.md @@ -23,7 +23,7 @@ Examples: - "Satz 7.18" (no description) → **verwerfen** (mere reference). Rules: -- When in doubt about standalone-ness → lean toward including. Duplicates are removed separately later; here only this counts: real block or junk. +- Decide by the standalone Define test above: an entry that only makes sense INSIDE a specific proof, reduction, or spot in the script is NOT a block → discard it. Include only entries that stand on their own (a concept you could teach on its own); duplicates are removed separately later. This is single-mention material, so hold a firm bar — but never discard a genuine standalone concept hiding behind a reference title (see the rename rule above). - Copy included entries VERBATIM ("Title — Kurzbeschreibung"), do not rephrase.{final} Write ONLY the JSON file to: {out_path} diff --git a/templates/Prompt/Blocks-Paar-Filter.md b/templates/Prompt/Blocks-Paar-Filter.md index ce6e421..f3847aa 100644 --- a/templates/Prompt/Blocks-Paar-Filter.md +++ b/templates/Prompt/Blocks-Paar-Filter.md @@ -1,20 +1,36 @@ -Two research passes have noted blocks for the topic "{topic}". For EACH pair, decide whether A and B denote the SAME block (the same concept, just worded differently) → **ja**, or whether they are TWO DIFFERENT blocks → **nein**. +Two research passes noted blocks for the topic "{topic}" as "Title — description". For EACH pair, decide: do A and B denote the SAME block → **ja**, or TWO DIFFERENT blocks → **nein**? PAIRS: {pairs} -Rules: -- **Watch the CORE ENTITY first** (the problem/object in question): Clique, Vertex Cover, Independent Set, Dominating Set, Set Cover, FVS, Knapsack … If the entities are DIFFERENT → **nein**, no matter how identical the phrasing. -- Identical phrasing is deceptive. These pairs are **nein** (different entity despite nearly identical wording): - - "Lower Bound **Clique** bzgl. Knoten" ↔ "Lower Bound **Vertex Cover** bzgl. Knoten" - - "Lower Bound Clique bzgl. **Knoten**" ↔ "Lower Bound Clique bzgl. **Kanten**" - - "Verifizierer für **FVS**" ↔ "Verifizierer für **Knapsack**" - - "**Cliquenproblem**" ↔ "**Vertex-Cover-Problem**" -- **ja** only on genuine semantic equivalence: same solution to the same problem, the same entity, just different wording/naming (e.g. "SET COVER" ↔ "Mengenüberdeckungsproblem", "Cliquenproblem" ↔ "k-CLIQUE", "List Scheduling" ↔ "LPT-Algorithmus"). -- **nein** also for different aspects of the same problem: "Set Cover (Problem)" ↔ "Set Cover ETH-Schranke"; a problem ↔ its reduction to another; a problem ↔ its verifier. -- When in doubt **nein** — better two separate blocks than wrongly merging two concepts. +## How to decide (per pair) +**STEP 1 — Name the CANONICAL ENTITY of each side.** Strip catalogue numbers („Definition 6.19", „Satz 7.8"), drop generic tags like „(Problem)"/„-Problem", ignore case, spacing and hyphenation. So „HITTING SET" and „HittingSet (Problem)" share one canonical entity; „Definition 6.19 NP" and „NP" both name the entity **NP**; „Algorithmus ΔTSP1" and „ΔTSP1" both name **ΔTSP1**. + +**STEP 2 — Same entity, or different?** +- **DIFFERENT canonical entity → nein**, however identical the wording: + - two different problems/objects: „Clique" ≠ „Vertex Cover"; „Lower Bound Clique bzgl. Knoten" ≠ „… bzgl. Kanten" (a different parameter is a different result); + - a distinct **variant** is its own entity: „GA" ≠ „ModifiedGreedy (MGA)"; „SAT" ≠ „3-SAT"; „Knapsack" ≠ „Multiple-Choice-Knapsack"; + - a **reduction between two problems** is its own entity: „Clique" ≠ „3-SAT ≤ Clique"; + - two reductions/relations that share ONE side but differ on the OTHER (or run in the opposite direction) are DIFFERENT results → **nein**: „SAT ≤ Clique" ≠ „SAT ≤ 3-Dim-Matching", „VertexCover ≤ FVS" ≠ „VertexCover ≤ Δ-Cover". A restriction/special case („3-SAT ≤ X") is NARROWER than the general („SAT ≤ X"), never the same. +- **SAME canonical entity → ja**, even when A and B emphasize DIFFERENT FACETS of it. Facets of one and the same object include: its **formal definition**, a **mechanism/step** (how it works), a **property** (approximation ratio, a bound, complexity, ∈ NP), a **characterization**, a **naming variant**. Two entries describing different facets of the SAME entity are duplicates. + +**STEP 3 — „When in doubt → nein" applies ONLY when STEP 1 is ambiguous** (you genuinely cannot tell whether the two names denote the same object). It does NOT fire merely because the two descriptions differ — differing descriptions of the SAME entity are **ja**. + +## Examples +DUPLICATE (ja) — same entity, different facet/wording: +- A: „Algorithmus ΔTSP1 — MST, Kanten verdoppeln, Eulerkreis, Abkürzungen" B: „ΔTSP1 — TSP-Approximationsalgorithmus mit Rate 2" → both = algorithm **ΔTSP1** (steps vs. its ratio) → **ja** +- A: „Definition 6.19 NP — L ∈ NP ⇔ ∃ NDTM …" B: „NP — Klasse aller polynomiell verifizierbaren Sprachen" → both = **NP** (definition vs. characterization) → **ja** +- A: „HITTING SET" B: „HittingSet (Problem)" → same entity, only naming → **ja** +- A: „SET COVER" B: „Mengenüberdeckungsproblem" → **ja** + +NOT A DUPLICATE (nein) — different entity: +- A: „Greedy-Algorithmus GA" B: „ModifiedGreedy (MGA)" → two different algorithms → **nein** +- A: „Lower Bound Clique bzgl. Knoten" B: „Lower Bound Clique bzgl. Kanten" → different parameter → **nein** +- A: „Clique" B: „3-SAT ≤ Clique" → a problem vs. a reduction (its own block) → **nein** +- A: „VertexCover ≤ FVS" B: „VertexCover ≤ Δ-Cover" → same source, different target → different reductions → **nein** +- A: „Cliquenproblem" B: „Vertex-Cover-Problem" → different problems → **nein** Write ONLY the JSON file to: {out_path} -Format (each pair number from the list with "ja" or "nein"; no other text in the file): +Format (each pair number from the list with „ja" or „nein"; no other text in the file): {{"pairs": {{"1": "ja", "2": "nein"}}}} diff --git a/templates/Prompt/Blocks-Research.md b/templates/Prompt/Blocks-Research.md index f239fa6..e9cdc98 100644 --- a/templates/Prompt/Blocks-Research.md +++ b/templates/Prompt/Blocks-Research.md @@ -12,14 +12,15 @@ Rules: - Pure NOTATION/symbols belong to their definition: „|x|", „Σ∗" — not their own entry. - A standalone REDUCTION between two problems, however, is its own block („3-SAT ≤ Clique"). - NO categories, NO ranking, NO ordering by importance — only a flat, numbered list. -- There is NO target count. Stop only when the research yields nothing new. -- Invent nothing: include only blocks you have backed by research. Note the source per block (URL or file path). If there is no individual source, the collective source suffices (handbook chapter, textbook, overview page, directory). +- Aim for the natural number of genuine learning units for this material — there is no hard quota, but do NOT split hairs to inflate the count (prefer the "families learned together = ONE block" rule above). STOP when the only remaining candidates are exercises, meta-questions, administrative notes, or duplicates of blocks you already listed. +- **EXCLUDE non-content — these are NEVER learning blocks:** exercise/task/assignment scaffolding ("Aufgabe 3", "Bonusaufgabe", "Übung", "Blatt 11", point values), meta/assessment items ("welche der folgenden…", true/false prompts, hand-in / exam-date notes), administrative & boilerplate (headings, page/room/exam numbers, names, copyright), and any **file names, paths, or URLs** (those are provenance, recorded separately — never a block or part of one). +- Invent nothing: include only blocks actually supported by the provided material. - Write title and description in GERMAN (technical terms/code identifiers stay original). - Description at most ~12 words. Write ONLY the Markdown file to: {blocks_path} -Format: EXACTLY one line per block: `N. Title — Kurzbeschreibung — Source` -The source (3rd segment) MUST be the exact file name or URL of the crawl page the block comes from — it drives the coverage check. +Format: EXACTLY one line per block: `N. Title — Kurzbeschreibung` +Use the em-dash " — " (a space on EACH side) ONLY to separate the title from the short description — never elsewhere in the line, and never inside the title or the description. Do NOT append the source file name, path, or URL to the line — provenance is recorded separately by the pipeline. {focus} {extra} \ No newline at end of file