From d25824229e819b6ac4825611c61cb75da3547aca Mon Sep 17 00:00:00 2001 From: Team3 Date: Sun, 5 Jul 2026 00:25:53 +0200 Subject: [PATCH] update --- backend/config.py | 1 - backend/fake_agents.py | 11 +- backend/guide_board.py | 370 ++++++++++++------------- backend/repair.py | 32 ++- backend/tests/test_events.py | 4 +- backend/tests/test_guide_board.py | 139 ++++++---- backend/tests/test_repair.py | 41 +++ backend/train_params.py | 1 - templates/Prompt/Guide-Coverage.md | 19 -- templates/Prompt/Guide-Fakten-Fix.md | 19 -- templates/Prompt/Guide-Fakten-Gate.md | 23 -- templates/Prompt/Guide-Fix.md | 27 ++ templates/Prompt/Guide-Lese-Check.md | 33 --- templates/Prompt/Guide-Pruefer.md | 36 +++ templates/Prompt/Guide-Sections-Fix.md | 38 --- 15 files changed, 392 insertions(+), 402 deletions(-) delete mode 100644 templates/Prompt/Guide-Coverage.md delete mode 100644 templates/Prompt/Guide-Fakten-Fix.md delete mode 100644 templates/Prompt/Guide-Fakten-Gate.md create mode 100644 templates/Prompt/Guide-Fix.md delete mode 100644 templates/Prompt/Guide-Lese-Check.md create mode 100644 templates/Prompt/Guide-Pruefer.md delete mode 100644 templates/Prompt/Guide-Sections-Fix.md diff --git a/backend/config.py b/backend/config.py index 5642775..935252c 100644 --- a/backend/config.py +++ b/backend/config.py @@ -201,7 +201,6 @@ GEN_PANEL = 2 # unabhängige Generator-Calls pro Block VERIFY_PANEL = 2 # unabhängige Prüfer-Calls pro Block (+ Ersatz bei 1 Ausfall) ART_SPLIT_SUBS = 20 # Artefakt-Generator splittet ab so vielen Subs in 2 parallele Calls FILTER_RECHECK_PANEL = 3 # judges in the survivor-recheck -MAX_WRITER_ROUNDS = 2 # guide coverage→writer loop cap GATE_FIX_MIN = 3 # fact-gate: unbelegt-claims below this → log only (falsch fixt immer) WRITER_SPLIT_SUBS = 30 # guide writer splits sections above this sub count KANBAN_BATCH = 5 # cards a worker pulls per micro-batch diff --git a/backend/fake_agents.py b/backend/fake_agents.py index b114b3e..6458fbb 100644 --- a/backend/fake_agents.py +++ b/backend/fake_agents.py @@ -199,15 +199,12 @@ class Welt: ziele = [{"id": f"z{i}", "text": f"Verstehen von {s}", "sub": s} for i, s in enumerate(self._subs_im_prompt(prompt), 1)][:8] return j({"ziele": ziele or [{"id": "z1", "text": "Grundlagen verstehen", "sub": ""}]}) - if "-gatefix-" in key or "-lesefix-" in key: + if "-gfix-" in key: return self._section_aus_prompt(prompt) or "\nRepariert." - if "-gate-" in key: - return j({"ok": True}) - if "-cov-" in key: + if "-pruef-" in key: # verschmolzener Prüfer: Fakten + Coverage + Lesbarkeit ids = sorted(set(_ZIEL_RE.findall(prompt))) - return j({"ziele": {z: True for z in ids}, "luecken": [], "ballast": []}) - if "-lese-" in key: - return j({"ok": True}) + return j({"claims": [], "ziele": {z: True for z in ids}, "luecken": [], + "ballast": [], "lese_probleme": []}) if "-w-" in key: return self._writer_md(prompt) diff --git a/backend/guide_board.py b/backend/guide_board.py index 5802874..284d989 100644 --- a/backend/guide_board.py +++ b/backend/guide_board.py @@ -3,9 +3,10 @@ lernziele judge-Rolle Backward Design — objectives BEFORE writing zuweisung code chapter/order from the outline artefact + facts grounding writer guide-Rolle ONE coherent per-block text, only from VERIFIED FACTS - fakten_gate judge-Rolle CoVe: atomic claims, each binary against the facts → minimal fix - coverage judge-Rolle objective↔section mapping; gap → back to writer (max 2 rounds) - lesbarkeit judge-Rolle Lese-Check + deterministic readability gate → fix → done + pruefer judge-Rolle EIN Call: CoVe-Fakten + Coverage + Lesbarkeit (lasen vorher + denselben Text in 3 seriellen Calls) + deterministische Gates + fix guide-Rolle EIN Rewrite unter allen Auflagen; bei falsch/Lücken danach + genau ein Re-Prüfer-Pass (der alte Lese-Fix blieb ungeprüft) Runner: one asyncio task per card (cards are fixed from the start — no queue engine needed); stage transitions are persisted in guide_cards, so the board is live and @@ -31,11 +32,10 @@ from textkit import _norm_title, _parse_fragment, _title log = logging.getLogger("creator.guide_board") -GUIDE_STAGES = ("lernziele", "zuweisung", "writer", "fakten_gate", "coverage", "lesbarkeit") +GUIDE_STAGES = ("lernziele", "zuweisung", "writer", "pruefer", "fix") STAGE_LABELS = {"lernziele": "Lernziele", "zuweisung": "Zuweisung", "writer": "Writer", - "fakten_gate": "Fakten-Gate", "coverage": "Coverage", - "lesbarkeit": "Lesbarkeit", "done": "Fertig"} -from config import GATE_FIX_MIN, MAX_WRITER_ROUNDS, WRITER_SPLIT_SUBS # zentral tunebar + "pruefer": "Prüfen", "fix": "Fix", "done": "Fertig"} +from config import GATE_FIX_MIN, WRITER_SPLIT_SUBS # zentral tunebar # Simultaneous cards = the per-topic agent cap: every card busies exactly ONE agent at a # time (its stages run serially), so a lower number just idles slots (was hardcoded 10 # from the old 10-slot era while the .env already allowed 24). @@ -87,33 +87,31 @@ def _gate_schema(data): return out -def _coverage_schema(data, ziel_ids: set[str]): - """{"ziele":{id:bool}, "luecken":[{ziel,fehlt}], "ballast":[str]} — ziele must cover all ids.""" - if not isinstance(data, dict) or not isinstance(data.get("ziele"), dict): +def _pruefer_schema(data, ziel_ids: set[str]): + """Verschmolzenes Prüfer-Verdikt: Claims (Fakten-Gate-Semantik via _gate_schema) + + Coverage (ziele/luecken/ballast) + Lesbarkeit (lese_probleme). {"ok":true} = leeres + Verdikt. `ziele` muss alle ids abdecken, wenn Ziele existieren — sonst optional.""" + if not isinstance(data, dict): + return None + if data.get("ok") is True: + return {"claims": [], "ziele": {}, "luecken": [], "ballast": [], "lese_probleme": []} + if not any(k in data for k in ("claims", "ziele", "luecken", "ballast", "lese_probleme")): + return None + claims = _gate_schema({"claims": data["claims"]}) if data.get("claims") else [] + if claims is None: return None ziele = {} - for k, v in data["ziele"].items(): + for k, v in (data.get("ziele") or {}).items() if isinstance(data.get("ziele"), dict) else []: ziele[str(k)] = str(v).strip().casefold() in ("true", "ja", "yes", "1") - if not ziel_ids <= set(ziele): + if ziel_ids and not ziel_ids <= set(ziele): return None luecken = [{"ziel": str(l.get("ziel", "")), "fehlt": str(l.get("fehlt", ""))} for l in data.get("luecken", []) if isinstance(l, dict) and str(l.get("fehlt", "")).strip()] ballast = [str(b).strip() for b in data.get("ballast", []) if str(b).strip()] - return {"ziele": ziele, "luecken": luecken, "ballast": ballast} - - -def _problems_schema(data): - """Lese-Check: {"ok":true} → [] · {"problems":[{section,problem}]} → [problem…].""" - if not isinstance(data, dict): - return None - if data.get("ok") is True: - return [] - probs = data.get("problems") - if not isinstance(probs, list) or not probs: - return None - out = [str(p.get("problem", "")).strip() for p in probs - if isinstance(p, dict) and str(p.get("problem", "")).strip()] - return out or None + lese = [str(p.get("problem", "")).strip() for p in data.get("lese_probleme", []) + if isinstance(p, dict) and str(p.get("problem", "")).strip()] + return {"claims": claims, "ziele": ziele, "luecken": luecken, "ballast": ballast, + "lese_probleme": lese} def _first_section(md: str) -> dict | None: @@ -335,11 +333,6 @@ async def _stage_writer(env: _Env, card: dict) -> bool: norm = card["block_norm"] ziele = await db.list_lernziele(env.topic, norm) ziele_text = "\n".join(f"- ({z['ziel_id']}) {z['text']}" for z in ziele) or "(keine definiert)" - gaps = "" - if card["writer_rounds"] > 0 and card.get("gate_info"): - gaps = ("\nREVISION ROUND — a previous version exists. Revise it: close exactly the gaps " - "below, cut the listed ballast, keep everything else as-is.\n" - f"PREVIOUS VERSION:\n{card.get('md', '')}\n\nGAPS/BALLAST:\n{card['gate_info']}\n") # oversized first drafts: two halves, merged into one canonical section if card["writer_rounds"] == 0 and len(env.subs_by_title.get(card["block"], [])) > WRITER_SPLIT_SUBS: text = await _write_split(env, card, ziele_text) @@ -348,7 +341,7 @@ async def _stage_writer(env: _Env, card: dict) -> bool: if text is None: await _set(env, card, status="error", gate_info="Writer (Split) ohne Ergebnis") return False - await _set(env, card, md=text, stage="fakten_gate", status="open") + await _set(env, card, md=text, stage="pruefer", status="open") return True path = env.slot(f"card-{_safe(norm)}-r{card['writer_rounds']}.md") @@ -366,7 +359,7 @@ async def _stage_writer(env: _Env, card: dict) -> bool: assignment=_card_assignment(env, card), ziele=ziele_text, facts=_card_facts(env, card["block"]), examples=await _card_examples(env, norm, env.subs_by_title.get(card["block"], [])), - gaps=gaps, spec=env.spec, + gaps="", spec=env.spec, budget=_writer_budget(len(env.subs_by_title.get(card["block"], []))), out_path=path, extra=_extra(env.instructions)), role="guide", capabilities="files", payload=_payload, @@ -376,190 +369,169 @@ async def _stage_writer(env: _Env, card: dict) -> bool: if status == FAILED: await _set(env, card, status="error", gate_info="Writer ohne Ergebnis") return False - await _set(env, card, md=text, stage="fakten_gate", status="open") + await _set(env, card, md=text, stage="pruefer", status="open") return True -async def _stage_fakten_gate(env: _Env, card: dict) -> bool: - norm = card["block_norm"] - sec = _first_section(card["md"]) - if sec is None: - await _set(env, card, status="error", gate_info="Writer-Fragment unlesbar") - return False - facts = _card_facts(env, card["block"]) - ex = await _card_examples(env, norm, env.subs_by_title.get(card["block"], [])) - if ex: # the fix agent sees the same facts variable — examples survive the fix pass - facts += "\n\nVERIFIED WORKED EXAMPLES (count as verified facts for this check):\n" + ex - path = env.slot(f"gate-{_safe(norm)}-r{card['writer_rounds']}.json") - status, claims = await run_single_slot( - env.ctx, f"Fakten-Gate {card['block']}", key=f"{env.guide_id}-gate-{_safe(norm)}-r{card['writer_rounds']}", - prompt=_prompt("Guide-Fakten-Gate", topic=env.topic, block=card["block"], - section=sec["md"], facts=facts, out_path=path, - extra=_extra(env.instructions)), - role="judge", capabilities="files", - payload=lambda result: _gate_schema(_json_file(path)), - timeout=_timeout("fakten_gate", 1)) - if status == CANCELLED: - return False - if status == FAILED: - claims = [] # gate failure must not block the card — logged, text stands - _log(env.topic, f"Fakten-Gate {card['block']}: kein Ergebnis — Text bleibt ungeprüft") - falsch = [c for c in claims if c.get("urteil") == "falsch"] if claims else [] - if claims and not falsch and len(claims) < GATE_FIX_MIN: - # 1–2 merely UNSUPPORTED claims don't justify a fix pass (it ran for 19/20 blocks, - # 40 agent-minutes) — but a WRONG claim always does: one slipped through this - # threshold and cost the guide 1.5 QA points - _log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claim(s) unter Schwelle — kein Fix") - claims = [] - if claims: - _log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claims ({len(falsch)} falsch) → Fix") - fixp = env.slot(f"gatefix-{_safe(norm)}-r{card['writer_rounds']}.md") - fixp.unlink(missing_ok=True) - claims_text = "\n".join(f"- {c['text']}" + (f" ({c['grund']})" if c["grund"] else "") - for c in claims) - - def _fixload(result): - text = fixp.read_text(encoding="utf-8") if fixp.exists() else "" - return text if _first_section(text) else None - - fstatus, fixed = await run_single_slot( - env.ctx, f"Fakten-Fix {card['block']}", key=f"{env.guide_id}-gatefix-{_safe(norm)}-r{card['writer_rounds']}", - prompt=_prompt("Guide-Fakten-Fix", topic=env.topic, block=card["block"], - section=card["md"], claims=claims_text, facts=facts, - out_path=fixp, extra=_extra(env.instructions)), - role="guide", capabilities="files", payload=_fixload, # Fix auf der Schreib-Rolle, Gate auf der Judge-Rolle - timeout=_timeout("fakten_gate", 1)) - if fstatus == CANCELLED: - return False - if fstatus == OK and fixed: - new_sec = _first_section(fixed) - # marker invariant: a fix that loses the sub markers kills the level filter → discard - if sec.get("subs") and not (new_sec and new_sec.get("subs")): - _log(env.topic, f"Fakten-Fix {card['block']} ohne Sub-Marker — verworfen") - else: - card["md"] = fixed - await _set(env, card, md=card["md"], stage="coverage", status="open") - return True - - -async def _stage_coverage(env: _Env, card: dict) -> bool: - norm = card["block_norm"] - ziele = await db.list_lernziele(env.topic, norm) - if not ziele: - await _set(env, card, stage="lesbarkeit") - return True - sec = _first_section(card["md"]) - ziele_text = "\n".join(f"- ({z['ziel_id']}) {z['text']}" for z in ziele) - path = env.slot(f"coverage-{_safe(norm)}-r{card['writer_rounds']}.json") - ids = {z["ziel_id"] for z in ziele} - status, res = await run_single_slot( - env.ctx, f"Coverage {card['block']}", key=f"{env.guide_id}-cov-{_safe(norm)}-r{card['writer_rounds']}", - prompt=_prompt("Guide-Coverage", topic=env.topic, block=card["block"], - ziele=ziele_text, section=sec["md"] if sec else card["md"], - out_path=path, extra=_extra(env.instructions)), - role="judge", capabilities="files", - payload=lambda result: _coverage_schema(_json_file(path), ids), - timeout=_timeout("coverage", len(ziele))) - if status == CANCELLED: - return False - if status == FAILED: - _log(env.topic, f"Coverage {card['block']}: kein Ergebnis — weiter ohne Gate") - await _set(env, card, stage="lesbarkeit") - return True - for zid, ok in res["ziele"].items(): - if zid in ids: - await db.set_ziel_covered(env.topic, norm, zid, ok) - if res["luecken"] and card["writer_rounds"] < MAX_WRITER_ROUNDS: - info = "\n".join(f"- Lücke ({l['ziel']}): {l['fehlt']}" for l in res["luecken"]) - if res["ballast"]: - info += "\n" + "\n".join(f"- Ballast (kürzen): {b}" for b in res["ballast"]) - _log(env.topic, f"Coverage {card['block']}: {len(res['luecken'])} Lücke(n) → Writer-Runde " - f"{card['writer_rounds'] + 1}") - await _set(env, card, writer_rounds=card["writer_rounds"] + 1, gate_info=info, - stage="writer", status="open") - return True - if res["luecken"]: - _log(env.topic, f"Coverage {card['block']}: Lücken bleiben nach {MAX_WRITER_ROUNDS} Runden") - await _set(env, card, gate_info="", stage="lesbarkeit") - return True - - -async def _stage_lesbarkeit(env: _Env, card: dict) -> bool: - norm = card["block_norm"] - sec = _first_section(card["md"]) - if sec is None: - await _set(env, card, status="error", gate_info="Fragment unlesbar") - return False - problems: list[str] = [] - path = env.slot(f"lese-{_safe(norm)}-r{card['writer_rounds']}.json") - # Text-Antwort + Engine-Sink: Datei-schreibende Judges lieferten invalides JSON - # (3 kaputte Check-Dateien im Messlauf) — der Sink validiert vor dem Persistieren - status, res = await run_single_slot( - env.ctx, f"Lese-Check {card['block']}", key=f"{env.guide_id}-lese-{_safe(norm)}", - prompt=_prompt("Guide-Lese-Check", topic=env.topic, format_name=env.format, - spec=env.spec, sections=f"SECTION: {card['block']}\n{sec['md']}", - extra=_extra(env.instructions)), - role="judge", capabilities="none", - payload=lambda result: _sink_json(result, path, _problems_schema), - timeout=_timeout("lese_check", 1)) - if status == CANCELLED: - return False - if status == OK and res: - problems += res - if READABILITY_ACTIVE: # deterministic gate, external grounding - hints = await asyncio.to_thread(readability.rate_sections, {1: sec["md"]}) - if hints.get(1): - problems.append(hints[1]) - # deterministic length trigger, same formula as the QA detector: prompt guidelines - # alone left writers 2.7–4.1× over target — a measured overshoot forces the fix pass +def _det_hinweise(env: _Env, card: dict, sec: dict) -> list[str]: + """Deterministische Befunde (extern geerdet, kein LLM): Readability-Modell + + Längen-Rahmen — dieselbe Formel wie der QA-Detektor. Gehen direkt in den Fix + und als „nicht wiederholen"-Notiz in den Prüfer-Prompt.""" + out: list[str] = [] subs_all = env.subs_by_title.get(card["block"], []) n_rel = max(sum(1 for s in subs_all if s.get("relevance") != "peripheral"), 1) aus = re.split(r"", sec["md"], maxsplit=1) pro_sub = len(aus[1] if len(aus) == 2 else sec["md"]) / n_rel if not (GUIDE_LAENGE_MIN <= pro_sub <= GUIDE_LAENGE_MAX * 0.9): ziel = _writer_budget(len(subs_all)) - problems.append( + out.append( f"Länge {round(pro_sub)} Zeichen/Sub (Rahmen {GUIDE_LAENGE_MIN}–{round(GUIDE_LAENGE_MAX * 0.9)}): " f"schreibe den ausführlich-Teil auf etwa {ziel} Zeichen GESAMT um — Sockel-Prosa und " f"Wiederholungen streichen, alle Sub-Marker und Beispiele behalten") - if problems: - from guide import _level_label - subs = env.subs_by_title.get(card["block"], []) - sub_list = "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)" - tasks = (f"SECTION: {card['block']}\n" - f"SUBBLOCKS (set one `` marker each, label/order as here):\n{sub_list}\n" - f"LENGTH TARGET: about {_writer_budget(len(subs))} characters for the detailed " - f"version (guideline — covering every subblock beats brevity).\n" - f"PROBLEM: {' · '.join(problems)}\nCURRENT CONTENT:\n{sec['md']}") - fixp = env.slot(f"lesefix-{_safe(norm)}.md") - fixp.unlink(missing_ok=True) + return out - def _fixload(result): - text = fixp.read_text(encoding="utf-8") if fixp.exists() else "" - return text if _first_section(text) else None - fstatus, fixed = await run_single_slot( - env.ctx, f"Lese-Fix {card['block']}", key=f"{env.guide_id}-lesefix-{_safe(norm)}", - prompt=_prompt("Guide-Sections-Fix", topic=env.topic, format_name=env.format, - facts=_card_facts(env, card["block"]), spec=env.spec, tasks=tasks, - out_path=fixp, extra=_extra(env.instructions)), - role="guide", capabilities="files", payload=_fixload, - timeout=_timeout("writer", 1)) - if fstatus == CANCELLED: +async def _det_readability(sec: dict) -> list[str]: + if not READABILITY_ACTIVE: + return [] + hints = await asyncio.to_thread(readability.rate_sections, {1: sec["md"]}) + return [hints[1]] if hints.get(1) else [] + + +def _auftraege(verdict: dict, det: list[str]) -> tuple[list[str], bool]: + """Prüfer-Verdikt → Fix-Auftragszeilen. kritisch = falsch-Claims oder Lücken + (nur die rechtfertigen den Re-Prüfer-Pass — Fakten/Coverage sind der Qualitätskern). + Claims-Schwelle wie beim alten Gate: 1–2 nur-„unbelegt" lohnen keinen Fix-Pass.""" + claims = verdict["claims"] + falsch = [c for c in claims if c["urteil"] == "falsch"] + if claims and not falsch and len(claims) < GATE_FIX_MIN: + claims = [] + zeilen = [f"- CLAIM ({c['urteil']}): {c['text']}" + (f" — {c['grund']}" if c['grund'] else "") + for c in claims] + zeilen += [f"- LÜCKE ({l['ziel']}): {l['fehlt']}" for l in verdict["luecken"]] + zeilen += [f"- BALLAST (kürzen): {b}" for b in verdict["ballast"]] + zeilen += [f"- LESBARKEIT: {p}" for p in verdict["lese_probleme"]] + zeilen += [f"- LESBARKEIT: {p}" for p in det] + return zeilen, bool(falsch or verdict["luecken"]) + + +async def _pruefer_call(env: _Env, card: dict, sec: dict, tag: str, det: list[str]) -> dict | None: + """EIN Judge-Call prüft Fakten + Coverage + Lesbarkeit (die drei lasen vorher denselben + Section-Text in drei seriellen Calls). Text-Antwort + Engine-Sink (Datei-schreibende + Judges lieferten invalides JSON). → Verdikt | None (FAILED/CANCELLED).""" + norm = card["block_norm"] + ziele = await db.list_lernziele(env.topic, norm) + ziele_text = "\n".join(f"- ({z['ziel_id']}) {z['text']}" for z in ziele) or "(keine definiert)" + ids = {z["ziel_id"] for z in ziele} + facts = _card_facts(env, card["block"]) + ex = await _card_examples(env, norm, env.subs_by_title.get(card["block"], [])) + if ex: # der Fix sieht dieselben Facts — Beispiele überleben den Fix-Pass + facts += "\n\nVERIFIED WORKED EXAMPLES (count as verified facts for this check):\n" + ex + hinweise = ("\nALREADY NOTED deterministically (do NOT repeat, they go to the fix anyway):\n" + + "\n".join(f"- {d}" for d in det) + "\n") if det else "\n" + path = env.slot(f"pruefer-{_safe(norm)}-{tag}.json") + status, verdict = await run_single_slot( + env.ctx, f"Prüfer {card['block']}", key=f"{env.guide_id}-pruef-{_safe(norm)}-{tag}", + prompt=_prompt("Guide-Pruefer", topic=env.topic, block=card["block"], + section=sec["md"], facts=facts, ziele=ziele_text, spec=env.spec, + hinweise=hinweise, extra=_extra(env.instructions)), + role="judge", capabilities="none", + payload=lambda result: _sink_json(result, path, lambda d: _pruefer_schema(d, ids)), + timeout=_timeout("fakten_gate", 1)) + if status != OK or verdict is None: + return None + for zid, ok in verdict["ziele"].items(): + if zid in ids: + await db.set_ziel_covered(env.topic, norm, zid, ok) + return verdict + + +async def _stage_pruefer(env: _Env, card: dict) -> bool: + """Verschmolzener Qualitäts-Pass: Fakten-Gate + Coverage + Lese-Check in EINEM Call + (vorher 3 serielle Judges + bis zu 3 Edit-Pässe, die einander überschrieben und deren + letzter ungeprüft blieb). Befunde → Fix-Stage; ohne Befund → done.""" + sec = _first_section(card["md"]) + if sec is None: + await _set(env, card, status="error", gate_info="Writer-Fragment unlesbar") + return False + det = (await _det_readability(sec)) + _det_hinweise(env, card, sec) + verdict = await _pruefer_call(env, card, sec, f"r{card['writer_rounds']}", det) + if verdict is None: + if is_guide_cancelled(env.guide_id): return False - if fstatus == OK and fixed: - new_sec = _first_section(fixed) - if sec.get("subs") and not (new_sec and new_sec.get("subs")): - _log(env.topic, f"Lese-Fix {card['block']} ohne Sub-Marker — verworfen") - else: - card["md"] = fixed - await _set(env, card, md=card["md"], stage="done", status="ok", gate_info="") + # fail-open wie das alte Gate: Karte nie blockieren — deterministische Befunde + # gehen trotzdem in den Fix + _log(env.topic, f"Prüfer {card['block']}: kein Ergebnis — nur deterministische Checks") + verdict = {"claims": [], "ziele": {}, "luecken": [], "ballast": [], "lese_probleme": []} + zeilen, kritisch = _auftraege(verdict, det) + if not zeilen: + await _set(env, card, md=card["md"], stage="done", status="ok", gate_info="") + return True + _log(env.topic, f"Prüfer {card['block']}: {len(zeilen)} Befund(e){' (kritisch)' if kritisch else ''} → Fix") + await _set(env, card, gate_info=("KRITISCH\n" if kritisch else "") + "\n".join(zeilen), + stage="fix", status="open") + return True + + +async def _stage_fix(env: _Env, card: dict) -> bool: + """EIN kompletter Section-Rewrite unter allen Auflagen (ersetzt Fakten-Fix + + Writer-Revision + Lese-Fix). Danach GENAU EIN Re-Prüfer-Pass, wenn der Fix wegen + falsch-Claims/Lücken lief — der alte Lese-Fix blieb ungeprüft. Rest-Befunde bleiben + sichtbar (gate_info), keine weitere Fix-Runde.""" + from guide import _level_label + norm = card["block_norm"] + sec = _first_section(card["md"]) + if sec is None: + await _set(env, card, status="error", gate_info="Fragment unlesbar") + return False + info = card.get("gate_info") or "" + kritisch = info.startswith("KRITISCH\n") + auftraege = info.removeprefix("KRITISCH\n") + subs = env.subs_by_title.get(card["block"], []) + sub_list = "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)" + fixp = env.slot(f"fix-{_safe(norm)}-r{card['writer_rounds']}.md") + fixp.unlink(missing_ok=True) + + def _fixload(result): + text = fixp.read_text(encoding="utf-8") if fixp.exists() else "" + return text if _first_section(text) else None + + fstatus, fixed = await run_single_slot( + env.ctx, f"Fix {card['block']}", key=f"{env.guide_id}-gfix-{_safe(norm)}-r{card['writer_rounds']}", + prompt=_prompt("Guide-Fix", topic=env.topic, format_name=env.format, block=card["block"], + section=card["md"], facts=_card_facts(env, card["block"]), spec=env.spec, + auftraege=auftraege, sub_list=sub_list, out_path=fixp, + extra=_extra(env.instructions)), + role="guide", capabilities="files", payload=_fixload, + timeout=_timeout("writer", 1)) + if fstatus == CANCELLED: + return False + angewandt = False + if fstatus == OK and fixed: + new_sec = _first_section(fixed) + # marker invariant: a fix that loses the sub markers kills the level filter → discard + if sec.get("subs") and not (new_sec and new_sec.get("subs")): + _log(env.topic, f"Fix {card['block']} ohne Sub-Marker — verworfen") + else: + card["md"] = fixed + angewandt = True + rest = "" + if kritisch and angewandt: + sec2 = _first_section(card["md"]) + verdict = await _pruefer_call(env, card, sec2, "re", []) + if verdict is None and is_guide_cancelled(env.guide_id): + return False + if verdict: + zeilen, _k = _auftraege(verdict, []) + if zeilen: + rest = "Rest-Befunde nach Fix:\n" + "\n".join(zeilen) + _log(env.topic, f"Re-Prüfer {card['block']}: {len(zeilen)} Rest-Befund(e) bleiben") + await _set(env, card, md=card["md"], stage="done", status="ok", gate_info=rest) return True _STAGE_FN = {"lernziele": _stage_lernziele, "zuweisung": _stage_zuweisung, - "writer": _stage_writer, "fakten_gate": _stage_fakten_gate, - "coverage": _stage_coverage, "lesbarkeit": _stage_lesbarkeit} + "writer": _stage_writer, "pruefer": _stage_pruefer, "fix": _stage_fix} async def _run_card(env: _Env, card: dict, sem: asyncio.Semaphore) -> None: diff --git a/backend/repair.py b/backend/repair.py index b2fc95c..9bd6d9f 100644 --- a/backend/repair.py +++ b/backend/repair.py @@ -69,6 +69,24 @@ async def _judge(template: str, topic: str, key: str, slot: str, items: list[str return verdicts +async def _mit_stichentscheid(template: str, topic: str, key: str, slot: str, + lines: list[str], befund: str) -> dict[int, str]: + """Zweitmeinung + Stichentscheid: Der Repair-Judge kann den QA-Befund kippen — bei + Dissens (QA sagt Befund, Judge sagt behalten) entscheidet ein DRITTER Judge nur über + die strittigen Items, Mehrheit 2/3 (Muster Crossblock-Tiebreaker). Ohne ihn pendelte + die Note dauerhaft unter 10 ohne Fix-Pfad (gemessen: aak-fremd 9.2, kanban-smoke- + Dublette 9.4 — „keine behebbaren Befunde" trotz Befund). j3 „behalten" oder Ausfall + → Item bleibt (fail-open).""" + v = await _judge(template, topic, key, slot, lines) + strittig = [i for i in range(1, len(lines) + 1) if v.get(i) != befund] + if strittig: + v3 = await _judge(template, topic, f"{key}-st", slot, [lines[i - 1] for i in strittig]) + for pos, i in enumerate(strittig, 1): + if v3.get(pos) == befund: + v[i] = befund # 2:1 für den QA-Befund → handeln + return v + + async def _fix_hygiene(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]: """Nur der norm-invariante Teil (`**`/Backticks); `(n)`-Suffix und leere Beschreibung ändern die Norm bzw. brauchen Inhalt — bleiben Befund.""" @@ -98,8 +116,8 @@ async def _merge_dubletten(topic: str, report: dict, by_norm: dict, files: dict) and _norm_title(p.get("a", "")) in by_norm and _norm_title(p.get("b", "")) in by_norm] if not paare: return [] - v = await _judge("QA-Dubletten", topic, "dubletten", "pairs", - [f"A: {p['a']}\nB: {p['b']}" for p in paare]) + v = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs", + [f"A: {p['a']}\nB: {p['b']}" for p in paare], "ja") merged = [] for i, p in enumerate(paare, 1): a, b = by_norm.get(_norm_title(p["a"])), by_norm.get(_norm_title(p["b"])) @@ -175,9 +193,9 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st and (a["block_norm"], a["sub_norm"]) != (b["block_norm"], b["sub_norm"])] if not paare: return [] - v = await _judge("QA-Sub-Dubletten", topic, "sub-dubletten", "pairs", - [f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}" - for a, b in paare]) + v = await _mit_stichentscheid("QA-Sub-Dubletten", topic, "sub-dubletten", "pairs", + [f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}" + for a, b in paare], "ja") merged: list[str] = [] gone: set[tuple] = set() for i, (a, b) in enumerate(paare, 1): @@ -229,7 +247,7 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: srcs = by_norm[_norm_title(t)]["payload"].get("sources") or None ev = _evidence_pack(folder, srcs, [t], budget=EVIDENCE_PER_BLOCK) if folder else "" lines.append(f"{t}\n{ev or '(keine Treffer im Material)'}") - v = await _judge("QA-Repair-Beleg", topic, "fremd", "blocks", lines) + v = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines, "nein") for i, t in enumerate(fremd, 1): if v.get(i) == "nein": await _reject(topic, t, by_norm, files, "qa-fremd") @@ -238,7 +256,7 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: if unecht: lines = [f"{t} — {by_norm[_norm_title(t)]['payload'].get('description') or '(ohne Beschreibung)'}" for t in unecht] - v = await _judge("QA-Bausteine", topic, "unecht", "blocks", lines) + v = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines, "nein") for i, t in enumerate(unecht, 1): if v.get(i) == "nein": await _reject(topic, t, by_norm, files, "qa-unecht") diff --git a/backend/tests/test_events.py b/backend/tests/test_events.py index 169f216..086a3af 100644 --- a/backend/tests/test_events.py +++ b/backend/tests/test_events.py @@ -175,10 +175,10 @@ async def test_guide_reset_card_single(testdb): assert cards["alpha"]["stage"] == "lernziele" and cards["alpha"]["md"] == "" and cards["alpha"]["writer_rounds"] == 0 assert cards["beta"]["stage"] == "done" and cards["beta"]["md"] # untouched assert await db.list_lernziele(TOPIC) and all(z["block_norm"] != "alpha" for z in await db.list_lernziele(TOPIC)) - # ab_stage 3 (fakten_gate) behält md + # ab_stage 3 (pruefer) behält md assert await gb.reset_card(TOPIC, "Guide", "beta", 3) is True cards = {c["block_norm"]: c for c in await db.list_guide_cards(TOPIC, "Guide")} - assert cards["beta"]["stage"] == "fakten_gate" and cards["beta"]["md"] + assert cards["beta"]["stage"] == "pruefer" and cards["beta"]["md"] async def test_completeness_route(testdb, tmp_path, monkeypatch): diff --git a/backend/tests/test_guide_board.py b/backend/tests/test_guide_board.py index 88733aa..b3b1235 100644 --- a/backend/tests/test_guide_board.py +++ b/backend/tests/test_guide_board.py @@ -30,19 +30,32 @@ def test_gate_schema(): assert [c["text"] for c in claims] == ["B"] -def test_coverage_schema(): - res = gb._coverage_schema({"ziele": {"z1": True, "z2": "false"}, - "luecken": [{"ziel": "z2", "fehlt": "Beweis"}], - "ballast": ["Abschweifung"]}, {"z1", "z2"}) +def test_pruefer_schema(): + """Verschmolzenes Verdikt: ok-Kurzform, Claims-Normalisierung, Ziel-Vollständigkeit.""" + assert gb._pruefer_schema({"ok": True}, {"z1"}) == { + "claims": [], "ziele": {}, "luecken": [], "ballast": [], "lese_probleme": []} + res = gb._pruefer_schema({"claims": [{"text": "A", "grund": "x", "urteil": "FALSCH"}], + "ziele": {"z1": True, "z2": "false"}, + "luecken": [{"ziel": "z2", "fehlt": "Beweis"}], + "ballast": ["Abschweifung"], + "lese_probleme": [{"problem": "zu lang"}]}, {"z1", "z2"}) + assert res["claims"] == [{"text": "A", "grund": "x", "urteil": "falsch"}] assert res["ziele"] == {"z1": True, "z2": False} - assert res["luecken"][0]["fehlt"] == "Beweis" - assert gb._coverage_schema({"ziele": {"z1": True}}, {"z1", "z2"}) is None # z2 missing + assert res["luecken"][0]["fehlt"] == "Beweis" and res["lese_probleme"] == ["zu lang"] + assert gb._pruefer_schema({"ziele": {"z1": True}}, {"z1", "z2"}) is None # z2 fehlt + assert gb._pruefer_schema({"lese_probleme": []}, set()) is not None # leeres Verdikt ok + assert gb._pruefer_schema("quatsch", set()) is None -def test_problems_schema(): - assert gb._problems_schema({"ok": True}) == [] - assert gb._problems_schema({"problems": [{"section": "S", "problem": "zu lang"}]}) == ["zu lang"] - assert gb._problems_schema({"problems": []}) is None +def test_auftraege_schwelle_und_kritisch(): + """1–2 nur-unbelegt-Claims verfallen (GATE_FIX_MIN); falsch/Lücken sind kritisch.""" + leer = {"claims": [], "ziele": {}, "luecken": [], "ballast": [], "lese_probleme": []} + z, k = gb._auftraege({**leer, "claims": [{"text": "c", "grund": "", "urteil": "unbelegt"}] * 2}, []) + assert z == [] and k is False + z, k = gb._auftraege({**leer, "claims": [{"text": "c", "grund": "w", "urteil": "falsch"}]}, []) + assert len(z) == 1 and k is True + z, k = gb._auftraege({**leer, "luecken": [{"ziel": "z1", "fehlt": "X"}]}, ["Länge 2000"]) + assert len(z) == 2 and k is True async def test_reset_from_stage(testdb): @@ -50,7 +63,7 @@ async def test_reset_from_stage(testdb): await db.upsert_guide_card(TOPIC, FMT, "a", "A") await db.upsert_guide_card(TOPIC, FMT, "b", "B") await db.set_guide_card(TOPIC, FMT, "a", stage="done", md="text", writer_rounds=2) - await db.set_guide_card(TOPIC, FMT, "b", stage="coverage", md="text") + await db.set_guide_card(TOPIC, FMT, "b", stage="pruefer", md="text") await db.put_lernziel(TOPIC, "a", "z1", "Ziel") # reset ab writer (idx 2): beide Karten zurück, md geleert, Ziele bleiben moved = await gb.reset_from_stage(TOPIC, FMT, 2) @@ -70,8 +83,8 @@ async def test_done_step(testdb): assert await gb.done_step(TOPIC, FMT) == -1 await db.upsert_guide_card(TOPIC, FMT, "a", "A") assert await gb.done_step(TOPIC, FMT) == -1 # alles in lernziele - await db.set_guide_card(TOPIC, FMT, "a", stage="coverage") - assert await gb.done_step(TOPIC, FMT) == 3 # bis fakten_gate fertig + await db.set_guide_card(TOPIC, FMT, "a", stage="pruefer") + assert await gb.done_step(TOPIC, FMT) == 2 # bis writer fertig await db.set_guide_card(TOPIC, FMT, "a", stage="done") assert await gb.done_step(TOPIC, FMT) == len(gb.GUIDE_STAGES) @@ -162,8 +175,8 @@ def test_writer_template_has_examples_placeholder(): assert "VERIFIED FACTS" in text and "2000 characters" in text -async def test_fakten_gate_counts_examples_as_facts(testdb, monkeypatch, tmp_path): - """Gate-Prompt enthält die Beispiele als verifizierte Fakten — sonst fliegen +async def test_pruefer_counts_examples_as_facts(testdb, monkeypatch, tmp_path): + """Prüfer-Prompt enthält die Beispiele als verifizierte Fakten — sonst fliegen gerechnete Beispielwerte als „nicht belegt" raus.""" import json as _json import guide_board as gb @@ -177,18 +190,19 @@ async def test_fakten_gate_counts_examples_as_facts(testdb, monkeypatch, tmp_pat async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None): captured["prompt"] = prompt - return "ok", [] + return "ok", payload((0, '{"ok": true}', "")) monkeypatch.setattr(gb, "run_single_slot", fake_slot) monkeypatch.setattr(gb, "_card_facts", lambda e, b: "FAKT X") + monkeypatch.setattr(gb, "READABILITY_ACTIVE", False) env = SimpleNamespace(ctx=SimpleNamespace(topic="t", provider="p", is_cancelled=lambda: False), guide_id="g", topic="t", format="Guide", instructions="", subs_by_title={"Gross": [{"title": "Sub Eins", "level": "beginner"}]}, spec="", slot=lambda name: tmp_path / name) - card = {"block_norm": "gross", "block": "Gross", "stage": "fakten_gate", "status": "open", - "writer_rounds": 0, "gate_info": "", - "md": "\n\nText."} - ok = await gb._stage_fakten_gate(env, card) + md = ("\n\n" + "Text im Rahmen. " * 20) + card = {"block_norm": "gross", "block": "Gross", "stage": "pruefer", "status": "open", + "writer_rounds": 0, "gate_info": "", "md": md} + ok = await gb._stage_pruefer(env, card) assert ok is True assert "VERIFIED WORKED EXAMPLES" in captured["prompt"] and "P1" in captured["prompt"] @@ -229,7 +243,7 @@ async def test_writer_splits_oversized_first_draft(testdb, monkeypatch, tmp_path from textkit import _parse_fragment secs = _parse_fragment(card["md"]) assert len(secs) == 1 and [s["title"] for s in secs[0]["subs"]] == ["Sub 1", "Sub 2"] - assert card["stage"] == "fakten_gate" + assert card["stage"] == "pruefer" async def test_lernziele_retry_bei_leerer_liste(testdb, tmp_path, monkeypatch): @@ -268,8 +282,8 @@ async def test_lernziele_zweimal_leer_laeuft_weiter(testdb, tmp_path, monkeypatc assert not await db.list_lernziele(TOPIC, "alpha") -async def test_lese_check_text_sink(testdb, tmp_path, monkeypatch): - """Lese-Check antwortet als Text, Engine-Sink persistiert; capabilities none.""" +async def test_pruefer_text_sink_ohne_befund_done(testdb, tmp_path, monkeypatch): + """Prüfer antwortet als Text (Engine-Sink, capabilities none); ohne Befund → done.""" db = testdb await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha") env = gb._Env(None, "g-l", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec") @@ -284,7 +298,7 @@ async def test_lese_check_text_sink(testdb, tmp_path, monkeypatch): monkeypatch.setattr(gb, "run_single_slot", fake_slot) monkeypatch.setattr(gb, "READABILITY_ACTIVE", False) - assert await gb._stage_lesbarkeit(env, card) + assert await gb._stage_pruefer(env, card) assert seen["caps"] == "none" assert (await db.list_guide_cards(TOPIC, FMT))[0]["stage"] == "done" @@ -307,24 +321,22 @@ async def test_writer_prompt_traegt_budget(testdb, tmp_path, monkeypatch): assert str(gb._writer_budget(2)) in seen["prompt"] # 800 + 2×400 -async def test_fakten_gate_schwelle(testdb, tmp_path, monkeypatch): - """1–2 Claims → kein Fix-Rewrite (nur Log); ab GATE_FIX_MIN läuft der Fix.""" +async def test_pruefer_claims_schwelle(testdb, tmp_path, monkeypatch): + """1–2 nur-unbelegt-Claims → kein Fix (Karte direkt done); die Schwelle lebt in _auftraege.""" db = testdb await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha") env = gb._Env(None, "g-g", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec") - md = "\n\nText." + md = "\n\n" + "Text im Rahmen. " * 20 card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md} - keys = [] async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout): - keys.append(key) - if "-gate-" in key: - return gb.OK, [{"text": "c1", "grund": ""}, {"text": "c2", "grund": ""}] - raise AssertionError("Fix darf unter der Schwelle nicht laufen") + antwort = '{"claims": [{"text": "c1", "grund": ""}, {"text": "c2", "grund": ""}]}' + return gb.OK, payload((0, antwort, "")) monkeypatch.setattr(gb, "run_single_slot", fake_slot) - assert await gb._stage_fakten_gate(env, card) - assert all("-gate-" in k for k in keys) + monkeypatch.setattr(gb, "READABILITY_ACTIVE", False) + assert await gb._stage_pruefer(env, card) + assert (await db.list_guide_cards(TOPIC, FMT))[0]["stage"] == "done" async def test_load_subblocks_defaultet_levellose(testdb): @@ -340,30 +352,46 @@ async def test_load_subblocks_defaultet_levellose(testdb): assert by_title == {"Mit Level": "beginner", "Ohne Level": "advanced"} -async def test_fakten_gate_falsch_claim_erzwingt_fix(testdb, tmp_path, monkeypatch): - """Ein einzelner FALSCH-Claim läuft in den Fix, auch unter GATE_FIX_MIN — - ein durchgerutschter kostete den Guide 1.5 QA-Punkte.""" +async def test_falsch_claim_erzwingt_fix_und_repruefer(testdb, tmp_path, monkeypatch): + """Ein einzelner FALSCH-Claim läuft in den Fix, auch unter GATE_FIX_MIN; nach + angewandtem Fix läuft GENAU EIN Re-Prüfer-Pass (der alte Lese-Fix blieb ungeprüft).""" db = testdb await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha") env = gb._Env(None, "g-gf", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec") - md = "\n\nText." - card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md} + md = "\n\n" + "Text im Rahmen. " * 20 + card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md, "gate_info": ""} keys = [] async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout): keys.append(key) - if "-gate-" in key: - return gb.OK, [{"text": "c1", "grund": "widerspricht", "urteil": "falsch"}] - return gb.FAILED, None # Fix-Agent liefert nichts — Text bleibt, aber der Call MUSS kommen + if "-pruef-" in key and key.endswith("-re"): + return gb.OK, payload((0, '{"ok": true}', "")) + if "-pruef-" in key: + return gb.OK, payload((0, '{"claims": [{"text": "c1", "grund": "widerspricht", "urteil": "falsch"}]}', "")) + if "-gfix-" in key: # Fix liefert eine valide Section über die Slot-Datei + import re as _re + m = _re.search(r"(/\S+\.md)", prompt) + with open(m.group(1), "w", encoding="utf-8") as f: + f.write(md.replace("Text im Rahmen.", "Korrigiert.")) + return gb.OK, payload(None) + raise AssertionError(key) monkeypatch.setattr(gb, "run_single_slot", fake_slot) - assert await gb._stage_fakten_gate(env, card) - assert any("-gatefix-" in k for k in keys) + monkeypatch.setattr(gb, "READABILITY_ACTIVE", False) + assert await gb._stage_pruefer(env, card) + karte = (await db.list_guide_cards(TOPIC, FMT))[0] + assert karte["stage"] == "fix" and karte["gate_info"].startswith("KRITISCH") + card.update(stage="fix", gate_info=karte["gate_info"]) + assert await gb._stage_fix(env, card) + assert any("-gfix-" in k for k in keys) + assert any(k.endswith("-re") for k in keys) # Re-Prüfer lief + karte = (await db.list_guide_cards(TOPIC, FMT))[0] + assert karte["stage"] == "done" and "Korrigiert." in karte["md"] -async def test_laengen_trigger_startet_lesefix(testdb, tmp_path, monkeypatch): - """Ausführlich-Teil über der Obergrenze → deterministisches Längen-Problem - mit hartem Zeichenziel landet im Lese-Fix-Auftrag.""" +async def test_laengen_trigger_startet_fix(testdb, tmp_path, monkeypatch): + """Ausführlich-Teil über der Obergrenze → deterministisches Längen-Problem mit hartem + Zeichenziel landet als Auftrag in der Fix-Stage (unkritisch → kein Re-Prüfer).""" db = testdb await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha") env = gb._Env(None, "g-lz", TOPIC, FMT, "", tmp_path / "Guide.json", @@ -371,16 +399,21 @@ async def test_laengen_trigger_startet_lesefix(testdb, tmp_path, monkeypatch): {}, "(q)", "spec") md = ("\n\n- x\n\n" + "Viel zu langer Sockeltext. " * 80) # ~2160 Z./Sub > 1200×0.9 - card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md} + card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md, "gate_info": ""} seen = {} async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout): - if "-lesefix-" in key: - seen["tasks"] = prompt + if "-gfix-" in key: + seen["auftraege"] = prompt return gb.FAILED, None - return gb.OK, payload((0, '{"ok": true}', "")) # Lese-Check: keine Probleme + if key.endswith("-re"): + raise AssertionError("unkritischer Befund darf keinen Re-Prüfer starten") + return gb.OK, payload((0, '{"ok": true}', "")) # Prüfer: keine LLM-Befunde monkeypatch.setattr(gb, "run_single_slot", fake_slot) monkeypatch.setattr(gb, "READABILITY_ACTIVE", False) - assert await gb._stage_lesbarkeit(env, card) - assert "Länge" in seen["tasks"] and str(gb._writer_budget(1)) in seen["tasks"] + assert await gb._stage_pruefer(env, card) + assert (await db.list_guide_cards(TOPIC, FMT))[0]["stage"] == "fix" + card.update(stage="fix", gate_info=(await db.list_guide_cards(TOPIC, FMT))[0]["gate_info"]) + assert await gb._stage_fix(env, card) + assert "Länge" in seen["auftraege"] and str(gb._writer_budget(1)) in seen["auftraege"] diff --git a/backend/tests/test_repair.py b/backend/tests/test_repair.py index a372f42..1de4271 100644 --- a/backend/tests/test_repair.py +++ b/backend/tests/test_repair.py @@ -86,6 +86,8 @@ async def test_fremd_removed_only_on_nein(env, monkeypatch): write_report(_report(fremd=["Fremdling", "Echter"])) async def fake_agent(key, prompt, timeout, **kw): + if "-st-" in key: # Stichentscheid über den strittigen „Echter": behalten + return 0, '{"relevant": {"1": "ja"}}', "" return 0, '{"relevant": {"1": "nein", "2": "ja"}}', "" monkeypatch.setattr(repair, "run_agent", fake_agent) @@ -204,6 +206,45 @@ async def test_sub_dubletten_zweitmeinung_nein(env, monkeypatch): assert rows["sub a"] == rows["sub b"] == "consensus" +async def test_sub_dubletten_stichentscheid_faltet(env, monkeypatch): + """Dissens QA (Befund) vs. Zweitmeinung (behalten) → Stichentscheid-Judge (Key -st) + entscheidet mit 2:1 für den Befund → Merge. Vorher pendelte die Note dauerhaft + unter 10 ohne Fix-Pfad („keine behebbaren Befunde" trotz Befund).""" + db, seed, files, write_report = env + await seed("Alpha", "beschr") + norm = repair._norm_title("Alpha") + await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A", + facts='{"key_points": ["a"]}', status="consensus") + await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus") + write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}])) + + async def fake_agent(key, prompt, timeout, **kw): + if "-st-" in key: # Stichentscheid bestätigt den QA-Befund + return 0, '{"relevant": {"1": "ja"}}', "" + return 0, '{"relevant": {"1": "nein"}}', "" # Zweitmeinung widerspricht + + monkeypatch.setattr(repair, "run_agent", fake_agent) + res = await repair.repair_befunde(TOPIC) + assert res["sub_merges"] == ["Sub B → Sub A"] + rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)} + assert rows["sub b"] == "variant" and rows["sub a"] == "consensus" + + +async def test_fremd_stichentscheid_behalten(env, monkeypatch): + """Dissens bei fremd, Stichentscheid sagt ebenfalls behalten (ja) → Block bleibt + (fail-open bei 1:2 gegen den Befund).""" + db, seed, files, write_report = env + await seed("Alpha", "beschr") + write_report(_report(fremd=["Alpha"])) + + async def fake_agent(key, prompt, timeout, **kw): + return 0, '{"relevant": {"1": "ja"}}', "" # beide: belegt/behalten + + monkeypatch.setattr(repair, "run_agent", fake_agent) + res = await repair.repair_befunde(TOPIC) + assert res["entfernt"] == [] + + async def test_waisen_cleanup(env, monkeypatch): """Artefakte/Fragen auf verworfene oder fehlende Subs fliegen; lebende und mehrdeutig-präfixige bleiben.""" diff --git a/backend/train_params.py b/backend/train_params.py index 7c87246..2443394 100644 --- a/backend/train_params.py +++ b/backend/train_params.py @@ -29,7 +29,6 @@ PARAMS: dict[str, dict] = { "EMBEDDING_BLOCK_FLOOR": {"default": 0.5, "min": 0.35, "max": 0.65, "step": 0.05, "kategorie": "auswahl", "fidelity": "voll"}, "CROSS_CHUNK_PAARE": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit", "fidelity": "board2"}, # Guide - "MAX_WRITER_ROUNDS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet", "fidelity": "board2"}, "GATE_FIX_MIN": {"default": 3, "min": 1, "max": 6, "step": 1, "kategorie": "qualitaet", "fidelity": "board2"}, "WRITER_SPLIT_SUBS": {"default": 30, "min": 15, "max": 45, "step": 5, "kategorie": "qualitaet", "fidelity": "board2"}, # Engine / Kosten diff --git a/templates/Prompt/Guide-Coverage.md b/templates/Prompt/Guide-Coverage.md deleted file mode 100644 index 50046ec..0000000 --- a/templates/Prompt/Guide-Coverage.md +++ /dev/null @@ -1,19 +0,0 @@ -Coverage gate ("Was fehlt?") for ONE written guide section on the topic "{topic}": check the text against its learning objectives. Objective without content = gap; content without objective = ballast. - -LEARNING OBJECTIVES of block "{block}": -{ziele} - -SECTION — current content (subblocks are marked with ``): -{section} - -Procedure: -1. For EACH objective decide binary: does the text teach it well enough that a beginner could achieve the objective afterwards? Mentioning a keyword is NOT teaching — the how/why must be there. -2. For each NOT-covered objective state precisely WHAT is missing (German, concrete — the writer will patch exactly this). -3. List BALLAST: passages that serve none of the objectives (digressions, redundant repetition). Shortening candidates only — never a whole subblock. Worked-example passages that apply a concept belonging to an objective are teaching, not ballast. -4. Judge strictly binary per objective; no partial credit. - -Write ONLY the JSON file to: {out_path} - -Format (`ziele` maps EVERY objective id to true/false): -{{"ziele": {{"z1": true, "z2": false}}, "luecken": [{{"ziel": "z2", "fehlt": "…"}}], "ballast": ["passage …"]}} -{extra} diff --git a/templates/Prompt/Guide-Fakten-Fix.md b/templates/Prompt/Guide-Fakten-Fix.md deleted file mode 100644 index 97a3447..0000000 --- a/templates/Prompt/Guide-Fakten-Fix.md +++ /dev/null @@ -1,19 +0,0 @@ -Remove unsupported claims from ONE guide section on the topic "{topic}" — minimal edit, everything else stays VERBATIM. - -SECTION (block "{block}") — current content: -{section} - -UNSUPPORTED CLAIMS (from the fact gate) — ONLY these may be touched: -{claims} - -VERIFIED FACTS (the only allowed factual basis — for rephrasing, if a claim can be corrected instead of removed): -{facts} - -Rules: -- Per claim: correct it IF the verified facts state the right version; otherwise DELETE it (smooth the surrounding sentence so the text stays fluent). -- Everything else stays word-for-word identical — no rewriting, no shortening, no new content. -- STRUCTURE INVARIANT (mandatory, the level filter dies without it): keep ALL marker lines exactly — ``, ``, ``, `` and every `` in BOTH blocks, same order. Never delete a whole subblock; if all its claims fall, keep a minimal supported sentence. -- German text, same tone as the original. - -Write ONLY the file {out_path} — the COMPLETE corrected section in exactly the original marker format. -{extra} diff --git a/templates/Prompt/Guide-Fakten-Gate.md b/templates/Prompt/Guide-Fakten-Gate.md deleted file mode 100644 index f9ffcd6..0000000 --- a/templates/Prompt/Guide-Fakten-Gate.md +++ /dev/null @@ -1,23 +0,0 @@ -Fact-check ONE written guide section on the topic "{topic}" — Chain-of-Verification style, binary per claim. - -SECTION (block "{block}"): -{section} - -VERIFIED FACTS — the ONLY allowed factual basis (extract-once from the source): -{facts} - -Procedure: -1. Decompose the section text into its ATOMIC factual claims (definitions, properties, numbers, procedure steps, causal statements). Ignore pure didactics (transitions, framing, mnemonic phrasing). -2. Worked-example passages (a concrete problem worked through in steps to a result) are DIDACTICS when they merely APPLY or ILLUSTRATE a verified fact or a provided worked example: their concretely chosen values and computed intermediates do NOT count as over-specific. Flag an example step ONLY if it contradicts the facts or smuggles in a NEW general claim not derivable from them. -3. CONTEXT sentences are DIDACTICS too, not claims: introductions and summaries that only preview/recap the section, uncontroversial general knowledge that merely places the topic (history, origin, what an adjacent well-known technology is), and paraphrases of the verified facts. Ignore them — a guide needs connective tissue. This exemption ends the moment a sentence makes a checkable statement about THIS block's own syntax, behavior or rules: that is a claim. -4. Check EACH claim INDIVIDUALLY and BINARY against the VERIFIED FACTS above: derivable from them → belegt. Not derivable, contradicting, or over-specific beyond the facts → nicht belegt. For unsupported claims set "urteil": **"falsch"** when the claim CONTRADICTS the verified facts or contradicts the section itself (e.g. a rule its own example violates); **"unbelegt"** when it is merely not derivable from the facts. -5. Do NOT search the web, do NOT use outside knowledge as EVIDENCE — a claim about the block that is true in the world but absent from the facts is still "nicht belegt". -6. When a sentence is genuinely ambiguous between context and claim → treat it as a claim (safety before cost). - -Write ONLY the JSON file to: {out_path} - -Format — everything supported: -{{"ok": true}} -Otherwise (ONLY unsupported claims, VERBATIM as they appear in the text — NEVER list supported ones): -{{"claims": [{{"text": "verbatim claim from the section", "grund": "why unsupported (German, short)", "urteil": "falsch|unbelegt"}}]}} -{extra} diff --git a/templates/Prompt/Guide-Fix.md b/templates/Prompt/Guide-Fix.md new file mode 100644 index 0000000..5a2d1b3 --- /dev/null +++ b/templates/Prompt/Guide-Fix.md @@ -0,0 +1,27 @@ +Revise ONE guide section on the topic "{topic}" (format: {format_name}) in a single pass — fix exactly the findings below, everything else stays unchanged in substance. + +SECTION (block "{block}") — current content: +{section} + +VERIFIED FACTS (the only allowed factual basis): +{facts} + +SECTION SPECIFICATION: +{spec} + +FINDINGS TO FIX: +{auftraege} + +Rules per finding type: +- **Claims (falsch/unbelegt):** correct IF the verified facts state the right version, otherwise DELETE and smooth the surrounding sentence. Touch nothing else. +- **Lücken:** add the missing content for the named objective — grounded in the verified facts, at the fitting subblock, beginner-friendly. +- **Ballast:** shorten the named passages without information loss — never drop a subblock. +- **Lesbarkeit/Länge:** fix exactly the noted problems (split sentences, lists instead of prose enumerations, cut repetition/filler; hit the stated length target when one is given). + +STRUCTURE INVARIANT (mandatory — the level filter dies without it): keep the marker skeleton exactly — ``, ``, ``, `` and one `` per subblock in BOTH blocks, labels/titles/order as in the subblock list: +{sub_list} + +Write the revised section in GERMAN, same tone as the original. + +Write ONLY the file {out_path} — the COMPLETE revised section in exactly the original marker format. No text outside the section. +{extra} diff --git a/templates/Prompt/Guide-Lese-Check.md b/templates/Prompt/Guide-Lese-Check.md deleted file mode 100644 index 1275b07..0000000 --- a/templates/Prompt/Guide-Lese-Check.md +++ /dev/null @@ -1,33 +0,0 @@ -Review written sections of a learning guide on the topic "{topic}" (format: {format_name}) for readability. -Audience: beginners. - -SECTION SPECIFICATION (target state): -{spec} - -SECTIONS: -{sections} - -Review each section: -1. Does the section teach the concept understandably for a beginner with no prior knowledge — does it frame it, explain the how/why, make an example concrete? Not so dense that only someone who already knows the topic can follow it. -2. Readability (note genuine flaws): - - Sentences over ~20 words (never over 25), or nested sentences with several interjections. - - An enumeration (steps/options/requirements) written as one long prose sentence that should be a Markdown list. - - Wall of text: one dense block without paragraphs that could be split into several. - - More than ~4 new technical terms left unexplained at first occurrence. -3. Conciseness (superfluous material harms learning — note genuine flaws): - - Repetition: the same statement multiple times, just reworded. - - Trivia spelled out at length, filler sentences, preambles, meta-comments with no new content. - - An example that contributes nothing to understanding. - → note as "too long/redundant — shorten without loss of information". IMPORTANT: shortening never means dropping a subblock — each one stays. -4. Are the examples short, simple, plausibly correct — and in the topic-appropriate format per the specification (no code block around prose examples, no prose pseudo-example where code is required)? -5. Is the Markdown clean (no broken code blocks, no placeholders, no foreign text)? - -You only REVIEW and note problems — you change nothing. Note only genuine flaws, no matters of taste. - -Reply with ONLY the JSON — no other text, no code fences. - -Format — all in order: -{{"ok": true}} -Otherwise (section title EXACTLY as above): -{{"problems": [{{"section": "exact section title", "problem": "…"}}]}} -{extra} diff --git a/templates/Prompt/Guide-Pruefer.md b/templates/Prompt/Guide-Pruefer.md new file mode 100644 index 0000000..3bb120c --- /dev/null +++ b/templates/Prompt/Guide-Pruefer.md @@ -0,0 +1,36 @@ +You are the combined quality gate for ONE written guide section on the topic "{topic}" — fact check (Chain-of-Verification style), coverage against the learning objectives, and readability. One verdict, three lenses. + +SECTION (block "{block}") — subblocks are marked with ``: +{section} + +VERIFIED FACTS — the ONLY allowed factual basis (extract-once from the source): +{facts} + +LEARNING OBJECTIVES of this block: +{ziele} + +SECTION SPECIFICATION (target state for readability): +{spec} +{hinweise} +LENS 1 — FACTS (binary per claim): +1. Decompose the text into its ATOMIC factual claims (definitions, properties, numbers, procedure steps, causal statements). Ignore pure didactics: transitions, framing, previews/recaps, uncontroversial placement knowledge, paraphrases of the verified facts. Worked-example passages that merely APPLY a verified fact are didactics; flag an example step ONLY if it contradicts the facts or smuggles in a NEW general claim. +2. Check each claim against the VERIFIED FACTS only — no web, no outside knowledge as evidence. Not derivable → claim with "urteil": **"falsch"** when it CONTRADICTS the facts or the section itself, **"unbelegt"** when merely not derivable. List ONLY unsupported claims, VERBATIM. +3. Genuinely ambiguous between context and claim → treat as claim. + +LENS 2 — COVERAGE (binary per objective): +4. For EACH objective id: does the text teach it well enough that a beginner could achieve it afterwards (mentioning a keyword is NOT teaching)? `ziele` maps EVERY id to true/false. +5. Per uncovered objective one `luecken`-entry: WHAT exactly is missing (German, concrete — the fix will patch exactly this). +6. `ballast`: passages serving none of the objectives (digressions, redundant repetition) — shortening candidates only, never a whole subblock. + +LENS 3 — READABILITY (note genuine flaws only, no taste): +7. Beginner-followable (framing, how/why, concrete example)? Sentences over ~25 words or nested; enumerations as prose instead of lists; walls of text; >4 unexplained new terms; repetition/filler/preambles; examples that add nothing; broken Markdown. → one `lese_probleme`-entry each (German, short). + +Everything in order → `{{"ok": true}}`. + +Reply with ONLY the JSON — no other text, no code fences. Format: +{{"claims": [{{"text": "verbatim claim", "grund": "warum unbelegt (kurz)", "urteil": "falsch|unbelegt"}}], + "ziele": {{"z1": true, "z2": false}}, + "luecken": [{{"ziel": "z2", "fehlt": "…"}}], + "ballast": ["passage …"], + "lese_probleme": [{{"problem": "…"}}]}} +{extra} diff --git a/templates/Prompt/Guide-Sections-Fix.md b/templates/Prompt/Guide-Sections-Fix.md deleted file mode 100644 index 7c95106..0000000 --- a/templates/Prompt/Guide-Sections-Fix.md +++ /dev/null @@ -1,38 +0,0 @@ -Revise individual sections of a learning guide on the topic "{topic}" (format: {format_name}). - -{facts} - -SECTION SPECIFICATION: -{spec} - -TO REVISE — per section, its subblocks, the noted problem, and the current content: -{tasks} - -Per section, fix ONLY the noted problem; whatever is in order stays unchanged in substance. - -MANDATORY STRUCTURE (same as the writer) — otherwise the staged display breaks: -- Each section has TWO versions: **compact** (one mnemonic per subblock) and **detailed** (the `ausführlich` layer: a coherent beginner text). -- In BOTH versions, each subblock carries a `` marker. LABEL and title EXACTLY from the subblock list above, same order in both blocks. The detailed text starts with a short anchor (framing) BEFORE the first marker. -- The markers are invisible interfaces, NOT visible headings — write fluently regardless. - -KEEP IT CONCISE (if the problem is "too long/redundant"): -- Every sentence carries new information. Cut repetition, filler and meta sentences, preambles. -- Explain trivia briefly; give depth only where the material needs it. When in doubt, leave it out, don't add. -- An example only where it genuinely carries the understanding — not dutifully for every subblock. -- Shortening never means dropping a subblock. Each one stays with its marker. - -Write all revised sections in GERMAN (the guide is for German-speaking learners), even though these instructions are in English. - -Write ONLY the file {out_path} in EXACTLY this format — one section marker per flagged section (title EXACTLY as above): - - - - -- one mnemonic per subblock (concise, no explanation) - -Anchor: brief framing of the block — before the first subblock. - -Beginner-friendly prose for this subblock. - -Write the marker lines exactly like that. No text outside the sections. -{extra}