From 219993ffd5149acd4844728cfb6c106250bbfba2 Mon Sep 17 00:00:00 2001 From: Team3 Date: Sun, 5 Jul 2026 00:38:03 +0200 Subject: [PATCH] update --- backend/qa.py | 36 ++++++++++- backend/repair.py | 69 ++++++++++++++++------ backend/tests/test_repair.py | 29 +++++++++ frontend/src/components/GenerationView.vue | 4 +- 4 files changed, 115 insertions(+), 23 deletions(-) diff --git a/backend/qa.py b/backend/qa.py index ed8a862..482c1a5 100644 --- a/backend/qa.py +++ b/backend/qa.py @@ -294,6 +294,24 @@ async def _llm_verdicts(template: str, topic: str, key: str, items: list[str]) - # ── Report ────────────────────────────────────────────────────────────────────────── +def freispruch_pfad(topic: str) -> Path: + return QA_DIR / topic / "freispruch.json" + + +def _paar_key(a: str, b: str) -> str: + return "||".join(sorted((_norm_title(a), _norm_title(b)))) + + +def lade_freispruch(topic: str) -> dict[str, list[str]]: + """Persistierte 2:1-Freisprüche des Repair-Stichentscheids (repair._mit_stichentscheid): + mehrheitlich als „behalten" geurteilte Befunde zählen nicht mehr in die Note — sonst + pendelte sie dauerhaft unter 10 ohne Fix-Pfad (gemessen: kanban-smoke 9.4, aak 9.2). + Die Detektoren bleiben unverändert; ein Freispruch ist ein persistiertes Urteil, + kein Detektor-Tuning. Freigesprochene bleiben im Report sichtbar.""" + d = _json_file(freispruch_pfad(topic)) + return d if isinstance(d, dict) else {} + + async def qa_report(topic: str, llm: bool = False) -> dict | None: cards = await db.kanban_cards(topic, board="inventory", stage="done_block") if not cards: @@ -317,6 +335,11 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None: hy = hygiene(blocks) n_sections = sum(len(_sections(t)) for t in corpus.values()) or 1 + frei = lade_freispruch(topic) + frei_fremd = set(frei.get("fremd") or []) + fremd_frei = [t for t in fr if _norm_title(t) in frei_fremd] + fr = [t for t in fr if _norm_title(t) not in frei_fremd] + if llm and d: v = await _llm_verdicts("QA-Dubletten", topic, "dubletten", [f"A: {p['a']}\nB: {p['b']}" for p in d[:LLM_SAMPLE]]) @@ -334,6 +357,10 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None: [f"A: {p['a']}\nB: {p['b']}" for p in chunk]) for k, p in enumerate(chunk, 1): p["llm"] = v.get(k, "?") + frei_sub = set(frei.get("sub_dubletten") or []) + for p in sd: + if p.get("llm") == "ja" and _paar_key(p["a"], p["b"]) in frei_sub: + p["freispruch"] = True # 2:1-Urteil „behalten" — sichtbar, aber notenfrei unecht: list[str] | None = None if llm and blocks: verdacht = [] @@ -350,6 +377,8 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None: v2 = await _llm_verdicts("QA-Bausteine", topic, "bausteine-b2", [f"{b['title']} — {b['description'] or '(ohne Beschreibung)'}" for b in verdacht]) unecht = [b["title"] for k, b in enumerate(verdacht, 1) if v2.get(k) == "nein"] + frei_unecht = set(frei.get("unecht") or []) + unecht = [t for t in unecht if _norm_title(t) not in frei_unecht] art_rows = [dict(r) for r in await db.get_sub_artefakte(topic)] fragen = [dict(r) for r in await db.list_question_pattern(topic)] @@ -362,8 +391,10 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None: n_cons = sum(1 for r in sub_rows if r["status"] == "consensus") if n_cons: quoten_art["sub_dubletten_verdacht"] = round(len(sd) / n_cons, 3) - if llm: # confirmed pairs only — the bare candidate list is suspicion, not damage - quoten_art["sub_dubletten"] = round(sum(1 for p in sd if p.get("llm") == "ja") / n_cons, 3) + if llm: # confirmed pairs only — the bare candidate list is suspicion, not damage; + # freigesprochene (2:1 „behalten") zählen nicht mehr + quoten_art["sub_dubletten"] = round( + sum(1 for p in sd if p.get("llm") == "ja" and not p.get("freispruch")) / n_cons, 3) summary = _json_file(arbeit_dir(topic) / "lauf-summary.json") or {} report = { "topic": topic, "erstellt": datetime.now(timezone.utc).isoformat(), @@ -377,6 +408,7 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None: }, "quoten_artefakte": quoten_art, **({"unecht": unecht} if unecht is not None else {}), + **({"fremd_freigesprochen": fremd_frei} if fremd_frei else {}), "dubletten": d, "sub_dubletten": sd, "luecken": lk, "fremd": fr, "beleg": bl, "hygiene": hy, "artefakte": art, "lauf": summary, diff --git a/backend/repair.py b/backend/repair.py index 9bd6d9f..2347718 100644 --- a/backend/repair.py +++ b/backend/repair.py @@ -38,8 +38,8 @@ async def repair_befunde(topic: str) -> dict: hygiene = await _fix_hygiene(topic, report, by_norm, files) merges = await _merge_dubletten(topic, report, by_norm, files) - sub_merges = await _merge_sub_dubletten(topic, report, files) - entfernt = await _entferne_fremd_unecht(topic, report, by_norm, files) + sub_merges, frei_subs = await _merge_sub_dubletten(topic, report, files) + entfernt, frei_bloecke = await _entferne_fremd_unecht(topic, report, by_norm, files) aufgeraeumt = await _raeume_waisen(topic) # llm=True: gleiche Messlatte wie QA-Button/Abschluss-QA — der llm=False-Report @@ -48,7 +48,8 @@ async def repair_befunde(topic: str) -> dict: if neu: await qa.write_report(neu) return {"hygiene": hygiene, "merges": merges, "sub_merges": sub_merges, "entfernt": entfernt, - "aufgeraeumt": aufgeraeumt, "braucht_research": len(report.get("luecken", []))} + "aufgeraeumt": aufgeraeumt, "freigesprochen": frei_subs + frei_bloecke, + "braucht_research": len(report.get("luecken", []))} async def _judge(template: str, topic: str, key: str, slot: str, items: list[str]) -> dict[int, str]: @@ -69,22 +70,42 @@ async def _judge(template: str, topic: str, key: str, slot: str, items: list[str return verdicts +def _speichere_freispruch(topic: str, kategorie: str, keys: list[str]) -> None: + d = qa.lade_freispruch(topic) + alt = set(d.get(kategorie) or []) + d[kategorie] = sorted(alt | set(keys)) + qa.freispruch_pfad(topic).parent.mkdir(parents=True, exist_ok=True) + atomic_write_json(qa.freispruch_pfad(topic), d, indent=1) + + async def _mit_stichentscheid(template: str, topic: str, key: str, slot: str, - lines: list[str], befund: str) -> dict[int, str]: + lines: list[str], befund: str, kategorie: str = "", + ids: list[str] | None = None) -> tuple[dict[int, str], list[str]]: """Zweitmeinung + Stichentscheid: Der Repair-Judge kann den QA-Befund kippen — bei Dissens (QA sagt Befund, Judge sagt behalten) entscheidet ein DRITTER Judge nur über die strittigen Items, Mehrheit 2/3 (Muster Crossblock-Tiebreaker). Ohne ihn pendelte die Note dauerhaft unter 10 ohne Fix-Pfad (gemessen: aak-fremd 9.2, kanban-smoke- - Dublette 9.4 — „keine behebbaren Befunde" trotz Befund). j3 „behalten" oder Ausfall - → Item bleibt (fail-open).""" + Dublette 9.4 — „keine behebbaren Befunde" trotz Befund). + Explizites 2:1-„behalten" wird als FREISPRUCH persistiert (kategorie+ids) — die QA + zählt das Item ab dann nicht mehr (qa.lade_freispruch). j3-AUSFALL persistiert nicht + (fail-open ist kein Urteil). → (verdicts, freigesprochene Zeilen).""" v = await _judge(template, topic, key, slot, lines) strittig = [i for i in range(1, len(lines) + 1) if v.get(i) != befund] + frei: list[str] = [] if strittig: v3 = await _judge(template, topic, f"{key}-st", slot, [lines[i - 1] for i in strittig]) + gegen = "nein" if befund == "ja" else "ja" + frei_keys: list[str] = [] for pos, i in enumerate(strittig, 1): if v3.get(pos) == befund: v[i] = befund # 2:1 für den QA-Befund → handeln - return v + elif v3.get(pos) == gegen and kategorie and ids: + frei_keys.append(ids[i - 1]) + frei.append(lines[i - 1].splitlines()[0][:80]) + if frei_keys: + _speichere_freispruch(topic, kategorie, frei_keys) + log.info("[%s] Repair %s: %d Befund(e) per 2:1 freigesprochen", topic, kategorie, len(frei_keys)) + return v, frei async def _fix_hygiene(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]: @@ -116,8 +137,8 @@ async def _merge_dubletten(topic: str, report: dict, by_norm: dict, files: dict) and _norm_title(p.get("a", "")) in by_norm and _norm_title(p.get("b", "")) in by_norm] if not paare: return [] - v = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs", - [f"A: {p['a']}\nB: {p['b']}" for p in paare], "ja") + v, _frei = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs", + [f"A: {p['a']}\nB: {p['b']}" for p in paare], "ja") merged = [] for i, p in enumerate(paare, 1): a, b = by_norm.get(_norm_title(p["a"])), by_norm.get(_norm_title(p["b"])) @@ -177,7 +198,7 @@ async def falte_sub(topic: str, files: dict, win: dict, lose: dict) -> None: _entferne_sub_in_files(files, lose["block_norm"], lose["sub_norm"]) -async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[str]: +async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> tuple[list[str], list[str]]: """QA-bestätigte Sub-Paare (llm=ja) nach Zweitmeinung falten: Verlierer → variant, seine Fragen/Artefakte wandern zum Gewinner (oder fallen weg, wenn er den Typ hat). Repair hatte dafür keinen Handler — die Paare überlebten jeden Repair-Zyklus.""" @@ -192,10 +213,13 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st and (a := _row(p.get("a"))) and (b := _row(p.get("b"))) and (a["block_norm"], a["sub_norm"]) != (b["block_norm"], b["sub_norm"])] if not paare: - return [] - v = await _mit_stichentscheid("QA-Sub-Dubletten", topic, "sub-dubletten", "pairs", - [f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}" - for a, b in paare], "ja") + return [], [] + v, frei = await _mit_stichentscheid( + "QA-Sub-Dubletten", topic, "sub-dubletten", "pairs", + [f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}" for a, b in paare], + "ja", kategorie="sub_dubletten", + ids=[qa._paar_key(f"[{a['block']}] {a['sub_title']}", f"[{b['block']}] {b['sub_title']}") + for a, b in paare]) merged: list[str] = [] gone: set[tuple] = set() for i, (a, b) in enumerate(paare, 1): @@ -206,7 +230,7 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st await falte_sub(topic, files, win, lose) gone.add(lk) merged.append(f"{lose['sub_title'][:40]} → {win['sub_title'][:40]}") - return merged + return merged, frei def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None: @@ -237,8 +261,9 @@ def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None: atomic_write_json(files["artefakte"], neu, indent=1) -async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]: +async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: dict) -> tuple[list[str], list[str]]: out = [] + frei_alle: list[str] = [] fremd = [t for t in report.get("fremd", []) if _norm_title(t) in by_norm] if fremd: folder = source_folder(topic) @@ -247,7 +272,10 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: srcs = by_norm[_norm_title(t)]["payload"].get("sources") or None ev = _evidence_pack(folder, srcs, [t], budget=EVIDENCE_PER_BLOCK) if folder else "" lines.append(f"{t}\n{ev or '(keine Treffer im Material)'}") - v = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines, "nein") + v, frei = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines, + "nein", kategorie="fremd", + ids=[_norm_title(t) for t in fremd]) + frei_alle += frei for i, t in enumerate(fremd, 1): if v.get(i) == "nein": await _reject(topic, t, by_norm, files, "qa-fremd") @@ -256,12 +284,15 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: if unecht: lines = [f"{t} — {by_norm[_norm_title(t)]['payload'].get('description') or '(ohne Beschreibung)'}" for t in unecht] - v = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines, "nein") + v, frei = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines, + "nein", kategorie="unecht", + ids=[_norm_title(t) for t in unecht]) + frei_alle += frei for i, t in enumerate(unecht, 1): if v.get(i) == "nein": await _reject(topic, t, by_norm, files, "qa-unecht") out.append(t) - return out + return out, frei_alle async def _raeume_waisen(topic: str) -> int: diff --git a/backend/tests/test_repair.py b/backend/tests/test_repair.py index 1de4271..74e2382 100644 --- a/backend/tests/test_repair.py +++ b/backend/tests/test_repair.py @@ -230,6 +230,35 @@ async def test_sub_dubletten_stichentscheid_faltet(env, monkeypatch): assert rows["sub b"] == "variant" and rows["sub a"] == "consensus" +async def test_stichentscheid_behalten_persistiert_freispruch(env, monkeypatch): + """2:1 „behalten" (Zweitmeinung + j3 einig gegen den QA-Befund) → Freispruch wird + persistiert und der Report des nächsten qa_report zählt das Paar nicht mehr — + vorher pendelte die Note dauerhaft unter 10 ohne Fix-Pfad.""" + import qa as qa_mod + db, seed, files, write_report = env + await seed("Alpha", "beschr") + norm = repair._norm_title("Alpha") + await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A", status="consensus") + await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus") + write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}])) + + async def fake_agent(key, prompt, timeout, **kw): + return 0, '{"relevant": {"1": "nein"}}', "" # beide Repair-Judges: behalten + + monkeypatch.setattr(repair, "run_agent", fake_agent) + res = await repair.repair_befunde(TOPIC) + assert res["sub_merges"] == [] and len(res["freigesprochen"]) == 1 + frei = qa_mod.lade_freispruch(TOPIC) + key = qa_mod._paar_key("[Alpha] Sub A", "[Alpha] Sub B") + assert key in set(frei.get("sub_dubletten") or []) + # QA-Seite: bestätigtes, aber freigesprochenes Paar zählt nicht in die Quote + sd = [{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}] + frei_sub = set(frei["sub_dubletten"]) + zaehlt = sum(1 for p in sd if p.get("llm") == "ja" + and qa_mod._paar_key(p["a"], p["b"]) not in frei_sub) + assert zaehlt == 0 + + async def test_fremd_stichentscheid_behalten(env, monkeypatch): """Dissens bei fremd, Stichentscheid sagt ebenfalls behalten (ja) → Block bleibt (fail-open bei 1:2 gegen den Befund).""" diff --git a/frontend/src/components/GenerationView.vue b/frontend/src/components/GenerationView.vue index c4741a7..f921b9b 100644 --- a/frontend/src/components/GenerationView.vue +++ b/frontend/src/components/GenerationView.vue @@ -106,10 +106,10 @@ async function repairClick() { try { const r = await runRepair(props.topic) const n = (r.hygiene || []).length + (r.merges || []).length + (r.sub_merges || []).length - + (r.entfernt || []).length + (r.aufgeraeumt || 0) + + (r.entfernt || []).length + (r.aufgeraeumt || 0) + (r.freigesprochen || []).length repairInfo.value = n === 0 ? 'keine behebbaren Befunde' - : `${(r.hygiene || []).length} Titel · ${(r.merges || []).length} Merges · ${(r.sub_merges || []).length} Sub-Merges · ${(r.entfernt || []).length} entfernt · ${r.aufgeraeumt || 0} aufgeräumt` + : `${(r.hygiene || []).length} Titel · ${(r.merges || []).length} Merges · ${(r.sub_merges || []).length} Sub-Merges · ${(r.entfernt || []).length} entfernt · ${r.aufgeraeumt || 0} aufgeräumt · ${(r.freigesprochen || []).length} freigesprochen` } catch (e) { repairInfo.value = String(e.message || e) } finally {