This commit is contained in:
Team3
2026-07-05 00:38:03 +02:00
parent d25824229e
commit 219993ffd5
4 changed files with 115 additions and 23 deletions

View File

@@ -294,6 +294,24 @@ async def _llm_verdicts(template: str, topic: str, key: str, items: list[str]) -
# ── Report ────────────────────────────────────────────────────────────────────────── # ── Report ──────────────────────────────────────────────────────────────────────────
def freispruch_pfad(topic: str) -> Path:
return QA_DIR / topic / "freispruch.json"
def _paar_key(a: str, b: str) -> str:
return "||".join(sorted((_norm_title(a), _norm_title(b))))
def lade_freispruch(topic: str) -> dict[str, list[str]]:
"""Persistierte 2:1-Freisprüche des Repair-Stichentscheids (repair._mit_stichentscheid):
mehrheitlich als „behalten" geurteilte Befunde zählen nicht mehr in die Note — sonst
pendelte sie dauerhaft unter 10 ohne Fix-Pfad (gemessen: kanban-smoke 9.4, aak 9.2).
Die Detektoren bleiben unverändert; ein Freispruch ist ein persistiertes Urteil,
kein Detektor-Tuning. Freigesprochene bleiben im Report sichtbar."""
d = _json_file(freispruch_pfad(topic))
return d if isinstance(d, dict) else {}
async def qa_report(topic: str, llm: bool = False) -> dict | None: async def qa_report(topic: str, llm: bool = False) -> dict | None:
cards = await db.kanban_cards(topic, board="inventory", stage="done_block") cards = await db.kanban_cards(topic, board="inventory", stage="done_block")
if not cards: if not cards:
@@ -317,6 +335,11 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
hy = hygiene(blocks) hy = hygiene(blocks)
n_sections = sum(len(_sections(t)) for t in corpus.values()) or 1 n_sections = sum(len(_sections(t)) for t in corpus.values()) or 1
frei = lade_freispruch(topic)
frei_fremd = set(frei.get("fremd") or [])
fremd_frei = [t for t in fr if _norm_title(t) in frei_fremd]
fr = [t for t in fr if _norm_title(t) not in frei_fremd]
if llm and d: if llm and d:
v = await _llm_verdicts("QA-Dubletten", topic, "dubletten", v = await _llm_verdicts("QA-Dubletten", topic, "dubletten",
[f"A: {p['a']}\nB: {p['b']}" for p in d[:LLM_SAMPLE]]) [f"A: {p['a']}\nB: {p['b']}" for p in d[:LLM_SAMPLE]])
@@ -334,6 +357,10 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
[f"A: {p['a']}\nB: {p['b']}" for p in chunk]) [f"A: {p['a']}\nB: {p['b']}" for p in chunk])
for k, p in enumerate(chunk, 1): for k, p in enumerate(chunk, 1):
p["llm"] = v.get(k, "?") p["llm"] = v.get(k, "?")
frei_sub = set(frei.get("sub_dubletten") or [])
for p in sd:
if p.get("llm") == "ja" and _paar_key(p["a"], p["b"]) in frei_sub:
p["freispruch"] = True # 2:1-Urteil „behalten" — sichtbar, aber notenfrei
unecht: list[str] | None = None unecht: list[str] | None = None
if llm and blocks: if llm and blocks:
verdacht = [] verdacht = []
@@ -350,6 +377,8 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
v2 = await _llm_verdicts("QA-Bausteine", topic, "bausteine-b2", v2 = await _llm_verdicts("QA-Bausteine", topic, "bausteine-b2",
[f"{b['title']}{b['description'] or '(ohne Beschreibung)'}" for b in verdacht]) [f"{b['title']}{b['description'] or '(ohne Beschreibung)'}" for b in verdacht])
unecht = [b["title"] for k, b in enumerate(verdacht, 1) if v2.get(k) == "nein"] unecht = [b["title"] for k, b in enumerate(verdacht, 1) if v2.get(k) == "nein"]
frei_unecht = set(frei.get("unecht") or [])
unecht = [t for t in unecht if _norm_title(t) not in frei_unecht]
art_rows = [dict(r) for r in await db.get_sub_artefakte(topic)] art_rows = [dict(r) for r in await db.get_sub_artefakte(topic)]
fragen = [dict(r) for r in await db.list_question_pattern(topic)] fragen = [dict(r) for r in await db.list_question_pattern(topic)]
@@ -362,8 +391,10 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
n_cons = sum(1 for r in sub_rows if r["status"] == "consensus") n_cons = sum(1 for r in sub_rows if r["status"] == "consensus")
if n_cons: if n_cons:
quoten_art["sub_dubletten_verdacht"] = round(len(sd) / n_cons, 3) quoten_art["sub_dubletten_verdacht"] = round(len(sd) / n_cons, 3)
if llm: # confirmed pairs only — the bare candidate list is suspicion, not damage if llm: # confirmed pairs only — the bare candidate list is suspicion, not damage;
quoten_art["sub_dubletten"] = round(sum(1 for p in sd if p.get("llm") == "ja") / n_cons, 3) # freigesprochene (2:1 „behalten") zählen nicht mehr
quoten_art["sub_dubletten"] = round(
sum(1 for p in sd if p.get("llm") == "ja" and not p.get("freispruch")) / n_cons, 3)
summary = _json_file(arbeit_dir(topic) / "lauf-summary.json") or {} summary = _json_file(arbeit_dir(topic) / "lauf-summary.json") or {}
report = { report = {
"topic": topic, "erstellt": datetime.now(timezone.utc).isoformat(), "topic": topic, "erstellt": datetime.now(timezone.utc).isoformat(),
@@ -377,6 +408,7 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
}, },
"quoten_artefakte": quoten_art, "quoten_artefakte": quoten_art,
**({"unecht": unecht} if unecht is not None else {}), **({"unecht": unecht} if unecht is not None else {}),
**({"fremd_freigesprochen": fremd_frei} if fremd_frei else {}),
"dubletten": d, "sub_dubletten": sd, "luecken": lk, "fremd": fr, "beleg": bl, "hygiene": hy, "dubletten": d, "sub_dubletten": sd, "luecken": lk, "fremd": fr, "beleg": bl, "hygiene": hy,
"artefakte": art, "artefakte": art,
"lauf": summary, "lauf": summary,

View File

@@ -38,8 +38,8 @@ async def repair_befunde(topic: str) -> dict:
hygiene = await _fix_hygiene(topic, report, by_norm, files) hygiene = await _fix_hygiene(topic, report, by_norm, files)
merges = await _merge_dubletten(topic, report, by_norm, files) merges = await _merge_dubletten(topic, report, by_norm, files)
sub_merges = await _merge_sub_dubletten(topic, report, files) sub_merges, frei_subs = await _merge_sub_dubletten(topic, report, files)
entfernt = await _entferne_fremd_unecht(topic, report, by_norm, files) entfernt, frei_bloecke = await _entferne_fremd_unecht(topic, report, by_norm, files)
aufgeraeumt = await _raeume_waisen(topic) aufgeraeumt = await _raeume_waisen(topic)
# llm=True: gleiche Messlatte wie QA-Button/Abschluss-QA — der llm=False-Report # llm=True: gleiche Messlatte wie QA-Button/Abschluss-QA — der llm=False-Report
@@ -48,7 +48,8 @@ async def repair_befunde(topic: str) -> dict:
if neu: if neu:
await qa.write_report(neu) await qa.write_report(neu)
return {"hygiene": hygiene, "merges": merges, "sub_merges": sub_merges, "entfernt": entfernt, return {"hygiene": hygiene, "merges": merges, "sub_merges": sub_merges, "entfernt": entfernt,
"aufgeraeumt": aufgeraeumt, "braucht_research": len(report.get("luecken", []))} "aufgeraeumt": aufgeraeumt, "freigesprochen": frei_subs + frei_bloecke,
"braucht_research": len(report.get("luecken", []))}
async def _judge(template: str, topic: str, key: str, slot: str, items: list[str]) -> dict[int, str]: async def _judge(template: str, topic: str, key: str, slot: str, items: list[str]) -> dict[int, str]:
@@ -69,22 +70,42 @@ async def _judge(template: str, topic: str, key: str, slot: str, items: list[str
return verdicts return verdicts
def _speichere_freispruch(topic: str, kategorie: str, keys: list[str]) -> None:
d = qa.lade_freispruch(topic)
alt = set(d.get(kategorie) or [])
d[kategorie] = sorted(alt | set(keys))
qa.freispruch_pfad(topic).parent.mkdir(parents=True, exist_ok=True)
atomic_write_json(qa.freispruch_pfad(topic), d, indent=1)
async def _mit_stichentscheid(template: str, topic: str, key: str, slot: str, async def _mit_stichentscheid(template: str, topic: str, key: str, slot: str,
lines: list[str], befund: str) -> dict[int, str]: lines: list[str], befund: str, kategorie: str = "",
ids: list[str] | None = None) -> tuple[dict[int, str], list[str]]:
"""Zweitmeinung + Stichentscheid: Der Repair-Judge kann den QA-Befund kippen — bei """Zweitmeinung + Stichentscheid: Der Repair-Judge kann den QA-Befund kippen — bei
Dissens (QA sagt Befund, Judge sagt behalten) entscheidet ein DRITTER Judge nur über Dissens (QA sagt Befund, Judge sagt behalten) entscheidet ein DRITTER Judge nur über
die strittigen Items, Mehrheit 2/3 (Muster Crossblock-Tiebreaker). Ohne ihn pendelte die strittigen Items, Mehrheit 2/3 (Muster Crossblock-Tiebreaker). Ohne ihn pendelte
die Note dauerhaft unter 10 ohne Fix-Pfad (gemessen: aak-fremd 9.2, kanban-smoke- die Note dauerhaft unter 10 ohne Fix-Pfad (gemessen: aak-fremd 9.2, kanban-smoke-
Dublette 9.4 — „keine behebbaren Befunde" trotz Befund). j3 „behalten" oder Ausfall Dublette 9.4 — „keine behebbaren Befunde" trotz Befund).
→ Item bleibt (fail-open).""" Explizites 2:1-„behalten" wird als FREISPRUCH persistiert (kategorie+ids) — die QA
zählt das Item ab dann nicht mehr (qa.lade_freispruch). j3-AUSFALL persistiert nicht
(fail-open ist kein Urteil). → (verdicts, freigesprochene Zeilen)."""
v = await _judge(template, topic, key, slot, lines) v = await _judge(template, topic, key, slot, lines)
strittig = [i for i in range(1, len(lines) + 1) if v.get(i) != befund] strittig = [i for i in range(1, len(lines) + 1) if v.get(i) != befund]
frei: list[str] = []
if strittig: if strittig:
v3 = await _judge(template, topic, f"{key}-st", slot, [lines[i - 1] for i in strittig]) v3 = await _judge(template, topic, f"{key}-st", slot, [lines[i - 1] for i in strittig])
gegen = "nein" if befund == "ja" else "ja"
frei_keys: list[str] = []
for pos, i in enumerate(strittig, 1): for pos, i in enumerate(strittig, 1):
if v3.get(pos) == befund: if v3.get(pos) == befund:
v[i] = befund # 2:1 für den QA-Befund → handeln v[i] = befund # 2:1 für den QA-Befund → handeln
return v elif v3.get(pos) == gegen and kategorie and ids:
frei_keys.append(ids[i - 1])
frei.append(lines[i - 1].splitlines()[0][:80])
if frei_keys:
_speichere_freispruch(topic, kategorie, frei_keys)
log.info("[%s] Repair %s: %d Befund(e) per 2:1 freigesprochen", topic, kategorie, len(frei_keys))
return v, frei
async def _fix_hygiene(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]: async def _fix_hygiene(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]:
@@ -116,8 +137,8 @@ async def _merge_dubletten(topic: str, report: dict, by_norm: dict, files: dict)
and _norm_title(p.get("a", "")) in by_norm and _norm_title(p.get("b", "")) in by_norm] and _norm_title(p.get("a", "")) in by_norm and _norm_title(p.get("b", "")) in by_norm]
if not paare: if not paare:
return [] return []
v = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs", v, _frei = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs",
[f"A: {p['a']}\nB: {p['b']}" for p in paare], "ja") [f"A: {p['a']}\nB: {p['b']}" for p in paare], "ja")
merged = [] merged = []
for i, p in enumerate(paare, 1): for i, p in enumerate(paare, 1):
a, b = by_norm.get(_norm_title(p["a"])), by_norm.get(_norm_title(p["b"])) a, b = by_norm.get(_norm_title(p["a"])), by_norm.get(_norm_title(p["b"]))
@@ -177,7 +198,7 @@ async def falte_sub(topic: str, files: dict, win: dict, lose: dict) -> None:
_entferne_sub_in_files(files, lose["block_norm"], lose["sub_norm"]) _entferne_sub_in_files(files, lose["block_norm"], lose["sub_norm"])
async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[str]: async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> tuple[list[str], list[str]]:
"""QA-bestätigte Sub-Paare (llm=ja) nach Zweitmeinung falten: Verlierer → variant, """QA-bestätigte Sub-Paare (llm=ja) nach Zweitmeinung falten: Verlierer → variant,
seine Fragen/Artefakte wandern zum Gewinner (oder fallen weg, wenn er den Typ hat). seine Fragen/Artefakte wandern zum Gewinner (oder fallen weg, wenn er den Typ hat).
Repair hatte dafür keinen Handler — die Paare überlebten jeden Repair-Zyklus.""" Repair hatte dafür keinen Handler — die Paare überlebten jeden Repair-Zyklus."""
@@ -192,10 +213,13 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st
and (a := _row(p.get("a"))) and (b := _row(p.get("b"))) and (a := _row(p.get("a"))) and (b := _row(p.get("b")))
and (a["block_norm"], a["sub_norm"]) != (b["block_norm"], b["sub_norm"])] and (a["block_norm"], a["sub_norm"]) != (b["block_norm"], b["sub_norm"])]
if not paare: if not paare:
return [] return [], []
v = await _mit_stichentscheid("QA-Sub-Dubletten", topic, "sub-dubletten", "pairs", v, frei = await _mit_stichentscheid(
[f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}" "QA-Sub-Dubletten", topic, "sub-dubletten", "pairs",
for a, b in paare], "ja") [f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}" for a, b in paare],
"ja", kategorie="sub_dubletten",
ids=[qa._paar_key(f"[{a['block']}] {a['sub_title']}", f"[{b['block']}] {b['sub_title']}")
for a, b in paare])
merged: list[str] = [] merged: list[str] = []
gone: set[tuple] = set() gone: set[tuple] = set()
for i, (a, b) in enumerate(paare, 1): for i, (a, b) in enumerate(paare, 1):
@@ -206,7 +230,7 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st
await falte_sub(topic, files, win, lose) await falte_sub(topic, files, win, lose)
gone.add(lk) gone.add(lk)
merged.append(f"{lose['sub_title'][:40]}{win['sub_title'][:40]}") merged.append(f"{lose['sub_title'][:40]}{win['sub_title'][:40]}")
return merged return merged, frei
def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None: def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None:
@@ -237,8 +261,9 @@ def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None:
atomic_write_json(files["artefakte"], neu, indent=1) atomic_write_json(files["artefakte"], neu, indent=1)
async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]: async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: dict) -> tuple[list[str], list[str]]:
out = [] out = []
frei_alle: list[str] = []
fremd = [t for t in report.get("fremd", []) if _norm_title(t) in by_norm] fremd = [t for t in report.get("fremd", []) if _norm_title(t) in by_norm]
if fremd: if fremd:
folder = source_folder(topic) folder = source_folder(topic)
@@ -247,7 +272,10 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files:
srcs = by_norm[_norm_title(t)]["payload"].get("sources") or None srcs = by_norm[_norm_title(t)]["payload"].get("sources") or None
ev = _evidence_pack(folder, srcs, [t], budget=EVIDENCE_PER_BLOCK) if folder else "" ev = _evidence_pack(folder, srcs, [t], budget=EVIDENCE_PER_BLOCK) if folder else ""
lines.append(f"{t}\n{ev or '(keine Treffer im Material)'}") lines.append(f"{t}\n{ev or '(keine Treffer im Material)'}")
v = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines, "nein") v, frei = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines,
"nein", kategorie="fremd",
ids=[_norm_title(t) for t in fremd])
frei_alle += frei
for i, t in enumerate(fremd, 1): for i, t in enumerate(fremd, 1):
if v.get(i) == "nein": if v.get(i) == "nein":
await _reject(topic, t, by_norm, files, "qa-fremd") await _reject(topic, t, by_norm, files, "qa-fremd")
@@ -256,12 +284,15 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files:
if unecht: if unecht:
lines = [f"{t}{by_norm[_norm_title(t)]['payload'].get('description') or '(ohne Beschreibung)'}" lines = [f"{t}{by_norm[_norm_title(t)]['payload'].get('description') or '(ohne Beschreibung)'}"
for t in unecht] for t in unecht]
v = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines, "nein") v, frei = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines,
"nein", kategorie="unecht",
ids=[_norm_title(t) for t in unecht])
frei_alle += frei
for i, t in enumerate(unecht, 1): for i, t in enumerate(unecht, 1):
if v.get(i) == "nein": if v.get(i) == "nein":
await _reject(topic, t, by_norm, files, "qa-unecht") await _reject(topic, t, by_norm, files, "qa-unecht")
out.append(t) out.append(t)
return out return out, frei_alle
async def _raeume_waisen(topic: str) -> int: async def _raeume_waisen(topic: str) -> int:

View File

@@ -230,6 +230,35 @@ async def test_sub_dubletten_stichentscheid_faltet(env, monkeypatch):
assert rows["sub b"] == "variant" and rows["sub a"] == "consensus" assert rows["sub b"] == "variant" and rows["sub a"] == "consensus"
async def test_stichentscheid_behalten_persistiert_freispruch(env, monkeypatch):
"""2:1 „behalten" (Zweitmeinung + j3 einig gegen den QA-Befund) → Freispruch wird
persistiert und der Report des nächsten qa_report zählt das Paar nicht mehr —
vorher pendelte die Note dauerhaft unter 10 ohne Fix-Pfad."""
import qa as qa_mod
db, seed, files, write_report = env
await seed("Alpha", "beschr")
norm = repair._norm_title("Alpha")
await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A", status="consensus")
await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus")
write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]))
async def fake_agent(key, prompt, timeout, **kw):
return 0, '{"relevant": {"1": "nein"}}', "" # beide Repair-Judges: behalten
monkeypatch.setattr(repair, "run_agent", fake_agent)
res = await repair.repair_befunde(TOPIC)
assert res["sub_merges"] == [] and len(res["freigesprochen"]) == 1
frei = qa_mod.lade_freispruch(TOPIC)
key = qa_mod._paar_key("[Alpha] Sub A", "[Alpha] Sub B")
assert key in set(frei.get("sub_dubletten") or [])
# QA-Seite: bestätigtes, aber freigesprochenes Paar zählt nicht in die Quote
sd = [{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]
frei_sub = set(frei["sub_dubletten"])
zaehlt = sum(1 for p in sd if p.get("llm") == "ja"
and qa_mod._paar_key(p["a"], p["b"]) not in frei_sub)
assert zaehlt == 0
async def test_fremd_stichentscheid_behalten(env, monkeypatch): async def test_fremd_stichentscheid_behalten(env, monkeypatch):
"""Dissens bei fremd, Stichentscheid sagt ebenfalls behalten (ja) → Block bleibt """Dissens bei fremd, Stichentscheid sagt ebenfalls behalten (ja) → Block bleibt
(fail-open bei 1:2 gegen den Befund).""" (fail-open bei 1:2 gegen den Befund)."""

View File

@@ -106,10 +106,10 @@ async function repairClick() {
try { try {
const r = await runRepair(props.topic) const r = await runRepair(props.topic)
const n = (r.hygiene || []).length + (r.merges || []).length + (r.sub_merges || []).length const n = (r.hygiene || []).length + (r.merges || []).length + (r.sub_merges || []).length
+ (r.entfernt || []).length + (r.aufgeraeumt || 0) + (r.entfernt || []).length + (r.aufgeraeumt || 0) + (r.freigesprochen || []).length
repairInfo.value = n === 0 repairInfo.value = n === 0
? 'keine behebbaren Befunde' ? 'keine behebbaren Befunde'
: `${(r.hygiene || []).length} Titel · ${(r.merges || []).length} Merges · ${(r.sub_merges || []).length} Sub-Merges · ${(r.entfernt || []).length} entfernt · ${r.aufgeraeumt || 0} aufgeräumt` : `${(r.hygiene || []).length} Titel · ${(r.merges || []).length} Merges · ${(r.sub_merges || []).length} Sub-Merges · ${(r.entfernt || []).length} entfernt · ${r.aufgeraeumt || 0} aufgeräumt · ${(r.freigesprochen || []).length} freigesprochen`
} catch (e) { } catch (e) {
repairInfo.value = String(e.message || e) repairInfo.value = String(e.message || e)
} finally { } finally {