update
This commit is contained in:
@@ -294,6 +294,24 @@ async def _llm_verdicts(template: str, topic: str, key: str, items: list[str]) -
|
||||
|
||||
# ── Report ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def freispruch_pfad(topic: str) -> Path:
|
||||
return QA_DIR / topic / "freispruch.json"
|
||||
|
||||
|
||||
def _paar_key(a: str, b: str) -> str:
|
||||
return "||".join(sorted((_norm_title(a), _norm_title(b))))
|
||||
|
||||
|
||||
def lade_freispruch(topic: str) -> dict[str, list[str]]:
|
||||
"""Persistierte 2:1-Freisprüche des Repair-Stichentscheids (repair._mit_stichentscheid):
|
||||
mehrheitlich als „behalten" geurteilte Befunde zählen nicht mehr in die Note — sonst
|
||||
pendelte sie dauerhaft unter 10 ohne Fix-Pfad (gemessen: kanban-smoke 9.4, aak 9.2).
|
||||
Die Detektoren bleiben unverändert; ein Freispruch ist ein persistiertes Urteil,
|
||||
kein Detektor-Tuning. Freigesprochene bleiben im Report sichtbar."""
|
||||
d = _json_file(freispruch_pfad(topic))
|
||||
return d if isinstance(d, dict) else {}
|
||||
|
||||
|
||||
async def qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
cards = await db.kanban_cards(topic, board="inventory", stage="done_block")
|
||||
if not cards:
|
||||
@@ -317,6 +335,11 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
hy = hygiene(blocks)
|
||||
n_sections = sum(len(_sections(t)) for t in corpus.values()) or 1
|
||||
|
||||
frei = lade_freispruch(topic)
|
||||
frei_fremd = set(frei.get("fremd") or [])
|
||||
fremd_frei = [t for t in fr if _norm_title(t) in frei_fremd]
|
||||
fr = [t for t in fr if _norm_title(t) not in frei_fremd]
|
||||
|
||||
if llm and d:
|
||||
v = await _llm_verdicts("QA-Dubletten", topic, "dubletten",
|
||||
[f"A: {p['a']}\nB: {p['b']}" for p in d[:LLM_SAMPLE]])
|
||||
@@ -334,6 +357,10 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
[f"A: {p['a']}\nB: {p['b']}" for p in chunk])
|
||||
for k, p in enumerate(chunk, 1):
|
||||
p["llm"] = v.get(k, "?")
|
||||
frei_sub = set(frei.get("sub_dubletten") or [])
|
||||
for p in sd:
|
||||
if p.get("llm") == "ja" and _paar_key(p["a"], p["b"]) in frei_sub:
|
||||
p["freispruch"] = True # 2:1-Urteil „behalten" — sichtbar, aber notenfrei
|
||||
unecht: list[str] | None = None
|
||||
if llm and blocks:
|
||||
verdacht = []
|
||||
@@ -350,6 +377,8 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
v2 = await _llm_verdicts("QA-Bausteine", topic, "bausteine-b2",
|
||||
[f"{b['title']} — {b['description'] or '(ohne Beschreibung)'}" for b in verdacht])
|
||||
unecht = [b["title"] for k, b in enumerate(verdacht, 1) if v2.get(k) == "nein"]
|
||||
frei_unecht = set(frei.get("unecht") or [])
|
||||
unecht = [t for t in unecht if _norm_title(t) not in frei_unecht]
|
||||
|
||||
art_rows = [dict(r) for r in await db.get_sub_artefakte(topic)]
|
||||
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
|
||||
@@ -362,8 +391,10 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
n_cons = sum(1 for r in sub_rows if r["status"] == "consensus")
|
||||
if n_cons:
|
||||
quoten_art["sub_dubletten_verdacht"] = round(len(sd) / n_cons, 3)
|
||||
if llm: # confirmed pairs only — the bare candidate list is suspicion, not damage
|
||||
quoten_art["sub_dubletten"] = round(sum(1 for p in sd if p.get("llm") == "ja") / n_cons, 3)
|
||||
if llm: # confirmed pairs only — the bare candidate list is suspicion, not damage;
|
||||
# freigesprochene (2:1 „behalten") zählen nicht mehr
|
||||
quoten_art["sub_dubletten"] = round(
|
||||
sum(1 for p in sd if p.get("llm") == "ja" and not p.get("freispruch")) / n_cons, 3)
|
||||
summary = _json_file(arbeit_dir(topic) / "lauf-summary.json") or {}
|
||||
report = {
|
||||
"topic": topic, "erstellt": datetime.now(timezone.utc).isoformat(),
|
||||
@@ -377,6 +408,7 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
},
|
||||
"quoten_artefakte": quoten_art,
|
||||
**({"unecht": unecht} if unecht is not None else {}),
|
||||
**({"fremd_freigesprochen": fremd_frei} if fremd_frei else {}),
|
||||
"dubletten": d, "sub_dubletten": sd, "luecken": lk, "fremd": fr, "beleg": bl, "hygiene": hy,
|
||||
"artefakte": art,
|
||||
"lauf": summary,
|
||||
|
||||
@@ -38,8 +38,8 @@ async def repair_befunde(topic: str) -> dict:
|
||||
|
||||
hygiene = await _fix_hygiene(topic, report, by_norm, files)
|
||||
merges = await _merge_dubletten(topic, report, by_norm, files)
|
||||
sub_merges = await _merge_sub_dubletten(topic, report, files)
|
||||
entfernt = await _entferne_fremd_unecht(topic, report, by_norm, files)
|
||||
sub_merges, frei_subs = await _merge_sub_dubletten(topic, report, files)
|
||||
entfernt, frei_bloecke = await _entferne_fremd_unecht(topic, report, by_norm, files)
|
||||
aufgeraeumt = await _raeume_waisen(topic)
|
||||
|
||||
# llm=True: gleiche Messlatte wie QA-Button/Abschluss-QA — der llm=False-Report
|
||||
@@ -48,7 +48,8 @@ async def repair_befunde(topic: str) -> dict:
|
||||
if neu:
|
||||
await qa.write_report(neu)
|
||||
return {"hygiene": hygiene, "merges": merges, "sub_merges": sub_merges, "entfernt": entfernt,
|
||||
"aufgeraeumt": aufgeraeumt, "braucht_research": len(report.get("luecken", []))}
|
||||
"aufgeraeumt": aufgeraeumt, "freigesprochen": frei_subs + frei_bloecke,
|
||||
"braucht_research": len(report.get("luecken", []))}
|
||||
|
||||
|
||||
async def _judge(template: str, topic: str, key: str, slot: str, items: list[str]) -> dict[int, str]:
|
||||
@@ -69,22 +70,42 @@ async def _judge(template: str, topic: str, key: str, slot: str, items: list[str
|
||||
return verdicts
|
||||
|
||||
|
||||
def _speichere_freispruch(topic: str, kategorie: str, keys: list[str]) -> None:
|
||||
d = qa.lade_freispruch(topic)
|
||||
alt = set(d.get(kategorie) or [])
|
||||
d[kategorie] = sorted(alt | set(keys))
|
||||
qa.freispruch_pfad(topic).parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write_json(qa.freispruch_pfad(topic), d, indent=1)
|
||||
|
||||
|
||||
async def _mit_stichentscheid(template: str, topic: str, key: str, slot: str,
|
||||
lines: list[str], befund: str) -> dict[int, str]:
|
||||
lines: list[str], befund: str, kategorie: str = "",
|
||||
ids: list[str] | None = None) -> tuple[dict[int, str], list[str]]:
|
||||
"""Zweitmeinung + Stichentscheid: Der Repair-Judge kann den QA-Befund kippen — bei
|
||||
Dissens (QA sagt Befund, Judge sagt behalten) entscheidet ein DRITTER Judge nur über
|
||||
die strittigen Items, Mehrheit 2/3 (Muster Crossblock-Tiebreaker). Ohne ihn pendelte
|
||||
die Note dauerhaft unter 10 ohne Fix-Pfad (gemessen: aak-fremd 9.2, kanban-smoke-
|
||||
Dublette 9.4 — „keine behebbaren Befunde" trotz Befund). j3 „behalten" oder Ausfall
|
||||
→ Item bleibt (fail-open)."""
|
||||
Dublette 9.4 — „keine behebbaren Befunde" trotz Befund).
|
||||
Explizites 2:1-„behalten" wird als FREISPRUCH persistiert (kategorie+ids) — die QA
|
||||
zählt das Item ab dann nicht mehr (qa.lade_freispruch). j3-AUSFALL persistiert nicht
|
||||
(fail-open ist kein Urteil). → (verdicts, freigesprochene Zeilen)."""
|
||||
v = await _judge(template, topic, key, slot, lines)
|
||||
strittig = [i for i in range(1, len(lines) + 1) if v.get(i) != befund]
|
||||
frei: list[str] = []
|
||||
if strittig:
|
||||
v3 = await _judge(template, topic, f"{key}-st", slot, [lines[i - 1] for i in strittig])
|
||||
gegen = "nein" if befund == "ja" else "ja"
|
||||
frei_keys: list[str] = []
|
||||
for pos, i in enumerate(strittig, 1):
|
||||
if v3.get(pos) == befund:
|
||||
v[i] = befund # 2:1 für den QA-Befund → handeln
|
||||
return v
|
||||
elif v3.get(pos) == gegen and kategorie and ids:
|
||||
frei_keys.append(ids[i - 1])
|
||||
frei.append(lines[i - 1].splitlines()[0][:80])
|
||||
if frei_keys:
|
||||
_speichere_freispruch(topic, kategorie, frei_keys)
|
||||
log.info("[%s] Repair %s: %d Befund(e) per 2:1 freigesprochen", topic, kategorie, len(frei_keys))
|
||||
return v, frei
|
||||
|
||||
|
||||
async def _fix_hygiene(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]:
|
||||
@@ -116,8 +137,8 @@ async def _merge_dubletten(topic: str, report: dict, by_norm: dict, files: dict)
|
||||
and _norm_title(p.get("a", "")) in by_norm and _norm_title(p.get("b", "")) in by_norm]
|
||||
if not paare:
|
||||
return []
|
||||
v = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs",
|
||||
[f"A: {p['a']}\nB: {p['b']}" for p in paare], "ja")
|
||||
v, _frei = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs",
|
||||
[f"A: {p['a']}\nB: {p['b']}" for p in paare], "ja")
|
||||
merged = []
|
||||
for i, p in enumerate(paare, 1):
|
||||
a, b = by_norm.get(_norm_title(p["a"])), by_norm.get(_norm_title(p["b"]))
|
||||
@@ -177,7 +198,7 @@ async def falte_sub(topic: str, files: dict, win: dict, lose: dict) -> None:
|
||||
_entferne_sub_in_files(files, lose["block_norm"], lose["sub_norm"])
|
||||
|
||||
|
||||
async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[str]:
|
||||
async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> tuple[list[str], list[str]]:
|
||||
"""QA-bestätigte Sub-Paare (llm=ja) nach Zweitmeinung falten: Verlierer → variant,
|
||||
seine Fragen/Artefakte wandern zum Gewinner (oder fallen weg, wenn er den Typ hat).
|
||||
Repair hatte dafür keinen Handler — die Paare überlebten jeden Repair-Zyklus."""
|
||||
@@ -192,10 +213,13 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st
|
||||
and (a := _row(p.get("a"))) and (b := _row(p.get("b")))
|
||||
and (a["block_norm"], a["sub_norm"]) != (b["block_norm"], b["sub_norm"])]
|
||||
if not paare:
|
||||
return []
|
||||
v = await _mit_stichentscheid("QA-Sub-Dubletten", topic, "sub-dubletten", "pairs",
|
||||
[f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}"
|
||||
for a, b in paare], "ja")
|
||||
return [], []
|
||||
v, frei = await _mit_stichentscheid(
|
||||
"QA-Sub-Dubletten", topic, "sub-dubletten", "pairs",
|
||||
[f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}" for a, b in paare],
|
||||
"ja", kategorie="sub_dubletten",
|
||||
ids=[qa._paar_key(f"[{a['block']}] {a['sub_title']}", f"[{b['block']}] {b['sub_title']}")
|
||||
for a, b in paare])
|
||||
merged: list[str] = []
|
||||
gone: set[tuple] = set()
|
||||
for i, (a, b) in enumerate(paare, 1):
|
||||
@@ -206,7 +230,7 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st
|
||||
await falte_sub(topic, files, win, lose)
|
||||
gone.add(lk)
|
||||
merged.append(f"{lose['sub_title'][:40]} → {win['sub_title'][:40]}")
|
||||
return merged
|
||||
return merged, frei
|
||||
|
||||
|
||||
def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None:
|
||||
@@ -237,8 +261,9 @@ def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None:
|
||||
atomic_write_json(files["artefakte"], neu, indent=1)
|
||||
|
||||
|
||||
async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]:
|
||||
async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: dict) -> tuple[list[str], list[str]]:
|
||||
out = []
|
||||
frei_alle: list[str] = []
|
||||
fremd = [t for t in report.get("fremd", []) if _norm_title(t) in by_norm]
|
||||
if fremd:
|
||||
folder = source_folder(topic)
|
||||
@@ -247,7 +272,10 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files:
|
||||
srcs = by_norm[_norm_title(t)]["payload"].get("sources") or None
|
||||
ev = _evidence_pack(folder, srcs, [t], budget=EVIDENCE_PER_BLOCK) if folder else ""
|
||||
lines.append(f"{t}\n{ev or '(keine Treffer im Material)'}")
|
||||
v = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines, "nein")
|
||||
v, frei = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines,
|
||||
"nein", kategorie="fremd",
|
||||
ids=[_norm_title(t) for t in fremd])
|
||||
frei_alle += frei
|
||||
for i, t in enumerate(fremd, 1):
|
||||
if v.get(i) == "nein":
|
||||
await _reject(topic, t, by_norm, files, "qa-fremd")
|
||||
@@ -256,12 +284,15 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files:
|
||||
if unecht:
|
||||
lines = [f"{t} — {by_norm[_norm_title(t)]['payload'].get('description') or '(ohne Beschreibung)'}"
|
||||
for t in unecht]
|
||||
v = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines, "nein")
|
||||
v, frei = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines,
|
||||
"nein", kategorie="unecht",
|
||||
ids=[_norm_title(t) for t in unecht])
|
||||
frei_alle += frei
|
||||
for i, t in enumerate(unecht, 1):
|
||||
if v.get(i) == "nein":
|
||||
await _reject(topic, t, by_norm, files, "qa-unecht")
|
||||
out.append(t)
|
||||
return out
|
||||
return out, frei_alle
|
||||
|
||||
|
||||
async def _raeume_waisen(topic: str) -> int:
|
||||
|
||||
@@ -230,6 +230,35 @@ async def test_sub_dubletten_stichentscheid_faltet(env, monkeypatch):
|
||||
assert rows["sub b"] == "variant" and rows["sub a"] == "consensus"
|
||||
|
||||
|
||||
async def test_stichentscheid_behalten_persistiert_freispruch(env, monkeypatch):
|
||||
"""2:1 „behalten" (Zweitmeinung + j3 einig gegen den QA-Befund) → Freispruch wird
|
||||
persistiert und der Report des nächsten qa_report zählt das Paar nicht mehr —
|
||||
vorher pendelte die Note dauerhaft unter 10 ohne Fix-Pfad."""
|
||||
import qa as qa_mod
|
||||
db, seed, files, write_report = env
|
||||
await seed("Alpha", "beschr")
|
||||
norm = repair._norm_title("Alpha")
|
||||
await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A", status="consensus")
|
||||
await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus")
|
||||
write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]))
|
||||
|
||||
async def fake_agent(key, prompt, timeout, **kw):
|
||||
return 0, '{"relevant": {"1": "nein"}}', "" # beide Repair-Judges: behalten
|
||||
|
||||
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
||||
res = await repair.repair_befunde(TOPIC)
|
||||
assert res["sub_merges"] == [] and len(res["freigesprochen"]) == 1
|
||||
frei = qa_mod.lade_freispruch(TOPIC)
|
||||
key = qa_mod._paar_key("[Alpha] Sub A", "[Alpha] Sub B")
|
||||
assert key in set(frei.get("sub_dubletten") or [])
|
||||
# QA-Seite: bestätigtes, aber freigesprochenes Paar zählt nicht in die Quote
|
||||
sd = [{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]
|
||||
frei_sub = set(frei["sub_dubletten"])
|
||||
zaehlt = sum(1 for p in sd if p.get("llm") == "ja"
|
||||
and qa_mod._paar_key(p["a"], p["b"]) not in frei_sub)
|
||||
assert zaehlt == 0
|
||||
|
||||
|
||||
async def test_fremd_stichentscheid_behalten(env, monkeypatch):
|
||||
"""Dissens bei fremd, Stichentscheid sagt ebenfalls behalten (ja) → Block bleibt
|
||||
(fail-open bei 1:2 gegen den Befund)."""
|
||||
|
||||
Reference in New Issue
Block a user