marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Team3
2026-07-05 00:38:03 +02:00
parent d25824229e
commit 219993ffd5
4 changed files with 115 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
backend/qa.py
View File

@@ -294,6 +294,24 @@ async def _llm_verdicts(template: str, topic: str, key: str, items: list[str]) -
# ── Report ──────────────────────────────────────────────────────────────────────────
def freispruch_pfad(topic: str) -> Path:
return QA_DIR / topic / "freispruch.json"
def _paar_key(a: str, b: str) -> str:
return "||".join(sorted((_norm_title(a), _norm_title(b))))
def lade_freispruch(topic: str) -> dict[str, list[str]]:
"""Persistierte 2:1-Freisprüche des Repair-Stichentscheids (repair._mit_stichentscheid):
mehrheitlich als „behalten" geurteilte Befunde zählen nicht mehr in die Note — sonst
pendelte sie dauerhaft unter 10 ohne Fix-Pfad (gemessen: kanban-smoke 9.4, aak 9.2).
Die Detektoren bleiben unverändert; ein Freispruch ist ein persistiertes Urteil,
kein Detektor-Tuning. Freigesprochene bleiben im Report sichtbar."""
d = _json_file(freispruch_pfad(topic))
return d if isinstance(d, dict) else {}
async def qa_report(topic: str, llm: bool = False) -> dict | None:
cards = await db.kanban_cards(topic, board="inventory", stage="done_block")
if not cards:
@@ -317,6 +335,11 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
hy = hygiene(blocks)
n_sections = sum(len(_sections(t)) for t in corpus.values()) or 1
frei = lade_freispruch(topic)
frei_fremd = set(frei.get("fremd") or [])
fremd_frei = [t for t in fr if _norm_title(t) in frei_fremd]
fr = [t for t in fr if _norm_title(t) not in frei_fremd]
if llm and d:
v = await _llm_verdicts("QA-Dubletten", topic, "dubletten",
[f"A: {p['a']}\nB: {p['b']}" for p in d[:LLM_SAMPLE]])
@@ -334,6 +357,10 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
[f"A: {p['a']}\nB: {p['b']}" for p in chunk])
for k, p in enumerate(chunk, 1):
p["llm"] = v.get(k, "?")
frei_sub = set(frei.get("sub_dubletten") or [])
for p in sd:
if p.get("llm") == "ja" and _paar_key(p["a"], p["b"]) in frei_sub:
p["freispruch"] = True # 2:1-Urteil „behalten" — sichtbar, aber notenfrei
unecht: list[str] | None = None
if llm and blocks:
verdacht = []
@@ -350,6 +377,8 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
v2 = await _llm_verdicts("QA-Bausteine", topic, "bausteine-b2",
[f"{b['title']} — {b['description'] or '(ohne Beschreibung)'}" for b in verdacht])
unecht = [b["title"] for k, b in enumerate(verdacht, 1) if v2.get(k) == "nein"]
frei_unecht = set(frei.get("unecht") or [])
unecht = [t for t in unecht if _norm_title(t) not in frei_unecht]
art_rows = [dict(r) for r in await db.get_sub_artefakte(topic)]
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
@@ -362,8 +391,10 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
n_cons = sum(1 for r in sub_rows if r["status"] == "consensus")
if n_cons:
quoten_art["sub_dubletten_verdacht"] = round(len(sd) / n_cons, 3)
if llm: # confirmed pairs only — the bare candidate list is suspicion, not damage
quoten_art["sub_dubletten"] = round(sum(1 for p in sd if p.get("llm") == "ja") / n_cons, 3)
if llm: # confirmed pairs only — the bare candidate list is suspicion, not damage;
# freigesprochene (2:1 „behalten") zählen nicht mehr
quoten_art["sub_dubletten"] = round(
sum(1 for p in sd if p.get("llm") == "ja" and not p.get("freispruch")) / n_cons, 3)
summary = _json_file(arbeit_dir(topic) / "lauf-summary.json") or {}
report = {
"topic": topic, "erstellt": datetime.now(timezone.utc).isoformat(),
@@ -377,6 +408,7 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
},
"quoten_artefakte": quoten_art,
**({"unecht": unecht} if unecht is not None else {}),
**({"fremd_freigesprochen": fremd_frei} if fremd_frei else {}),
"dubletten": d, "sub_dubletten": sd, "luecken": lk, "fremd": fr, "beleg": bl, "hygiene": hy,
"artefakte": art,
"lauf": summary,

67
backend/repair.py
View File

@@ -38,8 +38,8 @@ async def repair_befunde(topic: str) -> dict:
hygiene = await _fix_hygiene(topic, report, by_norm, files)
merges = await _merge_dubletten(topic, report, by_norm, files)
sub_merges = await _merge_sub_dubletten(topic, report, files)
entfernt = await _entferne_fremd_unecht(topic, report, by_norm, files)
sub_merges, frei_subs = await _merge_sub_dubletten(topic, report, files)
entfernt, frei_bloecke = await _entferne_fremd_unecht(topic, report, by_norm, files)
aufgeraeumt = await _raeume_waisen(topic)
# llm=True: gleiche Messlatte wie QA-Button/Abschluss-QA — der llm=False-Report
@@ -48,7 +48,8 @@ async def repair_befunde(topic: str) -> dict:
if neu:
await qa.write_report(neu)
return {"hygiene": hygiene, "merges": merges, "sub_merges": sub_merges, "entfernt": entfernt,
"aufgeraeumt": aufgeraeumt, "braucht_research": len(report.get("luecken", []))}
"aufgeraeumt": aufgeraeumt, "freigesprochen": frei_subs + frei_bloecke,
"braucht_research": len(report.get("luecken", []))}
async def _judge(template: str, topic: str, key: str, slot: str, items: list[str]) -> dict[int, str]:
@@ -69,22 +70,42 @@ async def _judge(template: str, topic: str, key: str, slot: str, items: list[str
return verdicts
def _speichere_freispruch(topic: str, kategorie: str, keys: list[str]) -> None:
d = qa.lade_freispruch(topic)
alt = set(d.get(kategorie) or [])
d[kategorie] = sorted(alt | set(keys))
qa.freispruch_pfad(topic).parent.mkdir(parents=True, exist_ok=True)
atomic_write_json(qa.freispruch_pfad(topic), d, indent=1)
async def _mit_stichentscheid(template: str, topic: str, key: str, slot: str,
lines: list[str], befund: str) -> dict[int, str]:
lines: list[str], befund: str, kategorie: str = "",
ids: list[str] | None = None) -> tuple[dict[int, str], list[str]]:
"""Zweitmeinung + Stichentscheid: Der Repair-Judge kann den QA-Befund kippen — bei
Dissens (QA sagt Befund, Judge sagt behalten) entscheidet ein DRITTER Judge nur über
die strittigen Items, Mehrheit 2/3 (Muster Crossblock-Tiebreaker). Ohne ihn pendelte
die Note dauerhaft unter 10 ohne Fix-Pfad (gemessen: aak-fremd 9.2, kanban-smoke-
Dublette 9.4 — „keine behebbaren Befunde" trotz Befund). j3 „behalten" oder Ausfall
→ Item bleibt (fail-open)."""
Dublette 9.4 — „keine behebbaren Befunde" trotz Befund).
Explizites 2:1-„behalten" wird als FREISPRUCH persistiert (kategorie+ids) — die QA
zählt das Item ab dann nicht mehr (qa.lade_freispruch). j3-AUSFALL persistiert nicht
(fail-open ist kein Urteil). → (verdicts, freigesprochene Zeilen)."""
v = await _judge(template, topic, key, slot, lines)
strittig = [i for i in range(1, len(lines) + 1) if v.get(i) != befund]
frei: list[str] = []
if strittig:
v3 = await _judge(template, topic, f"{key}-st", slot, [lines[i - 1] for i in strittig])
gegen = "nein" if befund == "ja" else "ja"
frei_keys: list[str] = []
for pos, i in enumerate(strittig, 1):
if v3.get(pos) == befund:
v[i] = befund # 2:1 für den QA-Befund → handeln
return v
elif v3.get(pos) == gegen and kategorie and ids:
frei_keys.append(ids[i - 1])
frei.append(lines[i - 1].splitlines()[0][:80])
if frei_keys:
_speichere_freispruch(topic, kategorie, frei_keys)
log.info("[%s] Repair %s: %d Befund(e) per 2:1 freigesprochen", topic, kategorie, len(frei_keys))
return v, frei
async def _fix_hygiene(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]:
@@ -116,7 +137,7 @@ async def _merge_dubletten(topic: str, report: dict, by_norm: dict, files: dict)
and _norm_title(p.get("a", "")) in by_norm and _norm_title(p.get("b", "")) in by_norm]
if not paare:
return []
v = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs",
v, _frei = await _mit_stichentscheid("QA-Dubletten", topic, "dubletten", "pairs",
[f"A: {p['a']}\nB: {p['b']}" for p in paare], "ja")
merged = []
for i, p in enumerate(paare, 1):
@@ -177,7 +198,7 @@ async def falte_sub(topic: str, files: dict, win: dict, lose: dict) -> None:
_entferne_sub_in_files(files, lose["block_norm"], lose["sub_norm"])
async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[str]:
async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> tuple[list[str], list[str]]:
"""QA-bestätigte Sub-Paare (llm=ja) nach Zweitmeinung falten: Verlierer → variant,
seine Fragen/Artefakte wandern zum Gewinner (oder fallen weg, wenn er den Typ hat).
Repair hatte dafür keinen Handler — die Paare überlebten jeden Repair-Zyklus."""
@@ -192,10 +213,13 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st
and (a := _row(p.get("a"))) and (b := _row(p.get("b")))
and (a["block_norm"], a["sub_norm"]) != (b["block_norm"], b["sub_norm"])]
if not paare:
return []
v = await _mit_stichentscheid("QA-Sub-Dubletten", topic, "sub-dubletten", "pairs",
[f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}"
for a, b in paare], "ja")
return [], []
v, frei = await _mit_stichentscheid(
"QA-Sub-Dubletten", topic, "sub-dubletten", "pairs",
[f"A: [{a['block']}] {a['sub_title']}\nB: [{b['block']}] {b['sub_title']}" for a, b in paare],
"ja", kategorie="sub_dubletten",
ids=[qa._paar_key(f"[{a['block']}] {a['sub_title']}", f"[{b['block']}] {b['sub_title']}")
for a, b in paare])
merged: list[str] = []
gone: set[tuple] = set()
for i, (a, b) in enumerate(paare, 1):
@@ -206,7 +230,7 @@ async def _merge_sub_dubletten(topic: str, report: dict, files: dict) -> list[st
await falte_sub(topic, files, win, lose)
gone.add(lk)
merged.append(f"{lose['sub_title'][:40]} → {win['sub_title'][:40]}")
return merged
return merged, frei
def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None:
@@ -237,8 +261,9 @@ def _entferne_sub_in_files(files: dict, bnorm: str, sub_norm: str) -> None:
atomic_write_json(files["artefakte"], neu, indent=1)
async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: dict) -> list[str]:
async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files: dict) -> tuple[list[str], list[str]]:
out = []
frei_alle: list[str] = []
fremd = [t for t in report.get("fremd", []) if _norm_title(t) in by_norm]
if fremd:
folder = source_folder(topic)
@@ -247,7 +272,10 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files:
srcs = by_norm[_norm_title(t)]["payload"].get("sources") or None
ev = _evidence_pack(folder, srcs, [t], budget=EVIDENCE_PER_BLOCK) if folder else ""
lines.append(f"{t}\n{ev or '(keine Treffer im Material)'}")
v = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines, "nein")
v, frei = await _mit_stichentscheid("QA-Repair-Beleg", topic, "fremd", "blocks", lines,
"nein", kategorie="fremd",
ids=[_norm_title(t) for t in fremd])
frei_alle += frei
for i, t in enumerate(fremd, 1):
if v.get(i) == "nein":
await _reject(topic, t, by_norm, files, "qa-fremd")
@@ -256,12 +284,15 @@ async def _entferne_fremd_unecht(topic: str, report: dict, by_norm: dict, files:
if unecht:
lines = [f"{t} — {by_norm[_norm_title(t)]['payload'].get('description') or '(ohne Beschreibung)'}"
for t in unecht]
v = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines, "nein")
v, frei = await _mit_stichentscheid("QA-Bausteine", topic, "unecht", "blocks", lines,
"nein", kategorie="unecht",
ids=[_norm_title(t) for t in unecht])
frei_alle += frei
for i, t in enumerate(unecht, 1):
if v.get(i) == "nein":
await _reject(topic, t, by_norm, files, "qa-unecht")
out.append(t)
return out
return out, frei_alle
async def _raeume_waisen(topic: str) -> int:

29
backend/tests/test_repair.py
View File

@@ -230,6 +230,35 @@ async def test_sub_dubletten_stichentscheid_faltet(env, monkeypatch):
assert rows["sub b"] == "variant" and rows["sub a"] == "consensus"
async def test_stichentscheid_behalten_persistiert_freispruch(env, monkeypatch):
"""2:1 „behalten" (Zweitmeinung + j3 einig gegen den QA-Befund) → Freispruch wird
persistiert und der Report des nächsten qa_report zählt das Paar nicht mehr —
vorher pendelte die Note dauerhaft unter 10 ohne Fix-Pfad."""
import qa as qa_mod
db, seed, files, write_report = env
await seed("Alpha", "beschr")
norm = repair._norm_title("Alpha")
await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A", status="consensus")
await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus")
write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]))
async def fake_agent(key, prompt, timeout, **kw):
return 0, '{"relevant": {"1": "nein"}}', "" # beide Repair-Judges: behalten
monkeypatch.setattr(repair, "run_agent", fake_agent)
res = await repair.repair_befunde(TOPIC)
assert res["sub_merges"] == [] and len(res["freigesprochen"]) == 1
frei = qa_mod.lade_freispruch(TOPIC)
key = qa_mod._paar_key("[Alpha] Sub A", "[Alpha] Sub B")
assert key in set(frei.get("sub_dubletten") or [])
# QA-Seite: bestätigtes, aber freigesprochenes Paar zählt nicht in die Quote
sd = [{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]
frei_sub = set(frei["sub_dubletten"])
zaehlt = sum(1 for p in sd if p.get("llm") == "ja"
and qa_mod._paar_key(p["a"], p["b"]) not in frei_sub)
assert zaehlt == 0
async def test_fremd_stichentscheid_behalten(env, monkeypatch):
"""Dissens bei fremd, Stichentscheid sagt ebenfalls behalten (ja) → Block bleibt
(fail-open bei 1:2 gegen den Befund)."""

4
frontend/src/components/GenerationView.vue
View File

@@ -106,10 +106,10 @@ async function repairClick() {
try {
const r = await runRepair(props.topic)
const n = (r.hygiene || []).length + (r.merges || []).length + (r.sub_merges || []).length
+ (r.entfernt || []).length + (r.aufgeraeumt || 0)
+ (r.entfernt || []).length + (r.aufgeraeumt || 0) + (r.freigesprochen || []).length
repairInfo.value = n === 0
? 'keine behebbaren Befunde'
: `${(r.hygiene || []).length} Titel · ${(r.merges || []).length} Merges · ${(r.sub_merges || []).length} Sub-Merges · ${(r.entfernt || []).length} entfernt · ${r.aufgeraeumt || 0} aufgeräumt`
: `${(r.hygiene || []).length} Titel · ${(r.merges || []).length} Merges · ${(r.sub_merges || []).length} Sub-Merges · ${(r.entfernt || []).length} entfernt · ${r.aufgeraeumt || 0} aufgeräumt · ${(r.freigesprochen || []).length} freigesprochen`
} catch (e) {
repairInfo.value = String(e.message || e)
} finally {