301 lines
14 KiB
Python
301 lines
14 KiB
Python
"""Befund-Repair: gezielte Aktionen aus dem QA-Report (repair.py) — ohne Flow, gegen Test-DB."""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
import repair
|
|
import qa as qa_mod
|
|
|
|
TOPIC = "reparatur"
|
|
|
|
|
|
def _report(**over):
|
|
r = {"topic": TOPIC, "note": 9.0, "quoten": {}, "hygiene": [], "dubletten": [],
|
|
"fremd": [], "unecht": [], "luecken": [], "artefakte": {"verwaiste": []}}
|
|
r.update(over)
|
|
return r
|
|
|
|
|
|
@pytest.fixture
|
|
async def env(testdb, tmp_path, monkeypatch):
|
|
"""Zwei fertige Blöcke auf beiden Boards + Subs/Artefakte + Sidecar-Dateien."""
|
|
db = testdb
|
|
monkeypatch.setattr(qa_mod, "QA_DIR", tmp_path / "qa")
|
|
files = {"sidecar": tmp_path / "sidecar.json", "facts": tmp_path / "facts.json",
|
|
"question_pattern": tmp_path / "qp.json", "sub_roh": tmp_path / "roh.json",
|
|
"artefakte": tmp_path / "artefakte.json"}
|
|
monkeypatch.setattr(repair, "_blocks_files", lambda t: files)
|
|
# frisches Abschluss-QA im Repair stumm schalten (eigener Test deckt qa_report ab)
|
|
async def _no_qa(topic, llm=False):
|
|
return None
|
|
monkeypatch.setattr(qa_mod, "qa_report", _no_qa)
|
|
|
|
async def _seed(title, desc, subs=1):
|
|
norm = repair._norm_title(title)
|
|
cid = "b-" + norm.replace(" ", "")[:10]
|
|
await db.kanban_upsert_card(TOPIC, "inventory", cid, "block", "done_block",
|
|
{"title": title, "description": desc, "sources": [f"{title}.txt"],
|
|
"readers": ["r1"], "mirrored_norm": norm})
|
|
await db.kanban_upsert_card(TOPIC, "artefacts", norm, "ablock", "done_artefact", {"title": title})
|
|
await db.upsert_block(TOPIC, norm, title, desc, [f"{title}.txt"])
|
|
await db.set_block_status(TOPIC, norm, "consensus")
|
|
for i in range(subs):
|
|
await db.put_subblock(TOPIC, norm, f"sub{i}", title, f"Sub {i}")
|
|
await db.put_sub_artifact(TOPIC, norm, "sub0", "flashcard", "{}", title, "Sub 0")
|
|
return cid
|
|
|
|
for p in files.values():
|
|
p.write_text("{}", encoding="utf-8")
|
|
(tmp_path / "qa" / TOPIC).mkdir(parents=True)
|
|
|
|
def write_report(r):
|
|
(tmp_path / "qa" / TOPIC / "r.json").write_text(json.dumps(r), encoding="utf-8")
|
|
|
|
return db, _seed, files, write_report
|
|
|
|
|
|
async def test_merge_confirmed_duplicate(env, monkeypatch):
|
|
db, seed, files, write_report = env
|
|
cid_a = await seed("Alpha", "kurz")
|
|
cid_b = await seed("Alpha Problem", "deutlich längere Beschreibung — Gewinner")
|
|
write_report(_report(dubletten=[{"a": "Alpha", "b": "Alpha Problem", "llm": "ja"},
|
|
{"a": "Alpha", "b": "Beta", "llm": "nein"}]))
|
|
calls = []
|
|
|
|
async def fake_agent(key, prompt, timeout, **kw):
|
|
calls.append(prompt)
|
|
return 0, '{"relevant": {"1": "ja"}}', ""
|
|
|
|
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["merges"] == ["Alpha → Alpha Problem"]
|
|
assert len(calls) == 1 and "Beta" not in calls[0] # nur das llm=ja-Paar zum Judge
|
|
verlierer = await db.kanban_get_card(TOPIC, "inventory", cid_a)
|
|
assert verlierer["stage"] == "grouped" and verlierer["payload"]["merged_into"] == "Alpha Problem"
|
|
gewinner = await db.kanban_get_card(TOPIC, "inventory", cid_b)
|
|
assert "Alpha.txt" in gewinner["payload"]["sources"] # Union
|
|
assert await db.kanban_get_card(TOPIC, "artefacts", "alpha") is None
|
|
assert not [r for r in await db.list_subblocks(TOPIC, "alpha")]
|
|
|
|
|
|
async def test_fremd_removed_only_on_nein(env, monkeypatch):
|
|
db, seed, files, write_report = env
|
|
cid_f = await seed("Fremdling", "gehört nicht rein")
|
|
cid_e = await seed("Echter", "belegt")
|
|
write_report(_report(fremd=["Fremdling", "Echter"]))
|
|
|
|
async def fake_agent(key, prompt, timeout, **kw):
|
|
if "-st-" in key: # Stichentscheid über den strittigen „Echter": behalten
|
|
return 0, '{"relevant": {"1": "ja"}}', ""
|
|
return 0, '{"relevant": {"1": "nein", "2": "ja"}}', ""
|
|
|
|
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
|
monkeypatch.setattr(repair, "source_folder", lambda t: None)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["entfernt"] == ["Fremdling"]
|
|
weg = await db.kanban_get_card(TOPIC, "inventory", cid_f)
|
|
assert weg["stage"] == "rejected" and weg["payload"]["reason"] == "qa-fremd"
|
|
bleibt = await db.kanban_get_card(TOPIC, "inventory", cid_e)
|
|
assert bleibt["stage"] == "done_block"
|
|
|
|
|
|
async def test_judge_failure_keeps_everything(env, monkeypatch):
|
|
db, seed, files, write_report = env
|
|
cid = await seed("Wackelig", "unsicher")
|
|
write_report(_report(unecht=["Wackelig"]))
|
|
|
|
async def broken_agent(key, prompt, timeout, **kw):
|
|
raise RuntimeError("boom")
|
|
|
|
monkeypatch.setattr(repair, "run_agent", broken_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["entfernt"] == []
|
|
card = await db.kanban_get_card(TOPIC, "inventory", cid)
|
|
assert card["stage"] == "done_block" # fail-open
|
|
|
|
|
|
async def test_hygiene_cleans_title_norm_invariant(env, monkeypatch):
|
|
db, seed, files, write_report = env
|
|
cid = await seed("**Fetter Titel**", "beschreibung")
|
|
files["sidecar"].write_text(json.dumps({"**Fetter Titel**": ["s"]}), encoding="utf-8")
|
|
write_report(_report(hygiene=[{"titel": "**Fetter Titel**", "probleme": ["markdown"]}]))
|
|
|
|
async def no_agent(*a, **kw):
|
|
raise AssertionError("Hygiene braucht keinen Agenten")
|
|
|
|
monkeypatch.setattr(repair, "run_agent", no_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["hygiene"] == ["**Fetter Titel** → Fetter Titel"]
|
|
card = await db.kanban_get_card(TOPIC, "inventory", cid)
|
|
assert card["payload"]["title"] == "Fetter Titel"
|
|
assert json.loads(files["sidecar"].read_text()) == {"Fetter Titel": ["s"]}
|
|
rows = await db.list_blocks(TOPIC)
|
|
assert any(r["title"] == "Fetter Titel" and r["status"] == "consensus" for r in rows)
|
|
|
|
|
|
async def test_no_report_is_clean_error(env):
|
|
db, seed, files, write_report = env
|
|
res = await repair.repair_befunde("gibtsnicht")
|
|
assert "fehler" in res
|
|
|
|
|
|
async def test_abschluss_qa_misst_mit_llm(env, monkeypatch):
|
|
"""Repair-Abschlussreport misst mit LLM — der llm=False-Report blendete
|
|
sub_dubletten aus und ließ die Note zwischen 10.0 und ~9 pendeln."""
|
|
db, seed, files, write_report = env
|
|
write_report(_report())
|
|
import qa as qa_mod
|
|
seen = {}
|
|
|
|
async def spy(topic, llm=False):
|
|
seen["llm"] = llm
|
|
return None
|
|
|
|
monkeypatch.setattr(qa_mod, "qa_report", spy)
|
|
await repair.repair_befunde(TOPIC)
|
|
assert seen["llm"] is True
|
|
|
|
|
|
async def test_sub_dubletten_merge(env, monkeypatch):
|
|
"""Bestätigtes Sub-Paar + Zweitmeinung ja → Verlierer variant, Frage/Artefakt
|
|
wandern zum Gewinner (bzw. fallen weg, wenn er den Typ schon hat)."""
|
|
db, seed, files, write_report = env
|
|
await seed("Alpha", "beschr")
|
|
norm = repair._norm_title("Alpha")
|
|
await db.put_subblock(TOPIC, norm, "gewinner sub", "Alpha", "Gewinner Sub",
|
|
facts='{"key_points": ["a", "b"]}', status="consensus")
|
|
await db.put_subblock(TOPIC, norm, "verlierer sub", "Alpha", "Verlierer Sub",
|
|
facts='{"key_points": ["x"]}', status="consensus")
|
|
await db.put_sub_artifact(TOPIC, norm, "verlierer sub", "example", "{}", "Alpha", "Verlierer Sub")
|
|
await db.upsert_question_pattern(TOPIC, norm, "verlierer sub", "Alpha", "Verlierer Sub", "Frage V?")
|
|
write_report(_report(sub_dubletten=[
|
|
{"a": "[Alpha] Gewinner Sub", "b": "[Alpha] Verlierer Sub", "llm": "ja"},
|
|
{"a": "[Alpha] Gibtsnicht", "b": "[Alpha] Verlierer Sub", "llm": "ja"}])) # tote Zeile → skip
|
|
|
|
async def fake_agent(key, prompt, timeout, **kw):
|
|
assert "Gibtsnicht" not in prompt
|
|
return 0, '{"relevant": {"1": "ja"}}', ""
|
|
|
|
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["sub_merges"] == ["Verlierer Sub → Gewinner Sub"]
|
|
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)}
|
|
assert rows["verlierer sub"] == "variant" and rows["gewinner sub"] == "consensus"
|
|
arts = {(r["sub_norm"], r["type"]) for r in await db.get_sub_artefakte(TOPIC)}
|
|
assert ("gewinner sub", "example") in arts and ("verlierer sub", "example") not in arts
|
|
fragen = {r["sub_norm"]: r["question"] for r in await db.list_question_pattern(TOPIC)}
|
|
assert fragen.get("gewinner sub") == "Frage V?" and "verlierer sub" not in fragen
|
|
|
|
|
|
async def test_sub_dubletten_zweitmeinung_nein(env, monkeypatch):
|
|
db, seed, files, write_report = env
|
|
await seed("Alpha", "beschr")
|
|
norm = repair._norm_title("Alpha")
|
|
await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A", status="consensus")
|
|
await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus")
|
|
write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]))
|
|
|
|
async def fake_agent(key, prompt, timeout, **kw):
|
|
return 0, '{"relevant": {"1": "nein"}}', ""
|
|
|
|
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["sub_merges"] == []
|
|
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)}
|
|
assert rows["sub a"] == rows["sub b"] == "consensus"
|
|
|
|
|
|
async def test_sub_dubletten_stichentscheid_faltet(env, monkeypatch):
|
|
"""Dissens QA (Befund) vs. Zweitmeinung (behalten) → Stichentscheid-Judge (Key -st)
|
|
entscheidet mit 2:1 für den Befund → Merge. Vorher pendelte die Note dauerhaft
|
|
unter 10 ohne Fix-Pfad („keine behebbaren Befunde" trotz Befund)."""
|
|
db, seed, files, write_report = env
|
|
await seed("Alpha", "beschr")
|
|
norm = repair._norm_title("Alpha")
|
|
await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A",
|
|
facts='{"key_points": ["a"]}', status="consensus")
|
|
await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus")
|
|
write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]))
|
|
|
|
async def fake_agent(key, prompt, timeout, **kw):
|
|
if "-st-" in key: # Stichentscheid bestätigt den QA-Befund
|
|
return 0, '{"relevant": {"1": "ja"}}', ""
|
|
return 0, '{"relevant": {"1": "nein"}}', "" # Zweitmeinung widerspricht
|
|
|
|
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["sub_merges"] == ["Sub B → Sub A"]
|
|
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)}
|
|
assert rows["sub b"] == "variant" and rows["sub a"] == "consensus"
|
|
|
|
|
|
async def test_stichentscheid_behalten_persistiert_freispruch(env, monkeypatch):
|
|
"""2:1 „behalten" (Zweitmeinung + j3 einig gegen den QA-Befund) → Freispruch wird
|
|
persistiert und der Report des nächsten qa_report zählt das Paar nicht mehr —
|
|
vorher pendelte die Note dauerhaft unter 10 ohne Fix-Pfad."""
|
|
import qa as qa_mod
|
|
db, seed, files, write_report = env
|
|
await seed("Alpha", "beschr")
|
|
norm = repair._norm_title("Alpha")
|
|
await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A", status="consensus")
|
|
await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus")
|
|
write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]))
|
|
|
|
async def fake_agent(key, prompt, timeout, **kw):
|
|
return 0, '{"relevant": {"1": "nein"}}', "" # beide Repair-Judges: behalten
|
|
|
|
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["sub_merges"] == [] and len(res["freigesprochen"]) == 1
|
|
frei = qa_mod.lade_freispruch(TOPIC)
|
|
key = qa_mod._paar_key("[Alpha] Sub A", "[Alpha] Sub B")
|
|
assert key in set(frei.get("sub_dubletten") or [])
|
|
# QA-Seite: bestätigtes, aber freigesprochenes Paar zählt nicht in die Quote
|
|
sd = [{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]
|
|
frei_sub = set(frei["sub_dubletten"])
|
|
zaehlt = sum(1 for p in sd if p.get("llm") == "ja"
|
|
and qa_mod._paar_key(p["a"], p["b"]) not in frei_sub)
|
|
assert zaehlt == 0
|
|
|
|
|
|
async def test_fremd_stichentscheid_behalten(env, monkeypatch):
|
|
"""Dissens bei fremd, Stichentscheid sagt ebenfalls behalten (ja) → Block bleibt
|
|
(fail-open bei 1:2 gegen den Befund)."""
|
|
db, seed, files, write_report = env
|
|
await seed("Alpha", "beschr")
|
|
write_report(_report(fremd=["Alpha"]))
|
|
|
|
async def fake_agent(key, prompt, timeout, **kw):
|
|
return 0, '{"relevant": {"1": "ja"}}', "" # beide: belegt/behalten
|
|
|
|
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["entfernt"] == []
|
|
|
|
|
|
async def test_waisen_cleanup(env, monkeypatch):
|
|
"""Artefakte/Fragen auf verworfene oder fehlende Subs fliegen; lebende und
|
|
mehrdeutig-präfixige bleiben."""
|
|
db, seed, files, write_report = env
|
|
await seed("Alpha", "beschreibung") # legt sub0 (consensus) + flashcard auf sub0 an
|
|
norm = repair._norm_title("Alpha")
|
|
await db.put_subblock(TOPIC, norm, "weg", "Alpha", "Weg", status="discarded")
|
|
await db.put_subblock(TOPIC, norm, "doppel: eins", "Alpha", "Doppel eins")
|
|
await db.put_subblock(TOPIC, norm, "doppel: zwei", "Alpha", "Doppel zwei")
|
|
await db.put_sub_artifact(TOPIC, norm, "weg", "flashcard", "{}", "Alpha", "Weg") # tot
|
|
await db.put_sub_artifact(TOPIC, norm, "fehlt", "example", "{}", "Alpha", "Fehlt") # tot
|
|
await db.put_sub_artifact(TOPIC, norm, "doppel", "example", "{}", "Alpha", "Doppel") # mehrdeutig → bleibt
|
|
await db.upsert_question_pattern(TOPIC, norm, "fehlt", "Alpha", "Fehlt", "Frage?") # tot
|
|
write_report(_report())
|
|
|
|
async def no_agent(*a, **kw):
|
|
raise AssertionError("Aufräumen braucht keinen Agenten")
|
|
|
|
monkeypatch.setattr(repair, "run_agent", no_agent)
|
|
res = await repair.repair_befunde(TOPIC)
|
|
assert res["aufgeraeumt"] == 3
|
|
rest = {(r["sub_norm"], r["type"]) for r in await db.get_sub_artefakte(TOPIC)}
|
|
assert rest == {("sub0", "flashcard"), ("doppel", "example")}
|
|
assert not [r for r in await db.list_question_pattern(TOPIC)]
|