update
This commit is contained in:
@@ -56,6 +56,16 @@ async def board_env(testdb, tmp_path, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", _fake_single_slot(tmp_path))
|
||||
|
||||
# QA-Gate: standardmäßig saubere Fake-Note (kein Embedding-Load in Tests);
|
||||
# Gate-Tests überschreiben qa_report gezielt.
|
||||
import qa as qa_mod
|
||||
|
||||
async def _fake_qa(topic, llm=False):
|
||||
return {"note": 10.0, "topic": TOPIC, "quoten": {}, "fremd": [], "artefakte": {"status": "nicht generiert"}}
|
||||
monkeypatch.setattr(qa_mod, "qa_report", _fake_qa)
|
||||
monkeypatch.setattr(qa_mod, "_write_report", lambda r: None)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
|
||||
async def no_emb(flow):
|
||||
return False
|
||||
monkeypatch.setattr(bi, "_emb_ok", no_emb)
|
||||
@@ -86,12 +96,21 @@ async def board_env(testdb, tmp_path, monkeypatch):
|
||||
async def fake_outline(ctx, set_p, files, entries, instructions):
|
||||
return {"chapters": [{"title": "Kapitel 1", "numbers": sorted(entries)}]}
|
||||
|
||||
async def fake_konsolidierung(ctx, files, raw, facts_map, instructions="", ns="", lbl=""):
|
||||
return None
|
||||
|
||||
for name, fn in [("_subblocks_block", fake_subblocks), ("_facts_block", fake_facts),
|
||||
("_levels_block", fake_levels), ("_relevance_block", fake_relevance),
|
||||
("_question_pattern_block", fake_pattern), ("_artefacts_block", fake_artefacts),
|
||||
("_outline_block", fake_outline)]:
|
||||
("_outline_block", fake_outline), ("_konsolidiere_subblocks", fake_konsolidierung)]:
|
||||
monkeypatch.setattr(ba, name, fn)
|
||||
|
||||
class _EmbOff: # Cross-Block-Barrier reicht ohne Modell alle Karten durch
|
||||
@staticmethod
|
||||
def available():
|
||||
return False
|
||||
monkeypatch.setattr(ba, "embedding", _EmbOff)
|
||||
|
||||
work = tmp_path / "arbeit"
|
||||
work.mkdir()
|
||||
files = {"arbeit": work, "final": tmp_path / "blocks.md",
|
||||
@@ -151,6 +170,10 @@ async def test_board1_full_flow(board_env):
|
||||
assert {s["sub_title"] for s in subs} == {"Sub Eins", "Sub Zwei"}
|
||||
outline = await db.get_outline(TOPIC)
|
||||
assert outline and "Kapitel 1" in outline
|
||||
# Lauf-Summary am Flow-Ende: run_id + Zähler (QA diffed dagegen)
|
||||
summary = json.loads((files["arbeit"] / "lauf-summary.json").read_text(encoding="utf-8"))
|
||||
assert summary["run_id"] and summary["topic"] == TOPIC
|
||||
assert summary["boards"].get("inventory", {}).get("done_block") == 4
|
||||
|
||||
|
||||
async def test_filter_judges_run_parallel(board_env, monkeypatch):
|
||||
@@ -839,3 +862,204 @@ async def test_ingest_strips_markdown_title(testdb, tmp_path):
|
||||
assert n == 1
|
||||
card = await testdb.kanban_get_card(TOPIC, B, "listscheduling")
|
||||
assert card["payload"]["title"] == "ListScheduling"
|
||||
|
||||
|
||||
# ── QA-Gate: Inventar-Prüfung vor Board 2 ────────────────────────────────────────────
|
||||
|
||||
async def _run_flow(ctx, files, timeout=30, **kw):
|
||||
import asyncio
|
||||
return await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "", research=False, **kw),
|
||||
timeout=timeout)
|
||||
|
||||
|
||||
async def test_qa_gate_pauses_on_bad_note(board_env, monkeypatch):
|
||||
"""Note unter Schwelle → Flow endet sauber, Board-2-Karten warten in subblocks."""
|
||||
import qa as qa_mod
|
||||
db, ctx, files = board_env
|
||||
|
||||
async def bad_qa(topic, llm=False):
|
||||
return {"note": 5.0, "topic": TOPIC, "quoten": {}, "fremd": [], "artefakte": {}}
|
||||
monkeypatch.setattr(qa_mod, "qa_report", bad_qa)
|
||||
monkeypatch.setattr(qa_mod, "_write_report", lambda r: None)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
await _seed(db)
|
||||
ok = await _run_flow(ctx, files)
|
||||
assert ok
|
||||
warten = await db.kanban_cards(TOPIC, board="artefacts", stage="subblocks")
|
||||
assert len(warten) == 4 # alle Blöcke gespawnt, keiner verarbeitet
|
||||
assert await db.kanban_count(TOPIC, "done_artefact", board="artefacts") == 0
|
||||
|
||||
|
||||
async def test_qa_gate_force_overrides(board_env, monkeypatch):
|
||||
"""qa_force=True („Trotzdem fortsetzen") übersteuert die schlechte Note."""
|
||||
import qa as qa_mod
|
||||
db, ctx, files = board_env
|
||||
called = {"n": 0}
|
||||
|
||||
async def bad_qa(topic, llm=False):
|
||||
called["n"] += 1
|
||||
return {"note": 5.0, "topic": TOPIC, "quoten": {}, "fremd": [], "artefakte": {}}
|
||||
monkeypatch.setattr(qa_mod, "qa_report", bad_qa)
|
||||
monkeypatch.setattr(qa_mod, "_write_report", lambda r: None)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
await _seed(db)
|
||||
ok = await _run_flow(ctx, files, qa_force=True)
|
||||
assert ok
|
||||
assert await db.kanban_count(TOPIC, "done_artefact", board="artefacts") >= 5
|
||||
assert called["n"] <= 1 # Gate-Lauf übersprungen; höchstens Abschluss-QA
|
||||
|
||||
|
||||
async def test_qa_gate_off_means_no_qa_call(board_env, monkeypatch):
|
||||
import qa as qa_mod
|
||||
db, ctx, files = board_env
|
||||
called = {"n": 0}
|
||||
|
||||
async def spy_qa(topic, llm=False):
|
||||
called["n"] += 1
|
||||
return {"note": 10.0, "topic": TOPIC, "quoten": {}, "fremd": [], "artefakte": {}}
|
||||
monkeypatch.setattr(qa_mod, "qa_report", spy_qa)
|
||||
monkeypatch.setattr(qa_mod, "_write_report", lambda r: None)
|
||||
monkeypatch.setattr(bi, "QA_GATE_NOTE", 0)
|
||||
await _seed(db)
|
||||
ok = await _run_flow(ctx, files)
|
||||
assert ok
|
||||
assert called["n"] == 1 # kein Gate-Lauf; nur die Abschluss-QA der Lauf-Summary
|
||||
|
||||
|
||||
async def test_qa_gate_fail_open(board_env, monkeypatch):
|
||||
"""QA crasht → Gate öffnet, Flow läuft komplett durch (fail-open)."""
|
||||
import qa as qa_mod
|
||||
db, ctx, files = board_env
|
||||
|
||||
async def broken_qa(topic, llm=False):
|
||||
raise RuntimeError("kaputt")
|
||||
monkeypatch.setattr(qa_mod, "qa_report", broken_qa)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
await _seed(db)
|
||||
ok = await _run_flow(ctx, files)
|
||||
assert ok
|
||||
assert await db.kanban_count(TOPIC, "done_artefact", board="artefacts") >= 5
|
||||
|
||||
|
||||
def test_qa_view_pausiert_logic(tmp_path, monkeypatch):
|
||||
import qa as qa_mod
|
||||
import json as _json
|
||||
monkeypatch.setattr(qa_mod, "QA_DIR", tmp_path)
|
||||
(tmp_path / TOPIC).mkdir()
|
||||
(tmp_path / TOPIC / "r1.json").write_text(_json.dumps(
|
||||
{"note": 5.0, "quoten": {"fremd": 0.2}, "fremd": ["X"], "unecht": ["Y"]}), encoding="utf-8")
|
||||
counts = {"inventory": {"done_block": 3}, "artefacts": {"subblocks": 4}}
|
||||
v = bi._qa_view(TOPIC, counts, None)
|
||||
assert v["pausiert"] is True and v["note"] == 5.0 and v["befunde"] == ["X", "Y"]
|
||||
from types import SimpleNamespace
|
||||
laufend = SimpleNamespace(state={})
|
||||
assert bi._qa_view(TOPIC, counts, laufend)["pausiert"] is False # Flow läuft noch
|
||||
# Bausteine gelöscht → kein Badge, obwohl der Report noch existiert
|
||||
assert bi._qa_view(TOPIC, {}, None) is None
|
||||
|
||||
|
||||
def test_qa_view_picks_newest_by_mtime(tmp_path, monkeypatch):
|
||||
"""Run-id-Namen (…-1311-5e5c) sortieren lexikographisch VOR Zeitstempel-Namen —
|
||||
ein Re-Run überschreibt die run-id-Datei, das Badge muss trotzdem sie zeigen."""
|
||||
import os
|
||||
import qa as qa_mod
|
||||
import json as _json
|
||||
monkeypatch.setattr(qa_mod, "QA_DIR", tmp_path)
|
||||
(tmp_path / TOPIC).mkdir()
|
||||
alt = tmp_path / TOPIC / "20260703-141649.json"
|
||||
alt.write_text(_json.dumps({"note": 10.0, "quoten": {}}), encoding="utf-8")
|
||||
os.utime(alt, (1000, 1000))
|
||||
neu = tmp_path / TOPIC / "20260703-1311-5e5c.json"
|
||||
neu.write_text(_json.dumps({"note": 8.9, "quoten": {}}), encoding="utf-8")
|
||||
os.utime(neu, (2000, 2000))
|
||||
v = bi._qa_view(TOPIC, {"inventory": {"done_block": 3}}, None)
|
||||
assert v["note"] == 8.9
|
||||
|
||||
|
||||
async def test_supplement_material_mode_for_source_topics(board_env, tmp_path, monkeypatch):
|
||||
"""Quellen-Thema: Supplement vergleicht gegen das MATERIAL (files, kein Web);
|
||||
thema-Modus behält die Websuche (voller Zugriff)."""
|
||||
db, ctx, files = board_env
|
||||
seen = {}
|
||||
|
||||
async def spy_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
|
||||
seen[key] = (capabilities, prompt)
|
||||
m = _PATH_RE.search(prompt)
|
||||
if m and "-supplement" in key and "-beleg" not in key:
|
||||
with open(m.group(1), "w", encoding="utf-8") as f:
|
||||
json.dump({"blocks": []}, f)
|
||||
return "ok", payload(None)
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", spy_slot)
|
||||
flow = _mk_flow(tmp_path)
|
||||
korpus = tmp_path / "korpus"
|
||||
korpus.mkdir()
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: korpus)
|
||||
await bi._supplement_producer(ctx, flow, ["Alpha"])
|
||||
caps, prompt = seen[f"blocks-{TOPIC}-supplement"]
|
||||
assert caps == "files"
|
||||
assert "LEARNING MATERIAL" in prompt and "Do NOT search the web" in prompt
|
||||
|
||||
seen.clear()
|
||||
(tmp_path / "supplement.json").unlink() # Resume-Guard zurücksetzen
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: None)
|
||||
await bi._supplement_producer(ctx, flow, ["Alpha"])
|
||||
caps, prompt = seen[f"blocks-{TOPIC}-supplement"]
|
||||
assert caps == "full"
|
||||
assert "Research the subject area" in prompt
|
||||
|
||||
|
||||
# ── Anker-Gate: Quorum-Titel ohne Korpus-Beleg (Reader-Ko-Halluzination) ────────────
|
||||
|
||||
async def _anker_env(db, tmp_path, monkeypatch, titel_map):
|
||||
(tmp_path / "korpus.txt").write_text("Der Graph ist zusammenhängend und endlich.", encoding="utf-8")
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: tmp_path)
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"title": titel_map[cid], "description": "", "readers": ["r1", "r2"], "supplement": False}]
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "_rep", lambda rows: rows[0])
|
||||
for cid in titel_map:
|
||||
await db.kanban_upsert_card(TOPIC, B, cid, "cluster", "consensus_gate", {})
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
return ctx, [{"card_id": c, "payload": {}} for c in titel_map]
|
||||
|
||||
|
||||
async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
|
||||
"""Titel ohne Korpus-Anker → Beleg-Judge; „nein" → rejected/kein-beleg.
|
||||
Titel MIT Anker geht ohne Judge nach naming."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch,
|
||||
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"})
|
||||
|
||||
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "Brückenproblem" in prompt and "Zusammenhang" not in prompt # nur der Anker-lose
|
||||
return "ok", payload((0, json.dumps({"relevant": {"1": "nein"}}), ""))
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), cards)
|
||||
assert (await db.kanban_get_card(TOPIC, B, "c1"))["stage"] == "naming"
|
||||
c2 = await db.kanban_get_card(TOPIC, B, "c2")
|
||||
assert c2["stage"] == "rejected" and c2["payload"]["reason"] == "kein-beleg"
|
||||
|
||||
|
||||
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Judge-Ausfall → Titel bleibt (2-Reader-Rückhalt)."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"})
|
||||
|
||||
async def broken_slot(*a, **kw):
|
||||
return "failed", None
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", broken_slot)
|
||||
await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), cards)
|
||||
assert (await db.kanban_get_card(TOPIC, B, "c9"))["stage"] == "naming"
|
||||
|
||||
|
||||
def test_hat_anker_ziffern_suffix():
|
||||
ctoks = {"tsp", "graph", "kanten"}
|
||||
assert bi._hat_anker("ΔTSP1-Algorithmus", ctoks) # tsp1 → tsp
|
||||
assert not bi._hat_anker("Königsberger Brückenproblem", ctoks)
|
||||
assert not bi._hat_anker("Algorithmus Verfahren", ctoks) # nur Stopwörter → kein Anker
|
||||
|
||||
@@ -283,3 +283,37 @@ async def test_run_agent_logs_opencode_tokens(testdb, monkeypatch):
|
||||
rc, *_ = await agents.run_agent("blocks-t-tok", "p", 5, provider="minimax", scope=TOPIC)
|
||||
assert rc == 0
|
||||
assert recorded and recorded[0]["meta"]["tokens"]["cache_read"] == 100
|
||||
|
||||
|
||||
# ── run_id-Registry + Lauf-Summary ───────────────────────────────────────────────────
|
||||
|
||||
async def test_run_id_stamped_on_events(testdb):
|
||||
"""Registry gesetzt → Agent- und Stage-Events tragen die run_id; geleert → leer."""
|
||||
db = testdb
|
||||
db.set_current_run(TOPIC, "20260703-1200-abcd")
|
||||
await db.add_event(TOPIC, "agent", key="k1", status="ok",
|
||||
meta={"tokens": {"input": 10, "output": 2, "cache_read": 50, "cache_write": 1}})
|
||||
await db.kanban_upsert_card(TOPIC, "inventory", "c1", "block", "ingest", {})
|
||||
await db.kanban_advance(TOPIC, "inventory", "c1", "cluster")
|
||||
db.set_current_run(TOPIC, None)
|
||||
await db.add_event(TOPIC, "agent", key="k2", status="ok")
|
||||
conn = await db.get_db()
|
||||
rows = await (await conn.execute("SELECT key, run_id FROM events WHERE topic=? ORDER BY id", (TOPIC,))).fetchall()
|
||||
by_key = {k: r for k, r in rows}
|
||||
assert by_key["k1"] == "20260703-1200-abcd"
|
||||
assert by_key["inventory:c1"] == "20260703-1200-abcd"
|
||||
assert by_key["k2"] == ""
|
||||
|
||||
|
||||
async def test_events_run_summary_aggregates(testdb):
|
||||
db = testdb
|
||||
db.set_current_run(TOPIC, "r1")
|
||||
await db.add_event(TOPIC, "agent", key="a", status="ok", dur_ms=1000,
|
||||
meta={"tokens": {"input": 10, "output": 2, "cache_read": 50, "cache_write": 1}})
|
||||
await db.add_event(TOPIC, "agent", key="b", status="timeout", dur_ms=120000,
|
||||
meta={"tokens": {"input": 5, "output": 0, "cache_read": 30, "cache_write": 0}})
|
||||
db.set_current_run(TOPIC, None)
|
||||
s = await db.events_run_summary(TOPIC, "r1")
|
||||
assert s["agents"]["gesamt"] == 2 and s["agents"]["ok"] == 1 and s["agents"]["timeout"] == 1
|
||||
assert s["agents"]["verlorene_min"] == 2
|
||||
assert s["tokens"] == {"input": 15, "output": 2, "cache_read": 80, "cache_write": 1}
|
||||
|
||||
@@ -150,8 +150,8 @@ def test_writer_template_has_examples_placeholder():
|
||||
from pipeline import _prompt
|
||||
text = _prompt("Guide-Writer-Board", topic="t", format_name="Guide", chapter="K1",
|
||||
assignment="- B", ziele="- z", facts="F", examples="", gaps="",
|
||||
spec="", out_path="/tmp/x.md", extra="")
|
||||
assert "VERIFIED FACTS" in text
|
||||
budget=2000, spec="", out_path="/tmp/x.md", extra="")
|
||||
assert "VERIFIED FACTS" in text and "2000 characters" in text
|
||||
|
||||
|
||||
async def test_fakten_gate_counts_examples_as_facts(testdb, monkeypatch, tmp_path):
|
||||
@@ -222,3 +222,78 @@ async def test_writer_splits_oversized_first_draft(testdb, monkeypatch, tmp_path
|
||||
secs = _parse_fragment(card["md"])
|
||||
assert len(secs) == 1 and [s["title"] for s in secs[0]["subs"]] == ["Sub 1", "Sub 2"]
|
||||
assert card["stage"] == "fakten_gate"
|
||||
|
||||
|
||||
async def test_lernziele_retry_bei_leerer_liste(testdb, tmp_path, monkeypatch):
|
||||
"""Leere Ziele-Liste → genau EIN Ersatz-Versuch (Key-Suffix -2); dessen Ziele landen in der DB."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-r", TOPIC, FMT, "", tmp_path / "Guide.json",
|
||||
{"Alpha": [{"title": "S1", "level": "beginner"}]}, {}, "(quelle)", "spec")
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0}
|
||||
calls = []
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
calls.append(key)
|
||||
if len(calls) == 1:
|
||||
return gb.OK, []
|
||||
return gb.OK, [{"id": "z1", "text": "Ziel", "sub": "S1"}]
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
assert await gb._stage_lernziele(env, card)
|
||||
assert len(calls) == 2 and calls[1].endswith("-2")
|
||||
assert [z["ziel_id"] for z in await db.list_lernziele(TOPIC, "alpha")] == ["z1"]
|
||||
|
||||
|
||||
async def test_lernziele_zweimal_leer_laeuft_weiter(testdb, tmp_path, monkeypatch):
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-r2", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0}
|
||||
|
||||
async def leer(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
return gb.OK, []
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", leer)
|
||||
assert await gb._stage_lernziele(env, card)
|
||||
assert (await db.list_guide_cards(TOPIC, FMT))[0]["stage"] == "zuweisung"
|
||||
assert not await db.list_lernziele(TOPIC, "alpha")
|
||||
|
||||
|
||||
async def test_lese_check_text_sink(testdb, tmp_path, monkeypatch):
|
||||
"""Lese-Check antwortet als Text, Engine-Sink persistiert; capabilities none."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-l", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
md = ("<!-- kapitel: K -->\n<!-- section: Alpha -->\n<!-- compact -->\n- x\n"
|
||||
"<!-- ausführlich -->\nText.")
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
seen = {}
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
seen["caps"] = capabilities
|
||||
return gb.OK, payload((0, '{"ok": true}', ""))
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
monkeypatch.setattr(gb, "READABILITY_ACTIVE", False)
|
||||
assert await gb._stage_lesbarkeit(env, card)
|
||||
assert seen["caps"] == "none"
|
||||
assert (await db.list_guide_cards(TOPIC, FMT))[0]["stage"] == "done"
|
||||
|
||||
|
||||
async def test_writer_prompt_traegt_budget(testdb, tmp_path, monkeypatch):
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-w", TOPIC, FMT, "", tmp_path / "Guide.json",
|
||||
{"Alpha": [{"title": "S1", "level": "beginner"},
|
||||
{"title": "S2", "level": "beginner"}]}, {}, "(q)", "spec")
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "chapter": "K", "gate_info": ""}
|
||||
seen = {}
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
seen["prompt"] = prompt
|
||||
return gb.FAILED, None
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
await gb._stage_writer(env, card)
|
||||
assert str(gb._writer_budget(2)) in seen["prompt"] # 800 + 2×400
|
||||
|
||||
65
backend/tests/test_guide_qa.py
Normal file
65
backend/tests/test_guide_qa.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""Guide-QA: Fehler-Injektion auf Mini-Guide-Karten — deterministisch, ohne LLM."""
|
||||
|
||||
import guide_qa as gq
|
||||
import qa
|
||||
|
||||
|
||||
def _card(block, md):
|
||||
return {"block": block, "block_norm": block.casefold(), "md": md}
|
||||
|
||||
|
||||
AUSF = ("<!-- section: Alpha -->\n<!-- compact -->\n- m\n<!-- ausführlich -->\n"
|
||||
"Einstieg in den Block.\n"
|
||||
"<!-- sub: beginner | Kantenzug Definition -->\n"
|
||||
"Ein Kantenzug verbindet Knoten über Kanten im Graphen.\n")
|
||||
|
||||
|
||||
def test_ausfuehrlich_extrahiert_lerntext():
|
||||
assert gq._ausfuehrlich(AUSF).startswith("\nEinstieg")
|
||||
assert gq._ausfuehrlich("nur text") == "nur text"
|
||||
|
||||
|
||||
def test_marker_fehlend():
|
||||
cards = [_card("Alpha", AUSF)]
|
||||
rel = {"alpha": {"kantenzug definition", "fehlender aspekt"}}
|
||||
out = gq.marker_fehlend(cards, rel)
|
||||
assert out == ["Alpha · fehlender aspekt"]
|
||||
|
||||
|
||||
def test_ziel_ohne_anker():
|
||||
cards = [_card("Alpha", AUSF)]
|
||||
ziele = [{"block_norm": "alpha", "ziel_id": "z1", "text": "Kantenzug im Graphen erklären"},
|
||||
{"block_norm": "alpha", "ziel_id": "z2", "text": "Adjazenzmatrix aufstellen können"}]
|
||||
out = gq.ziel_ohne_anker(cards, ziele)
|
||||
assert len(out) == 1 and "z2" in out[0]
|
||||
|
||||
|
||||
def test_laengen_ausreisser():
|
||||
duenn = _card("Alpha", "<!-- ausführlich -->\nkurz")
|
||||
ok = _card("Beta", "<!-- ausführlich -->\n" + "x" * 500)
|
||||
out = gq.laengen_ausreisser([duenn, ok], {"alpha": {"s1"}, "beta": {"s1"}})
|
||||
assert [x["block"] for x in out] == ["Alpha"]
|
||||
|
||||
|
||||
def test_redundanz_findet_absatz_doppel():
|
||||
a = "Der Kantenzug verbindet Knoten über mehrere Kanten und darf Knoten wiederholen. " * 3
|
||||
b = "Der Kantenzug verbindet Knoten über mehrere Kanten und darf Knoten wiederholen, genau. " * 3
|
||||
c = "Völlig anderes Thema: Matrizen, Determinanten und lineare Abbildungen im Vektorraum. " * 3
|
||||
cards = [_card("Alpha", f"<!-- ausführlich -->\n{a}\n\n{c}"),
|
||||
_card("Beta", f"<!-- ausführlich -->\n{b}")]
|
||||
out = gq.redundanz(cards)
|
||||
assert len(out) == 1 and out[0]["a"].startswith("Alpha")
|
||||
|
||||
|
||||
def test_lesbarkeit_fail_open(monkeypatch):
|
||||
def kaputt(md_by_num):
|
||||
raise RuntimeError("Modell fehlt")
|
||||
monkeypatch.setattr(gq.readability, "rate_sections", kaputt)
|
||||
assert gq.lesbarkeit([_card("Alpha", AUSF)]) == []
|
||||
|
||||
|
||||
def test_note_guide_kalibrierung():
|
||||
"""Gewicht = Punktabzug bei 100 %: 10 % fachlich falsch × 3.0 → 7.0; ungemessen zählt nicht."""
|
||||
assert qa.note({"fachlich_falsch": 0.1}, gq.NOTE_GEWICHTE_GUIDE) == 7.0
|
||||
ohne = {"marker_fehlend": 0.0, "ziel_ohne_anker": 0.0}
|
||||
assert qa.note(ohne, gq.NOTE_GEWICHTE_GUIDE) == 10.0
|
||||
507
backend/tests/test_konsolidierung.py
Normal file
507
backend/tests/test_konsolidierung.py
Normal file
@@ -0,0 +1,507 @@
|
||||
"""Sub-Konsolidierung: In-Block-Panel (blocks._konsolidiere_subblocks) und
|
||||
Cross-Block-Barrier (board_artefacts._proc_konsolidierung) — Judges gefaked, gegen Test-DB."""
|
||||
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import blocks
|
||||
import board_artefacts as ba
|
||||
from kanban import Flow
|
||||
from pipeline import FAILED, OK, GenContext
|
||||
|
||||
TOPIC = "konsolidierung"
|
||||
|
||||
|
||||
def _ctx():
|
||||
return GenContext(topic=TOPIC, provider="test", is_cancelled=lambda: False)
|
||||
|
||||
|
||||
def _fake_slot(antworten):
|
||||
"""run_single_slot-Fake: pro Judge-Key eine Antwort; schreibt via payload (wie der Engine-Sink)."""
|
||||
calls = []
|
||||
|
||||
async def fake(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
calls.append({"key": key, "prompt": prompt})
|
||||
j = key.rsplit("-", 1)[-1] # "j1"/"j2"
|
||||
antwort = antworten.get(j)
|
||||
if antwort is None:
|
||||
return FAILED, None
|
||||
return OK, payload((0, json.dumps(antwort), ""))
|
||||
|
||||
fake.calls = calls
|
||||
return fake
|
||||
|
||||
|
||||
async def _seed_block(db, bnorm, subs):
|
||||
for s in subs:
|
||||
await db.put_subblock(TOPIC, bnorm, blocks._norm_title(s), bnorm.title(), s, status="consensus")
|
||||
|
||||
|
||||
# ── In-Block ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
async def test_merge_on_unanimity(testdb, tmp_path, monkeypatch):
|
||||
"""Beide Judges gruppieren 1+2 → Gewinner (mehr key_points) bleibt, facts-Union,
|
||||
Verlierer wird DB-variant und fliegt aus raw/facts_map."""
|
||||
db = testdb
|
||||
subs = ["Durchstreichung: ~~text~~", "Durchstreichung: ~~text~~ streicht Text durch", "Fett: **text**"]
|
||||
await _seed_block(db, "betonung", subs)
|
||||
raw = {"Betonung": list(subs)}
|
||||
facts = {"Betonung": {
|
||||
blocks._norm_title(subs[0]): {"key_points": ["kp-a"], "cited_facts": [{"text": "z1"}]},
|
||||
blocks._norm_title(subs[1]): {"key_points": ["kp-b", "kp-c"], "cited_facts": [{"text": "z1"}, {"text": "z2"}]},
|
||||
}}
|
||||
fake = _fake_slot({"j1": {"gruppen": [[1, 2]], "luecken": ["Marker-Escaping fehlt"]},
|
||||
"j2": {"gruppen": [[2, 1]], "luecken": ["Escaping von Markern"]}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, facts)
|
||||
assert raw["Betonung"] == [subs[1], "Fett: **text**"] # Gewinner: 2 key_points > 1
|
||||
wf = facts["Betonung"][blocks._norm_title(subs[1])]
|
||||
assert wf["key_points"] == ["kp-b", "kp-c", "kp-a"]
|
||||
assert wf["cited_facts"] == [{"text": "z1"}, {"text": "z2"}] # Union ohne Doppel
|
||||
assert blocks._norm_title(subs[0]) not in facts["Betonung"]
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "betonung")}
|
||||
assert rows[blocks._norm_title(subs[0])] == "variant"
|
||||
assert rows[blocks._norm_title(subs[1])] == "consensus"
|
||||
journale = list(tmp_path.glob("sub-konsolidierung-*.json"))
|
||||
j = json.loads([p for p in journale if "-j" not in p.stem][0].read_text())
|
||||
# Lücken-Schnitt: Token-Überlappung beider Judges, Formulierung von j1 gewinnt
|
||||
assert j["gruppen"][0]["behalten"] == subs[1] and j["luecken"] == ["Marker-Escaping fehlt"]
|
||||
|
||||
|
||||
async def test_dissent_keeps_everything(testdb, tmp_path, monkeypatch):
|
||||
"""Nur ein Judge gruppiert → keine Einstimmigkeit → kein Merge."""
|
||||
db = testdb
|
||||
subs = ["Eintrag eins", "Eintrag zwei"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
fake = _fake_slot({"j1": {"gruppen": [[1, 2]], "luecken": []},
|
||||
"j2": {"gruppen": [], "luecken": []}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, {})
|
||||
assert raw["Block"] == subs
|
||||
assert all(r["status"] == "consensus" for r in await db.list_subblocks(TOPIC, "block"))
|
||||
|
||||
|
||||
async def test_judge_failure_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Ein Judge UND der Ersatz ohne Ergebnis → fail-open, nichts ändert sich."""
|
||||
db = testdb
|
||||
subs = ["Eintrag eins", "Eintrag zwei"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
fake = _fake_slot({"j1": {"gruppen": [[1, 2]], "luecken": []}}) # j2 UND j3 → FAILED
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, {})
|
||||
assert raw["Block"] == subs
|
||||
assert len(fake.calls) == 3 # j1, j2, Ersatz j3
|
||||
|
||||
|
||||
async def test_ersatzrichter_bei_ausfall(testdb, tmp_path, monkeypatch):
|
||||
"""j1 fällt aus → Ersatz j3 springt ein; Einstimmigkeit j2+j3 faltet.
|
||||
Vorher entwertete EIN Timeout die gute Stimme (13 Links-Dubletten überlebten)."""
|
||||
db = testdb
|
||||
subs = ["Kurz", "Deutlich längerer Eintrag"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
fake = _fake_slot({"j2": {"gruppen": [[1, 2]], "luecken": []},
|
||||
"j3": {"gruppen": [[2, 1]], "luecken": []}}) # j1 → FAILED
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, {})
|
||||
assert raw["Block"] == ["Deutlich längerer Eintrag"]
|
||||
|
||||
|
||||
async def test_negation_guard_blocks_merge(testdb, tmp_path, monkeypatch):
|
||||
"""Gegensätzliche Aussagen werden selbst bei einstimmigen Judges nicht gefaltet."""
|
||||
db = testdb
|
||||
subs = ["Tabs werden expandiert", "Tabs werden nicht expandiert"]
|
||||
await _seed_block(db, "tabs", subs)
|
||||
raw = {"Tabs": list(subs)}
|
||||
fake = _fake_slot({"j1": {"gruppen": [[1, 2]], "luecken": []},
|
||||
"j2": {"gruppen": [[1, 2]], "luecken": []}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, {})
|
||||
assert raw["Tabs"] == subs
|
||||
|
||||
|
||||
async def test_resume_skips_judges(testdb, tmp_path, monkeypatch):
|
||||
"""Vorhandene j-Dateien → kein neuer Agenten-Call, Ergebnis wird übernommen."""
|
||||
db = testdb
|
||||
subs = ["Eintrag eins", "Eintrag zwei lang"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
import hashlib
|
||||
h = hashlib.md5("\n".join(subs).encode()).hexdigest()[:8]
|
||||
for j in (1, 2):
|
||||
(tmp_path / f"sub-konsolidierung-{h}-j{j}.json").write_text(
|
||||
json.dumps({"gruppen": [[1, 2]], "luecken": []}), encoding="utf-8")
|
||||
|
||||
async def kein_agent(*a, **kw):
|
||||
raise AssertionError("Resume darf keinen Agenten starten")
|
||||
|
||||
monkeypatch.setattr(blocks, "run_single_slot", kein_agent)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, {})
|
||||
assert raw["Block"] == ["Eintrag zwei lang"]
|
||||
|
||||
|
||||
def test_schema_accepts_both_group_forms():
|
||||
"""Alte Listenform [1,4] und neue {haupt, weitere}-Form parsen beide; kataloge/fremd optional."""
|
||||
alt = blocks._konsolidierung_schema({"gruppen": [[1, 4]], "luecken": []}, 5)
|
||||
assert alt["gruppen"] == [{"haupt": None, "ids": [1, 4]}] and alt["fremd"] == set()
|
||||
neu = blocks._konsolidierung_schema(
|
||||
{"gruppen": [{"haupt": 4, "weitere": [1]}],
|
||||
"kataloge": [{"titel": "Katalog: Symbole", "mitglieder": [2, 3]}],
|
||||
"fremd": [5], "luecken": ["x"]}, 5)
|
||||
assert neu["gruppen"] == [{"haupt": 4, "ids": [1, 4]}]
|
||||
assert neu["kataloge"] == [{"titel": "Katalog: Symbole", "ids": [2, 3]}]
|
||||
assert neu["fremd"] == {5} and neu["luecken"] == ["x"]
|
||||
assert blocks._konsolidierung_schema({"gruppen": [{"haupt": 9, "weitere": [1]}]}, 5) == \
|
||||
{"gruppen": [], "kataloge": [], "fremd": set(), "luecken": []} # id out of range
|
||||
|
||||
|
||||
async def test_haupt_beats_heuristic(testdb, tmp_path, monkeypatch):
|
||||
"""Judges nennen den kürzeren Eintrag als haupt → er gewinnt trotz weniger key_points."""
|
||||
db = testdb
|
||||
subs = ["Basis", "Detailregel mit sehr langem Titel und Facts"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
facts = {"Block": {blocks._norm_title(subs[1]): {"key_points": ["a", "b", "c"]}}}
|
||||
fake = _fake_slot({"j1": {"gruppen": [{"haupt": 1, "weitere": [2]}], "luecken": []},
|
||||
"j2": {"gruppen": [{"haupt": 1, "weitere": [2]}], "luecken": []}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, facts)
|
||||
assert raw["Block"] == ["Basis"]
|
||||
assert facts["Block"][blocks._norm_title("Basis")]["key_points"] == ["a", "b", "c"] # Union geerbt
|
||||
|
||||
|
||||
async def test_katalog_bundles_to_new_row(testdb, tmp_path, monkeypatch):
|
||||
"""Einstimmige Katalog-Mitglieder → neue consensus-Zeile mit Facts-Union, Mitglieder variant."""
|
||||
db = testdb
|
||||
subs = ["Pfeilsymbole: a b c", "Mengensymbole: d e f", "Eigene Regel"]
|
||||
await _seed_block(db, "mathe", subs)
|
||||
raw = {"Mathe": list(subs)}
|
||||
facts = {"Mathe": {blocks._norm_title(subs[0]): {"key_points": ["kp1"]},
|
||||
blocks._norm_title(subs[1]): {"key_points": ["kp2"]}}}
|
||||
kat = {"titel": "Symbolkatalog: Pfeile und Mengen", "mitglieder": [1, 2]}
|
||||
fake = _fake_slot({"j1": {"gruppen": [], "kataloge": [kat], "luecken": []},
|
||||
"j2": {"gruppen": [], "kataloge": [kat], "luecken": []}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, facts)
|
||||
assert raw["Mathe"] == ["Eigene Regel", "Symbolkatalog: Pfeile und Mengen"]
|
||||
kn = blocks._norm_title("Symbolkatalog: Pfeile und Mengen")
|
||||
assert sorted(facts["Mathe"][kn]["key_points"]) == ["kp1", "kp2"]
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "mathe")}
|
||||
assert rows[kn] == "consensus"
|
||||
assert rows[blocks._norm_title(subs[0])] == "variant"
|
||||
|
||||
|
||||
async def test_katalog_dissent_keeps_members(testdb, tmp_path, monkeypatch):
|
||||
db = testdb
|
||||
subs = ["Pfeilsymbole: a b c", "Mengensymbole: d e f"]
|
||||
await _seed_block(db, "mathe", subs)
|
||||
raw = {"Mathe": list(subs)}
|
||||
fake = _fake_slot({"j1": {"gruppen": [], "kataloge": [{"titel": "K", "mitglieder": [1, 2]}], "luecken": []},
|
||||
"j2": {"gruppen": [], "kataloge": [], "luecken": []}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, {})
|
||||
assert raw["Mathe"] == subs
|
||||
|
||||
|
||||
async def test_fremd_unanimous_discards(testdb, tmp_path, monkeypatch):
|
||||
"""Einstimmig fremd → discarded + raus; einseitig fremd → bleibt."""
|
||||
db = testdb
|
||||
subs = ["CSS display überschreibt Verhalten", "Echte Markdown-Regel", "Nur einer hält es für fremd"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
fake = _fake_slot({"j1": {"gruppen": [], "fremd": [1, 3], "luecken": []},
|
||||
"j2": {"gruppen": [], "fremd": [1], "luecken": []}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
luecken = await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, {})
|
||||
assert raw["Block"] == [subs[1], subs[2]]
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "block")}
|
||||
assert rows[blocks._norm_title(subs[0])] == "discarded"
|
||||
assert rows[blocks._norm_title(subs[2])] == "consensus"
|
||||
assert luecken == {}
|
||||
|
||||
|
||||
async def test_luecken_nur_bei_einstimmigkeit(testdb, tmp_path, monkeypatch):
|
||||
"""Nur Lücken mit Token-Überlappung BEIDER Judges überleben; einseitige fallen weg."""
|
||||
db = testdb
|
||||
subs = ["Eintrag eins", "Eintrag zwei"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
fake = _fake_slot({"j1": {"gruppen": [], "luecken": ["Inline-HTML fehlt", "Front-Matter"]},
|
||||
"j2": {"gruppen": [], "luecken": ["nichts zu Inline-HTML"]}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
luecken = await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, {})
|
||||
assert luecken == {"Block": ["Inline-HTML fehlt"]}
|
||||
|
||||
|
||||
async def test_kp_deckel_im_judge_prompt(testdb, tmp_path, monkeypatch):
|
||||
"""Prompt zeigt max. 3 key_points je Sub (Timeout-Schutz); die Union bleibt voll."""
|
||||
db = testdb
|
||||
subs = ["Eintrag eins", "Eintrag zwei"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
facts = {"Block": {blocks._norm_title(subs[0]): {"key_points": [f"kp{i}" for i in range(1, 6)]}}}
|
||||
fake = _fake_slot({"j1": {"gruppen": [], "luecken": []}, "j2": {"gruppen": [], "luecken": []}})
|
||||
monkeypatch.setattr(blocks, "run_single_slot", fake)
|
||||
await blocks._konsolidiere_subblocks(_ctx(), {"arbeit": tmp_path}, raw, facts)
|
||||
prompt = fake.calls[0]["prompt"]
|
||||
assert "kp3" in prompt and "kp4" not in prompt
|
||||
|
||||
|
||||
def test_luecken_schnitt_cap():
|
||||
l1 = [f"Aspekt-{k} fehlt" for k in ("eins", "zwei", "drei", "vier", "fünf")]
|
||||
assert blocks._luecken_schnitt(l1, list(l1)) == l1[:3] # Cap 3
|
||||
assert blocks._luecken_schnitt(["Inline-HTML"], ["Tabellen-Syntax"]) == []
|
||||
|
||||
|
||||
def test_neg_set_lemmatisiert():
|
||||
"""kein/keine/keinen falten auf einen Stamm; nicht vs. ohne bleiben verschieden."""
|
||||
a = blocks._neg_set("Fehlerverhalten (kein Syntaxfehler)")
|
||||
b = blocks._neg_set("Fehlerverhalten (keine Syntax-Fehlermeldung)")
|
||||
assert a == b == frozenset({"kein"})
|
||||
assert blocks._neg_set("nicht expandiert") != blocks._neg_set("ohne Expansion")
|
||||
assert blocks._neg_set("niemals gerendert") == blocks._neg_set("nie gerendert")
|
||||
|
||||
|
||||
# ── Lücken-Nachfass ─────────────────────────────────────────────────────────────────
|
||||
|
||||
async def _nachfass_env(db, monkeypatch, facts_result):
|
||||
subs = ["Eintrag eins"]
|
||||
await _seed_block(db, "block", subs)
|
||||
raw = {"Block": list(subs)}
|
||||
facts_map = {"Block": {}}
|
||||
|
||||
async def fake_race(topic, label, slots, quorum, timeout, provider, cancelled=None, grace=0):
|
||||
return [{"Block": ["Eintrag eins", "Neuer Aspekt"]}]
|
||||
|
||||
async def fake_facts(ctx, set_p, files, fraw, q, folder, instructions, ns="", lbl="", sources=None, slim=False):
|
||||
assert slim is True # Nachfass nutzt die schlanke Facts-Variante
|
||||
assert list(fraw["Block"]) == ["Neuer Aspekt"] # nur der frische Fund geht ins Gate
|
||||
return facts_result
|
||||
|
||||
monkeypatch.setattr(blocks, "_race", fake_race)
|
||||
monkeypatch.setattr(blocks, "_facts_block", fake_facts)
|
||||
monkeypatch.setattr(blocks, "EMBEDDING_AKTIV", False)
|
||||
return raw, facts_map
|
||||
|
||||
|
||||
async def test_nachfass_adopts_backed_find(testdb, tmp_path, monkeypatch):
|
||||
db = testdb
|
||||
nn = blocks._norm_title("Neuer Aspekt")
|
||||
raw, facts_map = await _nachfass_env(db, monkeypatch,
|
||||
({"Block": {nn: {"key_points": ["kp"]}}}, {}))
|
||||
n = await blocks._luecken_runde(_ctx(), {"arbeit": tmp_path}, "Block", ["Aspekt"],
|
||||
raw, facts_map, {"type": "thema"}, None)
|
||||
assert n == 1 and raw["Block"] == ["Eintrag eins", "Neuer Aspekt"]
|
||||
assert facts_map["Block"][nn]["key_points"] == ["kp"]
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "block")}
|
||||
assert rows[nn] == "consensus"
|
||||
|
||||
|
||||
async def test_nachfass_drops_unbacked_find(testdb, tmp_path, monkeypatch):
|
||||
db = testdb
|
||||
nn = blocks._norm_title("Neuer Aspekt")
|
||||
raw, facts_map = await _nachfass_env(db, monkeypatch, ({}, {"Block": {nn}}))
|
||||
n = await blocks._luecken_runde(_ctx(), {"arbeit": tmp_path}, "Block", ["Aspekt"],
|
||||
raw, facts_map, {"type": "thema"}, None)
|
||||
assert n == 0 and raw["Block"] == ["Eintrag eins"]
|
||||
assert not any(r["sub_norm"] == nn for r in await db.list_subblocks(TOPIC, "block"))
|
||||
|
||||
|
||||
async def test_nachfass_drops_find_without_facts(testdb, tmp_path, monkeypatch):
|
||||
"""HARTES Gate: kein Facts-Eintrag = kein Beleg = keine Übernahme — nicht nur
|
||||
aktiv Verworfenes fliegt (Bilder-Lauf: 13 von 18 kamen ohne Beleg durch)."""
|
||||
db = testdb
|
||||
raw, facts_map = await _nachfass_env(db, monkeypatch, ({}, {})) # Facts fand NICHTS
|
||||
n = await blocks._luecken_runde(_ctx(), {"arbeit": tmp_path}, "Block", ["Aspekt"],
|
||||
raw, facts_map, {"type": "thema"}, None)
|
||||
assert n == 0 and raw["Block"] == ["Eintrag eins"]
|
||||
|
||||
|
||||
async def test_facts_stage_konsolidiert_nachfass_funde_erneut(testdb, tmp_path, monkeypatch):
|
||||
"""Kreis geschlossen: nach Übernahmen läuft die Konsolidierung ein zweites Mal;
|
||||
deren Lücken lösen KEINEN weiteren Nachfass aus."""
|
||||
db = testdb
|
||||
payload = {"title": "Alpha", "raw": {"Alpha": ["s1"]}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "facts", payload)
|
||||
calls = {"kons": 0, "nf": 0}
|
||||
|
||||
async def fake_facts(ctx, set_p, files, raw, q, folder, instructions, ns="", lbl="", sources=None, slim=False):
|
||||
return {"Alpha": {}}, {}
|
||||
|
||||
async def fake_kons(ctx, files, raw, facts_map, instructions="", ns="", lbl=""):
|
||||
calls["kons"] += 1
|
||||
return {"Alpha": ["Lücke X"]} # meldet auch in Runde 2 — darf nicht erneut nachfassen
|
||||
|
||||
async def fake_nf(ctx, files, title, luecken, raw, facts_map, q, folder,
|
||||
instructions="", ns="", lbl="", sources=None):
|
||||
calls["nf"] += 1
|
||||
return 2
|
||||
|
||||
monkeypatch.setattr(ba, "_facts_block", fake_facts)
|
||||
monkeypatch.setattr(ba, "_konsolidiere_subblocks", fake_kons)
|
||||
monkeypatch.setattr(ba, "_luecken_runde", fake_nf)
|
||||
flow = Flow(TOPIC, work_dir=tmp_path)
|
||||
await ba._proc_facts(_ctx(), flow, {"arbeit": tmp_path}, {"type": "thema"}, None, "",
|
||||
[{"card_id": "alpha", "payload": payload}])
|
||||
assert calls == {"kons": 2, "nf": 1}
|
||||
assert (await db.kanban_get_card(TOPIC, "artefacts", "alpha"))["stage"] == "levels"
|
||||
|
||||
|
||||
async def test_finalize_purges_stale_rows(testdb, tmp_path):
|
||||
"""Re-Run-Waisen: Finalize löscht Alt-Fragen/-Artefakte des Blocks vor dem Upsert."""
|
||||
db = testdb
|
||||
await db.upsert_question_pattern(TOPIC, "alpha", "alt-sub", "Alpha", "Alt", "Alte Frage?")
|
||||
await db.put_sub_artifact(TOPIC, "alpha", "alt-sub", "flashcard", "{}", "Alpha", "Alt")
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "finalize", {})
|
||||
files = {k: tmp_path / f"{k}.json" for k in
|
||||
("sub_roh", "facts", "sidecar", "question_pattern", "artefakte")}
|
||||
card = {"card_id": "alpha", "payload": {
|
||||
"title": "Alpha", "raw": {"Alpha": ["Neu"]}, "facts": {},
|
||||
"sidecar": {"Alpha": [{"title": "Neu", "level": "beginner"}]},
|
||||
"pattern": {"Alpha": [{"subblock": "Neu", "question": "F?"}]},
|
||||
"artefacts": {"flashcard": [{"block": "Alpha", "subblock": "Neu", "front": "F", "back": "B"}]}}}
|
||||
flow = Flow(TOPIC, work_dir=tmp_path)
|
||||
await ba._proc_finalize(_ctx(), flow, files, [card])
|
||||
assert {r["sub_norm"] for r in await db.list_question_pattern(TOPIC)} == {"neu"}
|
||||
assert {(r["sub_norm"], r["type"]) for r in await db.get_sub_artefakte(TOPIC)} == {("neu", "flashcard")}
|
||||
|
||||
|
||||
# ── Cross-Block ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
class _FakeEmb:
|
||||
"""Gleicher Text → gleicher Einheitsvektor, sonst orthogonal (cos 1.0 / 0.0)."""
|
||||
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def embed_sims(texts):
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr @ arr.T
|
||||
|
||||
|
||||
async def _cross_env(db, tmp_path):
|
||||
flow = Flow(TOPIC, work_dir=tmp_path)
|
||||
cards = []
|
||||
for bnorm, subs in (("alpha", ["Gleiche Aussage", "Nur in Alpha"]),
|
||||
("beta", ["Gleiche Aussage", "Nur in Beta"])):
|
||||
payload = {"title": bnorm.title(),
|
||||
"raw": {bnorm.title(): list(subs)},
|
||||
"sidecar": {bnorm.title(): [{"title": s, "level": "beginner"} for s in subs]},
|
||||
"facts": {bnorm.title(): {blocks._norm_title(s): {"key_points": [f"kp {s}"]} for s in subs}}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", bnorm, "ablock", "konsolidierung", payload)
|
||||
await _seed_block(db, bnorm, subs)
|
||||
cards.append({"card_id": bnorm, "payload": payload})
|
||||
return flow, cards
|
||||
|
||||
|
||||
async def test_crossblock_folds_loser(testdb, tmp_path, monkeypatch):
|
||||
"""Einstimmig „a" → Beta verliert die geteilte Aussage, Karten wandern zu levels."""
|
||||
db = testdb
|
||||
flow, cards = await _cross_env(db, tmp_path)
|
||||
monkeypatch.setattr(ba, "embedding", _FakeEmb)
|
||||
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "j2": {"pairs": {"1": "a"}}})
|
||||
monkeypatch.setattr(ba, "run_single_slot", fake)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
assert "Gleiche Aussage" in fake.calls[0]["prompt"]
|
||||
beta = await db.kanban_get_card(TOPIC, "artefacts", "beta")
|
||||
assert beta["stage"] == "question_pattern"
|
||||
assert beta["payload"]["raw"]["Beta"] == ["Nur in Beta"]
|
||||
assert blocks._norm_title("Gleiche Aussage") not in beta["payload"]["facts"]["Beta"]
|
||||
# Barriere liegt jetzt hinter levels/relevance → auch die sidecar muss den Fold tragen
|
||||
assert [e["title"] for e in beta["payload"]["sidecar"]["Beta"]] == ["Nur in Beta"]
|
||||
alpha = await db.kanban_get_card(TOPIC, "artefacts", "alpha")
|
||||
assert alpha["stage"] == "question_pattern"
|
||||
assert alpha["payload"]["raw"]["Alpha"] == ["Gleiche Aussage", "Nur in Alpha"]
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "beta")}
|
||||
assert rows[blocks._norm_title("Gleiche Aussage")] == "variant"
|
||||
|
||||
|
||||
async def test_crossblock_tiebreaker_folds(testdb, tmp_path, monkeypatch):
|
||||
"""j1/j2 uneinig → j3 entscheidet mit Mehrheit; hier „a" → Beta verliert."""
|
||||
db = testdb
|
||||
flow, cards = await _cross_env(db, tmp_path)
|
||||
monkeypatch.setattr(ba, "embedding", _FakeEmb)
|
||||
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "j2": {"pairs": {"1": "nein"}},
|
||||
"j3": {"pairs": {"1": "a"}}})
|
||||
monkeypatch.setattr(ba, "run_single_slot", fake)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
assert len(fake.calls) == 3
|
||||
beta = await db.kanban_get_card(TOPIC, "artefacts", "beta")
|
||||
assert beta["payload"]["raw"]["Beta"] == ["Nur in Beta"]
|
||||
|
||||
|
||||
async def test_crossblock_dissent_without_tiebreaker_keeps_both(testdb, tmp_path, monkeypatch):
|
||||
"""j3 liefert nichts (FAILED) → fail-open, Paar bleibt."""
|
||||
db = testdb
|
||||
flow, cards = await _cross_env(db, tmp_path)
|
||||
monkeypatch.setattr(ba, "embedding", _FakeEmb)
|
||||
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "j2": {"pairs": {"1": "b"}}}) # j3 fehlt → FAILED
|
||||
monkeypatch.setattr(ba, "run_single_slot", fake)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
beta = await db.kanban_get_card(TOPIC, "artefacts", "beta")
|
||||
assert beta["stage"] == "question_pattern"
|
||||
assert beta["payload"]["raw"]["Beta"] == ["Gleiche Aussage", "Nur in Beta"]
|
||||
|
||||
|
||||
async def test_crossblock_ersatzrichter(testdb, tmp_path, monkeypatch):
|
||||
"""Nur ein Richter liefert → Ersatz jE als zweite Stimme; Einstimmigkeit faltet."""
|
||||
db = testdb
|
||||
flow, cards = await _cross_env(db, tmp_path)
|
||||
monkeypatch.setattr(ba, "embedding", _FakeEmb)
|
||||
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "jE": {"pairs": {"1": "a"}}}) # j2 → FAILED
|
||||
monkeypatch.setattr(ba, "run_single_slot", fake)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
beta = await db.kanban_get_card(TOPIC, "artefacts", "beta")
|
||||
assert beta["payload"]["raw"]["Beta"] == ["Nur in Beta"]
|
||||
|
||||
|
||||
async def test_crossblock_without_embedding_advances(testdb, tmp_path, monkeypatch):
|
||||
db = testdb
|
||||
flow, cards = await _cross_env(db, tmp_path)
|
||||
|
||||
class _Aus:
|
||||
@staticmethod
|
||||
def available():
|
||||
return False
|
||||
|
||||
async def kein_agent(*a, **kw):
|
||||
raise AssertionError("ohne Embedding kein Judge")
|
||||
|
||||
monkeypatch.setattr(ba, "embedding", _Aus)
|
||||
monkeypatch.setattr(ba, "run_single_slot", kein_agent)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
for cid in ("alpha", "beta"):
|
||||
assert (await db.kanban_get_card(TOPIC, "artefacts", cid))["stage"] == "question_pattern"
|
||||
|
||||
|
||||
async def test_crossblock_context_wins(testdb, tmp_path, monkeypatch):
|
||||
"""Kontext-Sub (Block schon hinter der Barrier) gewinnt auch bei Verdict „b" —
|
||||
die Paket-Seite fällt, der Kontext bleibt unangetastet."""
|
||||
db = testdb
|
||||
flow = Flow(TOPIC, work_dir=tmp_path)
|
||||
payload = {"title": "Alpha", "raw": {"Alpha": ["Gleiche Aussage"]}, "facts": {"Alpha": {}}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "konsolidierung", payload)
|
||||
await _seed_block(db, "alpha", ["Gleiche Aussage"])
|
||||
cards = [{"card_id": "alpha", "payload": payload}]
|
||||
# Kontext-Block "gamma" ist bereits weiter (Stage levels) und hält dieselbe Aussage
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "gamma", "ablock", "levels",
|
||||
{"title": "Gamma", "raw": {"Gamma": ["Gleiche Aussage"]}, "facts": {}})
|
||||
await _seed_block(db, "gamma", ["Gleiche Aussage"])
|
||||
monkeypatch.setattr(ba, "embedding", _FakeEmb)
|
||||
# Verdict „a": das Paket (A) soll behalten — Kontext faltet trotzdem nie
|
||||
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "j2": {"pairs": {"1": "a"}}})
|
||||
monkeypatch.setattr(ba, "run_single_slot", fake)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
alpha = await db.kanban_get_card(TOPIC, "artefacts", "alpha")
|
||||
assert alpha["payload"]["raw"].get("Alpha", []) == [] # Paket-Seite gefaltet
|
||||
gamma_rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "gamma")}
|
||||
assert gamma_rows[blocks._norm_title("Gleiche Aussage")] == "consensus" # Kontext unberührt
|
||||
216
backend/tests/test_qa.py
Normal file
216
backend/tests/test_qa.py
Normal file
@@ -0,0 +1,216 @@
|
||||
"""QA-Detektoren: Fehler-Injektion auf Mini-Korpus — deterministisch, ohne LLM/Embedding."""
|
||||
|
||||
import qa
|
||||
|
||||
|
||||
CORPUS = {"Skript.txt": (
|
||||
"Kapitel 1: Vertex Cover — Definition, Approximation und Beweis der Guete.\n\n"
|
||||
"Kapitel 2: Matching in Graphen — perfektes Matching und Augmentationswege.")}
|
||||
BLOCKS = [
|
||||
{"title": "Vertex Cover", "description": "Knotenüberdeckung", "sources": ["Skript.txt"]},
|
||||
{"title": "Matching", "description": "Paarung in Graphen", "sources": ["Skript.txt"]},
|
||||
]
|
||||
SUBS = {"vertex cover": ["Approximation der Guete"], "matching": ["Matching in Graphen", "Augmentationswege"]}
|
||||
|
||||
|
||||
def test_baseline_clean(monkeypatch):
|
||||
"""Sauberes Soll-Inventar → alle Detektoren still (jeder Absatz ein Abschnitt)."""
|
||||
monkeypatch.setattr(qa, "SECTION_CHARS", 20)
|
||||
assert qa.dubletten(BLOCKS, emb_on=False) == []
|
||||
assert qa.luecken(BLOCKS, SUBS, CORPUS) == []
|
||||
assert qa.fremd(BLOCKS, CORPUS) == []
|
||||
assert qa.hygiene(BLOCKS) == []
|
||||
|
||||
|
||||
def test_injected_duplicate_found():
|
||||
b = BLOCKS + [{"title": "Vertex-Cover-Problem", "description": "", "sources": []}]
|
||||
pairs = qa.dubletten(b, emb_on=False)
|
||||
assert any({p["a"], p["b"]} == {"Vertex Cover", "Vertex-Cover-Problem"} for p in pairs)
|
||||
|
||||
|
||||
def test_acronym_signal():
|
||||
b = BLOCKS + [{"title": "VC (Vertex Cover)", "description": "", "sources": []}]
|
||||
pairs = qa.dubletten(b, emb_on=False)
|
||||
hit = next(p for p in pairs if "VC (Vertex Cover)" in (p["a"], p["b"]) and "Vertex Cover" in (p["a"], p["b"]))
|
||||
assert hit["signale"].get("akronym") is True
|
||||
|
||||
|
||||
def test_relation_operand_not_suspicious():
|
||||
"""Relation vs. Operand ist per Design getrennt — kein Verdachtspaar."""
|
||||
b = BLOCKS + [{"title": "3-SAT ≤ Vertex Cover", "description": "", "sources": []}]
|
||||
pairs = qa.dubletten(b, emb_on=False)
|
||||
assert not any("≤" in p["a"] + p["b"] for p in pairs)
|
||||
|
||||
|
||||
def test_removed_block_creates_gap(monkeypatch):
|
||||
monkeypatch.setattr(qa, "SECTION_CHARS", 20)
|
||||
only_vc = [BLOCKS[0]]
|
||||
gaps = qa.luecken(only_vc, {"vertex cover": SUBS["vertex cover"]}, CORPUS)
|
||||
assert len(gaps) == 1 and "Matching" in gaps[0]["vorschau"]
|
||||
|
||||
|
||||
def test_foreign_block_flagged():
|
||||
b = BLOCKS + [{"title": "Quantencomputer Grundlagen", "description": "", "sources": []}]
|
||||
assert qa.fremd(b, CORPUS) == ["Quantencomputer Grundlagen"]
|
||||
|
||||
|
||||
def test_beleg_flags_unbacked_sub():
|
||||
rows = [{"block": "Matching", "sub_title": "Erfunden", "mentions": 0, "status": "consensus"},
|
||||
{"block": "Matching", "sub_title": "Belegt", "mentions": 3, "status": "consensus"}]
|
||||
r = qa.beleg([{"title": "Matching", "sources": []}], rows)
|
||||
assert r["subs_ohne_beleg"] == ["Matching · Erfunden"]
|
||||
assert r["bloecke_ohne_quelle"] == ["Matching"]
|
||||
|
||||
|
||||
def test_hygiene_flags():
|
||||
b = [{"title": "**Fett**", "description": "", "sources": []},
|
||||
{"title": "Block (2)", "description": "ok", "sources": []}]
|
||||
h = {x["titel"]: x["probleme"] for x in qa.hygiene(b)}
|
||||
assert "markdown" in h["**Fett**"] and "leere-beschreibung" in h["**Fett**"]
|
||||
assert h["Block (2)"] == ["kollisions-suffix"]
|
||||
|
||||
|
||||
def test_sections_split_on_paragraphs():
|
||||
secs = qa._sections("a\n\nb\n\nc", goal=3)
|
||||
assert len(secs) >= 2 and "".join(secs).replace("\n", "") == "abc"
|
||||
|
||||
|
||||
def test_note_deterministic_and_monotonic():
|
||||
"""Saubere Quoten → 10; jede zusätzliche Quote drückt die Note."""
|
||||
sauber = {k: 0 for k in qa.NOTE_GEWICHTE}
|
||||
assert qa.note(sauber) == 10.0
|
||||
schlechter = dict(sauber, luecken=0.05)
|
||||
noch_schlechter = dict(schlechter, fremd=0.05)
|
||||
assert 10.0 > qa.note(schlechter) > qa.note(noch_schlechter) >= 0.0
|
||||
assert qa.note({k: 1 for k in qa.NOTE_GEWICHTE}) == 0.0
|
||||
|
||||
|
||||
def test_note_kalibrierung():
|
||||
"""Gewicht = Punktabzug bei 100 %: 5 % Fremd × 2.5 → −1.25 → 8.8 gerundet."""
|
||||
assert qa.note({"fremd": 0.05}) == 8.8
|
||||
assert qa.note({"fremd": 1.0}) == 0.0 # komplett fremdes Inventar = 0, nicht 7.7
|
||||
|
||||
|
||||
def test_note_verdacht_zaehlt_nicht():
|
||||
"""dubletten_verdacht ist Verdachtsliste, kein Urteil — beeinflusst die Note nicht."""
|
||||
assert qa.note({"dubletten_verdacht": 1.0}) == 10.0
|
||||
|
||||
|
||||
def test_note_artefakte_getrennt():
|
||||
"""Subs/Artefakte haben eigene Gewichte — zur Gate-Zeit existieren sie noch nicht
|
||||
und dürfen die Inventar-Note weder schönen noch drücken."""
|
||||
assert "subs_ohne_beleg" not in qa.NOTE_GEWICHTE
|
||||
assert qa.note({"subs_ohne_beleg": 0.0, "verwaiste": 0.1}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 9.0
|
||||
assert qa.note({"subs_ohne_beleg": 1.0}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 0.0
|
||||
|
||||
|
||||
def test_artefakte_coverage_and_orphans():
|
||||
subs = [{"block_norm": "b", "sub_norm": "s1: lange beschreibung", "status": "consensus"},
|
||||
{"block_norm": "b", "sub_norm": "s2", "status": "consensus"},
|
||||
{"block_norm": "b", "sub_norm": "alt", "status": "variant"},
|
||||
{"block_norm": "b", "sub_norm": "weg", "status": "discarded"}]
|
||||
arts = [{"block_norm": "b", "sub_norm": "s1", "type": "flashcard"}, # Präfix-Treffer
|
||||
{"block_norm": "b", "sub_norm": "alt", "type": "flashcard"}, # variant → lebt, keine Waise
|
||||
{"block_norm": "b", "sub_norm": "weg", "type": "flashcard"}, # verworfen → Waise
|
||||
{"block_norm": "b", "sub_norm": "tot", "type": "flashcard"}] # fehlt → Waise
|
||||
fragen = [{"block_norm": "b", "sub_norm": "s1: lange beschreibung"},
|
||||
{"block_norm": "b", "sub_norm": "s2"}]
|
||||
r = qa.artefakte(subs, arts, fragen)
|
||||
assert r["frage_abdeckung"] == 1.0
|
||||
assert r["flashcard_abdeckung"] == 0.5 # nur s1 der beiden consensus-Subs
|
||||
assert r["verwaiste"] == ["flashcard: b · tot", "flashcard: b · weg"]
|
||||
|
||||
|
||||
def test_artefakte_prefix_family_resolves_to_consensus():
|
||||
"""Kurz-Key trifft consensus-Sub PLUS gefaltete Varianten mit gleichem Präfix —
|
||||
das ist keine Waise, das Ziel ist der consensus-Sub."""
|
||||
subs = [{"block_norm": "b", "sub_norm": "auto: echte fassung", "status": "consensus"},
|
||||
{"block_norm": "b", "sub_norm": "auto: variante eins", "status": "variant"},
|
||||
{"block_norm": "b", "sub_norm": "auto: variante zwei", "status": "variant"}]
|
||||
arts = [{"block_norm": "b", "sub_norm": "auto", "type": "example"}]
|
||||
r = qa.artefakte(subs, arts, [])
|
||||
assert r["verwaiste"] == []
|
||||
assert r["beispiel_abdeckung"] == 1.0
|
||||
|
||||
|
||||
def test_artefakte_not_generated():
|
||||
assert qa.artefakte([{"block_norm": "b", "sub_norm": "s", "status": "consensus"}], [], []) == {"status": "nicht generiert"}
|
||||
|
||||
|
||||
def test_fremd_glued_prefix_not_whitewashed():
|
||||
"""'αÜbergang' darf nicht über den Substring 'bergang'⊂'Übergang' als belegt gelten;
|
||||
Symbol-Varianten (Δ/∆) bleiben über die ASCII-Form gedeckt."""
|
||||
corpus = {"S.txt": "Der Übergang ist wichtig.\n\nDer ∆TSP1 Algorithmus folgt."}
|
||||
b = [{"title": "αÜbergang", "description": "", "sources": []},
|
||||
{"title": "ΔTSP1-Algorithmus", "description": "", "sources": []}]
|
||||
assert qa.fremd(b, corpus) == ["αÜbergang"]
|
||||
|
||||
|
||||
def test_note_ignores_unmeasured_quotes():
|
||||
"""unechte_bloecke zählt nur, wenn gemessen (--llm) — sonst weder Schaden noch Schönung."""
|
||||
ohne = {k: 0.02 for k in qa.NOTE_GEWICHTE if k != "unechte_bloecke"}
|
||||
mit_null = dict(ohne, unechte_bloecke=0.0)
|
||||
mit_schaden = dict(ohne, unechte_bloecke=0.5)
|
||||
assert qa.note(mit_null) == qa.note(ohne)
|
||||
assert qa.note(mit_schaden) < qa.note(ohne)
|
||||
|
||||
|
||||
class _FakeEmb:
|
||||
"""Gleicher Text → gleicher Einheitsvektor, sonst orthogonal (cos 1.0 / 0.0)."""
|
||||
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def embed(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr
|
||||
|
||||
|
||||
def test_sub_dubletten_detector(monkeypatch):
|
||||
"""Kandidaten in-block UND cross-block; nur consensus-Subs zählen."""
|
||||
monkeypatch.setattr(qa, "embedding", _FakeEmb)
|
||||
rows = [{"block": "Alpha", "block_norm": "alpha", "sub_title": "Gleiche Aussage", "status": "consensus"},
|
||||
{"block": "Beta", "block_norm": "beta", "sub_title": "Gleiche Aussage", "status": "consensus"},
|
||||
{"block": "Beta", "block_norm": "beta", "sub_title": "Andere Aussage", "status": "consensus"},
|
||||
{"block": "Beta", "block_norm": "beta", "sub_title": "Gleiche Aussage", "status": "variant"}]
|
||||
pairs = qa.sub_dubletten(rows)
|
||||
assert len(pairs) == 1
|
||||
assert pairs[0]["cross"] is True and pairs[0]["cos"] == 1.0
|
||||
assert qa.sub_dubletten(rows, emb_on=False) == []
|
||||
|
||||
|
||||
def test_note_sub_dubletten():
|
||||
"""Bestätigte Sub-Dubletten drücken die Artefakt-Note; der bloße Verdacht nicht."""
|
||||
assert qa.note({"sub_dubletten": 0.1}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 9.0
|
||||
assert qa.note({"sub_dubletten_verdacht": 1.0}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 10.0
|
||||
|
||||
|
||||
def test_zaehlbare_luecken():
|
||||
"""Mit LLM zählen widerlegte Lücken nicht; unbeurteilte ('?'/ohne Key) konservativ schon."""
|
||||
lk = [{"llm": "ja"}, {"llm": "nein"}, {"llm": "?"}, {}]
|
||||
assert len(qa._zaehlbare_luecken(lk, llm=True)) == 3
|
||||
assert len(qa._zaehlbare_luecken(lk, llm=False)) == 4
|
||||
|
||||
|
||||
def test_description_anchors_cover(monkeypatch):
|
||||
"""Am Gate existieren keine Subs — Beschreibungs-Tokens müssen Abschnitte decken."""
|
||||
monkeypatch.setattr(qa, "SECTION_CHARS", 20)
|
||||
corpus = {"S.txt": "Kapitel 9: Augmentationswege und perfektes Matching."}
|
||||
block = [{"title": "Paarungen", "description": "perfektes Matching mit Augmentationswege", "sources": []}]
|
||||
assert qa.luecken(block, {}, corpus) == []
|
||||
ohne = [{"title": "Paarungen", "description": "", "sources": []}]
|
||||
assert len(qa.luecken(ohne, {}, corpus)) == 1
|
||||
|
||||
|
||||
def test_fremd_digit_suffix_tolerant():
|
||||
"""'ΔTSP1' matcht Korpus-'∆TSP' (tokenisiert zu 'tsp') via Ziffern-Suffix-Fallback."""
|
||||
corpus = {"S.txt": "Der ∆TSP Algorithmus verdoppelt Kanten im Graphen."}
|
||||
b = [{"title": "ΔTSP1-Algorithmus", "description": "", "sources": []},
|
||||
{"title": "Quantencomputer", "description": "", "sources": []}]
|
||||
assert qa.fremd(b, corpus) == ["Quantencomputer"]
|
||||
230
backend/tests/test_repair.py
Normal file
230
backend/tests/test_repair.py
Normal file
@@ -0,0 +1,230 @@
|
||||
"""Befund-Repair: gezielte Aktionen aus dem QA-Report (repair.py) — ohne Flow, gegen Test-DB."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import repair
|
||||
import qa as qa_mod
|
||||
|
||||
TOPIC = "reparatur"
|
||||
|
||||
|
||||
def _report(**over):
|
||||
r = {"topic": TOPIC, "note": 9.0, "quoten": {}, "hygiene": [], "dubletten": [],
|
||||
"fremd": [], "unecht": [], "luecken": [], "artefakte": {"verwaiste": []}}
|
||||
r.update(over)
|
||||
return r
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def env(testdb, tmp_path, monkeypatch):
|
||||
"""Zwei fertige Blöcke auf beiden Boards + Subs/Artefakte + Sidecar-Dateien."""
|
||||
db = testdb
|
||||
monkeypatch.setattr(qa_mod, "QA_DIR", tmp_path / "qa")
|
||||
files = {"sidecar": tmp_path / "sidecar.json", "facts": tmp_path / "facts.json",
|
||||
"question_pattern": tmp_path / "qp.json", "sub_roh": tmp_path / "roh.json",
|
||||
"artefakte": tmp_path / "artefakte.json"}
|
||||
monkeypatch.setattr(repair, "_blocks_files", lambda t: files)
|
||||
# frisches Abschluss-QA im Repair stumm schalten (eigener Test deckt qa_report ab)
|
||||
async def _no_qa(topic, llm=False):
|
||||
return None
|
||||
monkeypatch.setattr(qa_mod, "qa_report", _no_qa)
|
||||
|
||||
async def _seed(title, desc, subs=1):
|
||||
norm = repair._norm_title(title)
|
||||
cid = "b-" + norm.replace(" ", "")[:10]
|
||||
await db.kanban_upsert_card(TOPIC, "inventory", cid, "block", "done_block",
|
||||
{"title": title, "description": desc, "sources": [f"{title}.txt"],
|
||||
"readers": ["r1"], "mirrored_norm": norm})
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", norm, "ablock", "done_artefact", {"title": title})
|
||||
await db.upsert_block(TOPIC, norm, title, desc, [f"{title}.txt"])
|
||||
await db.set_block_status(TOPIC, norm, "consensus")
|
||||
for i in range(subs):
|
||||
await db.put_subblock(TOPIC, norm, f"sub{i}", title, f"Sub {i}")
|
||||
await db.put_sub_artifact(TOPIC, norm, "sub0", "flashcard", "{}", title, "Sub 0")
|
||||
return cid
|
||||
|
||||
for p in files.values():
|
||||
p.write_text("{}", encoding="utf-8")
|
||||
(tmp_path / "qa" / TOPIC).mkdir(parents=True)
|
||||
|
||||
def write_report(r):
|
||||
(tmp_path / "qa" / TOPIC / "r.json").write_text(json.dumps(r), encoding="utf-8")
|
||||
|
||||
return db, _seed, files, write_report
|
||||
|
||||
|
||||
async def test_merge_confirmed_duplicate(env, monkeypatch):
|
||||
db, seed, files, write_report = env
|
||||
cid_a = await seed("Alpha", "kurz")
|
||||
cid_b = await seed("Alpha Problem", "deutlich längere Beschreibung — Gewinner")
|
||||
write_report(_report(dubletten=[{"a": "Alpha", "b": "Alpha Problem", "llm": "ja"},
|
||||
{"a": "Alpha", "b": "Beta", "llm": "nein"}]))
|
||||
calls = []
|
||||
|
||||
async def fake_agent(key, prompt, timeout, **kw):
|
||||
calls.append(prompt)
|
||||
return 0, '{"relevant": {"1": "ja"}}', ""
|
||||
|
||||
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
||||
res = await repair.repair_befunde(TOPIC)
|
||||
assert res["merges"] == ["Alpha → Alpha Problem"]
|
||||
assert len(calls) == 1 and "Beta" not in calls[0] # nur das llm=ja-Paar zum Judge
|
||||
verlierer = await db.kanban_get_card(TOPIC, "inventory", cid_a)
|
||||
assert verlierer["stage"] == "grouped" and verlierer["payload"]["merged_into"] == "Alpha Problem"
|
||||
gewinner = await db.kanban_get_card(TOPIC, "inventory", cid_b)
|
||||
assert "Alpha.txt" in gewinner["payload"]["sources"] # Union
|
||||
assert await db.kanban_get_card(TOPIC, "artefacts", "alpha") is None
|
||||
assert not [r for r in await db.list_subblocks(TOPIC, "alpha")]
|
||||
|
||||
|
||||
async def test_fremd_removed_only_on_nein(env, monkeypatch):
|
||||
db, seed, files, write_report = env
|
||||
cid_f = await seed("Fremdling", "gehört nicht rein")
|
||||
cid_e = await seed("Echter", "belegt")
|
||||
write_report(_report(fremd=["Fremdling", "Echter"]))
|
||||
|
||||
async def fake_agent(key, prompt, timeout, **kw):
|
||||
return 0, '{"relevant": {"1": "nein", "2": "ja"}}', ""
|
||||
|
||||
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
||||
monkeypatch.setattr(repair, "source_folder", lambda t: None)
|
||||
res = await repair.repair_befunde(TOPIC)
|
||||
assert res["entfernt"] == ["Fremdling"]
|
||||
weg = await db.kanban_get_card(TOPIC, "inventory", cid_f)
|
||||
assert weg["stage"] == "rejected" and weg["payload"]["reason"] == "qa-fremd"
|
||||
bleibt = await db.kanban_get_card(TOPIC, "inventory", cid_e)
|
||||
assert bleibt["stage"] == "done_block"
|
||||
|
||||
|
||||
async def test_judge_failure_keeps_everything(env, monkeypatch):
|
||||
db, seed, files, write_report = env
|
||||
cid = await seed("Wackelig", "unsicher")
|
||||
write_report(_report(unecht=["Wackelig"]))
|
||||
|
||||
async def broken_agent(key, prompt, timeout, **kw):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(repair, "run_agent", broken_agent)
|
||||
res = await repair.repair_befunde(TOPIC)
|
||||
assert res["entfernt"] == []
|
||||
card = await db.kanban_get_card(TOPIC, "inventory", cid)
|
||||
assert card["stage"] == "done_block" # fail-open
|
||||
|
||||
|
||||
async def test_hygiene_cleans_title_norm_invariant(env, monkeypatch):
|
||||
db, seed, files, write_report = env
|
||||
cid = await seed("**Fetter Titel**", "beschreibung")
|
||||
files["sidecar"].write_text(json.dumps({"**Fetter Titel**": ["s"]}), encoding="utf-8")
|
||||
write_report(_report(hygiene=[{"titel": "**Fetter Titel**", "probleme": ["markdown"]}]))
|
||||
|
||||
async def no_agent(*a, **kw):
|
||||
raise AssertionError("Hygiene braucht keinen Agenten")
|
||||
|
||||
monkeypatch.setattr(repair, "run_agent", no_agent)
|
||||
res = await repair.repair_befunde(TOPIC)
|
||||
assert res["hygiene"] == ["**Fetter Titel** → Fetter Titel"]
|
||||
card = await db.kanban_get_card(TOPIC, "inventory", cid)
|
||||
assert card["payload"]["title"] == "Fetter Titel"
|
||||
assert json.loads(files["sidecar"].read_text()) == {"Fetter Titel": ["s"]}
|
||||
rows = await db.list_blocks(TOPIC)
|
||||
assert any(r["title"] == "Fetter Titel" and r["status"] == "consensus" for r in rows)
|
||||
|
||||
|
||||
async def test_no_report_is_clean_error(env):
|
||||
db, seed, files, write_report = env
|
||||
res = await repair.repair_befunde("gibtsnicht")
|
||||
assert "fehler" in res
|
||||
|
||||
|
||||
async def test_abschluss_qa_misst_mit_llm(env, monkeypatch):
|
||||
"""Repair-Abschlussreport misst mit LLM — der llm=False-Report blendete
|
||||
sub_dubletten aus und ließ die Note zwischen 10.0 und ~9 pendeln."""
|
||||
db, seed, files, write_report = env
|
||||
write_report(_report())
|
||||
import qa as qa_mod
|
||||
seen = {}
|
||||
|
||||
async def spy(topic, llm=False):
|
||||
seen["llm"] = llm
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(qa_mod, "qa_report", spy)
|
||||
await repair.repair_befunde(TOPIC)
|
||||
assert seen["llm"] is True
|
||||
|
||||
|
||||
async def test_sub_dubletten_merge(env, monkeypatch):
|
||||
"""Bestätigtes Sub-Paar + Zweitmeinung ja → Verlierer variant, Frage/Artefakt
|
||||
wandern zum Gewinner (bzw. fallen weg, wenn er den Typ schon hat)."""
|
||||
db, seed, files, write_report = env
|
||||
await seed("Alpha", "beschr")
|
||||
norm = repair._norm_title("Alpha")
|
||||
await db.put_subblock(TOPIC, norm, "gewinner sub", "Alpha", "Gewinner Sub",
|
||||
facts='{"key_points": ["a", "b"]}', status="consensus")
|
||||
await db.put_subblock(TOPIC, norm, "verlierer sub", "Alpha", "Verlierer Sub",
|
||||
facts='{"key_points": ["x"]}', status="consensus")
|
||||
await db.put_sub_artifact(TOPIC, norm, "verlierer sub", "example", "{}", "Alpha", "Verlierer Sub")
|
||||
await db.upsert_question_pattern(TOPIC, norm, "verlierer sub", "Alpha", "Verlierer Sub", "Frage V?")
|
||||
write_report(_report(sub_dubletten=[
|
||||
{"a": "[Alpha] Gewinner Sub", "b": "[Alpha] Verlierer Sub", "llm": "ja"},
|
||||
{"a": "[Alpha] Gibtsnicht", "b": "[Alpha] Verlierer Sub", "llm": "ja"}])) # tote Zeile → skip
|
||||
|
||||
async def fake_agent(key, prompt, timeout, **kw):
|
||||
assert "Gibtsnicht" not in prompt
|
||||
return 0, '{"relevant": {"1": "ja"}}', ""
|
||||
|
||||
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
||||
res = await repair.repair_befunde(TOPIC)
|
||||
assert res["sub_merges"] == ["Verlierer Sub → Gewinner Sub"]
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)}
|
||||
assert rows["verlierer sub"] == "variant" and rows["gewinner sub"] == "consensus"
|
||||
arts = {(r["sub_norm"], r["type"]) for r in await db.get_sub_artefakte(TOPIC)}
|
||||
assert ("gewinner sub", "example") in arts and ("verlierer sub", "example") not in arts
|
||||
fragen = {r["sub_norm"]: r["question"] for r in await db.list_question_pattern(TOPIC)}
|
||||
assert fragen.get("gewinner sub") == "Frage V?" and "verlierer sub" not in fragen
|
||||
|
||||
|
||||
async def test_sub_dubletten_zweitmeinung_nein(env, monkeypatch):
|
||||
db, seed, files, write_report = env
|
||||
await seed("Alpha", "beschr")
|
||||
norm = repair._norm_title("Alpha")
|
||||
await db.put_subblock(TOPIC, norm, "sub a", "Alpha", "Sub A", status="consensus")
|
||||
await db.put_subblock(TOPIC, norm, "sub b", "Alpha", "Sub B", status="consensus")
|
||||
write_report(_report(sub_dubletten=[{"a": "[Alpha] Sub A", "b": "[Alpha] Sub B", "llm": "ja"}]))
|
||||
|
||||
async def fake_agent(key, prompt, timeout, **kw):
|
||||
return 0, '{"relevant": {"1": "nein"}}', ""
|
||||
|
||||
monkeypatch.setattr(repair, "run_agent", fake_agent)
|
||||
res = await repair.repair_befunde(TOPIC)
|
||||
assert res["sub_merges"] == []
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)}
|
||||
assert rows["sub a"] == rows["sub b"] == "consensus"
|
||||
|
||||
|
||||
async def test_waisen_cleanup(env, monkeypatch):
|
||||
"""Artefakte/Fragen auf verworfene oder fehlende Subs fliegen; lebende und
|
||||
mehrdeutig-präfixige bleiben."""
|
||||
db, seed, files, write_report = env
|
||||
await seed("Alpha", "beschreibung") # legt sub0 (consensus) + flashcard auf sub0 an
|
||||
norm = repair._norm_title("Alpha")
|
||||
await db.put_subblock(TOPIC, norm, "weg", "Alpha", "Weg", status="discarded")
|
||||
await db.put_subblock(TOPIC, norm, "doppel: eins", "Alpha", "Doppel eins")
|
||||
await db.put_subblock(TOPIC, norm, "doppel: zwei", "Alpha", "Doppel zwei")
|
||||
await db.put_sub_artifact(TOPIC, norm, "weg", "flashcard", "{}", "Alpha", "Weg") # tot
|
||||
await db.put_sub_artifact(TOPIC, norm, "fehlt", "example", "{}", "Alpha", "Fehlt") # tot
|
||||
await db.put_sub_artifact(TOPIC, norm, "doppel", "example", "{}", "Alpha", "Doppel") # mehrdeutig → bleibt
|
||||
await db.upsert_question_pattern(TOPIC, norm, "fehlt", "Alpha", "Fehlt", "Frage?") # tot
|
||||
write_report(_report())
|
||||
|
||||
async def no_agent(*a, **kw):
|
||||
raise AssertionError("Aufräumen braucht keinen Agenten")
|
||||
|
||||
monkeypatch.setattr(repair, "run_agent", no_agent)
|
||||
res = await repair.repair_befunde(TOPIC)
|
||||
assert res["aufgeraeumt"] == 3
|
||||
rest = {(r["sub_norm"], r["type"]) for r in await db.get_sub_artefakte(TOPIC)}
|
||||
assert rest == {("sub0", "flashcard"), ("doppel", "example")}
|
||||
assert not [r for r in await db.list_question_pattern(TOPIC)]
|
||||
@@ -394,10 +394,12 @@ async def test_facts_check_inline_cited_evidence(sub_env, monkeypatch, tmp_path)
|
||||
"cited_facts": [{"text": "T", "source": "Skript.txt, Z.2"}],
|
||||
"example_idea": ""}]}
|
||||
|
||||
sh = blx._subs_hash({"Alpha": ["Sub Eins"]}) # Resume-Dateien tragen den Sub-Satz-Hash
|
||||
|
||||
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
|
||||
if "-facts-erg-" in key:
|
||||
return blx.FAILED, None
|
||||
(tmp_path / "facts-c0.json").write_text(json.dumps(facts), encoding="utf-8")
|
||||
(tmp_path / f"facts-{sh}-c0.json").write_text(json.dumps(facts), encoding="utf-8")
|
||||
return blx.OK, None
|
||||
|
||||
seen = []
|
||||
@@ -416,4 +418,29 @@ async def test_facts_check_inline_cited_evidence(sub_env, monkeypatch, tmp_path)
|
||||
assert len(seen) == blx.FACTS_CHECK_PANEL
|
||||
key, caps, prompt = seen[0]
|
||||
assert caps == "none" and "── Skript.txt · Z." in prompt
|
||||
assert (tmp_path / "facts-check-c0-j1.json").exists() # Engine persistiert die Antwort
|
||||
assert (tmp_path / f"facts-check-{sh}-c0-j1.json").exists() # Engine persistiert die Antwort
|
||||
|
||||
|
||||
def test_sub_key_resolves_short_titles():
|
||||
"""Artefakt-Agenten echoen den Kurztitel; der Sub-Key heißt 'kurztitel: beschreibung'.
|
||||
Eindeutiger Präfix wird aufgelöst, Mehrdeutiges und Fehlendes bleibt unverändert."""
|
||||
import board_artefacts as ba
|
||||
existing = {"autolink mit url: erzeugt link", "bilder: bindet bilder ein",
|
||||
"doppel: eins", "doppel: zwei", "exakt"}
|
||||
assert ba._sub_key(existing, "exakt") == "exakt"
|
||||
assert ba._sub_key(existing, "autolink mit url") == "autolink mit url: erzeugt link"
|
||||
assert ba._sub_key(existing, "doppel") == "doppel" # mehrdeutig → unverändert
|
||||
assert ba._sub_key(existing, "fehlt") == "fehlt" # kein Treffer → unverändert
|
||||
# Fuzzy: Paraphrase/Kürzung ohne Doppelpunkt-Präfix löst eindeutig auf
|
||||
lang = {"der backslash selbst muss mit escaped werden, um literal zu erscheinen"}
|
||||
assert ba._sub_key(lang, "der backslash selbst muss mit escaped werden") == next(iter(lang))
|
||||
assert ba._sub_key(lang | {"der backslash am zeilenende"}, "der backslash") == "der backslash" # mehrdeutig
|
||||
|
||||
|
||||
def test_subs_hash_invalidiert_bei_neuem_zuschnitt():
|
||||
"""Gleicher Sub-Satz → gleicher Hash (Resume greift); geänderter → neuer Hash.
|
||||
raw-Form (Strings) und sidecar-Form (dicts) hashen identisch."""
|
||||
a = {"Block": ["s1", "s2"]}
|
||||
assert blx._subs_hash(a) == blx._subs_hash({"Block": ["s1", "s2"]})
|
||||
assert blx._subs_hash(a) != blx._subs_hash({"Block": ["s1", "s3"]})
|
||||
assert blx._subs_hash(a) == blx._subs_hash({"Block": [{"title": "s1"}, {"title": "s2"}]})
|
||||
|
||||
Reference in New Issue
Block a user