update
This commit is contained in:
@@ -56,6 +56,16 @@ async def board_env(testdb, tmp_path, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", _fake_single_slot(tmp_path))
|
||||
|
||||
# QA-Gate: standardmäßig saubere Fake-Note (kein Embedding-Load in Tests);
|
||||
# Gate-Tests überschreiben qa_report gezielt.
|
||||
import qa as qa_mod
|
||||
|
||||
async def _fake_qa(topic, llm=False):
|
||||
return {"note": 10.0, "topic": TOPIC, "quoten": {}, "fremd": [], "artefakte": {"status": "nicht generiert"}}
|
||||
monkeypatch.setattr(qa_mod, "qa_report", _fake_qa)
|
||||
monkeypatch.setattr(qa_mod, "_write_report", lambda r: None)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
|
||||
async def no_emb(flow):
|
||||
return False
|
||||
monkeypatch.setattr(bi, "_emb_ok", no_emb)
|
||||
@@ -86,12 +96,21 @@ async def board_env(testdb, tmp_path, monkeypatch):
|
||||
async def fake_outline(ctx, set_p, files, entries, instructions):
|
||||
return {"chapters": [{"title": "Kapitel 1", "numbers": sorted(entries)}]}
|
||||
|
||||
async def fake_konsolidierung(ctx, files, raw, facts_map, instructions="", ns="", lbl=""):
|
||||
return None
|
||||
|
||||
for name, fn in [("_subblocks_block", fake_subblocks), ("_facts_block", fake_facts),
|
||||
("_levels_block", fake_levels), ("_relevance_block", fake_relevance),
|
||||
("_question_pattern_block", fake_pattern), ("_artefacts_block", fake_artefacts),
|
||||
("_outline_block", fake_outline)]:
|
||||
("_outline_block", fake_outline), ("_konsolidiere_subblocks", fake_konsolidierung)]:
|
||||
monkeypatch.setattr(ba, name, fn)
|
||||
|
||||
class _EmbOff: # Cross-Block-Barrier reicht ohne Modell alle Karten durch
|
||||
@staticmethod
|
||||
def available():
|
||||
return False
|
||||
monkeypatch.setattr(ba, "embedding", _EmbOff)
|
||||
|
||||
work = tmp_path / "arbeit"
|
||||
work.mkdir()
|
||||
files = {"arbeit": work, "final": tmp_path / "blocks.md",
|
||||
@@ -151,6 +170,10 @@ async def test_board1_full_flow(board_env):
|
||||
assert {s["sub_title"] for s in subs} == {"Sub Eins", "Sub Zwei"}
|
||||
outline = await db.get_outline(TOPIC)
|
||||
assert outline and "Kapitel 1" in outline
|
||||
# Lauf-Summary am Flow-Ende: run_id + Zähler (QA diffed dagegen)
|
||||
summary = json.loads((files["arbeit"] / "lauf-summary.json").read_text(encoding="utf-8"))
|
||||
assert summary["run_id"] and summary["topic"] == TOPIC
|
||||
assert summary["boards"].get("inventory", {}).get("done_block") == 4
|
||||
|
||||
|
||||
async def test_filter_judges_run_parallel(board_env, monkeypatch):
|
||||
@@ -839,3 +862,204 @@ async def test_ingest_strips_markdown_title(testdb, tmp_path):
|
||||
assert n == 1
|
||||
card = await testdb.kanban_get_card(TOPIC, B, "listscheduling")
|
||||
assert card["payload"]["title"] == "ListScheduling"
|
||||
|
||||
|
||||
# ── QA-Gate: Inventar-Prüfung vor Board 2 ────────────────────────────────────────────
|
||||
|
||||
async def _run_flow(ctx, files, timeout=30, **kw):
|
||||
import asyncio
|
||||
return await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "", research=False, **kw),
|
||||
timeout=timeout)
|
||||
|
||||
|
||||
async def test_qa_gate_pauses_on_bad_note(board_env, monkeypatch):
|
||||
"""Note unter Schwelle → Flow endet sauber, Board-2-Karten warten in subblocks."""
|
||||
import qa as qa_mod
|
||||
db, ctx, files = board_env
|
||||
|
||||
async def bad_qa(topic, llm=False):
|
||||
return {"note": 5.0, "topic": TOPIC, "quoten": {}, "fremd": [], "artefakte": {}}
|
||||
monkeypatch.setattr(qa_mod, "qa_report", bad_qa)
|
||||
monkeypatch.setattr(qa_mod, "_write_report", lambda r: None)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
await _seed(db)
|
||||
ok = await _run_flow(ctx, files)
|
||||
assert ok
|
||||
warten = await db.kanban_cards(TOPIC, board="artefacts", stage="subblocks")
|
||||
assert len(warten) == 4 # alle Blöcke gespawnt, keiner verarbeitet
|
||||
assert await db.kanban_count(TOPIC, "done_artefact", board="artefacts") == 0
|
||||
|
||||
|
||||
async def test_qa_gate_force_overrides(board_env, monkeypatch):
|
||||
"""qa_force=True („Trotzdem fortsetzen") übersteuert die schlechte Note."""
|
||||
import qa as qa_mod
|
||||
db, ctx, files = board_env
|
||||
called = {"n": 0}
|
||||
|
||||
async def bad_qa(topic, llm=False):
|
||||
called["n"] += 1
|
||||
return {"note": 5.0, "topic": TOPIC, "quoten": {}, "fremd": [], "artefakte": {}}
|
||||
monkeypatch.setattr(qa_mod, "qa_report", bad_qa)
|
||||
monkeypatch.setattr(qa_mod, "_write_report", lambda r: None)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
await _seed(db)
|
||||
ok = await _run_flow(ctx, files, qa_force=True)
|
||||
assert ok
|
||||
assert await db.kanban_count(TOPIC, "done_artefact", board="artefacts") >= 5
|
||||
assert called["n"] <= 1 # Gate-Lauf übersprungen; höchstens Abschluss-QA
|
||||
|
||||
|
||||
async def test_qa_gate_off_means_no_qa_call(board_env, monkeypatch):
|
||||
import qa as qa_mod
|
||||
db, ctx, files = board_env
|
||||
called = {"n": 0}
|
||||
|
||||
async def spy_qa(topic, llm=False):
|
||||
called["n"] += 1
|
||||
return {"note": 10.0, "topic": TOPIC, "quoten": {}, "fremd": [], "artefakte": {}}
|
||||
monkeypatch.setattr(qa_mod, "qa_report", spy_qa)
|
||||
monkeypatch.setattr(qa_mod, "_write_report", lambda r: None)
|
||||
monkeypatch.setattr(bi, "QA_GATE_NOTE", 0)
|
||||
await _seed(db)
|
||||
ok = await _run_flow(ctx, files)
|
||||
assert ok
|
||||
assert called["n"] == 1 # kein Gate-Lauf; nur die Abschluss-QA der Lauf-Summary
|
||||
|
||||
|
||||
async def test_qa_gate_fail_open(board_env, monkeypatch):
|
||||
"""QA crasht → Gate öffnet, Flow läuft komplett durch (fail-open)."""
|
||||
import qa as qa_mod
|
||||
db, ctx, files = board_env
|
||||
|
||||
async def broken_qa(topic, llm=False):
|
||||
raise RuntimeError("kaputt")
|
||||
monkeypatch.setattr(qa_mod, "qa_report", broken_qa)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
await _seed(db)
|
||||
ok = await _run_flow(ctx, files)
|
||||
assert ok
|
||||
assert await db.kanban_count(TOPIC, "done_artefact", board="artefacts") >= 5
|
||||
|
||||
|
||||
def test_qa_view_pausiert_logic(tmp_path, monkeypatch):
|
||||
import qa as qa_mod
|
||||
import json as _json
|
||||
monkeypatch.setattr(qa_mod, "QA_DIR", tmp_path)
|
||||
(tmp_path / TOPIC).mkdir()
|
||||
(tmp_path / TOPIC / "r1.json").write_text(_json.dumps(
|
||||
{"note": 5.0, "quoten": {"fremd": 0.2}, "fremd": ["X"], "unecht": ["Y"]}), encoding="utf-8")
|
||||
counts = {"inventory": {"done_block": 3}, "artefacts": {"subblocks": 4}}
|
||||
v = bi._qa_view(TOPIC, counts, None)
|
||||
assert v["pausiert"] is True and v["note"] == 5.0 and v["befunde"] == ["X", "Y"]
|
||||
from types import SimpleNamespace
|
||||
laufend = SimpleNamespace(state={})
|
||||
assert bi._qa_view(TOPIC, counts, laufend)["pausiert"] is False # Flow läuft noch
|
||||
# Bausteine gelöscht → kein Badge, obwohl der Report noch existiert
|
||||
assert bi._qa_view(TOPIC, {}, None) is None
|
||||
|
||||
|
||||
def test_qa_view_picks_newest_by_mtime(tmp_path, monkeypatch):
|
||||
"""Run-id-Namen (…-1311-5e5c) sortieren lexikographisch VOR Zeitstempel-Namen —
|
||||
ein Re-Run überschreibt die run-id-Datei, das Badge muss trotzdem sie zeigen."""
|
||||
import os
|
||||
import qa as qa_mod
|
||||
import json as _json
|
||||
monkeypatch.setattr(qa_mod, "QA_DIR", tmp_path)
|
||||
(tmp_path / TOPIC).mkdir()
|
||||
alt = tmp_path / TOPIC / "20260703-141649.json"
|
||||
alt.write_text(_json.dumps({"note": 10.0, "quoten": {}}), encoding="utf-8")
|
||||
os.utime(alt, (1000, 1000))
|
||||
neu = tmp_path / TOPIC / "20260703-1311-5e5c.json"
|
||||
neu.write_text(_json.dumps({"note": 8.9, "quoten": {}}), encoding="utf-8")
|
||||
os.utime(neu, (2000, 2000))
|
||||
v = bi._qa_view(TOPIC, {"inventory": {"done_block": 3}}, None)
|
||||
assert v["note"] == 8.9
|
||||
|
||||
|
||||
async def test_supplement_material_mode_for_source_topics(board_env, tmp_path, monkeypatch):
|
||||
"""Quellen-Thema: Supplement vergleicht gegen das MATERIAL (files, kein Web);
|
||||
thema-Modus behält die Websuche (voller Zugriff)."""
|
||||
db, ctx, files = board_env
|
||||
seen = {}
|
||||
|
||||
async def spy_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
|
||||
seen[key] = (capabilities, prompt)
|
||||
m = _PATH_RE.search(prompt)
|
||||
if m and "-supplement" in key and "-beleg" not in key:
|
||||
with open(m.group(1), "w", encoding="utf-8") as f:
|
||||
json.dump({"blocks": []}, f)
|
||||
return "ok", payload(None)
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", spy_slot)
|
||||
flow = _mk_flow(tmp_path)
|
||||
korpus = tmp_path / "korpus"
|
||||
korpus.mkdir()
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: korpus)
|
||||
await bi._supplement_producer(ctx, flow, ["Alpha"])
|
||||
caps, prompt = seen[f"blocks-{TOPIC}-supplement"]
|
||||
assert caps == "files"
|
||||
assert "LEARNING MATERIAL" in prompt and "Do NOT search the web" in prompt
|
||||
|
||||
seen.clear()
|
||||
(tmp_path / "supplement.json").unlink() # Resume-Guard zurücksetzen
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: None)
|
||||
await bi._supplement_producer(ctx, flow, ["Alpha"])
|
||||
caps, prompt = seen[f"blocks-{TOPIC}-supplement"]
|
||||
assert caps == "full"
|
||||
assert "Research the subject area" in prompt
|
||||
|
||||
|
||||
# ── Anker-Gate: Quorum-Titel ohne Korpus-Beleg (Reader-Ko-Halluzination) ────────────
|
||||
|
||||
async def _anker_env(db, tmp_path, monkeypatch, titel_map):
|
||||
(tmp_path / "korpus.txt").write_text("Der Graph ist zusammenhängend und endlich.", encoding="utf-8")
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: tmp_path)
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"title": titel_map[cid], "description": "", "readers": ["r1", "r2"], "supplement": False}]
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "_rep", lambda rows: rows[0])
|
||||
for cid in titel_map:
|
||||
await db.kanban_upsert_card(TOPIC, B, cid, "cluster", "consensus_gate", {})
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
return ctx, [{"card_id": c, "payload": {}} for c in titel_map]
|
||||
|
||||
|
||||
async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
|
||||
"""Titel ohne Korpus-Anker → Beleg-Judge; „nein" → rejected/kein-beleg.
|
||||
Titel MIT Anker geht ohne Judge nach naming."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch,
|
||||
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"})
|
||||
|
||||
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "Brückenproblem" in prompt and "Zusammenhang" not in prompt # nur der Anker-lose
|
||||
return "ok", payload((0, json.dumps({"relevant": {"1": "nein"}}), ""))
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), cards)
|
||||
assert (await db.kanban_get_card(TOPIC, B, "c1"))["stage"] == "naming"
|
||||
c2 = await db.kanban_get_card(TOPIC, B, "c2")
|
||||
assert c2["stage"] == "rejected" and c2["payload"]["reason"] == "kein-beleg"
|
||||
|
||||
|
||||
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Judge-Ausfall → Titel bleibt (2-Reader-Rückhalt)."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"})
|
||||
|
||||
async def broken_slot(*a, **kw):
|
||||
return "failed", None
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", broken_slot)
|
||||
await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), cards)
|
||||
assert (await db.kanban_get_card(TOPIC, B, "c9"))["stage"] == "naming"
|
||||
|
||||
|
||||
def test_hat_anker_ziffern_suffix():
|
||||
ctoks = {"tsp", "graph", "kanten"}
|
||||
assert bi._hat_anker("ΔTSP1-Algorithmus", ctoks) # tsp1 → tsp
|
||||
assert not bi._hat_anker("Königsberger Brückenproblem", ctoks)
|
||||
assert not bi._hat_anker("Algorithmus Verfahren", ctoks) # nur Stopwörter → kein Anker
|
||||
|
||||
Reference in New Issue
Block a user