195 lines
9.6 KiB
Python
195 lines
9.6 KiB
Python
"""E2E über die ECHTE Engine mit Fake-Agenten: kompletter Generierungspfad in Sekunden.
|
|
|
|
Anders als test_board_inventory (dort sind die Block-Funktionen gefakt) läuft hier alles
|
|
bis run_agent echt — _race, Quorum, Panels, In-Block-Konsolidierung, QA-Gate.
|
|
"""
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
|
|
import board_inventory as bi
|
|
from pipeline import GenContext
|
|
from invarianten import pruefe_invarianten, pruefe_guide_invarianten
|
|
|
|
TOPIC = "t"
|
|
|
|
|
|
def _files(tmp_path):
|
|
work = tmp_path / "arbeit"
|
|
work.mkdir(exist_ok=True)
|
|
return {"arbeit": work, "final": tmp_path / "blocks.md",
|
|
"sub_roh": tmp_path / "sub_roh.json", "sidecar": tmp_path / "subblocks.json",
|
|
"facts": tmp_path / "facts.json", "question_pattern": tmp_path / "question_pattern.json",
|
|
"artefakte": tmp_path / "artefakte.json", "outline": tmp_path / "outline.json",
|
|
"outline_slots": [tmp_path / f"outline-{i}.json" for i in (1, 2, 3)],
|
|
"research": [work / f"research-{i}.md" for i in (1, 2, 3, 4, 5)]}
|
|
|
|
|
|
async def _lauf(tmp_path, research=True, qa_force=False, timeout=120):
|
|
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
|
files = _files(tmp_path)
|
|
ok = await asyncio.wait_for(
|
|
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "",
|
|
research=research, qa_force=qa_force), timeout=timeout)
|
|
return ok, files
|
|
|
|
|
|
async def test_e2e_thema_vollpfad(fake_welt, testdb, tmp_path):
|
|
"""Research → Inventar → QA-Gate → Artefakte → Finalize, alle Schichten echt."""
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
done = [c for c in await db.kanban_cards(TOPIC, board="inventory", stage="done_block")]
|
|
titel = {c["payload"]["title"] for c in done}
|
|
# Bottom-up: 7 Atome zu 3 Themen-Clustern gruppiert
|
|
assert titel == {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"}
|
|
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
|
# jedes Atom liegt in genau EINEM Cluster — keine Cross-Block-Dublette mehr
|
|
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
|
assert [r["status"] for r in gemeinsam] == ["consensus"]
|
|
# Vollständigkeit: alle 7 Atome sind als consensus-Sub erhalten
|
|
consensus = {r["sub_norm"] for r in subs if r["status"] == "consensus"}
|
|
assert consensus == {"definition alpha", "alpha eigenschaften", "gemeinsamer grundbegriff",
|
|
"beta ablauf", "beta grenzen", "gamma praxisfall", "gamma werkzeuge"}
|
|
fehler = await pruefe_invarianten(TOPIC, files)
|
|
assert fehler == []
|
|
|
|
|
|
async def test_e2e_guide(fake_welt, testdb, tmp_path):
|
|
"""Auf den Vollpfad folgt der Guide-Bau — Gate/Coverage/Lese-Stages laufen echt."""
|
|
import guide_board
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
|
entries = {i: f"{c['payload']['title']} — {c['payload'].get('description', '')}"
|
|
for i, c in enumerate(done, 1)}
|
|
chapters = await asyncio.wait_for(
|
|
guide_board.run_guide_board("g-e2e", TOPIC, "Guide", entries, "", "claude",
|
|
tmp_path / "guides" / "Guide.json"), timeout=120)
|
|
assert chapters is not None
|
|
assert await pruefe_guide_invarianten(TOPIC) == []
|
|
|
|
|
|
async def test_e2e_only_artefacts(fake_welt, testdb, tmp_path):
|
|
"""Artefakt-only (inventory=False): nach removeArtefacts läuft NUR Board 2 auf den
|
|
fertigen Blocks — kein Board 1, keine Research-Agenten. Karten aus done_block geseedet."""
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
subs_vorher = {r["sub_norm"] for r in await db.list_subblocks(TOPIC)}
|
|
assert subs_vorher
|
|
# Artefakte entfernen (Karten zurück auf generate, Board-1-Subs bleiben)
|
|
await bi.reset_board_from_stage(TOPIC, "artefacts", "generate", files)
|
|
fake_welt.calls.clear()
|
|
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
|
ok2 = await asyncio.wait_for(
|
|
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "",
|
|
research=False, inventory=False), timeout=60)
|
|
assert ok2
|
|
assert not any("-research-" in k for k in fake_welt.calls) # Board 1 lief NICHT
|
|
assert any("-sb-enrich-" in k or "-sb-verify-" in k for k in fake_welt.calls) # Board 2 lief
|
|
assert {r["sub_norm"] for r in await db.list_subblocks(TOPIC)} == subs_vorher # vollständig
|
|
|
|
|
|
async def test_e2e_rerun_idempotent(fake_welt, testdb, tmp_path):
|
|
"""Zweiter Lauf (Continue, research=False) hinterlässt keine Waisen/Reste."""
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
vorher = {(r["block_norm"], r["sub_norm"], r["status"])
|
|
for r in await db.list_subblocks(TOPIC)}
|
|
ok2, _f = await _lauf(tmp_path, research=False)
|
|
assert ok2
|
|
nachher = {(r["block_norm"], r["sub_norm"], r["status"])
|
|
for r in await db.list_subblocks(TOPIC)}
|
|
assert nachher == vorher
|
|
assert await pruefe_invarianten(TOPIC, files) == []
|
|
|
|
|
|
@pytest.mark.parametrize("stoerung", [
|
|
{"muster": r"-sb-verify-.*-j1$", "modus": "garbage", "mal": 1}, # Ersatz-Richter jE
|
|
{"muster": r"-art-gen-.*-t1$", "modus": "fehler", "mal": 1}, # Slot-Restart
|
|
{"muster": r"-research-2$", "modus": "fehler", "mal": 3}, # 1 Producer tot
|
|
])
|
|
async def test_e2e_stoerungen_flow_endet(fake_welt, testdb, tmp_path, stoerung):
|
|
"""Einzel-Ausfälle dürfen weder den Flow stoppen noch Invarianten reißen."""
|
|
fake_welt.stoerungen.append(dict(stoerung, rest=stoerung["mal"]))
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
assert await pruefe_invarianten(TOPIC, files) == []
|
|
|
|
|
|
async def test_e2e_gate_vollinventur_ohne_fix(fake_welt, testdb, tmp_path):
|
|
"""Gate-Judge liefert eine Voll-Inventur (belegte Claims mit „Belegt…"-Grund) —
|
|
der Schema-Filter wirft sie raus, es läuft KEIN Fakten-Fix."""
|
|
import json
|
|
antwort = json.dumps({"claims": [
|
|
{"text": "Aussage 1", "grund": "Belegt durch Quelle", "urteil": "unbelegt"},
|
|
{"text": "Aussage 2", "grund": "Belegt durch Fakten", "urteil": "unbelegt"},
|
|
{"text": "Aussage 3", "grund": "Belegt: steht im Skript", "urteil": "unbelegt"}]})
|
|
fake_welt.stoerungen.append({"muster": r"-gate-", "modus": "antwort",
|
|
"antwort": antwort, "mal": 99, "rest": 99})
|
|
import guide_board
|
|
ok, _files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
|
entries = {i: c["payload"]["title"] for i, c in enumerate(done, 1)}
|
|
chapters = await asyncio.wait_for(
|
|
guide_board.run_guide_board("g-vi", TOPIC, "Guide", entries, "", "claude",
|
|
tmp_path / "guides" / "Guide.json"), timeout=120)
|
|
assert chapters is not None
|
|
assert not any("-gatefix-" in k for k in fake_welt.calls)
|
|
|
|
|
|
async def test_e2e_echtheits_flattern_gestoppt(fake_welt, testdb, tmp_path):
|
|
"""QA-Pass 1 flaggt alle Blöcke als unecht (Judge-Flattern) — der Bestätiger-Pass
|
|
widerspricht, die Gate-Note bleibt sauber, der Flow läuft durch."""
|
|
import json
|
|
fake_welt.stoerungen.append({"muster": r"^qa-t-bausteine-0$", "modus": "antwort",
|
|
"antwort": json.dumps({"relevant": {"1": "nein", "2": "nein", "3": "nein"}}),
|
|
"mal": 1, "rest": 1})
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok # Gate hat nicht pausiert — der Zufalls-Verdacht wurde nicht bestätigt
|
|
assert any("bausteine-b2" in k for k in fake_welt.calls)
|
|
assert await pruefe_invarianten(TOPIC, files) == []
|
|
|
|
|
|
async def test_e2e_uni_anker_gate(fake_welt, testdb, tmp_path, monkeypatch):
|
|
"""uni-Modus mit Mini-Korpus: der Kanon-Titel ohne Korpus-Anker wird deterministisch
|
|
rejected (leeres Evidence-Pack), die belegten Blöcke laufen durch; QA misst gegen
|
|
den echten Korpus."""
|
|
import blocks as blx
|
|
fake_welt.bloecke["Kanon-Klassiker"] = {
|
|
"beschreibung": "Beruehmtes Lehrbuchproblem", "subs": ["Klassiker Detail"]}
|
|
korpus = tmp_path / "korpus"
|
|
korpus.mkdir()
|
|
zeilen = []
|
|
for t, b in fake_welt.bloecke.items():
|
|
if t == "Kanon-Klassiker":
|
|
continue # kommt bewusst NICHT im Material vor
|
|
zeilen.append(f"Kapitel {t}: {b['beschreibung']}. " +
|
|
" ".join(f"Wir behandeln {s}." for s in b["subs"]))
|
|
(korpus / "skript.txt").write_text("\n\n".join(zeilen), encoding="utf-8")
|
|
monkeypatch.setattr(bi, "source_folder", lambda t: korpus)
|
|
monkeypatch.setattr(blx, "source_folder", lambda t: korpus)
|
|
|
|
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
|
files = _files(tmp_path)
|
|
ok = await asyncio.wait_for(
|
|
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "uni", "location": str(korpus)},
|
|
korpus, "", research=True, qa_force=True), timeout=120)
|
|
assert ok
|
|
db = testdb
|
|
alle = [dict(c) for c in await db.kanban_cards(TOPIC, board="inventory")]
|
|
# das unbelegte Atom „Klassiker Detail" (einziges Atom von Kanon-Klassiker) wird rejected
|
|
assert any(c["stage"] == "rejected" and c["payload"].get("title") == "Klassiker Detail"
|
|
for c in alle)
|
|
assert not any(c["payload"].get("title") in ("Klassiker Detail", "Kanon-Klassiker")
|
|
and c["stage"] == "done_block" for c in alle) # nie zum Block geworden
|
|
done = {c["payload"].get("title") for c in alle
|
|
if c["kind"] == "block" and c["stage"] == "done_block"}
|
|
assert {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"} <= done
|