198 lines
9.5 KiB
Python
198 lines
9.5 KiB
Python
"""E2E über die ECHTE Engine mit Fake-Agenten: kompletter Generierungspfad in Sekunden.
|
|
|
|
Anders als test_board_inventory (dort sind die Block-Funktionen gefakt) läuft hier alles
|
|
bis run_agent echt — _race, Quorum, Panels, Konsolidierung, Cross-Block, QA-Gate.
|
|
"""
|
|
|
|
import asyncio
|
|
|
|
import pytest
|
|
|
|
import board_inventory as bi
|
|
from pipeline import GenContext
|
|
from tests.invarianten import pruefe_invarianten, pruefe_guide_invarianten
|
|
|
|
TOPIC = "t"
|
|
|
|
|
|
def _files(tmp_path):
|
|
work = tmp_path / "arbeit"
|
|
work.mkdir(exist_ok=True)
|
|
return {"arbeit": work, "final": tmp_path / "blocks.md",
|
|
"sub_roh": tmp_path / "sub_roh.json", "sidecar": tmp_path / "subblocks.json",
|
|
"facts": tmp_path / "facts.json", "question_pattern": tmp_path / "question_pattern.json",
|
|
"artefakte": tmp_path / "artefakte.json", "outline": tmp_path / "outline.json",
|
|
"outline_slots": [tmp_path / f"outline-{i}.json" for i in (1, 2, 3)],
|
|
"research": [work / f"research-{i}.md" for i in (1, 2, 3, 4, 5)]}
|
|
|
|
|
|
async def _lauf(tmp_path, research=True, qa_force=False, timeout=120):
|
|
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
|
files = _files(tmp_path)
|
|
ok = await asyncio.wait_for(
|
|
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "",
|
|
research=research, qa_force=qa_force), timeout=timeout)
|
|
return ok, files
|
|
|
|
|
|
async def test_e2e_thema_vollpfad(fake_welt, testdb, tmp_path):
|
|
"""Research → Inventar → QA-Gate → Artefakte → Finalize, alle Schichten echt."""
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
done = [c for c in await db.kanban_cards(TOPIC, board="inventory", stage="done_block")]
|
|
titel = {c["payload"]["title"] for c in done}
|
|
assert titel == {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"}
|
|
# Cross-Block-Dublette: „Gemeinsamer Grundbegriff" überlebt in genau EINEM Block
|
|
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
|
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
|
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "variant"]
|
|
fehler = await pruefe_invarianten(TOPIC, files)
|
|
assert fehler == []
|
|
|
|
|
|
async def test_e2e_guide(fake_welt, testdb, tmp_path):
|
|
"""Auf den Vollpfad folgt der Guide-Bau — Gate/Coverage/Lese-Stages laufen echt."""
|
|
import guide_board
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
|
entries = {i: f"{c['payload']['title']} — {c['payload'].get('description', '')}"
|
|
for i, c in enumerate(done, 1)}
|
|
chapters = await asyncio.wait_for(
|
|
guide_board.run_guide_board("g-e2e", TOPIC, "Guide", entries, "", "claude",
|
|
tmp_path / "guides" / "Guide.json"), timeout=120)
|
|
assert chapters is not None
|
|
assert await pruefe_guide_invarianten(TOPIC) == []
|
|
|
|
|
|
async def test_e2e_rerun_idempotent(fake_welt, testdb, tmp_path):
|
|
"""Zweiter Lauf (Continue, research=False) hinterlässt keine Waisen/Reste."""
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
vorher = {(r["block_norm"], r["sub_norm"], r["status"])
|
|
for r in await db.list_subblocks(TOPIC)}
|
|
ok2, _f = await _lauf(tmp_path, research=False)
|
|
assert ok2
|
|
nachher = {(r["block_norm"], r["sub_norm"], r["status"])
|
|
for r in await db.list_subblocks(TOPIC)}
|
|
assert nachher == vorher
|
|
assert await pruefe_invarianten(TOPIC, files) == []
|
|
|
|
|
|
@pytest.mark.parametrize("stoerung", [
|
|
{"muster": r"-sub-crossblock-.*-j1$", "modus": "fehler", "mal": 3}, # Ersatzrichter jE
|
|
{"muster": r"-sub-konsolidierung-.*-j1$", "modus": "garbage", "mal": 1}, # Retry heilt
|
|
{"muster": r"-facts-c\d+$", "modus": "fehler", "mal": 1}, # Slot-Restart
|
|
{"muster": r"-research-2$", "modus": "fehler", "mal": 3}, # 1 Producer tot
|
|
])
|
|
async def test_e2e_stoerungen_flow_endet(fake_welt, testdb, tmp_path, stoerung):
|
|
"""Einzel-Ausfälle dürfen weder den Flow stoppen noch Invarianten reißen."""
|
|
fake_welt.stoerungen.append(dict(stoerung, rest=stoerung["mal"]))
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
assert await pruefe_invarianten(TOPIC, files) == []
|
|
|
|
|
|
async def test_e2e_crossblock_dissent_failopen(fake_welt, testdb, tmp_path):
|
|
"""j1 sagt a, j2 sagt b, j3 fällt aus → Paar bleibt (fail-open), Rest konsistent."""
|
|
fake_welt.stoerungen += [
|
|
{"muster": r"-sub-crossblock-.*-j2$", "modus": "antwort",
|
|
"antwort": '{"pairs": {"1": "b"}}', "mal": 1, "rest": 1},
|
|
{"muster": r"-sub-crossblock-.*-j3$", "modus": "fehler", "mal": 3, "rest": 3},
|
|
]
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
|
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
|
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "consensus"] # kein Fold
|
|
assert await pruefe_invarianten(TOPIC, files) == []
|
|
|
|
|
|
async def test_e2e_inblock_gruppe_faltet(fake_welt, testdb, tmp_path):
|
|
"""Welt-Regel: „Alpha Eigenschaften" faltet unter „Definition Alpha" — beide Judges
|
|
liefern die Gruppe, der Verlierer wird variant, seine facts wandern zum Gewinner."""
|
|
fake_welt.gruppen.append(("Definition Alpha", ["Alpha Eigenschaften"]))
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha-konzept")}
|
|
assert rows.get("alpha eigenschaften") == "variant"
|
|
assert rows.get("definition alpha") == "consensus"
|
|
assert await pruefe_invarianten(TOPIC, files) == []
|
|
|
|
|
|
async def test_e2e_gate_vollinventur_ohne_fix(fake_welt, testdb, tmp_path):
|
|
"""Gate-Judge liefert eine Voll-Inventur (belegte Claims mit „Belegt…"-Grund) —
|
|
der Schema-Filter wirft sie raus, es läuft KEIN Fakten-Fix."""
|
|
import json
|
|
antwort = json.dumps({"claims": [
|
|
{"text": "Aussage 1", "grund": "Belegt durch Quelle", "urteil": "unbelegt"},
|
|
{"text": "Aussage 2", "grund": "Belegt durch Fakten", "urteil": "unbelegt"},
|
|
{"text": "Aussage 3", "grund": "Belegt: steht im Skript", "urteil": "unbelegt"}]})
|
|
fake_welt.stoerungen.append({"muster": r"-gate-", "modus": "antwort",
|
|
"antwort": antwort, "mal": 99, "rest": 99})
|
|
import guide_board
|
|
ok, _files = await _lauf(tmp_path)
|
|
assert ok
|
|
db = testdb
|
|
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
|
entries = {i: c["payload"]["title"] for i, c in enumerate(done, 1)}
|
|
chapters = await asyncio.wait_for(
|
|
guide_board.run_guide_board("g-vi", TOPIC, "Guide", entries, "", "claude",
|
|
tmp_path / "guides" / "Guide.json"), timeout=120)
|
|
assert chapters is not None
|
|
assert not any("-gatefix-" in k for k in fake_welt.calls)
|
|
|
|
|
|
async def test_e2e_echtheits_flattern_gestoppt(fake_welt, testdb, tmp_path):
|
|
"""QA-Pass 1 flaggt alle Blöcke als unecht (Judge-Flattern) — der Bestätiger-Pass
|
|
widerspricht, die Gate-Note bleibt sauber, der Flow läuft durch."""
|
|
import json
|
|
fake_welt.stoerungen.append({"muster": r"^qa-t-bausteine-0$", "modus": "antwort",
|
|
"antwort": json.dumps({"relevant": {"1": "nein", "2": "nein", "3": "nein"}}),
|
|
"mal": 1, "rest": 1})
|
|
ok, files = await _lauf(tmp_path)
|
|
assert ok # Gate hat nicht pausiert — der Zufalls-Verdacht wurde nicht bestätigt
|
|
assert any("bausteine-b2" in k for k in fake_welt.calls)
|
|
assert await pruefe_invarianten(TOPIC, files) == []
|
|
|
|
|
|
async def test_e2e_uni_anker_gate(fake_welt, testdb, tmp_path, monkeypatch):
|
|
"""uni-Modus mit Mini-Korpus: der Kanon-Titel ohne Korpus-Anker wird deterministisch
|
|
rejected (leeres Evidence-Pack), die belegten Blöcke laufen durch; QA misst gegen
|
|
den echten Korpus."""
|
|
import blocks as blx
|
|
fake_welt.bloecke["Kanon-Klassiker"] = {
|
|
"beschreibung": "Beruehmtes Lehrbuchproblem", "subs": ["Klassiker Detail"]}
|
|
korpus = tmp_path / "korpus"
|
|
korpus.mkdir()
|
|
zeilen = []
|
|
for t, b in fake_welt.bloecke.items():
|
|
if t == "Kanon-Klassiker":
|
|
continue # kommt bewusst NICHT im Material vor
|
|
zeilen.append(f"Kapitel {t}: {b['beschreibung']}. " +
|
|
" ".join(f"Wir behandeln {s}." for s in b["subs"]))
|
|
(korpus / "skript.txt").write_text("\n\n".join(zeilen), encoding="utf-8")
|
|
monkeypatch.setattr(bi, "source_folder", lambda t: korpus)
|
|
monkeypatch.setattr(blx, "source_folder", lambda t: korpus)
|
|
|
|
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
|
files = _files(tmp_path)
|
|
ok = await asyncio.wait_for(
|
|
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "uni", "location": str(korpus)},
|
|
korpus, "", research=True, qa_force=True), timeout=120)
|
|
assert ok
|
|
db = testdb
|
|
alle = [dict(c) for c in await db.kanban_cards(TOPIC, board="inventory")]
|
|
assert any(c["stage"] == "rejected" and c["payload"].get("title") == "Kanon-Klassiker"
|
|
for c in alle)
|
|
assert not any(c["kind"] == "block" and c["payload"].get("title") == "Kanon-Klassiker"
|
|
for c in alle) # nie zum Block geworden
|
|
done = {c["payload"].get("title") for c in alle
|
|
if c["kind"] == "block" and c["stage"] == "done_block"}
|
|
assert {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"} <= done
|