This commit is contained in:
team3
2026-07-04 12:21:45 +02:00
parent 2f5d5b9ca1
commit 8d8f6c8e51
43 changed files with 1920 additions and 236 deletions

View File

@@ -16,3 +16,69 @@ async def testdb(tmp_path, monkeypatch):
await database.init_db()
yield database
await database.close_db()
@pytest.fixture
async def fake_welt(testdb, tmp_path, monkeypatch):
"""E2E ohne LLM: run_agent überall durch die Fake-Welt ersetzt, Tempo-Bremsen raus.
Alle echten Schichten (_race, Quorum, Panels, Producer, QA-Gate) laufen mit."""
import agents
import blocks
import board_inventory as bi
import guide
import kanban
import pipeline
import qa
import repair
from fake_agents import Welt
welt = Welt()
async def fake_run_agent(agent_key, prompt, timeout, provider="claude", role="fast",
capabilities="none", lane="batch", scope=None, on_line=None, label=""):
return welt.respond(agent_key, prompt, capabilities)
for mod in (agents, pipeline, blocks, guide, repair):
monkeypatch.setattr(mod, "run_agent", fake_run_agent)
# Tempo: grace/poll/backoff bremsen echte Läufe, nicht den Fake
monkeypatch.setattr(blocks, "CONSENSUS_GRACE", 0)
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
monkeypatch.setattr(kanban, "RETRY_BACKOFF", 0.05)
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
import guide_board
monkeypatch.setattr(guide_board, "READABILITY_ACTIVE", False) # kein Modell-Load im Test
import asyncio as _aio
monkeypatch.setattr(bi, "_ingest_lock", _aio.Lock()) # Modul-Lock klebt sonst am Vortest-Loop
class _FakeEmb: # identischer Text → cos 1.0, sonst 0.0 (deterministisch, ohne Modell)
@staticmethod
def available():
return True
@staticmethod
def embed_sims(texts):
import numpy as np
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
arr = np.zeros((len(texts), max(len(uniq), 1)))
for r, t in enumerate(texts):
arr[r, uniq[t]] = 1.0
return arr @ arr.T
@staticmethod
def embed(texts):
import numpy as np
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
arr = np.zeros((len(texts), max(len(uniq), 1)))
for r, t in enumerate(texts):
arr[r, uniq[t]] = 1.0
return arr
import board_artefacts as ba
for mod in (blocks, ba, qa):
monkeypatch.setattr(mod, "embedding", _FakeEmb)
async def emb_ok(flow): # Board-1-Vektorpfade aus (wie board_env) — Judge-Wellen reichen
return False
monkeypatch.setattr(bi, "_emb_ok", emb_ok)
return welt

View File

@@ -0,0 +1,96 @@
"""Invarianten nach einem (Fake-)E2E-Lauf: was IMMER gelten muss, egal welches Szenario.
Nutzt bewusst eigene, schlichte Prüfungen statt Pipeline-Heuristiken (Muster qa.py) —
geteilte blinde Flecken machen den Check wertlos. Rückgabe: Liste von Verstößen,
leer = alles konsistent.
"""
import json
import database as db
from textkit import _norm_title
_LEVELS_OK = {"beginner", "advanced", "expert"}
_RELEVANZ_OK = {"relevant", "peripheral"}
def _json(path):
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
async def pruefe_invarianten(topic: str, files: dict | None = None,
mit_artefakten: bool = True) -> list[str]:
fehler: list[str] = []
subs = [dict(r) for r in await db.list_subblocks(topic)]
cons = [r for r in subs if r["status"] == "consensus"]
for r in cons:
wo = f"{r['block']}/{r['sub_title']}"
fk = None
try:
fk = json.loads(r["facts"]) if r["facts"] else None
except ValueError:
fehler.append(f"facts unparsebar: {wo}")
if not (isinstance(fk, dict) and (fk.get("key_points") or fk.get("cited_facts"))):
fehler.append(f"consensus-Sub ohne facts: {wo}")
if r["level"] not in _LEVELS_OK:
fehler.append(f"consensus-Sub ohne gültiges level: {wo}")
if r["relevance"] not in _RELEVANZ_OK:
fehler.append(f"consensus-Sub ohne relevance: {wo}")
if mit_artefakten:
art = [dict(r) for r in await db.get_sub_artefakte(topic)]
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
versorgt = {(r["block_norm"], r["sub_norm"]) for r in art}
versorgt |= {(r["block_norm"], r["sub_norm"]) for r in fragen}
lebend = {(r["block_norm"], r["sub_norm"]) for r in subs if r["status"] in ("consensus", "variant")}
for r in cons:
if r["relevance"] == "relevant" and (r["block_norm"], r["sub_norm"]) not in versorgt:
fehler.append(f"relevanter Sub ohne Frage/Artefakt: {r['block']}/{r['sub_title']}")
for bn, sn in sorted({(r["block_norm"], r["sub_norm"]) for r in art} |
{(r["block_norm"], r["sub_norm"]) for r in fragen}):
if (bn, sn) not in lebend:
fehler.append(f"Waise (Ziel-Sub existiert nicht): {bn}/{sn}")
# keine hängengebliebenen Karten
for c in await db.kanban_cards(topic):
if c["stage"] == "dead":
fehler.append(f"dead-Karte: {c['board']}/{c['card_id']}")
if files is not None:
if not files["final"].exists():
fehler.append("blocks.md fehlt")
sc = _json(files["sidecar"])
if not isinstance(sc, dict):
fehler.append("sidecar-Datei fehlt/unparsebar")
else: # Sidecar und DB-consensus müssen dieselbe Sub-Menge tragen
db_menge = {(r["block_norm"], r["sub_norm"]) for r in cons}
sc_menge = {(_norm_title(bt), _norm_title(str(s.get("title", ""))))
for bt, ss in sc.items() for s in ss if isinstance(s, dict)}
for extra in sorted(sc_menge - db_menge):
fehler.append(f"Sidecar-Sub fehlt in DB: {extra}")
for extra in sorted(db_menge - sc_menge):
fehler.append(f"DB-consensus fehlt im Sidecar: {extra}")
return fehler
async def pruefe_guide_invarianten(topic: str, format_name: str = "Guide") -> list[str]:
"""Jeder relevante consensus-Sub trägt einen Sub-Marker im Guide (Muster
guide_qa.marker_fehlend, ohne LLM)."""
import guide_qa
fehler: list[str] = []
cards = [dict(r) for r in await db.list_guide_cards(topic, format_name)]
if not cards:
return ["keine Guide-Karten"]
for c in cards:
if c["status"] != "ok" or not (c.get("md") or "").strip():
fehler.append(f"Guide-Karte nicht ok: {c['block']} ({c['status']})")
subs_rel: dict[str, set] = {}
for r in await db.list_subblocks(topic):
if r["status"] == "consensus" and r["relevance"] != "peripheral":
subs_rel.setdefault(r["block_norm"], set()).add(r["sub_norm"])
fehler += [f"Sub-Marker fehlt: {m}" for m in guide_qa.marker_fehlend(cards, subs_rel)]
return fehler

View File

@@ -1012,12 +1012,13 @@ async def test_supplement_material_mode_for_source_topics(board_env, tmp_path, m
# ── Anker-Gate: Quorum-Titel ohne Korpus-Beleg (Reader-Ko-Halluzination) ────────────
async def _anker_env(db, tmp_path, monkeypatch, titel_map):
async def _anker_env(db, tmp_path, monkeypatch, titel_map, desc_map=None):
(tmp_path / "korpus.txt").write_text("Der Graph ist zusammenhängend und endlich.", encoding="utf-8")
monkeypatch.setattr(bi, "source_folder", lambda t: tmp_path)
async def fake_members(topic, cid):
return [{"title": titel_map[cid], "description": "", "readers": ["r1", "r2"], "supplement": False}]
return [{"title": titel_map[cid], "description": (desc_map or {}).get(cid, ""),
"readers": ["r1", "r2"], "supplement": False}]
monkeypatch.setattr(bi, "_member_rows", fake_members)
monkeypatch.setattr(bi, "_rep", lambda rows: rows[0])
@@ -1028,11 +1029,12 @@ async def _anker_env(db, tmp_path, monkeypatch, titel_map):
async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
"""Titel ohne Korpus-Anker → Beleg-Judge; „nein" → rejected/kein-beleg.
"""Titel ohne Korpus-Anker, aber mit Evidenz → Beleg-Judge; „nein" → rejected/kein-beleg.
Titel MIT Anker geht ohne Judge nach naming."""
db = testdb
ctx, cards = await _anker_env(db, tmp_path, monkeypatch,
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"})
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"},
{"c2": "Der Graph ist endlich."}) # Evidenz da → Judge entscheidet
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout):
assert "Brückenproblem" in prompt and "Zusammenhang" not in prompt # nur der Anker-lose
@@ -1045,11 +1047,27 @@ async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
assert c2["stage"] == "rejected" and c2["payload"]["reason"] == "kein-beleg"
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
"""Judge-Ausfall → Titel bleibt (2-Reader-Rückhalt)."""
async def test_anker_gate_leeres_pack_hart_nein(testdb, tmp_path, monkeypatch):
"""KEIN distinktives Token im Korpus → deterministisch rejected, Judge läuft NICHT
(der Judge winkte 3 Kanon-Titel auf Schein-Auszügen durch)."""
db = testdb
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"})
async def never_slot(*a, **kw):
raise AssertionError("Judge darf bei leerem Evidence-Pack nicht laufen")
monkeypatch.setattr(bi, "run_single_slot", never_slot)
await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), cards)
c9 = await db.kanban_get_card(TOPIC, B, "c9")
assert c9["stage"] == "rejected" and c9["payload"]["reason"] == "kein-beleg"
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
"""Judge-Ausfall bei VORHANDENER Evidenz → Titel bleibt (2-Reader-Rückhalt)."""
db = testdb
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"},
{"c9": "Der Graph ist endlich."})
async def broken_slot(*a, **kw):
return "failed", None
@@ -1063,3 +1081,103 @@ def test_hat_anker_ziffern_suffix():
assert bi._hat_anker("ΔTSP1-Algorithmus", ctoks) # tsp1 → tsp
assert not bi._hat_anker("Königsberger Brückenproblem", ctoks)
assert not bi._hat_anker("Algorithmus Verfahren", ctoks) # nur Stopwörter → kein Anker
async def test_reset_subblocks_loescht_globale_dateien(testdb, tmp_path):
"""Reset auf Spalte subblocks: DB-Spiegel UND globale Sidecar-Dateien + ab-*-Resume-Slots
weg — Reste des Vor-Laufs würden sonst in den frischen Lauf zurückmergen."""
db = testdb
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "artefacts", {"title": "Alpha"})
await db.put_subblock(TOPIC, "alpha", "s1", "Alpha", "S1", status="consensus")
arbeit = tmp_path / "arbeit"
(arbeit / "ab-alpha").mkdir(parents=True)
(arbeit / "ab-alpha" / "facts.json").write_text("{}", encoding="utf-8")
files = {"arbeit": arbeit}
for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"):
files[k] = tmp_path / f"{k}.json"
files[k].write_text("{}", encoding="utf-8")
moved = await bi.reset_board_from_stage(TOPIC, "artefacts", "subblocks", files)
assert moved == 1
assert not await db.list_subblocks(TOPIC)
assert not (arbeit / "ab-alpha").exists()
assert all(not files[k].exists() for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"))
# ── Naming-Abstraktion: freier Name nur mit Anker ───────────────────────────────────
def test_naming_schema_varianten():
assert bi._naming_schema({"best": 2}, 3) == (2, None, False)
assert bi._naming_schema({"best": 1, "name": "Kurzer Titel"}, 3) == (1, "Kurzer Titel", False)
assert bi._naming_schema({"ok": True}, 3) == (None, None, True)
assert bi._naming_schema({"best": 9}, 3) is None
assert bi._naming_schema({"best": 1, "name": "x" * 90}, 3) == (1, None, False) # zu lang
def test_name_verankert_thema_subset():
rows = [{"title": "Bubblesort-Schleife und Tauschoperation", "description": "innere Schleife"},
{"title": "Bubblesort Durchläufe", "description": ""}]
assert bi._name_verankert("Bubblesort Tauschoperation", rows, None)
assert not bi._name_verankert("Königsberger Brückenproblem", rows, None) # fremde Begriffe
assert not bi._name_verankert("und der", rows, None) # nur Stopwörter → kein Anker
def test_name_verankert_korpus():
ctoks = {"partition", "problem", "vollständigkeit"}
rows = [{"title": "irrelevant", "description": ""}]
assert bi._name_verankert("Partition-Problem", rows, ctoks)
assert not bi._name_verankert("Rucksackproblem Optimierung", rows, ctoks)
async def test_naming_vergibt_verankerten_namen(testdb, tmp_path, monkeypatch):
"""Judge liefert best+name; verankerter Name gewinnt, unverankerter fällt auf Member zurück."""
db = testdb
antwort = {"val": {"best": 1, "name": "Bubblesort Grundprinzip"}}
async def fake_members(topic, cid):
return [{"norm": "bubblesort - grundprinzip und ablauf (kap. 2)",
"title": "Bubblesort - Grundprinzip und Ablauf (Kap. 2)",
"description": "Sortieren durch Tauschen", "readers": ["r1"], "sources": []},
{"norm": "sortieren durch tauschen", "title": "Sortieren durch Tauschen",
"description": "", "readers": ["r2"], "sources": []}]
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
import json as _json
return "ok", payload((0, _json.dumps(antwort["val"]), ""))
monkeypatch.setattr(bi, "_member_rows", fake_members)
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
await db.kanban_upsert_card(TOPIC, B, "c1", "cluster", "naming", {})
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
await bi._name_one(ctx, _mk_flow(tmp_path), {"card_id": "c1", "payload": {}})
card = await db.kanban_get_card(TOPIC, B, "c1")
assert card["payload"]["title"] == "Bubblesort Grundprinzip"
# unverankerter Name → Member-Titel gewinnt
antwort["val"] = {"best": 2, "name": "Vergleichsbasierte Sortierverfahren"}
await db.kanban_upsert_card(TOPIC, B, "c2", "cluster", "naming", {})
await bi._name_one(ctx, _mk_flow(tmp_path), {"card_id": "c2", "payload": {}})
card = await db.kanban_get_card(TOPIC, B, "c2")
assert card["payload"]["title"] == "Sortieren durch Tauschen"
async def test_namecheck_ok_behaelt_titel(testdb, tmp_path, monkeypatch):
"""Check-Judge bestätigt mit ok:true → Titel und Beschreibung bleiben unverändert."""
db = testdb
async def fake_members(topic, cid):
return [{"norm": "a", "title": "A", "description": "da", "readers": ["r1"], "sources": []},
{"norm": "b", "title": "B", "description": "db", "readers": ["r2"], "sources": []}]
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
assert "Eigener Titel" in prompt # current_title steht im Prompt
return "ok", payload((0, '{"ok": true}', ""))
monkeypatch.setattr(bi, "_member_rows", fake_members)
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
payload = {"title": "Eigener Titel", "description": "Eigene Beschreibung", "main_norm": "a"}
await db.kanban_upsert_card(TOPIC, B, "c9", "cluster", "naming_check", payload)
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
await bi._namecheck_one(ctx, _mk_flow(tmp_path), {"card_id": "c9", "payload": payload})
block = await db.kanban_get_card(TOPIC, B, "b-c9")
assert block["payload"]["title"] == "Eigener Titel"
assert block["payload"]["description"] == "Eigene Beschreibung"

View File

@@ -0,0 +1,197 @@
"""E2E über die ECHTE Engine mit Fake-Agenten: kompletter Generierungspfad in Sekunden.
Anders als test_board_inventory (dort sind die Block-Funktionen gefakt) läuft hier alles
bis run_agent echt — _race, Quorum, Panels, Konsolidierung, Cross-Block, QA-Gate.
"""
import asyncio
import pytest
import board_inventory as bi
from pipeline import GenContext
from tests.invarianten import pruefe_invarianten, pruefe_guide_invarianten
TOPIC = "t"
def _files(tmp_path):
work = tmp_path / "arbeit"
work.mkdir(exist_ok=True)
return {"arbeit": work, "final": tmp_path / "blocks.md",
"sub_roh": tmp_path / "sub_roh.json", "sidecar": tmp_path / "subblocks.json",
"facts": tmp_path / "facts.json", "question_pattern": tmp_path / "question_pattern.json",
"artefakte": tmp_path / "artefakte.json", "outline": tmp_path / "outline.json",
"outline_slots": [tmp_path / f"outline-{i}.json" for i in (1, 2, 3)],
"research": [work / f"research-{i}.md" for i in (1, 2, 3, 4, 5)]}
async def _lauf(tmp_path, research=True, qa_force=False, timeout=120):
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
files = _files(tmp_path)
ok = await asyncio.wait_for(
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "",
research=research, qa_force=qa_force), timeout=timeout)
return ok, files
async def test_e2e_thema_vollpfad(fake_welt, testdb, tmp_path):
"""Research → Inventar → QA-Gate → Artefakte → Finalize, alle Schichten echt."""
ok, files = await _lauf(tmp_path)
assert ok
db = testdb
done = [c for c in await db.kanban_cards(TOPIC, board="inventory", stage="done_block")]
titel = {c["payload"]["title"] for c in done}
assert titel == {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"}
# Cross-Block-Dublette: „Gemeinsamer Grundbegriff" überlebt in genau EINEM Block
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "variant"]
fehler = await pruefe_invarianten(TOPIC, files)
assert fehler == []
async def test_e2e_guide(fake_welt, testdb, tmp_path):
"""Auf den Vollpfad folgt der Guide-Bau — Gate/Coverage/Lese-Stages laufen echt."""
import guide_board
ok, files = await _lauf(tmp_path)
assert ok
db = testdb
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
entries = {i: f"{c['payload']['title']}{c['payload'].get('description', '')}"
for i, c in enumerate(done, 1)}
chapters = await asyncio.wait_for(
guide_board.run_guide_board("g-e2e", TOPIC, "Guide", entries, "", "claude",
tmp_path / "guides" / "Guide.json"), timeout=120)
assert chapters is not None
assert await pruefe_guide_invarianten(TOPIC) == []
async def test_e2e_rerun_idempotent(fake_welt, testdb, tmp_path):
"""Zweiter Lauf (Continue, research=False) hinterlässt keine Waisen/Reste."""
ok, files = await _lauf(tmp_path)
assert ok
db = testdb
vorher = {(r["block_norm"], r["sub_norm"], r["status"])
for r in await db.list_subblocks(TOPIC)}
ok2, _f = await _lauf(tmp_path, research=False)
assert ok2
nachher = {(r["block_norm"], r["sub_norm"], r["status"])
for r in await db.list_subblocks(TOPIC)}
assert nachher == vorher
assert await pruefe_invarianten(TOPIC, files) == []
@pytest.mark.parametrize("stoerung", [
{"muster": r"-sub-crossblock-.*-j1$", "modus": "fehler", "mal": 3}, # Ersatzrichter jE
{"muster": r"-sub-konsolidierung-.*-j1$", "modus": "garbage", "mal": 1}, # Retry heilt
{"muster": r"-facts-c\d+$", "modus": "fehler", "mal": 1}, # Slot-Restart
{"muster": r"-research-2$", "modus": "fehler", "mal": 3}, # 1 Producer tot
])
async def test_e2e_stoerungen_flow_endet(fake_welt, testdb, tmp_path, stoerung):
"""Einzel-Ausfälle dürfen weder den Flow stoppen noch Invarianten reißen."""
fake_welt.stoerungen.append(dict(stoerung, rest=stoerung["mal"]))
ok, files = await _lauf(tmp_path)
assert ok
assert await pruefe_invarianten(TOPIC, files) == []
async def test_e2e_crossblock_dissent_failopen(fake_welt, testdb, tmp_path):
"""j1 sagt a, j2 sagt b, j3 fällt aus → Paar bleibt (fail-open), Rest konsistent."""
fake_welt.stoerungen += [
{"muster": r"-sub-crossblock-.*-j2$", "modus": "antwort",
"antwort": '{"pairs": {"1": "b"}}', "mal": 1, "rest": 1},
{"muster": r"-sub-crossblock-.*-j3$", "modus": "fehler", "mal": 3, "rest": 3},
]
ok, files = await _lauf(tmp_path)
assert ok
db = testdb
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "consensus"] # kein Fold
assert await pruefe_invarianten(TOPIC, files) == []
async def test_e2e_inblock_gruppe_faltet(fake_welt, testdb, tmp_path):
"""Welt-Regel: „Alpha Eigenschaften" faltet unter „Definition Alpha" — beide Judges
liefern die Gruppe, der Verlierer wird variant, seine facts wandern zum Gewinner."""
fake_welt.gruppen.append(("Definition Alpha", ["Alpha Eigenschaften"]))
ok, files = await _lauf(tmp_path)
assert ok
db = testdb
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha-konzept")}
assert rows.get("alpha eigenschaften") == "variant"
assert rows.get("definition alpha") == "consensus"
assert await pruefe_invarianten(TOPIC, files) == []
async def test_e2e_gate_vollinventur_ohne_fix(fake_welt, testdb, tmp_path):
"""Gate-Judge liefert eine Voll-Inventur (belegte Claims mit „Belegt…"-Grund) —
der Schema-Filter wirft sie raus, es läuft KEIN Fakten-Fix."""
import json
antwort = json.dumps({"claims": [
{"text": "Aussage 1", "grund": "Belegt durch Quelle", "urteil": "unbelegt"},
{"text": "Aussage 2", "grund": "Belegt durch Fakten", "urteil": "unbelegt"},
{"text": "Aussage 3", "grund": "Belegt: steht im Skript", "urteil": "unbelegt"}]})
fake_welt.stoerungen.append({"muster": r"-gate-", "modus": "antwort",
"antwort": antwort, "mal": 99, "rest": 99})
import guide_board
ok, _files = await _lauf(tmp_path)
assert ok
db = testdb
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
entries = {i: c["payload"]["title"] for i, c in enumerate(done, 1)}
chapters = await asyncio.wait_for(
guide_board.run_guide_board("g-vi", TOPIC, "Guide", entries, "", "claude",
tmp_path / "guides" / "Guide.json"), timeout=120)
assert chapters is not None
assert not any("-gatefix-" in k for k in fake_welt.calls)
async def test_e2e_echtheits_flattern_gestoppt(fake_welt, testdb, tmp_path):
"""QA-Pass 1 flaggt alle Blöcke als unecht (Judge-Flattern) — der Bestätiger-Pass
widerspricht, die Gate-Note bleibt sauber, der Flow läuft durch."""
import json
fake_welt.stoerungen.append({"muster": r"^qa-t-bausteine-0$", "modus": "antwort",
"antwort": json.dumps({"relevant": {"1": "nein", "2": "nein", "3": "nein"}}),
"mal": 1, "rest": 1})
ok, files = await _lauf(tmp_path)
assert ok # Gate hat nicht pausiert — der Zufalls-Verdacht wurde nicht bestätigt
assert any("bausteine-b2" in k for k in fake_welt.calls)
assert await pruefe_invarianten(TOPIC, files) == []
async def test_e2e_uni_anker_gate(fake_welt, testdb, tmp_path, monkeypatch):
"""uni-Modus mit Mini-Korpus: der Kanon-Titel ohne Korpus-Anker wird deterministisch
rejected (leeres Evidence-Pack), die belegten Blöcke laufen durch; QA misst gegen
den echten Korpus."""
import blocks as blx
fake_welt.bloecke["Kanon-Klassiker"] = {
"beschreibung": "Beruehmtes Lehrbuchproblem", "subs": ["Klassiker Detail"]}
korpus = tmp_path / "korpus"
korpus.mkdir()
zeilen = []
for t, b in fake_welt.bloecke.items():
if t == "Kanon-Klassiker":
continue # kommt bewusst NICHT im Material vor
zeilen.append(f"Kapitel {t}: {b['beschreibung']}. " +
" ".join(f"Wir behandeln {s}." for s in b["subs"]))
(korpus / "skript.txt").write_text("\n\n".join(zeilen), encoding="utf-8")
monkeypatch.setattr(bi, "source_folder", lambda t: korpus)
monkeypatch.setattr(blx, "source_folder", lambda t: korpus)
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
files = _files(tmp_path)
ok = await asyncio.wait_for(
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "uni", "location": str(korpus)},
korpus, "", research=True, qa_force=True), timeout=120)
assert ok
db = testdb
alle = [dict(c) for c in await db.kanban_cards(TOPIC, board="inventory")]
assert any(c["stage"] == "rejected" and c["payload"].get("title") == "Kanon-Klassiker"
for c in alle)
assert not any(c["kind"] == "block" and c["payload"].get("title") == "Kanon-Klassiker"
for c in alle) # nie zum Block geworden
done = {c["payload"].get("title") for c in alle
if c["kind"] == "block" and c["stage"] == "done_block"}
assert {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"} <= done

View File

@@ -18,8 +18,16 @@ def test_ziele_schema():
def test_gate_schema():
assert gb._gate_schema({"ok": True}) == []
claims = gb._gate_schema({"claims": [{"text": "Falsch", "grund": "fehlt"}]})
assert claims == [{"text": "Falsch", "grund": "fehlt"}]
assert claims == [{"text": "Falsch", "grund": "fehlt", "urteil": "unbelegt"}]
assert gb._gate_schema({}) is None
# urteil "falsch" wird durchgereicht, alles andere defaultet auf unbelegt
claims = gb._gate_schema({"claims": [{"text": "A", "grund": "widerspricht", "urteil": "FALSCH"},
{"text": "B", "grund": "x", "urteil": "quatsch"}]})
assert [c["urteil"] for c in claims] == ["falsch", "unbelegt"]
# Voll-Inventur-Rauschen: als belegt begründete Einträge fliegen raus
claims = gb._gate_schema({"claims": [{"text": "A", "grund": "Belegt durch Quelle X"},
{"text": "B", "grund": "nicht ableitbar"}]})
assert [c["text"] for c in claims] == ["B"]
def test_coverage_schema():
@@ -266,7 +274,7 @@ async def test_lese_check_text_sink(testdb, tmp_path, monkeypatch):
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
env = gb._Env(None, "g-l", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
md = ("<!-- kapitel: K -->\n<!-- section: Alpha -->\n<!-- compact -->\n- x\n"
"<!-- ausführlich -->\nText.")
"<!-- ausführlich -->\n" + "Text im Längen-Rahmen. " * 20) # ~460 Z. — kein Längen-Trigger
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
seen = {}
@@ -330,3 +338,49 @@ async def test_load_subblocks_defaultet_levellose(testdb):
subs = await _load_subblocks("t")
by_title = {s["title"]: s["level"] for s in subs["Block"]}
assert by_title == {"Mit Level": "beginner", "Ohne Level": "advanced"}
async def test_fakten_gate_falsch_claim_erzwingt_fix(testdb, tmp_path, monkeypatch):
"""Ein einzelner FALSCH-Claim läuft in den Fix, auch unter GATE_FIX_MIN —
ein durchgerutschter kostete den Guide 1.5 QA-Punkte."""
db = testdb
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
env = gb._Env(None, "g-gf", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
md = "<!-- section: Alpha -->\n<!-- ausführlich -->\nText."
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
keys = []
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
keys.append(key)
if "-gate-" in key:
return gb.OK, [{"text": "c1", "grund": "widerspricht", "urteil": "falsch"}]
return gb.FAILED, None # Fix-Agent liefert nichts — Text bleibt, aber der Call MUSS kommen
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
assert await gb._stage_fakten_gate(env, card)
assert any("-gatefix-" in k for k in keys)
async def test_laengen_trigger_startet_lesefix(testdb, tmp_path, monkeypatch):
"""Ausführlich-Teil über der Obergrenze → deterministisches Längen-Problem
mit hartem Zeichenziel landet im Lese-Fix-Auftrag."""
db = testdb
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
env = gb._Env(None, "g-lz", TOPIC, FMT, "", tmp_path / "Guide.json",
{"Alpha": [{"title": "S1", "level": "beginner", "relevance": "relevant"}]},
{}, "(q)", "spec")
md = ("<!-- section: Alpha -->\n<!-- compact -->\n- x\n<!-- ausführlich -->\n"
+ "Viel zu langer Sockeltext. " * 80) # ~2160 Z./Sub > 1200×0.9
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
seen = {}
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
if "-lesefix-" in key:
seen["tasks"] = prompt
return gb.FAILED, None
return gb.OK, payload((0, '{"ok": true}', "")) # Lese-Check: keine Probleme
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
monkeypatch.setattr(gb, "READABILITY_ACTIVE", False)
assert await gb._stage_lesbarkeit(env, card)
assert "Länge" in seen["tasks"] and str(gb._writer_budget(1)) in seen["tasks"]

View File

@@ -519,3 +519,90 @@ async def test_finalize_defaultet_level_nachzuegler(testdb, tmp_path):
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
row = next(r for r in await db.list_subblocks(TOPIC, "alpha"))
assert row["level"] == "advanced" and row["relevance"] == "relevant"
async def test_finalize_loescht_stale_consensus(testdb, tmp_path):
"""Alt-consensus-Rows, die der Lauf-Sidecar nicht mehr trägt, fliegen raus —
variant-Rows bleiben (QA liest die Status). Wurzel der 25 Board-2-losen Waisen."""
db = testdb
await db.put_subblock(TOPIC, "alpha", "alt-rest", "Alpha", "Alt-Rest", status="consensus")
await db.put_subblock(TOPIC, "alpha", "alte-variante", "Alpha", "Alte Variante", status="variant")
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "finalize", {})
files = {k: tmp_path / f"{k}.json" for k in
("sub_roh", "facts", "sidecar", "question_pattern", "artefakte")}
card = {"card_id": "alpha", "payload": {
"title": "Alpha", "raw": {"Alpha": ["Neu"]}, "facts": {},
"sidecar": {"Alpha": [{"title": "Neu", "level": "beginner"}]},
"pattern": {}, "artefacts": {}}}
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
assert rows == {"neu": "consensus", "alte-variante": "variant"}
async def test_crossblock_chunking_faltet_global(testdb, tmp_path, monkeypatch):
"""Paare werden gechunkt beurteilt (ein Hänger blockiert nur noch seinen Chunk);
die Verdicts falten global über alle Chunks."""
db = testdb
flow = Flow(TOPIC, work_dir=tmp_path)
cards = []
for bnorm, subs in (("alpha", ["Gleiche Aussage", "Zweite gleiche Aussage", "Nur in Alpha"]),
("beta", ["Gleiche Aussage", "Zweite gleiche Aussage"])):
payload = {"title": bnorm.title(),
"raw": {bnorm.title(): list(subs)},
"sidecar": {bnorm.title(): [{"title": s, "level": "beginner"} for s in subs]},
"facts": {bnorm.title(): {blocks._norm_title(s): {"key_points": []} for s in subs}}}
await db.kanban_upsert_card(TOPIC, "artefacts", bnorm, "ablock", "konsolidierung", payload)
await _seed_block(db, bnorm, subs)
cards.append({"card_id": bnorm, "payload": payload})
monkeypatch.setattr(ba, "embedding", _FakeEmb)
monkeypatch.setattr(ba, "CROSS_CHUNK_PAARE", 1) # 2 Paare → 2 Chunks
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "j2": {"pairs": {"1": "a"}}})
monkeypatch.setattr(ba, "run_single_slot", fake)
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
assert len(fake.calls) == 4 # 2 Chunks × j1/j2
beta = await db.kanban_get_card(TOPIC, "artefacts", "beta")
assert beta["payload"]["raw"].get("Beta", []) == [] # beide Dubletten global gefaltet
async def test_facts_nachfass_holt_nur_fehlende(testdb, tmp_path, monkeypatch):
"""Nachfass ruft den slim-Facts-Lauf NUR mit den facts-losen Subs und merged die Funde;
Vorhandenes bleibt unberührt, Subs werden nie verworfen."""
gesehen = {}
async def fake_facts_block(ctx, set_p, files, raw, q, folder, instructions, ns="", lbl="",
sources=None, slim=False):
gesehen["raw"] = raw
gesehen["slim"] = slim
return ({"Alpha": {"ohne beleg": {"key_points": ["kp neu"]},
"mit beleg": {"key_points": ["DARF NICHT GEWINNEN"]}}},
{"Alpha": {"ohne beleg"}}) # discard-Urteil wird ignoriert
monkeypatch.setattr(blocks, "_facts_block", fake_facts_block)
raw = {"Alpha": ["Mit Beleg", "Ohne Beleg"]}
facts_map = {"Alpha": {"mit beleg": {"key_points": ["kp alt"]}}}
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
n = await blocks._facts_nachfass(ctx, {"arbeit": tmp_path}, raw, facts_map, {}, None)
assert n == 1
assert gesehen["slim"] and gesehen["raw"] == {"Alpha": ["Ohne Beleg"]}
assert facts_map["Alpha"]["ohne beleg"]["key_points"] == ["kp neu"]
assert facts_map["Alpha"]["mit beleg"]["key_points"] == ["kp alt"]
assert raw["Alpha"] == ["Mit Beleg", "Ohne Beleg"] # kein Verwurf
async def test_levels_merge_fuzzy_match(testdb, tmp_path, monkeypatch):
"""Levels-Agent paraphrasiert den Sub-Titel → facts hängen trotzdem am Sidecar-Eintrag
(eindeutiger Präfix-Match statt stillem Grounding-Verlust)."""
db = testdb
payload = {"title": "Alpha", "raw": {"Alpha": ["Marker Regel: Details dazu"]},
"facts": {"Alpha": {blocks._norm_title("Marker Regel: Details dazu"):
{"key_points": ["kp"]}}}}
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "levels", payload)
cards = [{"card_id": "alpha", "payload": payload}]
async def fake_levels_block(ctx, set_p, files, raw, instructions, ns="", lbl=""):
return {"Alpha": [{"title": "Marker Regel", "level": "beginner"}]} # gekürzter Titel
monkeypatch.setattr(ba, "_levels_block", fake_levels_block)
await ba._proc_levels(_ctx(), Flow(TOPIC, work_dir=tmp_path), {"arbeit": tmp_path}, "", cards)
card = await db.kanban_get_card(TOPIC, "artefacts", "alpha")
assert card["payload"]["sidecar"]["Alpha"][0]["facts"] == {"key_points": ["kp"]}

View File

@@ -214,3 +214,37 @@ def test_fremd_digit_suffix_tolerant():
b = [{"title": "ΔTSP1-Algorithmus", "description": "", "sources": []},
{"title": "Quantencomputer", "description": "", "sources": []}]
assert qa.fremd(b, corpus) == ["Quantencomputer"]
async def test_unecht_braucht_doppelt_nein(testdb, tmp_path, monkeypatch):
"""Echtheits-Urteil zählt nur nach Bestätiger-Pass: der Einzel-Judge flaggte pro Lauf
andere Blöcke und pendelte die Note (aak: 9.3↔10.0 bei identischem Bestand)."""
db = testdb
for cid, titel in (("b1", "Wackelkandidat"), ("b2", "Zufallstreffer"), ("b3", "Solide")):
await db.kanban_upsert_card("t", "inventory", cid, "block", "done_block",
{"title": titel, "description": "d"})
monkeypatch.setattr(qa, "QA_DIR", tmp_path)
async def fake_verdicts(template, topic, key, items):
if template != "QA-Bausteine":
return {}
if key.startswith("bausteine-b2"): # Bestätiger sieht nur die Geflaggten
assert len(items) == 2
return {1: "nein", 2: "ja"} # nur der erste wird bestätigt
return {1: "nein", 2: "nein", 3: "ja"} # Pass 1 flaggt zwei
monkeypatch.setattr(qa, "_llm_verdicts", fake_verdicts)
report = await qa.qa_report("t", llm=True)
assert report["unecht"] == ["Wackelkandidat"]
async def test_topic_delete_entfernt_qa_ordner(testdb, tmp_path, monkeypatch):
"""DELETE /topics räumt auch storage/qa/<topic>/ — Reports gehören zum Topic."""
import routes
monkeypatch.setattr(routes, "topic_dir", lambda t: tmp_path / "topics" / t)
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
qdir = tmp_path / "qa" / "t"
qdir.mkdir(parents=True)
(qdir / "alt.json").write_text("{}", encoding="utf-8")
await routes.remove_topic("t")
assert not qdir.exists()

View File

@@ -226,10 +226,8 @@ async def test_outline_review_moves_block(testdb, tmp_path, monkeypatch):
out = plan_a
elif key.endswith("outline-review"):
out = review_out["val"]
if out is not None:
m = re.search(r"(/\S+\.json)", prompt)
with open(m.group(1), "w", encoding="utf-8") as f:
json.dump(out, f)
if out is not None: # neue Semantik: Antwort als TEXT, der Sink persistiert
return "ok", payload((0, json.dumps(out), ""))
return "ok", payload(None)
monkeypatch.setattr(blx, "run_single_slot", fake_slot)

113
backend/tests/test_train.py Normal file
View File

@@ -0,0 +1,113 @@
"""Training-Harness: Registry↔config-Konsistenz, ENV-Override, Trainer-Logik (Stub-Runner)."""
import json
import subprocess
import sys
from pathlib import Path
import config
import train
import train_params
from train import Trainer, score
BACKEND = Path(__file__).resolve().parent.parent
def test_registry_spiegelt_config():
"""Jeder Registry-Parameter existiert in config mit identischem Default und
flow-sicheren Rändern — sonst optimiert der Trainer Phantome."""
for name, p in train_params.PARAMS.items():
assert getattr(config, name, None) == p["default"], name
assert p["min"] <= p["default"] <= p["max"], name
assert p["step"] > 0, name
def test_creator_params_override_wirkt_im_subprozess():
out = subprocess.run(
[sys.executable, "-c", "import config; print(config.FACTS_CHUNK_SUBS, config.TIMEOUTS['subblock_check'][0])"],
capture_output=True, text=True, cwd=BACKEND,
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"FACTS_CHUNK_SUBS": 6, "TIMEOUT_subblock_check_base": 77}'})
assert out.stdout.split() == ["6", "77"], out.stderr
def test_creator_params_unbekannter_name_bricht_ab():
out = subprocess.run([sys.executable, "-c", "import config"],
capture_output=True, text=True, cwd=BACKEND,
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"GIBT_ES_NICHT": 1}'})
assert out.returncode != 0 and "GIBT_ES_NICHT" in out.stderr
def _metrics(note=8.0, dauer=10.0, tokens=1_000_000, **quoten):
return {"note": note, "quoten": quoten, "quoten_artefakte": {},
"dauer_min": dauer, "tokens": {"input": tokens, "output": 0}, "agents": {}}
def _stub_runner(antworten):
"""params-abhängige Metriken; zählt echte Aufrufe (Cache-Treffer zählen nicht)."""
calls = []
async def runner(params, thema):
calls.append((dict(params), thema[0]))
for muster, m in antworten:
if muster(params):
return dict(m)
return _metrics()
runner.calls = calls
return runner
async def test_screening_filtert_rauschen(tmp_path):
"""Nur Parameter mit Effekt über der Rausch-Schwelle kommen in die Feinphase;
ein bestätigter Gewinner wird übernommen."""
wirksam = "FACTS_CHUNK_SUBS"
runner = _stub_runner([
(lambda p: p.get(wirksam) == 8, _metrics(note=9.5, dauer=8.0)), # klar besser
])
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
best = await t.run()
assert best.get(wirksam) == 8
# kein anderer Parameter übernommen (alle anderen Δ=0 < Schwelle)
assert set(best) == {wirksam}
async def test_uebernahme_braucht_bestaetigung(tmp_path):
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird verworfen."""
zustand = {"mal": 0}
async def runner(params, thema):
if params.get("FACTS_CHUNK_SUBS") == 8:
zustand["mal"] += 1
return _metrics(note=9.5) if zustand["mal"] == 1 else _metrics(note=8.0)
return _metrics()
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
best = await t.run()
assert best == {}
async def test_cache_resume_wiederholt_keine_trials(tmp_path):
runner = _stub_runner([])
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
await t.run()
erste = len(runner.calls)
t2 = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
await t2.run()
assert len(runner.calls) == erste # alles aus trials.jsonl bedient
async def test_budget_stoppt(tmp_path):
runner = _stub_runner([])
t = Trainer(tmp_path / "s", max_trials=3, max_stunden=1, runner=runner)
await t.run()
assert len(runner.calls) <= 3
def test_score_richtungen():
basis = _metrics()
besser = _metrics(note=9.0)
teurer = _metrics(dauer=20.0, tokens=2_000_000)
assert score(besser, basis) > score(basis, basis)
assert score(teurer, basis) < score(basis, basis)
mit_befunden = _metrics(fremd=0.2, luecken=0.1)
assert score(mit_befunden, basis) < score(basis, basis)