update
This commit is contained in:
@@ -16,3 +16,69 @@ async def testdb(tmp_path, monkeypatch):
|
||||
await database.init_db()
|
||||
yield database
|
||||
await database.close_db()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fake_welt(testdb, tmp_path, monkeypatch):
|
||||
"""E2E ohne LLM: run_agent überall durch die Fake-Welt ersetzt, Tempo-Bremsen raus.
|
||||
Alle echten Schichten (_race, Quorum, Panels, Producer, QA-Gate) laufen mit."""
|
||||
import agents
|
||||
import blocks
|
||||
import board_inventory as bi
|
||||
import guide
|
||||
import kanban
|
||||
import pipeline
|
||||
import qa
|
||||
import repair
|
||||
from fake_agents import Welt
|
||||
|
||||
welt = Welt()
|
||||
|
||||
async def fake_run_agent(agent_key, prompt, timeout, provider="claude", role="fast",
|
||||
capabilities="none", lane="batch", scope=None, on_line=None, label=""):
|
||||
return welt.respond(agent_key, prompt, capabilities)
|
||||
|
||||
for mod in (agents, pipeline, blocks, guide, repair):
|
||||
monkeypatch.setattr(mod, "run_agent", fake_run_agent)
|
||||
|
||||
# Tempo: grace/poll/backoff bremsen echte Läufe, nicht den Fake
|
||||
monkeypatch.setattr(blocks, "CONSENSUS_GRACE", 0)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
monkeypatch.setattr(kanban, "RETRY_BACKOFF", 0.05)
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
|
||||
import guide_board
|
||||
monkeypatch.setattr(guide_board, "READABILITY_ACTIVE", False) # kein Modell-Load im Test
|
||||
import asyncio as _aio
|
||||
monkeypatch.setattr(bi, "_ingest_lock", _aio.Lock()) # Modul-Lock klebt sonst am Vortest-Loop
|
||||
|
||||
class _FakeEmb: # identischer Text → cos 1.0, sonst 0.0 (deterministisch, ohne Modell)
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def embed_sims(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr @ arr.T
|
||||
|
||||
@staticmethod
|
||||
def embed(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr
|
||||
|
||||
import board_artefacts as ba
|
||||
for mod in (blocks, ba, qa):
|
||||
monkeypatch.setattr(mod, "embedding", _FakeEmb)
|
||||
|
||||
async def emb_ok(flow): # Board-1-Vektorpfade aus (wie board_env) — Judge-Wellen reichen
|
||||
return False
|
||||
monkeypatch.setattr(bi, "_emb_ok", emb_ok)
|
||||
return welt
|
||||
|
||||
96
backend/tests/invarianten.py
Normal file
96
backend/tests/invarianten.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""Invarianten nach einem (Fake-)E2E-Lauf: was IMMER gelten muss, egal welches Szenario.
|
||||
|
||||
Nutzt bewusst eigene, schlichte Prüfungen statt Pipeline-Heuristiken (Muster qa.py) —
|
||||
geteilte blinde Flecken machen den Check wertlos. Rückgabe: Liste von Verstößen,
|
||||
leer = alles konsistent.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import database as db
|
||||
from textkit import _norm_title
|
||||
|
||||
_LEVELS_OK = {"beginner", "advanced", "expert"}
|
||||
_RELEVANZ_OK = {"relevant", "peripheral"}
|
||||
|
||||
|
||||
def _json(path):
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
async def pruefe_invarianten(topic: str, files: dict | None = None,
|
||||
mit_artefakten: bool = True) -> list[str]:
|
||||
fehler: list[str] = []
|
||||
subs = [dict(r) for r in await db.list_subblocks(topic)]
|
||||
cons = [r for r in subs if r["status"] == "consensus"]
|
||||
|
||||
for r in cons:
|
||||
wo = f"{r['block']}/{r['sub_title']}"
|
||||
fk = None
|
||||
try:
|
||||
fk = json.loads(r["facts"]) if r["facts"] else None
|
||||
except ValueError:
|
||||
fehler.append(f"facts unparsebar: {wo}")
|
||||
if not (isinstance(fk, dict) and (fk.get("key_points") or fk.get("cited_facts"))):
|
||||
fehler.append(f"consensus-Sub ohne facts: {wo}")
|
||||
if r["level"] not in _LEVELS_OK:
|
||||
fehler.append(f"consensus-Sub ohne gültiges level: {wo}")
|
||||
if r["relevance"] not in _RELEVANZ_OK:
|
||||
fehler.append(f"consensus-Sub ohne relevance: {wo}")
|
||||
|
||||
if mit_artefakten:
|
||||
art = [dict(r) for r in await db.get_sub_artefakte(topic)]
|
||||
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
|
||||
versorgt = {(r["block_norm"], r["sub_norm"]) for r in art}
|
||||
versorgt |= {(r["block_norm"], r["sub_norm"]) for r in fragen}
|
||||
lebend = {(r["block_norm"], r["sub_norm"]) for r in subs if r["status"] in ("consensus", "variant")}
|
||||
for r in cons:
|
||||
if r["relevance"] == "relevant" and (r["block_norm"], r["sub_norm"]) not in versorgt:
|
||||
fehler.append(f"relevanter Sub ohne Frage/Artefakt: {r['block']}/{r['sub_title']}")
|
||||
for bn, sn in sorted({(r["block_norm"], r["sub_norm"]) for r in art} |
|
||||
{(r["block_norm"], r["sub_norm"]) for r in fragen}):
|
||||
if (bn, sn) not in lebend:
|
||||
fehler.append(f"Waise (Ziel-Sub existiert nicht): {bn}/{sn}")
|
||||
|
||||
# keine hängengebliebenen Karten
|
||||
for c in await db.kanban_cards(topic):
|
||||
if c["stage"] == "dead":
|
||||
fehler.append(f"dead-Karte: {c['board']}/{c['card_id']}")
|
||||
|
||||
if files is not None:
|
||||
if not files["final"].exists():
|
||||
fehler.append("blocks.md fehlt")
|
||||
sc = _json(files["sidecar"])
|
||||
if not isinstance(sc, dict):
|
||||
fehler.append("sidecar-Datei fehlt/unparsebar")
|
||||
else: # Sidecar und DB-consensus müssen dieselbe Sub-Menge tragen
|
||||
db_menge = {(r["block_norm"], r["sub_norm"]) for r in cons}
|
||||
sc_menge = {(_norm_title(bt), _norm_title(str(s.get("title", ""))))
|
||||
for bt, ss in sc.items() for s in ss if isinstance(s, dict)}
|
||||
for extra in sorted(sc_menge - db_menge):
|
||||
fehler.append(f"Sidecar-Sub fehlt in DB: {extra}")
|
||||
for extra in sorted(db_menge - sc_menge):
|
||||
fehler.append(f"DB-consensus fehlt im Sidecar: {extra}")
|
||||
return fehler
|
||||
|
||||
|
||||
async def pruefe_guide_invarianten(topic: str, format_name: str = "Guide") -> list[str]:
|
||||
"""Jeder relevante consensus-Sub trägt einen Sub-Marker im Guide (Muster
|
||||
guide_qa.marker_fehlend, ohne LLM)."""
|
||||
import guide_qa
|
||||
fehler: list[str] = []
|
||||
cards = [dict(r) for r in await db.list_guide_cards(topic, format_name)]
|
||||
if not cards:
|
||||
return ["keine Guide-Karten"]
|
||||
for c in cards:
|
||||
if c["status"] != "ok" or not (c.get("md") or "").strip():
|
||||
fehler.append(f"Guide-Karte nicht ok: {c['block']} ({c['status']})")
|
||||
subs_rel: dict[str, set] = {}
|
||||
for r in await db.list_subblocks(topic):
|
||||
if r["status"] == "consensus" and r["relevance"] != "peripheral":
|
||||
subs_rel.setdefault(r["block_norm"], set()).add(r["sub_norm"])
|
||||
fehler += [f"Sub-Marker fehlt: {m}" for m in guide_qa.marker_fehlend(cards, subs_rel)]
|
||||
return fehler
|
||||
@@ -1012,12 +1012,13 @@ async def test_supplement_material_mode_for_source_topics(board_env, tmp_path, m
|
||||
|
||||
# ── Anker-Gate: Quorum-Titel ohne Korpus-Beleg (Reader-Ko-Halluzination) ────────────
|
||||
|
||||
async def _anker_env(db, tmp_path, monkeypatch, titel_map):
|
||||
async def _anker_env(db, tmp_path, monkeypatch, titel_map, desc_map=None):
|
||||
(tmp_path / "korpus.txt").write_text("Der Graph ist zusammenhängend und endlich.", encoding="utf-8")
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: tmp_path)
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"title": titel_map[cid], "description": "", "readers": ["r1", "r2"], "supplement": False}]
|
||||
return [{"title": titel_map[cid], "description": (desc_map or {}).get(cid, ""),
|
||||
"readers": ["r1", "r2"], "supplement": False}]
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "_rep", lambda rows: rows[0])
|
||||
@@ -1028,11 +1029,12 @@ async def _anker_env(db, tmp_path, monkeypatch, titel_map):
|
||||
|
||||
|
||||
async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
|
||||
"""Titel ohne Korpus-Anker → Beleg-Judge; „nein" → rejected/kein-beleg.
|
||||
"""Titel ohne Korpus-Anker, aber mit Evidenz → Beleg-Judge; „nein" → rejected/kein-beleg.
|
||||
Titel MIT Anker geht ohne Judge nach naming."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch,
|
||||
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"})
|
||||
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"},
|
||||
{"c2": "Der Graph ist endlich."}) # Evidenz da → Judge entscheidet
|
||||
|
||||
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "Brückenproblem" in prompt and "Zusammenhang" not in prompt # nur der Anker-lose
|
||||
@@ -1045,11 +1047,27 @@ async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
|
||||
assert c2["stage"] == "rejected" and c2["payload"]["reason"] == "kein-beleg"
|
||||
|
||||
|
||||
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Judge-Ausfall → Titel bleibt (2-Reader-Rückhalt)."""
|
||||
async def test_anker_gate_leeres_pack_hart_nein(testdb, tmp_path, monkeypatch):
|
||||
"""KEIN distinktives Token im Korpus → deterministisch rejected, Judge läuft NICHT
|
||||
(der Judge winkte 3 Kanon-Titel auf Schein-Auszügen durch)."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"})
|
||||
|
||||
async def never_slot(*a, **kw):
|
||||
raise AssertionError("Judge darf bei leerem Evidence-Pack nicht laufen")
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", never_slot)
|
||||
await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), cards)
|
||||
c9 = await db.kanban_get_card(TOPIC, B, "c9")
|
||||
assert c9["stage"] == "rejected" and c9["payload"]["reason"] == "kein-beleg"
|
||||
|
||||
|
||||
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Judge-Ausfall bei VORHANDENER Evidenz → Titel bleibt (2-Reader-Rückhalt)."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"},
|
||||
{"c9": "Der Graph ist endlich."})
|
||||
|
||||
async def broken_slot(*a, **kw):
|
||||
return "failed", None
|
||||
|
||||
@@ -1063,3 +1081,103 @@ def test_hat_anker_ziffern_suffix():
|
||||
assert bi._hat_anker("ΔTSP1-Algorithmus", ctoks) # tsp1 → tsp
|
||||
assert not bi._hat_anker("Königsberger Brückenproblem", ctoks)
|
||||
assert not bi._hat_anker("Algorithmus Verfahren", ctoks) # nur Stopwörter → kein Anker
|
||||
|
||||
|
||||
async def test_reset_subblocks_loescht_globale_dateien(testdb, tmp_path):
|
||||
"""Reset auf Spalte subblocks: DB-Spiegel UND globale Sidecar-Dateien + ab-*-Resume-Slots
|
||||
weg — Reste des Vor-Laufs würden sonst in den frischen Lauf zurückmergen."""
|
||||
db = testdb
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "artefacts", {"title": "Alpha"})
|
||||
await db.put_subblock(TOPIC, "alpha", "s1", "Alpha", "S1", status="consensus")
|
||||
arbeit = tmp_path / "arbeit"
|
||||
(arbeit / "ab-alpha").mkdir(parents=True)
|
||||
(arbeit / "ab-alpha" / "facts.json").write_text("{}", encoding="utf-8")
|
||||
files = {"arbeit": arbeit}
|
||||
for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"):
|
||||
files[k] = tmp_path / f"{k}.json"
|
||||
files[k].write_text("{}", encoding="utf-8")
|
||||
moved = await bi.reset_board_from_stage(TOPIC, "artefacts", "subblocks", files)
|
||||
assert moved == 1
|
||||
assert not await db.list_subblocks(TOPIC)
|
||||
assert not (arbeit / "ab-alpha").exists()
|
||||
assert all(not files[k].exists() for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"))
|
||||
|
||||
|
||||
# ── Naming-Abstraktion: freier Name nur mit Anker ───────────────────────────────────
|
||||
|
||||
def test_naming_schema_varianten():
|
||||
assert bi._naming_schema({"best": 2}, 3) == (2, None, False)
|
||||
assert bi._naming_schema({"best": 1, "name": "Kurzer Titel"}, 3) == (1, "Kurzer Titel", False)
|
||||
assert bi._naming_schema({"ok": True}, 3) == (None, None, True)
|
||||
assert bi._naming_schema({"best": 9}, 3) is None
|
||||
assert bi._naming_schema({"best": 1, "name": "x" * 90}, 3) == (1, None, False) # zu lang
|
||||
|
||||
|
||||
def test_name_verankert_thema_subset():
|
||||
rows = [{"title": "Bubblesort-Schleife und Tauschoperation", "description": "innere Schleife"},
|
||||
{"title": "Bubblesort Durchläufe", "description": ""}]
|
||||
assert bi._name_verankert("Bubblesort Tauschoperation", rows, None)
|
||||
assert not bi._name_verankert("Königsberger Brückenproblem", rows, None) # fremde Begriffe
|
||||
assert not bi._name_verankert("und der", rows, None) # nur Stopwörter → kein Anker
|
||||
|
||||
|
||||
def test_name_verankert_korpus():
|
||||
ctoks = {"partition", "problem", "vollständigkeit"}
|
||||
rows = [{"title": "irrelevant", "description": ""}]
|
||||
assert bi._name_verankert("Partition-Problem", rows, ctoks)
|
||||
assert not bi._name_verankert("Rucksackproblem Optimierung", rows, ctoks)
|
||||
|
||||
|
||||
async def test_naming_vergibt_verankerten_namen(testdb, tmp_path, monkeypatch):
|
||||
"""Judge liefert best+name; verankerter Name gewinnt, unverankerter fällt auf Member zurück."""
|
||||
db = testdb
|
||||
antwort = {"val": {"best": 1, "name": "Bubblesort Grundprinzip"}}
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"norm": "bubblesort - grundprinzip und ablauf (kap. 2)",
|
||||
"title": "Bubblesort - Grundprinzip und Ablauf (Kap. 2)",
|
||||
"description": "Sortieren durch Tauschen", "readers": ["r1"], "sources": []},
|
||||
{"norm": "sortieren durch tauschen", "title": "Sortieren durch Tauschen",
|
||||
"description": "", "readers": ["r2"], "sources": []}]
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
import json as _json
|
||||
return "ok", payload((0, _json.dumps(antwort["val"]), ""))
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
await db.kanban_upsert_card(TOPIC, B, "c1", "cluster", "naming", {})
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
await bi._name_one(ctx, _mk_flow(tmp_path), {"card_id": "c1", "payload": {}})
|
||||
card = await db.kanban_get_card(TOPIC, B, "c1")
|
||||
assert card["payload"]["title"] == "Bubblesort Grundprinzip"
|
||||
|
||||
# unverankerter Name → Member-Titel gewinnt
|
||||
antwort["val"] = {"best": 2, "name": "Vergleichsbasierte Sortierverfahren"}
|
||||
await db.kanban_upsert_card(TOPIC, B, "c2", "cluster", "naming", {})
|
||||
await bi._name_one(ctx, _mk_flow(tmp_path), {"card_id": "c2", "payload": {}})
|
||||
card = await db.kanban_get_card(TOPIC, B, "c2")
|
||||
assert card["payload"]["title"] == "Sortieren durch Tauschen"
|
||||
|
||||
|
||||
async def test_namecheck_ok_behaelt_titel(testdb, tmp_path, monkeypatch):
|
||||
"""Check-Judge bestätigt mit ok:true → Titel und Beschreibung bleiben unverändert."""
|
||||
db = testdb
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"norm": "a", "title": "A", "description": "da", "readers": ["r1"], "sources": []},
|
||||
{"norm": "b", "title": "B", "description": "db", "readers": ["r2"], "sources": []}]
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "Eigener Titel" in prompt # current_title steht im Prompt
|
||||
return "ok", payload((0, '{"ok": true}', ""))
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
payload = {"title": "Eigener Titel", "description": "Eigene Beschreibung", "main_norm": "a"}
|
||||
await db.kanban_upsert_card(TOPIC, B, "c9", "cluster", "naming_check", payload)
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
await bi._namecheck_one(ctx, _mk_flow(tmp_path), {"card_id": "c9", "payload": payload})
|
||||
block = await db.kanban_get_card(TOPIC, B, "b-c9")
|
||||
assert block["payload"]["title"] == "Eigener Titel"
|
||||
assert block["payload"]["description"] == "Eigene Beschreibung"
|
||||
|
||||
197
backend/tests/test_e2e_fake.py
Normal file
197
backend/tests/test_e2e_fake.py
Normal file
@@ -0,0 +1,197 @@
|
||||
"""E2E über die ECHTE Engine mit Fake-Agenten: kompletter Generierungspfad in Sekunden.
|
||||
|
||||
Anders als test_board_inventory (dort sind die Block-Funktionen gefakt) läuft hier alles
|
||||
bis run_agent echt — _race, Quorum, Panels, Konsolidierung, Cross-Block, QA-Gate.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
import board_inventory as bi
|
||||
from pipeline import GenContext
|
||||
from tests.invarianten import pruefe_invarianten, pruefe_guide_invarianten
|
||||
|
||||
TOPIC = "t"
|
||||
|
||||
|
||||
def _files(tmp_path):
|
||||
work = tmp_path / "arbeit"
|
||||
work.mkdir(exist_ok=True)
|
||||
return {"arbeit": work, "final": tmp_path / "blocks.md",
|
||||
"sub_roh": tmp_path / "sub_roh.json", "sidecar": tmp_path / "subblocks.json",
|
||||
"facts": tmp_path / "facts.json", "question_pattern": tmp_path / "question_pattern.json",
|
||||
"artefakte": tmp_path / "artefakte.json", "outline": tmp_path / "outline.json",
|
||||
"outline_slots": [tmp_path / f"outline-{i}.json" for i in (1, 2, 3)],
|
||||
"research": [work / f"research-{i}.md" for i in (1, 2, 3, 4, 5)]}
|
||||
|
||||
|
||||
async def _lauf(tmp_path, research=True, qa_force=False, timeout=120):
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
files = _files(tmp_path)
|
||||
ok = await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "",
|
||||
research=research, qa_force=qa_force), timeout=timeout)
|
||||
return ok, files
|
||||
|
||||
|
||||
async def test_e2e_thema_vollpfad(fake_welt, testdb, tmp_path):
|
||||
"""Research → Inventar → QA-Gate → Artefakte → Finalize, alle Schichten echt."""
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = [c for c in await db.kanban_cards(TOPIC, board="inventory", stage="done_block")]
|
||||
titel = {c["payload"]["title"] for c in done}
|
||||
assert titel == {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"}
|
||||
# Cross-Block-Dublette: „Gemeinsamer Grundbegriff" überlebt in genau EINEM Block
|
||||
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
||||
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
||||
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "variant"]
|
||||
fehler = await pruefe_invarianten(TOPIC, files)
|
||||
assert fehler == []
|
||||
|
||||
|
||||
async def test_e2e_guide(fake_welt, testdb, tmp_path):
|
||||
"""Auf den Vollpfad folgt der Guide-Bau — Gate/Coverage/Lese-Stages laufen echt."""
|
||||
import guide_board
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
||||
entries = {i: f"{c['payload']['title']} — {c['payload'].get('description', '')}"
|
||||
for i, c in enumerate(done, 1)}
|
||||
chapters = await asyncio.wait_for(
|
||||
guide_board.run_guide_board("g-e2e", TOPIC, "Guide", entries, "", "claude",
|
||||
tmp_path / "guides" / "Guide.json"), timeout=120)
|
||||
assert chapters is not None
|
||||
assert await pruefe_guide_invarianten(TOPIC) == []
|
||||
|
||||
|
||||
async def test_e2e_rerun_idempotent(fake_welt, testdb, tmp_path):
|
||||
"""Zweiter Lauf (Continue, research=False) hinterlässt keine Waisen/Reste."""
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
vorher = {(r["block_norm"], r["sub_norm"], r["status"])
|
||||
for r in await db.list_subblocks(TOPIC)}
|
||||
ok2, _f = await _lauf(tmp_path, research=False)
|
||||
assert ok2
|
||||
nachher = {(r["block_norm"], r["sub_norm"], r["status"])
|
||||
for r in await db.list_subblocks(TOPIC)}
|
||||
assert nachher == vorher
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stoerung", [
|
||||
{"muster": r"-sub-crossblock-.*-j1$", "modus": "fehler", "mal": 3}, # Ersatzrichter jE
|
||||
{"muster": r"-sub-konsolidierung-.*-j1$", "modus": "garbage", "mal": 1}, # Retry heilt
|
||||
{"muster": r"-facts-c\d+$", "modus": "fehler", "mal": 1}, # Slot-Restart
|
||||
{"muster": r"-research-2$", "modus": "fehler", "mal": 3}, # 1 Producer tot
|
||||
])
|
||||
async def test_e2e_stoerungen_flow_endet(fake_welt, testdb, tmp_path, stoerung):
|
||||
"""Einzel-Ausfälle dürfen weder den Flow stoppen noch Invarianten reißen."""
|
||||
fake_welt.stoerungen.append(dict(stoerung, rest=stoerung["mal"]))
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_crossblock_dissent_failopen(fake_welt, testdb, tmp_path):
|
||||
"""j1 sagt a, j2 sagt b, j3 fällt aus → Paar bleibt (fail-open), Rest konsistent."""
|
||||
fake_welt.stoerungen += [
|
||||
{"muster": r"-sub-crossblock-.*-j2$", "modus": "antwort",
|
||||
"antwort": '{"pairs": {"1": "b"}}', "mal": 1, "rest": 1},
|
||||
{"muster": r"-sub-crossblock-.*-j3$", "modus": "fehler", "mal": 3, "rest": 3},
|
||||
]
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
||||
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
||||
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "consensus"] # kein Fold
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_inblock_gruppe_faltet(fake_welt, testdb, tmp_path):
|
||||
"""Welt-Regel: „Alpha Eigenschaften" faltet unter „Definition Alpha" — beide Judges
|
||||
liefern die Gruppe, der Verlierer wird variant, seine facts wandern zum Gewinner."""
|
||||
fake_welt.gruppen.append(("Definition Alpha", ["Alpha Eigenschaften"]))
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha-konzept")}
|
||||
assert rows.get("alpha eigenschaften") == "variant"
|
||||
assert rows.get("definition alpha") == "consensus"
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_gate_vollinventur_ohne_fix(fake_welt, testdb, tmp_path):
|
||||
"""Gate-Judge liefert eine Voll-Inventur (belegte Claims mit „Belegt…"-Grund) —
|
||||
der Schema-Filter wirft sie raus, es läuft KEIN Fakten-Fix."""
|
||||
import json
|
||||
antwort = json.dumps({"claims": [
|
||||
{"text": "Aussage 1", "grund": "Belegt durch Quelle", "urteil": "unbelegt"},
|
||||
{"text": "Aussage 2", "grund": "Belegt durch Fakten", "urteil": "unbelegt"},
|
||||
{"text": "Aussage 3", "grund": "Belegt: steht im Skript", "urteil": "unbelegt"}]})
|
||||
fake_welt.stoerungen.append({"muster": r"-gate-", "modus": "antwort",
|
||||
"antwort": antwort, "mal": 99, "rest": 99})
|
||||
import guide_board
|
||||
ok, _files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
||||
entries = {i: c["payload"]["title"] for i, c in enumerate(done, 1)}
|
||||
chapters = await asyncio.wait_for(
|
||||
guide_board.run_guide_board("g-vi", TOPIC, "Guide", entries, "", "claude",
|
||||
tmp_path / "guides" / "Guide.json"), timeout=120)
|
||||
assert chapters is not None
|
||||
assert not any("-gatefix-" in k for k in fake_welt.calls)
|
||||
|
||||
|
||||
async def test_e2e_echtheits_flattern_gestoppt(fake_welt, testdb, tmp_path):
|
||||
"""QA-Pass 1 flaggt alle Blöcke als unecht (Judge-Flattern) — der Bestätiger-Pass
|
||||
widerspricht, die Gate-Note bleibt sauber, der Flow läuft durch."""
|
||||
import json
|
||||
fake_welt.stoerungen.append({"muster": r"^qa-t-bausteine-0$", "modus": "antwort",
|
||||
"antwort": json.dumps({"relevant": {"1": "nein", "2": "nein", "3": "nein"}}),
|
||||
"mal": 1, "rest": 1})
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok # Gate hat nicht pausiert — der Zufalls-Verdacht wurde nicht bestätigt
|
||||
assert any("bausteine-b2" in k for k in fake_welt.calls)
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_uni_anker_gate(fake_welt, testdb, tmp_path, monkeypatch):
|
||||
"""uni-Modus mit Mini-Korpus: der Kanon-Titel ohne Korpus-Anker wird deterministisch
|
||||
rejected (leeres Evidence-Pack), die belegten Blöcke laufen durch; QA misst gegen
|
||||
den echten Korpus."""
|
||||
import blocks as blx
|
||||
fake_welt.bloecke["Kanon-Klassiker"] = {
|
||||
"beschreibung": "Beruehmtes Lehrbuchproblem", "subs": ["Klassiker Detail"]}
|
||||
korpus = tmp_path / "korpus"
|
||||
korpus.mkdir()
|
||||
zeilen = []
|
||||
for t, b in fake_welt.bloecke.items():
|
||||
if t == "Kanon-Klassiker":
|
||||
continue # kommt bewusst NICHT im Material vor
|
||||
zeilen.append(f"Kapitel {t}: {b['beschreibung']}. " +
|
||||
" ".join(f"Wir behandeln {s}." for s in b["subs"]))
|
||||
(korpus / "skript.txt").write_text("\n\n".join(zeilen), encoding="utf-8")
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: korpus)
|
||||
monkeypatch.setattr(blx, "source_folder", lambda t: korpus)
|
||||
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
files = _files(tmp_path)
|
||||
ok = await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "uni", "location": str(korpus)},
|
||||
korpus, "", research=True, qa_force=True), timeout=120)
|
||||
assert ok
|
||||
db = testdb
|
||||
alle = [dict(c) for c in await db.kanban_cards(TOPIC, board="inventory")]
|
||||
assert any(c["stage"] == "rejected" and c["payload"].get("title") == "Kanon-Klassiker"
|
||||
for c in alle)
|
||||
assert not any(c["kind"] == "block" and c["payload"].get("title") == "Kanon-Klassiker"
|
||||
for c in alle) # nie zum Block geworden
|
||||
done = {c["payload"].get("title") for c in alle
|
||||
if c["kind"] == "block" and c["stage"] == "done_block"}
|
||||
assert {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"} <= done
|
||||
@@ -18,8 +18,16 @@ def test_ziele_schema():
|
||||
def test_gate_schema():
|
||||
assert gb._gate_schema({"ok": True}) == []
|
||||
claims = gb._gate_schema({"claims": [{"text": "Falsch", "grund": "fehlt"}]})
|
||||
assert claims == [{"text": "Falsch", "grund": "fehlt"}]
|
||||
assert claims == [{"text": "Falsch", "grund": "fehlt", "urteil": "unbelegt"}]
|
||||
assert gb._gate_schema({}) is None
|
||||
# urteil "falsch" wird durchgereicht, alles andere defaultet auf unbelegt
|
||||
claims = gb._gate_schema({"claims": [{"text": "A", "grund": "widerspricht", "urteil": "FALSCH"},
|
||||
{"text": "B", "grund": "x", "urteil": "quatsch"}]})
|
||||
assert [c["urteil"] for c in claims] == ["falsch", "unbelegt"]
|
||||
# Voll-Inventur-Rauschen: als belegt begründete Einträge fliegen raus
|
||||
claims = gb._gate_schema({"claims": [{"text": "A", "grund": "Belegt durch Quelle X"},
|
||||
{"text": "B", "grund": "nicht ableitbar"}]})
|
||||
assert [c["text"] for c in claims] == ["B"]
|
||||
|
||||
|
||||
def test_coverage_schema():
|
||||
@@ -266,7 +274,7 @@ async def test_lese_check_text_sink(testdb, tmp_path, monkeypatch):
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-l", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
md = ("<!-- kapitel: K -->\n<!-- section: Alpha -->\n<!-- compact -->\n- x\n"
|
||||
"<!-- ausführlich -->\nText.")
|
||||
"<!-- ausführlich -->\n" + "Text im Längen-Rahmen. " * 20) # ~460 Z. — kein Längen-Trigger
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
seen = {}
|
||||
|
||||
@@ -330,3 +338,49 @@ async def test_load_subblocks_defaultet_levellose(testdb):
|
||||
subs = await _load_subblocks("t")
|
||||
by_title = {s["title"]: s["level"] for s in subs["Block"]}
|
||||
assert by_title == {"Mit Level": "beginner", "Ohne Level": "advanced"}
|
||||
|
||||
|
||||
async def test_fakten_gate_falsch_claim_erzwingt_fix(testdb, tmp_path, monkeypatch):
|
||||
"""Ein einzelner FALSCH-Claim läuft in den Fix, auch unter GATE_FIX_MIN —
|
||||
ein durchgerutschter kostete den Guide 1.5 QA-Punkte."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-gf", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
md = "<!-- section: Alpha -->\n<!-- ausführlich -->\nText."
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
keys = []
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
keys.append(key)
|
||||
if "-gate-" in key:
|
||||
return gb.OK, [{"text": "c1", "grund": "widerspricht", "urteil": "falsch"}]
|
||||
return gb.FAILED, None # Fix-Agent liefert nichts — Text bleibt, aber der Call MUSS kommen
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
assert await gb._stage_fakten_gate(env, card)
|
||||
assert any("-gatefix-" in k for k in keys)
|
||||
|
||||
|
||||
async def test_laengen_trigger_startet_lesefix(testdb, tmp_path, monkeypatch):
|
||||
"""Ausführlich-Teil über der Obergrenze → deterministisches Längen-Problem
|
||||
mit hartem Zeichenziel landet im Lese-Fix-Auftrag."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-lz", TOPIC, FMT, "", tmp_path / "Guide.json",
|
||||
{"Alpha": [{"title": "S1", "level": "beginner", "relevance": "relevant"}]},
|
||||
{}, "(q)", "spec")
|
||||
md = ("<!-- section: Alpha -->\n<!-- compact -->\n- x\n<!-- ausführlich -->\n"
|
||||
+ "Viel zu langer Sockeltext. " * 80) # ~2160 Z./Sub > 1200×0.9
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
seen = {}
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
if "-lesefix-" in key:
|
||||
seen["tasks"] = prompt
|
||||
return gb.FAILED, None
|
||||
return gb.OK, payload((0, '{"ok": true}', "")) # Lese-Check: keine Probleme
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
monkeypatch.setattr(gb, "READABILITY_ACTIVE", False)
|
||||
assert await gb._stage_lesbarkeit(env, card)
|
||||
assert "Länge" in seen["tasks"] and str(gb._writer_budget(1)) in seen["tasks"]
|
||||
|
||||
@@ -519,3 +519,90 @@ async def test_finalize_defaultet_level_nachzuegler(testdb, tmp_path):
|
||||
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
|
||||
row = next(r for r in await db.list_subblocks(TOPIC, "alpha"))
|
||||
assert row["level"] == "advanced" and row["relevance"] == "relevant"
|
||||
|
||||
|
||||
async def test_finalize_loescht_stale_consensus(testdb, tmp_path):
|
||||
"""Alt-consensus-Rows, die der Lauf-Sidecar nicht mehr trägt, fliegen raus —
|
||||
variant-Rows bleiben (QA liest die Status). Wurzel der 25 Board-2-losen Waisen."""
|
||||
db = testdb
|
||||
await db.put_subblock(TOPIC, "alpha", "alt-rest", "Alpha", "Alt-Rest", status="consensus")
|
||||
await db.put_subblock(TOPIC, "alpha", "alte-variante", "Alpha", "Alte Variante", status="variant")
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "finalize", {})
|
||||
files = {k: tmp_path / f"{k}.json" for k in
|
||||
("sub_roh", "facts", "sidecar", "question_pattern", "artefakte")}
|
||||
card = {"card_id": "alpha", "payload": {
|
||||
"title": "Alpha", "raw": {"Alpha": ["Neu"]}, "facts": {},
|
||||
"sidecar": {"Alpha": [{"title": "Neu", "level": "beginner"}]},
|
||||
"pattern": {}, "artefacts": {}}}
|
||||
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||||
assert rows == {"neu": "consensus", "alte-variante": "variant"}
|
||||
|
||||
|
||||
async def test_crossblock_chunking_faltet_global(testdb, tmp_path, monkeypatch):
|
||||
"""Paare werden gechunkt beurteilt (ein Hänger blockiert nur noch seinen Chunk);
|
||||
die Verdicts falten global über alle Chunks."""
|
||||
db = testdb
|
||||
flow = Flow(TOPIC, work_dir=tmp_path)
|
||||
cards = []
|
||||
for bnorm, subs in (("alpha", ["Gleiche Aussage", "Zweite gleiche Aussage", "Nur in Alpha"]),
|
||||
("beta", ["Gleiche Aussage", "Zweite gleiche Aussage"])):
|
||||
payload = {"title": bnorm.title(),
|
||||
"raw": {bnorm.title(): list(subs)},
|
||||
"sidecar": {bnorm.title(): [{"title": s, "level": "beginner"} for s in subs]},
|
||||
"facts": {bnorm.title(): {blocks._norm_title(s): {"key_points": []} for s in subs}}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", bnorm, "ablock", "konsolidierung", payload)
|
||||
await _seed_block(db, bnorm, subs)
|
||||
cards.append({"card_id": bnorm, "payload": payload})
|
||||
monkeypatch.setattr(ba, "embedding", _FakeEmb)
|
||||
monkeypatch.setattr(ba, "CROSS_CHUNK_PAARE", 1) # 2 Paare → 2 Chunks
|
||||
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "j2": {"pairs": {"1": "a"}}})
|
||||
monkeypatch.setattr(ba, "run_single_slot", fake)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
assert len(fake.calls) == 4 # 2 Chunks × j1/j2
|
||||
beta = await db.kanban_get_card(TOPIC, "artefacts", "beta")
|
||||
assert beta["payload"]["raw"].get("Beta", []) == [] # beide Dubletten global gefaltet
|
||||
|
||||
|
||||
async def test_facts_nachfass_holt_nur_fehlende(testdb, tmp_path, monkeypatch):
|
||||
"""Nachfass ruft den slim-Facts-Lauf NUR mit den facts-losen Subs und merged die Funde;
|
||||
Vorhandenes bleibt unberührt, Subs werden nie verworfen."""
|
||||
gesehen = {}
|
||||
|
||||
async def fake_facts_block(ctx, set_p, files, raw, q, folder, instructions, ns="", lbl="",
|
||||
sources=None, slim=False):
|
||||
gesehen["raw"] = raw
|
||||
gesehen["slim"] = slim
|
||||
return ({"Alpha": {"ohne beleg": {"key_points": ["kp neu"]},
|
||||
"mit beleg": {"key_points": ["DARF NICHT GEWINNEN"]}}},
|
||||
{"Alpha": {"ohne beleg"}}) # discard-Urteil wird ignoriert
|
||||
|
||||
monkeypatch.setattr(blocks, "_facts_block", fake_facts_block)
|
||||
raw = {"Alpha": ["Mit Beleg", "Ohne Beleg"]}
|
||||
facts_map = {"Alpha": {"mit beleg": {"key_points": ["kp alt"]}}}
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
n = await blocks._facts_nachfass(ctx, {"arbeit": tmp_path}, raw, facts_map, {}, None)
|
||||
assert n == 1
|
||||
assert gesehen["slim"] and gesehen["raw"] == {"Alpha": ["Ohne Beleg"]}
|
||||
assert facts_map["Alpha"]["ohne beleg"]["key_points"] == ["kp neu"]
|
||||
assert facts_map["Alpha"]["mit beleg"]["key_points"] == ["kp alt"]
|
||||
assert raw["Alpha"] == ["Mit Beleg", "Ohne Beleg"] # kein Verwurf
|
||||
|
||||
|
||||
async def test_levels_merge_fuzzy_match(testdb, tmp_path, monkeypatch):
|
||||
"""Levels-Agent paraphrasiert den Sub-Titel → facts hängen trotzdem am Sidecar-Eintrag
|
||||
(eindeutiger Präfix-Match statt stillem Grounding-Verlust)."""
|
||||
db = testdb
|
||||
payload = {"title": "Alpha", "raw": {"Alpha": ["Marker Regel: Details dazu"]},
|
||||
"facts": {"Alpha": {blocks._norm_title("Marker Regel: Details dazu"):
|
||||
{"key_points": ["kp"]}}}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "levels", payload)
|
||||
cards = [{"card_id": "alpha", "payload": payload}]
|
||||
|
||||
async def fake_levels_block(ctx, set_p, files, raw, instructions, ns="", lbl=""):
|
||||
return {"Alpha": [{"title": "Marker Regel", "level": "beginner"}]} # gekürzter Titel
|
||||
|
||||
monkeypatch.setattr(ba, "_levels_block", fake_levels_block)
|
||||
await ba._proc_levels(_ctx(), Flow(TOPIC, work_dir=tmp_path), {"arbeit": tmp_path}, "", cards)
|
||||
card = await db.kanban_get_card(TOPIC, "artefacts", "alpha")
|
||||
assert card["payload"]["sidecar"]["Alpha"][0]["facts"] == {"key_points": ["kp"]}
|
||||
|
||||
@@ -214,3 +214,37 @@ def test_fremd_digit_suffix_tolerant():
|
||||
b = [{"title": "ΔTSP1-Algorithmus", "description": "", "sources": []},
|
||||
{"title": "Quantencomputer", "description": "", "sources": []}]
|
||||
assert qa.fremd(b, corpus) == ["Quantencomputer"]
|
||||
|
||||
|
||||
async def test_unecht_braucht_doppelt_nein(testdb, tmp_path, monkeypatch):
|
||||
"""Echtheits-Urteil zählt nur nach Bestätiger-Pass: der Einzel-Judge flaggte pro Lauf
|
||||
andere Blöcke und pendelte die Note (aak: 9.3↔10.0 bei identischem Bestand)."""
|
||||
db = testdb
|
||||
for cid, titel in (("b1", "Wackelkandidat"), ("b2", "Zufallstreffer"), ("b3", "Solide")):
|
||||
await db.kanban_upsert_card("t", "inventory", cid, "block", "done_block",
|
||||
{"title": titel, "description": "d"})
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path)
|
||||
|
||||
async def fake_verdicts(template, topic, key, items):
|
||||
if template != "QA-Bausteine":
|
||||
return {}
|
||||
if key.startswith("bausteine-b2"): # Bestätiger sieht nur die Geflaggten
|
||||
assert len(items) == 2
|
||||
return {1: "nein", 2: "ja"} # nur der erste wird bestätigt
|
||||
return {1: "nein", 2: "nein", 3: "ja"} # Pass 1 flaggt zwei
|
||||
|
||||
monkeypatch.setattr(qa, "_llm_verdicts", fake_verdicts)
|
||||
report = await qa.qa_report("t", llm=True)
|
||||
assert report["unecht"] == ["Wackelkandidat"]
|
||||
|
||||
|
||||
async def test_topic_delete_entfernt_qa_ordner(testdb, tmp_path, monkeypatch):
|
||||
"""DELETE /topics räumt auch storage/qa/<topic>/ — Reports gehören zum Topic."""
|
||||
import routes
|
||||
monkeypatch.setattr(routes, "topic_dir", lambda t: tmp_path / "topics" / t)
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
|
||||
qdir = tmp_path / "qa" / "t"
|
||||
qdir.mkdir(parents=True)
|
||||
(qdir / "alt.json").write_text("{}", encoding="utf-8")
|
||||
await routes.remove_topic("t")
|
||||
assert not qdir.exists()
|
||||
|
||||
@@ -226,10 +226,8 @@ async def test_outline_review_moves_block(testdb, tmp_path, monkeypatch):
|
||||
out = plan_a
|
||||
elif key.endswith("outline-review"):
|
||||
out = review_out["val"]
|
||||
if out is not None:
|
||||
m = re.search(r"(/\S+\.json)", prompt)
|
||||
with open(m.group(1), "w", encoding="utf-8") as f:
|
||||
json.dump(out, f)
|
||||
if out is not None: # neue Semantik: Antwort als TEXT, der Sink persistiert
|
||||
return "ok", payload((0, json.dumps(out), ""))
|
||||
return "ok", payload(None)
|
||||
|
||||
monkeypatch.setattr(blx, "run_single_slot", fake_slot)
|
||||
|
||||
113
backend/tests/test_train.py
Normal file
113
backend/tests/test_train.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Training-Harness: Registry↔config-Konsistenz, ENV-Override, Trainer-Logik (Stub-Runner)."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import config
|
||||
import train
|
||||
import train_params
|
||||
from train import Trainer, score
|
||||
|
||||
BACKEND = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def test_registry_spiegelt_config():
|
||||
"""Jeder Registry-Parameter existiert in config mit identischem Default und
|
||||
flow-sicheren Rändern — sonst optimiert der Trainer Phantome."""
|
||||
for name, p in train_params.PARAMS.items():
|
||||
assert getattr(config, name, None) == p["default"], name
|
||||
assert p["min"] <= p["default"] <= p["max"], name
|
||||
assert p["step"] > 0, name
|
||||
|
||||
|
||||
def test_creator_params_override_wirkt_im_subprozess():
|
||||
out = subprocess.run(
|
||||
[sys.executable, "-c", "import config; print(config.FACTS_CHUNK_SUBS, config.TIMEOUTS['subblock_check'][0])"],
|
||||
capture_output=True, text=True, cwd=BACKEND,
|
||||
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"FACTS_CHUNK_SUBS": 6, "TIMEOUT_subblock_check_base": 77}'})
|
||||
assert out.stdout.split() == ["6", "77"], out.stderr
|
||||
|
||||
|
||||
def test_creator_params_unbekannter_name_bricht_ab():
|
||||
out = subprocess.run([sys.executable, "-c", "import config"],
|
||||
capture_output=True, text=True, cwd=BACKEND,
|
||||
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"GIBT_ES_NICHT": 1}'})
|
||||
assert out.returncode != 0 and "GIBT_ES_NICHT" in out.stderr
|
||||
|
||||
|
||||
def _metrics(note=8.0, dauer=10.0, tokens=1_000_000, **quoten):
|
||||
return {"note": note, "quoten": quoten, "quoten_artefakte": {},
|
||||
"dauer_min": dauer, "tokens": {"input": tokens, "output": 0}, "agents": {}}
|
||||
|
||||
|
||||
def _stub_runner(antworten):
|
||||
"""params-abhängige Metriken; zählt echte Aufrufe (Cache-Treffer zählen nicht)."""
|
||||
calls = []
|
||||
|
||||
async def runner(params, thema):
|
||||
calls.append((dict(params), thema[0]))
|
||||
for muster, m in antworten:
|
||||
if muster(params):
|
||||
return dict(m)
|
||||
return _metrics()
|
||||
|
||||
runner.calls = calls
|
||||
return runner
|
||||
|
||||
|
||||
async def test_screening_filtert_rauschen(tmp_path):
|
||||
"""Nur Parameter mit Effekt über der Rausch-Schwelle kommen in die Feinphase;
|
||||
ein bestätigter Gewinner wird übernommen."""
|
||||
wirksam = "FACTS_CHUNK_SUBS"
|
||||
runner = _stub_runner([
|
||||
(lambda p: p.get(wirksam) == 8, _metrics(note=9.5, dauer=8.0)), # klar besser
|
||||
])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
best = await t.run()
|
||||
assert best.get(wirksam) == 8
|
||||
# kein anderer Parameter übernommen (alle anderen Δ=0 < Schwelle)
|
||||
assert set(best) == {wirksam}
|
||||
|
||||
|
||||
async def test_uebernahme_braucht_bestaetigung(tmp_path):
|
||||
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird verworfen."""
|
||||
zustand = {"mal": 0}
|
||||
|
||||
async def runner(params, thema):
|
||||
if params.get("FACTS_CHUNK_SUBS") == 8:
|
||||
zustand["mal"] += 1
|
||||
return _metrics(note=9.5) if zustand["mal"] == 1 else _metrics(note=8.0)
|
||||
return _metrics()
|
||||
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
best = await t.run()
|
||||
assert best == {}
|
||||
|
||||
|
||||
async def test_cache_resume_wiederholt_keine_trials(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
await t.run()
|
||||
erste = len(runner.calls)
|
||||
t2 = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
await t2.run()
|
||||
assert len(runner.calls) == erste # alles aus trials.jsonl bedient
|
||||
|
||||
|
||||
async def test_budget_stoppt(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=3, max_stunden=1, runner=runner)
|
||||
await t.run()
|
||||
assert len(runner.calls) <= 3
|
||||
|
||||
|
||||
def test_score_richtungen():
|
||||
basis = _metrics()
|
||||
besser = _metrics(note=9.0)
|
||||
teurer = _metrics(dauer=20.0, tokens=2_000_000)
|
||||
assert score(besser, basis) > score(basis, basis)
|
||||
assert score(teurer, basis) < score(basis, basis)
|
||||
mit_befunden = _metrics(fremd=0.2, luecken=0.1)
|
||||
assert score(mit_befunden, basis) < score(basis, basis)
|
||||
Reference in New Issue
Block a user