This commit is contained in:
Team3
2026-07-05 15:26:22 +02:00
parent 07e14fb82e
commit 250ea0b764
45 changed files with 1468 additions and 1452 deletions

View File

@@ -1,96 +0,0 @@
"""Invarianten nach einem (Fake-)E2E-Lauf: was IMMER gelten muss, egal welches Szenario.
Nutzt bewusst eigene, schlichte Prüfungen statt Pipeline-Heuristiken (Muster qa.py) —
geteilte blinde Flecken machen den Check wertlos. Rückgabe: Liste von Verstößen,
leer = alles konsistent.
"""
import json
import database as db
from textkit import _norm_title
_LEVELS_OK = {"beginner", "advanced", "expert"}
_RELEVANZ_OK = {"relevant", "peripheral"}
def _json(path):
try:
return json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
async def pruefe_invarianten(topic: str, files: dict | None = None,
mit_artefakten: bool = True) -> list[str]:
fehler: list[str] = []
subs = [dict(r) for r in await db.list_subblocks(topic)]
cons = [r for r in subs if r["status"] == "consensus"]
for r in cons:
wo = f"{r['block']}/{r['sub_title']}"
fk = None
try:
fk = json.loads(r["facts"]) if r["facts"] else None
except ValueError:
fehler.append(f"facts unparsebar: {wo}")
if not (isinstance(fk, dict) and (fk.get("key_points") or fk.get("cited_facts"))):
fehler.append(f"consensus-Sub ohne facts: {wo}")
if r["level"] not in _LEVELS_OK:
fehler.append(f"consensus-Sub ohne gültiges level: {wo}")
if r["relevance"] not in _RELEVANZ_OK:
fehler.append(f"consensus-Sub ohne relevance: {wo}")
if mit_artefakten:
art = [dict(r) for r in await db.get_sub_artefakte(topic)]
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
versorgt = {(r["block_norm"], r["sub_norm"]) for r in art}
versorgt |= {(r["block_norm"], r["sub_norm"]) for r in fragen}
lebend = {(r["block_norm"], r["sub_norm"]) for r in subs if r["status"] in ("consensus", "variant")}
for r in cons:
if r["relevance"] == "relevant" and (r["block_norm"], r["sub_norm"]) not in versorgt:
fehler.append(f"relevanter Sub ohne Frage/Artefakt: {r['block']}/{r['sub_title']}")
for bn, sn in sorted({(r["block_norm"], r["sub_norm"]) for r in art} |
{(r["block_norm"], r["sub_norm"]) for r in fragen}):
if (bn, sn) not in lebend:
fehler.append(f"Waise (Ziel-Sub existiert nicht): {bn}/{sn}")
# keine hängengebliebenen Karten
for c in await db.kanban_cards(topic):
if c["stage"] == "dead":
fehler.append(f"dead-Karte: {c['board']}/{c['card_id']}")
if files is not None:
if not files["final"].exists():
fehler.append("blocks.md fehlt")
sc = _json(files["sidecar"])
if not isinstance(sc, dict):
fehler.append("sidecar-Datei fehlt/unparsebar")
else: # Sidecar und DB-consensus müssen dieselbe Sub-Menge tragen
db_menge = {(r["block_norm"], r["sub_norm"]) for r in cons}
sc_menge = {(_norm_title(bt), _norm_title(str(s.get("title", ""))))
for bt, ss in sc.items() for s in ss if isinstance(s, dict)}
for extra in sorted(sc_menge - db_menge):
fehler.append(f"Sidecar-Sub fehlt in DB: {extra}")
for extra in sorted(db_menge - sc_menge):
fehler.append(f"DB-consensus fehlt im Sidecar: {extra}")
return fehler
async def pruefe_guide_invarianten(topic: str, format_name: str = "Guide") -> list[str]:
"""Jeder relevante consensus-Sub trägt einen Sub-Marker im Guide (Muster
guide_qa.marker_fehlend, ohne LLM)."""
import guide_qa
fehler: list[str] = []
cards = [dict(r) for r in await db.list_guide_cards(topic, format_name)]
if not cards:
return ["keine Guide-Karten"]
for c in cards:
if c["status"] != "ok" or not (c.get("md") or "").strip():
fehler.append(f"Guide-Karte nicht ok: {c['block']} ({c['status']})")
subs_rel: dict[str, set] = {}
for r in await db.list_subblocks(topic):
if r["status"] == "consensus" and r["relevance"] != "peripheral":
subs_rel.setdefault(r["block_norm"], set()).add(r["sub_norm"])
fehler += [f"Sub-Marker fehlt: {m}" for m in guide_qa.marker_fehlend(cards, subs_rel)]
return fehler

View File

@@ -10,7 +10,7 @@ import pytest
import board_inventory as bi
from pipeline import GenContext
from tests.invarianten import pruefe_invarianten, pruefe_guide_invarianten
from invarianten import pruefe_invarianten, pruefe_guide_invarianten
TOPIC = "t"

View File

@@ -131,8 +131,8 @@ async def test_guide_error_event(testdb):
def test_timeout_calibration_smoke():
from pipeline import _timeout
assert _timeout("subblock", 10) == 400 + 150
assert _timeout("content", 10) == 450 + 300
assert _timeout("subblock_check", 10) == 150 + 100
assert _timeout("writer", 10) == 450 + 600
def test_env_file_wins(tmp_path, monkeypatch):
@@ -317,3 +317,71 @@ async def test_events_run_summary_aggregates(testdb):
assert s["agents"]["gesamt"] == 2 and s["agents"]["ok"] == 1 and s["agents"]["timeout"] == 1
assert s["agents"]["verlorene_min"] == 2
assert s["tokens"] == {"input": 15, "output": 2, "cache_read": 80, "cache_write": 1}
async def test_topic_delete_entfernt_guides_und_kanban(testdb, tmp_path, monkeypatch):
"""DELETE /topics: guides/guide_cards/kanban_cards mitlöschen — GET /topics leitet
Topics aus guides ab, sonst taucht das gelöschte Topic sofort wieder auf."""
import routes, qa
db = testdb
monkeypatch.setattr(routes, "topic_dir", lambda t: tmp_path / "topics" / t)
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
await db.create_topic(TOPIC)
await db.create_guide({"id": "g1", "topic": TOPIC, "format": "Guide", "instructions": "",
"status": "done", "progress": None,
"created_at": "2026-01-01", "updated_at": "2026-01-01"})
await db.upsert_guide_card(TOPIC, "Guide", "alpha", "Alpha")
await db.kanban_upsert_card(TOPIC, "inventory", "b-1", "block", "done_block", {"title": "Alpha"})
res = await routes.remove_topic(TOPIC)
assert res["ok"]
assert all(g["topic"] != TOPIC for g in await db.list_guides())
assert await db.list_guide_cards(TOPIC, "Guide") == []
assert await db.kanban_cards(TOPIC, "inventory") == []
assert TOPIC not in await routes.get_topics()
async def test_topic_delete_409_bei_laufendem_guide(testdb, tmp_path, monkeypatch):
"""Läuft eine Generierung, wird nicht gelöscht (409) — ein laufender Flow schrieb
sonst nach dem Löschen munter neue Rows/Dateien."""
import pytest
from fastapi import HTTPException
import routes
db = testdb
monkeypatch.setattr(routes, "topic_dir", lambda t: tmp_path / "topics" / t)
await db.create_guide({"id": "g1", "topic": TOPIC, "format": "Guide", "instructions": "",
"status": "generating", "progress": None,
"created_at": "2026-01-01", "updated_at": "2026-01-01"})
with pytest.raises(HTTPException) as e:
await routes.remove_topic(TOPIC)
assert e.value.status_code == 409
assert any(g["topic"] == TOPIC for g in await db.list_guides())
async def test_runs_endpoint_liefert_bilanz(testdb):
"""GET /api/runs: pro run_id Zeitspanne + Agent-/Token-Bilanz + Fails, jüngster zuerst,
aktiv-Flag aus dem Run-Registry."""
import routes
db = testdb
db.set_current_run(TOPIC, "r1")
await db.add_event(TOPIC, "agent", key="a", status="ok", dur_ms=1000,
meta={"tokens": {"input": 10, "output": 20, "cache_read": 0, "cache_write": 0}})
await db.add_event(TOPIC, "agent", key="b", status="timeout", dur_ms=120000)
await db.add_event(TOPIC, "fail", key="inventory:b-1", status="dead", meta={"error": "kaputt"})
res = await routes.get_runs(TOPIC)
runs = res["runs"]
assert len(runs) == 1 and runs[0]["run_id"] == "r1" and runs[0]["aktiv"] is True
assert runs[0]["agents"]["gesamt"] == 2 and runs[0]["agents"]["timeout"] == 1
assert runs[0]["tokens"]["output"] == 20
assert runs[0]["fails"][0]["error"] == "kaputt" and runs[0]["fails"][0]["status"] == "dead"
assert runs[0]["start"] <= runs[0]["ende"]
db.set_current_run(TOPIC, "r2")
await db.add_event(TOPIC, "agent", key="c", status="ok")
db.set_current_run(TOPIC, None)
runs = (await routes.get_runs(TOPIC))["runs"]
assert [r["run_id"] for r in runs] == ["r2", "r1"]
assert runs[0]["aktiv"] is False # Registry geräumt → Lauf beendet
async def test_health(testdb):
import routes
assert (await routes.health())["ok"] is True

View File

@@ -444,3 +444,78 @@ async def test_repair_karten_setzt_befundkarten_auf_pruefer(testdb, tmp_path, mo
assert cards["alpha"]["stage"] == cards["beta"]["stage"] == "pruefer"
assert cards["alpha"]["md"] # Text bleibt — der Prüfer arbeitet auf dem Bestand
assert cards["gamma"]["stage"] == "done"
async def test_fix_failed_behaelt_befunde(testdb, tmp_path, monkeypatch):
"""Scheitert der Fix, dürfen die Prüfer-Befunde nicht stumm verschwinden — sie
bleiben im gate_info sichtbar (vorher wurde gate_info geleert)."""
db = testdb
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
env = gb._Env(None, "g-ff", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
md = "<!-- section: Alpha -->\n<!-- ausführlich -->\nText."
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md,
"stage": "fix", "gate_info": "KRITISCH\n- Claim c1 (falsch): korrigieren"}
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
assert "-gfix-" in key
return gb.FAILED, None
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
assert await gb._stage_fix(env, card)
karte = (await db.list_guide_cards(TOPIC, FMT))[0]
assert karte["stage"] == "done"
assert "offene Befunde" in karte["gate_info"] and "Claim c1" in karte["gate_info"]
async def test_repruefer_ausfall_wird_vermerkt(testdb, tmp_path, monkeypatch):
"""Fällt der Re-Prüfer aus (kein Cancel), wird das im gate_info vermerkt statt die
Karte stumm als geprüft durchzuwinken."""
db = testdb
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
env = gb._Env(None, "g-ra", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
md = "<!-- section: Alpha -->\n<!-- ausführlich -->\n" + "Text im Rahmen. " * 20
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md,
"stage": "fix", "gate_info": "KRITISCH\n- Claim c1 (falsch): korrigieren"}
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
if "-gfix-" in key:
import re as _re
m = _re.search(r"(/\S+\.md)", prompt)
with open(m.group(1), "w", encoding="utf-8") as f:
f.write(md.replace("Text im Rahmen.", "Korrigiert."))
return gb.OK, payload(None)
return gb.FAILED, None # Re-Prüfer fällt aus
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
assert await gb._stage_fix(env, card)
karte = (await db.list_guide_cards(TOPIC, FMT))[0]
assert karte["stage"] == "done" and "Korrigiert." in karte["md"]
assert "Re-Prüfer ohne Ergebnis" in karte["gate_info"]
async def test_progress_reporter_ueberlebt_db_fehler(testdb, monkeypatch):
"""Ein DB-Fehler beendet den Reporter nicht (der Fortschritt fror sonst still ein);
unveränderter Stand wird nicht erneut geschrieben."""
import asyncio
db = testdb
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
calls = {"n": 0}
writes = []
orig = db.guide_stage_counts
async def flaky(topic, fmt):
calls["n"] += 1
if calls["n"] == 1:
raise RuntimeError("kaputt")
return await orig(topic, fmt)
async def fake_update(guide_id, **kw):
writes.append(kw["progress"])
monkeypatch.setattr(gb.db, "guide_stage_counts", flaky)
monkeypatch.setattr(gb.db, "update_guide", fake_update)
task = asyncio.create_task(gb._progress_reporter("g-pr", TOPIC, FMT, takt=0.01))
await asyncio.sleep(0.08)
task.cancel()
assert calls["n"] >= 3 # lief nach dem Fehler weiter
assert writes.count(writes[0]) == 1 # gleicher Stand nur einmal geschrieben

View File

@@ -227,7 +227,7 @@ async def test_unecht_braucht_doppelt_nein(testdb, tmp_path, monkeypatch):
{"title": titel, "description": "d"})
monkeypatch.setattr(qa, "QA_DIR", tmp_path)
async def fake_verdicts(template, topic, key, items):
async def fake_wave(template, topic, key, slot, items, **kw):
if template != "QA-Bausteine":
return {}
if key.startswith("bausteine-b2"): # Bestätiger sieht nur die Geflaggten
@@ -235,7 +235,7 @@ async def test_unecht_braucht_doppelt_nein(testdb, tmp_path, monkeypatch):
return {1: "nein", 2: "ja"} # nur der erste wird bestätigt
return {1: "nein", 2: "nein", 3: "ja"} # Pass 1 flaggt zwei
monkeypatch.setattr(qa, "_llm_verdicts", fake_verdicts)
monkeypatch.setattr(qa, "judge_wave", fake_wave)
report = await qa.qa_report("t", llm=True)
assert report["unecht"] == ["Wackelkandidat"]

View File

@@ -82,59 +82,6 @@ async def test_hedge_schwelle_skaliert_mit_timeout(monkeypatch):
assert calls == ["k1"]
async def test_late_fold_nachzuegler_zaehlt_nach(monkeypatch):
"""Quorum 2 kehrt sofort zurück; der dritte Slot wird nicht gekillt, sein Ergebnis
geht an `late` (ersetzt den grace-Timer der Finder-Runden)."""
import time
killed, spaet = [], []
async def fake_agent(key, prompt, timeout, **kw):
if key == "k3":
await asyncio.sleep(0.2)
return (0, "dritter", "")
return (0, key, "")
async def late(val):
spaet.append(val)
monkeypatch.setattr(pipeline, "run_agent", fake_agent)
monkeypatch.setattr(pipeline, "kill_process", lambda k: killed.append(k))
monkeypatch.setattr(pipeline, "_HEDGE_NACH_S", 0)
slots = [{"key": f"k{i}", "prompt": "p", "role": "quick", "capabilities": "none",
"payload": lambda r: r[1]} for i in (1, 2, 3)]
t0 = time.monotonic()
res = await pipeline._race("t", "Test", slots, 2, 60, "claude", late=late)
assert time.monotonic() - t0 < 0.15 # kein Warten auf k3
assert sorted(res) == ["k1", "k2"]
assert "k3" not in killed
await asyncio.sleep(0.3)
assert spaet == ["dritter"]
async def test_late_fold_invalider_nachzuegler_ignoriert(monkeypatch):
"""Nachzügler mit invalidem Payload löst late NICHT aus (best-effort)."""
spaet = []
async def fake_agent(key, prompt, timeout, **kw):
if key == "k3":
await asyncio.sleep(0.1)
return (1, "", "kaputt")
return (0, key, "")
async def late(val):
spaet.append(val)
monkeypatch.setattr(pipeline, "run_agent", fake_agent)
monkeypatch.setattr(pipeline, "kill_process", lambda k: None)
monkeypatch.setattr(pipeline, "_HEDGE_NACH_S", 0)
slots = [{"key": f"k{i}", "prompt": "p", "role": "quick", "capabilities": "none",
"payload": lambda r: r[1]} for i in (1, 2, 3)]
res = await pipeline._race("t", "Test", slots, 2, 60, "claude", late=late)
assert res is not None
await asyncio.sleep(0.25)
assert spaet == []
async def test_hedge_zwilling_ersetzt_restart(monkeypatch):
"""Scheitert das Original, während der Zwilling noch läuft, gibt es KEINEN
zusätzlichen Restart — der Zwilling ist der Retry."""

View File

@@ -67,7 +67,7 @@ async def test_merge_confirmed_duplicate(env, monkeypatch):
calls.append(prompt)
return 0, '{"relevant": {"1": "ja"}}', ""
monkeypatch.setattr(repair, "run_agent", fake_agent)
import agents; monkeypatch.setattr(agents, "run_agent", fake_agent)
res = await repair.repair_befunde(TOPIC)
assert res["merges"] == ["Alpha → Alpha Problem"]
assert len(calls) == 1 and "Beta" not in calls[0] # nur das llm=ja-Paar zum Judge
@@ -90,7 +90,7 @@ async def test_fremd_removed_only_on_nein(env, monkeypatch):
return 0, '{"relevant": {"1": "ja"}}', ""
return 0, '{"relevant": {"1": "nein", "2": "ja"}}', ""
monkeypatch.setattr(repair, "run_agent", fake_agent)
import agents; monkeypatch.setattr(agents, "run_agent", fake_agent)
monkeypatch.setattr(repair, "source_folder", lambda t: None)
res = await repair.repair_befunde(TOPIC)
assert res["entfernt"] == ["Fremdling"]
@@ -108,7 +108,7 @@ async def test_judge_failure_keeps_everything(env, monkeypatch):
async def broken_agent(key, prompt, timeout, **kw):
raise RuntimeError("boom")
monkeypatch.setattr(repair, "run_agent", broken_agent)
import agents; monkeypatch.setattr(agents, "run_agent", broken_agent)
res = await repair.repair_befunde(TOPIC)
assert res["entfernt"] == []
card = await db.kanban_get_card(TOPIC, "inventory", cid)
@@ -124,7 +124,7 @@ async def test_hygiene_cleans_title_norm_invariant(env, monkeypatch):
async def no_agent(*a, **kw):
raise AssertionError("Hygiene braucht keinen Agenten")
monkeypatch.setattr(repair, "run_agent", no_agent)
import agents; monkeypatch.setattr(agents, "run_agent", no_agent)
res = await repair.repair_befunde(TOPIC)
assert res["hygiene"] == ["**Fetter Titel** → Fetter Titel"]
card = await db.kanban_get_card(TOPIC, "inventory", cid)
@@ -177,7 +177,7 @@ async def test_sub_dubletten_merge(env, monkeypatch):
assert "Gibtsnicht" not in prompt
return 0, '{"relevant": {"1": "ja"}}', ""
monkeypatch.setattr(repair, "run_agent", fake_agent)
import agents; monkeypatch.setattr(agents, "run_agent", fake_agent)
res = await repair.repair_befunde(TOPIC)
assert res["sub_merges"] == ["Verlierer Sub → Gewinner Sub"]
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)}
@@ -199,7 +199,7 @@ async def test_sub_dubletten_zweitmeinung_nein(env, monkeypatch):
async def fake_agent(key, prompt, timeout, **kw):
return 0, '{"relevant": {"1": "nein"}}', ""
monkeypatch.setattr(repair, "run_agent", fake_agent)
import agents; monkeypatch.setattr(agents, "run_agent", fake_agent)
res = await repair.repair_befunde(TOPIC)
assert res["sub_merges"] == []
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)}
@@ -223,7 +223,7 @@ async def test_sub_dubletten_stichentscheid_faltet(env, monkeypatch):
return 0, '{"relevant": {"1": "ja"}}', ""
return 0, '{"relevant": {"1": "nein"}}', "" # Zweitmeinung widerspricht
monkeypatch.setattr(repair, "run_agent", fake_agent)
import agents; monkeypatch.setattr(agents, "run_agent", fake_agent)
res = await repair.repair_befunde(TOPIC)
assert res["sub_merges"] == ["Sub B → Sub A"]
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, norm)}
@@ -245,7 +245,7 @@ async def test_stichentscheid_behalten_persistiert_freispruch(env, monkeypatch):
async def fake_agent(key, prompt, timeout, **kw):
return 0, '{"relevant": {"1": "nein"}}', "" # beide Repair-Judges: behalten
monkeypatch.setattr(repair, "run_agent", fake_agent)
import agents; monkeypatch.setattr(agents, "run_agent", fake_agent)
res = await repair.repair_befunde(TOPIC)
assert res["sub_merges"] == [] and len(res["freigesprochen"]) == 1
frei = qa_mod.lade_freispruch(TOPIC)
@@ -269,7 +269,7 @@ async def test_fremd_stichentscheid_behalten(env, monkeypatch):
async def fake_agent(key, prompt, timeout, **kw):
return 0, '{"relevant": {"1": "ja"}}', "" # beide: belegt/behalten
monkeypatch.setattr(repair, "run_agent", fake_agent)
import agents; monkeypatch.setattr(agents, "run_agent", fake_agent)
res = await repair.repair_befunde(TOPIC)
assert res["entfernt"] == []
@@ -292,7 +292,7 @@ async def test_waisen_cleanup(env, monkeypatch):
async def no_agent(*a, **kw):
raise AssertionError("Aufräumen braucht keinen Agenten")
monkeypatch.setattr(repair, "run_agent", no_agent)
import agents; monkeypatch.setattr(agents, "run_agent", no_agent)
res = await repair.repair_befunde(TOPIC)
assert res["aufgeraeumt"] == 3
rest = {(r["sub_norm"], r["type"]) for r in await db.get_sub_artefakte(TOPIC)}

View File

@@ -143,8 +143,10 @@ def test_cited_evidence_lines_and_fallback(tmp_path):
def test_sink_json_writes_only_valid(tmp_path):
p = tmp_path / "level-final-c1.json"
from pipeline import _enum_map_schema
levels = _enum_map_schema("levels", ("beginner", "advanced", "expert"))
ok = blx._sink_json((0, 'Vorab {"levels": {"1": "beginner"}} nach', ""), p,
lambda d: blx._levels_schema(d, {1}))
lambda d: levels(d, {1}))
assert ok == {1: "beginner"}
assert json.loads(p.read_text(encoding="utf-8"))["levels"]["1"] == "beginner"
bad = blx._sink_json((0, "kein json", ""), tmp_path / "x.json", lambda d: d)

View File

@@ -25,9 +25,9 @@ def test_registry_spiegelt_config():
def test_creator_params_override_wirkt_im_subprozess():
out = subprocess.run(
[sys.executable, "-c", "import config; print(config.FACTS_CHUNK_SUBS, config.TIMEOUTS['subblock_check'][0])"],
[sys.executable, "-c", "import config; print(config.GATE_FIX_MIN, config.TIMEOUTS['subblock_check'][0])"],
capture_output=True, text=True, cwd=BACKEND,
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"FACTS_CHUNK_SUBS": 6, "TIMEOUT_subblock_check_base": 77}'})
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"GATE_FIX_MIN": 6, "TIMEOUT_subblock_check_base": 77}'})
assert out.stdout.split() == ["6", "77"], out.stderr