534 lines
27 KiB
Python
534 lines
27 KiB
Python
"""Verschmolzene Board-2-Calls (block_calls.py): Generate-Konsens, Verify-Faltung mit
|
||
Fix-Tail, Artefakte in einem Durchgang — Agenten gefaked, gegen Test-DB."""
|
||
|
||
import json
|
||
|
||
import pytest
|
||
|
||
import block_calls as bc
|
||
import blocks as blx
|
||
import board_artefacts as ba
|
||
from pipeline import FAILED, OK, GenContext
|
||
from textkit import _norm_title
|
||
|
||
TOPIC = "t"
|
||
|
||
|
||
def _ctx():
|
||
return GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||
|
||
|
||
def _sub(title, level="beginner", relevance="relevant", kp=None, cf=None):
|
||
return {"title": title, "level": level, "relevance": relevance,
|
||
"key_points": [f"kp {title}"] if kp is None else kp,
|
||
"prerequisites": "", "hurdles": "", "cited_facts": cf or [], "example_idea": ""}
|
||
|
||
|
||
# ── Schemas ─────────────────────────────────────────────────────────────────────────
|
||
|
||
def test_gen_schema_normalisiert():
|
||
"""Gültige Einträge werden normalisiert; ungültiges level/relevance fällt auf ""
|
||
(Stimme entfällt, der Sub bleibt); Einträge ohne Titel fliegen."""
|
||
out = bc._gen_schema({"subs": [
|
||
{"title": " **A** ", "level": "Beginner", "relevance": "RELEVANT",
|
||
"key_points": ["k", ""], "cited_facts": [{"text": "t", "source": " s "},
|
||
{"text": ""}, "quatsch"]},
|
||
{"title": "B", "level": "profi", "relevance": "mittel"},
|
||
{"title": " "},
|
||
]})
|
||
assert [e["title"] for e in out] == ["A", "B"]
|
||
assert out[0]["level"] == "beginner" and out[0]["relevance"] == "relevant"
|
||
assert out[0]["key_points"] == ["k"]
|
||
assert out[0]["cited_facts"] == [{"text": "t", "source": "s"}]
|
||
assert out[1]["level"] == "" and out[1]["relevance"] == ""
|
||
|
||
|
||
def test_gen_schema_kaputt_ist_none():
|
||
assert bc._gen_schema(None) is None
|
||
assert bc._gen_schema({"subs": "x"}) is None
|
||
assert bc._gen_schema({"subs": []}) is None
|
||
assert bc._gen_schema({"subs": [{"level": "beginner"}]}) is None # nur titellose Einträge
|
||
|
||
|
||
def test_verify_schema_pflichtkeys_und_leeres_verdikt():
|
||
"""Mindestens EIN Pflicht-Key muss da sein; leere Listen heißen „alles ok"."""
|
||
assert bc._verify_schema({}, 3) is None
|
||
assert bc._verify_schema({"irgendwas": 1}, 3) is None
|
||
v = bc._verify_schema({"gruppen": []}, 3)
|
||
assert v["gruppen"] == [] and v["fremd"] == set() and v["luecken"] == []
|
||
assert v["uebernehmen"] == {} and v["facts_probleme"] == [] and v["levels"] == {}
|
||
|
||
|
||
def test_verify_schema_grenzen_und_normalisierung():
|
||
"""ids außerhalb 1..n und bools fallen raus; Ein-Element-Gruppen zählen nicht;
|
||
uebernehmen/levels werden casefolded bzw. enum-geprüft."""
|
||
v = bc._verify_schema({
|
||
"gruppen": [{"haupt": 2, "weitere": [1, 9, True]}, {"haupt": 3, "weitere": []}],
|
||
"kataloge": [{"titel": " K ", "mitglieder": [1, 2]}, {"titel": "", "mitglieder": [1, 2]}],
|
||
"fremd": [True, 1, "2", 9],
|
||
"luecken": [" x ", "", 7],
|
||
"uebernehmen": {"3": " JA ", "9": "ja"},
|
||
"facts_probleme": [{"nr": 2, "discard": 1, "hinweis": " h "}, {"nr": 9}, "quatsch"],
|
||
"levels": {"1": "expert", "2": "quatsch"},
|
||
"relevanz": {"1": "peripheral"},
|
||
}, 3)
|
||
assert v["gruppen"] == [{"haupt": 2, "ids": [1, 2]}]
|
||
assert v["kataloge"] == [{"titel": "K", "ids": [1, 2]}]
|
||
assert v["fremd"] == {1, 2}
|
||
assert v["luecken"] == ["x"]
|
||
assert v["uebernehmen"] == {3: "ja"}
|
||
assert v["facts_probleme"] == [{"nr": 2, "discard": True, "hinweis": "h"}]
|
||
assert v["levels"] == {1: "expert"} and v["relevanz"] == {1: "peripheral"}
|
||
|
||
|
||
def test_art_gen_schema_pattern_ist_pflicht():
|
||
"""Ohne verwertbares pattern kein Verdikt (Leitner hängt an den Fragen);
|
||
cards/examples sind best-effort und werden einzeln validiert."""
|
||
assert bc._art_gen_schema({"cards": [], "examples": []}) is None
|
||
assert bc._art_gen_schema("x") is None
|
||
out = bc._art_gen_schema({
|
||
"pattern": [{"block": "B", "subblock": "S", "question": "F?"},
|
||
{"block": "B", "subblock": "", "question": "F?"}],
|
||
"cards": [{"block": "B", "subblock": "S", "question": "F?", "answer": "A"},
|
||
{"block": "B", "subblock": "S", "question": "F?"}],
|
||
"examples": [{"block": "B", "subblock": "S", "problem": "P", "steps": ["s1", ""], "result": ""},
|
||
{"block": "B", "subblock": "S", "problem": "P", "steps": []}],
|
||
})
|
||
assert len(out["pattern"]) == 1 and len(out["cards"]) == 1
|
||
assert out["examples"] == [{"block": "B", "subblock": "S", "problem": "P",
|
||
"steps": ["s1"], "result": ""}]
|
||
|
||
|
||
def test_art_check_schema_varianten():
|
||
"""{"ok": true} → leeres Verdikt; ohne bekannten Key None; Beispiel-Indizes sind
|
||
1-basiert, bools/0 zählen nicht."""
|
||
ok = bc._art_check_schema({"ok": True})
|
||
assert ok == {"pattern": [], "pattern_ergaenzt": [], "examples_probleme": set()}
|
||
assert bc._art_check_schema({"foo": 1}) is None
|
||
v = bc._art_check_schema({"examples_probleme": [1, "2", {"index": 3}, True, 0, -1],
|
||
"pattern_ergaenzt": [{"block": "B", "subblock": "S", "question": "F?"}]})
|
||
assert v["examples_probleme"] == {1, 2, 3}
|
||
assert len(v["pattern_ergaenzt"]) == 1 and v["pattern"] == []
|
||
|
||
|
||
# ── Generate ────────────────────────────────────────────────────────────────────────
|
||
|
||
@pytest.fixture
|
||
def env(testdb, tmp_path, monkeypatch):
|
||
"""Ohne Modell (exakte Norm-Gleichheit), ohne Korpus (thema-Selbst-Recherche)."""
|
||
monkeypatch.setattr(bc, "EMBEDDING_AKTIV", False)
|
||
monkeypatch.setattr(blx, "EMBEDDING_AKTIV", False) # _dedup_subblocks aus
|
||
monkeypatch.setattr(bc, "material_folder", lambda t: None)
|
||
monkeypatch.setattr(bc, "load_source", lambda t: {"type": "thema"})
|
||
return testdb, _ctx(), {"arbeit": tmp_path}
|
||
|
||
|
||
def _mk_enrich_slot(output):
|
||
"""run_single_slot-Fake für die Anreicherung: schreibt die gescriptete Antwort via
|
||
payload (wie der Engine-Sink); output=None → Ausfall."""
|
||
calls = []
|
||
|
||
async def fake(ctx, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
|
||
calls.append({"key": key, "prompt": prompt})
|
||
if output is None:
|
||
return FAILED, None
|
||
return OK, payload((0, json.dumps(output), ""))
|
||
|
||
fake.calls = calls
|
||
return fake
|
||
|
||
|
||
async def test_enrich_reichert_feste_subs_an(env, monkeypatch):
|
||
"""Die Subs stehen fest (Board 1). EIN Call füllt Facts + Level + Relevanz — er
|
||
erfindet und entfernt nichts; die Board-1-Subs bleiben in Reihenfolge."""
|
||
db, ctx, files = env
|
||
await _seed_rows(db, "alpha", ["Sub A", "Sub B"])
|
||
monkeypatch.setattr(bc, "run_single_slot", _mk_enrich_slot({"subs": [
|
||
_sub("Sub A", kp=["k1"]),
|
||
_sub("Sub B", level="expert", relevance="peripheral", kp=["k2"])]}))
|
||
gen = await bc._generate_block(ctx, files, "Alpha", "Grundkonzept")
|
||
assert gen["raw"] == {"Alpha": ["Sub A", "Sub B"]}
|
||
assert gen["facts"]["Alpha"]["sub a"]["key_points"] == ["k1"]
|
||
assert gen["unsicher"] == []
|
||
assert gen["votes"]["sub b"] == {"level": ["expert"], "relevance": ["peripheral"]}
|
||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||
assert rows["sub a"] == rows["sub b"] == "consensus"
|
||
|
||
|
||
async def test_enrich_ohne_subs_kein_call(env, monkeypatch):
|
||
"""Kein Board-1-Sub → nichts anzureichern, kein Modell-Call."""
|
||
db, ctx, files = env
|
||
called = []
|
||
|
||
async def spy(*a, **k):
|
||
called.append(1)
|
||
return OK, None
|
||
monkeypatch.setattr(bc, "run_single_slot", spy)
|
||
gen = await bc._generate_block(ctx, files, "Alpha", "d")
|
||
assert gen == {"raw": {"Alpha": []}, "facts": {"Alpha": {}}, "unsicher": [], "votes": {}}
|
||
assert called == []
|
||
|
||
|
||
async def test_enrich_ausgelassener_sub_bleibt_leer(env, monkeypatch):
|
||
"""Lässt der Call einen Sub aus, bleibt er erhalten (Vollständigkeit) — mit leeren
|
||
Facts, die der Prüfer/QA als Lücke sieht."""
|
||
db, ctx, files = env
|
||
await _seed_rows(db, "alpha", ["Sub A", "Sub B"])
|
||
monkeypatch.setattr(bc, "run_single_slot", _mk_enrich_slot({"subs": [_sub("Sub A", kp=["k1"])]}))
|
||
gen = await bc._generate_block(ctx, files, "Alpha", "d")
|
||
assert gen["raw"] == {"Alpha": ["Sub A", "Sub B"]}
|
||
assert gen["facts"]["Alpha"]["sub b"]["key_points"] == []
|
||
assert gen["facts"]["Alpha"]["sub b"]["cited_facts"] == []
|
||
|
||
|
||
async def test_enrich_resume_ohne_neue_calls(env, monkeypatch):
|
||
"""Vorhandene enrich-Datei → kein neuer Call, Ergebnis wird übernommen."""
|
||
db, ctx, files = env
|
||
await _seed_rows(db, "alpha", ["Sub A"])
|
||
fake = _mk_enrich_slot({"subs": [_sub("Sub A", kp=["k1"])]})
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
gen1 = await bc._generate_block(ctx, files, "Alpha", "d")
|
||
n = len(fake.calls)
|
||
gen2 = await bc._generate_block(ctx, files, "Alpha", "d")
|
||
assert len(fake.calls) == n # aus Datei resumed
|
||
assert gen2["raw"] == gen1["raw"]
|
||
|
||
|
||
# ── Verify (+ Fix-Tail) ─────────────────────────────────────────────────────────────
|
||
|
||
def _gen_von(title, subs, unsicher=None, votes=None):
|
||
"""Karten-Payload wie aus _generate_block: raw/facts/unsicher/votes."""
|
||
return {"raw": {title: list(subs)},
|
||
"facts": {title: {_norm_title(s): {"key_points": [f"kp {s}"], "prerequisites": "",
|
||
"hurdles": "", "cited_facts": [], "example_idea": ""}
|
||
for s in subs}},
|
||
"unsicher": unsicher or [], "votes": votes or {}}
|
||
|
||
|
||
def _judge_slot(antworten, fix=None):
|
||
"""run_single_slot-Fake: Prüfer-Antwort je j-Suffix, Fix-Antwort für -sb-fix-;
|
||
fehlender Eintrag = FAILED."""
|
||
calls = []
|
||
|
||
async def fake(ctx, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
|
||
calls.append({"key": key, "prompt": prompt})
|
||
if "-sb-fix-" in key:
|
||
if fix is None:
|
||
return FAILED, None
|
||
return OK, payload((0, json.dumps(fix), ""))
|
||
j = key.rsplit("-j", 1)[-1]
|
||
antwort = antworten.get(j)
|
||
if antwort is None:
|
||
return FAILED, None
|
||
return OK, payload((0, json.dumps(antwort), ""))
|
||
|
||
fake.calls = calls
|
||
return fake
|
||
|
||
|
||
async def _seed_rows(db, bnorm, titles, status="consensus"):
|
||
for t in titles:
|
||
await db.put_subblock(TOPIC, bnorm, _norm_title(t), bnorm.title(), t, status=status)
|
||
|
||
|
||
async def test_verify_gruppe_faltet_einstimmig(env, monkeypatch):
|
||
"""Beide Prüfer gruppieren 1+2 → haupt gewinnt, Verlierer wird variant und seine
|
||
Facts wandern per Union zum Gewinner; Dissens-Gruppen falten nicht."""
|
||
db, ctx, files = env
|
||
subs = ["Marker Regel", "Marker Regel im Detail erklärt", "Eigenes Thema"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
verdikt = {"gruppen": [{"haupt": 2, "weitere": [1]}]}
|
||
fake = _judge_slot({"1": verdikt, "2": {"gruppen": [{"haupt": 2, "weitere": [1]},
|
||
{"haupt": 3, "weitere": [2]}]}})
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {})
|
||
assert res["raw"] == {"Alpha": [subs[1], subs[2]]} # Gruppe 2+3 war einseitig → kein Fold
|
||
wf = res["facts"]["Alpha"][_norm_title(subs[1])]
|
||
assert wf["key_points"] == [f"kp {subs[1]}", f"kp {subs[0]}"] # Union geerbt
|
||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||
assert rows[_norm_title(subs[0])] == "variant"
|
||
assert rows[_norm_title(subs[1])] == "consensus"
|
||
|
||
|
||
async def test_verify_fremd_nur_einstimmig(env, monkeypatch):
|
||
"""Fremd 2/2 → discarded + raus; einseitig fremd → bleibt."""
|
||
db, ctx, files = env
|
||
subs = ["CSS Regel", "Echte Regel"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
fake = _judge_slot({"1": {"fremd": [1, 2]}, "2": {"fremd": [1]}})
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {})
|
||
assert res["raw"] == {"Alpha": ["Echte Regel"]}
|
||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||
assert rows["css regel"] == "discarded" and rows["echte regel"] == "consensus"
|
||
|
||
|
||
async def test_verify_keep_all_behaelt_alle_atome(env, monkeypatch):
|
||
"""Bottom-up (keep_all=True): kein Board-1-Atom wird entfernt oder gefaltet, selbst
|
||
wenn beide Prüfer fremd/gruppen/discard melden. Nur Facts-Korrektur (Hinweis) und
|
||
Level/Relevanz greifen weiter."""
|
||
db, ctx, files = env
|
||
subs = ["Sub A", "Sub B", "Sub C"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
# Fix-Call liefert die Korrektur für Sub A UND einen belegten NEUEN Fund — der darf
|
||
# bei keep_all NICHT als Sub eingeschleust werden (Sub-Zahl bleibt bei den Board-1-Atomen).
|
||
fix = {"subs": [_sub("Sub A", kp=["korrigiert"]),
|
||
_sub("Extra Fund", kp=["belegt"], cf=[{"text": "x", "source": "s"}])]}
|
||
verdikt = {"fremd": [3], "gruppen": [{"haupt": 1, "weitere": [2]}],
|
||
"facts_probleme": [{"nr": 1, "discard": True, "hinweis": "Zahl falsch"},
|
||
{"nr": 2, "discard": True}],
|
||
"levels": {"2": "expert"}}
|
||
fake = _judge_slot({"1": verdikt, "2": verdikt}, fix=fix)
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {}, keep_all=True)
|
||
assert res["raw"] == {"Alpha": subs} # exakt die drei Atome — nichts entfernt, nichts neu
|
||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||
assert all(rows[_norm_title(s)] == "consensus" for s in subs)
|
||
assert "extra fund" not in rows # der Fix-Neuzugang wurde unterdrückt
|
||
side = {s["title"]: s for s in res["sidecar"]["Alpha"]}
|
||
assert side["Sub A"]["facts"]["key_points"] == ["korrigiert"] # Hinweis trotz discard=True
|
||
assert side["Sub B"]["level"] == "expert" # Level-Korrektur greift
|
||
|
||
|
||
async def test_verify_facts_discard_nur_2von2(env, monkeypatch):
|
||
"""Facts-Discard ist irreversibel → nur 2/2; die einseitige Stimme ohne Hinweis
|
||
löst auch keinen Fix aus."""
|
||
db, ctx, files = env
|
||
subs = ["Sub A", "Sub B"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
fake = _judge_slot({"1": {"facts_probleme": [{"nr": 1, "discard": True},
|
||
{"nr": 2, "discard": True}]},
|
||
"2": {"facts_probleme": [{"nr": 1, "discard": True}]}})
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {})
|
||
assert res["raw"] == {"Alpha": ["Sub B"]}
|
||
assert not any("-sb-fix-" in c["key"] for c in fake.calls)
|
||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||
assert rows["sub a"] == "discarded" and rows["sub b"] == "consensus"
|
||
|
||
|
||
async def test_verify_korrektur_ab_einer_stimme(env, monkeypatch):
|
||
"""Ein Hinweis EINES Prüfers reicht: der Fix-Call läuft und ersetzt die Facts des
|
||
beanstandeten Subs; die Einstufung bleibt."""
|
||
db, ctx, files = env
|
||
subs = ["Sub A", "Sub B"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
fix = {"subs": [_sub("Sub A", kp=["korrigierte Aussage"])]}
|
||
fake = _judge_slot({"1": {"facts_probleme": [{"nr": 1, "hinweis": "Zahl falsch"}]},
|
||
"2": {"gruppen": []}}, fix=fix)
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {})
|
||
assert any("-sb-fix-" in c["key"] for c in fake.calls)
|
||
side = {s["title"]: s for s in res["sidecar"]["Alpha"]}
|
||
assert side["Sub A"]["facts"]["key_points"] == ["korrigierte Aussage"]
|
||
assert res["facts"]["Alpha"]["sub a"]["key_points"] == ["korrigierte Aussage"]
|
||
assert side["Sub B"]["facts"]["key_points"] == ["kp Sub B"] # unbeanstandet
|
||
|
||
|
||
async def test_verify_luecke_belegt_wird_neuer_sub(env, monkeypatch):
|
||
"""Lücken-Schnitt beider Prüfer → Fix legt den belegten Fund als neuen consensus-Sub
|
||
an; ein unbelegter „Fund" verfällt am Beleg-Gate."""
|
||
db, ctx, files = env
|
||
subs = ["Sub A"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
fix = {"subs": [_sub("Escaping von Sonderzeichen", level="expert", kp=["belegt"]),
|
||
_sub("Unbelegte Behauptung", kp=[])]}
|
||
fake = _judge_slot({"1": {"luecken": ["Escaping fehlt"]},
|
||
"2": {"luecken": ["Escaping unbehandelt"]}}, fix=fix)
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {})
|
||
assert res["raw"] == {"Alpha": ["Sub A", "Escaping von Sonderzeichen"]}
|
||
neu = next(s for s in res["sidecar"]["Alpha"] if s["title"] == "Escaping von Sonderzeichen")
|
||
assert neu["level"] == "expert" and neu["facts"]["key_points"] == ["belegt"]
|
||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||
assert rows[_norm_title("Escaping von Sonderzeichen")] == "consensus"
|
||
assert _norm_title("Unbelegte Behauptung") not in rows
|
||
|
||
|
||
async def test_verify_ersatzrichter_bei_ausfall(env, monkeypatch):
|
||
"""Fällt EIN Prüfer aus, springt der Ersatz jE ein — Einstimmigkeit mit ihm faltet."""
|
||
db, ctx, files = env
|
||
subs = ["CSS Regel", "Echte Regel"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
fake = _judge_slot({"1": {"fremd": [1]}, "E": {"fremd": [1]}}) # j2 → FAILED
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {})
|
||
assert res["raw"] == {"Alpha": ["Echte Regel"]}
|
||
assert [c["key"].rsplit("-j", 1)[-1] for c in fake.calls] == ["1", "2", "E"]
|
||
|
||
|
||
async def test_verify_failopen_verwirft_nur_unsicher(env, monkeypatch):
|
||
"""Nur 1 Prüfer (auch der Ersatz fällt aus) → fail-open: consensus bleibt unangetastet,
|
||
unsicher wird verworfen (ohne Panel keine Übernahme-Entscheidung)."""
|
||
db, ctx, files = env
|
||
subs = ["Sub A"]
|
||
unsicher = [_sub("Unsicher B")]
|
||
await _seed_rows(db, "alpha", subs)
|
||
await _seed_rows(db, "alpha", ["Unsicher B"], status="candidate")
|
||
fake = _judge_slot({"1": {"fremd": [1], "uebernehmen": {"2": "ja"}}}) # j2+jE → FAILED
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs, unsicher=unsicher), {})
|
||
assert res["raw"] == {"Alpha": ["Sub A"]} # fremd-Einzelstimme wirkt NICHT
|
||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||
assert rows["sub a"] == "consensus" and rows["unsicher b"] == "discarded"
|
||
|
||
|
||
async def test_verify_level_korrektur_wiegt_doppelt(env, monkeypatch):
|
||
"""Explizite Prüfer-Korrektur (×2) schlägt die Generator-Stimme; Patt fällt auf
|
||
advanced/relevant (heutige Defaults)."""
|
||
db, ctx, files = env
|
||
subs = ["Sub A", "Sub B"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
votes = {"sub a": {"level": ["beginner"], "relevance": []},
|
||
"sub b": {"level": ["beginner", "expert"], "relevance": []}}
|
||
fake = _judge_slot({"1": {"levels": {"1": "expert"}}, "2": {"gruppen": []}})
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs, votes=votes), {})
|
||
side = {s["title"]: s for s in res["sidecar"]["Alpha"]}
|
||
assert side["Sub A"]["level"] == "expert" # 2× Korrektur > 1× Generator
|
||
assert side["Sub B"]["level"] == "advanced" # 1:1-Patt → Default
|
||
assert side["Sub A"]["relevance"] == "relevant" # keine Stimme → Default
|
||
|
||
|
||
async def test_verify_resume_ohne_neue_calls(env, monkeypatch):
|
||
"""Vorhandene verify-j-Dateien → kein neuer Prüfer-Call."""
|
||
db, ctx, files = env
|
||
subs = ["Sub A"]
|
||
await _seed_rows(db, "alpha", subs)
|
||
fake = _judge_slot({"1": {"gruppen": []}, "2": {"gruppen": []}})
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {})
|
||
n = len(fake.calls)
|
||
await bc._verify_block(ctx, files, "Alpha", _gen_von("Alpha", subs), {})
|
||
assert len(fake.calls) == n
|
||
|
||
|
||
# ── Artefakte ───────────────────────────────────────────────────────────────────────
|
||
|
||
def _sidecar(titles):
|
||
return [{"title": t, "level": "beginner", "relevance": "relevant",
|
||
"facts": {"key_points": [f"kp {t}"]}} for t in titles]
|
||
|
||
|
||
def _art_slot(gen_out, check_out):
|
||
"""run_single_slot-Fake für Artefakte: gen_out je Teil (dict oder callable(prompt)),
|
||
check_out fürs Prüfer-Verdikt."""
|
||
calls = []
|
||
|
||
async def fake(ctx, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
|
||
calls.append({"key": key, "prompt": prompt})
|
||
if "-art-gen-" in key:
|
||
out = gen_out(prompt) if callable(gen_out) else gen_out
|
||
return OK, payload((0, json.dumps(out), ""))
|
||
if "-art-check-" in key:
|
||
if check_out is None:
|
||
return FAILED, None
|
||
return OK, payload((0, json.dumps(check_out), ""))
|
||
raise AssertionError(f"unerwarteter Call {key}")
|
||
|
||
fake.calls = calls
|
||
return fake
|
||
|
||
|
||
async def test_artefakte_ein_call_liefert_alles(env, monkeypatch):
|
||
"""EIN Generator-Call liefert pattern+cards+examples, der Prüfer sagt ok →
|
||
Rohfassung wird übernommen, block-Feld auf den Karten-Block normiert."""
|
||
db, ctx, files = env
|
||
gen_out = {"pattern": [{"block": "Echo", "subblock": "Sub A", "question": "F?"}],
|
||
"cards": [{"block": "Echo", "subblock": "Sub A", "question": "F?", "answer": "A"}],
|
||
"examples": [{"block": "Echo", "subblock": "Sub A", "problem": "P",
|
||
"steps": ["s1"], "result": "R"}]}
|
||
fake = _art_slot(gen_out, {"ok": True})
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._artefakte_block(ctx, files, "Alpha", _sidecar(["Sub A"]))
|
||
assert [c["key"] for c in fake.calls if "-art-gen-" in c["key"]].__len__() == 1
|
||
assert res["pattern"] == {"Alpha": [{"subblock": "Sub A", "question": "F?"}]}
|
||
assert res["artefacts"]["flashcard"] == [{"block": "Alpha", "subblock": "Sub A",
|
||
"question": "F?", "answer": "A"}]
|
||
assert res["artefacts"]["example"][0]["block"] == "Alpha" # Agent-Echo „Echo" normiert
|
||
|
||
|
||
async def test_artefakte_check_entfernt_beispiel_und_ergaenzt_frage(env, monkeypatch):
|
||
"""examples_probleme wirft das beanstandete Beispiel; pattern_ergaenzt füllt die
|
||
fehlende Frage nach — der Prüfer-Prompt listet den fraglosen Sub."""
|
||
db, ctx, files = env
|
||
gen_out = {"pattern": [{"block": "Alpha", "subblock": "Sub A", "question": "F?"}],
|
||
"cards": [],
|
||
"examples": [{"block": "Alpha", "subblock": "Sub A", "problem": "P1", "steps": ["x"], "result": ""},
|
||
{"block": "Alpha", "subblock": "Sub A", "problem": "P2", "steps": ["y"], "result": ""}]}
|
||
check = {"examples_probleme": [{"index": 1}],
|
||
"pattern_ergaenzt": [{"block": "Alpha", "subblock": "Sub B", "question": "F B?"}]}
|
||
fake = _art_slot(gen_out, check)
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._artefakte_block(ctx, files, "Alpha", _sidecar(["Sub A", "Sub B"]))
|
||
assert [e["problem"] for e in res["artefacts"]["example"]] == ["P2"]
|
||
assert res["pattern"]["Alpha"] == [{"subblock": "Sub A", "question": "F?"},
|
||
{"subblock": "Sub B", "question": "F B?"}]
|
||
check_prompt = next(c["prompt"] for c in fake.calls if "-art-check-" in c["key"])
|
||
assert "STILL MISSING A QUESTION" in check_prompt and "Sub B" in check_prompt
|
||
|
||
|
||
async def test_artefakte_split_ab_schwelle(env, monkeypatch):
|
||
"""> ART_SPLIT_SUBS Subs → ZWEI parallele Generator-Calls, jeder sieht nur seine
|
||
Hälfte; die Ergebnisse werden zusammengeführt."""
|
||
db, ctx, files = env
|
||
monkeypatch.setattr(bc, "ART_SPLIT_SUBS", 2)
|
||
|
||
def gen_out(prompt):
|
||
subs = [t for t in ("Sub A", "Sub B", "Sub C") if f"- {t}" in prompt]
|
||
return {"pattern": [{"block": "Alpha", "subblock": s, "question": f"F {s}?"} for s in subs],
|
||
"cards": [], "examples": []}
|
||
|
||
fake = _art_slot(gen_out, {"ok": True})
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._artefakte_block(ctx, files, "Alpha", _sidecar(["Sub A", "Sub B", "Sub C"]))
|
||
gen_keys = [c["key"] for c in fake.calls if "-art-gen-" in c["key"]]
|
||
assert len(gen_keys) == 2 and gen_keys[0].endswith("-t1") and gen_keys[1].endswith("-t2")
|
||
assert [p["subblock"] for p in res["pattern"]["Alpha"]] == ["Sub A", "Sub B", "Sub C"]
|
||
|
||
|
||
async def test_artefakte_check_ausfall_uebernimmt_rohfassung(env, monkeypatch):
|
||
"""Prüfer ohne Ergebnis → fail-open, die Generator-Rohfassung zählt."""
|
||
db, ctx, files = env
|
||
gen_out = {"pattern": [{"block": "Alpha", "subblock": "Sub A", "question": "F?"}],
|
||
"cards": [], "examples": []}
|
||
fake = _art_slot(gen_out, None)
|
||
monkeypatch.setattr(bc, "run_single_slot", fake)
|
||
res = await bc._artefakte_block(ctx, files, "Alpha", _sidecar(["Sub A"]))
|
||
assert res["pattern"] == {"Alpha": [{"subblock": "Sub A", "question": "F?"}]}
|
||
|
||
|
||
async def test_artefakte_leerer_block(env):
|
||
db, ctx, files = env
|
||
res = await bc._artefakte_block(ctx, files, "Alpha", [])
|
||
assert res == {"pattern": {"Alpha": []}, "artefacts": {"flashcard": [], "example": []}}
|
||
|
||
|
||
# ── Migration der alten Stage-Treppe ────────────────────────────────────────────────
|
||
|
||
async def test_migriere_alt_karten(testdb):
|
||
"""Karten in alten Stages gehen mit reduziertem Payload zurück nach generate
|
||
(alte Zwischenstände sind für die verschmolzenen Calls wertlos); Terminal- und
|
||
Neu-Struktur-Karten bleiben unangetastet."""
|
||
db = testdb
|
||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "facts",
|
||
{"title": "Alpha", "description": "d", "n_size": 3,
|
||
"sources": ["s1"], "raw": {"Alpha": ["alt"]},
|
||
"facts": {"Alpha": {}}})
|
||
await db.kanban_upsert_card(TOPIC, "artefacts", "beta", "ablock", "question_pattern",
|
||
{"title": "Beta", "sidecar": {}})
|
||
await db.kanban_upsert_card(TOPIC, "artefacts", "gamma", "ablock", "done_artefact",
|
||
{"title": "Gamma", "raw": {"Gamma": ["bleibt"]}})
|
||
await db.kanban_upsert_card(TOPIC, "artefacts", "delta", "ablock", "verify",
|
||
{"title": "Delta", "raw": {"Delta": ["neu"]}})
|
||
n = await ba.migriere_alt_karten(TOPIC)
|
||
assert n == 2
|
||
alpha = await db.kanban_get_card(TOPIC, "artefacts", "alpha")
|
||
assert alpha["stage"] == "generate"
|
||
assert alpha["payload"] == {"title": "Alpha", "description": "d", "n_size": 3, "sources": ["s1"]}
|
||
assert (await db.kanban_get_card(TOPIC, "artefacts", "beta"))["stage"] == "generate"
|
||
gamma = await db.kanban_get_card(TOPIC, "artefacts", "gamma")
|
||
assert gamma["stage"] == "done_artefact" and gamma["payload"]["raw"] == {"Gamma": ["bleibt"]}
|
||
delta = await db.kanban_get_card(TOPIC, "artefacts", "delta")
|
||
assert delta["stage"] == "verify" and delta["payload"]["raw"] == {"Delta": ["neu"]}
|