marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Team3
2026-07-04 23:47:04 +02:00
parent 2d9ca00b47
commit 4105146c59
40 changed files with 2053 additions and 3208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

357
backend/tests/test_subblocks.py
View File

@@ -1,16 +1,14 @@
"""Subbaustein-Qualität: Varianten-Konsens, Seed-Garantie, Nachfass, Outline-Review."""
"""Subbaustein-Helfer: Varianten-Cluster, Evidence-Packs, Outline-Review, Hash/Key-Auflösung.
Die verschmolzene Block-Pipeline (Generate/Verify/Artefakte) testet tests/test_block_calls.py."""
import json
import re
import numpy as np
import pytest
import blocks as blx
from pipeline import GenContext
TOPIC = "t"
_MD_PATH = re.compile(r"(/\S+\.md)")
# ── _variant_clusters (pure) ─────────────────────────────────────────────────────────
@@ -37,157 +35,6 @@ def test_variant_clusters_negation_guard():
assert len(cl) == 2 # antonyms never merge, no matter the cosine
# ── _subblocks_block integration (fake race + fake embeddings) ──────────────────────
def _fake_sims(texts):
"""Markertoken matrix: same first word → 0.95, else 0."""
n = len(texts)
m = np.eye(n)
key = lambda t: t.split()[0].casefold()
for i in range(n):
for j in range(n):
if i != j and key(texts[i]) == key(texts[j]):
m[i][j] = 0.95
return m
def _mk_race(finder_by_agent):
"""Key-routed _race fake. Finder round 1 → scripted per-agent subs; later finder and
catch-up rounds → nothing; clarify judges echo the consensus lines from their prompt."""
prompts = []
async def fake_race(topic, label, slots, quorum, timeout, provider, on_update=None,
cancelled=None, *, grace=None, min_runtime=None, max_runtime=None, late=None):
outs = []
for slot in slots:
key, prompt = slot["key"], slot["prompt"]
prompts.append((key, prompt))
fake_race.slots_seen.append(slot)
if "-subblock-final-" in key:
# no-tool judges reply as TEXT; the payload sink writes the j-file itself
kons = re.search(r"Konsens \(≥2 finders\):\n(.*?)\nUnsicher", prompt, re.S)
subs = [l[2:] for l in (kons.group(1).splitlines() if kons else [])
if l.startswith("- ") and l != "- (keiner)"]
if subs:
text = "<!-- block: Alpha -->\n" + "\n".join(f"- {s}" for s in subs)
outs.append(slot["payload"]((0, text, "")))
continue
if "-r1-" in key:
agent = int(key.rsplit("-", 1)[1])
subs = finder_by_agent.get(agent) or []
if subs: # Finder antworten als TEXT (Marker-Format), kein out_path mehr
text = "<!-- block: Alpha -->\n" + "\n".join(f"- {s}" for s in subs)
outs.append(slot["payload"]((0, text, "")))
outs = [o for o in outs if o]
return outs or None
fake_race.slots_seen = []
return fake_race, prompts
@pytest.fixture
def sub_env(testdb, tmp_path, monkeypatch):
monkeypatch.setattr(blx, "EMBEDDING_AKTIV", True)
monkeypatch.setattr(blx.embedding, "available", lambda: True)
monkeypatch.setattr(blx.embedding, "embed_sims", _fake_sims)
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
files = {"arbeit": tmp_path}
return testdb, ctx, files
async def _run(ctx, files, monkeypatch, finder_by_agent, seeds=None):
fake, prompts = _mk_race(finder_by_agent)
monkeypatch.setattr(blx, "_race", fake)
raw = await blx._subblocks_block(ctx, lambda *a, **k: None, files, {1: "Alpha — Grundkonzept"},
"", wipe=False, ns="x-", seeds=seeds)
return raw, prompts
async def test_variant_consensus_end_to_end(sub_env, monkeypatch):
"""3 Einzelfunde in 3 Formulierungen → EIN consensus-Repräsentant; Varianten gehen
nicht als „Unsicher" ins Panel."""
db, ctx, files = sub_env
raw, prompts = await _run(ctx, files, monkeypatch, {
1: ["Umbruch braucht Marker"],
2: ["Umbruch erfordert explizite Marker!"],
3: ["Umbruch verlangt zwei Leerzeichen als Marker"],
})
assert raw == {"Alpha": ["Umbruch verlangt zwei Leerzeichen als Marker"]} # longest = rep
rows = await db.list_subblocks(TOPIC, "alpha")
status = sorted(r["status"] for r in rows)
assert status == ["consensus", "variant", "variant"]
clarify_prompts = [p for k, p in prompts if "-subblock-final-" in k]
assert clarify_prompts and "Umbruch braucht Marker" not in clarify_prompts[0]
async def test_seed_promotes_single_find(sub_env, monkeypatch):
"""Seed deckt einen verworfenen Einzelfund lexikalisch → Promotion zu consensus."""
db, ctx, files = sub_env
raw, _ = await _run(ctx, files, monkeypatch, {
1: ["Alpha Grundlagen", "Zeilenumbruch Regeln im Detail"],
2: ["Alpha Grundlagen"],
}, seeds=["Zeilenumbruch Regeln"])
assert "Zeilenumbruch Regeln im Detail" in raw["Alpha"]
row = next(r for r in await db.list_subblocks(TOPIC, "alpha")
if r["sub_norm"] == "zeilenumbruch regeln im detail")
assert row["status"] == "consensus"
async def test_seed_inserted_when_nothing_found(sub_env, monkeypatch):
"""Seed ohne jeden Fund wird als eigener consensus-Sub eingefügt (Facts-Gate prüft später)."""
db, ctx, files = sub_env
raw, _ = await _run(ctx, files, monkeypatch, {
1: ["Alpha Grundlagen"], 2: ["Alpha Grundlagen"],
}, seeds=["Fußnoten Syntax"])
assert "Fußnoten Syntax" in raw["Alpha"]
row = next(r for r in await db.list_subblocks(TOPIC, "alpha")
if r["sub_title"] == "Fußnoten Syntax")
assert row["status"] == "consensus"
async def test_seed_covered_no_duplicate(sub_env, monkeypatch):
"""Seed lexikalisch von einem consensus-Sub abgedeckt → nichts eingefügt."""
db, ctx, files = sub_env
raw, _ = await _run(ctx, files, monkeypatch, {
1: ["Tabs werden zu Leerzeichen expandiert"], 2: ["Tabs werden zu Leerzeichen expandiert"],
}, seeds=["Tabs"])
assert raw == {"Alpha": ["Tabs werden zu Leerzeichen expandiert"]}
async def test_wipe_false_is_idempotent(sub_env, monkeypatch):
"""Zweiter Karten-Lauf kumuliert keine Mentions (per-Block-Wipe)."""
db, ctx, files = sub_env
await _run(ctx, files, monkeypatch, {1: ["Alpha Grundlagen"], 2: ["Alpha Grundlagen"]})
first = {r["sub_norm"]: r["mentions"] for r in await db.list_subblocks(TOPIC, "alpha")}
await _run(ctx, files, monkeypatch, {1: ["Alpha Grundlagen"], 2: ["Alpha Grundlagen"]})
second = {r["sub_norm"]: r["mentions"] for r in await db.list_subblocks(TOPIC, "alpha")}
assert first == second
async def test_catchup_adds_and_stops(sub_env, monkeypatch, tmp_path):
"""Block unter SUBBLOCK_MIN: Nachfass-Runde findet Neues → eigenes Final-File,
Konsens wächst; zweite Runde ohne Neues → Ende."""
db, ctx, files = sub_env
fake, prompts = _mk_race({1: ["Alpha Grundlagen"], 2: ["Alpha Grundlagen"]})
base = fake
hit = {"n": 0}
async def with_catchup(topic, label, slots, *a, **k):
if any("-subblock-x" in s["key"] for s in slots):
hit["n"] += 1
if hit["n"] == 1: # first catch-up round: both agents agree on one new sub
text = "<!-- block: Alpha -->\n- Vertiefung der Konzepte"
return [slot["payload"]((0, text, "")) for slot in slots[:2]]
return None
return await base(topic, label, slots, *a, **k)
monkeypatch.setattr(blx, "_race", with_catchup)
raw = await blx._subblocks_block(ctx, lambda *a, **k: None, files, {1: "Alpha — Grundkonzept"},
"", wipe=False, ns="x-")
assert set(raw["Alpha"]) == {"Alpha Grundlagen", "Vertiefung der Konzepte"}
assert (tmp_path / "subblock-final-c1-x1.md").exists()
assert hit["n"] == 2 # round 2 ran, found nothing, loop ended
# ── Outline-Review ───────────────────────────────────────────────────────────────────
def test_outline_review_schema():
@@ -238,52 +85,8 @@ async def test_outline_review_moves_block(testdb, tmp_path, monkeypatch):
plan2 = await blx._outline_block(ctx, lambda *a, **k: None, files, entries, "")
assert plan2["chapters"][0]["numbers"] == [1, 2, 6]
async def test_paraphrase_saturation_stops_early(sub_env, monkeypatch):
"""Runde 2 liefert nur eine Paraphrase → zählt nicht als neu, Schleife endet ohne r3.
Die Paraphrase liegt trotzdem in der DB (Mention fürs Cluster-Voting)."""
db, ctx, files = sub_env
base_fake, prompts = _mk_race({1: ["Umbruch braucht Marker"], 2: ["Umbruch braucht Marker"]})
async def with_r2(topic, label, slots, *a, **k):
if any("-r2-" in s["key"] for s in slots):
text = "<!-- block: Alpha -->\n- Umbruch erfordert explizite Marker!"
return [slot["payload"]((0, text, "")) for slot in slots[:2]]
return await base_fake(topic, label, slots, *a, **k)
monkeypatch.setattr(blx, "_race", with_r2)
raw = await blx._subblocks_block(ctx, lambda *a, **k: None, files, {1: "Alpha — Grundkonzept"},
"", wipe=False, ns="x-")
assert raw["Alpha"] # Konsens steht
assert not any("-r3-" in k for k, _ in prompts) # Paraphrase hielt die Schleife NICHT am Leben
rows = await db.list_subblocks(TOPIC, "alpha")
assert any(r["sub_title"] == "Umbruch erfordert explizite Marker!" for r in rows)
async def test_round_cap_stops_endless_finders(sub_env, monkeypatch):
"""Jede Runde ein echt neues Konzept → hartes Cap stoppt bei SUBBLOCK_MAX_ROUNDS."""
db, ctx, files = sub_env
_, prompts = _mk_race({})
async def endless(topic, label, slots, *a, **k):
if "-subblock-final-" in slots[0]["key"]:
return None # panel fails → consensus fallback
outs = []
import re as _re
rn = _re.search(r"-r(\d+)-", slots[0]["key"])
n = rn.group(1) if rn else "x"
for slot in slots[:2]:
prompts.append((slot["key"], slot["prompt"]))
outs.append(slot["payload"]((0, f"<!-- block: Alpha -->\n- Konzept{n} ist eigenständig", "")))
return outs
monkeypatch.setattr(blx, "_race", endless)
await blx._subblocks_block(ctx, lambda *a, **k: None, files, {1: "Alpha — Grundkonzept"},
"", wipe=False, ns="x-")
max_round = max(int(k.split("-r")[1].split("-")[0]) for k, _ in prompts if "-r" in k and "-subblock-c" in k)
assert max_round == blx.SUBBLOCK_MAX_ROUNDS
# ── Inline-Evidenz für Judges (Token-Umbau) ──────────────────────────────────────────
# ── Inline-Evidenz (Evidence-Packs für die verschmolzenen Calls) ─────────────────────
def _corpus(tmp_path):
d = tmp_path / "korpus"
@@ -348,51 +151,6 @@ def test_sink_json_writes_only_valid(tmp_path):
assert bad is None and not (tmp_path / "x.json").exists()
async def test_clarify_inline_evidence_no_tools(sub_env, monkeypatch, tmp_path):
"""Mit Korpus: Judges UND Finder bekommen Auszüge inline und laufen ohne Tools
(Text-Antwort); die Dateien schreibt die Engine. Der Finder verlor vorher 3–13
Tool-Runden pro Call mit der Material-Suche via glob/grep/bash."""
db, ctx, files = sub_env
d = _corpus(tmp_path)
monkeypatch.setattr(blx, "source_folder", lambda t: d)
monkeypatch.setattr(blx, "load_source", lambda t: {"type": "uni"})
fake, prompts = _mk_race({1: ["Alpha Grundlagen"], 2: ["Alpha Grundlagen"]})
monkeypatch.setattr(blx, "_race", fake)
raw = await blx._subblocks_block(ctx, lambda *a, **k: None, files, {1: "Alpha — Grundkonzept"},
"", wipe=False, ns="x-", sources=["Skript.txt"])
assert raw == {"Alpha": ["Alpha Grundlagen"]}
judges = [s for s in fake.slots_seen if "-subblock-final-" in s["key"]]
finders = [s for s in fake.slots_seen if "-r1-" in s["key"]]
assert judges and all(s["capabilities"] == "none" for s in judges)
assert "── Skript.txt" in judges[0]["prompt"]
assert "ls/find" not in judges[0]["prompt"] # keine Selbst-Recherche-Anweisung mehr
assert finders and all(s["capabilities"] == "none" for s in finders)
assert "── Skript.txt" in finders[0]["prompt"] # Auszüge inline statt Dateisystem-Suche
assert "web search" not in finders[0]["prompt"]
assert list(tmp_path.glob("subblock-final-*-j*.md")) # Engine persistiert die Judge-Antwort
async def test_thema_nutzt_research_material_inline(sub_env, monkeypatch, tmp_path):
"""thema mit Research-Fundstellen (arbeit/material/*.txt): Finder bekommt sie inline
und läuft ohne Tools — vorher eigene Websuche pro Call (Reasoning-Schleifen, Retries)."""
db, ctx, files = sub_env
monkeypatch.setattr(blx, "source_folder", lambda t: None)
md = tmp_path / "arbeit" / "material"
md.mkdir(parents=True)
(md / "research-1.txt").write_text(
"https://example.org/alpha\nAlpha Grundlagen: der Kernbegriff, gut belegt.\n",
encoding="utf-8")
monkeypatch.setattr(blx, "arbeit_dir", lambda t: tmp_path / "arbeit")
fake, prompts = _mk_race({1: ["Alpha Grundlagen"], 2: ["Alpha Grundlagen"]})
monkeypatch.setattr(blx, "_race", fake)
raw = await blx._subblocks_block(ctx, lambda *a, **k: None, files, {1: "Alpha — Grundkonzept"},
"", wipe=False, ns="x-")
assert raw == {"Alpha": ["Alpha Grundlagen"]}
finders = [s for s in fake.slots_seen if "-r1-" in s["key"]]
assert finders and all(s["capabilities"] == "none" for s in finders)
assert "── research-1.txt" in finders[0]["prompt"] # Fundstellen inline
def test_material_folder_fallbacks(monkeypatch, tmp_path):
"""Echte Quelle gewinnt; sonst arbeit/material mit Inhalt; sonst None."""
monkeypatch.setattr(blx, "source_folder", lambda t: tmp_path / "quelle")
@@ -407,115 +165,6 @@ def test_material_folder_fallbacks(monkeypatch, tmp_path):
assert blx.material_folder("t") == md
async def test_panel_2of3_kehrt_bei_einigkeit_zurueck(tmp_path):
"""Zwei übereinstimmende Verdicts → Rückkehr ohne den langsamen Dritten; sein
Ergebnis wird detached nachpersistiert (Resume)."""
import asyncio as aio
gesunken = {}
async def judge(j, delay, antwort):
await aio.sleep(delay)
return (0, antwort, "")
tasks = {aio.create_task(judge(1, 0.01, "a")): 1,
aio.create_task(judge(2, 0.02, "a")): 2,
aio.create_task(judge(3, 5.0, "b")): 3}
def sink(j, r):
gesunken[j] = r[1]
import time
t0 = time.monotonic()
await blx._panel_2of3(tasks, sink, lambda: list(gesunken.values()), lambda s: s)
assert time.monotonic() - t0 < 1.0 # nicht auf j3 gewartet
assert gesunken == {1: "a", 2: "a"}
async def test_panel_2of3_dissens_wartet_auf_dritten():
"""Uneinige erste zwei → der dritte wird abgewartet (Mehrheit braucht ihn)."""
import asyncio as aio
gesunken = {}
async def judge(j, delay, antwort):
await aio.sleep(delay)
return (0, antwort, "")
tasks = {aio.create_task(judge(1, 0.01, "a")): 1,
aio.create_task(judge(2, 0.02, "b")): 2,
aio.create_task(judge(3, 0.1, "a")): 3}
await blx._panel_2of3(tasks, lambda j, r: gesunken.__setitem__(j, r[1]),
lambda: list(gesunken.values()), lambda s: s)
assert gesunken == {1: "a", 2: "b", 3: "a"}
async def test_facts_check_inline_cited_evidence(sub_env, monkeypatch, tmp_path):
"""Facts-Check: zitierte Zeilenbereiche gehen inline mit, Judge läuft ohne Tools,
die Check-Datei schreibt die Engine aus der Text-Antwort."""
db, ctx, files = sub_env
d = _corpus(tmp_path)
monkeypatch.setattr(blx, "source_folder", lambda t: d)
facts = {"facts": [{"block": "Alpha", "subblock": "Sub Eins", "key_points": ["k"],
"prerequisites": "", "hurdles": "",
"cited_facts": [{"text": "T", "source": "Skript.txt, Z.2"}],
"example_idea": ""}]}
sh = blx._subs_hash({"Alpha": ["Sub Eins"]}) # Resume-Dateien tragen den Sub-Satz-Hash
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
if "-facts-erg-" in key:
return blx.FAILED, None
(tmp_path / f"facts-{sh}-c0.json").write_text(json.dumps(facts), encoding="utf-8")
return blx.OK, None
seen = []
async def fake_agent(key, prompt, timeout, provider="", role="", capabilities="", scope=None, label="", **kw):
seen.append((key, capabilities, prompt))
return (0, '{"ok": true}', "")
monkeypatch.setattr(blx, "run_single_slot", fake_slot)
monkeypatch.setattr(blx, "run_agent", fake_agent)
res = await blx._facts_block(ctx, lambda *a, **k: None, {"arbeit": tmp_path},
{"Alpha": ["Sub Eins"]}, {"type": "uni"}, d, "", ns="x-")
assert res is not None
facts_map, discarded = res
assert "Alpha" in facts_map and not discarded
assert len(seen) == blx.FACTS_CHECK_PANEL
key, caps, prompt = seen[0]
assert caps == "none" and "── Skript.txt · Z." in prompt
assert (tmp_path / f"facts-check-{sh}-c0-j1.json").exists() # Engine persistiert die Antwort
async def test_facts_find_inline_evidence_no_tools(sub_env, monkeypatch, tmp_path):
"""Facts find/erg mit Korpus: Auszüge inline, Agent ohne Tools — Tool-Agenten
verloren sich in Reasoning-Schleifen und endeten mit leerem Turn (Retry-Wellen)."""
db, ctx, files = sub_env
d = _corpus(tmp_path)
monkeypatch.setattr(blx, "source_folder", lambda t: d)
seen = []
facts = {"facts": [{"block": "Alpha", "subblock": "Sub Eins", "key_points": ["k"],
"prerequisites": "", "hurdles": "",
"cited_facts": [{"text": "T", "source": "Skript.txt, Z.2"}],
"example_idea": ""}]}
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
seen.append((key, capabilities, prompt))
return blx.OK, payload((0, json.dumps(facts), ""))
async def fake_agent(key, prompt, timeout, **kw): # Check-Panel
return (0, '{"ok": true}', "")
monkeypatch.setattr(blx, "run_single_slot", fake_slot)
monkeypatch.setattr(blx, "run_agent", fake_agent)
res = await blx._facts_block(ctx, lambda *a, **k: None, {"arbeit": tmp_path},
{"Alpha": ["Sub Eins"]}, {"type": "uni"}, d, "", ns="x-")
assert res is not None
finder = [s for s in seen if "-facts-c0" in s[0] or "-facts-erg-" in s[0]]
assert finder and all(caps == "none" for _, caps, _ in finder)
assert all("── Skript.txt" in prompt for _, _, prompt in finder) # Auszüge inline
assert all("ls/find" not in prompt for _, _, prompt in finder)
def test_sub_key_resolves_short_titles():
"""Artefakt-Agenten echoen den Kurztitel; der Sub-Key heißt 'kurztitel: beschreibung'.
Eindeutiger Präfix wird aufgelöst, Mehrdeutiges und Fehlendes bleibt unverändert."""