marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Training-Harness (ACO, Multi-Fidelity), Prüfstand-Benchmark, Agenten-README

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
team3
2026-07-04 12:47:19 +02:00
parent 8d8f6c8e51
commit 8488737303
14 changed files with 772 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
backend/tests/conftest.py
View File

@@ -22,63 +22,10 @@ async def testdb(tmp_path, monkeypatch):
async def fake_welt(testdb, tmp_path, monkeypatch):
"""E2E ohne LLM: run_agent überall durch die Fake-Welt ersetzt, Tempo-Bremsen raus.
Alle echten Schichten (_race, Quorum, Panels, Producer, QA-Gate) laufen mit."""
import agents
import blocks
import board_inventory as bi
import guide
import kanban
import pipeline
import qa
import repair
from fake_agents import Welt
from fake_agents import Welt, aktivieren
welt = Welt()
async def fake_run_agent(agent_key, prompt, timeout, provider="claude", role="fast",
capabilities="none", lane="batch", scope=None, on_line=None, label=""):
return welt.respond(agent_key, prompt, capabilities)
for mod in (agents, pipeline, blocks, guide, repair):
monkeypatch.setattr(mod, "run_agent", fake_run_agent)
# Tempo: grace/poll/backoff bremsen echte Läufe, nicht den Fake
monkeypatch.setattr(blocks, "CONSENSUS_GRACE", 0)
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
monkeypatch.setattr(kanban, "RETRY_BACKOFF", 0.05)
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
import guide_board
monkeypatch.setattr(guide_board, "READABILITY_ACTIVE", False) # kein Modell-Load im Test
import asyncio as _aio
monkeypatch.setattr(bi, "_ingest_lock", _aio.Lock()) # Modul-Lock klebt sonst am Vortest-Loop
class _FakeEmb: # identischer Text → cos 1.0, sonst 0.0 (deterministisch, ohne Modell)
@staticmethod
def available():
return True
@staticmethod
def embed_sims(texts):
import numpy as np
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
arr = np.zeros((len(texts), max(len(uniq), 1)))
for r, t in enumerate(texts):
arr[r, uniq[t]] = 1.0
return arr @ arr.T
@staticmethod
def embed(texts):
import numpy as np
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
arr = np.zeros((len(texts), max(len(uniq), 1)))
for r, t in enumerate(texts):
arr[r, uniq[t]] = 1.0
return arr
import board_artefacts as ba
for mod in (blocks, ba, qa):
monkeypatch.setattr(mod, "embedding", _FakeEmb)
async def emb_ok(flow): # Board-1-Vektorpfade aus (wie board_env) — Judge-Wellen reichen
return False
monkeypatch.setattr(bi, "_emb_ok", emb_ok)
aktivieren(welt, setattr_fn=monkeypatch.setattr)
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa") # Reports nie in echte Nutzdaten
return welt

141
backend/tests/test_train.py
View File

@@ -1,4 +1,4 @@
"""Training-Harness: Registry↔config-Konsistenz, ENV-Override, Trainer-Logik (Stub-Runner)."""
"""Training-Harness: Registry↔config, ENV-Override, ACO-Trainer (Stub-Runner), Soll-Abgleich."""
import json
import subprocess
@@ -6,20 +6,21 @@ import sys
from pathlib import Path
import config
import train
import train_params
from train import Trainer, score
from train import AmeisenTrainer, score
from train_lauf import soll_abgleich
BACKEND = Path(__file__).resolve().parent.parent
def test_registry_spiegelt_config():
"""Jeder Registry-Parameter existiert in config mit identischem Default und
flow-sicheren Rändern — sonst optimiert der Trainer Phantome."""
"""Jeder Registry-Parameter existiert in config mit identischem Default, flow-sicheren
Rändern und einer Fidelity-Zuordnung — sonst optimiert der Trainer Phantome."""
for name, p in train_params.PARAMS.items():
assert getattr(config, name, None) == p["default"], name
assert p["min"] <= p["default"] <= p["max"], name
assert p["step"] > 0, name
assert p["fidelity"] in ("board2", "voll"), name
def test_creator_params_override_wirkt_im_subprozess():
@@ -42,72 +43,120 @@ def _metrics(note=8.0, dauer=10.0, tokens=1_000_000, **quoten):
"dauer_min": dauer, "tokens": {"input": tokens, "output": 0}, "agents": {}}
def _stub_runner(antworten):
"""params-abhängige Metriken; zählt echte Aufrufe (Cache-Treffer zählen nicht)."""
def _stub(score_fn):
"""Runner-Paar (F1/F2 + F0) für Tests: score_fn(params) → Metriken."""
calls = []
async def runner(params, thema):
calls.append((dict(params), thema[0]))
for muster, m in antworten:
if muster(params):
return dict(m)
return _metrics()
async def runner(params, fidelity, suffix=""):
calls.append((dict(params), fidelity))
return score_fn(params)
async def f0(params):
return {"ok": True, "invarianten_fehler": [], "calls": 100}
runner.calls = calls
return runner
return runner, f0
async def test_screening_filtert_rauschen(tmp_path):
"""Nur Parameter mit Effekt über der Rausch-Schwelle kommen in die Feinphase;
ein bestätigter Gewinner wird übernommen."""
wirksam = "FACTS_CHUNK_SUBS"
runner = _stub_runner([
(lambda p: p.get(wirksam) == 8, _metrics(note=9.5, dauer=8.0)), # klar besser
])
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
def _trainer(tmp_path, runner, f0, **kw):
args = dict(max_trials=999, max_stunden=1, ameisen=3, seed=7, f2_intervall=1000)
args.update(kw)
return AmeisenTrainer(tmp_path / "s", runner=runner, runner_f0=f0, **args)
async def test_aco_konvergiert_auf_optimum(tmp_path):
"""Gepflanztes Optimum (FACTS_CHUNK_SUBS=6) wird gefunden und bestätigt übernommen;
die Pheromon-Spur konzentriert sich dort."""
def bewertung(params):
return _metrics(note=9.5, dauer=7.0) if params.get("FACTS_CHUNK_SUBS") == 6 else _metrics()
runner, f0 = _stub(bewertung)
t = _trainer(tmp_path, runner, f0, max_trials=120)
best = await t.run()
assert best.get(wirksam) == 8
# kein anderer Parameter übernommen (alle anderen Δ=0 < Schwelle)
assert set(best) == {wirksam}
assert best.get("FACTS_CHUNK_SUBS") == 6
taus = t.pheromon["FACTS_CHUNK_SUBS"]
assert max(taus, key=lambda k: taus[k]) == "6"
async def test_uebernahme_braucht_bestaetigung(tmp_path):
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird verworfen."""
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird nicht Bester."""
zustand = {"mal": 0}
async def runner(params, thema):
if params.get("FACTS_CHUNK_SUBS") == 8:
def bewertung(params):
if params.get("FACTS_CHUNK_SUBS") == 6:
zustand["mal"] += 1
return _metrics(note=9.5) if zustand["mal"] == 1 else _metrics(note=8.0)
return _metrics()
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
runner, f0 = _stub(bewertung)
t = _trainer(tmp_path, runner, f0, max_trials=40)
best = await t.run()
assert best == {}
assert best.get("FACTS_CHUNK_SUBS") != 6
async def test_cache_resume_wiederholt_keine_trials(tmp_path):
runner = _stub_runner([])
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
async def test_f0_filter_verwirft_kaputte_kandidaten(tmp_path):
"""Kandidaten mit Invarianten-Fehlern erreichen nie einen bezahlten Lauf."""
runner, _f0 = _stub(lambda p: _metrics())
async def f0_kaputt(params):
if params: # nur Nicht-Baseline
return {"ok": True, "invarianten_fehler": ["kaputt"], "calls": 100}
return {"ok": True, "invarianten_fehler": [], "calls": 100}
t = _trainer(tmp_path, runner, f0_kaputt, max_trials=20)
await t.run()
erste = len(runner.calls)
t2 = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
await t2.run()
assert len(runner.calls) == erste # alles aus trials.jsonl bedient
bezahlt_mit_params = [c for c, _f in runner.calls if c]
assert bezahlt_mit_params == [] # nur Baselines liefen
async def test_resume_laedt_pheromon_und_cache(tmp_path):
def bewertung(params):
return _metrics(note=9.5) if params.get("FACTS_CHUNK_SUBS") == 6 else _metrics()
runner, f0 = _stub(bewertung)
t = _trainer(tmp_path, runner, f0, max_trials=60)
await t.run()
best, tau = t.best_params, dict(t.pheromon["FACTS_CHUNK_SUBS"])
runner2, f02 = _stub(bewertung)
t2 = _trainer(tmp_path, runner2, f02, max_trials=0) # kein Budget: alles aus Persistenz
assert t2.best_params == best
assert t2.pheromon["FACTS_CHUNK_SUBS"] == tau
async def test_budget_stoppt(tmp_path):
runner = _stub_runner([])
t = Trainer(tmp_path / "s", max_trials=3, max_stunden=1, runner=runner)
runner, f0 = _stub(lambda p: _metrics())
t = _trainer(tmp_path, runner, f0, max_trials=4)
await t.run()
assert len(runner.calls) <= 3
assert len(runner.calls) <= 4
def test_score_richtungen():
basis = _metrics()
besser = _metrics(note=9.0)
teurer = _metrics(dauer=20.0, tokens=2_000_000)
assert score(besser, basis) > score(basis, basis)
assert score(teurer, basis) < score(basis, basis)
mit_befunden = _metrics(fremd=0.2, luecken=0.1)
assert score(mit_befunden, basis) < score(basis, basis)
assert score(_metrics(note=9.0), basis) > score(basis, basis)
assert score(_metrics(dauer=20.0, tokens=2_000_000), basis) < score(basis, basis)
assert score(_metrics(fremd=0.2, luecken=0.1), basis) < score(basis, basis)
mit_soll = dict(_metrics(), soll={"f1": 1.0})
ohne_soll = dict(_metrics(), soll={"f1": 0.5})
assert score(mit_soll, basis) > score(ohne_soll, basis)
def test_soll_abgleich():
soll = {"bloecke": [{"titel": "Symmetrische Verschlüsselung"},
{"titel": "Asymmetrische Verschlüsselung",
"alternativen": ["Public-Key-Kryptographie"]},
{"titel": "Digitale Signaturen"}]}
r = soll_abgleich(["Symmetrische Verschlüsselung", "Public-Key-Kryptographie", "Quantencomputer"], soll)
assert r["fehlend"] == ["Digitale Signaturen"]
assert r["extra"] == ["Quantencomputer"]
assert 0 < r["f1"] < 1
async def test_copy_topic_dupliziert_karten_und_bloecke(testdb):
db = testdb
await db.kanban_upsert_card("q", "inventory", "b1", "block", "done_block", {"title": "Alpha"})
await db.upsert_block("q", "alpha", "Alpha", "Beschreibung", ["s1"], "r1")
await db.copy_topic("q", "z")
karten = await db.kanban_cards("z")
assert [c["card_id"] for c in karten] == ["b1"]
bloecke = await db.list_blocks("z")
assert [b["title"] for b in bloecke] == ["Alpha"]