Training-Harness (ACO, Multi-Fidelity), Prüfstand-Benchmark, Agenten-README
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
team3
@@ -1,4 +1,4 @@
|
||||
"""Training-Harness: Registry↔config-Konsistenz, ENV-Override, Trainer-Logik (Stub-Runner)."""
|
||||
"""Training-Harness: Registry↔config, ENV-Override, ACO-Trainer (Stub-Runner), Soll-Abgleich."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
@@ -6,20 +6,21 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
import config
|
||||
import train
|
||||
import train_params
|
||||
from train import Trainer, score
|
||||
from train import AmeisenTrainer, score
|
||||
from train_lauf import soll_abgleich
|
||||
|
||||
BACKEND = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def test_registry_spiegelt_config():
|
||||
"""Jeder Registry-Parameter existiert in config mit identischem Default und
|
||||
flow-sicheren Rändern — sonst optimiert der Trainer Phantome."""
|
||||
"""Jeder Registry-Parameter existiert in config mit identischem Default, flow-sicheren
|
||||
Rändern und einer Fidelity-Zuordnung — sonst optimiert der Trainer Phantome."""
|
||||
for name, p in train_params.PARAMS.items():
|
||||
assert getattr(config, name, None) == p["default"], name
|
||||
assert p["min"] <= p["default"] <= p["max"], name
|
||||
assert p["step"] > 0, name
|
||||
assert p["fidelity"] in ("board2", "voll"), name
|
||||
|
||||
|
||||
def test_creator_params_override_wirkt_im_subprozess():
|
||||
@@ -42,72 +43,120 @@ def _metrics(note=8.0, dauer=10.0, tokens=1_000_000, **quoten):
|
||||
"dauer_min": dauer, "tokens": {"input": tokens, "output": 0}, "agents": {}}
|
||||
|
||||
|
||||
def _stub_runner(antworten):
|
||||
"""params-abhängige Metriken; zählt echte Aufrufe (Cache-Treffer zählen nicht)."""
|
||||
def _stub(score_fn):
|
||||
"""Runner-Paar (F1/F2 + F0) für Tests: score_fn(params) → Metriken."""
|
||||
calls = []
|
||||
|
||||
async def runner(params, thema):
|
||||
calls.append((dict(params), thema[0]))
|
||||
for muster, m in antworten:
|
||||
if muster(params):
|
||||
return dict(m)
|
||||
return _metrics()
|
||||
async def runner(params, fidelity, suffix=""):
|
||||
calls.append((dict(params), fidelity))
|
||||
return score_fn(params)
|
||||
|
||||
async def f0(params):
|
||||
return {"ok": True, "invarianten_fehler": [], "calls": 100}
|
||||
|
||||
runner.calls = calls
|
||||
return runner
|
||||
return runner, f0
|
||||
|
||||
|
||||
async def test_screening_filtert_rauschen(tmp_path):
|
||||
"""Nur Parameter mit Effekt über der Rausch-Schwelle kommen in die Feinphase;
|
||||
ein bestätigter Gewinner wird übernommen."""
|
||||
wirksam = "FACTS_CHUNK_SUBS"
|
||||
runner = _stub_runner([
|
||||
(lambda p: p.get(wirksam) == 8, _metrics(note=9.5, dauer=8.0)), # klar besser
|
||||
])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
def _trainer(tmp_path, runner, f0, **kw):
|
||||
args = dict(max_trials=999, max_stunden=1, ameisen=3, seed=7, f2_intervall=1000)
|
||||
args.update(kw)
|
||||
return AmeisenTrainer(tmp_path / "s", runner=runner, runner_f0=f0, **args)
|
||||
|
||||
|
||||
async def test_aco_konvergiert_auf_optimum(tmp_path):
|
||||
"""Gepflanztes Optimum (FACTS_CHUNK_SUBS=6) wird gefunden und bestätigt übernommen;
|
||||
die Pheromon-Spur konzentriert sich dort."""
|
||||
def bewertung(params):
|
||||
return _metrics(note=9.5, dauer=7.0) if params.get("FACTS_CHUNK_SUBS") == 6 else _metrics()
|
||||
|
||||
runner, f0 = _stub(bewertung)
|
||||
t = _trainer(tmp_path, runner, f0, max_trials=120)
|
||||
best = await t.run()
|
||||
assert best.get(wirksam) == 8
|
||||
# kein anderer Parameter übernommen (alle anderen Δ=0 < Schwelle)
|
||||
assert set(best) == {wirksam}
|
||||
assert best.get("FACTS_CHUNK_SUBS") == 6
|
||||
taus = t.pheromon["FACTS_CHUNK_SUBS"]
|
||||
assert max(taus, key=lambda k: taus[k]) == "6"
|
||||
|
||||
|
||||
async def test_uebernahme_braucht_bestaetigung(tmp_path):
|
||||
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird verworfen."""
|
||||
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird nicht Bester."""
|
||||
zustand = {"mal": 0}
|
||||
|
||||
async def runner(params, thema):
|
||||
if params.get("FACTS_CHUNK_SUBS") == 8:
|
||||
def bewertung(params):
|
||||
if params.get("FACTS_CHUNK_SUBS") == 6:
|
||||
zustand["mal"] += 1
|
||||
return _metrics(note=9.5) if zustand["mal"] == 1 else _metrics(note=8.0)
|
||||
return _metrics()
|
||||
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
runner, f0 = _stub(bewertung)
|
||||
t = _trainer(tmp_path, runner, f0, max_trials=40)
|
||||
best = await t.run()
|
||||
assert best == {}
|
||||
assert best.get("FACTS_CHUNK_SUBS") != 6
|
||||
|
||||
|
||||
async def test_cache_resume_wiederholt_keine_trials(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
async def test_f0_filter_verwirft_kaputte_kandidaten(tmp_path):
|
||||
"""Kandidaten mit Invarianten-Fehlern erreichen nie einen bezahlten Lauf."""
|
||||
runner, _f0 = _stub(lambda p: _metrics())
|
||||
|
||||
async def f0_kaputt(params):
|
||||
if params: # nur Nicht-Baseline
|
||||
return {"ok": True, "invarianten_fehler": ["kaputt"], "calls": 100}
|
||||
return {"ok": True, "invarianten_fehler": [], "calls": 100}
|
||||
|
||||
t = _trainer(tmp_path, runner, f0_kaputt, max_trials=20)
|
||||
await t.run()
|
||||
erste = len(runner.calls)
|
||||
t2 = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
await t2.run()
|
||||
assert len(runner.calls) == erste # alles aus trials.jsonl bedient
|
||||
bezahlt_mit_params = [c for c, _f in runner.calls if c]
|
||||
assert bezahlt_mit_params == [] # nur Baselines liefen
|
||||
|
||||
|
||||
async def test_resume_laedt_pheromon_und_cache(tmp_path):
|
||||
def bewertung(params):
|
||||
return _metrics(note=9.5) if params.get("FACTS_CHUNK_SUBS") == 6 else _metrics()
|
||||
|
||||
runner, f0 = _stub(bewertung)
|
||||
t = _trainer(tmp_path, runner, f0, max_trials=60)
|
||||
await t.run()
|
||||
best, tau = t.best_params, dict(t.pheromon["FACTS_CHUNK_SUBS"])
|
||||
runner2, f02 = _stub(bewertung)
|
||||
t2 = _trainer(tmp_path, runner2, f02, max_trials=0) # kein Budget: alles aus Persistenz
|
||||
assert t2.best_params == best
|
||||
assert t2.pheromon["FACTS_CHUNK_SUBS"] == tau
|
||||
|
||||
|
||||
async def test_budget_stoppt(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=3, max_stunden=1, runner=runner)
|
||||
runner, f0 = _stub(lambda p: _metrics())
|
||||
t = _trainer(tmp_path, runner, f0, max_trials=4)
|
||||
await t.run()
|
||||
assert len(runner.calls) <= 3
|
||||
assert len(runner.calls) <= 4
|
||||
|
||||
|
||||
def test_score_richtungen():
|
||||
basis = _metrics()
|
||||
besser = _metrics(note=9.0)
|
||||
teurer = _metrics(dauer=20.0, tokens=2_000_000)
|
||||
assert score(besser, basis) > score(basis, basis)
|
||||
assert score(teurer, basis) < score(basis, basis)
|
||||
mit_befunden = _metrics(fremd=0.2, luecken=0.1)
|
||||
assert score(mit_befunden, basis) < score(basis, basis)
|
||||
assert score(_metrics(note=9.0), basis) > score(basis, basis)
|
||||
assert score(_metrics(dauer=20.0, tokens=2_000_000), basis) < score(basis, basis)
|
||||
assert score(_metrics(fremd=0.2, luecken=0.1), basis) < score(basis, basis)
|
||||
mit_soll = dict(_metrics(), soll={"f1": 1.0})
|
||||
ohne_soll = dict(_metrics(), soll={"f1": 0.5})
|
||||
assert score(mit_soll, basis) > score(ohne_soll, basis)
|
||||
|
||||
|
||||
def test_soll_abgleich():
|
||||
soll = {"bloecke": [{"titel": "Symmetrische Verschlüsselung"},
|
||||
{"titel": "Asymmetrische Verschlüsselung",
|
||||
"alternativen": ["Public-Key-Kryptographie"]},
|
||||
{"titel": "Digitale Signaturen"}]}
|
||||
r = soll_abgleich(["Symmetrische Verschlüsselung", "Public-Key-Kryptographie", "Quantencomputer"], soll)
|
||||
assert r["fehlend"] == ["Digitale Signaturen"]
|
||||
assert r["extra"] == ["Quantencomputer"]
|
||||
assert 0 < r["f1"] < 1
|
||||
|
||||
|
||||
async def test_copy_topic_dupliziert_karten_und_bloecke(testdb):
|
||||
db = testdb
|
||||
await db.kanban_upsert_card("q", "inventory", "b1", "block", "done_block", {"title": "Alpha"})
|
||||
await db.upsert_block("q", "alpha", "Alpha", "Beschreibung", ["s1"], "r1")
|
||||
await db.copy_topic("q", "z")
|
||||
karten = await db.kanban_cards("z")
|
||||
assert [c["card_id"] for c in karten] == ["b1"]
|
||||
bloecke = await db.list_blocks("z")
|
||||
assert [b["title"] for b in bloecke] == ["Alpha"]
|
||||
|
||||
Reference in New Issue
Block a user