114 lines
4.0 KiB
Python
114 lines
4.0 KiB
Python
"""Training-Harness: Registry↔config-Konsistenz, ENV-Override, Trainer-Logik (Stub-Runner)."""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import config
|
|
import train
|
|
import train_params
|
|
from train import Trainer, score
|
|
|
|
BACKEND = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def test_registry_spiegelt_config():
|
|
"""Jeder Registry-Parameter existiert in config mit identischem Default und
|
|
flow-sicheren Rändern — sonst optimiert der Trainer Phantome."""
|
|
for name, p in train_params.PARAMS.items():
|
|
assert getattr(config, name, None) == p["default"], name
|
|
assert p["min"] <= p["default"] <= p["max"], name
|
|
assert p["step"] > 0, name
|
|
|
|
|
|
def test_creator_params_override_wirkt_im_subprozess():
|
|
out = subprocess.run(
|
|
[sys.executable, "-c", "import config; print(config.FACTS_CHUNK_SUBS, config.TIMEOUTS['subblock_check'][0])"],
|
|
capture_output=True, text=True, cwd=BACKEND,
|
|
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"FACTS_CHUNK_SUBS": 6, "TIMEOUT_subblock_check_base": 77}'})
|
|
assert out.stdout.split() == ["6", "77"], out.stderr
|
|
|
|
|
|
def test_creator_params_unbekannter_name_bricht_ab():
|
|
out = subprocess.run([sys.executable, "-c", "import config"],
|
|
capture_output=True, text=True, cwd=BACKEND,
|
|
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"GIBT_ES_NICHT": 1}'})
|
|
assert out.returncode != 0 and "GIBT_ES_NICHT" in out.stderr
|
|
|
|
|
|
def _metrics(note=8.0, dauer=10.0, tokens=1_000_000, **quoten):
|
|
return {"note": note, "quoten": quoten, "quoten_artefakte": {},
|
|
"dauer_min": dauer, "tokens": {"input": tokens, "output": 0}, "agents": {}}
|
|
|
|
|
|
def _stub_runner(antworten):
|
|
"""params-abhängige Metriken; zählt echte Aufrufe (Cache-Treffer zählen nicht)."""
|
|
calls = []
|
|
|
|
async def runner(params, thema):
|
|
calls.append((dict(params), thema[0]))
|
|
for muster, m in antworten:
|
|
if muster(params):
|
|
return dict(m)
|
|
return _metrics()
|
|
|
|
runner.calls = calls
|
|
return runner
|
|
|
|
|
|
async def test_screening_filtert_rauschen(tmp_path):
|
|
"""Nur Parameter mit Effekt über der Rausch-Schwelle kommen in die Feinphase;
|
|
ein bestätigter Gewinner wird übernommen."""
|
|
wirksam = "FACTS_CHUNK_SUBS"
|
|
runner = _stub_runner([
|
|
(lambda p: p.get(wirksam) == 8, _metrics(note=9.5, dauer=8.0)), # klar besser
|
|
])
|
|
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
|
best = await t.run()
|
|
assert best.get(wirksam) == 8
|
|
# kein anderer Parameter übernommen (alle anderen Δ=0 < Schwelle)
|
|
assert set(best) == {wirksam}
|
|
|
|
|
|
async def test_uebernahme_braucht_bestaetigung(tmp_path):
|
|
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird verworfen."""
|
|
zustand = {"mal": 0}
|
|
|
|
async def runner(params, thema):
|
|
if params.get("FACTS_CHUNK_SUBS") == 8:
|
|
zustand["mal"] += 1
|
|
return _metrics(note=9.5) if zustand["mal"] == 1 else _metrics(note=8.0)
|
|
return _metrics()
|
|
|
|
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
|
best = await t.run()
|
|
assert best == {}
|
|
|
|
|
|
async def test_cache_resume_wiederholt_keine_trials(tmp_path):
|
|
runner = _stub_runner([])
|
|
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
|
await t.run()
|
|
erste = len(runner.calls)
|
|
t2 = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
|
await t2.run()
|
|
assert len(runner.calls) == erste # alles aus trials.jsonl bedient
|
|
|
|
|
|
async def test_budget_stoppt(tmp_path):
|
|
runner = _stub_runner([])
|
|
t = Trainer(tmp_path / "s", max_trials=3, max_stunden=1, runner=runner)
|
|
await t.run()
|
|
assert len(runner.calls) <= 3
|
|
|
|
|
|
def test_score_richtungen():
|
|
basis = _metrics()
|
|
besser = _metrics(note=9.0)
|
|
teurer = _metrics(dauer=20.0, tokens=2_000_000)
|
|
assert score(besser, basis) > score(basis, basis)
|
|
assert score(teurer, basis) < score(basis, basis)
|
|
mit_befunden = _metrics(fremd=0.2, luecken=0.1)
|
|
assert score(mit_befunden, basis) < score(basis, basis)
|