Training-Harness (ACO, Multi-Fidelity), Prüfstand-Benchmark, Agenten-README
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
team3
@@ -1701,7 +1701,8 @@ async def _write_run_summary(topic: str, flow: Flow):
|
||||
**await db.events_run_summary(topic, run_id)}
|
||||
try: # Abschluss-QA MIT Judges: sub_dubletten/unechte werden beurteilt — erst damit
|
||||
import qa # ist note_artefakte belastbar (Kandidatenliste allein zählt nicht)
|
||||
report = await qa.qa_report(topic, llm=True)
|
||||
from config import ABSCHLUSS_QA_LLM # Training schaltet die Judges ab (misst selbst)
|
||||
report = await qa.qa_report(topic, llm=bool(ABSCHLUSS_QA_LLM))
|
||||
if report:
|
||||
summary["note"] = report["note"]
|
||||
summary["note_artefakte"] = report.get("note_artefakte")
|
||||
|
||||
@@ -196,6 +196,7 @@ RETRY_BACKOFF = 30.0 # base seconds; backoff = base · 2^(retries-1)
|
||||
MAX_RESTARTS = 2 # agent restart cap per race slot
|
||||
JUDGE_CHUNK = 40 # repair: findings per judge call
|
||||
EVIDENCE_PER_BLOCK = 6000 # repair: excerpt chars per fremd candidate
|
||||
ABSCHLUSS_QA_LLM = 1 # 0 = Abschluss-QA ohne LLM-Judges (Training misst selbst; spart Minuten)
|
||||
|
||||
# Timeouts per agent step: (base seconds, seconds per block/section).
|
||||
# Applies equally to all providers — whoever is too slow gets restarted or overtaken.
|
||||
|
||||
@@ -1186,6 +1186,24 @@ async def default_subblock_levels(topic: str, block_norm: str) -> None:
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def copy_topic(quelle: str, ziel: str) -> None:
|
||||
"""Trainings-Helfer: Kanban-Karten + Block-Rows der Quelle unter neuem Topic duplizieren
|
||||
(Frozen-Inventar-Trials — Board 2 läuft auf identischem Board-1-Stand neu). Nur DB;
|
||||
Dateien (source.json/blocks.md) kopiert der Runner."""
|
||||
db = await get_db()
|
||||
await db.execute("DELETE FROM kanban_cards WHERE topic = ?", (ziel,))
|
||||
await db.execute("DELETE FROM blocks WHERE topic = ?", (ziel,))
|
||||
await db.execute(
|
||||
"""INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, retries, not_before, last_error, updated_at)
|
||||
SELECT ?, board, card_id, kind, stage, payload, 0, 0, '', updated_at
|
||||
FROM kanban_cards WHERE topic = ?""", (ziel, quelle))
|
||||
await db.execute(
|
||||
"""INSERT INTO blocks (topic, title_norm, title, description, mentions, status, sources, reader, updated_at)
|
||||
SELECT ?, title_norm, title, description, mentions, status, sources, reader, updated_at
|
||||
FROM blocks WHERE topic = ?""", (ziel, quelle))
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def delete_stale_consensus(topic: str, block_norm: str, keep: set[str]) -> None:
|
||||
"""Drop consensus rows of a block that are NOT in this run's sidecar (`keep`): finalize
|
||||
only upserts, so re-runs piled up orphan rows (measured: 25 subs without any board-2
|
||||
|
||||
@@ -287,3 +287,59 @@ async def respond(agent_key: str, prompt: str, capabilities: str) -> tuple[int,
|
||||
if _WELT is None:
|
||||
_WELT = Welt()
|
||||
return _WELT.respond(agent_key, prompt, capabilities)
|
||||
|
||||
|
||||
def aktivieren(welt: Welt, setattr_fn=setattr) -> None:
|
||||
"""Alle Patches für einen Fake-E2E-Lauf (run_agent überall, Tempo-Bremsen raus,
|
||||
Text-Identitäts-Embedding). pytest übergibt monkeypatch.setattr (auto-Rollback);
|
||||
train_f0 nutzt den Default — der Prozess stirbt nach dem Lauf sowieso."""
|
||||
import asyncio
|
||||
|
||||
import agents
|
||||
import blocks
|
||||
import board_artefacts as ba
|
||||
import board_inventory as bi
|
||||
import guide
|
||||
import guide_board
|
||||
import kanban
|
||||
import pipeline
|
||||
import qa
|
||||
import repair
|
||||
|
||||
async def fake_run_agent(agent_key, prompt, timeout, provider="claude", role="fast",
|
||||
capabilities="none", lane="batch", scope=None, on_line=None, label=""):
|
||||
return welt.respond(agent_key, prompt, capabilities)
|
||||
|
||||
for mod in (agents, pipeline, blocks, guide, repair):
|
||||
setattr_fn(mod, "run_agent", fake_run_agent)
|
||||
setattr_fn(blocks, "CONSENSUS_GRACE", 0)
|
||||
setattr_fn(bi, "_QA_GATE_POLL", 0.05)
|
||||
setattr_fn(kanban, "RETRY_BACKOFF", 0.05)
|
||||
setattr_fn(guide_board, "READABILITY_ACTIVE", False)
|
||||
setattr_fn(bi, "_ingest_lock", asyncio.Lock())
|
||||
|
||||
class _FakeEmb: # identischer Text → cos 1.0, sonst 0.0 (deterministisch, ohne Modell)
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def embed(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr
|
||||
|
||||
@staticmethod
|
||||
def embed_sims(texts):
|
||||
arr = _FakeEmb.embed(texts)
|
||||
return arr @ arr.T
|
||||
|
||||
for mod in (blocks, ba, qa):
|
||||
setattr_fn(mod, "embedding", _FakeEmb)
|
||||
|
||||
async def emb_ok(flow): # Board-1-Vektorpfade aus — Judge-Wellen reichen
|
||||
return False
|
||||
setattr_fn(bi, "_emb_ok", emb_ok)
|
||||
|
||||
@@ -22,63 +22,10 @@ async def testdb(tmp_path, monkeypatch):
|
||||
async def fake_welt(testdb, tmp_path, monkeypatch):
|
||||
"""E2E ohne LLM: run_agent überall durch die Fake-Welt ersetzt, Tempo-Bremsen raus.
|
||||
Alle echten Schichten (_race, Quorum, Panels, Producer, QA-Gate) laufen mit."""
|
||||
import agents
|
||||
import blocks
|
||||
import board_inventory as bi
|
||||
import guide
|
||||
import kanban
|
||||
import pipeline
|
||||
import qa
|
||||
import repair
|
||||
from fake_agents import Welt
|
||||
from fake_agents import Welt, aktivieren
|
||||
|
||||
welt = Welt()
|
||||
|
||||
async def fake_run_agent(agent_key, prompt, timeout, provider="claude", role="fast",
|
||||
capabilities="none", lane="batch", scope=None, on_line=None, label=""):
|
||||
return welt.respond(agent_key, prompt, capabilities)
|
||||
|
||||
for mod in (agents, pipeline, blocks, guide, repair):
|
||||
monkeypatch.setattr(mod, "run_agent", fake_run_agent)
|
||||
|
||||
# Tempo: grace/poll/backoff bremsen echte Läufe, nicht den Fake
|
||||
monkeypatch.setattr(blocks, "CONSENSUS_GRACE", 0)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
monkeypatch.setattr(kanban, "RETRY_BACKOFF", 0.05)
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
|
||||
import guide_board
|
||||
monkeypatch.setattr(guide_board, "READABILITY_ACTIVE", False) # kein Modell-Load im Test
|
||||
import asyncio as _aio
|
||||
monkeypatch.setattr(bi, "_ingest_lock", _aio.Lock()) # Modul-Lock klebt sonst am Vortest-Loop
|
||||
|
||||
class _FakeEmb: # identischer Text → cos 1.0, sonst 0.0 (deterministisch, ohne Modell)
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def embed_sims(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr @ arr.T
|
||||
|
||||
@staticmethod
|
||||
def embed(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr
|
||||
|
||||
import board_artefacts as ba
|
||||
for mod in (blocks, ba, qa):
|
||||
monkeypatch.setattr(mod, "embedding", _FakeEmb)
|
||||
|
||||
async def emb_ok(flow): # Board-1-Vektorpfade aus (wie board_env) — Judge-Wellen reichen
|
||||
return False
|
||||
monkeypatch.setattr(bi, "_emb_ok", emb_ok)
|
||||
aktivieren(welt, setattr_fn=monkeypatch.setattr)
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa") # Reports nie in echte Nutzdaten
|
||||
return welt
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Training-Harness: Registry↔config-Konsistenz, ENV-Override, Trainer-Logik (Stub-Runner)."""
|
||||
"""Training-Harness: Registry↔config, ENV-Override, ACO-Trainer (Stub-Runner), Soll-Abgleich."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
@@ -6,20 +6,21 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
import config
|
||||
import train
|
||||
import train_params
|
||||
from train import Trainer, score
|
||||
from train import AmeisenTrainer, score
|
||||
from train_lauf import soll_abgleich
|
||||
|
||||
BACKEND = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def test_registry_spiegelt_config():
|
||||
"""Jeder Registry-Parameter existiert in config mit identischem Default und
|
||||
flow-sicheren Rändern — sonst optimiert der Trainer Phantome."""
|
||||
"""Jeder Registry-Parameter existiert in config mit identischem Default, flow-sicheren
|
||||
Rändern und einer Fidelity-Zuordnung — sonst optimiert der Trainer Phantome."""
|
||||
for name, p in train_params.PARAMS.items():
|
||||
assert getattr(config, name, None) == p["default"], name
|
||||
assert p["min"] <= p["default"] <= p["max"], name
|
||||
assert p["step"] > 0, name
|
||||
assert p["fidelity"] in ("board2", "voll"), name
|
||||
|
||||
|
||||
def test_creator_params_override_wirkt_im_subprozess():
|
||||
@@ -42,72 +43,120 @@ def _metrics(note=8.0, dauer=10.0, tokens=1_000_000, **quoten):
|
||||
"dauer_min": dauer, "tokens": {"input": tokens, "output": 0}, "agents": {}}
|
||||
|
||||
|
||||
def _stub_runner(antworten):
|
||||
"""params-abhängige Metriken; zählt echte Aufrufe (Cache-Treffer zählen nicht)."""
|
||||
def _stub(score_fn):
|
||||
"""Runner-Paar (F1/F2 + F0) für Tests: score_fn(params) → Metriken."""
|
||||
calls = []
|
||||
|
||||
async def runner(params, thema):
|
||||
calls.append((dict(params), thema[0]))
|
||||
for muster, m in antworten:
|
||||
if muster(params):
|
||||
return dict(m)
|
||||
return _metrics()
|
||||
async def runner(params, fidelity, suffix=""):
|
||||
calls.append((dict(params), fidelity))
|
||||
return score_fn(params)
|
||||
|
||||
async def f0(params):
|
||||
return {"ok": True, "invarianten_fehler": [], "calls": 100}
|
||||
|
||||
runner.calls = calls
|
||||
return runner
|
||||
return runner, f0
|
||||
|
||||
|
||||
async def test_screening_filtert_rauschen(tmp_path):
|
||||
"""Nur Parameter mit Effekt über der Rausch-Schwelle kommen in die Feinphase;
|
||||
ein bestätigter Gewinner wird übernommen."""
|
||||
wirksam = "FACTS_CHUNK_SUBS"
|
||||
runner = _stub_runner([
|
||||
(lambda p: p.get(wirksam) == 8, _metrics(note=9.5, dauer=8.0)), # klar besser
|
||||
])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
def _trainer(tmp_path, runner, f0, **kw):
|
||||
args = dict(max_trials=999, max_stunden=1, ameisen=3, seed=7, f2_intervall=1000)
|
||||
args.update(kw)
|
||||
return AmeisenTrainer(tmp_path / "s", runner=runner, runner_f0=f0, **args)
|
||||
|
||||
|
||||
async def test_aco_konvergiert_auf_optimum(tmp_path):
|
||||
"""Gepflanztes Optimum (FACTS_CHUNK_SUBS=6) wird gefunden und bestätigt übernommen;
|
||||
die Pheromon-Spur konzentriert sich dort."""
|
||||
def bewertung(params):
|
||||
return _metrics(note=9.5, dauer=7.0) if params.get("FACTS_CHUNK_SUBS") == 6 else _metrics()
|
||||
|
||||
runner, f0 = _stub(bewertung)
|
||||
t = _trainer(tmp_path, runner, f0, max_trials=120)
|
||||
best = await t.run()
|
||||
assert best.get(wirksam) == 8
|
||||
# kein anderer Parameter übernommen (alle anderen Δ=0 < Schwelle)
|
||||
assert set(best) == {wirksam}
|
||||
assert best.get("FACTS_CHUNK_SUBS") == 6
|
||||
taus = t.pheromon["FACTS_CHUNK_SUBS"]
|
||||
assert max(taus, key=lambda k: taus[k]) == "6"
|
||||
|
||||
|
||||
async def test_uebernahme_braucht_bestaetigung(tmp_path):
|
||||
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird verworfen."""
|
||||
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird nicht Bester."""
|
||||
zustand = {"mal": 0}
|
||||
|
||||
async def runner(params, thema):
|
||||
if params.get("FACTS_CHUNK_SUBS") == 8:
|
||||
def bewertung(params):
|
||||
if params.get("FACTS_CHUNK_SUBS") == 6:
|
||||
zustand["mal"] += 1
|
||||
return _metrics(note=9.5) if zustand["mal"] == 1 else _metrics(note=8.0)
|
||||
return _metrics()
|
||||
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
runner, f0 = _stub(bewertung)
|
||||
t = _trainer(tmp_path, runner, f0, max_trials=40)
|
||||
best = await t.run()
|
||||
assert best == {}
|
||||
assert best.get("FACTS_CHUNK_SUBS") != 6
|
||||
|
||||
|
||||
async def test_cache_resume_wiederholt_keine_trials(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
async def test_f0_filter_verwirft_kaputte_kandidaten(tmp_path):
|
||||
"""Kandidaten mit Invarianten-Fehlern erreichen nie einen bezahlten Lauf."""
|
||||
runner, _f0 = _stub(lambda p: _metrics())
|
||||
|
||||
async def f0_kaputt(params):
|
||||
if params: # nur Nicht-Baseline
|
||||
return {"ok": True, "invarianten_fehler": ["kaputt"], "calls": 100}
|
||||
return {"ok": True, "invarianten_fehler": [], "calls": 100}
|
||||
|
||||
t = _trainer(tmp_path, runner, f0_kaputt, max_trials=20)
|
||||
await t.run()
|
||||
erste = len(runner.calls)
|
||||
t2 = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
await t2.run()
|
||||
assert len(runner.calls) == erste # alles aus trials.jsonl bedient
|
||||
bezahlt_mit_params = [c for c, _f in runner.calls if c]
|
||||
assert bezahlt_mit_params == [] # nur Baselines liefen
|
||||
|
||||
|
||||
async def test_resume_laedt_pheromon_und_cache(tmp_path):
|
||||
def bewertung(params):
|
||||
return _metrics(note=9.5) if params.get("FACTS_CHUNK_SUBS") == 6 else _metrics()
|
||||
|
||||
runner, f0 = _stub(bewertung)
|
||||
t = _trainer(tmp_path, runner, f0, max_trials=60)
|
||||
await t.run()
|
||||
best, tau = t.best_params, dict(t.pheromon["FACTS_CHUNK_SUBS"])
|
||||
runner2, f02 = _stub(bewertung)
|
||||
t2 = _trainer(tmp_path, runner2, f02, max_trials=0) # kein Budget: alles aus Persistenz
|
||||
assert t2.best_params == best
|
||||
assert t2.pheromon["FACTS_CHUNK_SUBS"] == tau
|
||||
|
||||
|
||||
async def test_budget_stoppt(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=3, max_stunden=1, runner=runner)
|
||||
runner, f0 = _stub(lambda p: _metrics())
|
||||
t = _trainer(tmp_path, runner, f0, max_trials=4)
|
||||
await t.run()
|
||||
assert len(runner.calls) <= 3
|
||||
assert len(runner.calls) <= 4
|
||||
|
||||
|
||||
def test_score_richtungen():
|
||||
basis = _metrics()
|
||||
besser = _metrics(note=9.0)
|
||||
teurer = _metrics(dauer=20.0, tokens=2_000_000)
|
||||
assert score(besser, basis) > score(basis, basis)
|
||||
assert score(teurer, basis) < score(basis, basis)
|
||||
mit_befunden = _metrics(fremd=0.2, luecken=0.1)
|
||||
assert score(mit_befunden, basis) < score(basis, basis)
|
||||
assert score(_metrics(note=9.0), basis) > score(basis, basis)
|
||||
assert score(_metrics(dauer=20.0, tokens=2_000_000), basis) < score(basis, basis)
|
||||
assert score(_metrics(fremd=0.2, luecken=0.1), basis) < score(basis, basis)
|
||||
mit_soll = dict(_metrics(), soll={"f1": 1.0})
|
||||
ohne_soll = dict(_metrics(), soll={"f1": 0.5})
|
||||
assert score(mit_soll, basis) > score(ohne_soll, basis)
|
||||
|
||||
|
||||
def test_soll_abgleich():
|
||||
soll = {"bloecke": [{"titel": "Symmetrische Verschlüsselung"},
|
||||
{"titel": "Asymmetrische Verschlüsselung",
|
||||
"alternativen": ["Public-Key-Kryptographie"]},
|
||||
{"titel": "Digitale Signaturen"}]}
|
||||
r = soll_abgleich(["Symmetrische Verschlüsselung", "Public-Key-Kryptographie", "Quantencomputer"], soll)
|
||||
assert r["fehlend"] == ["Digitale Signaturen"]
|
||||
assert r["extra"] == ["Quantencomputer"]
|
||||
assert 0 < r["f1"] < 1
|
||||
|
||||
|
||||
async def test_copy_topic_dupliziert_karten_und_bloecke(testdb):
|
||||
db = testdb
|
||||
await db.kanban_upsert_card("q", "inventory", "b1", "block", "done_block", {"title": "Alpha"})
|
||||
await db.upsert_block("q", "alpha", "Alpha", "Beschreibung", ["s1"], "r1")
|
||||
await db.copy_topic("q", "z")
|
||||
karten = await db.kanban_cards("z")
|
||||
assert [c["card_id"] for c in karten] == ["b1"]
|
||||
bloecke = await db.list_blocks("z")
|
||||
assert [b["title"] for b in bloecke] == ["Alpha"]
|
||||
|
||||
374
backend/train.py
374
backend/train.py
@@ -1,40 +1,66 @@
|
||||
"""make train: Parameter-Optimierung auf Mini-Themen (Baseline → Screening → Koordinaten-Suche).
|
||||
"""make train: Ameisen-Optimierung (ACO) der Pipeline-Parameter — anytime, multi-fidelity.
|
||||
|
||||
Jeder Trial ist ein Subprozess (train_lauf.py) mit CREATOR_PARAMS im ENV — so binden die
|
||||
Module die überschriebenen Werte beim Import. Metriken sind deterministisch (qa_report
|
||||
ohne LLM); gegen Judge-/Lauf-Rauschen gilt: Baseline mit Wiederholung liefert die
|
||||
Rausch-Schwelle, und eine Übernahme braucht einen BESTÄTIGUNGSLAUF (sonst Random Walk).
|
||||
Prinzip: Pheromon-Gewichte je (Parameter, Stufe) steuern, welche Kandidaten („Ameisen")
|
||||
als Nächstes getestet werden. Gute Kandidaten verstärken ihre Stufen, Verdunstung hält
|
||||
die Suche offen — je länger der Trainer läuft, desto gezielter werden die Tests.
|
||||
Jederzeit stoppbar; der Stand (beste_params.json/report.md) ist immer aktuell.
|
||||
|
||||
CLI: python3 train.py [--trials 40] [--stunden 8] [--sitzung NAME]
|
||||
Ergebnis: storage/train/<sitzung>/{trials.jsonl, report.md, beste_params.json}
|
||||
Fidelity-Kaskade pro Kandidat:
|
||||
F0 Fake-E2E (train_f0.py, Sekunden, 0 Tokens): Invarianten + Struktur-Proxy — Filter.
|
||||
F1 Frozen-Inventar (train_lauf.py --board2, ~5–8 min): misst Board-2/Guide-Parameter.
|
||||
F2 Volllauf inkl. Soll-Abgleich: alle N Runden für den Besten + Inventar-Parameter.
|
||||
|
||||
CLI: python3 train.py [--stunden 8] [--trials 60] [--ameisen 3] [--seed 0]
|
||||
[--sitzung NAME] [--f2-intervall 5]
|
||||
python3 train.py --init (baut das Frozen-Inventar-Vorlage-Topic, einmalig)
|
||||
Ergebnis: storage/train/<sitzung>/{trials.jsonl, pheromon.json, report.md, beste_params.json}
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from config import STORAGE_DIR
|
||||
from train_params import PARAMS, schritte
|
||||
from train_params import PARAMS
|
||||
|
||||
HAUPT_THEMA = ("train-sort", "benchmarks/sortierverfahren")
|
||||
VALIDIER_THEMA = ("train-foto", "benchmarks/fotografie")
|
||||
VORLAGE_TOPIC = "train-vorlage"
|
||||
BENCHMARK = "benchmarks/pruefstand"
|
||||
# Score-Gewichte: Qualität + Auswahl dominieren (Entwicklungsphase), Kosten ziehen ab.
|
||||
W_NOTE, W_AUSWAHL, W_ZEIT, W_TOKEN = 4.0, 4.0, 1.0, 1.0
|
||||
RHO = 0.2 # Pheromon-Verdunstung je Runde
|
||||
SPARSITY = 0.5 # Wahrscheinlichkeit, dass eine Ameise einen Parameter auf Default lässt
|
||||
F0_CALL_FAKTOR = 1.5 # Struktur-Proxy: mehr als 1.5× Baseline-Calls → Kandidat verworfen
|
||||
# Trainings-Fixa (Speed, kein Suchraum): Abschluss-QA ohne Judges, kurze Nachzügler-Gnade
|
||||
TRAIN_FIXA = {"ABSCHLUSS_QA_LLM": 0, "CONSENSUS_GRACE": 60}
|
||||
|
||||
|
||||
def stufen(name: str) -> list[float]:
|
||||
p = PARAMS[name]
|
||||
out, w = [], p["min"]
|
||||
while w <= p["max"] + 1e-9:
|
||||
out.append(round(w, 4))
|
||||
w += p["step"]
|
||||
return out
|
||||
|
||||
|
||||
def score(m: dict, basis: dict) -> float:
|
||||
"""Skalarer Vergleichswert eines Trials. note 0–10; auswahl aus den MECE-Quoten;
|
||||
Zeit/Tokens normiert auf die Baseline (1.0 = Baseline-Kosten)."""
|
||||
q = m.get("quoten") or {}
|
||||
qa_ = m.get("quoten_artefakte") or {}
|
||||
auswahl = 10.0 * max(0.0, 1.0 - min(1.0, (
|
||||
q.get("dubletten_verdacht", 0) + q.get("luecken", 0) + q.get("fremd", 0)
|
||||
+ qa_.get("sub_dubletten_verdacht", 0) + qa_.get("verwaiste", 0))))
|
||||
"""Skalarer Vergleichswert. note 0–10; auswahl aus Soll-Abgleich (F2) oder MECE-Quoten;
|
||||
Zeit/Tokens normiert auf die Baseline derselben Fidelity."""
|
||||
if m.get("soll"):
|
||||
auswahl = 10.0 * m["soll"]["f1"]
|
||||
else:
|
||||
q = m.get("quoten") or {}
|
||||
qa_ = m.get("quoten_artefakte") or {}
|
||||
auswahl = 10.0 * max(0.0, 1.0 - min(1.0, (
|
||||
q.get("dubletten_verdacht", 0) + q.get("luecken", 0) + q.get("fremd", 0)
|
||||
+ qa_.get("sub_dubletten_verdacht", 0) + qa_.get("verwaiste", 0))))
|
||||
zeit = (m.get("dauer_min") or 0) / max(basis.get("dauer_min") or 1, 0.1)
|
||||
tok = _tokens(m) / max(_tokens(basis), 1)
|
||||
return round(W_NOTE * (m.get("note") or 0) + W_AUSWAHL * auswahl
|
||||
@@ -46,151 +72,269 @@ def _tokens(m: dict) -> int:
|
||||
return int(t.get("input") or 0) + int(t.get("output") or 0)
|
||||
|
||||
|
||||
class Trainer:
|
||||
def __init__(self, sitzung: Path, max_trials: int, max_stunden: float, runner=None):
|
||||
class AmeisenTrainer:
|
||||
def __init__(self, sitzung: Path, *, max_trials: int, max_stunden: float, ameisen: int = 3,
|
||||
seed: int = 0, f2_intervall: int = 5, runner=None, runner_f0=None):
|
||||
self.dir = sitzung
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
self.cache_pfad = self.dir / "trials.jsonl"
|
||||
self.cache: dict[str, dict] = {}
|
||||
if self.cache_pfad.exists(): # Resume: bezahlte Trials nie wiederholen
|
||||
for line in self.cache_pfad.read_text(encoding="utf-8").splitlines():
|
||||
e = json.loads(line)
|
||||
self.cache[e["key"]] = e["metrics"]
|
||||
self.rng = random.Random(seed)
|
||||
self.ameisen = ameisen
|
||||
self.f2_intervall = max(f2_intervall, 1)
|
||||
self.max_trials = max_trials
|
||||
self.deadline = time.monotonic() + max_stunden * 3600
|
||||
self.gezahlt = 0
|
||||
self.runner = runner or self._subprozess
|
||||
self.log = []
|
||||
self.runner = runner or self._subprozess # (params, fidelity) -> metrics|None
|
||||
self.runner_f0 = runner_f0 or self._subprozess_f0 # (params) -> {"ok","calls",…}|None
|
||||
self.log: list[str] = []
|
||||
self.basis: dict[str, dict] = {} # Fidelity → Baseline-Metriken
|
||||
self.f0_basis: int | None = None
|
||||
self.best_params: dict = {}
|
||||
self.best_score: float | None = None
|
||||
self.rauschen = 0.5
|
||||
# Pheromon + Trial-Cache (Resume)
|
||||
self.pheromon: dict[str, dict[str, float]] = {
|
||||
n: {str(s): 1.0 for s in stufen(n)} for n in PARAMS}
|
||||
ph = self.dir / "pheromon.json"
|
||||
if ph.exists():
|
||||
gespeichert = json.loads(ph.read_text(encoding="utf-8"))
|
||||
for n, taus in gespeichert.get("pheromon", {}).items():
|
||||
if n in self.pheromon:
|
||||
self.pheromon[n].update({k: float(v) for k, v in taus.items()})
|
||||
self.best_params = gespeichert.get("best_params", {})
|
||||
self.best_score = gespeichert.get("best_score")
|
||||
self.cache_pfad = self.dir / "trials.jsonl"
|
||||
self.cache: dict[str, dict] = {}
|
||||
if self.cache_pfad.exists():
|
||||
for line in self.cache_pfad.read_text(encoding="utf-8").splitlines():
|
||||
e = json.loads(line)
|
||||
self.cache[e["key"]] = e["metrics"]
|
||||
|
||||
# ── Kandidaten ──────────────────────────────────────────────────────────────────
|
||||
def kandidat(self, fidelity: str) -> dict:
|
||||
"""Eine Ameise: je Parameter der Fidelity mit SPARSITY auf Default, sonst
|
||||
Pheromon-gewichtete Stufe. Sparsame Kandidaten → saubere Attribution."""
|
||||
params = {}
|
||||
for name, p in PARAMS.items():
|
||||
if fidelity == "board2" and p["fidelity"] != "board2":
|
||||
continue
|
||||
if self.rng.random() < SPARSITY:
|
||||
continue
|
||||
st = stufen(name)
|
||||
taus = [self.pheromon[name][str(s)] for s in st]
|
||||
wert = self.rng.choices(st, weights=taus)[0]
|
||||
if wert != p["default"]:
|
||||
params[name] = wert
|
||||
return params
|
||||
|
||||
# ── Trial-Ausführung ────────────────────────────────────────────────────────────
|
||||
def _key(self, params: dict, thema: tuple, tag: str = "") -> str:
|
||||
raw = json.dumps({"p": params, "t": thema[0], "tag": tag}, sort_keys=True)
|
||||
def _key(self, params: dict, fidelity: str, tag: str = "") -> str:
|
||||
raw = json.dumps({"p": params, "f": fidelity, "tag": tag}, sort_keys=True)
|
||||
return hashlib.md5(raw.encode()).hexdigest()[:12]
|
||||
|
||||
async def trial(self, params: dict, thema: tuple = HAUPT_THEMA, tag: str = "") -> dict | None:
|
||||
"""tag unterscheidet bewusste Wiederholungen (Baseline n=2, Bestätigung)."""
|
||||
key = self._key(params, thema, tag)
|
||||
async def trial(self, params: dict, fidelity: str, tag: str = "") -> dict | None:
|
||||
key = self._key(params, fidelity, tag)
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
if self.gezahlt >= self.max_trials or time.monotonic() > self.deadline:
|
||||
return None
|
||||
self.gezahlt += 1
|
||||
metrics = await self.runner(params, thema)
|
||||
metrics = await self.runner(params, fidelity, "0")
|
||||
if metrics is not None:
|
||||
with open(self.cache_pfad, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps({"key": key, "params": params, "thema": thema[0],
|
||||
f.write(json.dumps({"key": key, "params": params, "fidelity": fidelity,
|
||||
"tag": tag, "metrics": metrics}, ensure_ascii=False) + "\n")
|
||||
self.cache[key] = metrics
|
||||
return metrics
|
||||
|
||||
async def _subprozess(self, params: dict, thema: tuple) -> dict | None:
|
||||
out = self.dir / f"metrics-{self._key(params, thema)}.json"
|
||||
env = {"CREATOR_PARAMS": json.dumps(params)}
|
||||
import os
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "train_lauf.py", thema[0], thema[1], str(out),
|
||||
env={**os.environ, **env})
|
||||
async def _subprozess(self, params: dict, fidelity: str, topic_suffix: str = "0") -> dict | None:
|
||||
out = self.dir / f"metrics-{self._key(params, fidelity)}{topic_suffix}.json"
|
||||
topic = f"train-t{topic_suffix}"
|
||||
args = ([topic, VORLAGE_TOPIC, str(out), "--board2"] if fidelity == "board2"
|
||||
else [topic, BENCHMARK, str(out)])
|
||||
env = {**os.environ, "CREATOR_PARAMS": json.dumps({**TRAIN_FIXA, **params})}
|
||||
proc = await asyncio.create_subprocess_exec(sys.executable, "train_lauf.py", *args, env=env)
|
||||
rc = await proc.wait()
|
||||
if rc != 0 or not out.exists():
|
||||
self._log(f"Trial fehlgeschlagen (rc={rc}, params={params})")
|
||||
self._log(f"Trial fehlgeschlagen (rc={rc}, {fidelity}, params={params})")
|
||||
return None
|
||||
return json.loads(out.read_text(encoding="utf-8"))
|
||||
|
||||
async def _subprozess_f0(self, params: dict) -> dict | None:
|
||||
out = self.dir / f"f0-{self._key(params, 'f0')}.json"
|
||||
env = {**os.environ, "CREATOR_PARAMS": json.dumps(params)}
|
||||
proc = await asyncio.create_subprocess_exec(sys.executable, "train_f0.py", str(out), env=env)
|
||||
rc = await proc.wait()
|
||||
return json.loads(out.read_text(encoding="utf-8")) if rc == 0 and out.exists() else None
|
||||
|
||||
def _log(self, msg: str) -> None:
|
||||
line = f"{datetime.now(timezone.utc).isoformat()[11:19]} {msg}"
|
||||
print(line, flush=True)
|
||||
self.log.append(line)
|
||||
|
||||
# ── Trainings-Phasen ────────────────────────────────────────────────────────────
|
||||
# ── Pheromon ────────────────────────────────────────────────────────────────────
|
||||
def verstaerke(self, params: dict, delta: float) -> None:
|
||||
for name, wert in params.items():
|
||||
taus = self.pheromon[name]
|
||||
key = str(wert)
|
||||
if key in taus:
|
||||
taus[key] += delta
|
||||
|
||||
def verdunste(self) -> None:
|
||||
for taus in self.pheromon.values():
|
||||
for k in taus:
|
||||
taus[k] = max(0.1, (1 - RHO) * taus[k] + RHO * 1.0) # Drift zurück zu uniform
|
||||
|
||||
# ── Hauptschleife ───────────────────────────────────────────────────────────────
|
||||
async def run(self) -> dict:
|
||||
# Phase 0: Baseline zweimal → Score-Basis + Rausch-Schwelle
|
||||
self._log("Baseline (2 Läufe)…")
|
||||
b1 = await self.trial({}, tag="baseline-1")
|
||||
b2 = await self.trial({}, tag="baseline-2")
|
||||
# Baseline F1 ×2 → Score-Basis + Rausch-Schwelle; F0-Basis für den Struktur-Proxy
|
||||
f0 = await self.runner_f0({})
|
||||
self.f0_basis = (f0 or {}).get("calls")
|
||||
b1 = await self.trial({}, "board2", tag="baseline-1")
|
||||
b2 = await self.trial({}, "board2", tag="baseline-2")
|
||||
if not b1 or not b2:
|
||||
self._log("Baseline unvollständig — Abbruch.")
|
||||
return {}
|
||||
self.basis = b1
|
||||
return self.best_params
|
||||
self.basis["board2"] = b1
|
||||
s1, s2 = score(b1, b1), score(b2, b1)
|
||||
self.rauschen = max(abs(s1 - s2), 0.5) # Mindest-Schwelle gegen Glücks-Übernahmen
|
||||
best_params: dict = {}
|
||||
best_score = max(s1, s2)
|
||||
self._log(f"Baseline-Score {s1}/{s2}, Rausch-Schwelle {self.rauschen}")
|
||||
self.rauschen = max(abs(s1 - s2), 0.5)
|
||||
if self.best_score is None:
|
||||
self.best_score = max(s1, s2)
|
||||
self._log(f"Baseline {s1}/{s2}, Rauschen {self.rauschen}, F0-Basis {self.f0_basis} Calls")
|
||||
|
||||
# Phase 1: Screening — je Parameter ±1 Schritt, Effekt vs. Rauschen
|
||||
effekte: list[tuple[float, str, float]] = [] # (|effekt|, name, bester_wert)
|
||||
for name in PARAMS:
|
||||
lo, hi = schritte(name)
|
||||
for wert in dict.fromkeys((lo, hi)): # lo==hi am Rand nur einmal
|
||||
if wert == PARAMS[name]["default"]:
|
||||
continue
|
||||
m = await self.trial({**best_params, name: wert})
|
||||
if m is None:
|
||||
continue
|
||||
delta = score(m, self.basis) - best_score
|
||||
self._log(f"Screening {name}={wert}: Δ{delta:+.2f}")
|
||||
if delta > self.rauschen:
|
||||
effekte.append((delta, name, wert))
|
||||
effekte.sort(reverse=True)
|
||||
self._log(f"Wirksam: {[(n, w) for _, n, w in effekte]}")
|
||||
|
||||
# Phase 2: Koordinaten-Suche über ALLE wirksamen Parameter (keine feste Obergrenze),
|
||||
# Übernahme nur nach Bestätigungslauf
|
||||
for _, name, start_wert in effekte:
|
||||
wert = start_wert
|
||||
p = PARAMS[name]
|
||||
richtung = p["step"] if wert > p["default"] else -p["step"]
|
||||
while True:
|
||||
kandidat = {**best_params, name: wert}
|
||||
m = await self.trial(kandidat)
|
||||
if m is None:
|
||||
runde, stagnation, gezahlt_vorher = 0, 0, self.gezahlt
|
||||
while (self.gezahlt < self.max_trials and time.monotonic() < self.deadline
|
||||
and stagnation < 20): # konvergiert: nur noch Cache-Treffer → fertig
|
||||
if runde > 0:
|
||||
stagnation = stagnation + 1 if self.gezahlt == gezahlt_vorher else 0
|
||||
gezahlt_vorher = self.gezahlt
|
||||
runde += 1
|
||||
fidelity = "voll" if runde % self.f2_intervall == 0 else "board2"
|
||||
if fidelity == "voll" and "voll" not in self.basis:
|
||||
base = await self.trial({}, "voll", tag="baseline-voll")
|
||||
if base is None:
|
||||
break
|
||||
delta = score(m, self.basis) - best_score
|
||||
if delta <= self.rauschen:
|
||||
self.basis["voll"] = base
|
||||
kandidaten = []
|
||||
for _ in range(self.ameisen * 3): # ziehen bis K einzigartige nicht-leere da sind
|
||||
k = self.kandidat(fidelity)
|
||||
if k and k not in kandidaten and k != self.best_params:
|
||||
kandidaten.append(k)
|
||||
if len(kandidaten) >= self.ameisen:
|
||||
break
|
||||
m2 = await self.trial(kandidat, tag="bestaetigung")
|
||||
if m2 is None or score(m2, self.basis) - best_score <= self.rauschen:
|
||||
self._log(f"{name}={wert}: nicht bestätigt — verworfen")
|
||||
break
|
||||
best_params, best_score = kandidat, min(score(m, self.basis), score(m2, self.basis))
|
||||
self._log(f"ÜBERNOMMEN {name}={wert} → Score {best_score}")
|
||||
naechster = round(wert + richtung, 4)
|
||||
if not p["min"] <= naechster <= p["max"]:
|
||||
break
|
||||
wert = naechster
|
||||
if not kandidaten:
|
||||
continue
|
||||
# F0-Filter: Invarianten + Struktur-Proxy, parallel, kostenlos
|
||||
f0s = await asyncio.gather(*[self.runner_f0(k) for k in kandidaten])
|
||||
ueberlebende = []
|
||||
for k, f in zip(kandidaten, f0s):
|
||||
if f is None or not f.get("ok") or f.get("invarianten_fehler"):
|
||||
self._log(f"F0 verwirft {k} (Invarianten)")
|
||||
elif self.f0_basis and f.get("calls", 0) > self.f0_basis * F0_CALL_FAKTOR:
|
||||
self._log(f"F0 verwirft {k} (Calls {f['calls']} > {self.f0_basis}×{F0_CALL_FAKTOR})")
|
||||
else:
|
||||
ueberlebende.append(k)
|
||||
if not ueberlebende:
|
||||
self.verdunste()
|
||||
continue
|
||||
# F1/F2 parallel (eigene Topic-Namen)
|
||||
ergebnisse = await asyncio.gather(*[
|
||||
self._bewertet(k, fidelity, str(i + 1)) for i, k in enumerate(ueberlebende)])
|
||||
bewertet = [(k, m, score(m, self.basis[fidelity]))
|
||||
for k, m in ergebnisse if m is not None]
|
||||
if not bewertet:
|
||||
continue
|
||||
bewertet.sort(key=lambda x: -x[2])
|
||||
self.verdunste()
|
||||
top_k, _top_m, top_s = bewertet[0]
|
||||
self._log(f"Runde {runde} ({fidelity}): top {top_s} {top_k} "
|
||||
f"(best {self.best_score})")
|
||||
if top_s > (self.best_score or 0):
|
||||
self.verstaerke(top_k, delta=1.0)
|
||||
if top_s > (self.best_score or 0) + self.rauschen:
|
||||
m2 = await self.trial(top_k, fidelity, tag="bestaetigung")
|
||||
if m2 is not None and score(m2, self.basis[fidelity]) > self.best_score + self.rauschen:
|
||||
self.best_params = top_k
|
||||
self.best_score = min(top_s, score(m2, self.basis[fidelity]))
|
||||
self._log(f"NEUER BESTER {self.best_params} → {self.best_score}")
|
||||
else:
|
||||
self._log(f"{top_k}: nicht bestätigt")
|
||||
self.verstaerke(self.best_params, delta=0.5) # Elite hält die Spur warm
|
||||
self._speichern()
|
||||
self._speichern()
|
||||
return self.best_params
|
||||
|
||||
# Validierung auf dem zweiten Thema
|
||||
if best_params:
|
||||
v_base = await self.trial({}, thema=VALIDIER_THEMA, tag="val-base")
|
||||
v_best = await self.trial(best_params, thema=VALIDIER_THEMA, tag="val-best")
|
||||
if v_base and v_best:
|
||||
self._log(f"Validierung {VALIDIER_THEMA[0]}: Baseline {score(v_base, v_base)}"
|
||||
f" → Best {score(v_best, v_base)}")
|
||||
async def _bewertet(self, params: dict, fidelity: str, suffix: str):
|
||||
if self.gezahlt >= self.max_trials or time.monotonic() > self.deadline:
|
||||
return params, None
|
||||
key = self._key(params, fidelity)
|
||||
if key in self.cache:
|
||||
return params, self.cache[key]
|
||||
self.gezahlt += 1
|
||||
m = await self.runner(params, fidelity, suffix)
|
||||
if m is not None:
|
||||
with open(self.cache_pfad, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps({"key": key, "params": params, "fidelity": fidelity,
|
||||
"tag": "", "metrics": m}, ensure_ascii=False) + "\n")
|
||||
self.cache[key] = m
|
||||
return params, m
|
||||
|
||||
self._schreibe_report(best_params, best_score)
|
||||
return best_params
|
||||
|
||||
def _schreibe_report(self, best_params: dict, best_score: float) -> None:
|
||||
def _speichern(self) -> None:
|
||||
from fsutil import atomic_write_json, atomic_write_text
|
||||
atomic_write_json(self.dir / "beste_params.json", best_params, indent=1)
|
||||
report = ["# Trainings-Report", "",
|
||||
f"Trials bezahlt: {self.gezahlt}/{self.max_trials}",
|
||||
f"Bester Score: {best_score} (Baseline-Rauschen {self.rauschen})",
|
||||
f"Beste Parameter: `{json.dumps(best_params, ensure_ascii=False)}`",
|
||||
"", "Nutzung: `CREATOR_PARAMS=$(cat beste_params.json) make dev` —",
|
||||
atomic_write_json(self.dir / "pheromon.json",
|
||||
{"pheromon": self.pheromon, "best_params": self.best_params,
|
||||
"best_score": self.best_score}, indent=1)
|
||||
atomic_write_json(self.dir / "beste_params.json", self.best_params, indent=1)
|
||||
staerkste = sorted(((n, max(t.items(), key=lambda x: x[1]))
|
||||
for n, t in self.pheromon.items()),
|
||||
key=lambda x: -x[1][1])[:10]
|
||||
report = ["# Trainings-Report (Ameisen)", "",
|
||||
f"Bezahlte Läufe: {self.gezahlt}/{self.max_trials}",
|
||||
f"Bester Score: {self.best_score} (Rauschband {self.rauschen})",
|
||||
f"Beste Parameter: `{json.dumps(self.best_params, ensure_ascii=False)}`",
|
||||
"", "Stärkste Pheromon-Spuren:",
|
||||
*[f"- {n}={s} (τ={t:.1f})" for n, (s, t) in staerkste],
|
||||
"", "Nutzung: `CREATOR_PARAMS=$(cat beste_params.json)` —",
|
||||
"Übernahme nach config.py bleibt eine manuelle Entscheidung.", "", "## Log", ""]
|
||||
report += [f"- {l}" for l in self.log]
|
||||
report += [f"- {l}" for l in self.log[-200:]]
|
||||
atomic_write_text(self.dir / "report.md", "\n".join(report))
|
||||
print(f"\nReport: {self.dir / 'report.md'}")
|
||||
|
||||
|
||||
async def init_vorlage() -> None:
|
||||
"""Einmalig: Prüfstand-Volllauf mit Defaults, Ergebnis bleibt als Frozen-Inventar-Vorlage
|
||||
liegen (Topic train-vorlage). Nach Korpus-/Prompt-Änderungen neu ausführen."""
|
||||
import agents
|
||||
import database
|
||||
from blocks import generate_blocks
|
||||
from fsutil import atomic_write_json as awj
|
||||
from paths import source_path
|
||||
await database.init_db()
|
||||
agents.on_event = database.add_event
|
||||
await database.create_topic(VORLAGE_TOPIC)
|
||||
qp = source_path(VORLAGE_TOPIC)
|
||||
qp.parent.mkdir(parents=True, exist_ok=True)
|
||||
awj(qp, {"type": "uni", "location": BENCHMARK, "spec": ""})
|
||||
await generate_blocks(VORLAGE_TOPIC, provider="minimax", research=True, qa_force=True)
|
||||
await database.close_db()
|
||||
print(f"Vorlage {VORLAGE_TOPIC} steht — Training kann starten (make train).")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--trials", type=int, default=40)
|
||||
ap.add_argument("--stunden", type=float, default=12.0)
|
||||
ap.add_argument("--sitzung", default=datetime.now(timezone.utc).strftime("%Y%m%d-%H%M"))
|
||||
ap.add_argument("--init", action="store_true", help="Frozen-Inventar-Vorlage bauen")
|
||||
ap.add_argument("--trials", type=int, default=60)
|
||||
ap.add_argument("--stunden", type=float, default=8.0)
|
||||
ap.add_argument("--ameisen", type=int, default=3)
|
||||
ap.add_argument("--seed", type=int, default=0)
|
||||
ap.add_argument("--f2-intervall", type=int, default=5)
|
||||
ap.add_argument("--sitzung", default="aco") # fester Default: Resume über Sitzungen hinweg
|
||||
args = ap.parse_args()
|
||||
trainer = Trainer(STORAGE_DIR / "train" / args.sitzung, args.trials, args.stunden)
|
||||
if args.init:
|
||||
asyncio.run(init_vorlage())
|
||||
return
|
||||
trainer = AmeisenTrainer(STORAGE_DIR / "train" / args.sitzung,
|
||||
max_trials=args.trials, max_stunden=args.stunden,
|
||||
ameisen=args.ameisen, seed=args.seed,
|
||||
f2_intervall=args.f2_intervall)
|
||||
asyncio.run(trainer.run())
|
||||
|
||||
|
||||
|
||||
59
backend/train_f0.py
Normal file
59
backend/train_f0.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Fidelity 0 des Trainers: Fake-E2E-Lauf im Subprozess — Sekunden, null Tokens.
|
||||
|
||||
Misst mit den CREATOR_PARAMS des Kandidaten: (a) halten die Invarianten? (b) wie viele
|
||||
Agenten-Calls erzeugt die Struktur (Proxy für Tokens/Laufzeit)? Unsinnige Kandidaten
|
||||
fallen hier raus, bevor ein echter Lauf Geld kostet.
|
||||
|
||||
CLI: python3 train_f0.py <ausgabe.json> (CREATOR_PARAMS im ENV)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# WICHTIG: config (mit CREATOR_PARAMS) lädt vor allen Pipeline-Modulen
|
||||
import database
|
||||
from fake_agents import Welt, aktivieren
|
||||
from fsutil import atomic_write_json
|
||||
|
||||
|
||||
async def f0(out: str) -> None:
|
||||
tmp = Path(tempfile.mkdtemp(prefix="train-f0-"))
|
||||
database.DB_PATH = tmp / "f0.db"
|
||||
database._db = None
|
||||
await database.init_db()
|
||||
welt = Welt()
|
||||
aktivieren(welt)
|
||||
import board_inventory as bi
|
||||
import qa
|
||||
qa.QA_DIR = tmp / "qa"
|
||||
from pipeline import GenContext
|
||||
|
||||
work = tmp / "arbeit"
|
||||
work.mkdir()
|
||||
files = {"arbeit": work, "final": tmp / "blocks.md",
|
||||
"sub_roh": tmp / "sub_roh.json", "sidecar": tmp / "subblocks.json",
|
||||
"facts": tmp / "facts.json", "question_pattern": tmp / "question_pattern.json",
|
||||
"artefakte": tmp / "artefakte.json", "outline": tmp / "outline.json",
|
||||
"outline_slots": [tmp / f"outline-{i}.json" for i in (1, 2, 3)],
|
||||
"research": [work / f"research-{i}.md" for i in (1, 2, 3, 4, 5)]}
|
||||
ctx = GenContext(topic="f0", provider="claude", is_cancelled=lambda: False)
|
||||
start = time.monotonic()
|
||||
ok = await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "",
|
||||
research=True, qa_force=True), timeout=180)
|
||||
from tests.invarianten import pruefe_invarianten
|
||||
fehler = await pruefe_invarianten("f0", files)
|
||||
atomic_write_json(Path(out), {
|
||||
"ok": bool(ok), "invarianten_fehler": fehler, "calls": len(welt.calls),
|
||||
"dauer_s": round(time.monotonic() - start, 1)}, indent=1)
|
||||
await database.close_db()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
raise SystemExit("Nutzung: python3 train_f0.py <ausgabe.json>")
|
||||
asyncio.run(f0(sys.argv[1]))
|
||||
@@ -1,8 +1,12 @@
|
||||
"""EIN Trainings-Trial: frischer Prozess (CREATOR_PARAMS wirkt beim Import), ein
|
||||
kompletter Mini-Lauf, deterministische Metriken als JSON — danach ist das Topic weg.
|
||||
"""EIN Trainings-Trial: frischer Prozess (CREATOR_PARAMS wirkt beim Import), ein Mini-Lauf,
|
||||
deterministische Metriken als JSON — danach ist das Trial-Topic weg.
|
||||
|
||||
CLI: python3 train_lauf.py <topic> <benchmark-location> <ausgabe.json>
|
||||
(benchmark-location repo-relativ, z. B. "benchmarks/sortierverfahren")
|
||||
Fidelity-Modi:
|
||||
voll python3 train_lauf.py <topic> <benchmark-location> <ausgabe.json>
|
||||
— kompletter Lauf (Research + Board 1 + Board 2) + Soll-Abgleich gegen
|
||||
<benchmark-location>/soll.json (falls vorhanden)
|
||||
board2 python3 train_lauf.py <topic> <vorlage-topic> <ausgabe.json> --board2
|
||||
— Frozen-Inventar: Vorlage kopieren, Board 2 komplett neu (research=False)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -16,26 +20,55 @@ import database
|
||||
import qa
|
||||
from blocks import generate_blocks
|
||||
from fsutil import atomic_write_json
|
||||
from paths import source_path, topic_dir
|
||||
from paths import blocks_path, source_path, topic_dir
|
||||
from textkit import _norm_title
|
||||
|
||||
|
||||
async def trial(topic: str, location: str, out: str) -> None:
|
||||
def soll_abgleich(ist_titel: list[str], soll: dict) -> dict:
|
||||
"""Ground-Truth-Vergleich: welche Soll-Blöcke fehlen, was ist überzählig.
|
||||
Match über Norm-Gleichheit gegen Titel+Alternativen, Fallback beidseitiges Containment."""
|
||||
ist = {_norm_title(t): t for t in ist_titel}
|
||||
treffer, fehlend, belegt = [], [], set()
|
||||
for block in soll.get("bloecke", []):
|
||||
formen = {_norm_title(block["titel"])} | {_norm_title(a) for a in block.get("alternativen", [])}
|
||||
gefunden = next((n for n in ist if n in formen), None)
|
||||
if gefunden is None:
|
||||
gefunden = next((n for n in ist if any(f and (f in n or n in f) for f in formen)), None)
|
||||
if gefunden:
|
||||
treffer.append(block["titel"])
|
||||
belegt.add(gefunden)
|
||||
else:
|
||||
fehlend.append(block["titel"])
|
||||
extra = [t for n, t in ist.items() if n not in belegt]
|
||||
n_soll = max(len(soll.get("bloecke", [])), 1)
|
||||
praezision = len(treffer) / max(len(ist), 1)
|
||||
recall = len(treffer) / n_soll
|
||||
f1 = 2 * praezision * recall / max(praezision + recall, 1e-9)
|
||||
return {"treffer": treffer, "fehlend": fehlend, "extra": extra, "f1": round(f1, 3)}
|
||||
|
||||
|
||||
async def trial(topic: str, quelle: str, out: str, board2: bool) -> None:
|
||||
await database.init_db()
|
||||
agents.on_event = database.add_event # sonst keine Dauer-/Token-Events (main.py-lifespan-Pendant)
|
||||
try:
|
||||
await _aufraeumen(topic) # Reste eines abgebrochenen Trials
|
||||
await database.create_topic(topic)
|
||||
qp = source_path(topic)
|
||||
qp.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write_json(qp, {"type": "uni", "location": location, "spec": ""})
|
||||
start = datetime.now(timezone.utc)
|
||||
# qa_force=True: das Gate misst, pausiert den Trial aber nie
|
||||
await generate_blocks(topic, provider="minimax", research=True, qa_force=True)
|
||||
if board2:
|
||||
await _frozen_inventar(topic, vorlage=quelle)
|
||||
await generate_blocks(topic, provider="minimax", research=False, qa_force=True)
|
||||
else:
|
||||
qp = source_path(topic)
|
||||
qp.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write_json(qp, {"type": "uni", "location": quelle, "spec": ""})
|
||||
# qa_force=True: das Gate misst nichts und pausiert nie
|
||||
await generate_blocks(topic, provider="minimax", research=True, qa_force=True)
|
||||
dauer_min = round((datetime.now(timezone.utc) - start).total_seconds() / 60, 1)
|
||||
report = await qa.qa_report(topic, llm=False) or {}
|
||||
lauf = report.get("lauf") or {}
|
||||
metrics = {
|
||||
"topic": topic,
|
||||
"fidelity": "board2" if board2 else "voll",
|
||||
"note": report.get("note"),
|
||||
"note_artefakte": report.get("note_artefakte"),
|
||||
"quoten": report.get("quoten") or {},
|
||||
@@ -45,17 +78,42 @@ async def trial(topic: str, location: str, out: str) -> None:
|
||||
"tokens": (lauf.get("tokens") or {}),
|
||||
"agents": (lauf.get("agents") or {}),
|
||||
}
|
||||
if not board2:
|
||||
soll_pfad = Path(__file__).resolve().parent.parent / quelle / "soll.json"
|
||||
if soll_pfad.exists():
|
||||
import json
|
||||
done = await database.kanban_cards(topic, board="inventory", stage="done_block")
|
||||
titel = [c["payload"].get("title", "") for c in done if c["kind"] == "block"]
|
||||
metrics["soll"] = soll_abgleich(titel, json.loads(soll_pfad.read_text(encoding="utf-8")))
|
||||
atomic_write_json(Path(out), metrics, indent=1)
|
||||
finally:
|
||||
await _aufraeumen(topic)
|
||||
await database.close_db()
|
||||
|
||||
|
||||
async def _frozen_inventar(topic: str, vorlage: str) -> None:
|
||||
"""Board-1-Stand der Vorlage übernehmen und Board 2 auf Start zurücksetzen —
|
||||
reset_board_from_stage räumt DB-Spiegel, globale Dateien und Resume-Slots."""
|
||||
import board_inventory
|
||||
from blocks import _blocks_files
|
||||
await database.copy_topic(vorlage, topic)
|
||||
tdir = topic_dir(topic)
|
||||
tdir.mkdir(parents=True, exist_ok=True)
|
||||
for src, dst in ((source_path(vorlage), source_path(topic)),
|
||||
(blocks_path(vorlage), blocks_path(topic))):
|
||||
if src.exists():
|
||||
shutil.copy(src, dst)
|
||||
files = _blocks_files(topic)
|
||||
files["arbeit"].mkdir(parents=True, exist_ok=True)
|
||||
await board_inventory.reset_board_from_stage(topic, "artefacts", "subblocks", files)
|
||||
|
||||
|
||||
async def _aufraeumen(topic: str) -> None:
|
||||
"""Topic restlos entfernen (DELETE-/topics-Sequenz aus routes.py)."""
|
||||
await database.delete_topic(topic)
|
||||
await database.delete_block_data(topic)
|
||||
await database.delete_topic_pipeline(topic)
|
||||
await database.kanban_reset(topic)
|
||||
await database.delete_source(topic)
|
||||
await database.delete_guide_content(topic)
|
||||
shutil.rmtree(topic_dir(topic), ignore_errors=True)
|
||||
@@ -63,6 +121,7 @@ async def _aufraeumen(topic: str) -> None:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 4:
|
||||
raise SystemExit("Nutzung: python3 train_lauf.py <topic> <benchmark-location> <ausgabe.json>")
|
||||
asyncio.run(trial(sys.argv[1], sys.argv[2], sys.argv[3]))
|
||||
args = [a for a in sys.argv[1:] if a != "--board2"]
|
||||
if len(args) != 3:
|
||||
raise SystemExit("Nutzung: python3 train_lauf.py <topic> <quelle> <ausgabe.json> [--board2]")
|
||||
asyncio.run(trial(args[0], args[1], args[2], board2="--board2" in sys.argv))
|
||||
|
||||
@@ -8,43 +8,43 @@ bewusst NICHT hier — die Messlatte darf nie Teil des Suchraums sein.
|
||||
|
||||
PARAMS: dict[str, dict] = {
|
||||
# Recherche / Inventar
|
||||
"RESEARCH_THEMA_AGENTS": {"default": 5, "min": 2, "max": 8, "step": 1, "kategorie": "qualitaet"},
|
||||
"RESEARCH_READERS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet"},
|
||||
"RESEARCH_SECTION_CHARS": {"default": 12000, "min": 6000, "max": 24000, "step": 3000, "kategorie": "tokens"},
|
||||
"DEDUP_PAIR_FLOOR": {"default": 0.6, "min": 0.45, "max": 0.8, "step": 0.05, "kategorie": "auswahl"},
|
||||
"DEDUP_TITLE_AUTO": {"default": 0.95, "min": 0.9, "max": 0.99, "step": 0.01, "kategorie": "auswahl"},
|
||||
"DEDUP_GLOBAL_FLOOR": {"default": 0.65, "min": 0.5, "max": 0.8, "step": 0.05, "kategorie": "auswahl"},
|
||||
"DEDUP_PAIRS_CHUNK": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit"},
|
||||
"FILTER_CHUNK": {"default": 35, "min": 15, "max": 60, "step": 10, "kategorie": "laufzeit"},
|
||||
"FILTER_RECHECK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"CONSOLIDATION_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"RESEARCH_THEMA_AGENTS": {"default": 5, "min": 2, "max": 8, "step": 1, "kategorie": "qualitaet", "fidelity": "voll"},
|
||||
"RESEARCH_READERS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet", "fidelity": "voll"},
|
||||
"RESEARCH_SECTION_CHARS": {"default": 12000, "min": 6000, "max": 24000, "step": 3000, "kategorie": "tokens", "fidelity": "voll"},
|
||||
"DEDUP_PAIR_FLOOR": {"default": 0.6, "min": 0.45, "max": 0.8, "step": 0.05, "kategorie": "auswahl", "fidelity": "voll"},
|
||||
"DEDUP_TITLE_AUTO": {"default": 0.95, "min": 0.9, "max": 0.99, "step": 0.01, "kategorie": "auswahl", "fidelity": "voll"},
|
||||
"DEDUP_GLOBAL_FLOOR": {"default": 0.65, "min": 0.5, "max": 0.8, "step": 0.05, "kategorie": "auswahl", "fidelity": "voll"},
|
||||
"DEDUP_PAIRS_CHUNK": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit", "fidelity": "voll"},
|
||||
"FILTER_CHUNK": {"default": 35, "min": 15, "max": 60, "step": 10, "kategorie": "laufzeit", "fidelity": "voll"},
|
||||
"FILTER_RECHECK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet", "fidelity": "voll"},
|
||||
"CONSOLIDATION_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet", "fidelity": "voll"},
|
||||
# Subbausteine
|
||||
"SUBBLOCK_CHUNK": {"default": 10, "min": 4, "max": 20, "step": 2, "kategorie": "laufzeit"},
|
||||
"SUBBLOCK_MIN": {"default": 5, "min": 2, "max": 10, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_MAX_ROUNDS": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_EXTRA_ROUNDS": {"default": 2, "min": 0, "max": 4, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"SUBBLOCK_CHUNK": {"default": 10, "min": 4, "max": 20, "step": 2, "kategorie": "laufzeit", "fidelity": "board2"},
|
||||
"SUBBLOCK_MIN": {"default": 5, "min": 2, "max": 10, "step": 1, "kategorie": "auswahl", "fidelity": "board2"},
|
||||
"SUBBLOCK_MAX_ROUNDS": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "auswahl", "fidelity": "board2"},
|
||||
"SUBBLOCK_EXTRA_ROUNDS": {"default": 2, "min": 0, "max": 4, "step": 1, "kategorie": "auswahl", "fidelity": "board2"},
|
||||
"SUBBLOCK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet", "fidelity": "board2"},
|
||||
# Facts / Artefakte / Fragen
|
||||
"FACTS_CHUNK_SUBS": {"default": 10, "min": 4, "max": 20, "step": 2, "kategorie": "laufzeit"},
|
||||
"FACTS_CHECK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"QUESTION_CHUNK_SUBS": {"default": 25, "min": 10, "max": 50, "step": 5, "kategorie": "laufzeit"},
|
||||
"QUESTION_MAX_ROUNDS": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"ARTEFACT_CHUNK_SUBS": {"default": 25, "min": 10, "max": 50, "step": 5, "kategorie": "laufzeit"},
|
||||
"FACTS_CHUNK_SUBS": {"default": 10, "min": 4, "max": 20, "step": 2, "kategorie": "laufzeit", "fidelity": "board2"},
|
||||
"FACTS_CHECK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet", "fidelity": "board2"},
|
||||
"QUESTION_CHUNK_SUBS": {"default": 25, "min": 10, "max": 50, "step": 5, "kategorie": "laufzeit", "fidelity": "board2"},
|
||||
"QUESTION_MAX_ROUNDS": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet", "fidelity": "board2"},
|
||||
"ARTEFACT_CHUNK_SUBS": {"default": 25, "min": 10, "max": 50, "step": 5, "kategorie": "laufzeit", "fidelity": "board2"},
|
||||
# Embedding-Schwellen (Auswahl-Kern)
|
||||
"SUB_VARIANT_COS": {"default": 0.90, "min": 0.85, "max": 0.96, "step": 0.01, "kategorie": "auswahl"},
|
||||
"SEED_COVER_COS": {"default": 0.80, "min": 0.7, "max": 0.9, "step": 0.02, "kategorie": "auswahl"},
|
||||
"SUB_DUP_KANDIDAT_COS": {"default": 0.75, "min": 0.65, "max": 0.85, "step": 0.02, "kategorie": "auswahl"},
|
||||
"EMBEDDING_BLOCK_FLOOR": {"default": 0.5, "min": 0.35, "max": 0.65, "step": 0.05, "kategorie": "auswahl"},
|
||||
"CROSS_CHUNK_PAARE": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit"},
|
||||
"SUB_VARIANT_COS": {"default": 0.90, "min": 0.85, "max": 0.96, "step": 0.01, "kategorie": "auswahl", "fidelity": "board2"},
|
||||
"SEED_COVER_COS": {"default": 0.80, "min": 0.7, "max": 0.9, "step": 0.02, "kategorie": "auswahl", "fidelity": "board2"},
|
||||
"SUB_DUP_KANDIDAT_COS": {"default": 0.75, "min": 0.65, "max": 0.85, "step": 0.02, "kategorie": "auswahl", "fidelity": "board2"},
|
||||
"EMBEDDING_BLOCK_FLOOR": {"default": 0.5, "min": 0.35, "max": 0.65, "step": 0.05, "kategorie": "auswahl", "fidelity": "voll"},
|
||||
"CROSS_CHUNK_PAARE": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit", "fidelity": "board2"},
|
||||
# Guide
|
||||
"MAX_WRITER_ROUNDS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet"},
|
||||
"GATE_FIX_MIN": {"default": 3, "min": 1, "max": 6, "step": 1, "kategorie": "qualitaet"},
|
||||
"WRITER_SPLIT_SUBS": {"default": 30, "min": 15, "max": 45, "step": 5, "kategorie": "qualitaet"},
|
||||
"MAX_WRITER_ROUNDS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet", "fidelity": "board2"},
|
||||
"GATE_FIX_MIN": {"default": 3, "min": 1, "max": 6, "step": 1, "kategorie": "qualitaet", "fidelity": "board2"},
|
||||
"WRITER_SPLIT_SUBS": {"default": 30, "min": 15, "max": 45, "step": 5, "kategorie": "qualitaet", "fidelity": "board2"},
|
||||
# Engine / Kosten
|
||||
"CONSENSUS_GRACE": {"default": 300, "min": 0, "max": 600, "step": 60, "kategorie": "laufzeit"},
|
||||
"MAX_RESTARTS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "laufzeit"},
|
||||
"EVIDENCE_BUDGET_CHARS": {"default": 48000, "min": 16000, "max": 64000, "step": 8000, "kategorie": "tokens"},
|
||||
"QUELLE_RELEVANZ_CHUNK": {"default": 12, "min": 6, "max": 24, "step": 3, "kategorie": "laufzeit"},
|
||||
"CONSENSUS_GRACE": {"default": 300, "min": 0, "max": 600, "step": 60, "kategorie": "laufzeit", "fidelity": "board2"},
|
||||
"MAX_RESTARTS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "laufzeit", "fidelity": "board2"},
|
||||
"EVIDENCE_BUDGET_CHARS": {"default": 48000, "min": 16000, "max": 64000, "step": 8000, "kategorie": "tokens", "fidelity": "board2"},
|
||||
"QUELLE_RELEVANZ_CHUNK": {"default": 12, "min": 6, "max": 24, "step": 3, "kategorie": "laufzeit", "fidelity": "voll"},
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user