128 lines
5.7 KiB
Python
128 lines
5.7 KiB
Python
"""EIN Trainings-Trial: frischer Prozess (CREATOR_PARAMS wirkt beim Import), ein Mini-Lauf,
|
|
deterministische Metriken als JSON — danach ist das Trial-Topic weg.
|
|
|
|
Fidelity-Modi:
|
|
voll python3 train_lauf.py <topic> <benchmark-location> <ausgabe.json>
|
|
— kompletter Lauf (Research + Board 1 + Board 2) + Soll-Abgleich gegen
|
|
<benchmark-location>/soll.json (falls vorhanden)
|
|
board2 python3 train_lauf.py <topic> <vorlage-topic> <ausgabe.json> --board2
|
|
— Frozen-Inventar: Vorlage kopieren, Board 2 komplett neu (research=False)
|
|
"""
|
|
|
|
import asyncio
|
|
import shutil
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import agents
|
|
import database
|
|
import qa
|
|
from blocks import generate_blocks
|
|
from fsutil import atomic_write_json
|
|
from paths import blocks_path, source_path, topic_dir
|
|
from textkit import _norm_title
|
|
|
|
|
|
def soll_abgleich(ist_titel: list[str], soll: dict) -> dict:
|
|
"""Ground-Truth-Vergleich: welche Soll-Blöcke fehlen, was ist überzählig.
|
|
Match über Norm-Gleichheit gegen Titel+Alternativen, Fallback beidseitiges Containment."""
|
|
ist = {_norm_title(t): t for t in ist_titel}
|
|
treffer, fehlend, belegt = [], [], set()
|
|
for block in soll.get("bloecke", []):
|
|
formen = {_norm_title(block["titel"])} | {_norm_title(a) for a in block.get("alternativen", [])}
|
|
gefunden = next((n for n in ist if n in formen), None)
|
|
if gefunden is None:
|
|
gefunden = next((n for n in ist if any(f and (f in n or n in f) for f in formen)), None)
|
|
if gefunden:
|
|
treffer.append(block["titel"])
|
|
belegt.add(gefunden)
|
|
else:
|
|
fehlend.append(block["titel"])
|
|
extra = [t for n, t in ist.items() if n not in belegt]
|
|
n_soll = max(len(soll.get("bloecke", [])), 1)
|
|
praezision = len(treffer) / max(len(ist), 1)
|
|
recall = len(treffer) / n_soll
|
|
f1 = 2 * praezision * recall / max(praezision + recall, 1e-9)
|
|
return {"treffer": treffer, "fehlend": fehlend, "extra": extra, "f1": round(f1, 3)}
|
|
|
|
|
|
async def trial(topic: str, quelle: str, out: str, board2: bool) -> None:
|
|
await database.init_db()
|
|
agents.on_event = database.add_event # sonst keine Dauer-/Token-Events (main.py-lifespan-Pendant)
|
|
try:
|
|
await _aufraeumen(topic) # Reste eines abgebrochenen Trials
|
|
await database.create_topic(topic)
|
|
start = datetime.now(timezone.utc)
|
|
if board2:
|
|
await _frozen_inventar(topic, vorlage=quelle)
|
|
await generate_blocks(topic, provider="minimax", research=False, qa_force=True)
|
|
else:
|
|
qp = source_path(topic)
|
|
qp.parent.mkdir(parents=True, exist_ok=True)
|
|
atomic_write_json(qp, {"type": "uni", "location": quelle, "spec": ""})
|
|
# qa_force=True: das Gate misst nichts und pausiert nie
|
|
await generate_blocks(topic, provider="minimax", research=True, qa_force=True)
|
|
dauer_min = round((datetime.now(timezone.utc) - start).total_seconds() / 60, 1)
|
|
report = await qa.qa_report(topic, llm=False) or {}
|
|
lauf = report.get("lauf") or {}
|
|
metrics = {
|
|
"topic": topic,
|
|
"fidelity": "board2" if board2 else "voll",
|
|
"note": report.get("note"),
|
|
"note_artefakte": report.get("note_artefakte"),
|
|
"quoten": report.get("quoten") or {},
|
|
"quoten_artefakte": report.get("quoten_artefakte") or {},
|
|
"bloecke": report.get("bloecke"),
|
|
"dauer_min": lauf.get("dauer_min") or dauer_min,
|
|
"tokens": (lauf.get("tokens") or {}),
|
|
"agents": (lauf.get("agents") or {}),
|
|
}
|
|
if not board2:
|
|
soll_pfad = Path(__file__).resolve().parent.parent / quelle / "soll.json"
|
|
if soll_pfad.exists():
|
|
import json
|
|
done = await database.kanban_cards(topic, board="inventory", stage="done_block")
|
|
titel = [c["payload"].get("title", "") for c in done if c["kind"] == "block"]
|
|
metrics["soll"] = soll_abgleich(titel, json.loads(soll_pfad.read_text(encoding="utf-8")))
|
|
atomic_write_json(Path(out), metrics, indent=1)
|
|
finally:
|
|
await _aufraeumen(topic)
|
|
await database.close_db()
|
|
|
|
|
|
async def _frozen_inventar(topic: str, vorlage: str) -> None:
|
|
"""Board-1-Stand der Vorlage übernehmen und Board 2 auf Start zurücksetzen —
|
|
reset_board_from_stage räumt DB-Spiegel, globale Dateien und Resume-Slots."""
|
|
import board_inventory
|
|
from blocks import _blocks_files
|
|
await database.copy_topic(vorlage, topic)
|
|
tdir = topic_dir(topic)
|
|
tdir.mkdir(parents=True, exist_ok=True)
|
|
for src, dst in ((source_path(vorlage), source_path(topic)),
|
|
(blocks_path(vorlage), blocks_path(topic))):
|
|
if src.exists():
|
|
shutil.copy(src, dst)
|
|
files = _blocks_files(topic)
|
|
files["arbeit"].mkdir(parents=True, exist_ok=True)
|
|
await board_inventory.reset_board_from_stage(topic, "artefacts", "subblocks", files)
|
|
|
|
|
|
async def _aufraeumen(topic: str) -> None:
|
|
"""Topic restlos entfernen (DELETE-/topics-Sequenz aus routes.py)."""
|
|
await database.delete_topic(topic)
|
|
await database.delete_block_data(topic)
|
|
await database.delete_topic_pipeline(topic)
|
|
await database.kanban_reset(topic)
|
|
await database.delete_source(topic)
|
|
await database.delete_guide_content(topic)
|
|
shutil.rmtree(topic_dir(topic), ignore_errors=True)
|
|
shutil.rmtree(qa.QA_DIR / topic, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = [a for a in sys.argv[1:] if a != "--board2"]
|
|
if len(args) != 3:
|
|
raise SystemExit("Nutzung: python3 train_lauf.py <topic> <quelle> <ausgabe.json> [--board2]")
|
|
asyncio.run(trial(args[0], args[1], args[2], board2="--board2" in sys.argv))
|