199 lines
9.0 KiB
Python
199 lines
9.0 KiB
Python
"""Orchestrierung: fünf Ebenen strikt seriell, je Ebene bauen → QA → Repair-Loop
|
|
bis 100 %/Stillstand/Limit. Resume ist der Normalfall: der Fortschritt liegt
|
|
komplett in der DB (topics.status + Item-Status), ein neuer Start überspringt
|
|
Fertiges. Infra-Erschöpfung pausiert den Lauf (fail-closed), Budget stoppt hart."""
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
import agents
|
|
import artefakte
|
|
import auto_loop
|
|
import db
|
|
import guide
|
|
import inventar
|
|
import korpus
|
|
import ledger
|
|
import llm
|
|
import qa
|
|
import struktur
|
|
from config import RUN_BUDGET_TOKENS
|
|
|
|
log = logging.getLogger("creator2.pipeline")
|
|
|
|
EBENEN = [
|
|
("korpus", korpus, "korpus_fertig"),
|
|
("inventar", inventar, "inventar_fertig"),
|
|
("artefakte", artefakte, "artefakte_fertig"),
|
|
("struktur", struktur, "struktur_fertig"),
|
|
("guide", guide, "fertig"),
|
|
]
|
|
_ORDNUNG = ["neu", "korpus", "korpus_fertig", "inventar_fertig",
|
|
"artefakte_fertig", "struktur_fertig", "fertig"]
|
|
_laeufe: dict[str, asyncio.Task] = {}
|
|
|
|
|
|
def _fertig_ab(status: str, marke: str) -> bool:
|
|
try:
|
|
return _ORDNUNG.index(status) >= _ORDNUNG.index(marke)
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def lauf_starten(topic: str, budget: int | None = None) -> int:
|
|
if topic in _laeufe and not _laeufe[topic].done():
|
|
raise RuntimeError("Lauf läuft bereits")
|
|
agents.abbruch_aufheben(f"{topic}-")
|
|
git_hash, dirty = ledger.git_stand()
|
|
run_id = db.insert("runs", topic=topic, status="running", git_hash=git_hash,
|
|
git_dirty=int(dirty),
|
|
budget_tokens=budget if budget is not None else RUN_BUDGET_TOKENS)
|
|
_laeufe[topic] = asyncio.ensure_future(_lauf(topic, run_id))
|
|
return run_id
|
|
|
|
|
|
def lauf_stoppen(topic: str) -> None:
|
|
agents.abbrechen(f"{topic}-")
|
|
task = _laeufe.get(topic)
|
|
if task and not task.done():
|
|
task.cancel()
|
|
run = db.one("SELECT id FROM runs WHERE topic=? AND status='running'"
|
|
" ORDER BY id DESC", (topic,))
|
|
if run:
|
|
db.update("runs", "id", run["id"], status="stopped", beendet="jetzt")
|
|
|
|
|
|
_ROLLBACK = {"guide": "struktur_fertig", "struktur": "artefakte_fertig",
|
|
"artefakte": "inventar_fertig", "inventar": "korpus_fertig", "korpus": "neu"}
|
|
|
|
|
|
def ebenen_entfernen(topic: str, ebene: str) -> None:
|
|
"""Ergebnisse einer Ebene UND aller nachgelagerten verwerfen (rechts hängt an
|
|
links), Topic-Status entsprechend zurückrollen. Quellen-Snapshots bleiben,
|
|
außer bei korpus."""
|
|
if ebene not in _ROLLBACK:
|
|
raise ValueError(f"unbekannte Ebene: {ebene}")
|
|
lauf_stoppen(topic)
|
|
stufe = ["guide", "struktur", "artefakte", "inventar", "korpus"].index(ebene)
|
|
db.execute("DELETE FROM sections WHERE baustein_id IN"
|
|
" (SELECT id FROM bausteine WHERE topic=?)", (topic,))
|
|
db.execute("UPDATE bausteine SET status='neu' WHERE topic=?", (topic,))
|
|
db.execute("UPDATE kapitel SET intro='' WHERE topic=?", (topic,)) # Intro ist Guide-Text
|
|
if stufe >= 1: # struktur
|
|
db.execute("DELETE FROM bausteine WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM kapitel WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM lernziele WHERE topic=?", (topic,))
|
|
db.execute("UPDATE atome SET ziel_id=NULL, baustein_id=NULL, ord=0 WHERE topic=?",
|
|
(topic,))
|
|
if stufe >= 2: # artefakte
|
|
db.execute("DELETE FROM leitner WHERE artefakt_id IN (SELECT ar.id FROM artefakte ar"
|
|
" JOIN atome a ON ar.atom_id=a.id WHERE a.topic=?)", (topic,))
|
|
db.execute("DELETE FROM artefakte WHERE atom_id IN"
|
|
" (SELECT id FROM atome WHERE topic=?)", (topic,))
|
|
if stufe >= 3: # inventar
|
|
db.execute("DELETE FROM anker WHERE atom_id IN (SELECT id FROM atome WHERE topic=?)",
|
|
(topic,))
|
|
db.execute("DELETE FROM kanten WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM atome WHERE topic=?", (topic,))
|
|
db.execute("UPDATE quellen SET status='extrahiert' WHERE topic=?", (topic,))
|
|
if stufe >= 4: # korpus
|
|
db.execute("DELETE FROM soll WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM quellen WHERE topic=?", (topic,))
|
|
import shutil
|
|
from config import KORPUS_DIR
|
|
shutil.rmtree(KORPUS_DIR / topic, ignore_errors=True)
|
|
# Token-/Dauer-Anzeige der entfernten Ebenen nullen: Events bleiben (Ledger ist
|
|
# append-only), aber die Zählung beginnt hinter dem Reset-Marker neu.
|
|
letzte = db.one("SELECT MAX(id) AS m FROM events")["m"] or 0
|
|
resets = db.uj(db.one("SELECT resets FROM topics WHERE name=?", (topic,))["resets"], {})
|
|
for name in ["guide", "struktur", "artefakte", "inventar", "korpus"][:stufe + 1]:
|
|
resets[name] = letzte
|
|
db.update("topics", "name", topic, status=_ROLLBACK[ebene], resets=db.j(resets))
|
|
|
|
|
|
def topic_loeschen(topic: str) -> None:
|
|
lauf_stoppen(topic)
|
|
ebenen_entfernen(topic, "korpus")
|
|
for run in db.query("SELECT id FROM runs WHERE topic=?", (topic,)):
|
|
db.execute("DELETE FROM events WHERE run_id=?", (run["id"],))
|
|
db.execute("DELETE FROM befunde WHERE run_id=?", (run["id"],))
|
|
db.execute("DELETE FROM runs WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM topics WHERE name=?", (topic,))
|
|
|
|
|
|
def voll_reset(topic: str) -> None:
|
|
"""Alles Generierte verwerfen (auch Atome + Artefakte); Quellen-Snapshots und
|
|
Lauf-Historie bleiben. Für Neuextraktion nach Extraktions-Regeländerungen."""
|
|
lauf_stoppen(topic)
|
|
soll_reset(topic)
|
|
db.execute("DELETE FROM leitner WHERE artefakt_id IN (SELECT ar.id FROM artefakte ar"
|
|
" JOIN atome a ON ar.atom_id=a.id WHERE a.topic=?)", (topic,))
|
|
db.execute("DELETE FROM artefakte WHERE atom_id IN"
|
|
" (SELECT id FROM atome WHERE topic=?)", (topic,))
|
|
db.execute("DELETE FROM anker WHERE atom_id IN (SELECT id FROM atome WHERE topic=?)",
|
|
(topic,))
|
|
db.execute("DELETE FROM kanten WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM atome WHERE topic=?", (topic,))
|
|
db.update("topics", "name", topic, status="neu")
|
|
|
|
|
|
def soll_reset(topic: str) -> None:
|
|
"""Soll/Struktur/Guide verwerfen, Atome + Artefakte behalten. Der nächste Lauf
|
|
baut das Soll neu; der Extraktions-Guard in inventar verhindert Doppel-Atome."""
|
|
lauf_stoppen(topic)
|
|
db.execute("DELETE FROM sections WHERE baustein_id IN"
|
|
" (SELECT id FROM bausteine WHERE topic=?)", (topic,))
|
|
db.execute("DELETE FROM bausteine WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM kapitel WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM lernziele WHERE topic=?", (topic,))
|
|
db.execute("DELETE FROM soll WHERE topic=?", (topic,))
|
|
db.execute("UPDATE atome SET soll_id=NULL, ziel_id=NULL, baustein_id=NULL, ord=0"
|
|
" WHERE topic=?", (topic,))
|
|
db.execute("UPDATE quellen SET status='neu' WHERE topic=?", (topic,))
|
|
db.update("topics", "name", topic, status="korpus")
|
|
|
|
|
|
async def _lauf(topic: str, run_id: int) -> None:
|
|
t = db.one("SELECT * FROM topics WHERE name=?", (topic,))
|
|
ctx = llm.Kontext(run_id, topic, t["provider"])
|
|
try:
|
|
for name, modul, marke in EBENEN:
|
|
status = db.one("SELECT status FROM topics WHERE name=?", (topic,))["status"]
|
|
if _fertig_ab(status, marke):
|
|
continue
|
|
db.update("runs", "id", run_id, ebene=name)
|
|
await modul.bauen(ctx)
|
|
erst_note, _ = await qa.messen(ctx, name)
|
|
|
|
async def reparieren(n=name, m=modul):
|
|
offene = db.query("SELECT * FROM befunde WHERE run_id=? AND ebene=?"
|
|
" AND status='offen'", (run_id, n))
|
|
bewegt = await m.reparieren(ctx, offene)
|
|
neue_note, _ = await qa.messen(ctx, n)
|
|
return neue_note, bewegt
|
|
|
|
note, grund = await auto_loop.auto_repair_loop(name, erst_note, reparieren)
|
|
log.info("%s/%s: Note %.1f (%s)", topic, name, note, grund)
|
|
db.update("topics", "name", topic, status=marke)
|
|
flags = db.uj(db.one("SELECT auto FROM topics WHERE name=?", (topic,))["auto"], {})
|
|
if not flags.get(name, True) and marke != "fertig":
|
|
db.update("runs", "id", run_id, status="done", beendet="jetzt",
|
|
grund=f"Auto-Stopp nach {name}")
|
|
return
|
|
db.update("runs", "id", run_id, status="done", beendet="jetzt")
|
|
except llm.LaufPause as e:
|
|
agents.abbrechen(f"{topic}-") # Nachzügler-Tasks dürfen keine Tokens mehr ziehen
|
|
db.update("runs", "id", run_id, status="paused", grund=str(e)[:300], beendet="jetzt")
|
|
log.warning("%s: Lauf pausiert — %s", topic, e)
|
|
except ledger.BudgetErschoepft as e:
|
|
agents.abbrechen(f"{topic}-")
|
|
db.update("runs", "id", run_id, status="budget", grund=str(e)[:300], beendet="jetzt")
|
|
log.warning("%s: Budget erreicht", topic)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
agents.abbrechen(f"{topic}-")
|
|
db.update("runs", "id", run_id, status="failed", grund=f"{type(e).__name__}: {e}"[:300],
|
|
beendet="jetzt")
|
|
log.exception("%s: Lauf fehlgeschlagen", topic)
|