init
This commit is contained in:
team3
198
backend/pipeline.py
Normal file
198
backend/pipeline.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""Orchestrierung: fünf Ebenen strikt seriell, je Ebene bauen → QA → Repair-Loop
|
||||
bis 100 %/Stillstand/Limit. Resume ist der Normalfall: der Fortschritt liegt
|
||||
komplett in der DB (topics.status + Item-Status), ein neuer Start überspringt
|
||||
Fertiges. Infra-Erschöpfung pausiert den Lauf (fail-closed), Budget stoppt hart."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import agents
|
||||
import artefakte
|
||||
import auto_loop
|
||||
import db
|
||||
import guide
|
||||
import inventar
|
||||
import korpus
|
||||
import ledger
|
||||
import llm
|
||||
import qa
|
||||
import struktur
|
||||
from config import RUN_BUDGET_TOKENS
|
||||
|
||||
log = logging.getLogger("creator2.pipeline")
|
||||
|
||||
EBENEN = [
|
||||
("korpus", korpus, "korpus_fertig"),
|
||||
("inventar", inventar, "inventar_fertig"),
|
||||
("artefakte", artefakte, "artefakte_fertig"),
|
||||
("struktur", struktur, "struktur_fertig"),
|
||||
("guide", guide, "fertig"),
|
||||
]
|
||||
_ORDNUNG = ["neu", "korpus", "korpus_fertig", "inventar_fertig",
|
||||
"artefakte_fertig", "struktur_fertig", "fertig"]
|
||||
_laeufe: dict[str, asyncio.Task] = {}
|
||||
|
||||
|
||||
def _fertig_ab(status: str, marke: str) -> bool:
|
||||
try:
|
||||
return _ORDNUNG.index(status) >= _ORDNUNG.index(marke)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def lauf_starten(topic: str, budget: int | None = None) -> int:
|
||||
if topic in _laeufe and not _laeufe[topic].done():
|
||||
raise RuntimeError("Lauf läuft bereits")
|
||||
agents.abbruch_aufheben(f"{topic}-")
|
||||
git_hash, dirty = ledger.git_stand()
|
||||
run_id = db.insert("runs", topic=topic, status="running", git_hash=git_hash,
|
||||
git_dirty=int(dirty),
|
||||
budget_tokens=budget if budget is not None else RUN_BUDGET_TOKENS)
|
||||
_laeufe[topic] = asyncio.ensure_future(_lauf(topic, run_id))
|
||||
return run_id
|
||||
|
||||
|
||||
def lauf_stoppen(topic: str) -> None:
|
||||
agents.abbrechen(f"{topic}-")
|
||||
task = _laeufe.get(topic)
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
run = db.one("SELECT id FROM runs WHERE topic=? AND status='running'"
|
||||
" ORDER BY id DESC", (topic,))
|
||||
if run:
|
||||
db.update("runs", "id", run["id"], status="stopped", beendet="jetzt")
|
||||
|
||||
|
||||
_ROLLBACK = {"guide": "struktur_fertig", "struktur": "artefakte_fertig",
|
||||
"artefakte": "inventar_fertig", "inventar": "korpus_fertig", "korpus": "neu"}
|
||||
|
||||
|
||||
def ebenen_entfernen(topic: str, ebene: str) -> None:
|
||||
"""Ergebnisse einer Ebene UND aller nachgelagerten verwerfen (rechts hängt an
|
||||
links), Topic-Status entsprechend zurückrollen. Quellen-Snapshots bleiben,
|
||||
außer bei korpus."""
|
||||
if ebene not in _ROLLBACK:
|
||||
raise ValueError(f"unbekannte Ebene: {ebene}")
|
||||
lauf_stoppen(topic)
|
||||
stufe = ["guide", "struktur", "artefakte", "inventar", "korpus"].index(ebene)
|
||||
db.execute("DELETE FROM sections WHERE baustein_id IN"
|
||||
" (SELECT id FROM bausteine WHERE topic=?)", (topic,))
|
||||
db.execute("UPDATE bausteine SET status='neu' WHERE topic=?", (topic,))
|
||||
db.execute("UPDATE kapitel SET intro='' WHERE topic=?", (topic,)) # Intro ist Guide-Text
|
||||
if stufe >= 1: # struktur
|
||||
db.execute("DELETE FROM bausteine WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM kapitel WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM lernziele WHERE topic=?", (topic,))
|
||||
db.execute("UPDATE atome SET ziel_id=NULL, baustein_id=NULL, ord=0 WHERE topic=?",
|
||||
(topic,))
|
||||
if stufe >= 2: # artefakte
|
||||
db.execute("DELETE FROM leitner WHERE artefakt_id IN (SELECT ar.id FROM artefakte ar"
|
||||
" JOIN atome a ON ar.atom_id=a.id WHERE a.topic=?)", (topic,))
|
||||
db.execute("DELETE FROM artefakte WHERE atom_id IN"
|
||||
" (SELECT id FROM atome WHERE topic=?)", (topic,))
|
||||
if stufe >= 3: # inventar
|
||||
db.execute("DELETE FROM anker WHERE atom_id IN (SELECT id FROM atome WHERE topic=?)",
|
||||
(topic,))
|
||||
db.execute("DELETE FROM kanten WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM atome WHERE topic=?", (topic,))
|
||||
db.execute("UPDATE quellen SET status='extrahiert' WHERE topic=?", (topic,))
|
||||
if stufe >= 4: # korpus
|
||||
db.execute("DELETE FROM soll WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM quellen WHERE topic=?", (topic,))
|
||||
import shutil
|
||||
from config import KORPUS_DIR
|
||||
shutil.rmtree(KORPUS_DIR / topic, ignore_errors=True)
|
||||
# Token-/Dauer-Anzeige der entfernten Ebenen nullen: Events bleiben (Ledger ist
|
||||
# append-only), aber die Zählung beginnt hinter dem Reset-Marker neu.
|
||||
letzte = db.one("SELECT MAX(id) AS m FROM events")["m"] or 0
|
||||
resets = db.uj(db.one("SELECT resets FROM topics WHERE name=?", (topic,))["resets"], {})
|
||||
for name in ["guide", "struktur", "artefakte", "inventar", "korpus"][:stufe + 1]:
|
||||
resets[name] = letzte
|
||||
db.update("topics", "name", topic, status=_ROLLBACK[ebene], resets=db.j(resets))
|
||||
|
||||
|
||||
def topic_loeschen(topic: str) -> None:
|
||||
lauf_stoppen(topic)
|
||||
ebenen_entfernen(topic, "korpus")
|
||||
for run in db.query("SELECT id FROM runs WHERE topic=?", (topic,)):
|
||||
db.execute("DELETE FROM events WHERE run_id=?", (run["id"],))
|
||||
db.execute("DELETE FROM befunde WHERE run_id=?", (run["id"],))
|
||||
db.execute("DELETE FROM runs WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM topics WHERE name=?", (topic,))
|
||||
|
||||
|
||||
def voll_reset(topic: str) -> None:
|
||||
"""Alles Generierte verwerfen (auch Atome + Artefakte); Quellen-Snapshots und
|
||||
Lauf-Historie bleiben. Für Neuextraktion nach Extraktions-Regeländerungen."""
|
||||
lauf_stoppen(topic)
|
||||
soll_reset(topic)
|
||||
db.execute("DELETE FROM leitner WHERE artefakt_id IN (SELECT ar.id FROM artefakte ar"
|
||||
" JOIN atome a ON ar.atom_id=a.id WHERE a.topic=?)", (topic,))
|
||||
db.execute("DELETE FROM artefakte WHERE atom_id IN"
|
||||
" (SELECT id FROM atome WHERE topic=?)", (topic,))
|
||||
db.execute("DELETE FROM anker WHERE atom_id IN (SELECT id FROM atome WHERE topic=?)",
|
||||
(topic,))
|
||||
db.execute("DELETE FROM kanten WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM atome WHERE topic=?", (topic,))
|
||||
db.update("topics", "name", topic, status="neu")
|
||||
|
||||
|
||||
def soll_reset(topic: str) -> None:
|
||||
"""Soll/Struktur/Guide verwerfen, Atome + Artefakte behalten. Der nächste Lauf
|
||||
baut das Soll neu; der Extraktions-Guard in inventar verhindert Doppel-Atome."""
|
||||
lauf_stoppen(topic)
|
||||
db.execute("DELETE FROM sections WHERE baustein_id IN"
|
||||
" (SELECT id FROM bausteine WHERE topic=?)", (topic,))
|
||||
db.execute("DELETE FROM bausteine WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM kapitel WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM lernziele WHERE topic=?", (topic,))
|
||||
db.execute("DELETE FROM soll WHERE topic=?", (topic,))
|
||||
db.execute("UPDATE atome SET soll_id=NULL, ziel_id=NULL, baustein_id=NULL, ord=0"
|
||||
" WHERE topic=?", (topic,))
|
||||
db.execute("UPDATE quellen SET status='neu' WHERE topic=?", (topic,))
|
||||
db.update("topics", "name", topic, status="korpus")
|
||||
|
||||
|
||||
async def _lauf(topic: str, run_id: int) -> None:
|
||||
t = db.one("SELECT * FROM topics WHERE name=?", (topic,))
|
||||
ctx = llm.Kontext(run_id, topic, t["provider"])
|
||||
try:
|
||||
for name, modul, marke in EBENEN:
|
||||
status = db.one("SELECT status FROM topics WHERE name=?", (topic,))["status"]
|
||||
if _fertig_ab(status, marke):
|
||||
continue
|
||||
db.update("runs", "id", run_id, ebene=name)
|
||||
await modul.bauen(ctx)
|
||||
erst_note, _ = await qa.messen(ctx, name)
|
||||
|
||||
async def reparieren(n=name, m=modul):
|
||||
offene = db.query("SELECT * FROM befunde WHERE run_id=? AND ebene=?"
|
||||
" AND status='offen'", (run_id, n))
|
||||
bewegt = await m.reparieren(ctx, offene)
|
||||
neue_note, _ = await qa.messen(ctx, n)
|
||||
return neue_note, bewegt
|
||||
|
||||
note, grund = await auto_loop.auto_repair_loop(name, erst_note, reparieren)
|
||||
log.info("%s/%s: Note %.1f (%s)", topic, name, note, grund)
|
||||
db.update("topics", "name", topic, status=marke)
|
||||
flags = db.uj(db.one("SELECT auto FROM topics WHERE name=?", (topic,))["auto"], {})
|
||||
if not flags.get(name, True) and marke != "fertig":
|
||||
db.update("runs", "id", run_id, status="done", beendet="jetzt",
|
||||
grund=f"Auto-Stopp nach {name}")
|
||||
return
|
||||
db.update("runs", "id", run_id, status="done", beendet="jetzt")
|
||||
except llm.LaufPause as e:
|
||||
agents.abbrechen(f"{topic}-") # Nachzügler-Tasks dürfen keine Tokens mehr ziehen
|
||||
db.update("runs", "id", run_id, status="paused", grund=str(e)[:300], beendet="jetzt")
|
||||
log.warning("%s: Lauf pausiert — %s", topic, e)
|
||||
except ledger.BudgetErschoepft as e:
|
||||
agents.abbrechen(f"{topic}-")
|
||||
db.update("runs", "id", run_id, status="budget", grund=str(e)[:300], beendet="jetzt")
|
||||
log.warning("%s: Budget erreicht", topic)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
agents.abbrechen(f"{topic}-")
|
||||
db.update("runs", "id", run_id, status="failed", grund=f"{type(e).__name__}: {e}"[:300],
|
||||
beendet="jetzt")
|
||||
log.exception("%s: Lauf fehlgeschlagen", topic)
|
||||
Reference in New Issue
Block a user