marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-06-30 00:14:18 +02:00
parent 3e3559aa8f
commit c794fcaccf
152 changed files with 9485 additions and 9583 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
backend/pipeline.py
View File

@@ -1,8 +1,8 @@
"""Pipeline-Grundbausteine: Agent-Races (mit Grace), Single-Slot, Schemata, Prompts, Guide-Status.
"""Pipeline building blocks: agent races (with grace), single-slot, schemas, prompts, guide status.
Hält den mutablen Pipeline-Zustand (Generierungs-Semaphore, Cancel-Set).
Zugriff auf das Cancel-Set NUR über die Funktionen hier — kopierte Referenzen
in anderen Modulen würden bei einem Re-Assign auseinanderlaufen.
Holds the mutable pipeline state (generation semaphore, cancel set).
Access the cancel set ONLY through the functions here — copied references
in other modules would diverge on a re-assign.
"""
import asyncio
@@ -15,7 +15,7 @@ from typing import Callable
from agents import run_agent, kill_process, cancel_scope, clear_scope
from config import MAX_CONCURRENT_GENERATIONS, TEMPLATES_DIR, TIMEOUTS
from database import update_guide
from jsonio import read_json_file as _json_datei
from jsonio import read_json_file as _json_file
from textkit import _STUFEN
log = logging.getLogger("creator.pipeline")
@@ -26,10 +26,10 @@ _cancelled: set[str] = set()
async def cancel_guide(guide_id: str) -> bool:
_cancelled.add(guide_id)
cancel_scope(f"{guide_id}-") # wartende Agenten bailen vorm Spawn
kill_process(guide_id) # laufende Subprozesse killen
cancel_scope(f"{guide_id}-") # waiting agents bail before spawn
kill_process(guide_id) # kill running subprocesses
now = datetime.now(timezone.utc).isoformat()
await update_guide(guide_id, status="error", progress=None, error_msg="Abgebrochen — Fortschritt bleibt erhalten", updated_at=now)
await update_guide(guide_id, status="error", progress=None, error_msg="Cancelled — progress is preserved", updated_at=now)
return True
@@ -39,7 +39,7 @@ def is_guide_cancelled(guide_id: str) -> bool:
def clear_guide_cancelled(guide_id: str) -> None:
_cancelled.discard(guide_id)
clear_scope(f"{guide_id}-") # Scope leeren → Neustart blockiert nicht
clear_scope(f"{guide_id}-") # clear scope → restart not blocked
async def _set_progress(guide_id: str, progress: str) -> None:
@@ -63,7 +63,7 @@ def _prompt(name: str, **kwargs) -> str:
def _extra(instructions: str) -> str:
return f"\n\nZUSÄTZLICHE ANWEISUNGEN VOM NUTZER:\n{instructions}\n" if instructions else ""
return f"\n\nADDITIONAL INSTRUCTIONS FROM THE USER:\n{instructions}\n" if instructions else ""
def _log(topic: str, msg: str) -> None:
@@ -76,8 +76,8 @@ def _claude_error(label: str, returncode: int, stdout: str, stderr: str) -> str:
return f"{label}: {stderr[:1000]}"
tail = (stdout or "").strip()[-500:]
if tail:
return f"{label} (exit {returncode}, stderr leer): …{tail}"
return f"{label} (exit {returncode}, ohne Ausgabe)"
return f"{label} (exit {returncode}, stderr empty): …{tail}"
return f"{label} (exit {returncode}, no output)"
def _gather_error(label: str, results: list) -> str:
@@ -87,7 +87,7 @@ def _gather_error(label: str, results: list) -> str:
returncode, stdout, stderr = r
if returncode != 0:
return _claude_error(label, returncode, stdout, stderr)
return f"{label}: kein verwertbares Ergebnis"
return f"{label}: no usable result"
def _timeout(step: str, n: int = 0) -> int:
@@ -95,21 +95,21 @@ def _timeout(step: str, n: int = 0) -> int:
return base + per * n
def _probleme_schema(data):
"""{"ok": true} → [] · {"probleme": [str]} → Liste · sonst None."""
def _problems_schema(data):
"""{"ok": true} → [] · {"problems": [str]} → list · else None."""
if not isinstance(data, dict):
return None
if data.get("ok") is True:
return []
p = data.get("probleme")
p = data.get("problems")
if not isinstance(p, list) or not p:
return None
out = [str(x).strip() for x in p if str(x).strip()]
return out or None
def _str_liste(val) -> list[str] | None:
"""Liste nicht-leerer Strings → gestrippte Liste (leer erlaubt) · sonst None."""
def _str_list(val) -> list[str] | None:
"""List of non-empty strings → stripped list (empty allowed) · else None."""
if not isinstance(val, list) or not all(isinstance(x, str) for x in val):
return None
out = [x.strip() for x in val]
@@ -119,110 +119,67 @@ def _str_liste(val) -> list[str] | None:
def _runde_schema(data, final: bool = False):
"""{"aufnehmen": [str], "rest": [str]} → (aufnehmen, rest) · sonst None.
"""{"keep": [str], "rest": [str]} → (include, rest) · else None.
final=True: letzte Klärungs-Runde — ein nicht-leerer Rest ist ungültig.
final=True: last clarification round — a non-empty rest is invalid.
"""
if not isinstance(data, dict):
return None
aufnehmen = _str_liste(data.get("aufnehmen"))
rest = _str_liste(data.get("rest"))
if aufnehmen is None or rest is None or (final and rest):
include = _str_list(data.get("keep"))
rest = _str_list(data.get("rest"))
if include is None or rest is None or (final and rest):
return None
return aufnehmen, rest
return include, rest
def _stufen_schema(data, ids: set[int] | None = None):
"""{"stufen": {"1": "anfaenger", …}} → {id: stufe} · sonst None.
_RELEVANCE = ("relevant", "peripheral")
_YESNO = ("ja", "nein")
Stufe ∈ {anfaenger, fortgeschritten, experte} (alte Werte abwärtskompatibel). Sind `ids`
gegeben, müssen mindestens diese abgedeckt sein (Extras erlaubt); der Aufrufer filtert auf `ids`.
"""
if not isinstance(data, dict) or not isinstance(data.get("stufen"), dict) or not data["stufen"]:
return None
out: dict[int, str] = {}
for k, v in data["stufen"].items():
try:
num = int(k)
except (ValueError, TypeError):
def _enum_map_schema(key: str, allowed):
"""Factory for `{"<key>": {"1": value, …}}` → `{id: value}` parsers; value ∈ `allowed`
(casefolded). If `ids` are given, at least these must be covered (extras allowed). None
on any invalid id/value or wrong shape. The caller filters the result to `ids`."""
def parse(data, ids: set[int] | None = None):
if not isinstance(data, dict) or not isinstance(data.get(key), dict) or not data[key]:
return None
stufe = str(v).strip().casefold()
if stufe not in _STUFEN:
out: dict[int, str] = {}
for k, v in data[key].items():
try:
num = int(k)
except (ValueError, TypeError):
return None
value = str(v).strip().casefold()
if value not in allowed:
return None
out[num] = value
if ids is not None and not ids <= set(out):
return None
out[num] = stufe
if ids is not None and not ids <= set(out):
return None
return out
return out
return parse
_RELEVANZ = ("relevant", "rand")
def _relevanz_schema(data, ids: set[int] | None = None):
"""{"relevanz": {"1": "relevant", …}} → {id: relevanz} · sonst None.
Relevanz ∈ {relevant, rand} (binär). Wie `_stufen_schema`: sind `ids` gegeben,
müssen mindestens diese abgedeckt sein (Extras erlaubt).
"""
if not isinstance(data, dict) or not isinstance(data.get("relevanz"), dict) or not data["relevanz"]:
return None
out: dict[int, str] = {}
for k, v in data["relevanz"].items():
try:
num = int(k)
except (ValueError, TypeError):
return None
wert = str(v).strip().casefold()
if wert not in _RELEVANZ:
return None
out[num] = wert
if ids is not None and not ids <= set(out):
return None
return out
_JANEIN = ("ja", "nein")
def _janein_schema(data, ids: set[int] | None = None):
"""{"relevant": {"1": "ja", …}} → {id: ja/nein} · sonst None.
Binäres ja/nein — das Themen-Relevanz-Gate der Sichtung. Wie `_relevanz_schema`:
sind `ids` gegeben, müssen mindestens diese abgedeckt sein (Extras erlaubt).
"""
if not isinstance(data, dict) or not isinstance(data.get("relevant"), dict) or not data["relevant"]:
return None
out: dict[int, str] = {}
for k, v in data["relevant"].items():
try:
num = int(k)
except (ValueError, TypeError):
return None
wert = str(v).strip().casefold()
if wert not in _JANEIN:
return None
out[num] = wert
if ids is not None and not ids <= set(out):
return None
return out
_levels_schema = _enum_map_schema("levels", _STUFEN) # level ∈ beginner/advanced/expert
_relevance_schema = _enum_map_schema("relevance", _RELEVANCE) # relevance ∈ relevant/peripheral
_yesno_schema = _enum_map_schema("relevant", _YESNO) # triage gate ∈ ja/nein
_MAX_RESTARTS = 2
async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: int, provider: str, on_update=None, cancelled=None, *, grace: int | None = None) -> list | None:
"""Startet alle Slots parallel und sammelt `quorum` gültige Ergebnisse.
"""Starts all slots in parallel and collects `quorum` valid results.
Slot-Spec: {key, prompt, role, capabilities, payload}. `payload(result)`
prüft die Gültigkeit und liefert das Slot-Ergebnis oder None.
Fehler/Timeout/ungültig → Slot-Neustart (max. _MAX_RESTARTS). Sobald das
Quorum steht, werden die übrigen Agenten gekillt. None = Quorum verfehlt.
`cancelled()` → True bricht ab (keine Restarts, Rückgabe None).
Slot spec: {key, prompt, role, capabilities, payload}. `payload(result)`
checks validity and returns the slot result or None.
Error/timeout/invalid → slot restart (max. _MAX_RESTARTS). As soon as the
quorum stands, the remaining agents are killed. None = quorum missed.
`cancelled()` → True aborts (no restarts, returns None).
Mit `grace` wird `quorum` zum Minimum: Das erste gültige Ergebnis startet
einen Timer von `grace` Sekunden. Nach dessen Ablauf werden laufende
Agenten nur gekillt, wenn das Minimum steht — sonst läuft das Race samt
Restarts weiter, bis es steht. Rückgabe: `quorum` bis `len(slots)` Ergebnisse.
With `grace`, `quorum` becomes the minimum: the first valid result starts
a timer of `grace` seconds. After it expires, running agents are only
killed if the minimum stands — otherwise the race, including restarts,
keeps running until it stands. Returns: `quorum` to `len(slots)` results.
"""
attempts = {i: 0 for i in range(len(slots))}
tasks: dict[asyncio.Task, int] = {}
@@ -247,7 +204,7 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout:
return None
if deadline is not None and len(results) >= quorum and loop.time() >= deadline:
return results
# Grace gesetzt und Minimum erreicht → nur bis zum Deadline-Rest warten
# Grace set and minimum reached → only wait for the remaining deadline
wait_timeout = None
if deadline is not None and len(results) >= quorum:
wait_timeout = max(0.0, deadline - loop.time())
@@ -260,13 +217,13 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout:
try:
result = task.result()
if result[0] != 0:
err = _claude_error("Fehler", *result)
err = _claude_error("Error", *result)
else:
payload = slots[i]["payload"](result)
if payload is None:
err = "Ergebnis ungültig/nicht parsebar"
err = "result invalid/not parseable"
except asyncio.TimeoutError:
err = f"Timeout nach {timeout}s"
err = f"Timeout after {timeout}s"
except Exception as e:
err = f"{type(e).__name__}: {e}"
@@ -274,23 +231,23 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout:
results.append(payload)
if grace is not None and deadline is None:
deadline = loop.time() + grace
_log(topic, f"{label}: erstes Ergebnis — Grace {grace}s läuft")
_log(topic, f"{label}: first result — grace {grace}s running")
if on_update:
on_update(len(results))
if len(results) >= quorum and (grace is None or loop.time() >= deadline):
return results
continue
_log(topic, f"{label} {i + 1} (Versuch {attempts[i] + 1}): {err}")
_log(topic, f"{label} {i + 1} (attempt {attempts[i] + 1}): {err}")
attempts[i] += 1
# Steht das Minimum schon, sind Restarts sinnlos — der Neustart
# würde am Grace-Ende ohnehin gekillt.
satt = grace is not None and len(results) >= quorum
if attempts[i] <= _MAX_RESTARTS and not satt and not (cancelled and cancelled()):
# If the minimum already stands, restarts are pointless — the restart
# would be killed at the grace end anyway.
enough = grace is not None and len(results) >= quorum
if attempts[i] <= _MAX_RESTARTS and not enough and not (cancelled and cancelled()):
spawn(i)
if len(results) >= quorum: # alle Slots durch, Minimum steht (nur mit grace erreichbar)
if len(results) >= quorum: # all slots done, minimum stands (only reachable with grace)
return results
_log(topic, f"{label}: Quorum {quorum} nicht erreicht ({len(results)} gültig)")
_log(topic, f"{label}: quorum {quorum} not reached ({len(results)} valid)")
return None
finally:
for task, i in tasks.items():
@@ -302,14 +259,14 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout:
@dataclass
class GenContext:
"""Durchgereichte Pipeline-Parameter — erspart lange Argument-Signaturen."""
"""Pipeline parameters passed through — saves long argument signatures."""
topic: str
provider: str
is_cancelled: Callable[[], bool]
guide_id: str | None = None
# Ergebnis-Status von run_single_slot
# Result status of run_single_slot
OK, CANCELLED, FAILED = "ok", "cancelled", "failed"
@@ -317,9 +274,9 @@ async def run_single_slot(
ctx: GenContext, label: str, *,
key: str, prompt: str, role: str, capabilities: str, payload, timeout: int,
) -> tuple[str, object]:
"""Ein Agent, ein gültiges Ergebnis (Race mit Quorum 1).
"""One agent, one valid result (race with quorum 1).
→ (OK, wert) | (CANCELLED, None) | (FAILED, None)
→ (OK, value) | (CANCELLED, None) | (FAILED, None)
"""
slots = [{"key": key, "prompt": prompt, "role": role, "capabilities": capabilities, "payload": payload}]
res = await _race(ctx.topic, label, slots, 1, timeout, ctx.provider, cancelled=ctx.is_cancelled)
@@ -330,9 +287,9 @@ async def run_single_slot(
return OK, res[0]
async def _gather_fortschritt(coros, total, melde, start=0):
"""Läuft `coros` nebenläufig und meldet Live-Fortschritt: `await melde(fertig, total)`
nach jedem Abschluss (und einmal initial). Ergebnisse in Reihenfolge, return_exceptions=True."""
async def _gather_progress(coros, total, report, start=0):
"""Runs `coros` concurrently and reports live progress: `await report(done, total)`
after each completion (and once initially). Results in order, return_exceptions=True."""
done = start
async def wrap(c):
@@ -341,9 +298,7 @@ async def _gather_fortschritt(coros, total, melde, start=0):
return await c
finally:
done += 1
await melde(done, total)
await report(done, total)
await melde(done, total)
await report(done, total)
return await asyncio.gather(*[wrap(c) for c in coros], return_exceptions=True)