init
This commit is contained in:
team3
0
backend/__init__.py
Normal file
0
backend/__init__.py
Normal file
73
backend/belege.py
Normal file
73
backend/belege.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""Beleg-Fenster um Anker (−300/+1200): Faktenbasis für Writer und Prüfer.
|
||||
Überlappende Fenster derselben Quelle werden gemerged (creator2-Muster).
|
||||
Außerdem: Facetten-Abdeckung — jede Soll-BELEG-Stelle braucht ein Atom in
|
||||
der Nähe (das feine 100-%-Maß, Gutachten-Lücke „Namensherkunft")."""
|
||||
from . import config, db, laden, textkit
|
||||
|
||||
|
||||
def beleg_stellen(topic: str) -> list[dict]:
|
||||
"""Alle auffindbaren Beleg-Stellen bestätigter Soll-Punkte."""
|
||||
stellen = []
|
||||
for s in db.query("SELECT id, belege FROM soll WHERE topic=? AND "
|
||||
"status='bestaetigt' AND freispruch=''", topic):
|
||||
for beleg in db.uj(s["belege"], []):
|
||||
quelle = db.one("SELECT * FROM quellen WHERE id=? AND "
|
||||
"status='geladen'", beleg["quelle"])
|
||||
if not quelle:
|
||||
continue
|
||||
try:
|
||||
text = laden.snapshot_lesen(quelle)
|
||||
except OSError:
|
||||
continue
|
||||
pos = textkit.finde_zitat(text, beleg["zitat"])
|
||||
if pos:
|
||||
stellen.append({"soll_id": s["id"], "quelle_id": quelle["id"],
|
||||
"start": pos[0], "ende": pos[1],
|
||||
"zitat": beleg["zitat"][:120]})
|
||||
return stellen
|
||||
|
||||
|
||||
def unbedeckte_belege(topic: str) -> list[dict]:
|
||||
"""Beleg-Stellen OHNE aktives Atom im ±EXTRAKT_FENSTER derselben Quelle."""
|
||||
anker = {}
|
||||
for a in db.query(
|
||||
"SELECT an.quelle_id q, an.start s FROM anker an JOIN atome a ON "
|
||||
"a.id=an.atom_id WHERE a.topic=? AND a.status='aktiv' AND "
|
||||
"an.start>=0", topic):
|
||||
anker.setdefault(a["q"], []).append(a["s"])
|
||||
return [st for st in beleg_stellen(topic)
|
||||
if not any(abs(s - st["start"]) <= config.EXTRAKT_FENSTER
|
||||
for s in anker.get(st["quelle_id"], []))]
|
||||
|
||||
|
||||
def abdeckung_prozent(topic: str) -> float | None:
|
||||
stellen = beleg_stellen(topic)
|
||||
if not stellen:
|
||||
return None
|
||||
frei = len(unbedeckte_belege(topic))
|
||||
return round((len(stellen) - frei) / len(stellen) * 100, 1)
|
||||
|
||||
|
||||
def fenster_fuer_atome(atom_ids: list[int]) -> str:
|
||||
spans: dict[int, list[list[int]]] = {}
|
||||
for aid in atom_ids:
|
||||
for a in db.query("SELECT * FROM anker WHERE atom_id=? AND start>=0", aid):
|
||||
von = max(0, a["start"] - config.BELEG_FENSTER_VOR)
|
||||
bis = a["ende"] + config.BELEG_FENSTER_NACH
|
||||
spans.setdefault(a["quelle_id"], []).append([von, bis])
|
||||
teile = []
|
||||
for qid, fenster in spans.items():
|
||||
quelle = db.one("SELECT * FROM quellen WHERE id=?", qid)
|
||||
if not quelle or quelle["status"] != "geladen":
|
||||
continue
|
||||
text = laden.snapshot_lesen(quelle)
|
||||
fenster.sort()
|
||||
gemerged = [fenster[0]]
|
||||
for von, bis in fenster[1:]:
|
||||
if von <= gemerged[-1][1]:
|
||||
gemerged[-1][1] = max(gemerged[-1][1], bis)
|
||||
else:
|
||||
gemerged.append([von, bis])
|
||||
for von, bis in gemerged:
|
||||
teile.append(f"[Quelle: {quelle['titel'][:60]}]\n{text[von:bis]}")
|
||||
return "\n\n---\n\n".join(teile)
|
||||
150
backend/config.py
Normal file
150
backend/config.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""Konstanten + .env-Loader. Regel: .env-Datei gewinnt über geerbtes Env
|
||||
(creator2-Lektion 12: --reload-Master pinnte sonst stale Werte)."""
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _load_env() -> None:
|
||||
datei = ROOT / ".env"
|
||||
if not datei.exists():
|
||||
return
|
||||
for zeile in datei.read_text().splitlines():
|
||||
zeile = zeile.strip()
|
||||
if not zeile or zeile.startswith("#") or "=" not in zeile:
|
||||
continue
|
||||
k, v = zeile.split("=", 1)
|
||||
v = v.split("#", 1)[0].strip()
|
||||
if v:
|
||||
os.environ[k.strip()] = v
|
||||
|
||||
|
||||
_load_env()
|
||||
|
||||
FAKE = bool(os.getenv("CREATOR_FAKE"))
|
||||
LLM_LOG = bool(os.getenv("LLM_LOG")) # 1 = Prompt+Antwort je Call in call_texte
|
||||
DB_PFAD = Path(os.getenv("CREATOR_DB") or ROOT / "storage" / "creator3.db")
|
||||
KORPUS_DIR = ROOT / "storage" / "korpus"
|
||||
SKILLS_DIR = ROOT / "skills"
|
||||
PIPELINE_YAML = ROOT / "pipeline.yaml"
|
||||
|
||||
TOPIC_NAME_RE = re.compile(r"^[a-zA-Z0-9äöüÄÖÜß][a-zA-Z0-9äöüÄÖÜß _-]{0,63}$")
|
||||
|
||||
# ── LLM / MiniMax (R3/R9) — alles M3; M2.x kann Thinking nicht abschalten ──
|
||||
LLM_ENDPOINT = "https://api.minimax.io/anthropic/v1/messages"
|
||||
ROLLEN = {
|
||||
# thinking IMMER explizit ("enabled" gibt 400; Default widersprüchlich dokumentiert)
|
||||
"extraktion": dict(model="MiniMax-M3", thinking="disabled", temperature=0.2),
|
||||
"judge": dict(model="MiniMax-M3", thinking="disabled", temperature=0.1),
|
||||
"writer": dict(model="MiniMax-M3", thinking="adaptive", temperature=0.7),
|
||||
}
|
||||
LLM_MAX_TOKENS = 32_000
|
||||
MAX_PARALLEL_LLM = int(os.getenv("MAX_PARALLEL_LLM", "12")) # M3: 200 RPM → 8-16 sicher
|
||||
TAKT_SEKUNDEN = float(os.getenv("TAKT_SEKUNDEN", "3")) # 1 Call-Start je Takt; 0 = Breiten-Drossel
|
||||
BUENDEL = {"lang": 1, "mittel": 3, "kurz": 10} # Tasks je Call nach Knoten-Klasse
|
||||
INFRA_MAX_RETRIES = 3
|
||||
INFRA_BACKOFF_BASE = 8.0 # → 8/16/32 s, mit Jitter (Retry-After unzuverlässig)
|
||||
INHALT_MAX_RESTARTS = 2
|
||||
DROSSEL_MIN_BREITE = 4 # 429/1002/1041/2045: Breite halbieren, 20 Erfolge = +1
|
||||
DROSSEL_ERFOLGE_JE_PLUS = 20
|
||||
HEDGE_NACH_S = int(os.getenv("HEDGE_NACH_S", "90")) # Zwilling nach max(90, timeout/2); 0 = aus
|
||||
RUN_BUDGET_TOKENS = int(os.getenv("RUN_BUDGET_TOKENS", "10000000")) # DEFAULT AN; 0 = aus
|
||||
PREIS_IN, PREIS_OUT = 0.60, 2.40 # $/1M konservativ (R9), real messen
|
||||
|
||||
# Timeouts je Knoten: (basis_s, pro_item_s) — creator2-Lektion 14: nie fix;
|
||||
# pro_item skaliert mit der Bündel-Größe (n in llm.call)
|
||||
TIMEOUTS = {
|
||||
"recherche_plan": (60, 20), "soll_extraktion": (180, 60), "soll_konsens": (240, 2),
|
||||
"atom_extraktion": (240, 60), "anker_fix": (180, 4), "dedup": (120, 8),
|
||||
"soll_zuordnung": (120, 3), "luecke": (180, 45), "lernziele": (120, 3),
|
||||
"bausteine": (120, 3), "ordnung": (180, 3), "writer": (420, 0),
|
||||
"kapitel_intro": (90, 0), "llm_pruefung": (240, 60), "fix": (300, 0),
|
||||
"redundanz": (120, 8), "klaerung": (180, 0),
|
||||
}
|
||||
TIMEOUT_DEFAULT = (180, 0)
|
||||
|
||||
# ── Suche (R2) ──
|
||||
SUCHE_TIMEOUT_S = 10
|
||||
SUCHE_RETRIES = 1
|
||||
CIRCUIT_BREAKER_N = 3
|
||||
WIKI_MAX_PARALLEL = 3
|
||||
DDGS_PAUSE_S = 1.5
|
||||
SUCHE_MAX_TREFFER = 8
|
||||
|
||||
# ── Laden (R6) ──
|
||||
FETCH_CONNECT_S = 10
|
||||
FETCH_READ_S = 30
|
||||
DOMAIN_RATE_S = 1.0
|
||||
SNAPSHOT_MIN_ZEICHEN = 500
|
||||
|
||||
# ── Korpus ──
|
||||
RECHERCHE_LENSES = ("ueberblick", "tutorial", "referenz", "praxis") # generisch!
|
||||
RECHERCHE_RUNDEN_MAX = int(os.getenv("RECHERCHE_RUNDEN_MAX", "4"))
|
||||
QUERIES_JE_LENS = 3
|
||||
SOLL_CHUNK_CHARS = 40_000
|
||||
SOLL_MIN_BELEGE = 2 # ≥2 unabhängige Quellen je bestätigtem Punkt
|
||||
SOLL_PUNKTE_MIN = int(os.getenv("SOLL_PUNKTE_MIN", "5"))
|
||||
SOLL_BAND_FAKTOR = 1.0 # Ziel ≈ √Kandidaten (creator2-Band — Lektion 90)
|
||||
SOLL_BAND_TOLERANZ = 1.5 # mehr als Ziel×1.5 bestätigt → nochmal falten
|
||||
QUELLEN_MAX = int(os.getenv("QUELLEN_MAX", "24")) # harter Korpus-Deckel
|
||||
EXTRAKT_FENSTER = 4_000 # Atome nur ±Fenster um Soll-Belege (belegnah)
|
||||
|
||||
# ── Inventar ──
|
||||
ABSCHNITT_CHARS = 6_000 # 12k riss das 32k-Output-Cap (creator2 empirisch)
|
||||
READER_JE_ABSCHNITT = 2 # Konsens entsteht über Dedup, nicht Union
|
||||
CHUNK_HALBIER_TIEFE = 2
|
||||
ANKER_OVERLAP_MERGE = 0.5
|
||||
FUZZY_MIN_ZEICHEN = 40
|
||||
FUZZY_FEHLERQUOTE = 0.08
|
||||
WORT_MIN = 8
|
||||
WORT_FEHLERQUOTE = 0.25
|
||||
ANKER_FIX_BATCH = 20
|
||||
MERGE_PANEL = 2 # Merge nur einstimmig (Falsch-Merge >> Dublette)
|
||||
DEDUP_JACCARD = 0.3 # gemessener Paraphrasen-Floor
|
||||
DEDUP_TITEL_FUZZY = 85 # nur bei ≥2 Inhaltstoken (Subset-100-Falle)
|
||||
DEDUP_PAARE_PRO_CALL = 10
|
||||
DEDUP_KANDIDATEN_CAP = 400
|
||||
VERDICHTUNG_ZIEL = 8 # weiches Ziel Atome je Soll-Punkt (kein Cap!)
|
||||
LUECKEN_RUNDEN_MAX = 2
|
||||
WEB_NACHRECHERCHE_RUNDEN = 1
|
||||
LUECKE_FENSTER = 6_000
|
||||
BELEG_FENSTER_VOR, BELEG_FENSTER_NACH = 300, 1200
|
||||
|
||||
# ── Struktur ──
|
||||
KAPITEL_SOLL_PUNKTE = 8
|
||||
BAUSTEIN_MIN_ATOME, BAUSTEIN_MAX_ATOME = 4, 8
|
||||
ZIELE_CHUNK_ATOME = 30
|
||||
|
||||
# ── Guide ──
|
||||
SECTION_WOERTER_PRO_ATOM = (40, 220) # QA-Band = ×1.25
|
||||
PRUEF_PANEL = 2
|
||||
KLAERUNG_PANEL = 3
|
||||
FIX_MAX_VERSUCHE = 3
|
||||
REDUNDANZ_JACCARD = 0.5 # Satzpaare zwischen Bausteinen (Paraphrasen!)
|
||||
REDUNDANZ_SHINGLE = 5 # gemeinsame 5-Wort-Folge = Kandidat
|
||||
REDUNDANZ_SATZ_MIN_TOKEN = 6
|
||||
ZIEL_DEDUP_JACCARD = 0.5 # synonyme Lernziele deterministisch mergen
|
||||
ECHO_JACCARD = 0.7 # 1. Satz nach Marker ≈ Atom-Definition = Echo
|
||||
SOLL_MIN_BELEGE_WIKI = 1 # kuratierte Quelle (Wikipedia) = 1 Beleg genügt
|
||||
|
||||
# ── Engine (R8) ──
|
||||
POLL_SEKUNDEN = 2.0
|
||||
MAX_VERSUCHE = 2 # Inhalts-Versuche je Task; Infra zählt separat
|
||||
STILLSTAND_N = 2 # identischer Befund-Fingerprint n× → PAUSE
|
||||
GATE_RUNDEN_MAX = {"gate_korpus": 4, "gate_inventar": 8, "gate_struktur": 3, "gate_guide": 12}
|
||||
# gate_inventar 6: Anker-Fix, 2× Dedup/Lücke, Web-Nachrecherche, Nachzügler-Dedup
|
||||
# gate_guide 8: Echtlauf konvergierte 45→28→10 — der Stillstand-Fingerprint
|
||||
# fängt echte Schleifen, das Runden-Limit ist nur die Notbremse dahinter
|
||||
MAX_PARALLEL_GLOBAL = 24
|
||||
|
||||
|
||||
def timeout_fuer(knoten: str, n: int = 0) -> float:
|
||||
basis, pro = TIMEOUTS.get(knoten, TIMEOUT_DEFAULT)
|
||||
return basis + pro * n
|
||||
|
||||
|
||||
def user_agent() -> str:
|
||||
mail = os.getenv("KONTAKT_MAIL", "")
|
||||
return f"creator3/0.1 (+mailto:{mail})" if mail else "creator3/0.1"
|
||||
205
backend/db.py
Normal file
205
backend/db.py
Normal file
@@ -0,0 +1,205 @@
|
||||
"""SQLite-Schicht: EINE Writer-Connection (eliminiert 'database is locked'
|
||||
architektonisch, R8), WAL, kurze Transaktionen. Zustand liegt NUR hier —
|
||||
Resume = Prozess neu starten, fertige Tasks überspringen."""
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable
|
||||
|
||||
from . import config
|
||||
|
||||
_conn: sqlite3.Connection | None = None
|
||||
_lock = threading.RLock()
|
||||
on_change: Callable[[str, dict], None] | None = None # ws.Hub.push
|
||||
|
||||
_LIVE_TABELLEN = {"topics", "runs", "tasks", "gate_laeufe", "befunde"}
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS topics(
|
||||
name TEXT PRIMARY KEY, titel TEXT DEFAULT '', status TEXT DEFAULT 'neu',
|
||||
quellen_max INTEGER, erstellt TEXT DEFAULT (datetime('now')));
|
||||
CREATE TABLE IF NOT EXISTS runs(
|
||||
id INTEGER PRIMARY KEY, topic TEXT NOT NULL, status TEXT DEFAULT 'running',
|
||||
gestartet TEXT, beendet TEXT, stufe TEXT DEFAULT '',
|
||||
budget_tokens INTEGER DEFAULT 0, grund TEXT DEFAULT '', git_hash TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS tasks(
|
||||
id INTEGER PRIMARY KEY, run_id INTEGER NOT NULL, topic TEXT NOT NULL,
|
||||
knoten TEXT NOT NULL, item TEXT NOT NULL, status TEXT DEFAULT 'offen',
|
||||
art TEXT DEFAULT '', versuch INTEGER DEFAULT 0, infra_versuch INTEGER DEFAULT 0,
|
||||
runde INTEGER DEFAULT 0, payload TEXT DEFAULT '{}', ergebnis TEXT DEFAULT '{}',
|
||||
fehler TEXT DEFAULT '', erzeugt_von TEXT DEFAULT '', gestartet TEXT, beendet TEXT,
|
||||
UNIQUE(topic, knoten, item));
|
||||
CREATE INDEX IF NOT EXISTS idx_tasks_disp ON tasks(run_id, status, knoten);
|
||||
CREATE TABLE IF NOT EXISTS gate_laeufe(
|
||||
id INTEGER PRIMARY KEY, run_id INTEGER, topic TEXT, knoten TEXT, runde INTEGER,
|
||||
status TEXT DEFAULT '', fingerprint TEXT DEFAULT '', befunde TEXT DEFAULT '[]',
|
||||
ts TEXT, UNIQUE(topic, knoten, runde));
|
||||
CREATE TABLE IF NOT EXISTS events(
|
||||
id INTEGER PRIMARY KEY, run_id INTEGER, ts TEXT, stufe TEXT DEFAULT '',
|
||||
knoten TEXT DEFAULT '', item TEXT DEFAULT '', skills TEXT DEFAULT '[]',
|
||||
skill_hash TEXT DEFAULT '', model TEXT DEFAULT '', role TEXT DEFAULT '',
|
||||
status TEXT DEFAULT '', dur_ms INTEGER DEFAULT 0, wait_ms INTEGER DEFAULT 0,
|
||||
tok_in INTEGER DEFAULT 0, tok_out INTEGER DEFAULT 0,
|
||||
tok_cache_read INTEGER DEFAULT 0, tok_cache_write INTEGER DEFAULT 0,
|
||||
meta TEXT DEFAULT '{}');
|
||||
CREATE INDEX IF NOT EXISTS idx_events_run ON events(run_id);
|
||||
CREATE TABLE IF NOT EXISTS call_texte(
|
||||
event_id INTEGER PRIMARY KEY, prompt TEXT DEFAULT '', antwort TEXT DEFAULT '');
|
||||
CREATE TABLE IF NOT EXISTS quellen(
|
||||
id INTEGER PRIMARY KEY, topic TEXT, titel TEXT DEFAULT '', url TEXT DEFAULT '',
|
||||
url_norm TEXT DEFAULT '', backend TEXT DEFAULT '', snapshot TEXT DEFAULT '',
|
||||
roh TEXT DEFAULT '', hash TEXT DEFAULT '', runde INTEGER DEFAULT 0,
|
||||
zweck TEXT DEFAULT 'korpus', status TEXT DEFAULT 'neu', grund TEXT DEFAULT '',
|
||||
atome_stand TEXT DEFAULT '', UNIQUE(topic, url_norm));
|
||||
CREATE TABLE IF NOT EXISTS soll(
|
||||
id INTEGER PRIMARY KEY, topic TEXT, punkt TEXT, status TEXT DEFAULT 'kandidat',
|
||||
belege TEXT DEFAULT '[]', kapitel_id INTEGER, geprueft TEXT DEFAULT '[]',
|
||||
freispruch TEXT DEFAULT '', verdichtet INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS atome(
|
||||
id INTEGER PRIMARY KEY, topic TEXT, titel TEXT, typ TEXT DEFAULT '',
|
||||
definition TEXT DEFAULT '', status TEXT DEFAULT 'neu', soll_id INTEGER,
|
||||
soll_geprueft INTEGER DEFAULT 0,
|
||||
ziel_id INTEGER, baustein_id INTEGER, ord INTEGER DEFAULT 0, merged_into INTEGER);
|
||||
CREATE INDEX IF NOT EXISTS idx_atome_topic ON atome(topic);
|
||||
CREATE TABLE IF NOT EXISTS anker(
|
||||
id INTEGER PRIMARY KEY, atom_id INTEGER, quelle_id INTEGER,
|
||||
start INTEGER DEFAULT -1, ende INTEGER DEFAULT -1, zitat TEXT DEFAULT '');
|
||||
CREATE INDEX IF NOT EXISTS idx_anker_atom ON anker(atom_id);
|
||||
CREATE TABLE IF NOT EXISTS kanten(
|
||||
id INTEGER PRIMARY KEY, topic TEXT, von_atom INTEGER, zu_atom INTEGER,
|
||||
art TEXT DEFAULT 'verwandt', status TEXT DEFAULT 'aktiv',
|
||||
UNIQUE(von_atom, zu_atom, art));
|
||||
CREATE TABLE IF NOT EXISTS lernziele(
|
||||
id INTEGER PRIMARY KEY, topic TEXT, titel TEXT DEFAULT '', text TEXT DEFAULT '',
|
||||
soll_id INTEGER, status TEXT DEFAULT 'neu');
|
||||
CREATE TABLE IF NOT EXISTS bausteine(
|
||||
id INTEGER PRIMARY KEY, topic TEXT, ziel_id INTEGER, titel TEXT DEFAULT '',
|
||||
ord INTEGER DEFAULT 0, kapitel_id INTEGER, status TEXT DEFAULT 'neu');
|
||||
CREATE TABLE IF NOT EXISTS kapitel(
|
||||
id INTEGER PRIMARY KEY, topic TEXT, titel TEXT DEFAULT '', intro TEXT DEFAULT '',
|
||||
ord INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS sections(
|
||||
baustein_id INTEGER PRIMARY KEY, stage TEXT DEFAULT 'writer', text TEXT DEFAULT '',
|
||||
qa_hash TEXT DEFAULT '', fix_versuche INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS befunde(
|
||||
id INTEGER PRIMARY KEY, run_id INTEGER, stufe TEXT DEFAULT '', knoten TEXT DEFAULT '',
|
||||
art TEXT DEFAULT '', item TEXT DEFAULT '', detail TEXT DEFAULT '',
|
||||
status TEXT DEFAULT 'offen', quelle TEXT DEFAULT '', runde INTEGER DEFAULT 0,
|
||||
geklaert INTEGER DEFAULT 0);
|
||||
CREATE INDEX IF NOT EXISTS idx_befunde_run ON befunde(run_id, stufe);
|
||||
"""
|
||||
|
||||
|
||||
def connect(pfad: str | None = None) -> None:
|
||||
global _conn
|
||||
with _lock:
|
||||
if _conn is not None:
|
||||
_conn.close()
|
||||
ziel = pfad or str(config.DB_PFAD)
|
||||
if ziel != ":memory:":
|
||||
config.DB_PFAD.parent.mkdir(parents=True, exist_ok=True)
|
||||
_conn = sqlite3.connect(ziel, check_same_thread=False)
|
||||
_conn.row_factory = sqlite3.Row
|
||||
_conn.execute("PRAGMA journal_mode=WAL")
|
||||
_conn.execute("PRAGMA synchronous=NORMAL")
|
||||
_conn.execute("PRAGMA busy_timeout=5000")
|
||||
_conn.execute("PRAGMA foreign_keys=ON")
|
||||
_conn.executescript(SCHEMA)
|
||||
for migration in ( # Mini-Migrationen für Bestands-DBs
|
||||
"ALTER TABLE atome ADD COLUMN soll_geprueft INTEGER DEFAULT 0",
|
||||
"ALTER TABLE topics ADD COLUMN quellen_max INTEGER",
|
||||
"ALTER TABLE soll ADD COLUMN verdichtet INTEGER DEFAULT 0"):
|
||||
try:
|
||||
_conn.execute(migration)
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
_conn.commit()
|
||||
|
||||
|
||||
def reset_for_tests() -> None:
|
||||
connect(":memory:")
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def j(x: Any) -> str:
|
||||
return json.dumps(x, ensure_ascii=False)
|
||||
|
||||
|
||||
def uj(s: str, default: Any = None) -> Any:
|
||||
try:
|
||||
return json.loads(s)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _notify(tabelle: str, row: dict) -> None:
|
||||
if on_change and tabelle in _LIVE_TABELLEN:
|
||||
on_change(tabelle, row)
|
||||
|
||||
|
||||
def query(sql: str, *args) -> list[dict]:
|
||||
with _lock:
|
||||
return [dict(r) for r in _conn.execute(sql, args).fetchall()]
|
||||
|
||||
|
||||
def one(sql: str, *args) -> dict | None:
|
||||
rows = query(sql, *args)
|
||||
return rows[0] if rows else None
|
||||
|
||||
|
||||
def execute(sql: str, *args) -> int:
|
||||
"""Einzelnes Statement + Commit. Rückgabe: rowcount (für Claim-Muster)."""
|
||||
with _lock:
|
||||
cur = _conn.execute(sql, args)
|
||||
_conn.commit()
|
||||
tabelle = sql.split()[2] if sql.lstrip().upper().startswith(("UPDATE",)) else ""
|
||||
_notify(tabelle.lower(), {})
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
def insert(tabelle: str, ignore: bool = False, **felder) -> int | None:
|
||||
"""INSERT (OR IGNORE); Rückgabe lastrowid oder None bei ignoriertem Duplikat."""
|
||||
keys = ", ".join(felder)
|
||||
qs = ", ".join("?" for _ in felder)
|
||||
verb = "INSERT OR IGNORE" if ignore else "INSERT"
|
||||
with _lock:
|
||||
cur = _conn.execute(f"{verb} INTO {tabelle}({keys}) VALUES({qs})",
|
||||
tuple(felder.values()))
|
||||
_conn.commit()
|
||||
_notify(tabelle, dict(felder))
|
||||
return cur.lastrowid if cur.rowcount else None
|
||||
|
||||
|
||||
def update(tabelle: str, wo: str, wo_args: tuple, **felder) -> int:
|
||||
setzt = ", ".join(f"{k}=?" for k in felder)
|
||||
with _lock:
|
||||
cur = _conn.execute(f"UPDATE {tabelle} SET {setzt} WHERE {wo}",
|
||||
tuple(felder.values()) + wo_args)
|
||||
_conn.commit()
|
||||
_notify(tabelle, dict(felder))
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
class tx:
|
||||
"""Transaktion für atomare Mehrfach-Schreiber (Ergebnis + fertig + Folgetasks).
|
||||
Innerhalb: cursor.execute; niemals LLM-Calls."""
|
||||
|
||||
def __enter__(self):
|
||||
_lock.acquire()
|
||||
_conn.execute("BEGIN IMMEDIATE")
|
||||
return _conn
|
||||
|
||||
def __exit__(self, typ, wert, tb):
|
||||
try:
|
||||
if typ is None:
|
||||
_conn.commit()
|
||||
_notify("tasks", {})
|
||||
else:
|
||||
_conn.rollback()
|
||||
finally:
|
||||
_lock.release()
|
||||
return False
|
||||
189
backend/dedup.py
Normal file
189
backend/dedup.py
Normal file
@@ -0,0 +1,189 @@
|
||||
"""Dedup-Barriere (R11): 1) Anker-Overlap ≥0.5 = einziger Auto-Merge.
|
||||
2) Kandidaten-Kanäle (Union): Titel-Kern, Jaccard gestemmt ≥0.3, Titel-Fuzzy ≥85
|
||||
(nur ≥2 Inhaltstoken), Akronym-Kanal, Gleicher-Soll-Punkt. 3) 2er-Panel,
|
||||
Merge NUR einstimmig; geprüfte Paare als 'verwandt'-Kante — nie zweimal Tokens."""
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from . import config, db, engine, llm, panels, textkit
|
||||
|
||||
|
||||
def _aktive(topic: str) -> list[dict]:
|
||||
return db.query("SELECT * FROM atome WHERE topic=? AND status='aktiv'", topic)
|
||||
|
||||
|
||||
def _anker(atom_id: int) -> list[dict]:
|
||||
return db.query("SELECT * FROM anker WHERE atom_id=? AND start>=0", atom_id)
|
||||
|
||||
|
||||
def _merge(topic: str, gewinner: dict, verlierer: dict) -> None:
|
||||
"""Gewinner = längere Definition; Anker wandern, Kanten hängen um."""
|
||||
if len(verlierer["definition"]) > len(gewinner["definition"]):
|
||||
gewinner, verlierer = verlierer, gewinner
|
||||
db.update("atome", "id=?", (verlierer["id"],), status="verworfen",
|
||||
merged_into=gewinner["id"])
|
||||
db.update("anker", "atom_id=?", (verlierer["id"],), atom_id=gewinner["id"])
|
||||
db.execute("UPDATE OR IGNORE kanten SET von_atom=? WHERE von_atom=?",
|
||||
gewinner["id"], verlierer["id"])
|
||||
db.execute("UPDATE OR IGNORE kanten SET zu_atom=? WHERE zu_atom=?",
|
||||
gewinner["id"], verlierer["id"])
|
||||
db.execute("DELETE FROM kanten WHERE von_atom=zu_atom")
|
||||
if verlierer["soll_id"] and not gewinner["soll_id"]:
|
||||
db.update("atome", "id=?", (gewinner["id"],), soll_id=verlierer["soll_id"])
|
||||
|
||||
|
||||
def _auto_merge_anker(topic: str) -> int:
|
||||
"""Gleiche Quellstelle = gleiches Atom (deterministisch, kein Judge)."""
|
||||
n = 0
|
||||
atome = _aktive(topic)
|
||||
for i, a in enumerate(atome):
|
||||
if a["status"] != "aktiv":
|
||||
continue
|
||||
spans_a = [(x["quelle_id"], x["start"], x["ende"]) for x in _anker(a["id"])]
|
||||
for b in atome[i + 1:]:
|
||||
if b["status"] != "aktiv":
|
||||
continue
|
||||
spans_b = [(x["quelle_id"], x["start"], x["ende"]) for x in _anker(b["id"])]
|
||||
if any(qa == qb and textkit.ueberlappung((sa, ea), (sb, eb))
|
||||
>= config.ANKER_OVERLAP_MERGE
|
||||
for qa, sa, ea in spans_a for qb, sb, eb in spans_b):
|
||||
_merge(topic, a, b)
|
||||
b["status"] = "verworfen"
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def _kandidaten(topic: str) -> list[tuple[dict, dict, str]]:
|
||||
atome = _aktive(topic)
|
||||
geprueft = {(k["von_atom"], k["zu_atom"]) for k in db.query(
|
||||
"SELECT von_atom, zu_atom FROM kanten WHERE topic=?", topic)}
|
||||
paare = []
|
||||
for i, a in enumerate(atome):
|
||||
for b in atome[i + 1:]:
|
||||
key = (min(a["id"], b["id"]), max(a["id"], b["id"]))
|
||||
if key in geprueft:
|
||||
continue # verwandt-Kante: nie zweimal Tokens
|
||||
if textkit.negations_menge(a["definition"]) != \
|
||||
textkit.negations_menge(b["definition"]):
|
||||
continue # Antonyme messen 0.91-0.95 — harte Vorbedingung
|
||||
kanal = _kanal(a, b)
|
||||
if kanal:
|
||||
paare.append((a, b, kanal))
|
||||
if len(paare) >= config.DEDUP_KANDIDATEN_CAP:
|
||||
return paare
|
||||
return paare
|
||||
|
||||
|
||||
def _signatur(text: str) -> set:
|
||||
"""Zahlen + Eigennamen — dieselbe Studie in Paraphrase teilt sie
|
||||
(a20≈a78: 'Biwer', '2023', '25', '35'), auch über Soll-Grenzen."""
|
||||
import re
|
||||
zahlen = set(re.findall(r"\b\d{2,4}\b", text))
|
||||
namen = {w for w in re.findall(r"\b[A-ZÄÖÜ][a-zäöü]{3,}\b", text)}
|
||||
return zahlen | namen
|
||||
|
||||
|
||||
def _kanal(a: dict, b: dict) -> str:
|
||||
if textkit.titel_kern(a["titel"]) == textkit.titel_kern(b["titel"]):
|
||||
return "titel_kern"
|
||||
if textkit.ist_akronym_von(a["titel"], b["titel"]) or \
|
||||
textkit.ist_akronym_von(b["titel"], a["titel"]):
|
||||
return "akronym" # Lektion 28 — fehlte in creator2
|
||||
if a["soll_id"] and a["soll_id"] == b["soll_id"]:
|
||||
return "gleicher_soll" # nach Verdichtung wenige Paare — Panel gatet
|
||||
if len(_signatur(a["definition"]) & _signatur(b["definition"])) >= 2:
|
||||
return "zahlen_namen"
|
||||
if textkit.jaccard(a["definition"], b["definition"]) >= config.DEDUP_JACCARD:
|
||||
return "jaccard"
|
||||
ta, tb = textkit.tokens(a["titel"]), textkit.tokens(b["titel"])
|
||||
if len(ta) >= 2 and len(tb) >= 2 and \
|
||||
fuzz.token_set_ratio(a["titel"], b["titel"]) >= config.DEDUP_TITEL_FUZZY:
|
||||
return "titel_fuzzy" # nur ≥2 Inhaltstoken (Subset-100-Falle)
|
||||
return ""
|
||||
|
||||
|
||||
async def _chunk_urteilen(task: dict, chunk: list, tag: str) -> tuple[int, list]:
|
||||
"""Ein Panel über ≤ DEDUP_PAARE_PRO_CALL Paare. Entscheidet nur Paare, die
|
||||
in ALLEN gültigen Stimmen vorkommen; fehlende → Rückgabe zum Nachprüfen
|
||||
(nie still als „keine Dublette" werten)."""
|
||||
topic = task["topic"]
|
||||
liste = "\n\n".join(
|
||||
f"PAAR {j + 1}:\nA[{a['id']}] {a['titel']}: {a['definition'][:200]}\n"
|
||||
f"B[{b['id']}] {b['titel']}: {b['definition'][:200]}"
|
||||
for j, (a, b, _) in enumerate(chunk))
|
||||
|
||||
async def ruf(p):
|
||||
text = await llm.call(
|
||||
run_id=task["run_id"], stufe="inventar", knoten="dedup",
|
||||
item=f"{task['item']}:{tag}:p{p}", role="judge", n=len(chunk),
|
||||
skill_namen=["richter", "deutsch-praezise", "dedup-urteil"],
|
||||
werte={"paare": liste})
|
||||
if text is None:
|
||||
return None
|
||||
urteile = {}
|
||||
for b in textkit.bloecke(text, "URTEIL"):
|
||||
if b.get("paar", "").isdigit():
|
||||
urteile[int(b["paar"])] = b.get("urteil", "UNSICHER")
|
||||
return urteile
|
||||
|
||||
stimmen = await panels.panel(ruf, config.MERGE_PANEL)
|
||||
gueltig = [s for s in stimmen if s is not None]
|
||||
merges, unvollstaendig = 0, []
|
||||
for j, (a, b, kanal) in enumerate(chunk):
|
||||
if len(gueltig) < config.MERGE_PANEL or any((j + 1) not in s
|
||||
for s in gueltig):
|
||||
unvollstaendig.append((a, b, kanal))
|
||||
continue
|
||||
urteile = [s[j + 1] for s in gueltig]
|
||||
if all(u in ("dublette", "enthalten") for u in urteile):
|
||||
a_neu = db.one("SELECT * FROM atome WHERE id=?", a["id"])
|
||||
b_neu = db.one("SELECT * FROM atome WHERE id=?", b["id"])
|
||||
if a_neu["status"] == "aktiv" and b_neu["status"] == "aktiv":
|
||||
_merge(topic, a_neu, b_neu) # Gewinner = längere Definition
|
||||
merges += 1
|
||||
else: # kein einstimmiges Ja → Gedächtnis-Kante, nie wieder prüfen
|
||||
db.insert("kanten", ignore=True, topic=topic,
|
||||
von_atom=min(a["id"], b["id"]),
|
||||
zu_atom=max(a["id"], b["id"]), art="verwandt")
|
||||
return merges, unvollstaendig
|
||||
|
||||
|
||||
@engine.worker("dedup.dedup")
|
||||
async def dedup(task: dict) -> engine.Ergebnis:
|
||||
topic = task["topic"]
|
||||
auto = _auto_merge_anker(topic)
|
||||
paare = _kandidaten(topic)
|
||||
merges, offen = 0, []
|
||||
for i in range(0, len(paare), config.DEDUP_PAARE_PRO_CALL):
|
||||
chunk = paare[i:i + config.DEDUP_PAARE_PRO_CALL]
|
||||
m, unvollstaendig = await _chunk_urteilen(task, chunk, f"c{i}")
|
||||
merges += m
|
||||
offen += unvollstaendig
|
||||
rest = []
|
||||
for i in range(0, len(offen), config.DEDUP_PAARE_PRO_CALL):
|
||||
# EIN Nach-Call nur mit den unentschiedenen Paaren
|
||||
m, unvollstaendig = await _chunk_urteilen(
|
||||
task, offen[i:i + config.DEDUP_PAARE_PRO_CALL], f"nach{i}")
|
||||
merges += m
|
||||
rest += unvollstaendig
|
||||
for a, b, _ in rest: # immer noch ohne Urteil → sichtbar, KEINE Kante
|
||||
item = f"paar:{min(a['id'], b['id'])}:{max(a['id'], b['id'])}"
|
||||
if not db.one("SELECT id FROM befunde WHERE run_id=? AND art='parse' "
|
||||
"AND item=?", task["run_id"], item):
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="inventar",
|
||||
knoten="dedup", art="parse", item=item,
|
||||
detail="Paar-Urteil fehlt auch nach Nach-Call",
|
||||
runde=task["runde"])
|
||||
|
||||
stand = (db.one("SELECT COALESCE(MAX(id),0) m FROM atome WHERE topic=? AND "
|
||||
"status='aktiv'", topic))["m"]
|
||||
# Zuordnungs-Tasks nur für UNGEPRÜFTE Atome ohne Soll-Punkt — KEINER-Atome
|
||||
# werden nicht erneut gefragt (Endlosschleifen-Lektion, Echtlauf 2)
|
||||
ohne = [a["id"] for a in db.query(
|
||||
"SELECT id FROM atome WHERE topic=? AND status='aktiv' AND "
|
||||
"soll_id IS NULL AND soll_geprueft=0", topic)]
|
||||
neue = [{"knoten": "soll_zuordnung", "item": f"{task['item']}:z{i}",
|
||||
"payload": {"atom_ids": ohne[i:i + config.ZIELE_CHUNK_ATOME]}}
|
||||
for i in range(0, len(ohne), config.ZIELE_CHUNK_ATOME)]
|
||||
return engine.Ergebnis(daten={"auto": auto, "panel_merges": merges,
|
||||
"kandidaten": len(paare), "stand": stand},
|
||||
neue_tasks=neue)
|
||||
360
backend/engine.py
Normal file
360
backend/engine.py
Normal file
@@ -0,0 +1,360 @@
|
||||
"""Task-Engine (R8): DB ist die einzige Wahrheit/Queue. Scheduler blockiert auf
|
||||
einem Wecker-Event + 2-s-Fallback-Poll. Claim per UPDATE…WHERE status='offen'.
|
||||
Resume = derselbe Codepfad (Loop starten, DB lesen); Zombie-Reset beim Start.
|
||||
Ergebnis + status='fertig' + Folgetasks entstehen in EINER Transaktion."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import importlib
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from . import config, db, graph
|
||||
from .llm import BudgetErschoepft, LaufPause
|
||||
|
||||
WORKER: dict[str, object] = {}
|
||||
_laeufe: dict[str, asyncio.Task] = {}
|
||||
_wecker: dict[int, asyncio.Event] = {}
|
||||
|
||||
|
||||
def worker(name: str, buendel: bool = False):
|
||||
def deco(fn):
|
||||
fn.buendel = buendel # opt-in: Worker verarbeitet t["_buendel"]-Listen
|
||||
WORKER[name] = fn
|
||||
return fn
|
||||
return deco
|
||||
|
||||
|
||||
@dataclass
|
||||
class Ergebnis:
|
||||
"""Rückgabe jedes Workers. neue_tasks: {knoten, item, payload?, art?, runde?}
|
||||
— Ziel muss über eine Kante vom eigenen Knoten erreichbar sein."""
|
||||
daten: dict = field(default_factory=dict)
|
||||
neue_tasks: list = field(default_factory=list)
|
||||
befunde: list = field(default_factory=list) # nur Gates: {art, item, detail}
|
||||
gate_status: str = "" # nur Gates: gruen | rot
|
||||
teil_status: dict = field(default_factory=dict) # Bündel: task_id → fertig | neu
|
||||
|
||||
|
||||
def module_laden() -> None:
|
||||
"""Worker-Module importieren → Registrierung; danach Graph validieren."""
|
||||
for mod in ("suche", "laden", "korpus", "inventar", "dedup",
|
||||
"struktur", "guide", "redundanz", "montage"):
|
||||
importlib.import_module(f"backend.{mod}")
|
||||
graph.laden(set(WORKER))
|
||||
|
||||
|
||||
def wecken(run_id: int) -> None:
|
||||
ev = _wecker.get(run_id)
|
||||
if ev:
|
||||
ev.set()
|
||||
|
||||
|
||||
# ── Lauf-Verwaltung ──
|
||||
|
||||
def lauf_starten(topic: str, budget: int | None = None) -> int:
|
||||
if topic in _laeufe and not _laeufe[topic].done():
|
||||
raise RuntimeError("Lauf läuft bereits")
|
||||
run = db.one("SELECT * FROM runs WHERE topic=? AND status IN "
|
||||
"('paused','budget') ORDER BY id DESC LIMIT 1", topic)
|
||||
if run: # Resume: gleicher Run, gleiche Tasks
|
||||
run_id = run["id"]
|
||||
felder = {"status": "running", "grund": ""}
|
||||
if budget is not None: # Budget-Nachschlag beim Fortsetzen
|
||||
felder["budget_tokens"] = budget
|
||||
db.update("runs", "id=?", (run_id,), **felder)
|
||||
else:
|
||||
run_id = db.insert("runs", topic=topic, status="running",
|
||||
gestartet=db.now(),
|
||||
budget_tokens=budget if budget is not None
|
||||
else config.RUN_BUDGET_TOKENS)
|
||||
_laeufe[topic] = asyncio.ensure_future(_lauf(topic, run_id))
|
||||
return run_id
|
||||
|
||||
|
||||
def lauf_pausieren(topic: str) -> None:
|
||||
run = _aktiver_run(topic)
|
||||
if run:
|
||||
db.update("runs", "id=?", (run["id"],), status="paused", grund="manuell")
|
||||
wecken(run["id"])
|
||||
|
||||
|
||||
def lauf_stoppen(topic: str) -> None:
|
||||
run = _aktiver_run(topic)
|
||||
if run:
|
||||
db.update("runs", "id=?", (run["id"],), status="stopped", beendet=db.now())
|
||||
wecken(run["id"])
|
||||
task = _laeufe.get(topic)
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
|
||||
|
||||
def _aktiver_run(topic: str) -> dict | None:
|
||||
return db.one("SELECT * FROM runs WHERE topic=? AND status='running' "
|
||||
"ORDER BY id DESC LIMIT 1", topic)
|
||||
|
||||
|
||||
# ── Kern-Loop ──
|
||||
|
||||
async def _lauf(topic: str, run_id: int) -> None:
|
||||
g = graph.get()
|
||||
ev = _wecker[run_id] = asyncio.Event()
|
||||
sem_global = asyncio.Semaphore(config.MAX_PARALLEL_GLOBAL)
|
||||
sems = {k.id: asyncio.Semaphore(k.max_parallel) for k in g.knoten.values()}
|
||||
laufende: set[asyncio.Task] = set()
|
||||
# Zombie-Reset: ein Prozess ⇒ beim Start ist garantiert nichts in Arbeit
|
||||
db.execute("UPDATE tasks SET status='offen', fehler='zombie' "
|
||||
"WHERE topic=? AND status='laufend'", topic)
|
||||
_start_tasks(topic, run_id)
|
||||
try:
|
||||
while True:
|
||||
run = db.one("SELECT * FROM runs WHERE id=?", run_id)
|
||||
if run["status"] != "running":
|
||||
break
|
||||
stufe_i = _aktuelle_stufe(g, topic)
|
||||
db.update("runs", "id=? AND stufe!=?", (run_id, g.stufen[stufe_i]),
|
||||
stufe=g.stufen[stufe_i])
|
||||
_gate_pruefen(g, topic, run_id, stufe_i)
|
||||
gestartet = _dispatch(g, topic, run_id, stufe_i, sems, sem_global,
|
||||
laufende, ev)
|
||||
if not gestartet and not laufende and _leer(topic):
|
||||
_abschliessen(g, topic, run_id)
|
||||
break
|
||||
with suppress(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(ev.wait(), timeout=config.POLL_SEKUNDEN)
|
||||
ev.clear()
|
||||
laufende = {t for t in laufende if not t.done()}
|
||||
# Pause/Stop: Laufende sanft auslaufen lassen
|
||||
if laufende:
|
||||
await asyncio.gather(*laufende, return_exceptions=True)
|
||||
finally:
|
||||
_wecker.pop(run_id, None)
|
||||
|
||||
|
||||
def _leer(topic: str) -> bool:
|
||||
return not db.one("SELECT id FROM tasks WHERE topic=? AND "
|
||||
"status IN ('offen','laufend') LIMIT 1", topic)
|
||||
|
||||
|
||||
def _aktuelle_stufe(g: graph.Graph, topic: str) -> int:
|
||||
for i, stufe in enumerate(g.stufen):
|
||||
gate = g.gate_der_stufe(stufe)
|
||||
if gate is None:
|
||||
return i
|
||||
gruen = db.one("SELECT id FROM gate_laeufe WHERE topic=? AND knoten=? "
|
||||
"AND status='gruen'", topic, gate.id)
|
||||
if not gruen:
|
||||
return i
|
||||
return len(g.stufen) - 1
|
||||
|
||||
|
||||
def _dispatch(g, topic, run_id, stufe_i, sems, sem_global, laufende, ev) -> int:
|
||||
offen = db.query(
|
||||
"SELECT * FROM tasks WHERE topic=? AND status='offen' ORDER BY runde, id",
|
||||
topic)
|
||||
n = 0
|
||||
for t in offen:
|
||||
kn = g.knoten.get(t["knoten"])
|
||||
if kn is None or g.stufen_index(kn.stufe) > stufe_i:
|
||||
continue
|
||||
if sems[kn.id].locked() or sem_global.locked():
|
||||
continue
|
||||
if kn.barriere and _vorarbeit_offen(g, topic, kn, ausser=t["id"]):
|
||||
continue
|
||||
if kn.typ == "gate" and _vorarbeit_offen(g, topic, kn, ausser=t["id"]):
|
||||
continue
|
||||
if db.execute("UPDATE tasks SET status='laufend', gestartet=? "
|
||||
"WHERE id=? AND status='offen'", db.now(), t["id"]) != 1:
|
||||
continue
|
||||
t = dict(t)
|
||||
t["_buendel"] = _buendel_claimen(g, t, kn)
|
||||
task = asyncio.create_task(
|
||||
_ausfuehren(g, t, kn, sems[kn.id], sem_global, ev))
|
||||
laufende.add(task)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def _buendel_claimen(g, t: dict, kn: graph.Knoten) -> list[dict]:
|
||||
"""Bis zu buendel−1 weitere offene Tasks gleicher (knoten, art) mit-claimen.
|
||||
Nur für Worker mit buendel-Opt-in — sonst blieben Mitglieder 'laufend' hängen."""
|
||||
fn = WORKER[kn.worker]
|
||||
if kn.buendel <= 1 or not getattr(fn, "buendel", False):
|
||||
return []
|
||||
mitglieder = []
|
||||
kandidaten = db.query(
|
||||
"SELECT * FROM tasks WHERE topic=? AND knoten=? AND art=? AND "
|
||||
"status='offen' AND id!=? ORDER BY runde, id LIMIT ?",
|
||||
t["topic"], t["knoten"], t["art"], t["id"], kn.buendel - 1)
|
||||
for k in kandidaten:
|
||||
if db.execute("UPDATE tasks SET status='laufend', gestartet=? "
|
||||
"WHERE id=? AND status='offen'", db.now(), k["id"]) == 1:
|
||||
mitglieder.append(dict(k))
|
||||
return mitglieder
|
||||
|
||||
|
||||
def _vorarbeit_offen(g: graph.Graph, topic: str, kn: graph.Knoten,
|
||||
ausser: int) -> bool:
|
||||
"""Barriere/Gate: wartet, bis KEIN offener/laufender Task an Knoten der
|
||||
eigenen oder früherer Stufen existiert (außer sich selbst)."""
|
||||
max_i = g.stufen_index(kn.stufe)
|
||||
ids = [k.id for k in g.knoten.values()
|
||||
if g.stufen_index(k.stufe) <= max_i and k.id != kn.id]
|
||||
qs = ",".join("?" for _ in ids)
|
||||
return bool(db.one(
|
||||
f"SELECT id FROM tasks WHERE topic=? AND status IN ('offen','laufend') "
|
||||
f"AND knoten IN ({qs}) AND id!=? LIMIT 1", topic, *ids, ausser))
|
||||
|
||||
|
||||
def _gate_pruefen(g: graph.Graph, topic: str, run_id: int, stufe_i: int) -> None:
|
||||
"""Stufe leer gelaufen → Gate-Task für Runde N anlegen (idempotent)."""
|
||||
gate = g.gate_der_stufe(g.stufen[stufe_i])
|
||||
if gate is None:
|
||||
return
|
||||
vorhanden = db.one("SELECT id FROM tasks WHERE topic=? AND knoten=? AND "
|
||||
"status IN ('offen','laufend')", topic, gate.id)
|
||||
if vorhanden:
|
||||
return
|
||||
if _vorarbeit_offen(g, topic, gate, ausser=-1):
|
||||
return
|
||||
runde = (db.one("SELECT COUNT(*) c FROM gate_laeufe WHERE topic=? AND knoten=?",
|
||||
topic, gate.id) or {"c": 0})["c"] + 1
|
||||
if runde > config.GATE_RUNDEN_MAX.get(gate.id, 3):
|
||||
_pausieren(run_id, f"gate_cap:{gate.id}:{runde}")
|
||||
return
|
||||
db.insert("tasks", ignore=True, run_id=run_id, topic=topic, knoten=gate.id,
|
||||
item=f"runde:{runde}", runde=runde, erzeugt_von="engine")
|
||||
|
||||
|
||||
async def _ausfuehren(g, t: dict, kn: graph.Knoten, sem, sem_global, ev) -> None:
|
||||
fn = WORKER[kn.worker]
|
||||
alle = [t] + t.get("_buendel", [])
|
||||
try:
|
||||
async with sem_global, sem:
|
||||
erg: Ergebnis = await fn(t)
|
||||
except (LaufPause, BudgetErschoepft) as e:
|
||||
status = "budget" if isinstance(e, BudgetErschoepft) else "paused"
|
||||
for m in alle:
|
||||
db.update("tasks", "id=?", (m["id"],), status="offen",
|
||||
fehler=str(e)[:300])
|
||||
_pausieren(t["run_id"], f"{status}:{e}", status=status)
|
||||
ev.set()
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
for m in alle:
|
||||
db.update("tasks", "id=?", (m["id"],), status="offen", fehler="abbruch")
|
||||
raise
|
||||
except Exception as e: # Inhaltsfehler: begrenzte Neuversuche, dann sichtbar
|
||||
for m in alle:
|
||||
_task_neu(m, kn, f"{type(e).__name__}: {e}"[:300])
|
||||
ev.set()
|
||||
return
|
||||
_abschluss_schreiben(g, t, kn, erg)
|
||||
ev.set()
|
||||
|
||||
|
||||
def _task_neu(t: dict, kn: graph.Knoten, fehler: str) -> None:
|
||||
versuch = t["versuch"] + 1
|
||||
status = "offen" if versuch < kn.max_versuche else "fehler"
|
||||
db.update("tasks", "id=?", (t["id"],), status=status, versuch=versuch,
|
||||
fehler=fehler)
|
||||
|
||||
|
||||
def _abschluss_schreiben(g, t: dict, kn: graph.Knoten, erg: Ergebnis) -> None:
|
||||
erlaubt = g.erlaubte_ziele(kn.id)
|
||||
# Bündel-Mitglieder: kaputte Teil-Antworten → neuer Versuch, nie verwerfen
|
||||
for m in t.get("_buendel", []):
|
||||
if erg.teil_status.get(m["id"]) == "neu":
|
||||
_task_neu(m, kn, "buendel_teil_kaputt")
|
||||
else:
|
||||
db.update("tasks", "id=?", (m["id"],), status="fertig",
|
||||
ergebnis=db.j({"buendel_mit": t["id"]}), beendet=db.now())
|
||||
with db.tx() as c:
|
||||
if erg.teil_status.get(t["id"]) == "neu":
|
||||
c.execute("UPDATE tasks SET status=?, versuch=?, fehler=? WHERE id=?",
|
||||
("offen" if t["versuch"] + 1 < kn.max_versuche else "fehler",
|
||||
t["versuch"] + 1, "buendel_teil_kaputt", t["id"]))
|
||||
else:
|
||||
c.execute(
|
||||
"UPDATE tasks SET status='fertig', ergebnis=?, beendet=? WHERE id=?",
|
||||
(db.j(erg.daten), db.now(), t["id"]))
|
||||
eingefuegt = 0
|
||||
for nt in erg.neue_tasks:
|
||||
if nt["knoten"] not in erlaubt:
|
||||
raise graph.GraphFehler(
|
||||
f"{kn.id} darf keine Tasks für {nt['knoten']} erzeugen")
|
||||
cur = c.execute(
|
||||
"INSERT OR IGNORE INTO tasks(run_id, topic, knoten, item, art, "
|
||||
"runde, payload, erzeugt_von) VALUES(?,?,?,?,?,?,?,?)",
|
||||
(t["run_id"], t["topic"], nt["knoten"], nt["item"],
|
||||
nt.get("art", ""), nt.get("runde", t["runde"]),
|
||||
db.j(nt.get("payload", {})), kn.id))
|
||||
eingefuegt += cur.rowcount
|
||||
if kn.typ == "gate":
|
||||
_gate_abschluss(c, t, kn, erg, eingefuegt)
|
||||
|
||||
|
||||
def _gate_abschluss(c, t: dict, kn: graph.Knoten, erg: Ergebnis,
|
||||
neu_eingefuegt: int = 0) -> None:
|
||||
befunde = sorted(f"{b['art']}|{b['item']}|{b.get('detail', '')[:200]}"
|
||||
for b in erg.befunde)
|
||||
fingerprint = hashlib.sha256("\n".join(befunde).encode()).hexdigest()[:16]
|
||||
c.execute("INSERT OR REPLACE INTO gate_laeufe(run_id, topic, knoten, runde, "
|
||||
"status, fingerprint, befunde, ts) VALUES(?,?,?,?,?,?,?,?)",
|
||||
(t["run_id"], t["topic"], kn.id, t["runde"], erg.gate_status,
|
||||
fingerprint, db.j(erg.befunde), db.now()))
|
||||
for b in erg.befunde:
|
||||
# Spiegel-Zeile nur, wenn kein identischer Befund schon offen ist
|
||||
schon = c.execute(
|
||||
"SELECT id FROM befunde WHERE art=? AND item=? AND "
|
||||
"substr(detail,1,200)=? AND status='offen'",
|
||||
(b["art"], b["item"], b.get("detail", "")[:200])).fetchone()
|
||||
if not schon:
|
||||
c.execute("INSERT INTO befunde(run_id, stufe, knoten, art, item, "
|
||||
"detail, runde) VALUES(?,?,?,?,?,?,?)",
|
||||
(t["run_id"], kn.stufe, kn.id, b["art"], b["item"],
|
||||
b.get("detail", "")[:500], t["runde"]))
|
||||
if erg.gate_status == "gruen":
|
||||
# Stufe bestanden → offene Befunde dieser Stufe sind Geschichte
|
||||
c.execute("UPDATE befunde SET status='behoben' WHERE status='offen' AND "
|
||||
"stufe=? AND run_id IN (SELECT id FROM runs WHERE topic=?)",
|
||||
(kn.stufe, t["topic"]))
|
||||
if erg.gate_status == "rot":
|
||||
# Stillstand: identischer Fingerprint n-mal hintereinander UND keine
|
||||
# NEUEN Reparatur-Tasks entstanden (die Eskalations-Kette kann bei
|
||||
# gleicher Befundmenge trotzdem vorankommen: fix → klaerung → neu)
|
||||
alte = [r["fingerprint"] for r in c.execute(
|
||||
"SELECT fingerprint FROM gate_laeufe WHERE topic=? AND knoten=? "
|
||||
"ORDER BY runde DESC LIMIT ?",
|
||||
(t["topic"], kn.id, config.STILLSTAND_N)).fetchall()]
|
||||
if neu_eingefuegt == 0:
|
||||
grund = (f"stillstand:{kn.id}:r{t['runde']}"
|
||||
if len(alte) >= config.STILLSTAND_N and len(set(alte)) == 1
|
||||
else f"gate_rot_ohne_reparatur:{kn.id}")
|
||||
c.execute("UPDATE runs SET status='paused', grund=? WHERE id=?",
|
||||
(grund, t["run_id"]))
|
||||
|
||||
|
||||
def _pausieren(run_id: int, grund: str, status: str = "paused") -> None:
|
||||
db.update("runs", "id=? AND status='running'", (run_id,),
|
||||
status=status, grund=grund[:200])
|
||||
|
||||
|
||||
def _abschliessen(g: graph.Graph, topic: str, run_id: int) -> None:
|
||||
"""Kein Task mehr offen: done, wenn alle Gates grün + Montage fertig."""
|
||||
fertig = db.one("SELECT id FROM tasks WHERE topic=? AND knoten='montage' "
|
||||
"AND status='fertig'", topic)
|
||||
offen_fehler = db.one("SELECT id FROM tasks WHERE topic=? AND status='fehler' "
|
||||
"LIMIT 1", topic)
|
||||
if fertig and not offen_fehler:
|
||||
db.update("runs", "id=?", (run_id,), status="done", beendet=db.now())
|
||||
else:
|
||||
grund = "tasks_mit_fehler" if offen_fehler else "leerlauf_ohne_montage"
|
||||
_pausieren(run_id, grund)
|
||||
|
||||
|
||||
def _start_tasks(topic: str, run_id: int) -> None:
|
||||
"""Initiale Tasks der Stufe 1 (idempotent — Resume ist ein No-Op)."""
|
||||
for lens in config.RECHERCHE_LENSES:
|
||||
db.insert("tasks", ignore=True, run_id=run_id, topic=topic,
|
||||
knoten="recherche_plan", item=f"r1:lens:{lens}", runde=1,
|
||||
payload=db.j({"lens": lens}), erzeugt_von="start")
|
||||
316
backend/fakes.py
Normal file
316
backend/fakes.py
Normal file
@@ -0,0 +1,316 @@
|
||||
"""Deterministische Fake-Welt (CREATOR_FAKE=1): ersetzt LLM, Suche und Laden.
|
||||
Die Fakes LESEN den Prompt — Zitate kommen nur, wenn sie wörtlich im gelieferten
|
||||
Text stehen (Zitat-Gate wird mitgetestet). Präparierte Eigenschaften:
|
||||
- Dublette über 2 Quellen (HTTP ↔ Hypertext Transfer Protocol, Akronym-Kanal)
|
||||
- Soll-Punkt „Fehlerbehandlung" ohne Atom (erst die Web-Nachrecherche deckt ihn)
|
||||
- Kapitel-übergreifende Satz-Dopplung, die der Fix per Verweis ersetzt
|
||||
Fehlender Handler → harter Fehler (nie still raten)."""
|
||||
import re
|
||||
|
||||
from . import skills
|
||||
|
||||
FUELL = ("Dieser Absatz liefert zusätzlichen Zusammenhang für Lernende. " * 4).strip()
|
||||
|
||||
S1 = "Ein Protokoll regelt den Austausch von Nachrichten zwischen zwei Partnern im Netz."
|
||||
S2 = "HTTP steuert die Übertragung von Webseiten zwischen Browser und Server."
|
||||
S3 = "Das Hypertext Transfer Protocol überträgt Inhalte im World Wide Web."
|
||||
S4 = "Kommunikation braucht klare Regeln für beide Seiten."
|
||||
S5 = ("Beim Wiederholungsmuster wird eine fehlgeschlagene Anfrage nach kurzer "
|
||||
"Wartezeit erneut gesendet.")
|
||||
S6A = "Ohne saubere Fehlerbehandlung scheitern Systeme im Alltag."
|
||||
S7A = "Ein Vorteil ist bessere Konzentration bei der Arbeit."
|
||||
S7B = "Ein weiterer Vorteil ist weniger Stress im Alltag."
|
||||
S6B = "Fehlerbehandlung entscheidet über die Zuverlässigkeit eines Systems."
|
||||
DUP = ("Fehler lassen sich nie ganz vermeiden, entscheidend ist der richtige "
|
||||
"Umgang mit ihnen.")
|
||||
|
||||
P1 = "Grundlagen der Protokolle und Regeln erklären"
|
||||
P2 = "Das HTTP-Protokoll verstehen"
|
||||
P3 = "Fehlerbehandlung beherrschen"
|
||||
|
||||
QUELLEN = {
|
||||
"https://beispiel.de/grundlagen":
|
||||
f"Einführung in das Thema.\n\n{FUELL}\n\n{S1}\n\n{FUELL}\n\n{S2}\n\n"
|
||||
f"{FUELL}\n\n{S6A}\n\n{FUELL}",
|
||||
"https://beispiel.de/vertiefung":
|
||||
f"Vertiefende Betrachtung.\n\n{FUELL}\n\n{S4}\n\n{FUELL}\n\n{S3}\n\n"
|
||||
f"{FUELL}\n\n{S6B}\n\n{S7A} {S7B}\n\n{FUELL}",
|
||||
"https://beispiel.de/fehler":
|
||||
f"Umgang mit Fehlern.\n\n{FUELL}\n\n{S5}\n\n{FUELL}\n\n{FUELL}",
|
||||
}
|
||||
|
||||
SOLL = [(P1, [S1, S4]), (P2, [S2, S3]), (P3, [S6A, S6B])]
|
||||
ATOME = [ # (titel, typ, definition, beleg-satz)
|
||||
("Protokoll", "begriff",
|
||||
"Ein Protokoll ist die Vereinbarung, nach der zwei Partner Nachrichten austauschen.", S1),
|
||||
("HTTP", "begriff",
|
||||
"HTTP ist das Protokoll für die Übertragung von Webseiten.", S2),
|
||||
("Hypertext Transfer Protocol", "begriff",
|
||||
"Das Hypertext Transfer Protocol dient der Übertragung von Inhalten im Web.", S3),
|
||||
("Kommunikationsregeln", "aussage",
|
||||
"Erfolgreiche Kommunikation setzt klare, beidseitig bekannte Regeln voraus.", S4),
|
||||
("Vorteil Konzentration", "aussage",
|
||||
"Die Methode verbessert die Konzentration bei der Arbeit.", S7A),
|
||||
("Vorteil Stressabbau", "aussage",
|
||||
"Die Methode senkt den empfundenen Stress im Alltag.", S7B),
|
||||
("Wiederholungsmuster", "verfahren",
|
||||
"Beim Wiederholungsmuster wird eine fehlgeschlagene Anfrage nach Wartezeit wiederholt.", S5),
|
||||
]
|
||||
_ATOM_SOLL = {"Protokoll": P1, "Kommunikationsregeln": P1, "HTTP": P2,
|
||||
"Hypertext Transfer Protocol": P2, "Wiederholungsmuster": P3,
|
||||
"Vorteil Konzentration": P1, "Vorteil Stressabbau": P1}
|
||||
_DUBLETTEN = {frozenset({"HTTP", "Hypertext Transfer Protocol"})}
|
||||
_ENTHALTEN = {frozenset({"Basisregeln", "Basisregeln und Sonderfaelle"})}
|
||||
|
||||
|
||||
def suche(query: str) -> list[dict]:
|
||||
q = query.casefold()
|
||||
if "fehlerbehandlung" in q:
|
||||
return [{"titel": "Fehlerbehandlung", "url": "https://beispiel.de/fehler",
|
||||
"backend": "fake"}]
|
||||
if "r2" in q or " v2" in q:
|
||||
return [] # Runde 2: nichts Neues → Sättigung
|
||||
if "ueberblick" in q:
|
||||
return [{"titel": "Grundlagen", "url": "https://beispiel.de/grundlagen",
|
||||
"backend": "fake"}]
|
||||
if "tutorial" in q:
|
||||
return [{"titel": "Vertiefung", "url": "https://beispiel.de/vertiefung",
|
||||
"backend": "fake"}]
|
||||
return []
|
||||
|
||||
|
||||
def laden(url: str) -> str:
|
||||
return QUELLEN.get(url, "")
|
||||
|
||||
|
||||
def _abschnitt(prompt: str, marke: str) -> str:
|
||||
m = re.search(rf"{marke}:\n(.*?)(?=\n\n[A-ZÄÖÜ\-]{{3,}}[^a-z]*:\n|\nAusgabeformat|\Z)",
|
||||
prompt, re.DOTALL)
|
||||
return m.group(1) if m else prompt
|
||||
|
||||
|
||||
def _ids_liste(text: str) -> list[tuple[int, str]]:
|
||||
return [(int(m.group(1)), m.group(2).strip())
|
||||
for m in re.finditer(r"^\[(\d+)\]\s*(.+)$", text, re.MULTILINE)]
|
||||
|
||||
|
||||
# ── Handler je Aufgabe ──
|
||||
|
||||
def _nummerierte_abschnitte(p: str) -> dict[int, str]:
|
||||
"""`=== ABSCHNITT n ===`-Blöcke aus dem Bündel-Prompt lesen."""
|
||||
return {int(m.group(1)): m.group(2) for m in re.finditer(
|
||||
r"=== ABSCHNITT (\d+) ===\n(.*?)(?=\n\n=== ABSCHNITT |\n\nRegeln:|\Z)",
|
||||
p, re.DOTALL)}
|
||||
|
||||
|
||||
def _recherche_plan(p: str) -> str:
|
||||
runde = re.search(r"\(Runde (\d+)\)", p)
|
||||
r = runde.group(1) if runde else "1"
|
||||
m = re.search(r"BLICKWINKEL:\n(.*?)\n\nBereits", p, re.DOTALL)
|
||||
lenses = re.findall(r"^- (\w+)$", m.group(1) if m else "", re.MULTILINE)
|
||||
return "\n".join(f"===QUERY===\nLENS: {l}\nTEXT: thema {l} r{r} nr{i}"
|
||||
for l in (lenses or ["x"]) for i in (1, 2))
|
||||
|
||||
|
||||
def _soll_extraktion(p: str) -> str:
|
||||
out = []
|
||||
for i, text in _nummerierte_abschnitte(p).items():
|
||||
n = 0
|
||||
for punkt, belege in SOLL:
|
||||
for beleg in belege:
|
||||
if beleg in text:
|
||||
out.append(f"===PUNKT===\nABSCHNITT: {i}\nTEXT: {punkt}\n"
|
||||
f"BELEG: {beleg}")
|
||||
n += 1
|
||||
if not n:
|
||||
out.append(f"===LEER===\nABSCHNITT: {i}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _soll_konsens(p: str) -> str:
|
||||
gruppen: dict[str, list[int]] = {}
|
||||
for i, text in _ids_liste(_abschnitt(p, "KANDIDATEN")):
|
||||
gruppen.setdefault(text, []).append(i)
|
||||
return "\n".join(f"===GRUPPE===\nIDS: {','.join(map(str, ids))}\nTEXT: {text}"
|
||||
for text, ids in gruppen.items())
|
||||
|
||||
|
||||
def _atom_extraktion(p: str) -> str:
|
||||
out = []
|
||||
for i, text in _nummerierte_abschnitte(p).items():
|
||||
n = 0
|
||||
for titel, typ, definition, satz in ATOME:
|
||||
if satz in text:
|
||||
out.append(f"===ATOM===\nABSCHNITT: {i}\nTITEL: {titel}\n"
|
||||
f"TYP: {typ}\nDEFINITION: {definition}\nZITAT: {satz}")
|
||||
n += 1
|
||||
if not n:
|
||||
out.append(f"===LEER===\nABSCHNITT: {i}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _anker_fix(p: str) -> str:
|
||||
return "\n".join(f"===FIX===\nID: {i}\nZITAT: VERWERFEN"
|
||||
for i, _ in _ids_liste(_abschnitt(p, "ATOME")))
|
||||
|
||||
|
||||
def _dedup_urteil(p: str) -> str:
|
||||
out = []
|
||||
for m in re.finditer(r"PAAR (\d+):\nA\[\d+\] ([^:]+):.*?\nB\[\d+\] ([^:]+):",
|
||||
p, re.DOTALL):
|
||||
nr, ta, tb = m.group(1), m.group(2).strip(), m.group(3).strip()
|
||||
paar = frozenset({ta, tb})
|
||||
urteil = ("dublette" if paar in _DUBLETTEN
|
||||
else "enthalten" if paar in _ENTHALTEN else "verschieden")
|
||||
out.append(f"===URTEIL===\nPAAR: {nr}\nURTEIL: {urteil}\nGRUND: fake")
|
||||
return "\n".join(out) or "===URTEIL===\nPAAR: 1\nURTEIL: verschieden\nGRUND: leer"
|
||||
|
||||
|
||||
def _soll_zuordnung(p: str) -> str:
|
||||
soll_map = {text: i for i, text in _ids_liste(_abschnitt(p, "SOLL-PUNKTE"))}
|
||||
out = []
|
||||
for atom_id, zeile in _ids_liste(_abschnitt(p, "ATOME")):
|
||||
titel = zeile.split(":")[0].strip()
|
||||
punkt = _ATOM_SOLL.get(titel, "")
|
||||
ziel = soll_map.get(punkt)
|
||||
out.append(f"===ZUORDNUNG===\nATOM: {atom_id}\n"
|
||||
f"SOLL: {ziel if ziel is not None else 'KEINER'}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _luecke_nachextraktion(p: str) -> str:
|
||||
out = []
|
||||
for m in re.finditer(r"=== LUECKE ===\nSOLL: (\d+)\nPUNKT: .*?\nFENSTER:\n"
|
||||
r"(.*?)(?=\n\n=== LUECKE ===|\n\nRegeln:|\Z)",
|
||||
p, re.DOTALL):
|
||||
sid, fenster = m.group(1), m.group(2)
|
||||
if S5 in fenster:
|
||||
t, typ, d, s = next(a for a in ATOME if a[3] == S5)
|
||||
out.append(f"===ATOM===\nSOLL: {sid}\nTITEL: {t}\nTYP: {typ}\n"
|
||||
f"DEFINITION: {d}\nZITAT: {s}")
|
||||
else:
|
||||
out.append(f"===NICHTS===\nSOLL: {sid}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _verdichtung(p: str) -> str:
|
||||
vorteile, andere = [], []
|
||||
for i, zeile in _ids_liste(_abschnitt(p, "ATOME")):
|
||||
(vorteile if "Vorteil" in zeile else andere).append(str(i))
|
||||
out = []
|
||||
if vorteile:
|
||||
out.append(f"===EINHEIT===\nIDS: {','.join(vorteile)}")
|
||||
out += [f"===EINHEIT===\nIDS: {i}" for i in andere]
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _lernziele(p: str) -> str:
|
||||
out = []
|
||||
for m in re.finditer(r"=== SOLL (\d+) ===\nPUNKT: (.*?)\nATOME:\n"
|
||||
r"(.*?)(?=\n\n=== SOLL |\n\nRegeln:|\Z)", p, re.DOTALL):
|
||||
i, punkt = m.group(1), m.group(2)
|
||||
ids = [str(x) for x, _ in _ids_liste(m.group(3))]
|
||||
out.append(f"===ZIEL===\nSOLL: {i}\nTITEL: {punkt.split()[0][:30]}\n"
|
||||
f"TEXT: Der Lernende kann {punkt} anwenden.\n"
|
||||
f"ATOME: {','.join(ids)}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _bausteine(p: str) -> str:
|
||||
ids = [str(i) for i, _ in _ids_liste(_abschnitt(p, "ATOME (in Quellreihenfolge)"))]
|
||||
return f"===BAUSTEIN===\nTITEL: Baustein\nATOME: {','.join(ids)}"
|
||||
|
||||
|
||||
def _themen_schnitt(p: str) -> str:
|
||||
fehler, rest = [], []
|
||||
for i, text in _ids_liste(_abschnitt(p, "SOLL-PUNKTE")):
|
||||
(fehler if "Fehlerbehandlung" in text else rest).append(str(i))
|
||||
out = []
|
||||
if rest:
|
||||
out.append(f"===KAPITEL===\nTITEL: Grundlagen\nPUNKTE: {','.join(rest)}")
|
||||
if fehler:
|
||||
out.append(f"===KAPITEL===\nTITEL: Fehlerbehandlung\nPUNKTE: {','.join(fehler)}")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def _ordnung(p: str) -> str:
|
||||
prior = re.search(r"übernehmen\): ([\d,]+)", p)
|
||||
return f"===ORDNUNG===\nIDS: {prior.group(1) if prior else ''}"
|
||||
|
||||
|
||||
def _writer(p: str) -> str:
|
||||
atome = re.findall(r"<!-- atom: (\d+) -->\n\*\*(.+?)\*\* — (.+)", p)
|
||||
teile = []
|
||||
for aid, titel, definition in atome:
|
||||
# Erster Satz bewusst KEIN Definitions-Echo (semantischer det-Check)
|
||||
teile.append(f"<!-- atom: {aid} -->\n\nSchauen wir uns das Konzept "
|
||||
f"nun genauer an. {definition} "
|
||||
f"{FUELL} Damit ist {titel} eingeordnet.")
|
||||
teile.append(DUP) # gepflanzte Dopplung — Redundanz-Check muss sie finden
|
||||
return "===TEXT===\n" + "\n\n".join(teile)
|
||||
|
||||
|
||||
def _kapitel_intro(p: str) -> str:
|
||||
kap = re.search(r"Kapitel „(.+?)\"", p)
|
||||
return (f"===INTRO===\nDieses Kapitel behandelt {kap.group(1) if kap else 'X'}. "
|
||||
"Es baut auf dem bisher Gelernten auf.")
|
||||
|
||||
|
||||
def _guide_pruefung(p: str) -> str:
|
||||
ids = re.findall(r"^=== SECTION (\d+) ===$", p, re.MULTILINE)
|
||||
return "\n".join(f"===OK===\nSECTION: {i}"
|
||||
for i in dict.fromkeys(ids)) or "===OK===\nSECTION: 0"
|
||||
|
||||
|
||||
def _guide_fix(p: str) -> str:
|
||||
section = _abschnitt(p, "SECTION")
|
||||
zeilen = [z for z in section.splitlines()
|
||||
if "Fehler lassen sich nie ganz vermeiden" not in z]
|
||||
zeilen.append("Wie bereits bei den Grundlagen beschrieben, zählt der "
|
||||
"Umgang mit Fehlern.")
|
||||
return "===TEXT===\n" + "\n".join(zeilen).strip()
|
||||
|
||||
|
||||
def _redundanz_urteil(p: str) -> str:
|
||||
out = []
|
||||
for m in re.finditer(r"PAAR (\d+):\nKapitel A: (.*)\nKapitel B: (.*)", p):
|
||||
redundant = "Fehler lassen sich nie ganz vermeiden" in m.group(2)
|
||||
out.append(f"===URTEIL===\nPAAR: {m.group(1)}\n"
|
||||
f"URTEIL: {'redundant' if redundant else 'didaktisch_gewollt'}")
|
||||
return "\n".join(out) or "===URTEIL===\nPAAR: 1\nURTEIL: didaktisch_gewollt"
|
||||
|
||||
|
||||
_HANDLER = {
|
||||
"recherche-plan": _recherche_plan,
|
||||
"soll-extraktion": _soll_extraktion,
|
||||
"soll-konsens": _soll_konsens,
|
||||
"soll-beleg": lambda p: "===BELEG===\nID: 0\nBELEG: KEIN BELEG",
|
||||
"quellen-relevanz": lambda p: "===RELEVANT===\nIDS: " + ",".join(
|
||||
m for m in re.findall(r"^\[(\d+)\]", p, re.MULTILINE)),
|
||||
"atom-extraktion": _atom_extraktion,
|
||||
"anker-fix": _anker_fix,
|
||||
"dedup-urteil": _dedup_urteil,
|
||||
"soll-zuordnung": _soll_zuordnung,
|
||||
"luecke-nachextraktion": _luecke_nachextraktion,
|
||||
"luecke-freispruch": lambda p: "===URTEIL===\nURTEIL: unerfuellbar\nGRUND: fake",
|
||||
"verdichtung": _verdichtung,
|
||||
"lernziele": _lernziele,
|
||||
"bausteine": _bausteine,
|
||||
"themen-schnitt": _themen_schnitt,
|
||||
"ordnung": _ordnung,
|
||||
"writer": _writer,
|
||||
"kapitel-intro": _kapitel_intro,
|
||||
"guide-pruefung": _guide_pruefung,
|
||||
"guide-fix": _guide_fix,
|
||||
"redundanz-urteil": _redundanz_urteil,
|
||||
"klaerung": lambda p: "===URTEIL===\nURTEIL: kein_mangel\nGRUND: fake",
|
||||
}
|
||||
|
||||
|
||||
def antwort(prompt: str) -> tuple[int, str, str]:
|
||||
"""→ (rc, text, err). Fehlender Handler = harter Fehler, nie still raten."""
|
||||
name = skills.aufgaben_name(prompt)
|
||||
handler = _HANDLER.get(name)
|
||||
if handler is None:
|
||||
return 1, "", f"kein Fake-Handler für Aufgabe '{name}'"
|
||||
return 0, handler(prompt), ""
|
||||
154
backend/graph.py
Normal file
154
backend/graph.py
Normal file
@@ -0,0 +1,154 @@
|
||||
"""pipeline.yaml laden + statisch validieren (R8): Normal-Kanten müssen ein DAG
|
||||
sein (graphlib), Rück-/Reparatur-Kanten nur von Gates und stufen-abwärts,
|
||||
Worker registriert, Skill-Dateien vorhanden, genau ein Gate je Stufe (außer
|
||||
letzter). Liefert auch das vorberechnete Layout für die Graphensicht."""
|
||||
import graphlib
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import yaml
|
||||
|
||||
from . import config, skills
|
||||
|
||||
|
||||
class GraphFehler(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class Knoten:
|
||||
id: str
|
||||
typ: str # system | llm | gate
|
||||
worker: str
|
||||
stufe: str
|
||||
titel: str = ""
|
||||
beschreibung: str = ""
|
||||
skills: list = field(default_factory=list)
|
||||
klasse: str = "lang" # lang | mittel | kurz → Bündel-Größe je Call
|
||||
max_parallel: int = 4
|
||||
max_versuche: int = config.MAX_VERSUCHE
|
||||
barriere: bool = False
|
||||
|
||||
@property
|
||||
def buendel(self) -> int:
|
||||
return config.BUENDEL[self.klasse]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Graph:
|
||||
stufen: list[str]
|
||||
knoten: dict[str, Knoten]
|
||||
kanten: list[dict] # {von, nach, art}
|
||||
|
||||
def stufen_index(self, stufe: str) -> int:
|
||||
return self.stufen.index(stufe)
|
||||
|
||||
def gate_der_stufe(self, stufe: str) -> Knoten | None:
|
||||
return next((k for k in self.knoten.values()
|
||||
if k.typ == "gate" and k.stufe == stufe), None)
|
||||
|
||||
def normal_nachfolger(self, knoten_id: str) -> set[str]:
|
||||
return {k["nach"] for k in self.kanten
|
||||
if k["von"] == knoten_id and k["art"] == "normal"}
|
||||
|
||||
def erlaubte_ziele(self, knoten_id: str) -> set[str]:
|
||||
"""Alle Knoten, für die dieser Knoten Tasks erzeugen darf."""
|
||||
return {k["nach"] for k in self.kanten if k["von"] == knoten_id}
|
||||
|
||||
def layout(self) -> dict:
|
||||
"""Fluss-Layout (R7): Zeile = längster Pfad über Normal-Kanten,
|
||||
Spalte = Spur innerhalb der Zeile. Reine Reparatur-Ziele (ohne
|
||||
Normal-Kanten) rücken neben ihr Gate."""
|
||||
vor: dict[str, list[str]] = {k: [] for k in self.knoten}
|
||||
for ka in self.kanten:
|
||||
if ka["art"] == "normal":
|
||||
vor[ka["nach"]].append(ka["von"])
|
||||
ebene: dict[str, int] = {}
|
||||
for kid in graphlib.TopologicalSorter(
|
||||
{k: set(v) for k, v in vor.items()}).static_order():
|
||||
ebene[kid] = max((ebene[v] + 1 for v in vor[kid]), default=0)
|
||||
an_normal = {ka[e] for ka in self.kanten if ka["art"] == "normal"
|
||||
for e in ("von", "nach")}
|
||||
for ka in self.kanten:
|
||||
if ka["art"] != "normal" and ka["nach"] not in an_normal:
|
||||
ebene[ka["nach"]] = ebene[ka["von"]]
|
||||
|
||||
pos = {}
|
||||
spuren: dict[int, int] = {}
|
||||
for kid in sorted(ebene, key=lambda i: ebene[i]):
|
||||
k = self.knoten[kid]
|
||||
spur = spuren.get(ebene[kid], 0)
|
||||
spuren[ebene[kid]] = spur + 1
|
||||
pos[kid] = {"zeile": ebene[kid], "spalte": spur, "typ": k.typ,
|
||||
"barriere": k.barriere, "stufe": k.stufe,
|
||||
"titel": k.titel or kid,
|
||||
"beschreibung": k.beschreibung,
|
||||
"skills": [{"name": s, "art": skills.lade(s)["art"]}
|
||||
for s in k.skills]}
|
||||
return {"stufen": self.stufen, "knoten": pos, "kanten": self.kanten}
|
||||
|
||||
|
||||
_graph: Graph | None = None
|
||||
|
||||
|
||||
def laden(worker_namen: set[str] | None = None, pfad=None) -> Graph:
|
||||
global _graph
|
||||
daten = yaml.safe_load((pfad or config.PIPELINE_YAML).read_text())
|
||||
stufen = daten["stufen"]
|
||||
knoten = {}
|
||||
for k in daten["knoten"]:
|
||||
kn = Knoten(**{**k, "skills": k.get("skills", [])})
|
||||
if kn.id in knoten:
|
||||
raise GraphFehler(f"Knoten-id doppelt: {kn.id}")
|
||||
if kn.stufe not in stufen:
|
||||
raise GraphFehler(f"{kn.id}: unbekannte Stufe {kn.stufe}")
|
||||
if kn.typ not in ("system", "llm", "gate"):
|
||||
raise GraphFehler(f"{kn.id}: unbekannter Typ {kn.typ}")
|
||||
if kn.klasse not in config.BUENDEL:
|
||||
raise GraphFehler(f"{kn.id}: unbekannte Klasse {kn.klasse}")
|
||||
knoten[kn.id] = kn
|
||||
kanten = daten["kanten"]
|
||||
|
||||
for kante in kanten:
|
||||
for ende in ("von", "nach"):
|
||||
if kante[ende] not in knoten:
|
||||
raise GraphFehler(f"Kante {kante}: unbekannter Knoten {kante[ende]}")
|
||||
if kante["art"] not in ("normal", "reparatur", "ruecklauf"):
|
||||
raise GraphFehler(f"Kante {kante}: unbekannte Art")
|
||||
if kante["art"] != "normal":
|
||||
von = knoten[kante["von"]]
|
||||
if von.typ != "gate":
|
||||
raise GraphFehler(f"{kante['art']}-Kante nur von Gates: {kante}")
|
||||
if stufen.index(knoten[kante["nach"]].stufe) > stufen.index(von.stufe):
|
||||
raise GraphFehler(f"{kante['art']}-Kante muss stufen-abwärts zeigen: {kante}")
|
||||
|
||||
# Normal-Kanten: DAG-Pflicht (graphlib erwartet Vorgänger-Format)
|
||||
vorgaenger: dict[str, set] = {k: set() for k in knoten}
|
||||
for kante in kanten:
|
||||
if kante["art"] == "normal":
|
||||
vorgaenger[kante["nach"]].add(kante["von"])
|
||||
try:
|
||||
graphlib.TopologicalSorter(vorgaenger).prepare()
|
||||
except graphlib.CycleError as e:
|
||||
raise GraphFehler(f"Zyklus in Normal-Kanten: {e}") from e
|
||||
|
||||
for stufe in stufen[:-1]:
|
||||
gates = [k for k in knoten.values() if k.typ == "gate" and k.stufe == stufe]
|
||||
if len(gates) != 1:
|
||||
raise GraphFehler(f"Stufe {stufe}: genau 1 Gate nötig, {len(gates)} gefunden")
|
||||
|
||||
for kn in knoten.values():
|
||||
if kn.typ == "llm" and not kn.skills:
|
||||
raise GraphFehler(f"{kn.id}: llm-Knoten ohne Skills")
|
||||
for s in kn.skills:
|
||||
skills.lade(s) # wirft SkillFehler, wenn Datei fehlt/kaputt
|
||||
if worker_namen is not None and kn.worker not in worker_namen:
|
||||
raise GraphFehler(f"{kn.id}: Worker '{kn.worker}' nicht registriert")
|
||||
|
||||
_graph = Graph(stufen=stufen, knoten=knoten, kanten=kanten)
|
||||
return _graph
|
||||
|
||||
|
||||
def get() -> Graph:
|
||||
if _graph is None:
|
||||
raise GraphFehler("Graph nicht geladen")
|
||||
return _graph
|
||||
460
backend/guide.py
Normal file
460
backend/guide.py
Normal file
@@ -0,0 +1,460 @@
|
||||
"""Stufe 4: Writer (DELIMITED, „bekannt"-Kontext, Verweis statt Wiederholung)
|
||||
→ deterministische Prüfung → 2er-Prüfer-Panel (qa_hash, fail-closed) → Fix
|
||||
(Cap 3, Klärung 3er einmalig). Gate re-prüft die harten Checks selbst."""
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from . import belege, config, db, engine, llm, panels, textkit
|
||||
|
||||
_MARKER_RE = re.compile(r"<!--\s*atom:\s*(\d+)\s*-->")
|
||||
|
||||
|
||||
def _baustein(bid: int) -> dict:
|
||||
return db.one("SELECT * FROM bausteine WHERE id=?", bid)
|
||||
|
||||
|
||||
def _atome(bid: int) -> list[dict]:
|
||||
return db.query("SELECT * FROM atome WHERE baustein_id=? AND status='aktiv' "
|
||||
"ORDER BY ord", bid)
|
||||
|
||||
|
||||
def _band(n_atome: int) -> tuple[int, int]:
|
||||
lo, hi = config.SECTION_WOERTER_PRO_ATOM
|
||||
return lo * n_atome, hi * n_atome
|
||||
|
||||
|
||||
def _slug(titel: str) -> str:
|
||||
s = textkit.norm(titel).replace(" ", "-")
|
||||
return re.sub(r"[^a-z0-9äöüß-]", "", s)
|
||||
|
||||
|
||||
def _bekannt_kontext(topic: str, baustein: dict) -> str:
|
||||
"""Bereits behandelte Atome (frühere Kapitel/Bausteine) als Linkziele."""
|
||||
fruehere = db.query(
|
||||
"SELECT a.titel, b2.titel bt FROM atome a JOIN bausteine b2 ON "
|
||||
"b2.id=a.baustein_id JOIN kapitel k2 ON k2.id=b2.kapitel_id "
|
||||
"JOIN bausteine b ON b.id=? JOIN kapitel k ON k.id=b.kapitel_id "
|
||||
"WHERE a.topic=? AND a.status='aktiv' AND "
|
||||
"(k2.ord < k.ord OR (k2.ord=k.ord AND b2.ord < b.ord))",
|
||||
baustein["id"], topic)
|
||||
if not fruehere:
|
||||
return "-"
|
||||
return "\n".join(f"- {f['titel']} (im Abschnitt '{f['bt']}')"
|
||||
for f in fruehere[:60])
|
||||
|
||||
|
||||
@engine.worker("guide.writer")
|
||||
async def writer(task: dict) -> engine.Ergebnis:
|
||||
payload = db.uj(task["payload"], {})
|
||||
bid = payload["baustein_id"]
|
||||
b = _baustein(bid)
|
||||
atome = _atome(bid)
|
||||
if not atome:
|
||||
return engine.Ergebnis(daten={"skip": True})
|
||||
sec = db.one("SELECT * FROM sections WHERE baustein_id=?", bid)
|
||||
if sec and sec["text"] and sec["stage"] != "writer" and \
|
||||
not payload.get("neu"): # Resume-Skip — außer bei Neuschreiben
|
||||
return engine.Ergebnis(daten={"skip": True}, neue_tasks=[
|
||||
{"knoten": "det_pruefung", "item": task["item"].replace("baustein", "det")}])
|
||||
# Prosa-Form statt Feldnamen — „TITEL:/DEFINITION:" leakte 41× in Guides
|
||||
atome_text = "\n".join(
|
||||
f"<!-- atom: {a['id']} -->\n**{a['titel']}** — {a['definition']}"
|
||||
for a in atome)
|
||||
lo, hi = _band(len(atome))
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="guide", knoten="writer", item=task["item"],
|
||||
role="writer",
|
||||
skill_namen=["autor", "deutsch-praezise", "guide-lesbarkeit", "writer"],
|
||||
werte={"baustein": b["titel"], "atome": atome_text,
|
||||
"belege": belege.fenster_fuer_atome([a["id"] for a in atome])[:40000],
|
||||
"bekannt": _bekannt_kontext(task["topic"], b),
|
||||
"band": f"{lo}-{hi}"})
|
||||
if antwort is None:
|
||||
raise RuntimeError("writer ohne Ergebnis")
|
||||
text = textkit.block_text(antwort, "TEXT") or antwort
|
||||
text = re.sub(r"^=+\s*$", "", text, flags=re.MULTILINE).strip()
|
||||
# Führende Baustein-Titel-Überschrift entfernen — die setzt die Montage
|
||||
m = re.match(r"^#{1,3}\s+(.+)\n+", text)
|
||||
if m and textkit.jaccard(m.group(1), b["titel"], gestemmt=False) >= 0.6:
|
||||
text = text[m.end():]
|
||||
versuche = 0 if payload.get("neu") else \
|
||||
(sec or {}).get("fix_versuche") or 0 # Neuschreiben resettet den Fix-Cap
|
||||
db.execute("INSERT OR REPLACE INTO sections(baustein_id, stage, text, "
|
||||
"qa_hash, fix_versuche) VALUES(?,?,?,?,?)",
|
||||
bid, "geprueft", text, "", versuche)
|
||||
return engine.Ergebnis(daten={"woerter": len(text.split())}, neue_tasks=[
|
||||
{"knoten": "det_pruefung", "item": task["item"].replace("baustein", "det"),
|
||||
"payload": {"baustein_id": bid}}])
|
||||
|
||||
|
||||
def _det_befunde(topic: str, bid: int, text: str) -> list[dict]:
|
||||
atome = _atome(bid)
|
||||
eigene = {a["id"] for a in atome}
|
||||
marker = [int(m) for m in _MARKER_RE.findall(text)]
|
||||
befunde = []
|
||||
for a in atome:
|
||||
if marker.count(a["id"]) != 1:
|
||||
befunde.append({"art": "marker_fehlend" if a["id"] not in marker
|
||||
else "marker_doppelt",
|
||||
"item": f"baustein:{bid}",
|
||||
"detail": f"Atom {a['id']} ({a['titel']})"})
|
||||
for m in set(marker) - eigene:
|
||||
befunde.append({"art": "marker_fremd", "item": f"baustein:{bid}",
|
||||
"detail": f"Atom {m} gehört nicht zum Baustein"})
|
||||
lo, hi = _band(len(atome))
|
||||
woerter = len(re.sub(r"<!--.*?-->", "", text).split())
|
||||
if not lo * 0.75 <= woerter <= hi * 1.25:
|
||||
befunde.append({"art": "laenge", "item": f"baustein:{bid}",
|
||||
"detail": f"{woerter} Wörter, Band {lo}-{hi}"})
|
||||
leak = re.search(r"^(TITEL|TYP|DEFINITION|ZITAT|BELEG):", text, re.MULTILINE)
|
||||
if leak:
|
||||
befunde.append({"art": "format_leak", "item": f"baustein:{bid}",
|
||||
"detail": f"Rohformat im Text: {leak.group(0)}"})
|
||||
# Lesefluss-Checks: linear lesbar, keine Sprünge, keine Echo-Zeilen
|
||||
link = re.search(r"\]\(#[^)]*\)", text)
|
||||
if link:
|
||||
befunde.append({"art": "link_im_text", "item": f"baustein:{bid}",
|
||||
"detail": "Anker-Link im Fließtext (Rückbezug als Prosa)"})
|
||||
echo = re.search(r"^\*\*[^*\n]+\*\*\s+—", text, re.MULTILINE)
|
||||
if echo:
|
||||
befunde.append({"art": "atom_echo", "item": f"baustein:{bid}",
|
||||
"detail": f"Fettzeilen-Echo: {echo.group(0)[:60]}"})
|
||||
else:
|
||||
# Semantisches Echo: erster Satz nach dem Marker ≈ Atom-Definition
|
||||
# (fängt Fettzeilen OHNE Gedankenstrich, Gutachten-Rückfall Z.294/297)
|
||||
for a in atome:
|
||||
m = re.search(rf"<!--\s*atom:\s*{a['id']}\s*-->\s*\n+(.+)", text)
|
||||
if not m:
|
||||
continue
|
||||
erster = textkit.satz_split(m.group(1))
|
||||
if erster and textkit.jaccard(erster[0], a["definition"]) >= \
|
||||
config.ECHO_JACCARD:
|
||||
befunde.append({"art": "atom_echo", "item": f"baustein:{bid}",
|
||||
"detail": f"Definition-Echo bei Atom {a['id']}: "
|
||||
f"{erster[0][:60]}"})
|
||||
break
|
||||
h = re.search(r"^#{1,3}\s", text, re.MULTILINE)
|
||||
if h:
|
||||
befunde.append({"art": "h_ebene", "item": f"baustein:{bid}",
|
||||
"detail": "H1-H3 im Section-Text (nur #### erlaubt)"})
|
||||
# Vorwärtsverweise: Titel von Atomen SPÄTERER Kapitel fallen nicht früher
|
||||
b = _baustein(bid)
|
||||
spaetere = db.query(
|
||||
"SELECT a.titel FROM atome a JOIN bausteine b2 ON b2.id=a.baustein_id "
|
||||
"JOIN kapitel k2 ON k2.id=b2.kapitel_id JOIN bausteine b ON b.id=? "
|
||||
"JOIN kapitel k ON k.id=b.kapitel_id WHERE a.topic=? AND "
|
||||
"a.status='aktiv' AND k2.ord > k.ord", bid, topic)
|
||||
# Verlinkte Erwähnungen sind KEIN Vorwärtsverweis — „Verweis statt
|
||||
# Wiederholung" ist ja genau das gewollte Muster
|
||||
ohne_links = re.sub(r"\[[^\]]*\]\([^)]*\)", " ", text)
|
||||
text_norm = textkit.norm(ohne_links)
|
||||
for s in spaetere:
|
||||
titel_norm = textkit.norm(s["titel"])
|
||||
if len(titel_norm) >= 12 and titel_norm in text_norm:
|
||||
befunde.append({"art": "vorwaerts", "item": f"baustein:{bid}",
|
||||
"detail": f"'{s['titel']}' wird erst später gelehrt"})
|
||||
return befunde
|
||||
|
||||
|
||||
@engine.worker("guide.det_pruefung")
|
||||
async def det_pruefung(task: dict) -> engine.Ergebnis:
|
||||
payload = db.uj(task["payload"], {})
|
||||
bid = payload["baustein_id"]
|
||||
sec = db.one("SELECT * FROM sections WHERE baustein_id=?", bid)
|
||||
befunde = _det_befunde(task["topic"], bid, sec["text"]) if sec else []
|
||||
# Nur Befunde schließen, die es LIVE nicht mehr gibt — fortbestehende
|
||||
# Zeilen behalten ihre Historie (geklaert-Flag, Eskalations-Stand)
|
||||
frisch = {(b["art"], b["item"], b["detail"][:400]) for b in befunde}
|
||||
for row in db.query(
|
||||
"SELECT id, art, item, detail FROM befunde WHERE item=? AND "
|
||||
"status='offen' AND art IN ('marker_fehlend','marker_fremd',"
|
||||
"'marker_doppelt','vorwaerts','laenge','format_leak',"
|
||||
"'link_im_text','atom_echo','h_ebene')",
|
||||
f"baustein:{bid}"):
|
||||
if (row["art"], row["item"], row["detail"][:400]) not in frisch:
|
||||
db.update("befunde", "id=?", (row["id"],), status="veraltet")
|
||||
for b in befunde:
|
||||
befund_merken(task["run_id"], "det_pruefung", b["art"], b["item"],
|
||||
b["detail"], task["runde"])
|
||||
return engine.Ergebnis(daten={"befunde": len(befunde)}, neue_tasks=[
|
||||
{"knoten": "llm_pruefung", "item": task["item"].replace("det", "pruef"),
|
||||
"payload": {"baustein_id": bid}}])
|
||||
|
||||
|
||||
@engine.worker("guide.llm_pruefung", buendel=True)
|
||||
async def llm_pruefung(task: dict) -> engine.Ergebnis:
|
||||
mitglieder = [task] + task.get("_buendel", [])
|
||||
neue = [{"knoten": "redundanz", "item": f"r{task['runde']}:redundanz"}]
|
||||
teil, aktive = {}, {} # bid → (mitglied, section, hash)
|
||||
for m in mitglieder:
|
||||
bid = db.uj(m["payload"], {})["baustein_id"]
|
||||
sec = db.one("SELECT * FROM sections WHERE baustein_id=?", bid)
|
||||
if sec is None or not sec["text"]:
|
||||
teil[m["id"]] = "fertig"
|
||||
continue
|
||||
hash_ = hashlib.sha256(sec["text"].encode()).hexdigest()[:16]
|
||||
if sec["qa_hash"] == hash_: # unverändert seit letzter Prüfung
|
||||
teil[m["id"]] = "fertig"
|
||||
continue
|
||||
aktive[bid] = (m, sec, hash_)
|
||||
if not aktive:
|
||||
return engine.Ergebnis(daten={"skip": True}, neue_tasks=neue,
|
||||
teil_status=teil)
|
||||
sections_text = "\n\n".join(
|
||||
f"=== SECTION {bid} ===\n{sec['text']}\n\n"
|
||||
f"--- BELEGE für SECTION {bid} ---\n"
|
||||
+ belege.fenster_fuer_atome([a["id"] for a in _atome(bid)])[:30000]
|
||||
for bid, (_, sec, _) in aktive.items())
|
||||
|
||||
async def ruf(i):
|
||||
text = await llm.call(
|
||||
run_id=task["run_id"], stufe="guide", knoten="llm_pruefung",
|
||||
item=f"{task['item']}:p{i}", role="judge", n=len(aktive),
|
||||
skill_namen=["richter", "deutsch-praezise", "guide-pruefung"],
|
||||
werte={"sections": sections_text})
|
||||
if text is None:
|
||||
return None
|
||||
return {"befunde": [b for b in textkit.bloecke(text, "BEFUND")
|
||||
if b.get("art") in ("falsch", "luecke", "stil")
|
||||
and b.get("section", "").strip().isdigit()],
|
||||
"ok": {int(b["section"]) for b in textkit.bloecke(text, "OK")
|
||||
if b.get("section", "").strip().isdigit()}}
|
||||
|
||||
stimmen = await panels.panel(ruf, config.PRUEF_PANEL)
|
||||
gueltig = [s for s in stimmen if s is not None]
|
||||
if len(gueltig) < config.PRUEF_PANEL:
|
||||
# fail-closed: qa_hash NICHT setzen — nächste Runde prüft erneut
|
||||
for _, (m, _, _) in aktive.items():
|
||||
teil[m["id"]] = "fertig"
|
||||
return engine.Ergebnis(daten={"panel_ausfall": True}, neue_tasks=neue,
|
||||
teil_status=teil)
|
||||
gesamt = 0
|
||||
for bid, (m, sec, hash_) in aktive.items():
|
||||
je_stimme = [[b for b in s["befunde"] if int(b["section"]) == bid]
|
||||
for s in gueltig]
|
||||
if not all(bid in s["ok"] or je_stimme[k]
|
||||
for k, s in enumerate(gueltig)):
|
||||
teil[m["id"]] = "neu" # Section fehlt in einer Stimme → neuer Versuch
|
||||
continue
|
||||
bestaetigt = []
|
||||
for b in je_stimme[0]: # nur DOPPELT bestätigte Befunde zählen (Lektion 15)
|
||||
for anderes in je_stimme[1:]:
|
||||
if any(b["art"] == b2.get("art") and textkit.jaccard(
|
||||
b.get("stelle", ""), b2.get("stelle", ""),
|
||||
gestemmt=False) >= 0.3 for b2 in anderes):
|
||||
bestaetigt.append(b)
|
||||
break
|
||||
for b in bestaetigt:
|
||||
befund_merken(task["run_id"], "llm_pruefung", b["art"],
|
||||
f"baustein:{bid}",
|
||||
f"{b.get('stelle', '')[:150]} — {b.get('detail', '')[:200]}",
|
||||
task["runde"])
|
||||
db.update("sections", "baustein_id=?", (bid,), qa_hash=hash_)
|
||||
offen = db.query("SELECT id FROM befunde WHERE stufe='guide' AND item=? "
|
||||
"AND status='offen'", f"baustein:{bid}")
|
||||
if offen and (sec["fix_versuche"] or 0) < config.FIX_MAX_VERSUCHE:
|
||||
neue.append({"knoten": "fix", "item": f"r{task['runde']}:fix:{bid}",
|
||||
"payload": {"baustein_id": bid}})
|
||||
teil[m["id"]] = "fertig"
|
||||
gesamt += len(bestaetigt)
|
||||
return engine.Ergebnis(daten={"befunde": gesamt}, neue_tasks=neue,
|
||||
teil_status=teil)
|
||||
|
||||
|
||||
@engine.worker("guide.fix")
|
||||
async def fix(task: dict) -> engine.Ergebnis:
|
||||
payload = db.uj(task["payload"], {})
|
||||
bid = payload["baustein_id"]
|
||||
sec = db.one("SELECT * FROM sections WHERE baustein_id=?", bid)
|
||||
offen = db.query("SELECT * FROM befunde WHERE item=? AND status='offen'",
|
||||
f"baustein:{bid}")
|
||||
if sec is None or not offen:
|
||||
return engine.Ergebnis(daten={"skip": True})
|
||||
|
||||
if task["art"] == "klaerung": # 3er-Stichentscheid, je Befund genau einmal
|
||||
return await _klaerung(task, bid, sec, offen)
|
||||
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="guide", knoten="fix", item=task["item"],
|
||||
role="writer",
|
||||
skill_namen=["autor", "deutsch-praezise", "guide-lesbarkeit", "guide-fix"],
|
||||
werte={"section": sec["text"],
|
||||
"befunde": "\n".join(f"- [{b['art']}] {b['detail']}" for b in offen),
|
||||
"belege": belege.fenster_fuer_atome(
|
||||
[a["id"] for a in _atome(bid)])[:30000]})
|
||||
if antwort is None:
|
||||
raise RuntimeError("fix ohne Ergebnis")
|
||||
text = textkit.block_text(antwort, "TEXT")
|
||||
alte_marker = sorted(_MARKER_RE.findall(sec["text"]))
|
||||
if not text or sorted(_MARKER_RE.findall(text)) != alte_marker:
|
||||
# Fix, der die Marker-Invariante bricht, wird verworfen
|
||||
db.update("sections", "baustein_id=?", (bid,),
|
||||
fix_versuche=(sec["fix_versuche"] or 0) + 1)
|
||||
return engine.Ergebnis(daten={"verworfen": True})
|
||||
db.update("sections", "baustein_id=?", (bid,), text=text, qa_hash="",
|
||||
fix_versuche=(sec["fix_versuche"] or 0) + 1, stage="fix")
|
||||
for b in offen:
|
||||
db.update("befunde", "id=?", (b["id"],), status="behoben")
|
||||
return engine.Ergebnis(daten={"gefixt": len(offen)})
|
||||
|
||||
|
||||
async def _klaerung(task, bid, sec, offen) -> engine.Ergebnis:
|
||||
beleg_text = belege.fenster_fuer_atome([a["id"] for a in _atome(bid)])[:20000]
|
||||
for b in offen:
|
||||
if b["geklaert"]:
|
||||
continue
|
||||
|
||||
async def ruf(i, b=b):
|
||||
text = await llm.call(
|
||||
run_id=task["run_id"], stufe="guide", knoten="fix",
|
||||
item=f"{task['item']}:k{b['id']}:p{i}", role="judge",
|
||||
skill_namen=["richter", "deutsch-praezise", "klaerung"],
|
||||
werte={"befund": f"[{b['art']}] {b['detail']}", "belege": beleg_text})
|
||||
urteile = textkit.bloecke(text or "", "URTEIL")
|
||||
return urteile[0].get("urteil") if urteile else None
|
||||
|
||||
stimmen = await panels.panel(ruf, config.KLAERUNG_PANEL)
|
||||
ist_mangel = panels.mehrheit(
|
||||
stimmen, lambda s: True if s == "mangel"
|
||||
else (False if s == "kein_mangel" else None))
|
||||
db.update("befunde", "id=?", (b["id"],), geklaert=1,
|
||||
status="offen" if ist_mangel else "freispruch")
|
||||
return engine.Ergebnis(daten={"geklaert": len(offen)})
|
||||
|
||||
|
||||
@engine.worker("guide.kapitel_intro")
|
||||
async def kapitel_intro(task: dict) -> engine.Ergebnis:
|
||||
payload = db.uj(task["payload"], {})
|
||||
kap = db.one("SELECT * FROM kapitel WHERE id=?", payload["kapitel_id"])
|
||||
if kap is None:
|
||||
return engine.Ergebnis(daten={"skip": True})
|
||||
if kap["intro"]:
|
||||
return engine.Ergebnis(daten={"skip": True},
|
||||
neue_tasks=[{"knoten": "redundanz",
|
||||
"item": f"r{task['runde']}:redundanz"}])
|
||||
titel = db.query("SELECT titel FROM bausteine WHERE kapitel_id=? ORDER BY ord",
|
||||
kap["id"])
|
||||
vorherige = db.query("SELECT titel FROM kapitel WHERE topic=? AND ord<? "
|
||||
"ORDER BY ord", task["topic"], kap["ord"])
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="guide", knoten="kapitel_intro",
|
||||
item=task["item"], role="writer",
|
||||
skill_namen=["autor", "deutsch-praezise", "kapitel-intro"],
|
||||
werte={"kapitel": kap["titel"],
|
||||
"sections": "\n".join(f"- {t['titel']}" for t in titel),
|
||||
"vorherige": "\n".join(f"- {v['titel']}" for v in vorherige)
|
||||
or "- (keine)"})
|
||||
intro = textkit.block_text(antwort or "", "INTRO")
|
||||
if not intro: # Ausfall/Format kaputt: neuer Versuch statt leerem Intro
|
||||
raise RuntimeError("kapitel_intro ohne INTRO-Block")
|
||||
db.update("kapitel", "id=?", (kap["id"],), intro=intro[:800])
|
||||
return engine.Ergebnis(daten={"intro": True}, neue_tasks=[
|
||||
{"knoten": "redundanz", "item": f"r{task['runde']}:redundanz"}])
|
||||
|
||||
|
||||
def befund_merken(run_id: int, knoten: str, art: str, item: str, detail: str,
|
||||
runde: int) -> bool:
|
||||
"""Befund-Dedupe (creator2-Schlüssel art+item+detail): existiert derselbe
|
||||
Befund offen oder final (eskaliert/freispruch), KEINE neue Zeile — sonst
|
||||
startet die Eskalations-Kette (geklaert-Flag!) jede Runde von vorn."""
|
||||
kern = detail[:400]
|
||||
if db.one("SELECT id FROM befunde WHERE art=? AND item=? AND "
|
||||
"substr(detail,1,400)=? AND status IN ('offen','eskaliert',"
|
||||
"'freispruch')", art, item, kern):
|
||||
return False
|
||||
db.insert("befunde", run_id=run_id, stufe="guide", knoten=knoten, art=art,
|
||||
item=item, detail=detail[:500], runde=runde)
|
||||
return True
|
||||
|
||||
|
||||
def _rewrites(topic: str, bid) -> int:
|
||||
return db.one("SELECT COUNT(*) c FROM tasks WHERE topic=? AND knoten='writer' "
|
||||
"AND art='reparatur' AND item LIKE ?", topic,
|
||||
f"%:baustein:{bid}")["c"]
|
||||
|
||||
|
||||
@engine.worker("guide.gate")
|
||||
async def gate(task: dict) -> engine.Ergebnis:
|
||||
topic, runde = task["topic"], task["runde"]
|
||||
befunde, neue = [], []
|
||||
bausteine = db.query("SELECT b.id FROM bausteine b WHERE b.topic=?", topic)
|
||||
# Final entschiedene det-Befunde (Klärung/Eskalation) blocken nicht erneut
|
||||
final = {(r["art"], r["item"], r["detail"][:400]) for r in db.query(
|
||||
"SELECT art, item, detail FROM befunde WHERE status IN "
|
||||
"('freispruch','eskaliert') AND run_id IN (SELECT id FROM runs WHERE "
|
||||
"topic=?)", topic)}
|
||||
alle_marker: list[int] = []
|
||||
for b in bausteine:
|
||||
sec = db.one("SELECT * FROM sections WHERE baustein_id=?", b["id"])
|
||||
if sec is None or not sec["text"]:
|
||||
befunde.append({"art": "section_fehlt", "item": f"baustein:{b['id']}",
|
||||
"detail": ""})
|
||||
continue
|
||||
alle_marker += [int(m) for m in _MARKER_RE.findall(sec["text"])]
|
||||
for f in _det_befunde(topic, b["id"], sec["text"]):
|
||||
if f["art"] == "laenge": # Länge blockiert nicht hart (nur Befund)
|
||||
continue
|
||||
if (f["art"], f["item"], f["detail"][:400]) in final:
|
||||
continue
|
||||
befunde.append(f)
|
||||
for m in {m for m in alle_marker if alle_marker.count(m) > 1}:
|
||||
befunde.append({"art": "marker_global_doppelt", "item": f"atom:{m}",
|
||||
"detail": "Atom in mehreren Sections gelehrt"})
|
||||
# Kaputte Marker mit erschöpftem Fix → Section NEU schreiben (max 1×!
|
||||
# sonst resettet jedes Neuschreiben den Fix-Cap → Endlos-Karussell)
|
||||
for f in [f for f in befunde if f["art"].startswith("marker")]:
|
||||
bid = f["item"].split(":")[-1]
|
||||
sec = db.one("SELECT fix_versuche FROM sections WHERE baustein_id=?", bid)
|
||||
if sec and (sec["fix_versuche"] or 0) >= config.FIX_MAX_VERSUCHE \
|
||||
and _rewrites(topic, bid) < 1:
|
||||
neue.append({"knoten": "writer", "item": f"r{runde + 1}:baustein:{bid}",
|
||||
"art": "reparatur", "runde": runde + 1,
|
||||
"payload": {"baustein_id": int(bid), "neu": True}})
|
||||
# Inhaltliche Befunde (LLM/Redundanz): Fix, danach 3er-Klärung
|
||||
offene = db.query(
|
||||
"SELECT * FROM befunde b WHERE b.run_id IN (SELECT id FROM runs WHERE "
|
||||
"topic=?) AND b.stufe='guide' AND b.status='offen' AND b.art IN "
|
||||
"('falsch','luecke','redundanz','vorwaerts','format_leak')", topic)
|
||||
for b in offene:
|
||||
befunde.append({"art": b["art"], "item": b["item"],
|
||||
"detail": b["detail"][:120]})
|
||||
bid = b["item"].split(":")[-1]
|
||||
if not bid.isdigit():
|
||||
continue
|
||||
sec = db.one("SELECT fix_versuche FROM sections WHERE baustein_id=?", bid)
|
||||
versuche = (sec or {}).get("fix_versuche") or 0
|
||||
if versuche < config.FIX_MAX_VERSUCHE:
|
||||
neue.append({"knoten": "fix", "item": f"r{runde + 1}:fix:{bid}",
|
||||
"art": "reparatur", "runde": runde + 1,
|
||||
"payload": {"baustein_id": int(bid)}})
|
||||
elif not b["geklaert"]:
|
||||
neue.append({"knoten": "fix", "item": f"r{runde + 1}:klaerung:{bid}",
|
||||
"art": "klaerung", "runde": runde + 1,
|
||||
"payload": {"baustein_id": int(bid)}})
|
||||
elif _rewrites(topic, bid) < 1:
|
||||
neue.append({"knoten": "writer", "item": f"r{runde + 1}:baustein:{bid}",
|
||||
"art": "reparatur", "runde": runde + 1,
|
||||
"payload": {"baustein_id": int(bid), "neu": True}})
|
||||
else:
|
||||
# Exit nach Fix-Cap + 3er-Klärung + 1 Neuschreiben: eskalieren
|
||||
# statt ewig blocken — 3 Instanzen haben geprüft, Rest ist
|
||||
# dokumentierte Grenze (sichtbar in den Kennzahlen)
|
||||
db.update("befunde", "id=?", (b["id"],), status="eskaliert")
|
||||
befunde.pop() # zählt nicht mehr als gate-blockierend
|
||||
if befunde:
|
||||
# Re-Prüfung der gefixten Sections in der nächsten Runde
|
||||
for bid in {b["item"].split(":")[-1] for b in befunde
|
||||
if b["item"].startswith("baustein:")}:
|
||||
if bid.isdigit():
|
||||
neue.append({"knoten": "det_pruefung",
|
||||
"item": f"r{runde + 1}:det:{bid}", "art": "reparatur",
|
||||
"runde": runde + 1, "payload": {"baustein_id": int(bid)}})
|
||||
neue.append({"knoten": "redundanz", "item": f"r{runde + 1}:redundanz",
|
||||
"art": "reparatur", "runde": runde + 1})
|
||||
return engine.Ergebnis(gate_status="rot", befunde=befunde, neue_tasks=neue)
|
||||
|
||||
db.update("topics", "name=?", (topic,), status="guide_fertig")
|
||||
db.execute("UPDATE sections SET stage='fertig' WHERE baustein_id IN "
|
||||
"(SELECT id FROM bausteine WHERE topic=?)", topic)
|
||||
return engine.Ergebnis(gate_status="gruen", neue_tasks=[
|
||||
{"knoten": "montage", "item": "montage"}])
|
||||
518
backend/inventar.py
Normal file
518
backend/inventar.py
Normal file
@@ -0,0 +1,518 @@
|
||||
"""Stufe 2: Atome extrahieren (2 Reader, Anker-Pflicht), Anker-Fix
|
||||
(deterministisch vor LLM), Soll-Zuordnung, Lücken schließen (Nachextraktion →
|
||||
Web-Nachrecherche → Freispruch-Panel). Gate zählt Coverage deterministisch."""
|
||||
from . import config, db, engine, laden, llm, panels, textkit
|
||||
|
||||
|
||||
def _quelle(qid: int) -> dict:
|
||||
return db.one("SELECT * FROM quellen WHERE id=?", qid)
|
||||
|
||||
|
||||
def _snapshot(qid: int) -> str:
|
||||
return laden.snapshot_lesen(_quelle(qid))
|
||||
|
||||
|
||||
def _halbieren(m: dict, p: dict, run_id: int) -> list:
|
||||
"""Output-Cap: Chunk halbieren statt Retry (Lektion!); Tiefe gedeckelt."""
|
||||
tiefe = p.get("tiefe", 0)
|
||||
if tiefe >= config.CHUNK_HALBIER_TIEFE:
|
||||
db.insert("befunde", run_id=run_id, stufe="inventar",
|
||||
knoten="atom_extraktion", art="quelle_unvollstaendig",
|
||||
item=m["item"], detail="Cap trotz Halbierung")
|
||||
return []
|
||||
halb = p["chars"] // 2
|
||||
return [{"knoten": "atom_extraktion", "item": f"{m['item']}:h{i}",
|
||||
"payload": {**p, "offset": p["offset"] + i * halb,
|
||||
"chars": halb + (p["chars"] % 2 if i else 0),
|
||||
"tiefe": tiefe + 1}}
|
||||
for i in (0, 1)]
|
||||
|
||||
|
||||
@engine.worker("inventar.atom_extraktion", buendel=True)
|
||||
async def atom_extraktion(task: dict) -> engine.Ergebnis:
|
||||
mitglieder = [task] + task.get("_buendel", [])
|
||||
teil, aktive = {}, {} # idx → (mitglied, payload, chunk)
|
||||
idx = 0
|
||||
for m in mitglieder:
|
||||
p = db.uj(m["payload"], {})
|
||||
# Resume-Guard: Atome dieses Tasks (Item als Herkunft) nicht doppeln
|
||||
if db.one("SELECT id FROM atome WHERE topic=? AND typ LIKE ? LIMIT 1",
|
||||
task["topic"], f"%|{m['item']}"):
|
||||
teil[m["id"]] = "fertig"
|
||||
continue
|
||||
text = _snapshot(p["quelle_id"])
|
||||
idx += 1
|
||||
aktive[idx] = (m, p, text[p["offset"]:p["offset"] + p["chars"]])
|
||||
if not aktive:
|
||||
return engine.Ergebnis(daten={"skip": True}, teil_status=teil)
|
||||
thema = db.one("SELECT COALESCE(NULLIF(titel,''), name) t FROM topics "
|
||||
"WHERE name=?", task["topic"])["t"]
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="inventar", knoten="atom_extraktion",
|
||||
item=task["item"], role="extraktion", n=len(aktive),
|
||||
skill_namen=["extraktor", "deutsch-praezise", "atom-extraktion"],
|
||||
werte={"thema": thema,
|
||||
"abschnitte": "\n\n".join(f"=== ABSCHNITT {i} ===\n{c}"
|
||||
for i, (_, _, c) in aktive.items())})
|
||||
if antwort is None: # Output-Cap: alle Mitglieder halbieren, keiner verloren
|
||||
neue = []
|
||||
for i, (m, p, _) in aktive.items():
|
||||
neue += _halbieren(m, p, task["run_id"])
|
||||
teil[m["id"]] = "fertig"
|
||||
return engine.Ergebnis(daten={"halbiert": True}, neue_tasks=neue,
|
||||
teil_status=teil)
|
||||
je_abschnitt: dict[int, list] = {}
|
||||
for block in textkit.bloecke(antwort, "ATOM"):
|
||||
a = block.get("abschnitt", "").strip()
|
||||
if a.isdigit():
|
||||
je_abschnitt.setdefault(int(a), []).append(block)
|
||||
leer = {int(b["abschnitt"]) for b in textkit.bloecke(antwort, "LEER")
|
||||
if b.get("abschnitt", "").strip().isdigit()}
|
||||
n_gesamt, neue_quellen = 0, []
|
||||
for i, (m, p, chunk) in aktive.items():
|
||||
if i not in je_abschnitt and i not in leer:
|
||||
teil[m["id"]] = "neu" # Abschnitt fehlt in der Antwort → neuer Versuch
|
||||
continue
|
||||
for block in je_abschnitt.get(i, []):
|
||||
titel, zitat = block.get("titel", ""), block.get("zitat", "")
|
||||
if not titel or not zitat:
|
||||
continue
|
||||
span = textkit.finde_zitat(chunk, zitat)
|
||||
atom_id = db.insert(
|
||||
"atome", topic=task["topic"], titel=titel[:120],
|
||||
typ=f"{block.get('typ', 'begriff').split()[0]}|{m['item']}",
|
||||
definition=block.get("definition", "")[:600], status="neu",
|
||||
soll_id=p.get("soll_id"))
|
||||
start, ende = (p["offset"] + span[0], p["offset"] + span[1]) \
|
||||
if span else (-1, -1)
|
||||
# Zitat = exakter Quell-Slice (byte-wörtlich), nicht das LLM-Echo
|
||||
db.insert("anker", atom_id=atom_id, quelle_id=p["quelle_id"],
|
||||
start=start, ende=ende,
|
||||
zitat=(chunk[span[0]:span[1]][:600] if span else zitat[:600]))
|
||||
n_gesamt += 1
|
||||
teil[m["id"]] = "fertig"
|
||||
if p["quelle_id"] not in neue_quellen:
|
||||
neue_quellen.append(p["quelle_id"])
|
||||
return engine.Ergebnis(daten={"atome": n_gesamt}, teil_status=teil, neue_tasks=[
|
||||
{"knoten": "anker_fix", "item": f"quelle:{qid}:r{task['runde']}",
|
||||
"payload": {"quelle_id": qid}} for qid in neue_quellen])
|
||||
|
||||
|
||||
@engine.worker("inventar.anker_fix")
|
||||
async def anker_fix(task: dict) -> engine.Ergebnis:
|
||||
payload = db.uj(task["payload"], {})
|
||||
qid = payload["quelle_id"]
|
||||
text = _snapshot(qid)
|
||||
offen = db.query(
|
||||
"SELECT a.id atom_id, a.titel, an.id anker_id, an.zitat FROM atome a "
|
||||
"JOIN anker an ON an.atom_id=a.id AND an.quelle_id=? AND an.start=-1 "
|
||||
"WHERE a.topic=? AND a.status IN ('neu','aktiv')", qid, task["topic"])
|
||||
det_fix = 0
|
||||
rest = []
|
||||
for z in offen: # Stufe 1: deterministisches Re-Match (0-Call-Fall häufig)
|
||||
span = textkit.finde_zitat(text, z["zitat"])
|
||||
if span:
|
||||
db.update("anker", "id=?", (z["anker_id"],), start=span[0], ende=span[1])
|
||||
det_fix += 1
|
||||
else:
|
||||
rest.append(z)
|
||||
llm_fix = verworfen = 0
|
||||
for i in range(0, len(rest), config.ANKER_FIX_BATCH):
|
||||
batch = rest[i:i + config.ANKER_FIX_BATCH]
|
||||
liste = "\n".join(f"[{z['atom_id']}] {z['titel']}: {z['zitat'][:200]}"
|
||||
for z in batch)
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="inventar", knoten="anker_fix",
|
||||
item=f"{task['item']}:b{i}", role="extraktion", n=len(batch),
|
||||
skill_namen=["extraktor", "deutsch-praezise", "anker-fix"],
|
||||
werte={"atome": liste, "quelltext": text[:config.SOLL_CHUNK_CHARS]})
|
||||
if antwort is None: # Call-Ausfall: neuer Versuch statt stilles Auslassen
|
||||
raise RuntimeError("anker_fix ohne Ergebnis")
|
||||
antworten = {int(b["id"]): b.get("zitat", "") for b in
|
||||
textkit.bloecke(antwort, "FIX") if b.get("id", "").isdigit()}
|
||||
for z in batch:
|
||||
neu = antworten.get(z["atom_id"], "")
|
||||
span = textkit.finde_zitat(text, neu) if neu and neu != "VERWERFEN" else None
|
||||
if span:
|
||||
db.update("anker", "id=?", (z["anker_id"],),
|
||||
start=span[0], ende=span[1], zitat=neu[:600])
|
||||
llm_fix += 1
|
||||
elif neu == "VERWERFEN":
|
||||
db.update("atome", "id=?", (z["atom_id"],), status="verworfen")
|
||||
verworfen += 1
|
||||
# Verankerte sofort aktivieren — Dedup/Zuordnung sehen sonst nichts
|
||||
db.execute("UPDATE atome SET status='aktiv' WHERE topic=? AND status='neu' "
|
||||
"AND id IN (SELECT atom_id FROM anker WHERE quelle_id=? AND "
|
||||
"start>=0)", task["topic"], qid)
|
||||
return engine.Ergebnis(
|
||||
daten={"det": det_fix, "llm": llm_fix, "verworfen": verworfen},
|
||||
neue_tasks=[{"knoten": "dedup", "item": f"r{task['runde']}:dedup"}])
|
||||
|
||||
|
||||
@engine.worker("inventar.soll_zuordnung")
|
||||
async def soll_zuordnung(task: dict) -> engine.Ergebnis:
|
||||
payload = db.uj(task["payload"], {})
|
||||
atome = [a for i in payload["atom_ids"]
|
||||
if (a := db.one("SELECT * FROM atome WHERE id=? AND status='aktiv' "
|
||||
"AND soll_id IS NULL", i))]
|
||||
if not atome:
|
||||
return engine.Ergebnis(daten={"skip": True})
|
||||
soll = db.query("SELECT * FROM soll WHERE topic=? AND status='bestaetigt'",
|
||||
task["topic"])
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="inventar", knoten="soll_zuordnung",
|
||||
item=task["item"], role="judge", n=len(atome),
|
||||
skill_namen=["richter", "deutsch-praezise", "soll-zuordnung"],
|
||||
werte={"atome": "\n".join(f"[{a['id']}] {a['titel']}: {a['definition'][:150]}"
|
||||
for a in atome),
|
||||
"soll": "\n".join(f"[{s['id']}] {s['punkt']}" for s in soll)})
|
||||
if antwort is None:
|
||||
raise RuntimeError("soll_zuordnung ohne Ergebnis")
|
||||
soll_ids = {s["id"] for s in soll}
|
||||
n = 0
|
||||
for b in textkit.bloecke(antwort, "ZUORDNUNG"):
|
||||
if not b.get("atom", "").isdigit():
|
||||
continue
|
||||
ziel = b.get("soll", "")
|
||||
if ziel.isdigit() and int(ziel) in soll_ids:
|
||||
db.update("atome", "id=?", (int(b["atom"]),), soll_id=int(ziel))
|
||||
n += 1
|
||||
# ALLE Atome dieses Batches gelten als geprüft — auch die mit KEINER.
|
||||
# Sonst fragt jede Dedup-Runde dieselben Atome erneut (Endlosschleife).
|
||||
for a in atome:
|
||||
db.update("atome", "id=?", (a["id"],), soll_geprueft=1)
|
||||
return engine.Ergebnis(daten={"zugeordnet": n})
|
||||
|
||||
|
||||
async def _freispruch(task: dict, soll: dict) -> bool:
|
||||
auszuege = _beleg_fenster(soll)
|
||||
|
||||
async def ruf(i):
|
||||
text = await llm.call(
|
||||
run_id=task["run_id"], stufe="inventar", knoten="luecke",
|
||||
item=f"{task['item']}:p{i}", role="judge",
|
||||
skill_namen=["richter", "deutsch-praezise", "luecke-freispruch"],
|
||||
werte={"soll_punkt": soll["punkt"], "auszuege": auszuege[:20000]})
|
||||
urteile = textkit.bloecke(text or "", "URTEIL")
|
||||
return urteile[0].get("urteil", "") if urteile else None
|
||||
stimmen = await panels.panel(ruf, config.MERGE_PANEL)
|
||||
if panels.einstimmig(stimmen, config.MERGE_PANEL,
|
||||
lambda s: True if s == "unerfuellbar"
|
||||
else (False if s == "abgedeckt" else None)):
|
||||
db.update("soll", "id=?", (soll["id"],),
|
||||
freispruch=f"Panel einstimmig unerfuellbar (r{task['runde']})")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@engine.worker("inventar.luecke", buendel=True)
|
||||
async def luecke(task: dict) -> engine.Ergebnis:
|
||||
mitglieder = [task] + task.get("_buendel", [])
|
||||
teil, aktive = {}, {} # soll_id → (mitglied, soll, fenster)
|
||||
for m in mitglieder:
|
||||
p = db.uj(m["payload"], {})
|
||||
soll = db.one("SELECT * FROM soll WHERE id=?", p["soll_id"])
|
||||
if soll is None:
|
||||
teil[m["id"]] = "fertig"
|
||||
continue
|
||||
if not p.get("fenster") and db.one(
|
||||
"SELECT id FROM atome WHERE soll_id=? AND status='aktiv'",
|
||||
soll["id"]):
|
||||
# (Facetten-Tasks haben ein Fenster und laufen auch bei gedecktem Punkt)
|
||||
teil[m["id"]] = "fertig" # inzwischen gedeckt
|
||||
continue
|
||||
if task["art"] == "freispruch":
|
||||
# Panel je Soll-Punkt — Bündel läuft sequenziell durch dieselbe Logik
|
||||
await _freispruch(m, soll)
|
||||
teil[m["id"]] = "fertig"
|
||||
continue
|
||||
# Nachextraktion: gezieltes Facetten-Fenster ODER ±6k um alle Belegstellen
|
||||
if p.get("fenster"):
|
||||
f = p["fenster"]
|
||||
q = _quelle(f["quelle_id"])
|
||||
fenster = laden.snapshot_lesen(q)[f["von"]:f["bis"]] if q else ""
|
||||
else:
|
||||
fenster = _beleg_fenster(soll)
|
||||
aktive[soll["id"]] = (m, soll, fenster[:30000])
|
||||
if not aktive:
|
||||
return engine.Ergebnis(daten={"skip": True}, teil_status=teil)
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="inventar", knoten="luecke",
|
||||
item=task["item"], role="extraktion", n=len(aktive),
|
||||
skill_namen=["extraktor", "deutsch-praezise", "luecke-nachextraktion"],
|
||||
werte={"luecken": "\n\n".join(
|
||||
f"=== LUECKE ===\nSOLL: {sid}\nPUNKT: {s['punkt']}\nFENSTER:\n{f}"
|
||||
for sid, (_, s, f) in aktive.items())})
|
||||
if antwort is None:
|
||||
raise RuntimeError("luecke ohne Ergebnis")
|
||||
je_soll: dict[int, list] = {}
|
||||
for block in textkit.bloecke(antwort, "ATOM"):
|
||||
s = block.get("soll", "").strip()
|
||||
if s.isdigit():
|
||||
je_soll.setdefault(int(s), []).append(block)
|
||||
nichts = {int(b["soll"]) for b in textkit.bloecke(antwort, "NICHTS")
|
||||
if b.get("soll", "").strip().isdigit()}
|
||||
gesamt = 0
|
||||
for sid, (m, soll, _) in aktive.items():
|
||||
if sid not in je_soll and sid not in nichts:
|
||||
teil[m["id"]] = "neu" # Soll-Punkt fehlt in der Antwort → neuer Versuch
|
||||
continue
|
||||
n = 0
|
||||
for block in je_soll.get(sid, []):
|
||||
zitat = block.get("zitat", "")
|
||||
for beleg in db.uj(soll["belege"], []):
|
||||
q = _quelle(beleg["quelle"])
|
||||
if q is None or q["status"] != "geladen":
|
||||
continue
|
||||
text_q = laden.snapshot_lesen(q)
|
||||
span = textkit.finde_zitat(text_q, zitat)
|
||||
if span:
|
||||
atom_id = db.insert(
|
||||
"atome", topic=task["topic"],
|
||||
titel=block.get("titel", "")[:120],
|
||||
typ=f"{block.get('typ', 'begriff').split()[0]}|{m['item']}",
|
||||
definition=block.get("definition", "")[:600],
|
||||
status="aktiv", soll_id=soll["id"])
|
||||
db.insert("anker", atom_id=atom_id, quelle_id=q["id"],
|
||||
start=span[0], ende=span[1],
|
||||
zitat=text_q[span[0]:span[1]][:600])
|
||||
n += 1
|
||||
break
|
||||
teil[m["id"]] = "fertig"
|
||||
gesamt += n
|
||||
return engine.Ergebnis(daten={"gefunden": gesamt}, teil_status=teil)
|
||||
|
||||
|
||||
@engine.worker("inventar.verdichtung")
|
||||
async def verdichtung(task: dict) -> engine.Ergebnis:
|
||||
"""Minimalität (Ziel 2): Atome eines Soll-Punkts in möglichst wenige
|
||||
lehrbare Einheiten falten. Fusion NUR im Schnitt beider Panel-Stimmen
|
||||
(einstimmig, Lektion 18); Anker wandern mit — Abdeckung bleibt."""
|
||||
from .dedup import _merge
|
||||
payload = db.uj(task["payload"], {})
|
||||
soll = db.one("SELECT * FROM soll WHERE id=?", payload["soll_id"])
|
||||
if soll is None or soll["verdichtet"]:
|
||||
return engine.Ergebnis(daten={"skip": True})
|
||||
atome = db.query("SELECT * FROM atome WHERE soll_id=? AND status='aktiv'",
|
||||
soll["id"])
|
||||
if len(atome) <= config.VERDICHTUNG_ZIEL:
|
||||
db.update("soll", "id=?", (soll["id"],), verdichtet=1)
|
||||
return engine.Ergebnis(daten={"skip": True})
|
||||
nach_id = {a["id"]: a for a in atome}
|
||||
liste = "\n".join(f"[{a['id']}] {a['titel']}: {a['definition'][:150]}"
|
||||
for a in atome)
|
||||
|
||||
async def ruf(i):
|
||||
text = await llm.call(
|
||||
run_id=task["run_id"], stufe="inventar", knoten="verdichtung",
|
||||
item=f"{task['item']}:p{i}", role="judge", n=len(atome),
|
||||
skill_namen=["richter", "deutsch-praezise", "verdichtung"],
|
||||
werte={"soll_punkt": soll["punkt"], "atome": liste,
|
||||
"ziel": config.VERDICHTUNG_ZIEL})
|
||||
if text is None:
|
||||
return None
|
||||
gruppen, gesehen = [], set()
|
||||
for block in textkit.bloecke(text, "EINHEIT"):
|
||||
ids = [int(x) for x in block.get("ids", "").replace(" ", "").split(",")
|
||||
if x.isdigit() and int(x) in nach_id and int(x) not in gesehen]
|
||||
if ids:
|
||||
gruppen.append(ids)
|
||||
gesehen |= set(ids)
|
||||
gruppen += [[a["id"]] for a in atome if a["id"] not in gesehen]
|
||||
return gruppen
|
||||
|
||||
stimmen = [s for s in await panels.panel(ruf, config.MERGE_PANEL) if s]
|
||||
if len(stimmen) < config.MERGE_PANEL:
|
||||
raise RuntimeError("verdichtung: Panel unvollständig") # fail-closed
|
||||
# Schnitt: nur Paare, die BEIDE Stimmen zusammen gruppieren
|
||||
def paare(gruppen):
|
||||
out = set()
|
||||
for g in gruppen:
|
||||
for i, a in enumerate(g):
|
||||
for b in g[i + 1:]:
|
||||
out.add(frozenset((a, b)))
|
||||
return out
|
||||
gemeinsam = paare(stimmen[0])
|
||||
for s in stimmen[1:]:
|
||||
gemeinsam &= paare(s)
|
||||
# Connected Components über die einstimmigen Paare
|
||||
eltern = {a["id"]: a["id"] for a in atome}
|
||||
def wurzel(x):
|
||||
while eltern[x] != x:
|
||||
eltern[x] = eltern[eltern[x]]
|
||||
x = eltern[x]
|
||||
return x
|
||||
for paar in gemeinsam:
|
||||
a, b = tuple(paar)
|
||||
eltern[wurzel(a)] = wurzel(b)
|
||||
gruppen: dict[int, list[int]] = {}
|
||||
for aid in eltern:
|
||||
gruppen.setdefault(wurzel(aid), []).append(aid)
|
||||
fusionen = 0
|
||||
for mitglieder in gruppen.values():
|
||||
if len(mitglieder) < 2:
|
||||
continue
|
||||
kopf = max(mitglieder, key=lambda i: len(nach_id[i]["definition"]))
|
||||
for m in mitglieder:
|
||||
if m == kopf:
|
||||
continue
|
||||
k = db.one("SELECT * FROM atome WHERE id=?", kopf)
|
||||
v = db.one("SELECT * FROM atome WHERE id=?", m)
|
||||
if k["status"] == "aktiv" and v["status"] == "aktiv":
|
||||
_merge(task["topic"], k, v)
|
||||
fusionen += 1
|
||||
db.update("soll", "id=?", (soll["id"],), verdichtet=1)
|
||||
uebrig = db.one("SELECT COUNT(*) c FROM atome WHERE soll_id=? AND "
|
||||
"status='aktiv'", soll["id"])["c"]
|
||||
return engine.Ergebnis(daten={"vorher": len(atome), "nachher": uebrig,
|
||||
"fusionen": fusionen})
|
||||
|
||||
|
||||
def _beleg_fenster(soll: dict) -> str:
|
||||
teile = []
|
||||
for beleg in db.uj(soll["belege"], []):
|
||||
q = _quelle(beleg["quelle"])
|
||||
if q is None or q["status"] != "geladen":
|
||||
continue
|
||||
text = laden.snapshot_lesen(q)
|
||||
span = textkit.finde_zitat(text, beleg["zitat"])
|
||||
mitte = span[0] if span else 0
|
||||
von = max(0, mitte - config.LUECKE_FENSTER // 2)
|
||||
teile.append(text[von:von + config.LUECKE_FENSTER])
|
||||
return "\n\n---\n\n".join(teile)
|
||||
|
||||
|
||||
@engine.worker("inventar.gate")
|
||||
async def gate(task: dict) -> engine.Ergebnis:
|
||||
topic, runde = task["topic"], task["runde"]
|
||||
befunde, neue = [], []
|
||||
|
||||
# (a) aktive/neue Atome ohne Anker → Reparatur anker_fix
|
||||
unverankert = db.query(
|
||||
"SELECT DISTINCT an.quelle_id q FROM atome a JOIN anker an ON an.atom_id=a.id "
|
||||
"WHERE a.topic=? AND a.status IN ('neu','aktiv') AND an.start=-1", topic)
|
||||
for z in unverankert:
|
||||
befunde.append({"art": "anker", "item": f"quelle:{z['q']}", "detail": ""})
|
||||
neue.append({"knoten": "anker_fix", "item": f"quelle:{z['q']}:fix:r{runde}",
|
||||
"art": "reparatur", "runde": runde + 1,
|
||||
"payload": {"quelle_id": z["q"]}})
|
||||
# Atome 'neu' → 'aktiv' (verankerte)
|
||||
db.execute("UPDATE atome SET status='aktiv' WHERE topic=? AND status='neu' AND "
|
||||
"id IN (SELECT atom_id FROM anker WHERE start>=0)", topic)
|
||||
db.execute("UPDATE atome SET status='verworfen' WHERE topic=? AND status='neu' "
|
||||
"AND id NOT IN (SELECT atom_id FROM anker WHERE start>=0)", topic)
|
||||
|
||||
# (b) neue Atome seit letztem Dedup → Dedup-Runde
|
||||
max_atom = (db.one("SELECT COALESCE(MAX(id),0) m FROM atome WHERE topic=? "
|
||||
"AND status='aktiv'", topic))["m"]
|
||||
dedup_task = db.one("SELECT id, ergebnis FROM tasks WHERE topic=? AND "
|
||||
"knoten='dedup' AND status='fertig' ORDER BY id DESC "
|
||||
"LIMIT 1", topic) or {"id": 0, "ergebnis": "{}"}
|
||||
letzter_dedup = db.uj(dedup_task["ergebnis"], {}).get("stand", 0)
|
||||
# Zuordnung NACH dem letzten Dedup → Gleicher-Soll-Kanal konnte noch nicht
|
||||
# feuern → eine Dedup-Runde nachlegen
|
||||
zuordnung_danach = db.one(
|
||||
"SELECT id FROM tasks WHERE topic=? AND knoten='soll_zuordnung' AND "
|
||||
"status='fertig' AND id>? AND ergebnis LIKE '%zugeordnet%' AND "
|
||||
"ergebnis NOT LIKE '%\"zugeordnet\": 0%' LIMIT 1",
|
||||
topic, dedup_task["id"])
|
||||
if max_atom > letzter_dedup or zuordnung_danach:
|
||||
# Item trägt den BEWEGLICHEN Zustand mit — sonst sieht der Stillstand-
|
||||
# Fingerprint echten Fortschritt (neue Zuordnungen) als identisch an
|
||||
befunde.append({"art": "dedup_ausstehend",
|
||||
"item": f"stand:{max_atom}:z{(zuordnung_danach or {}).get('id', 0)}",
|
||||
"detail": ""})
|
||||
neue.append({"knoten": "dedup", "item": f"g{runde + 1}:dedup",
|
||||
"art": "reparatur", "runde": runde + 1})
|
||||
|
||||
# (c) Coverage: jeder bestätigte Soll-Punkt braucht ≥1 aktives Atom oder Freispruch
|
||||
luecken = db.query(
|
||||
"SELECT s.* FROM soll s WHERE s.topic=? AND s.status='bestaetigt' AND "
|
||||
"s.freispruch='' AND NOT EXISTS (SELECT 1 FROM atome a WHERE "
|
||||
"a.soll_id=s.id AND a.status='aktiv')", topic)
|
||||
web_runden = db.one(
|
||||
"SELECT COUNT(DISTINCT item) c FROM tasks WHERE topic=? AND knoten='suche' "
|
||||
"AND item LIKE 'luecke:%'", topic)["c"]
|
||||
for s in luecken:
|
||||
befunde.append({"art": "luecke", "item": f"soll:{s['id']}",
|
||||
"detail": s["punkt"][:120]})
|
||||
bisher = db.one("SELECT COUNT(*) c FROM tasks WHERE topic=? AND "
|
||||
"knoten='luecke' AND item LIKE ? AND item NOT LIKE '%:frei:%'",
|
||||
topic, f"%:soll:{s['id']}")["c"]
|
||||
if bisher < config.LUECKEN_RUNDEN_MAX:
|
||||
neue.append({"knoten": "luecke", "item": f"r{runde + 1}:soll:{s['id']}",
|
||||
"art": "reparatur", "runde": runde + 1,
|
||||
"payload": {"soll_id": s["id"]}})
|
||||
elif web_runden < config.WEB_NACHRECHERCHE_RUNDEN * max(1, len(luecken)):
|
||||
# Web-Nachrecherche: Query GEGROUNDET aus Punkt + Zitat-Kern (R11)
|
||||
zitat = (db.uj(s["belege"], []) or [{}])[0].get("zitat", "")
|
||||
query = " ".join(sorted(textkit.tokens(s["punkt"]))[:4] +
|
||||
sorted(textkit.tokens(zitat))[:3])
|
||||
neue.append({"knoten": "suche", "item": f"luecke:{s['id']}:r{runde + 1}",
|
||||
"art": "ruecklauf", "runde": runde + 1,
|
||||
"payload": {"query": query, "zweck": f"luecke:{s['id']}"}})
|
||||
else:
|
||||
neue.append({"knoten": "luecke",
|
||||
"item": f"r{runde + 1}:frei:soll:{s['id']}",
|
||||
"art": "freispruch", "runde": runde + 1,
|
||||
"payload": {"soll_id": s["id"]}})
|
||||
|
||||
# (d) Beifang raus: geprüfte Atome ohne Soll-Bezug verwerfen — Abdeckung
|
||||
# misst sich am Soll; alles andere bläht nur (32-Bausteine-Lektion)
|
||||
db.execute("UPDATE atome SET status='verworfen' WHERE topic=? AND "
|
||||
"status='aktiv' AND soll_id IS NULL AND soll_geprueft=1", topic)
|
||||
|
||||
# (d2) Facetten HART: jede Beleg-Stelle braucht ein Atom in der Nähe
|
||||
# (Gutachten: Namensherkunft fehlte trotz 7 Belegen). Cap: 2 gezielte
|
||||
# Nachextraktionen je Stelle, dann eskaliert (dokumentierte Grenze).
|
||||
from . import belege as belege_mod
|
||||
final = {r["item"] for r in db.query(
|
||||
"SELECT item FROM befunde WHERE art='facette' AND status IN "
|
||||
"('eskaliert','freispruch') AND run_id IN (SELECT id FROM runs WHERE "
|
||||
"topic=?)", topic)}
|
||||
for st in belege_mod.unbedeckte_belege(topic):
|
||||
key = f"beleg:{st['quelle_id']}:{st['start']}:soll:{st['soll_id']}"
|
||||
if key in final:
|
||||
continue
|
||||
versuche = db.one("SELECT COUNT(*) c FROM tasks WHERE topic=? AND "
|
||||
"knoten='luecke' AND item LIKE ?",
|
||||
topic, f"%:{key}")["c"]
|
||||
if versuche >= config.LUECKEN_RUNDEN_MAX:
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="inventar",
|
||||
knoten="gate_inventar", art="facette", item=key,
|
||||
detail=f"unabgedeckt trotz Nachextraktion: {st['zitat']}",
|
||||
status="eskaliert", runde=runde)
|
||||
continue
|
||||
befunde.append({"art": "facette", "item": key,
|
||||
"detail": st["zitat"][:120]})
|
||||
von = max(0, st["start"] - config.LUECKE_FENSTER // 2)
|
||||
neue.append({"knoten": "luecke", "item": f"r{runde + 1}:{key}",
|
||||
"art": "reparatur", "runde": runde + 1,
|
||||
"payload": {"soll_id": st["soll_id"],
|
||||
"fenster": {"quelle_id": st["quelle_id"],
|
||||
"von": von,
|
||||
"bis": von + config.LUECKE_FENSTER}}})
|
||||
|
||||
# (e) Minimalität: dicke Soll-Punkte einmalig verdichten (Ziel 2)
|
||||
dicke = db.query(
|
||||
"SELECT s.id, COUNT(a.id) n FROM soll s JOIN atome a ON a.soll_id=s.id "
|
||||
"AND a.status='aktiv' WHERE s.topic=? AND s.status='bestaetigt' AND "
|
||||
"s.verdichtet=0 GROUP BY s.id HAVING n > ?",
|
||||
topic, config.VERDICHTUNG_ZIEL)
|
||||
for d in dicke:
|
||||
befunde.append({"art": "unverdichtet", "item": f"soll:{d['id']}",
|
||||
"detail": f"{d['n']} Atome"})
|
||||
neue.append({"knoten": "verdichtung", "item": f"r{runde + 1}:verd:soll:{d['id']}",
|
||||
"art": "reparatur", "runde": runde + 1,
|
||||
"payload": {"soll_id": d["id"]}})
|
||||
|
||||
if befunde:
|
||||
return engine.Ergebnis(gate_status="rot", befunde=befunde, neue_tasks=neue)
|
||||
|
||||
# grün → Lernziele je Soll-Punkt (+ Sammelziel für Atome ohne Soll)
|
||||
db.update("topics", "name=?", (topic,), status="inventar_fertig")
|
||||
neue = [{"knoten": "lernziele", "item": f"soll:{s['id']}",
|
||||
"payload": {"soll_id": s["id"]}}
|
||||
for s in db.query("SELECT id FROM soll WHERE topic=? AND "
|
||||
"status='bestaetigt'", topic)]
|
||||
return engine.Ergebnis(gate_status="gruen", neue_tasks=neue)
|
||||
232
backend/korpus.py
Normal file
232
backend/korpus.py
Normal file
@@ -0,0 +1,232 @@
|
||||
"""Stufe 1: Recherche-Planung → Soll-Extraktion (Zitat-Gate) → Konsens
|
||||
(Judge gruppiert NUR, der Code zählt ≥2 unabhängige Quellen) → hartes Gate
|
||||
mit Sättigungslogik (Nullrunde nur mit neuen Queries gültig, R11)."""
|
||||
import hashlib
|
||||
|
||||
from . import config, db, engine, laden, llm, textkit
|
||||
|
||||
|
||||
def _thema(topic: str) -> str:
|
||||
t = db.one("SELECT titel, name FROM topics WHERE name=?", topic)
|
||||
return (t.get("titel") or t["name"]) if t else topic
|
||||
|
||||
|
||||
@engine.worker("korpus.recherche_plan", buendel=True)
|
||||
async def recherche_plan(task: dict) -> engine.Ergebnis:
|
||||
mitglieder = [task] + task.get("_buendel", [])
|
||||
lens_von = {m["id"]: db.uj(m["payload"], {}).get("lens", "")
|
||||
for m in mitglieder}
|
||||
seed = []
|
||||
if task["runde"] == 1:
|
||||
# Wikipedia-Seed: kuratierter Grundstock ist Pflicht, nicht Suchglück
|
||||
# (Audit Run 7: wikipedia_de fehlte komplett) — einmalig, idempotent
|
||||
seed = [{"knoten": "suche", "item": "r1:wikiseed",
|
||||
"payload": {"seed": True}}]
|
||||
bisherige = [db.uj(t["ergebnis"], {}).get("query", "") for t in db.query(
|
||||
"SELECT ergebnis FROM tasks WHERE topic=? AND knoten='suche'", task["topic"])]
|
||||
text = await llm.call(
|
||||
run_id=task["run_id"], stufe="korpus", knoten="recherche_plan",
|
||||
item=task["item"], role="judge", n=len(mitglieder),
|
||||
skill_namen=["planer", "deutsch-praezise", "recherche-plan"],
|
||||
werte={"thema": _thema(task["topic"]),
|
||||
"lenses": "\n".join(f"- {l}" for l in lens_von.values()),
|
||||
"runde": task["runde"],
|
||||
"bisherige_queries": "\n".join(f"- {q}" for q in bisherige if q) or "-"})
|
||||
if text is None:
|
||||
raise RuntimeError("recherche_plan ohne Ergebnis")
|
||||
je_lens: dict[str, list] = {}
|
||||
for block in textkit.bloecke(text, "QUERY"):
|
||||
q = block.get("text", "").strip()
|
||||
if q:
|
||||
je_lens.setdefault(block.get("lens", "").strip(), []).append(q)
|
||||
neue, teil = [], {}
|
||||
for m in mitglieder:
|
||||
queries = je_lens.get(lens_von[m["id"]], [])[:config.QUERIES_JE_LENS]
|
||||
if not queries: # Lens fehlt in der Antwort → neuer Versuch, nie stumm
|
||||
teil[m["id"]] = "neu"
|
||||
continue
|
||||
teil[m["id"]] = "fertig"
|
||||
for q in queries:
|
||||
h = hashlib.sha256(q.encode()).hexdigest()[:8]
|
||||
neue.append({"knoten": "suche", "item": f"r{m['runde']}:q:{h}",
|
||||
"payload": {"query": q}})
|
||||
return engine.Ergebnis(daten={"queries": len(neue)},
|
||||
neue_tasks=seed + neue, teil_status=teil)
|
||||
|
||||
|
||||
@engine.worker("korpus.soll_extraktion", buendel=True)
|
||||
async def soll_extraktion(task: dict) -> engine.Ergebnis:
|
||||
mitglieder = [task] + task.get("_buendel", [])
|
||||
chunks = {} # idx → (mitglied, quelle, chunk)
|
||||
for i, m in enumerate(mitglieder, 1):
|
||||
p = db.uj(m["payload"], {})
|
||||
quelle = db.one("SELECT * FROM quellen WHERE id=?", p["quelle_id"])
|
||||
text = laden.snapshot_lesen(quelle)
|
||||
chunks[i] = (m, quelle, text[p["offset"]:p["offset"] + p["chars"]])
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="korpus", knoten="soll_extraktion",
|
||||
item=task["item"], role="extraktion", n=len(mitglieder),
|
||||
skill_namen=["extraktor", "deutsch-praezise", "soll-extraktion"],
|
||||
werte={"thema": _thema(task["topic"]),
|
||||
"abschnitte": "\n\n".join(f"=== ABSCHNITT {i} ===\n{c}"
|
||||
for i, (_, _, c) in chunks.items())})
|
||||
if antwort is None:
|
||||
raise RuntimeError("soll_extraktion ohne Ergebnis")
|
||||
je_abschnitt: dict[int, list] = {}
|
||||
for block in textkit.bloecke(antwort, "PUNKT"):
|
||||
a = block.get("abschnitt", "").strip()
|
||||
if a.isdigit():
|
||||
je_abschnitt.setdefault(int(a), []).append(block)
|
||||
leer = {int(b["abschnitt"]) for b in textkit.bloecke(antwort, "LEER")
|
||||
if b.get("abschnitt", "").strip().isdigit()}
|
||||
n_gesamt, teil = 0, {}
|
||||
for i, (m, quelle, chunk) in chunks.items():
|
||||
if i not in je_abschnitt and i not in leer:
|
||||
teil[m["id"]] = "neu" # Abschnitt fehlt in der Antwort → neuer Versuch
|
||||
continue
|
||||
# Resume-Idempotenz: Kandidaten dieses Chunks vorher räumen
|
||||
db.execute("DELETE FROM soll WHERE topic=? AND status='kandidat' AND "
|
||||
"belege LIKE ?", task["topic"], f'%"chunk": "{m["item"]}"%')
|
||||
for block in je_abschnitt.get(i, []):
|
||||
punkt, beleg = block.get("text", ""), block.get("beleg", "")
|
||||
# Zitat-Gate: Beleg muss wörtlich in der Quelle stehen
|
||||
if punkt and beleg and textkit.finde_zitat(chunk, beleg):
|
||||
db.insert("soll", topic=task["topic"], punkt=punkt[:300],
|
||||
belege=db.j([{"quelle": quelle["id"],
|
||||
"zitat": beleg[:500], "chunk": m["item"]}]))
|
||||
n_gesamt += 1
|
||||
teil[m["id"]] = "fertig"
|
||||
return engine.Ergebnis(
|
||||
daten={"punkte": n_gesamt}, teil_status=teil,
|
||||
neue_tasks=[{"knoten": "soll_konsens", "item": f"r{task['runde']}:konsens"}])
|
||||
|
||||
|
||||
@engine.worker("korpus.soll_konsens")
|
||||
async def soll_konsens(task: dict) -> engine.Ergebnis:
|
||||
"""Judge gruppiert, Code zählt. NEU (Lektion 90): Ziel-Band ≈ √Kandidaten —
|
||||
ohne Band bestätigte creator3 353 Mikro-Punkte für ein Mini-Thema."""
|
||||
bestaetigt = 0
|
||||
for _ in range(2): # 2. Pass faltet zu feine Ergebnisse nochmal
|
||||
punkte = db.query("SELECT * FROM soll WHERE topic=? AND status IN "
|
||||
"('kandidat','bestaetigt')", task["topic"])
|
||||
if not punkte:
|
||||
return engine.Ergebnis(daten={"bestaetigt": 0})
|
||||
ziel = max(config.SOLL_PUNKTE_MIN,
|
||||
round(len(punkte) ** 0.5 * config.SOLL_BAND_FAKTOR))
|
||||
liste = "\n".join(f"[{k['id']}] {k['punkt']}" for k in punkte)
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="korpus", knoten="soll_konsens",
|
||||
item=f"{task['item']}:n{len(punkte)}", role="judge", n=len(punkte),
|
||||
skill_namen=["richter", "deutsch-praezise", "soll-konsens"],
|
||||
werte={"punkte": liste, "ziel": ziel})
|
||||
if antwort is None:
|
||||
raise RuntimeError("soll_konsens ohne Ergebnis")
|
||||
bekannt = {k["id"]: k for k in punkte}
|
||||
bestaetigt = 0
|
||||
for gruppe in textkit.bloecke(antwort, "GRUPPE"):
|
||||
ids = [int(i) for i in gruppe.get("ids", "").replace(" ", "").split(",")
|
||||
if i.isdigit() and int(i) in bekannt]
|
||||
if not ids:
|
||||
continue
|
||||
belege = []
|
||||
for i in ids:
|
||||
belege += db.uj(bekannt[i]["belege"], [])
|
||||
quellen = {b["quelle"] for b in belege}
|
||||
# DER CODE zählt — ≥2 unabhängige Quellen ODER 1 kuratierte
|
||||
# (Wikipedia = vertrauenswürdig, creator2-uni-Prinzip; sonst
|
||||
# verlieren Einzelquellen-Facetten wie „Overflow-Pomodoros")
|
||||
kuratiert = db.one(
|
||||
"SELECT id FROM quellen WHERE id IN ({}) AND backend LIKE "
|
||||
"'wikipedia%'".format(",".join("?" * len(quellen))),
|
||||
*quellen) if quellen else None
|
||||
noetig = config.SOLL_MIN_BELEGE_WIKI if kuratiert \
|
||||
else config.SOLL_MIN_BELEGE
|
||||
if len(quellen) >= noetig:
|
||||
kopf = ids[0]
|
||||
db.update("soll", "id=?", (kopf,), status="bestaetigt",
|
||||
punkt=gruppe.get("text", bekannt[kopf]["punkt"])[:300],
|
||||
belege=db.j(belege))
|
||||
for i in ids[1:]:
|
||||
db.update("soll", "id=?", (i,), status="gefaltet")
|
||||
bestaetigt += 1
|
||||
if bestaetigt <= ziel * config.SOLL_BAND_TOLERANZ:
|
||||
break # im Band — fertig
|
||||
return engine.Ergebnis(daten={"bestaetigt": bestaetigt})
|
||||
|
||||
|
||||
@engine.worker("korpus.gate")
|
||||
async def gate(task: dict) -> engine.Ergebnis:
|
||||
topic, runde = task["topic"], task["runde"]
|
||||
quellen = db.one("SELECT COUNT(*) c FROM quellen WHERE topic=? AND "
|
||||
"status='geladen'", topic)["c"]
|
||||
bestaetigt = db.one("SELECT COUNT(*) c FROM soll WHERE topic=? AND "
|
||||
"status='bestaetigt'", topic)["c"]
|
||||
vorher = db.uj((db.one(
|
||||
"SELECT ergebnis FROM tasks WHERE topic=? AND knoten='gate_korpus' AND "
|
||||
"runde=? AND status='fertig'", topic, runde - 1) or {}).get("ergebnis", ""),
|
||||
{}).get("bestaetigt", -1)
|
||||
befunde, neue = [], []
|
||||
|
||||
if quellen == 0:
|
||||
befunde.append({"art": "korpus_leer", "item": topic,
|
||||
"detail": "keine geladene Quelle"})
|
||||
if bestaetigt < config.SOLL_PUNKTE_MIN:
|
||||
befunde.append({"art": "soll_zu_wenig", "item": topic,
|
||||
"detail": f"{bestaetigt} < {config.SOLL_PUNKTE_MIN}"})
|
||||
gesaettigt = bestaetigt == vorher # Nullrunde (mit frischen Queries gelaufen)
|
||||
if not gesaettigt and runde < config.RECHERCHE_RUNDEN_MAX:
|
||||
befunde.append({"art": "unsaettigt", "item": topic,
|
||||
"detail": f"Runde {runde}: {vorher}→{bestaetigt}"})
|
||||
|
||||
if befunde:
|
||||
if runde < config.RECHERCHE_RUNDEN_MAX:
|
||||
neue = [{"knoten": "recherche_plan", "item": f"r{runde + 1}:lens:{lens}",
|
||||
"runde": runde + 1, "art": "ruecklauf",
|
||||
"payload": {"lens": lens}}
|
||||
for lens in config.RECHERCHE_LENSES]
|
||||
return engine.Ergebnis(gate_status="rot", befunde=befunde,
|
||||
neue_tasks=neue,
|
||||
daten={"bestaetigt": bestaetigt})
|
||||
|
||||
# grün → Rest-Kandidaten sind entschieden (Sättigung): verwerfen statt
|
||||
# als unadjudizierter Ballast liegen zu lassen
|
||||
db.execute("UPDATE soll SET status='verworfen' WHERE topic=? AND "
|
||||
"status='kandidat'", topic)
|
||||
# Stufe inventar: Extraktion NUR belegnah (Lektion 92) — 2 Reader je
|
||||
# Fenster um die Soll-Belege. Quellen ohne Beleg liefern keine Atome.
|
||||
db.update("topics", "name=?", (topic,), status="korpus_fertig")
|
||||
belege_je_quelle: dict[int, list] = {}
|
||||
for s in db.query("SELECT belege FROM soll WHERE topic=? AND "
|
||||
"status='bestaetigt'", topic):
|
||||
for b in db.uj(s["belege"], []):
|
||||
belege_je_quelle.setdefault(b["quelle"], []).append(b["zitat"])
|
||||
for q in db.query("SELECT * FROM quellen WHERE topic=? AND status='geladen' "
|
||||
"AND zweck='korpus'", topic):
|
||||
zitate = belege_je_quelle.get(q["id"])
|
||||
if not zitate:
|
||||
continue
|
||||
text = laden.snapshot_lesen(q)
|
||||
spans = []
|
||||
for zitat in zitate:
|
||||
span = textkit.finde_zitat(text, zitat)
|
||||
if span:
|
||||
spans.append([max(0, span[0] - config.EXTRAKT_FENSTER),
|
||||
min(len(text), span[1] + config.EXTRAKT_FENSTER)])
|
||||
spans.sort()
|
||||
gemerged = []
|
||||
for s in spans:
|
||||
if gemerged and s[0] <= gemerged[-1][1]:
|
||||
gemerged[-1][1] = max(gemerged[-1][1], s[1])
|
||||
else:
|
||||
gemerged.append(s)
|
||||
i = 0
|
||||
for von, bis in gemerged:
|
||||
for o, c in textkit.abschnitte(text[von:bis]):
|
||||
for r in range(1, config.READER_JE_ABSCHNITT + 1):
|
||||
neue.append({"knoten": "atom_extraktion",
|
||||
"item": f"quelle:{q['id']}:a{i}:r{r}",
|
||||
"payload": {"quelle_id": q["id"],
|
||||
"offset": von + o, "chars": len(c)}})
|
||||
i += 1
|
||||
return engine.Ergebnis(gate_status="gruen", neue_tasks=neue,
|
||||
daten={"bestaetigt": bestaetigt})
|
||||
203
backend/laden.py
Normal file
203
backend/laden.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""Ingestion (R6): Bytes → trafilatura(markdown) → ftfy NFC → Snapshot.
|
||||
Normalisierung passiert GENAU EINMAL, danach ist der Snapshot unveränderlich —
|
||||
alle Verbatim-Zitate stammen ausschließlich aus dem Snapshot."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.robotparser
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from . import config, db, engine
|
||||
|
||||
_robots: dict[str, urllib.robotparser.RobotFileParser | None] = {}
|
||||
_letzter_fetch: dict[str, float] = {}
|
||||
|
||||
|
||||
def _text_reparieren(text: str) -> str:
|
||||
text = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", " ", text) # VOR ftfy
|
||||
try:
|
||||
import ftfy
|
||||
return ftfy.fix_text(text, normalization="NFC")
|
||||
except ImportError:
|
||||
import unicodedata
|
||||
return unicodedata.normalize("NFC", text)
|
||||
|
||||
|
||||
def _robots_erlaubt(url: str) -> bool:
|
||||
domain = urlsplit(url).netloc
|
||||
if domain not in _robots:
|
||||
rp = urllib.robotparser.RobotFileParser()
|
||||
try:
|
||||
rp.set_url(f"https://{domain}/robots.txt")
|
||||
rp.read()
|
||||
_robots[domain] = rp
|
||||
except Exception:
|
||||
_robots[domain] = None # nicht erreichbar → fail-open
|
||||
rp = _robots[domain]
|
||||
return rp is None or rp.can_fetch(config.user_agent(), url)
|
||||
|
||||
|
||||
async def _domain_bremse(url: str) -> None:
|
||||
domain = urlsplit(url).netloc
|
||||
delta = time.monotonic() - _letzter_fetch.get(domain, 0)
|
||||
if delta < config.DOMAIN_RATE_S:
|
||||
await asyncio.sleep(config.DOMAIN_RATE_S - delta)
|
||||
_letzter_fetch[domain] = time.monotonic()
|
||||
|
||||
|
||||
_WIKI_RE = re.compile(r"https?://([a-z]{2})\.wikipedia\.org/wiki/(.+)$")
|
||||
|
||||
|
||||
async def _wikipedia_api(url: str) -> str:
|
||||
"""Wikipedia über die Action-API laden (bot-freundlich; die HTML-Seiten
|
||||
blockierten per robots.txt die Kernquelle des ersten Echtlaufs)."""
|
||||
m = _WIKI_RE.match(url)
|
||||
if not m:
|
||||
return ""
|
||||
sprache, titel = m.group(1), m.group(2).split("#")[0]
|
||||
async with httpx.AsyncClient(
|
||||
timeout=config.FETCH_READ_S,
|
||||
headers={"User-Agent": config.user_agent()}) as client:
|
||||
r = await client.get(
|
||||
f"https://{sprache}.wikipedia.org/w/api.php",
|
||||
params={"action": "query", "prop": "extracts", "explaintext": 1,
|
||||
"format": "json", "redirects": 1, "titles": titel})
|
||||
r.raise_for_status()
|
||||
seiten = r.json().get("query", {}).get("pages", {})
|
||||
return "\n\n".join(p.get("extract", "") for p in seiten.values())
|
||||
|
||||
|
||||
async def _fetch(url: str) -> tuple[bytes, str]:
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=True,
|
||||
timeout=httpx.Timeout(config.FETCH_READ_S, connect=config.FETCH_CONNECT_S),
|
||||
headers={"User-Agent": config.user_agent()},
|
||||
transport=httpx.AsyncHTTPTransport(retries=2)) as client:
|
||||
r = await client.get(url)
|
||||
r.raise_for_status()
|
||||
return r.content, r.headers.get("content-type", "")
|
||||
|
||||
|
||||
def _extrahieren(inhalt: bytes) -> str:
|
||||
import trafilatura
|
||||
return trafilatura.extract(inhalt, output_format="markdown",
|
||||
include_tables=True, include_links=False,
|
||||
include_images=False) or ""
|
||||
|
||||
|
||||
def snapshot_pfad(topic: str, hash_: str) -> str:
|
||||
return str(config.KORPUS_DIR / topic / f"q{hash_}.md")
|
||||
|
||||
|
||||
def snapshot_lesen(quelle: dict) -> str:
|
||||
from pathlib import Path
|
||||
return Path(quelle["snapshot"]).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _atomar_schreiben(pfad: str, daten: bytes) -> None:
|
||||
os.makedirs(os.path.dirname(pfad), exist_ok=True)
|
||||
tmp = pfad + ".tmp"
|
||||
with open(tmp, "wb") as f:
|
||||
f.write(daten)
|
||||
os.replace(tmp, pfad)
|
||||
|
||||
|
||||
@engine.worker("laden.laden")
|
||||
async def laden(task: dict) -> engine.Ergebnis:
|
||||
quelle = db.one("SELECT * FROM quellen WHERE id=?",
|
||||
db.uj(task["payload"], {}).get("quelle_id"))
|
||||
if quelle is None:
|
||||
raise RuntimeError("Quelle fehlt")
|
||||
if quelle["status"] == "geladen": # Resume
|
||||
return engine.Ergebnis(daten={"skip": True},
|
||||
neue_tasks=_folge_tasks(task, quelle))
|
||||
|
||||
if config.FAKE:
|
||||
from . import fakes
|
||||
text = fakes.laden(quelle["url"])
|
||||
roh = b""
|
||||
else:
|
||||
if quelle["url"].lower().endswith(".pdf"):
|
||||
db.update("quellen", "id=?", (quelle["id"],), status="fehler",
|
||||
grund="pdf_uebersprungen")
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="korpus",
|
||||
knoten="laden", art="pdf", item=quelle["url"][:120],
|
||||
detail="PDF in v1 übersprungen")
|
||||
return engine.Ergebnis(daten={"pdf": True})
|
||||
if _WIKI_RE.match(quelle["url"]):
|
||||
try:
|
||||
text = await _wikipedia_api(quelle["url"])
|
||||
except Exception:
|
||||
text = ""
|
||||
if text:
|
||||
text = _text_reparieren(text)
|
||||
return _snapshot_ablegen(task, quelle, text, b"")
|
||||
# API-Fehler → normaler Weg (inkl. robots-Check) als Fallback
|
||||
if not _robots_erlaubt(quelle["url"]):
|
||||
db.update("quellen", "id=?", (quelle["id"],), status="robots")
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="korpus",
|
||||
knoten="laden", art="robots", item=quelle["url"][:120])
|
||||
return engine.Ergebnis(daten={"robots": True})
|
||||
await _domain_bremse(quelle["url"])
|
||||
try:
|
||||
roh, _ = await _fetch(quelle["url"])
|
||||
except Exception as e:
|
||||
db.update("quellen", "id=?", (quelle["id"],), status="fehler",
|
||||
grund=f"{type(e).__name__}"[:80])
|
||||
return engine.Ergebnis(daten={"fetch_fehler": str(e)[:120]})
|
||||
text = _extrahieren(roh)
|
||||
|
||||
text = _text_reparieren(text)
|
||||
return _snapshot_ablegen(task, quelle, text, roh)
|
||||
|
||||
|
||||
def _snapshot_ablegen(task: dict, quelle: dict, text: str,
|
||||
roh: bytes) -> engine.Ergebnis:
|
||||
if len(text) < config.SNAPSHOT_MIN_ZEICHEN or "enable javascript" in text.lower():
|
||||
status = "braucht_js" if text and "javascript" in text.lower() else "leer"
|
||||
db.update("quellen", "id=?", (quelle["id"],), status=status)
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="korpus", knoten="laden",
|
||||
art=status, item=quelle["url"][:120])
|
||||
return engine.Ergebnis(daten={status: True})
|
||||
|
||||
hash_ = hashlib.sha256(text.encode()).hexdigest()[:16]
|
||||
dublette = db.one("SELECT id FROM quellen WHERE topic=? AND hash=? AND id!=?",
|
||||
task["topic"], hash_, quelle["id"])
|
||||
if dublette:
|
||||
db.update("quellen", "id=?", (quelle["id"],), status="fehler",
|
||||
grund="inhalt_dublette")
|
||||
return engine.Ergebnis(daten={"dublette": True})
|
||||
pfad = snapshot_pfad(task["topic"], hash_)
|
||||
_atomar_schreiben(pfad, text.encode("utf-8"))
|
||||
roh_pfad = ""
|
||||
if roh:
|
||||
roh_pfad = pfad.replace(".md", ".html")
|
||||
_atomar_schreiben(roh_pfad, roh) # Re-Extraktion ohne Re-Fetch (R6)
|
||||
db.update("quellen", "id=?", (quelle["id"],), status="geladen", hash=hash_,
|
||||
snapshot=pfad, roh=roh_pfad)
|
||||
quelle = {**quelle, "snapshot": pfad, "status": "geladen"}
|
||||
return engine.Ergebnis(daten={"zeichen": len(text)},
|
||||
neue_tasks=_folge_tasks(task, quelle))
|
||||
|
||||
|
||||
def _folge_tasks(task: dict, quelle: dict) -> list:
|
||||
"""korpus-Quellen → Soll-Extraktion je 40k-Chunk; Lücken-Quellen →
|
||||
direkte Atom-Extraktion (Soll steht schon fest)."""
|
||||
from . import textkit
|
||||
if quelle["zweck"].startswith("luecke:"):
|
||||
soll_id = int(quelle["zweck"].split(":")[1])
|
||||
text = snapshot_lesen(quelle)
|
||||
return [{"knoten": "atom_extraktion",
|
||||
"item": f"quelle:{quelle['id']}:a{i}:r1",
|
||||
"payload": {"quelle_id": quelle["id"], "offset": o,
|
||||
"chars": len(c), "soll_id": soll_id}}
|
||||
for i, (o, c) in enumerate(textkit.abschnitte(text))]
|
||||
text = snapshot_lesen(quelle)
|
||||
chunks = textkit.abschnitte(text, config.SOLL_CHUNK_CHARS)
|
||||
return [{"knoten": "soll_extraktion", "item": f"quelle:{quelle['id']}:c{i}",
|
||||
"payload": {"quelle_id": quelle["id"], "offset": o, "chars": len(c)}}
|
||||
for i, (o, c) in enumerate(chunks)]
|
||||
52
backend/ledger.py
Normal file
52
backend/ledger.py
Normal file
@@ -0,0 +1,52 @@
|
||||
"""Event-Ledger: eine Zeile je LLM-Versuch (im finally — jeder Ausgang zählt,
|
||||
auch Hedge-Verlierer). Budget = harte Grenze, geprüft VOR und NACH jedem Call."""
|
||||
from . import config, db
|
||||
|
||||
|
||||
class BudgetErschoepft(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def log_call(run_id: int, *, stufe: str = "", knoten: str = "", item: str = "",
|
||||
skills_liste: list | None = None, skill_hash: str = "",
|
||||
model: str = "", role: str = "", status: str = "",
|
||||
dur_ms: int = 0, wait_ms: int = 0,
|
||||
tokens: dict | None = None, meta: dict | None = None,
|
||||
prompt: str = "", antwort: str = "") -> int | None:
|
||||
t = tokens or {}
|
||||
event_id = db.insert(
|
||||
"events", run_id=run_id, ts=db.now(), stufe=stufe, knoten=knoten,
|
||||
item=item, skills=db.j(skills_liste or []), skill_hash=skill_hash,
|
||||
model=model, role=role, status=status, dur_ms=dur_ms,
|
||||
wait_ms=wait_ms, tok_in=t.get("input", 0), tok_out=t.get("output", 0),
|
||||
tok_cache_read=t.get("cache_read", 0),
|
||||
tok_cache_write=t.get("cache_write", 0), meta=db.j(meta or {}))
|
||||
if config.LLM_LOG and event_id and (prompt or antwort):
|
||||
db.insert("call_texte", event_id=event_id, prompt=prompt, antwort=antwort)
|
||||
return event_id
|
||||
|
||||
|
||||
def verbraucht(run_id: int) -> int:
|
||||
r = db.one("SELECT COALESCE(SUM(tok_in + tok_out), 0) s FROM events "
|
||||
"WHERE run_id=?", run_id)
|
||||
return r["s"]
|
||||
|
||||
|
||||
def budget_pruefen(run_id: int) -> None:
|
||||
run = db.one("SELECT budget_tokens FROM runs WHERE id=?", run_id)
|
||||
limit = (run or {}).get("budget_tokens") or 0
|
||||
if limit > 0 and verbraucht(run_id) >= limit:
|
||||
raise BudgetErschoepft(f"Budget {limit} Tokens erreicht")
|
||||
|
||||
|
||||
def kennzahlen(run_id: int) -> dict:
|
||||
zeilen = db.query(
|
||||
"SELECT stufe, knoten, COUNT(*) calls, SUM(tok_in) tok_in, "
|
||||
"SUM(tok_out) tok_out, SUM(tok_cache_read) cache_read, "
|
||||
"SUM(dur_ms) dur_ms, SUM(wait_ms) wait_ms, "
|
||||
"SUM(status='ok') ok, SUM(status IN ('error','infra','parse','cap')) fehler "
|
||||
"FROM events WHERE run_id=? GROUP BY stufe, knoten ORDER BY MIN(id)", run_id)
|
||||
gesamt = verbraucht(run_id)
|
||||
kosten = sum(z["tok_in"] or 0 for z in zeilen) / 1e6 * config.PREIS_IN + \
|
||||
sum(z["tok_out"] or 0 for z in zeilen) / 1e6 * config.PREIS_OUT
|
||||
return {"zeilen": zeilen, "verbraucht": gesamt, "kosten_usd": round(kosten, 2)}
|
||||
238
backend/llm.py
Normal file
238
backend/llm.py
Normal file
@@ -0,0 +1,238 @@
|
||||
"""MiniMax-Client (R3/R9). Fehlerklassen strikt getrennt: Infra (429/Timeout)
|
||||
→ 3 Retries mit Backoff 8/16/32 s + Jitter, erschöpft → LaufPause (fail-closed).
|
||||
Inhalt (leer/kaputt) → begrenzte Restarts, dann None. cap (stop=max_tokens)
|
||||
→ SOFORT None, kein Retry (deterministisch — Aufrufer halbiert den Chunk).
|
||||
Ein 429 drosselt ALLE Calls (globaler Cooldown + adaptive Breite)."""
|
||||
import asyncio
|
||||
import random
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import httpx
|
||||
|
||||
from . import config, db, skills
|
||||
from .ledger import BudgetErschoepft, budget_pruefen, log_call # noqa: F401
|
||||
|
||||
_INFRA_MARKER = ("429", "rate limit", "rate_limit", "timeout", "timed out",
|
||||
"connection", "network", "overloaded", "unavailable",
|
||||
'"status_code":1002', '"status_code":1041', '"status_code":2045')
|
||||
|
||||
|
||||
class LaufPause(Exception):
|
||||
"""Infrastruktur erschöpft — Lauf pausieren, nie fail-open weiterrechnen."""
|
||||
|
||||
|
||||
class ManuellePause(LaufPause):
|
||||
"""Nutzer hat pausiert — Wartende brechen sofort ab, Laufende laufen aus."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ApiErgebnis:
|
||||
rc: int
|
||||
text: str = ""
|
||||
err: str = ""
|
||||
tokens: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
# ── globale Drossel (ein 429 bremst alle) ──
|
||||
_sem = asyncio.Semaphore(config.MAX_PARALLEL_LLM)
|
||||
_cooldown_bis = 0.0
|
||||
_breite = config.MAX_PARALLEL_LLM
|
||||
_inflight = 0
|
||||
_erfolge = 0
|
||||
|
||||
|
||||
def drossel_melden(sekunden: float) -> None:
|
||||
global _cooldown_bis, _breite, _erfolge
|
||||
_cooldown_bis = max(_cooldown_bis, time.monotonic() + sekunden)
|
||||
_breite = max(config.DROSSEL_MIN_BREITE, _breite // 2)
|
||||
_erfolge = 0
|
||||
|
||||
|
||||
def _erfolg_melden() -> None:
|
||||
global _breite, _erfolge
|
||||
_erfolge += 1
|
||||
if _erfolge >= config.DROSSEL_ERFOLGE_JE_PLUS and _breite < config.MAX_PARALLEL_LLM:
|
||||
_breite += 1
|
||||
_erfolge = 0
|
||||
|
||||
|
||||
_naechster_start = 0.0
|
||||
|
||||
|
||||
async def _slot() -> float:
|
||||
"""Cooldown abwarten, dann Takt (1 Start je TAKT_SEKUNDEN) ODER — bei
|
||||
Takt 0 — adaptive Breite. Rückgabe: Wartezeit in Sekunden."""
|
||||
global _inflight, _naechster_start
|
||||
t0 = time.monotonic()
|
||||
while time.monotonic() < _cooldown_bis:
|
||||
await asyncio.sleep(min(1.0, _cooldown_bis - time.monotonic()))
|
||||
if config.TAKT_SEKUNDEN > 0:
|
||||
while True:
|
||||
jetzt = time.monotonic()
|
||||
if jetzt >= _naechster_start:
|
||||
_naechster_start = max(jetzt, _naechster_start) + config.TAKT_SEKUNDEN
|
||||
break
|
||||
await asyncio.sleep(min(0.2, _naechster_start - jetzt))
|
||||
else:
|
||||
while _inflight >= _breite:
|
||||
await asyncio.sleep(0.2)
|
||||
_inflight += 1
|
||||
return time.monotonic() - t0
|
||||
|
||||
|
||||
def _slot_frei() -> None:
|
||||
global _inflight
|
||||
_inflight = max(0, _inflight - 1)
|
||||
|
||||
|
||||
def _ist_infra(err: str) -> bool:
|
||||
e = err.lower()
|
||||
return any(m in e for m in _INFRA_MARKER)
|
||||
|
||||
|
||||
async def _api(prompt: str, role: str, timeout: float) -> ApiErgebnis:
|
||||
opts = dict(config.ROLLEN.get(role) or config.ROLLEN["judge"])
|
||||
body = {"model": opts.pop("model"), "max_tokens": config.LLM_MAX_TOKENS,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": opts.pop("temperature"),
|
||||
"thinking": {"type": opts.pop("thinking")}}
|
||||
headers = {"x-api-key": __import__("os").environ.get("MINIMAX_API_KEY", ""),
|
||||
"anthropic-version": "2023-06-01"}
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(timeout, connect=config.FETCH_CONNECT_S)) as client:
|
||||
resp = await asyncio.wait_for(
|
||||
client.post(config.LLM_ENDPOINT, json=body, headers=headers),
|
||||
timeout=timeout)
|
||||
except (httpx.TimeoutException, asyncio.TimeoutError):
|
||||
return ApiErgebnis(1, err="timeout")
|
||||
except httpx.HTTPError as e:
|
||||
return ApiErgebnis(1, err=f"{type(e).__name__}: {e}")
|
||||
if resp.status_code == 429:
|
||||
try:
|
||||
nach = float(resp.headers.get("retry-after", "15"))
|
||||
except ValueError:
|
||||
nach = 15.0
|
||||
drossel_melden(min(nach, 60.0))
|
||||
return ApiErgebnis(1, err="HTTP 429: rate limited")
|
||||
if resp.status_code != 200:
|
||||
return ApiErgebnis(1, err=f"HTTP {resp.status_code}: {resp.text[:300]}")
|
||||
daten = resp.json()
|
||||
text = "".join(b.get("text", "") for b in daten.get("content", [])
|
||||
if b.get("type") == "text")
|
||||
u = daten.get("usage") or {}
|
||||
tokens = {"input": int(u.get("input_tokens") or 0),
|
||||
"output": int(u.get("output_tokens") or 0),
|
||||
"cache_read": int(u.get("cache_read_input_tokens") or 0),
|
||||
"cache_write": int(u.get("cache_creation_input_tokens") or 0)}
|
||||
if _ist_infra(resp.text[:500]) and not text.strip():
|
||||
return ApiErgebnis(1, err=resp.text[:300], tokens=tokens)
|
||||
if not text.strip():
|
||||
return ApiErgebnis(1, err=f"leere Antwort (stop={daten.get('stop_reason')})",
|
||||
tokens=tokens)
|
||||
_erfolg_melden()
|
||||
return ApiErgebnis(0, text=text, tokens=tokens)
|
||||
|
||||
|
||||
async def _versuch(prompt: str, role: str, timeout: float) -> tuple[ApiErgebnis, bool]:
|
||||
"""Ein Versuch inkl. Hedge-Zwilling (Lektion 4/5). → (ergebnis, war_hedge)."""
|
||||
wait_s = await _slot()
|
||||
try:
|
||||
haupt = asyncio.ensure_future(_api(prompt, role, timeout))
|
||||
schwelle = max(config.HEDGE_NACH_S, timeout / 2) if config.HEDGE_NACH_S else None
|
||||
if schwelle is None:
|
||||
return await haupt, False
|
||||
fertig, _ = await asyncio.wait({haupt}, timeout=schwelle)
|
||||
if fertig:
|
||||
return haupt.result(), False
|
||||
zwilling = asyncio.ensure_future(_api(prompt, role, timeout))
|
||||
fertig, offen = await asyncio.wait({haupt, zwilling},
|
||||
return_when=asyncio.FIRST_COMPLETED)
|
||||
gewinner = fertig.pop()
|
||||
for t in offen:
|
||||
t.cancel() # Verlierer wird vom Aufrufer als hedge_cancel verbucht
|
||||
return gewinner.result(), True
|
||||
finally:
|
||||
_slot_frei()
|
||||
# wait_s in Meta des Aufrufers — hier via Attribut zurückgeben
|
||||
_versuch.last_wait = wait_s # type: ignore[attr-defined]
|
||||
|
||||
|
||||
async def call(*, run_id: int, stufe: str, knoten: str, item: str,
|
||||
skill_namen: list[str], werte: dict, role: str = "judge",
|
||||
n: int = 0) -> str | None:
|
||||
"""Zentraler LLM-Engpass. Rückgabe: Antworttext oder None (Inhaltsfehler/cap).
|
||||
Wirft LaufPause (Infra erschöpft/manuell) oder BudgetErschoepft."""
|
||||
prompt, skill_hash, paare = skills.komponieren(skill_namen, werte)
|
||||
timeout = config.timeout_fuer(knoten, n)
|
||||
infra_rest = config.INFRA_MAX_RETRIES
|
||||
inhalt_rest = config.INHALT_MAX_RESTARTS
|
||||
|
||||
while True:
|
||||
run = db.one("SELECT status FROM runs WHERE id=?", run_id)
|
||||
if run and run["status"] != "running":
|
||||
raise ManuellePause("Lauf nicht mehr running")
|
||||
budget_pruefen(run_id)
|
||||
|
||||
if config.FAKE:
|
||||
from . import fakes
|
||||
t0 = time.monotonic()
|
||||
rc, text, err = fakes.antwort(prompt)
|
||||
log_call(run_id, stufe=stufe, knoten=knoten, item=item,
|
||||
skills_liste=[p[0] for p in paare], skill_hash=skill_hash,
|
||||
model="fake", role=role, status="ok" if rc == 0 else "error",
|
||||
dur_ms=int((time.monotonic() - t0) * 1000),
|
||||
meta={"err": err[:200]} if err else {},
|
||||
prompt=prompt, antwort=text or err)
|
||||
if rc != 0:
|
||||
raise RuntimeError(f"fake: {err}")
|
||||
return text
|
||||
|
||||
t0 = time.monotonic()
|
||||
status, err, tokens, text = "error", "", {}, ""
|
||||
hedge = False
|
||||
try:
|
||||
res, hedge = await _versuch(prompt, role, timeout)
|
||||
tokens, err, text = res.tokens, res.err, res.text
|
||||
if res.rc == 0:
|
||||
status = "ok"
|
||||
elif "stop=max_tokens" in err:
|
||||
status = "cap"
|
||||
elif _ist_infra(err):
|
||||
status = "infra"
|
||||
else:
|
||||
status = "error"
|
||||
except asyncio.CancelledError:
|
||||
status, err = "pause", "abbruch"
|
||||
raise
|
||||
finally:
|
||||
log_call(run_id, stufe=stufe, knoten=knoten, item=item,
|
||||
skills_liste=[p[0] for p in paare], skill_hash=skill_hash,
|
||||
model=config.ROLLEN.get(role, {}).get("model", ""), role=role,
|
||||
status=status, dur_ms=int((time.monotonic() - t0) * 1000),
|
||||
wait_ms=int(getattr(_versuch, "last_wait", 0) * 1000),
|
||||
tokens=tokens, meta={"err": err[:300]} if err else {},
|
||||
prompt=prompt, antwort=text or err)
|
||||
if hedge: # Verlierer sichtbar machen — sonst fehlen Tokens/Kennzahlen
|
||||
log_call(run_id, stufe=stufe, knoten=knoten, item=f"{item}-hedge",
|
||||
skill_hash=skill_hash, role=role, status="hedge_cancel")
|
||||
budget_pruefen(run_id)
|
||||
|
||||
if status == "ok":
|
||||
return text
|
||||
if status == "cap":
|
||||
return None # deterministisch: gleicher Prompt läuft wieder ins Cap
|
||||
if status == "infra":
|
||||
infra_rest -= 1
|
||||
if infra_rest < 0:
|
||||
raise LaufPause(f"Infra erschöpft: {err[:120]}")
|
||||
pause = config.INFRA_BACKOFF_BASE * 2 ** (
|
||||
config.INFRA_MAX_RETRIES - 1 - infra_rest)
|
||||
pause *= 1 + random.random() * 0.25 # Jitter
|
||||
drossel_melden(pause)
|
||||
await asyncio.sleep(pause)
|
||||
continue
|
||||
inhalt_rest -= 1
|
||||
if inhalt_rest < 0:
|
||||
return None
|
||||
238
backend/main.py
Normal file
238
backend/main.py
Normal file
@@ -0,0 +1,238 @@
|
||||
"""FastAPI-App: Topics verwalten, Läufe starten/pausieren, Graph-Snapshot
|
||||
(kompakt: Zähler statt Monolith), Task-Listen lazy, Guide, Kennzahlen."""
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel
|
||||
|
||||
from . import config, db, engine, graph, ledger, montage
|
||||
from .ws import hub
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
db.connect()
|
||||
engine.module_laden()
|
||||
hub.bind_loop(asyncio.get_running_loop())
|
||||
db.on_change = hub.push
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app.add_middleware(CORSMiddleware, allow_origins=["http://localhost:5173"],
|
||||
allow_methods=["*"], allow_headers=["*"])
|
||||
|
||||
|
||||
class TopicNeu(BaseModel):
|
||||
name: str
|
||||
titel: str = ""
|
||||
|
||||
|
||||
class StartAuftrag(BaseModel):
|
||||
budget: int | None = None
|
||||
|
||||
|
||||
class TopicEinstellung(BaseModel):
|
||||
quellen_max: int | None = None # 0 = unbegrenzt, None = Default
|
||||
|
||||
|
||||
@app.patch("/api/topics/{topic}")
|
||||
async def topic_einstellen(topic: str, e: TopicEinstellung):
|
||||
if not db.one("SELECT name FROM topics WHERE name=?", topic):
|
||||
raise HTTPException(404)
|
||||
if e.quellen_max is not None and (not 0 <= e.quellen_max <= 200
|
||||
or e.quellen_max == 1):
|
||||
# 1 Quelle kann nie bestätigen — Konsens braucht ≥2 unabhängige Belege
|
||||
raise HTTPException(400, "quellen_max: 0 (unbegrenzt) oder 2-200")
|
||||
db.update("topics", "name=?", (topic,), quellen_max=e.quellen_max)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.get("/api/topics")
|
||||
def topics_liste():
|
||||
topics = db.query("SELECT * FROM topics ORDER BY erstellt DESC")
|
||||
for t in topics:
|
||||
t["run"] = db.one("SELECT id, status, grund, stufe FROM runs WHERE "
|
||||
"topic=? ORDER BY id DESC LIMIT 1", t["name"])
|
||||
return topics
|
||||
|
||||
|
||||
@app.post("/api/topics")
|
||||
def topic_anlegen(auftrag: TopicNeu):
|
||||
if not config.TOPIC_NAME_RE.match(auftrag.name):
|
||||
raise HTTPException(400, "ungültiger Name")
|
||||
if db.one("SELECT name FROM topics WHERE name=?", auftrag.name):
|
||||
raise HTTPException(409, "Topic existiert")
|
||||
db.insert("topics", name=auftrag.name, titel=auftrag.titel)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
def _pipeline_daten_loeschen(topic: str) -> None:
|
||||
"""Alle Pipeline-Daten eines Topics räumen (Snapshots inklusive)."""
|
||||
for tabelle in ("tasks", "gate_laeufe", "quellen", "soll", "atome", "kanten",
|
||||
"lernziele", "bausteine", "kapitel"):
|
||||
db.execute(f"DELETE FROM {tabelle} WHERE topic=?", topic)
|
||||
db.execute("DELETE FROM befunde WHERE run_id IN "
|
||||
"(SELECT id FROM runs WHERE topic=?)", topic)
|
||||
db.execute("DELETE FROM sections WHERE baustein_id NOT IN "
|
||||
"(SELECT id FROM bausteine)")
|
||||
db.execute("DELETE FROM anker WHERE atom_id NOT IN (SELECT id FROM atome)")
|
||||
ordner = (config.KORPUS_DIR / topic).resolve()
|
||||
if ordner.is_dir() and ordner.parent == config.KORPUS_DIR.resolve():
|
||||
import shutil
|
||||
shutil.rmtree(ordner) # Guard: nur direkt unter KORPUS_DIR (Lektion!)
|
||||
|
||||
|
||||
@app.delete("/api/topics/{topic}")
|
||||
async def topic_loeschen(topic: str):
|
||||
engine.lauf_stoppen(topic)
|
||||
_pipeline_daten_loeschen(topic)
|
||||
db.execute("DELETE FROM runs WHERE topic=?", topic)
|
||||
db.execute("DELETE FROM topics WHERE name=?", topic)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.post("/api/topics/{topic}/reset")
|
||||
async def topic_reset(topic: str):
|
||||
"""Pipeline auf null: Topic bleibt, Läufe/Ledger bleiben als Historie."""
|
||||
if not db.one("SELECT name FROM topics WHERE name=?", topic):
|
||||
raise HTTPException(404)
|
||||
engine.lauf_stoppen(topic)
|
||||
_pipeline_daten_loeschen(topic)
|
||||
# Alte Läufe schließen — sonst resumed der nächste Start den alten Run
|
||||
# samt altem Budget und Token-Zähler
|
||||
db.execute("UPDATE runs SET status='stopped', beendet=? WHERE topic=? AND "
|
||||
"status IN ('running','paused','budget')", db.now(), topic)
|
||||
db.update("topics", "name=?", (topic,), status="neu")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.post("/api/topics/{topic}/start")
|
||||
async def start(topic: str, auftrag: StartAuftrag):
|
||||
if not db.one("SELECT name FROM topics WHERE name=?", topic):
|
||||
raise HTTPException(404)
|
||||
try:
|
||||
run_id = engine.lauf_starten(topic, auftrag.budget)
|
||||
except RuntimeError as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
return {"run_id": run_id}
|
||||
|
||||
|
||||
@app.post("/api/topics/{topic}/pause")
|
||||
async def pause(topic: str):
|
||||
engine.lauf_pausieren(topic)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.post("/api/topics/{topic}/stop")
|
||||
async def stop(topic: str):
|
||||
engine.lauf_stoppen(topic)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.get("/api/topics/{topic}/graph")
|
||||
def graph_snapshot(topic: str):
|
||||
g = graph.get()
|
||||
zaehler = {r["knoten"]: {} for r in []}
|
||||
zaehler = {}
|
||||
for r in db.query("SELECT knoten, status, COUNT(*) c FROM tasks WHERE "
|
||||
"topic=? GROUP BY knoten, status", topic):
|
||||
zaehler.setdefault(r["knoten"], {})[r["status"]] = r["c"]
|
||||
gates = {r["knoten"]: {"status": r["status"], "runde": r["runde"]}
|
||||
for r in db.query(
|
||||
"SELECT knoten, status, MAX(runde) runde FROM gate_laeufe "
|
||||
"WHERE topic=? GROUP BY knoten", topic)}
|
||||
run = db.one("SELECT id, status, grund, stufe, budget_tokens FROM runs "
|
||||
"WHERE topic=? ORDER BY id DESC LIMIT 1", topic)
|
||||
if run:
|
||||
run["verbraucht"] = ledger.verbraucht(run["id"])
|
||||
return {"layout": g.layout(), "zaehler": zaehler, "gates": gates, "run": run}
|
||||
|
||||
|
||||
@app.get("/api/topics/{topic}/knoten/{knoten}/tasks")
|
||||
def knoten_tasks(topic: str, knoten: str):
|
||||
tasks = db.query("SELECT id, item, status, art, runde, versuch, fehler, "
|
||||
"gestartet, beendet FROM tasks WHERE topic=? AND knoten=? "
|
||||
"ORDER BY id DESC LIMIT 200", topic, knoten)
|
||||
befunde = db.query("SELECT art, item, detail, status FROM befunde WHERE "
|
||||
"knoten=? AND run_id IN (SELECT id FROM runs WHERE topic=?) "
|
||||
"ORDER BY id DESC LIMIT 100", knoten, topic)
|
||||
return {"tasks": tasks, "befunde": befunde}
|
||||
|
||||
|
||||
@app.get("/api/topics/{topic}/guide")
|
||||
def guide_lesen(topic: str):
|
||||
kapitel = []
|
||||
for k in db.query("SELECT * FROM kapitel WHERE topic=? ORDER BY ord", topic):
|
||||
sections = db.query(
|
||||
"SELECT b.id, b.titel, s.text FROM bausteine b LEFT JOIN sections s "
|
||||
"ON s.baustein_id=b.id WHERE b.kapitel_id=? ORDER BY b.ord", k["id"])
|
||||
kapitel.append({"id": k["id"], "titel": k["titel"], "intro": k["intro"],
|
||||
"sections": sections})
|
||||
return {"kapitel": kapitel,
|
||||
"fertig": (db.one("SELECT status FROM topics WHERE name=?", topic)
|
||||
or {}).get("status") == "fertig"}
|
||||
|
||||
|
||||
@app.get("/api/topics/{topic}/kennzahlen")
|
||||
def kennzahlen(topic: str):
|
||||
run = db.one("SELECT id, budget_tokens FROM runs WHERE topic=? "
|
||||
"ORDER BY id DESC LIMIT 1", topic)
|
||||
if not run:
|
||||
return {"zeilen": [], "verbraucht": 0, "kosten_usd": 0, "coverage": None}
|
||||
kz = ledger.kennzahlen(run["id"])
|
||||
gesamt = db.one("SELECT COUNT(*) c FROM soll WHERE topic=? AND "
|
||||
"status='bestaetigt'", topic)["c"]
|
||||
gedeckt = db.one(
|
||||
"SELECT COUNT(*) c FROM soll s WHERE s.topic=? AND s.status='bestaetigt' "
|
||||
"AND (s.freispruch!='' OR EXISTS (SELECT 1 FROM atome a WHERE "
|
||||
"a.soll_id=s.id AND a.status='aktiv' AND a.baustein_id IS NOT NULL))",
|
||||
topic)["c"]
|
||||
redundanz_offen = db.one(
|
||||
"SELECT COUNT(*) c FROM befunde WHERE art='redundanz' AND status='offen' "
|
||||
"AND run_id IN (SELECT id FROM runs WHERE topic=?)", topic)["c"]
|
||||
eskaliert = db.one(
|
||||
"SELECT COUNT(*) c FROM befunde WHERE status='eskaliert' AND run_id IN "
|
||||
"(SELECT id FROM runs WHERE topic=?)", topic)["c"]
|
||||
atome_aktiv = db.one("SELECT COUNT(*) c FROM atome WHERE topic=? AND "
|
||||
"status='aktiv'", topic)["c"]
|
||||
verdichtet = db.one("SELECT COUNT(*) c FROM soll WHERE topic=? AND "
|
||||
"status='bestaetigt' AND verdichtet=1", topic)["c"]
|
||||
from . import belege as belege_mod
|
||||
kz["beleg_abdeckung"] = belege_mod.abdeckung_prozent(topic)
|
||||
kz.update({"coverage": round(gedeckt / gesamt * 100, 1) if gesamt else None,
|
||||
"soll_gesamt": gesamt, "soll_gedeckt": gedeckt,
|
||||
"redundanz_offen": redundanz_offen, "eskaliert": eskaliert,
|
||||
"atome_aktiv": atome_aktiv, "verdichtet_punkte": verdichtet,
|
||||
"budget": run["budget_tokens"]})
|
||||
return kz
|
||||
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
def health():
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@app.websocket("/ws")
|
||||
async def ws_endpoint(ws: WebSocket):
|
||||
await ws.accept()
|
||||
hub.connect(ws)
|
||||
try:
|
||||
while True:
|
||||
await ws.receive_text()
|
||||
except WebSocketDisconnect:
|
||||
hub.disconnect(ws)
|
||||
|
||||
|
||||
_DIST = config.ROOT / "frontend" / "dist"
|
||||
if _DIST.exists():
|
||||
app.mount("/assets", StaticFiles(directory=_DIST / "assets"), name="assets")
|
||||
|
||||
@app.get("/")
|
||||
def index():
|
||||
return FileResponse(_DIST / "index.html")
|
||||
59
backend/montage.py
Normal file
59
backend/montage.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Stufe 5: Guide-Zusammenbau (Kapitel + Intros + Sections) + deterministische
|
||||
Abschluss-Checks: jeder Atom-Marker global genau einmal, Anker-Links auflösbar."""
|
||||
import re
|
||||
|
||||
from . import config, db, engine
|
||||
from .guide import _MARKER_RE, _slug
|
||||
|
||||
|
||||
def guide_pfad(topic: str) -> str:
|
||||
return str(config.KORPUS_DIR.parent / "guides" / f"{topic}.md")
|
||||
|
||||
|
||||
def bauen(topic: str) -> str:
|
||||
teile = []
|
||||
t = db.one("SELECT COALESCE(NULLIF(titel,''), name) t FROM topics WHERE name=?",
|
||||
topic)
|
||||
teile.append(f"# {t['t']}\n")
|
||||
for kap in db.query("SELECT * FROM kapitel WHERE topic=? ORDER BY ord", topic):
|
||||
teile.append(f"\n## {kap['titel']}\n")
|
||||
if kap["intro"]:
|
||||
teile.append(kap["intro"] + "\n")
|
||||
for b in db.query("SELECT * FROM bausteine WHERE kapitel_id=? ORDER BY ord",
|
||||
kap["id"]):
|
||||
sec = db.one("SELECT text FROM sections WHERE baustein_id=?", b["id"])
|
||||
if sec and sec["text"]:
|
||||
teile.append(f"\n### {b['titel']}\n")
|
||||
teile.append(sec["text"] + "\n")
|
||||
return "\n".join(teile)
|
||||
|
||||
|
||||
@engine.worker("montage.montieren")
|
||||
async def montieren(task: dict) -> engine.Ergebnis:
|
||||
topic = task["topic"]
|
||||
text = bauen(topic)
|
||||
# Check 1: jeder aktive Atom-Marker genau einmal
|
||||
marker = [int(m) for m in _MARKER_RE.findall(text)]
|
||||
aktive = {a["id"] for a in db.query(
|
||||
"SELECT id FROM atome WHERE topic=? AND status='aktiv' AND "
|
||||
"baustein_id IS NOT NULL", topic)}
|
||||
fehler = []
|
||||
for a in aktive:
|
||||
if marker.count(a) != 1:
|
||||
fehler.append(f"Atom-Marker {a}: {marker.count(a)}x")
|
||||
# Check 2: Anker-Links auflösbar (H2/H3-Slugs)
|
||||
slugs = {_slug(u) for u in re.findall(r"^#{2,3}\s+(.+)$", text, re.MULTILINE)}
|
||||
for link in re.findall(r"\]\(#([^)]+)\)", text):
|
||||
if link not in slugs:
|
||||
fehler.append(f"toter Anker-Link: #{link}")
|
||||
if fehler:
|
||||
for f in fehler:
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="montage",
|
||||
knoten="montage", art="montage", item=topic, detail=f[:300])
|
||||
raise RuntimeError(f"Montage-Checks: {len(fehler)} Fehler")
|
||||
from .laden import _atomar_schreiben
|
||||
_atomar_schreiben(guide_pfad(topic), text.encode("utf-8"))
|
||||
db.update("topics", "name=?", (topic,), status="fertig")
|
||||
return engine.Ergebnis(daten={"zeichen": len(text),
|
||||
"kapitel": len(db.query(
|
||||
"SELECT id FROM kapitel WHERE topic=?", topic))})
|
||||
33
backend/panels.py
Normal file
33
backend/panels.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""Panel-Primitive (creator2-Lektionen 15-24): 2 = Konsens-Minimum, Merge nur
|
||||
einstimmig, Ausfall = keine Stimme = kein Entscheid, Ersatz-Richter bei genau
|
||||
einem Ausfall."""
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
|
||||
def einstimmig(stimmen: list, n: int, urteil: Callable) -> bool:
|
||||
"""True nur bei vollzähligem Panel UND allen Stimmen True.
|
||||
urteil(s) → True/False/None; None (Ausfall/Parse) = keine Stimme."""
|
||||
urteile = [urteil(s) for s in stimmen if s is not None]
|
||||
urteile = [u for u in urteile if u is not None]
|
||||
return len(urteile) >= n and all(urteile)
|
||||
|
||||
|
||||
async def panel(ruf: Callable[[int], Awaitable], groesse: int) -> list:
|
||||
"""groesse unabhängige Calls; fällt GENAU eine Stimme aus, läuft ein
|
||||
Ersatz-Richter (Index -1). None-Stimmen bleiben in der Liste."""
|
||||
import asyncio
|
||||
stimmen = list(await asyncio.gather(*(ruf(i) for i in range(groesse))))
|
||||
gueltig = [s for s in stimmen if s is not None]
|
||||
if groesse >= 2 and len(gueltig) == groesse - 1:
|
||||
stimmen.append(await ruf(-1))
|
||||
return stimmen
|
||||
|
||||
|
||||
def mehrheit(stimmen: list, urteil: Callable) -> bool | None:
|
||||
"""Mehrheitsentscheid (Klärung, 3er): None wenn keine Mehrheit möglich."""
|
||||
urteile = [urteil(s) for s in stimmen if s is not None]
|
||||
urteile = [u for u in urteile if u is not None]
|
||||
if not urteile:
|
||||
return None
|
||||
ja = sum(1 for u in urteile if u)
|
||||
return ja * 2 > len(urteile)
|
||||
83
backend/redundanz.py
Normal file
83
backend/redundanz.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""Rückwärts-Redundanz-Check (Kernziel Länge minimieren, R11): Satzpaare aus
|
||||
VERSCHIEDENEN Kapiteln mit Jaccard ≥0.6 → 2er-Panel entscheidet „redundant vs.
|
||||
didaktisch gewollt" (Recap ist keine Dopplung). Befunde behebt der Fix-Knoten
|
||||
per Ein-Satz-Refresher + Anker-Link."""
|
||||
from . import config, db, engine, llm, panels, textkit
|
||||
|
||||
|
||||
def kandidaten_paare(topic: str) -> list[dict]:
|
||||
saetze = []
|
||||
for row in db.query(
|
||||
"SELECT s.baustein_id bid, s.text, b.ord bord, k.ord kord FROM "
|
||||
"sections s JOIN bausteine b ON b.id=s.baustein_id "
|
||||
"JOIN kapitel k ON k.id=b.kapitel_id WHERE b.topic=? AND s.text!=''",
|
||||
topic):
|
||||
for satz in textkit.satz_split(row["text"]):
|
||||
toks = textkit.stamm_tokens(satz)
|
||||
if len(toks) >= config.REDUNDANZ_SATZ_MIN_TOKEN:
|
||||
saetze.append({"satz": satz, "toks": toks, "bid": row["bid"],
|
||||
"pos": (row["kord"], row["bord"])})
|
||||
def shingles(satz: str) -> set:
|
||||
woerter = textkit.norm(satz).split()
|
||||
n = config.REDUNDANZ_SHINGLE
|
||||
return {" ".join(woerter[i:i + n]) for i in range(len(woerter) - n + 1)}
|
||||
|
||||
paare = []
|
||||
for i, a in enumerate(saetze):
|
||||
for b in saetze[i + 1:]:
|
||||
if a["bid"] == b["bid"]:
|
||||
continue # innerhalb einer Section: Sache des Prüfers, nicht hier
|
||||
schnitt = len(a["toks"] & b["toks"])
|
||||
jaccard_treffer = schnitt and schnitt / len(a["toks"] | b["toks"]) \
|
||||
>= config.REDUNDANZ_JACCARD
|
||||
# Shingle-Kanal: identische Wort-Folge in längeren Sätzen —
|
||||
# Jaccard übersieht sie, wenn der Restsatz verschieden ist
|
||||
if jaccard_treffer or (shingles(a["satz"]) & shingles(b["satz"])):
|
||||
paare.append({"a": a, "b": b})
|
||||
return paare
|
||||
|
||||
|
||||
@engine.worker("redundanz.pruefen")
|
||||
async def pruefen(task: dict) -> engine.Ergebnis:
|
||||
paare = kandidaten_paare(task["topic"])
|
||||
befunde = 0
|
||||
for i in range(0, len(paare), config.DEDUP_PAARE_PRO_CALL):
|
||||
chunk = paare[i:i + config.DEDUP_PAARE_PRO_CALL]
|
||||
liste = "\n\n".join(
|
||||
f"PAAR {j + 1}:\nKapitel A: {p['a']['satz']}\nKapitel B: {p['b']['satz']}"
|
||||
for j, p in enumerate(chunk))
|
||||
|
||||
async def ruf(p, liste=liste, n=len(chunk)):
|
||||
text = await llm.call(
|
||||
run_id=task["run_id"], stufe="guide", knoten="redundanz",
|
||||
item=f"{task['item']}:c{i}:p{p}", role="judge", n=n,
|
||||
skill_namen=["richter", "deutsch-praezise", "redundanz-urteil"],
|
||||
werte={"paare": liste})
|
||||
if text is None:
|
||||
return None
|
||||
return {int(b["paar"]): b.get("urteil", "UNSICHER")
|
||||
for b in textkit.bloecke(text, "URTEIL")
|
||||
if b.get("paar", "").isdigit()}
|
||||
|
||||
stimmen = await panels.panel(ruf, config.PRUEF_PANEL)
|
||||
for j, p in enumerate(chunk):
|
||||
def urteil(s, j=j):
|
||||
if s is None or (j + 1) not in s:
|
||||
return None
|
||||
u = s[j + 1]
|
||||
return True if u == "redundant" else (
|
||||
False if u == "didaktisch_gewollt" else None)
|
||||
if panels.einstimmig(stimmen, config.PRUEF_PANEL, urteil):
|
||||
# Der SPÄTERE Baustein trägt den Befund (dort wird verwiesen)
|
||||
spaeter = max(p["a"], p["b"], key=lambda x: x["pos"])
|
||||
schon = db.one(
|
||||
"SELECT id FROM befunde WHERE item=? AND art='redundanz' AND "
|
||||
"detail=? AND status IN ('offen','eskaliert','freispruch')",
|
||||
f"baustein:{spaeter['bid']}", spaeter["satz"][:400])
|
||||
if not schon:
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="guide",
|
||||
knoten="redundanz", art="redundanz",
|
||||
item=f"baustein:{spaeter['bid']}",
|
||||
detail=spaeter["satz"][:400], runde=task["runde"])
|
||||
befunde += 1
|
||||
return engine.Ergebnis(daten={"paare": len(paare), "befunde": befunde})
|
||||
12
backend/requirements.txt
Normal file
12
backend/requirements.txt
Normal file
@@ -0,0 +1,12 @@
|
||||
fastapi>=0.115
|
||||
uvicorn[standard]>=0.30
|
||||
httpx>=0.27
|
||||
pyyaml>=6.0
|
||||
trafilatura>=2.1
|
||||
ftfy>=6.3
|
||||
ddgs>=9.14
|
||||
fuzzysearch>=0.8
|
||||
rapidfuzz>=3.9
|
||||
PyStemmer>=2.2
|
||||
pytest>=8.0
|
||||
pytest-asyncio>=0.24
|
||||
98
backend/skills.py
Normal file
98
backend/skills.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""Skill-Loader + Komposition (R5): genau 1 Rolle + 1 Aufgabe + 0..n Stil.
|
||||
Nur die Aufgabe definiert das Ausgabeformat (Konfliktfreiheit per Konstruktion).
|
||||
{{platzhalter}} nur im Aufgabe-Body, str.replace statt format_map.
|
||||
Hashing zweistufig: je Datei sha256[:12], skill_hash über die geordnete Liste."""
|
||||
import hashlib
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from . import config
|
||||
|
||||
_FRONT_RE = re.compile(r"^---\n(.*?)\n---\n(.*)$", re.DOTALL)
|
||||
_PLATZHALTER_RE = re.compile(r"\{\{([a-z_]+)\}\}")
|
||||
_cache: dict[str, dict] = {}
|
||||
|
||||
|
||||
class SkillFehler(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _parse(pfad: Path) -> dict:
|
||||
text = pfad.read_text(encoding="utf-8")
|
||||
m = _FRONT_RE.match(text)
|
||||
if not m:
|
||||
raise SkillFehler(f"{pfad.name}: kein Frontmatter")
|
||||
meta: dict = {"platzhalter": []}
|
||||
for zeile in m.group(1).splitlines():
|
||||
if ":" not in zeile:
|
||||
continue
|
||||
k, v = zeile.split(":", 1)
|
||||
k, v = k.strip(), v.strip()
|
||||
if k == "platzhalter":
|
||||
meta[k] = [p.strip() for p in v.strip("[]").split(",") if p.strip()]
|
||||
else:
|
||||
meta[k] = v
|
||||
for pflicht in ("name", "art", "beschreibung"):
|
||||
if not meta.get(pflicht):
|
||||
raise SkillFehler(f"{pfad.name}: Frontmatter-Feld '{pflicht}' fehlt")
|
||||
if meta["art"] not in ("rolle", "aufgabe", "stil"):
|
||||
raise SkillFehler(f"{pfad.name}: unbekannte art '{meta['art']}'")
|
||||
body = m.group(2).strip()
|
||||
return {**meta, "body": body,
|
||||
"hash": hashlib.sha256(text.encode()).hexdigest()[:12]}
|
||||
|
||||
|
||||
def lade(name: str) -> dict:
|
||||
"""Skill per Name laden ('rolle/extraktor' oder nur 'extraktor' mit Suche)."""
|
||||
if name in _cache and not config.FAKE:
|
||||
return _cache[name]
|
||||
if "/" in name:
|
||||
pfad = config.SKILLS_DIR / f"{name}.md"
|
||||
else:
|
||||
treffer = list(config.SKILLS_DIR.glob(f"*/{name}.md"))
|
||||
if len(treffer) != 1:
|
||||
raise SkillFehler(f"Skill '{name}': {len(treffer)} Treffer")
|
||||
pfad = treffer[0]
|
||||
if not pfad.exists():
|
||||
raise SkillFehler(f"Skill-Datei fehlt: {pfad}")
|
||||
_cache[name] = _parse(pfad)
|
||||
return _cache[name]
|
||||
|
||||
|
||||
def komponieren(skill_namen: list[str], werte: dict) -> tuple[str, str, list]:
|
||||
"""→ (prompt, skill_hash, [(name, hash)]). Reihenfolge Rolle→Stil→Aufgabe;
|
||||
Prompt beginnt mit stabilem Präfix (MiniMax-Prompt-Caching ab 512 Tokens)."""
|
||||
geladen = [lade(n) for n in skill_namen]
|
||||
rollen = [s for s in geladen if s["art"] == "rolle"]
|
||||
aufgaben = [s for s in geladen if s["art"] == "aufgabe"]
|
||||
stile = [s for s in geladen if s["art"] == "stil"]
|
||||
if len(rollen) != 1 or len(aufgaben) != 1:
|
||||
raise SkillFehler(f"Komposition braucht genau 1 Rolle + 1 Aufgabe: {skill_namen}")
|
||||
aufgabe = aufgaben[0]
|
||||
|
||||
fehlt = [p for p in aufgabe["platzhalter"] if p not in werte]
|
||||
if fehlt:
|
||||
raise SkillFehler(f"Aufgabe '{aufgabe['name']}': Platzhalter fehlen: {fehlt}")
|
||||
body = aufgabe["body"]
|
||||
for p in aufgabe["platzhalter"]:
|
||||
body = body.replace("{{" + p + "}}", str(werte[p]))
|
||||
rest = _PLATZHALTER_RE.search(body)
|
||||
if rest:
|
||||
raise SkillFehler(f"Aufgabe '{aufgabe['name']}': unersetzt: {rest.group(0)}")
|
||||
|
||||
teile = [f"<!-- aufgabe:{aufgabe['name']} -->", # Fake-Registry-Schlüssel
|
||||
"## Rolle", rollen[0]["body"]]
|
||||
if stile:
|
||||
teile += ["## Arbeitsweise"] + [s["body"] for s in stile]
|
||||
teile += ["## Aufgabe", body]
|
||||
prompt = "\n\n".join(teile)
|
||||
|
||||
paare = [(s["name"], s["hash"]) for s in rollen + stile + [aufgabe]]
|
||||
skill_hash = hashlib.sha256(
|
||||
"\n".join(f"{n}:{h}" for n, h in paare).encode()).hexdigest()[:12]
|
||||
return prompt, skill_hash, paare
|
||||
|
||||
|
||||
def aufgaben_name(prompt: str) -> str:
|
||||
m = re.match(r"<!-- aufgabe:([a-z0-9-]+) -->", prompt)
|
||||
return m.group(1) if m else ""
|
||||
344
backend/struktur.py
Normal file
344
backend/struktur.py
Normal file
@@ -0,0 +1,344 @@
|
||||
"""Stufe 3: Lernziele je Soll-Punkt → Bausteine (Band 4-8) → Ordnung
|
||||
(Kapitel-Schnitt + Permutation, exakt validiert, deterministischer Fallback).
|
||||
Gate: exakte Partition + Coverage (jeder Soll-Punkt ≥1 Baustein)."""
|
||||
import math
|
||||
|
||||
from . import config, db, engine, llm, textkit
|
||||
|
||||
|
||||
def _atom_rang(atom_id: int) -> tuple:
|
||||
a = db.one("SELECT quelle_id, start FROM anker WHERE atom_id=? AND start>=0 "
|
||||
"ORDER BY quelle_id, start LIMIT 1", atom_id)
|
||||
return (a["quelle_id"], a["start"]) if a else (9999, 10**9)
|
||||
|
||||
|
||||
@engine.worker("struktur.lernziele", buendel=True)
|
||||
async def lernziele(task: dict) -> engine.Ergebnis:
|
||||
mitglieder = [task] + task.get("_buendel", [])
|
||||
teil, aktive, neue = {}, {}, [] # idx → (mitglied, soll_id, punkt, atome)
|
||||
idx = 0
|
||||
for m in mitglieder:
|
||||
soll_id = db.uj(m["payload"], {}).get("soll_id")
|
||||
if soll_id:
|
||||
soll = db.one("SELECT * FROM soll WHERE id=?", soll_id)
|
||||
atome = db.query("SELECT * FROM atome WHERE soll_id=? AND "
|
||||
"status='aktiv'", soll_id)
|
||||
punkt = soll["punkt"]
|
||||
else:
|
||||
atome = db.query("SELECT * FROM atome WHERE topic=? AND "
|
||||
"status='aktiv' AND soll_id IS NULL", task["topic"])
|
||||
punkt = "Weitere Grundlagen zum Thema"
|
||||
if not atome:
|
||||
teil[m["id"]] = "fertig"
|
||||
continue
|
||||
if db.one("SELECT id FROM lernziele WHERE topic=? AND "
|
||||
"COALESCE(soll_id,-1)=COALESCE(?,-1)", task["topic"], soll_id):
|
||||
teil[m["id"]] = "fertig"
|
||||
neue += _baustein_tasks(task["topic"], soll_id)
|
||||
continue
|
||||
idx += 1
|
||||
aktive[idx] = (m, soll_id, punkt, atome)
|
||||
if not aktive:
|
||||
return engine.Ergebnis(daten={"skip": True}, teil_status=teil,
|
||||
neue_tasks=neue)
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="struktur", knoten="lernziele",
|
||||
item=task["item"], role="judge",
|
||||
n=sum(len(a) for (_, _, _, a) in aktive.values()),
|
||||
skill_namen=["planer", "deutsch-praezise", "lernziele"],
|
||||
werte={"soll_punkte": "\n\n".join(
|
||||
f"=== SOLL {i} ===\nPUNKT: {punkt}\nATOME:\n" +
|
||||
"\n".join(f"[{a['id']}] {a['titel']}" for a in atome)
|
||||
for i, (_, _, punkt, atome) in aktive.items())})
|
||||
if antwort is None:
|
||||
raise RuntimeError("lernziele ohne Ergebnis")
|
||||
je_soll: dict[int, list] = {}
|
||||
for block in textkit.bloecke(antwort, "ZIEL"):
|
||||
s = block.get("soll", "").strip()
|
||||
if s.isdigit():
|
||||
je_soll.setdefault(int(s), []).append(block)
|
||||
for i, (m, soll_id, punkt, atome) in aktive.items():
|
||||
if i not in je_soll:
|
||||
teil[m["id"]] = "neu" # Soll-Punkt fehlt in der Antwort → neuer Versuch
|
||||
continue
|
||||
ids_alle = {a["id"] for a in atome}
|
||||
zugeordnet: set[int] = set()
|
||||
for block in je_soll[i]:
|
||||
atom_ids = [int(x) for x in
|
||||
block.get("atome", "").replace(" ", "").split(",")
|
||||
if x.isdigit() and int(x) in ids_alle - zugeordnet]
|
||||
if not atom_ids:
|
||||
continue
|
||||
ziel_id = db.insert("lernziele", topic=task["topic"],
|
||||
titel=block.get("titel", "")[:80],
|
||||
text=block.get("text", "")[:300], soll_id=soll_id)
|
||||
for aid in atom_ids:
|
||||
db.update("atome", "id=?", (aid,), ziel_id=ziel_id)
|
||||
zugeordnet |= set(atom_ids)
|
||||
rest = ids_alle - zugeordnet # deterministisches Sammelziel — nichts verlieren
|
||||
if rest:
|
||||
ziel_id = db.insert(
|
||||
"lernziele", topic=task["topic"], titel="Grundlagen",
|
||||
soll_id=soll_id,
|
||||
text=f"Der Lernende kann Grundlagen zu '{punkt}' erklären.")
|
||||
for aid in rest:
|
||||
db.update("atome", "id=?", (aid,), ziel_id=ziel_id)
|
||||
teil[m["id"]] = "fertig"
|
||||
neue += _baustein_tasks(task["topic"], soll_id)
|
||||
return engine.Ergebnis(daten={"ziele": True}, teil_status=teil,
|
||||
neue_tasks=neue)
|
||||
|
||||
|
||||
def _baustein_tasks(topic: str, soll_id) -> list:
|
||||
return [{"knoten": "bausteine", "item": f"ziel:{z['id']}",
|
||||
"payload": {"ziel_id": z["id"]}}
|
||||
for z in db.query("SELECT id FROM lernziele WHERE topic=? AND "
|
||||
"COALESCE(soll_id,-1)=COALESCE(?,-1)", topic, soll_id)]
|
||||
|
||||
|
||||
@engine.worker("struktur.bausteine")
|
||||
async def bausteine(task: dict) -> engine.Ergebnis:
|
||||
payload = db.uj(task["payload"], {})
|
||||
ziel = db.one("SELECT * FROM lernziele WHERE id=?", payload["ziel_id"])
|
||||
atome = db.query("SELECT * FROM atome WHERE ziel_id=? AND status='aktiv'",
|
||||
ziel["id"])
|
||||
if not atome:
|
||||
return engine.Ergebnis(daten={"skip": True})
|
||||
if db.one("SELECT id FROM bausteine WHERE ziel_id=?", ziel["id"]):
|
||||
return engine.Ergebnis(daten={"skip": True},
|
||||
neue_tasks=[{"knoten": "ordnung", "item": "ordnung"}])
|
||||
atome.sort(key=lambda a: _atom_rang(a["id"]))
|
||||
gruppen: list[list[dict]]
|
||||
if len(atome) <= config.BAUSTEIN_MAX_ATOME:
|
||||
gruppen = [atome]
|
||||
else:
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="struktur", knoten="bausteine",
|
||||
item=task["item"], role="judge", n=len(atome),
|
||||
skill_namen=["planer", "deutsch-praezise", "bausteine"],
|
||||
werte={"ziel": ziel["text"],
|
||||
"band": f"{config.BAUSTEIN_MIN_ATOME}-{config.BAUSTEIN_MAX_ATOME}",
|
||||
"atome": "\n".join(f"[{a['id']}] {a['titel']}" for a in atome)})
|
||||
if antwort is None: # Call-Ausfall: neuer Versuch statt stiller Fallback
|
||||
raise RuntimeError("bausteine ohne Ergebnis")
|
||||
gruppen = _gruppen_validieren(antwort, atome)
|
||||
for i, gruppe in enumerate(gruppen):
|
||||
titel = ziel["titel"] or gruppe[0]["titel"]
|
||||
if len(gruppen) > 1:
|
||||
titel = f"{titel} (Teil {i + 1})"
|
||||
bid = db.insert("bausteine", topic=task["topic"], ziel_id=ziel["id"],
|
||||
titel=titel[:120], ord=i)
|
||||
for j, a in enumerate(gruppe):
|
||||
db.update("atome", "id=?", (a["id"],), baustein_id=bid, ord=j)
|
||||
return engine.Ergebnis(daten={"bausteine": len(gruppen)},
|
||||
neue_tasks=[{"knoten": "ordnung", "item": "ordnung"}])
|
||||
|
||||
|
||||
def _gruppen_validieren(antwort: str | None, atome: list[dict]) -> list[list[dict]]:
|
||||
"""LLM-Split validieren (jede ID genau einmal); sonst deterministischer
|
||||
divmod-Split entlang der Quellreihenfolge."""
|
||||
nach_id = {a["id"]: a for a in atome}
|
||||
if antwort:
|
||||
gruppen, gesehen = [], set()
|
||||
for block in textkit.bloecke(antwort, "BAUSTEIN"):
|
||||
ids = [int(i) for i in block.get("atome", "").replace(" ", "").split(",")
|
||||
if i.isdigit()]
|
||||
if ids and all(i in nach_id and i not in gesehen for i in ids):
|
||||
gruppen.append([nach_id[i] for i in ids])
|
||||
gesehen |= set(ids)
|
||||
if gesehen == set(nach_id) and all(
|
||||
len(g) <= config.BAUSTEIN_MAX_ATOME for g in gruppen):
|
||||
return gruppen
|
||||
n_gruppen = math.ceil(len(atome) / config.BAUSTEIN_MAX_ATOME)
|
||||
basis, rest = divmod(len(atome), n_gruppen)
|
||||
gruppen, pos = [], 0
|
||||
for i in range(n_gruppen):
|
||||
k = basis + (1 if i < rest else 0)
|
||||
gruppen.append(atome[pos:pos + k])
|
||||
pos += k
|
||||
return gruppen
|
||||
|
||||
|
||||
def _ziele_dedup(topic: str) -> int:
|
||||
"""Synonyme Lernziele deterministisch mergen (Audit: „einordnen" vs.
|
||||
„verorten" erzeugte Doppel-Bausteine → Writer-Echos). Falsch-Merge ist
|
||||
harmlos: Atome bleiben erhalten, hängen nur am selben Ziel."""
|
||||
ziele = db.query("SELECT * FROM lernziele WHERE topic=? ORDER BY id", topic)
|
||||
merges = 0
|
||||
for i, a in enumerate(ziele):
|
||||
if a.get("_weg"):
|
||||
continue
|
||||
for b in ziele[i + 1:]:
|
||||
if b.get("_weg"):
|
||||
continue
|
||||
if textkit.titel_kern(a["titel"] or a["text"]) == \
|
||||
textkit.titel_kern(b["titel"] or b["text"]) or \
|
||||
textkit.jaccard(a["text"], b["text"]) >= \
|
||||
config.ZIEL_DEDUP_JACCARD:
|
||||
db.update("atome", "ziel_id=?", (b["id"],), ziel_id=a["id"])
|
||||
db.execute("DELETE FROM bausteine WHERE ziel_id=?", b["id"])
|
||||
db.execute("DELETE FROM lernziele WHERE id=?", b["id"])
|
||||
b["_weg"] = True
|
||||
merges += 1
|
||||
if merges: # verwaiste Baustein-Zuordnungen lösen → Gate-Reparatur baut neu
|
||||
db.execute("UPDATE atome SET baustein_id=NULL WHERE topic=? AND "
|
||||
"baustein_id IS NOT NULL AND baustein_id NOT IN "
|
||||
"(SELECT id FROM bausteine)", topic)
|
||||
return merges
|
||||
|
||||
|
||||
@engine.worker("struktur.ordnung")
|
||||
async def ordnung(task: dict) -> engine.Ergebnis:
|
||||
"""Barriere: Ziel-Dedup, dann Kapitel-Schnitt über Soll-Punkte + Baustein-
|
||||
Permutation je Kapitel. Prior = Median der Atom-Ränge; exakt validiert."""
|
||||
topic = task["topic"]
|
||||
ziel_merges = _ziele_dedup(topic)
|
||||
soll = db.query("SELECT * FROM soll WHERE topic=? AND status='bestaetigt' "
|
||||
"ORDER BY id", topic)
|
||||
db.execute("DELETE FROM kapitel WHERE topic=?", topic) # idempotent neu
|
||||
kapitel_gruppen = await _kapitel_schnitt(task, soll)
|
||||
for i, (titel, punkt_ids) in enumerate(kapitel_gruppen):
|
||||
kid = db.insert("kapitel", topic=topic, titel=titel[:120], ord=i)
|
||||
for sid in punkt_ids:
|
||||
db.update("soll", "id=?", (sid,), kapitel_id=kid)
|
||||
|
||||
# Bausteine → Kapitel (über Ziel→Soll); Sammel-Bausteine ans letzte Kapitel
|
||||
letztes = db.one("SELECT id FROM kapitel WHERE topic=? ORDER BY ord DESC "
|
||||
"LIMIT 1", topic)
|
||||
for b in db.query("SELECT b.*, z.soll_id sz FROM bausteine b JOIN lernziele z "
|
||||
"ON z.id=b.ziel_id WHERE b.topic=?", topic):
|
||||
kap = db.one("SELECT kapitel_id k FROM soll WHERE id=?", b["sz"]) \
|
||||
if b["sz"] else None
|
||||
db.update("bausteine", "id=?", (b["id"],),
|
||||
kapitel_id=(kap or {}).get("k") or letztes["id"])
|
||||
|
||||
# Permutation je Kapitel (Prior: Median-Anker-Rang)
|
||||
for kap in db.query("SELECT * FROM kapitel WHERE topic=? ORDER BY ord", topic):
|
||||
bs = db.query("SELECT * FROM bausteine WHERE topic=? AND kapitel_id=?",
|
||||
topic, kap["id"])
|
||||
prior = sorted(bs, key=lambda b: _baustein_rang(b["id"]))
|
||||
reihenfolge = prior
|
||||
if len(bs) > 1:
|
||||
ids = [b["id"] for b in prior]
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="struktur", knoten="ordnung",
|
||||
item=f"perm:kapitel:{kap['id']}", role="judge", n=len(bs),
|
||||
skill_namen=["planer", "deutsch-praezise", "ordnung"],
|
||||
werte={"art": "Bausteine",
|
||||
"liste": "\n".join(f"[{b['id']}] {b['titel']}" for b in prior),
|
||||
"prior": ",".join(map(str, ids))})
|
||||
perm = _permutation_validieren(antwort, ids)
|
||||
if perm:
|
||||
nach_id = {b["id"]: b for b in bs}
|
||||
reihenfolge = [nach_id[i] for i in perm]
|
||||
else:
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="struktur",
|
||||
knoten="ordnung", art="ordnung_fallback",
|
||||
item=f"kapitel:{kap['id']}", status="freispruch",
|
||||
detail="Judge-Permutation ungültig, Prior verwendet")
|
||||
for i, b in enumerate(reihenfolge):
|
||||
db.update("bausteine", "id=?", (b["id"],), ord=i, status="geordnet")
|
||||
return engine.Ergebnis(daten={"kapitel": len(kapitel_gruppen),
|
||||
"ziel_merges": ziel_merges})
|
||||
|
||||
|
||||
async def _kapitel_schnitt(task: dict, soll: list[dict]) -> list[tuple[str, list[int]]]:
|
||||
thema = db.one("SELECT COALESCE(NULLIF(titel,''), name) t FROM topics "
|
||||
"WHERE name=?", task["topic"])["t"]
|
||||
ids = [s["id"] for s in soll]
|
||||
if len(ids) <= config.KAPITEL_SOLL_PUNKTE:
|
||||
return [(thema, ids)]
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="struktur", knoten="ordnung",
|
||||
item="themen_schnitt", role="judge", n=len(ids),
|
||||
skill_namen=["planer", "deutsch-praezise", "themen-schnitt"],
|
||||
werte={"thema": thema, "richtwert": config.KAPITEL_SOLL_PUNKTE,
|
||||
"punkte": "\n".join(f"[{s['id']}] {s['punkt']}" for s in soll)})
|
||||
if antwort:
|
||||
gruppen, gesehen = [], set()
|
||||
for block in textkit.bloecke(antwort, "KAPITEL"):
|
||||
pids = [int(i) for i in block.get("punkte", "").replace(" ", "").split(",")
|
||||
if i.isdigit() and int(i) in set(ids) - gesehen]
|
||||
if pids:
|
||||
gruppen.append((block.get("titel", "Kapitel"), pids))
|
||||
gesehen |= set(pids)
|
||||
if gesehen == set(ids): # exakte Partition — sonst Fallback
|
||||
return gruppen
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="struktur", knoten="ordnung",
|
||||
art="themen_fallback", item=task["topic"], status="freispruch",
|
||||
detail="Schnitt ungültig, √n-Fallback")
|
||||
n_kap = max(1, round(math.sqrt(len(ids))))
|
||||
groesse = math.ceil(len(ids) / n_kap)
|
||||
return [(f"Teil {i + 1}", ids[i * groesse:(i + 1) * groesse])
|
||||
for i in range(n_kap) if ids[i * groesse:(i + 1) * groesse]]
|
||||
|
||||
|
||||
def _baustein_rang(baustein_id: int) -> tuple:
|
||||
raenge = sorted(_atom_rang(a["id"]) for a in db.query(
|
||||
"SELECT id FROM atome WHERE baustein_id=? AND status='aktiv'", baustein_id))
|
||||
return raenge[len(raenge) // 2] if raenge else (9999, 10**9)
|
||||
|
||||
|
||||
def _permutation_validieren(antwort: str | None, ids: list[int]) -> list[int] | None:
|
||||
if not antwort:
|
||||
return None
|
||||
block = textkit.bloecke(antwort, "ORDNUNG")
|
||||
if not block:
|
||||
return None
|
||||
perm = [int(i) for i in block[0].get("ids", "").replace(" ", "").split(",")
|
||||
if i.isdigit()]
|
||||
return perm if sorted(perm) == sorted(ids) else None
|
||||
|
||||
|
||||
@engine.worker("struktur.gate")
|
||||
async def gate(task: dict) -> engine.Ergebnis:
|
||||
topic, runde = task["topic"], task["runde"]
|
||||
befunde, neue = [], []
|
||||
ohne_baustein = db.query("SELECT id FROM atome WHERE topic=? AND "
|
||||
"status='aktiv' AND baustein_id IS NULL", topic)
|
||||
for a in ohne_baustein:
|
||||
befunde.append({"art": "partition", "item": f"atom:{a['id']}",
|
||||
"detail": "Atom ohne Baustein"})
|
||||
zu_gross = db.query(
|
||||
"SELECT baustein_id b, COUNT(*) n FROM atome WHERE topic=? AND "
|
||||
"status='aktiv' GROUP BY baustein_id HAVING n>?",
|
||||
topic, config.BAUSTEIN_MAX_ATOME)
|
||||
for z in zu_gross:
|
||||
befunde.append({"art": "band", "item": f"baustein:{z['b']}",
|
||||
"detail": f"{z['n']} Atome"})
|
||||
ziel = db.one("SELECT ziel_id z FROM bausteine WHERE id=?", z["b"])
|
||||
db.execute("DELETE FROM bausteine WHERE id=?", z["b"])
|
||||
neue.append({"knoten": "bausteine", "item": f"ziel:{ziel['z']}:r{runde + 1}",
|
||||
"art": "reparatur", "runde": runde + 1,
|
||||
"payload": {"ziel_id": ziel["z"]}})
|
||||
# Coverage Stufe 3: jeder bestätigte Soll-Punkt (ohne Freispruch) ≥1 Baustein
|
||||
ungedeckt = db.query(
|
||||
"SELECT s.id FROM soll s WHERE s.topic=? AND s.status='bestaetigt' AND "
|
||||
"s.freispruch='' AND NOT EXISTS (SELECT 1 FROM atome a WHERE a.soll_id=s.id "
|
||||
"AND a.status='aktiv' AND a.baustein_id IS NOT NULL)", topic)
|
||||
for s in ungedeckt:
|
||||
befunde.append({"art": "soll_ohne_baustein", "item": f"soll:{s['id']}",
|
||||
"detail": ""})
|
||||
# Kennzahl-Befund (blockiert nicht): Struktur aufgebläht?
|
||||
n_bausteine = db.one("SELECT COUNT(*) c FROM bausteine WHERE topic=?",
|
||||
topic)["c"]
|
||||
n_soll = db.one("SELECT COUNT(*) c FROM soll WHERE topic=? AND "
|
||||
"status='bestaetigt'", topic)["c"]
|
||||
if n_soll and n_bausteine > n_soll * 3:
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="struktur",
|
||||
knoten="gate_struktur", art="struktur_aufgeblaeht",
|
||||
item=topic, status="freispruch",
|
||||
detail=f"{n_bausteine} Bausteine aus {n_soll} Soll-Punkten")
|
||||
if befunde:
|
||||
return engine.Ergebnis(gate_status="rot", befunde=befunde, neue_tasks=neue)
|
||||
|
||||
db.update("topics", "name=?", (topic,), status="struktur_fertig")
|
||||
neue = []
|
||||
for b in db.query("SELECT id FROM bausteine WHERE topic=?", topic):
|
||||
db.insert("sections", ignore=True, baustein_id=b["id"])
|
||||
neue.append({"knoten": "writer", "item": f"r1:baustein:{b['id']}",
|
||||
"payload": {"baustein_id": b["id"]}})
|
||||
for k in db.query("SELECT id FROM kapitel WHERE topic=?", topic):
|
||||
neue.append({"knoten": "kapitel_intro", "item": f"kapitel:{k['id']}",
|
||||
"payload": {"kapitel_id": k["id"]}})
|
||||
return engine.Ergebnis(gate_status="gruen", neue_tasks=neue)
|
||||
200
backend/suche.py
Normal file
200
backend/suche.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""Such-Schicht (R2): Wikipedia de+en immer, dazu Kette Tavily→Brave→ddgs.
|
||||
Circuit Breaker je Backend; Teilausfall = degraded-Befund, nie Lauf-Crash."""
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from . import config, db, engine, llm, textkit
|
||||
|
||||
_TRACKING = {"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
|
||||
"fbclid", "gclid", "ref"}
|
||||
_ausfaelle: dict[str, int] = {}
|
||||
_wiki_sem = asyncio.Semaphore(config.WIKI_MAX_PARALLEL)
|
||||
|
||||
|
||||
def url_normieren(url: str) -> str:
|
||||
s = urlsplit(url)
|
||||
query = urlencode([(k, v) for k, v in parse_qsl(s.query)
|
||||
if k.lower() not in _TRACKING])
|
||||
return urlunsplit((s.scheme.lower(), s.netloc.lower(),
|
||||
s.path.rstrip("/") or "/", query, ""))
|
||||
|
||||
|
||||
def _tot(backend: str) -> bool:
|
||||
return _ausfaelle.get(backend, 0) >= config.CIRCUIT_BREAKER_N
|
||||
|
||||
|
||||
def _ausfall(backend: str) -> None:
|
||||
_ausfaelle[backend] = _ausfaelle.get(backend, 0) + 1
|
||||
|
||||
|
||||
async def _http_json(url: str, **kw) -> dict | list | None:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=config.SUCHE_TIMEOUT_S,
|
||||
headers={"User-Agent": config.user_agent()}) as client:
|
||||
r = await client.get(url, **kw)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
async def _wikipedia(query: str) -> list[dict]:
|
||||
treffer = []
|
||||
for sprache in ("de", "en"):
|
||||
backend = f"wikipedia_{sprache}"
|
||||
if _tot(backend):
|
||||
continue
|
||||
try:
|
||||
async with _wiki_sem:
|
||||
daten = await _http_json(
|
||||
f"https://{sprache}.wikipedia.org/w/rest.php/v1/search/page",
|
||||
params={"q": query, "limit": 4})
|
||||
for s in (daten or {}).get("pages", []):
|
||||
treffer.append({
|
||||
"titel": s.get("title", ""), "backend": backend,
|
||||
"url": f"https://{sprache}.wikipedia.org/wiki/"
|
||||
f"{s.get('key', s.get('title', ''))}"})
|
||||
_ausfaelle[backend] = 0
|
||||
except Exception:
|
||||
_ausfall(backend)
|
||||
return treffer
|
||||
|
||||
|
||||
async def _tavily(query: str) -> list[dict]:
|
||||
key = os.getenv("TAVILY_API_KEY")
|
||||
if not key:
|
||||
return []
|
||||
async with httpx.AsyncClient(timeout=config.SUCHE_TIMEOUT_S) as client:
|
||||
r = await client.post("https://api.tavily.com/search",
|
||||
json={"api_key": key, "query": query,
|
||||
"max_results": config.SUCHE_MAX_TREFFER})
|
||||
r.raise_for_status()
|
||||
return [{"titel": t.get("title", ""), "url": t["url"], "backend": "tavily"}
|
||||
for t in r.json().get("results", [])]
|
||||
|
||||
|
||||
async def _brave(query: str) -> list[dict]:
|
||||
key = os.getenv("BRAVE_API_KEY")
|
||||
if not key:
|
||||
return []
|
||||
daten = await _http_json("https://api.search.brave.com/res/v1/web/search",
|
||||
params={"q": query, "count": config.SUCHE_MAX_TREFFER,
|
||||
"country": "DE", "search_lang": "de"},
|
||||
headers={"X-Subscription-Token": key})
|
||||
return [{"titel": t.get("title", ""), "url": t["url"], "backend": "brave"}
|
||||
for t in (daten or {}).get("web", {}).get("results", [])]
|
||||
|
||||
|
||||
async def _ddgs(query: str) -> list[dict]:
|
||||
def _sync():
|
||||
from ddgs import DDGS
|
||||
return DDGS().text(query, region="de-de",
|
||||
max_results=config.SUCHE_MAX_TREFFER)
|
||||
await asyncio.sleep(config.DDGS_PAUSE_S) # Drosselung (ToS-Grauzone)
|
||||
rows = await asyncio.to_thread(_sync)
|
||||
return [{"titel": r.get("title", ""), "url": r.get("href") or r.get("link", ""),
|
||||
"backend": "ddgs"} for r in rows or [] if r.get("href") or r.get("link")]
|
||||
|
||||
|
||||
_KETTE = (("tavily", _tavily), ("brave", _brave), ("ddgs", _ddgs))
|
||||
|
||||
|
||||
async def web_suchen(query: str) -> tuple[list[dict], list[str]]:
|
||||
"""→ (treffer, fehler). Wikipedia immer; Kette bis zum ersten Backend mit
|
||||
Ergebnissen (Fallback statt Fan-out, R2)."""
|
||||
treffer = await _wikipedia(query)
|
||||
fehler = []
|
||||
for name, fn in _KETTE:
|
||||
if _tot(name):
|
||||
continue
|
||||
for versuch in range(config.SUCHE_RETRIES + 1):
|
||||
try:
|
||||
web = await asyncio.wait_for(fn(query), config.SUCHE_TIMEOUT_S * 2)
|
||||
if web:
|
||||
_ausfaelle[name] = 0
|
||||
return treffer + web, fehler
|
||||
break # Backend ohne Key/Treffer → nächstes
|
||||
except Exception as e:
|
||||
if versuch == config.SUCHE_RETRIES:
|
||||
_ausfall(name)
|
||||
fehler.append(f"{name}: {type(e).__name__}")
|
||||
return treffer, fehler
|
||||
|
||||
|
||||
async def _relevanz_filter(task: dict, thema: str, treffer: list) -> list:
|
||||
"""Kurzurteil je Trefferliste: Werbe-/App-Listen und Off-Topic raus
|
||||
(Lektion 91). AUCH Wikipedia-Treffer — Run 7 lud „Pasta" und „Xinjiang"."""
|
||||
web = treffer
|
||||
if len(web) <= 2:
|
||||
return treffer
|
||||
liste = "\n".join(f"[{i}] {t['titel']} — {t['url']}" for i, t in enumerate(web))
|
||||
antwort = await llm.call(
|
||||
run_id=task["run_id"], stufe="korpus", knoten="suche",
|
||||
item=f"{task['item']}:relevanz", role="judge",
|
||||
skill_namen=["richter", "deutsch-praezise", "quellen-relevanz"],
|
||||
werte={"thema": thema, "treffer": liste})
|
||||
if antwort is None:
|
||||
return treffer # Urteil ausgefallen → lieber behalten als verlieren
|
||||
bloecke = textkit.bloecke(antwort, "RELEVANT")
|
||||
ids = {int(i) for i in (bloecke[0].get("ids", "") if bloecke else "")
|
||||
.replace(" ", "").split(",") if i.isdigit()}
|
||||
return [t for i, t in enumerate(web) if i in ids]
|
||||
|
||||
|
||||
@engine.worker("suche.suchen")
|
||||
async def suchen(task: dict) -> engine.Ergebnis:
|
||||
payload = db.uj(task["payload"], {})
|
||||
query = payload.get("query", "")
|
||||
zweck = payload.get("zweck", "korpus")
|
||||
seed = bool(payload.get("seed"))
|
||||
if seed:
|
||||
# Wikipedia-Seed: deterministisch de+en zur Topic-Bezeichnung —
|
||||
# umgeht Suche, Relevanz-Judge und Quellen-Deckel (2 Pflichtquellen)
|
||||
from urllib.parse import quote
|
||||
titel = db.one("SELECT COALESCE(NULLIF(titel,''), name) t FROM topics "
|
||||
"WHERE name=?", task["topic"])["t"]
|
||||
treffer = [{"titel": f"Wikipedia ({s}): {titel}",
|
||||
"url": f"https://{s}.wikipedia.org/wiki/"
|
||||
f"{quote(titel.replace(' ', '_'))}",
|
||||
"backend": f"wikipedia_{s}"} for s in ("de", "en")]
|
||||
fehler = []
|
||||
elif config.FAKE:
|
||||
from . import fakes
|
||||
treffer, fehler = fakes.suche(query), []
|
||||
else:
|
||||
treffer, fehler = await web_suchen(query)
|
||||
# Wikipedia zuerst (kuratierter Grundstock schlägt Blog-Treffer)
|
||||
treffer.sort(key=lambda t: not t["backend"].startswith("wikipedia"))
|
||||
if zweck == "korpus" and treffer and not config.FAKE and not seed:
|
||||
thema = db.one("SELECT COALESCE(NULLIF(titel,''), name) t FROM topics "
|
||||
"WHERE name=?", task["topic"])["t"]
|
||||
treffer = await _relevanz_filter(task, thema, treffer)
|
||||
# Quellen-Deckel je Topic (0 = unbegrenzt, dann stoppt nur die Sättigung)
|
||||
einstellung = db.one("SELECT quellen_max FROM topics WHERE name=?",
|
||||
task["topic"]) or {}
|
||||
deckel = einstellung.get("quellen_max")
|
||||
if deckel is None:
|
||||
deckel = config.QUELLEN_MAX
|
||||
neue = []
|
||||
for t in treffer:
|
||||
if zweck == "korpus" and deckel > 0 and not seed:
|
||||
vorhanden = db.one("SELECT COUNT(*) c FROM quellen WHERE topic=? AND "
|
||||
"zweck='korpus' AND status NOT IN "
|
||||
"('fehler','leer','robots','braucht_js')",
|
||||
task["topic"])["c"]
|
||||
if vorhanden >= deckel:
|
||||
break
|
||||
url_norm = url_normieren(t["url"])
|
||||
qid = db.insert("quellen", ignore=True, topic=task["topic"],
|
||||
titel=t["titel"][:200], url=t["url"], url_norm=url_norm,
|
||||
backend=t["backend"], runde=task["runde"], zweck=zweck)
|
||||
if qid:
|
||||
neue.append({"knoten": "laden", "item": f"quelle:{qid}",
|
||||
"payload": {"quelle_id": qid}})
|
||||
if fehler and not treffer:
|
||||
db.insert("befunde", run_id=task["run_id"], stufe="korpus", knoten="suche",
|
||||
art="degraded", item=query[:80], detail="; ".join(fehler)[:300])
|
||||
return engine.Ergebnis(daten={"treffer": len(treffer), "neu": len(neue)},
|
||||
neue_tasks=neue)
|
||||
270
backend/textkit.py
Normal file
270
backend/textkit.py
Normal file
@@ -0,0 +1,270 @@
|
||||
"""Textwerkzeuge. Regel: NFKC+casefold vor JEDEM Vergleich (Lektion 27).
|
||||
finde_zitat = 5 Stufen (Hypothes.is-Muster, Lektion 83): exakt → casefold →
|
||||
alnum-gefaltet → fuzzy (8 %) → Wort-Alignment. Falsch-Anker >> fehlender Anker."""
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from . import config
|
||||
|
||||
try:
|
||||
import Stemmer
|
||||
_stemmer = Stemmer.Stemmer("german")
|
||||
except ImportError: # Fallback ohne C-Extension: ungestemmt
|
||||
_stemmer = None
|
||||
|
||||
_WORT_RE = re.compile(r"[a-zäöüß0-9]+")
|
||||
_NEGATION = {"nicht", "kein", "keine", "keinen", "keiner", "nie", "niemals", "ohne"}
|
||||
|
||||
|
||||
def norm(text: str) -> str:
|
||||
return " ".join(unicodedata.normalize("NFKC", text).casefold().split())
|
||||
|
||||
|
||||
def tokens(text: str) -> set[str]:
|
||||
return set(_WORT_RE.findall(norm(text)))
|
||||
|
||||
|
||||
def stamm_tokens(text: str) -> set[str]:
|
||||
t = _WORT_RE.findall(norm(text))
|
||||
return set(_stemmer.stemWords(t)) if _stemmer else set(t)
|
||||
|
||||
|
||||
def jaccard(a: str, b: str, gestemmt: bool = True) -> float:
|
||||
ta, tb = (stamm_tokens(a), stamm_tokens(b)) if gestemmt else (tokens(a), tokens(b))
|
||||
if not ta or not tb:
|
||||
return 0.0
|
||||
return len(ta & tb) / len(ta | tb)
|
||||
|
||||
|
||||
def negations_menge(text: str) -> frozenset:
|
||||
"""Antonyme messen 0.91-0.95 Cosinus (Lektion 31) — harte Merge-Vorbedingung."""
|
||||
t = tokens(text)
|
||||
return frozenset(w for w in t if w in _NEGATION or w.startswith("nicht"))
|
||||
|
||||
|
||||
def titel_kern(titel: str) -> str:
|
||||
"""Alnum-Faltung (NFKD, Kombinationszeichen raus) — deterministischer
|
||||
Dubletten-Kandidat neben der Ähnlichkeit."""
|
||||
d = unicodedata.normalize("NFKD", titel).casefold()
|
||||
return "".join(c for c in d if c.isalnum())
|
||||
|
||||
|
||||
def ist_akronym_von(kurz: str, lang: str) -> bool:
|
||||
"""Akronym↔Expansion (Lektion 28 — fehlte in creator2): 'HTTP' ↔
|
||||
'Hypertext Transfer Protocol'."""
|
||||
k = titel_kern(kurz)
|
||||
woerter = _WORT_RE.findall(norm(lang))
|
||||
if len(k) < 2 or len(woerter) < 2 or len(k) > len(woerter) + 2:
|
||||
return False
|
||||
initialen = "".join(w[0] for w in woerter)
|
||||
return k == initialen or k == "".join(w[0] for w in woerter if len(w) > 2)
|
||||
|
||||
|
||||
def abschnitte(text: str, max_chars: int = config.ABSCHNITT_CHARS) -> list[tuple[int, str]]:
|
||||
"""Split an Absatzgrenzen (lost-in-the-middle-Guard) → [(offset, chunk)]."""
|
||||
ergebnis, start = [], 0
|
||||
while start < len(text):
|
||||
ende = min(start + max_chars, len(text))
|
||||
if ende < len(text):
|
||||
bruch = text.rfind("\n\n", start, ende)
|
||||
if bruch > start + max_chars // 4:
|
||||
ende = bruch
|
||||
ergebnis.append((start, text[start:ende]))
|
||||
start = ende
|
||||
return [(o, c) for o, c in ergebnis if c.strip()]
|
||||
|
||||
|
||||
def ueberlappung(a: tuple[int, int], b: tuple[int, int]) -> float:
|
||||
"""Span-Überlappung relativ zum kürzeren Span [0..1]."""
|
||||
schnitt = max(0, min(a[1], b[1]) - max(a[0], b[0]))
|
||||
kuerzer = min(a[1] - a[0], b[1] - b[0])
|
||||
return schnitt / kuerzer if kuerzer > 0 else 0.0
|
||||
|
||||
|
||||
_SATZ_RE = re.compile(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ])")
|
||||
|
||||
|
||||
def satz_split(text: str) -> list[str]:
|
||||
saetze = []
|
||||
for absatz in text.split("\n"):
|
||||
absatz = absatz.strip()
|
||||
if not absatz or absatz.startswith(("#", "```", "|", "<!--")):
|
||||
continue
|
||||
saetze += [s.strip() for s in _SATZ_RE.split(absatz) if s.strip()]
|
||||
return saetze
|
||||
|
||||
|
||||
# ── Positions-Maps für Anker-Suche ──
|
||||
|
||||
def _map_bauen(text: str, wandler) -> tuple[str, list[int]]:
|
||||
teile, positionen = [], []
|
||||
for i, zeichen in enumerate(text):
|
||||
for t in wandler(zeichen):
|
||||
teile.append(t)
|
||||
positionen.append(i)
|
||||
return "".join(teile), positionen
|
||||
|
||||
|
||||
def _ws_falten(text: str) -> tuple[str, list[int]]:
|
||||
out, pos, war_ws = [], [], False
|
||||
for i, c in enumerate(text):
|
||||
if c.isspace():
|
||||
if not war_ws and out:
|
||||
out.append(" ")
|
||||
pos.append(i)
|
||||
war_ws = True
|
||||
else:
|
||||
out.append(c)
|
||||
pos.append(i)
|
||||
war_ws = False
|
||||
return "".join(out), pos
|
||||
|
||||
|
||||
def _locker(text: str) -> tuple[str, list[int]]:
|
||||
"""Nur alnum, NFKD-Basisbuchstaben (dekomponierte Umlaute! Lektion 82)."""
|
||||
def wandler(c: str):
|
||||
for d in unicodedata.normalize("NFKD", c).casefold():
|
||||
if d.isalnum() and not unicodedata.combining(d):
|
||||
yield d
|
||||
return _map_bauen(text, wandler)
|
||||
|
||||
|
||||
def finde_zitat(text: str, zitat: str) -> tuple[int, int] | None:
|
||||
zitat = zitat.strip()
|
||||
if not zitat:
|
||||
return None
|
||||
# Stufe 1+2: whitespace-tolerant, dann casefold
|
||||
t_flach, t_pos = _ws_falten(text)
|
||||
z_flach, _ = _ws_falten(zitat)
|
||||
for haystack, needle in ((t_flach, z_flach),
|
||||
(t_flach.casefold(), z_flach.casefold())):
|
||||
i = haystack.find(needle)
|
||||
if i >= 0:
|
||||
return t_pos[i], t_pos[min(i + len(needle), len(t_pos)) - 1] + 1
|
||||
# Stufe 3: alnum-gefaltet
|
||||
t_locker, tl_pos = _locker(text)
|
||||
z_locker, _ = _locker(zitat)
|
||||
if len(z_locker) >= 10:
|
||||
erst = t_locker.find(z_locker)
|
||||
if erst >= 0 and (len(z_locker) >= 20
|
||||
or t_locker.find(z_locker, erst + 1) < 0):
|
||||
return tl_pos[erst], tl_pos[min(erst + len(z_locker), len(tl_pos)) - 1] + 1
|
||||
# Stufe 4: fuzzy (Levenshtein auf der Faltung)
|
||||
span = _fuzzy_span(t_locker, tl_pos, z_locker)
|
||||
if span and negations_menge(text[span[0]:span[1]]) == negations_menge(zitat):
|
||||
return span
|
||||
# Stufe 5: Wort-Alignment
|
||||
span = _wort_span(text, zitat)
|
||||
if span and negations_menge(text[span[0]:span[1]]) == negations_menge(zitat):
|
||||
return span
|
||||
return None
|
||||
|
||||
|
||||
def _fuzzy_span(t_locker: str, tl_pos: list[int], z_locker: str):
|
||||
if len(z_locker) < config.FUZZY_MIN_ZEICHEN:
|
||||
return None
|
||||
try:
|
||||
from fuzzysearch import find_near_matches
|
||||
except ImportError:
|
||||
return None
|
||||
dist = max(1, int(len(z_locker) * config.FUZZY_FEHLERQUOTE))
|
||||
treffer = find_near_matches(z_locker, t_locker, max_l_dist=dist)
|
||||
if not treffer:
|
||||
return None
|
||||
treffer.sort(key=lambda m: m.dist)
|
||||
best = treffer[0]
|
||||
# Eindeutigkeits-Guard: nicht-überlappender Zweittreffer fast gleicher Distanz
|
||||
for m in treffer[1:]:
|
||||
if m.start >= best.end or m.end <= best.start:
|
||||
if m.dist <= best.dist + 1:
|
||||
return None
|
||||
break
|
||||
return tl_pos[best.start], tl_pos[min(best.end, len(tl_pos)) - 1] + 1
|
||||
|
||||
|
||||
def _woerter_mit_spans(text: str) -> list[tuple[str, int, int]]:
|
||||
out = []
|
||||
for m in re.finditer(r"\S+", text):
|
||||
gefaltet = "".join(c for c in unicodedata.normalize("NFKD", m.group()).casefold()
|
||||
if c.isalnum() and not unicodedata.combining(c))
|
||||
if gefaltet:
|
||||
out.append((gefaltet, m.start(), m.end()))
|
||||
return out
|
||||
|
||||
|
||||
def _wort_span(text: str, zitat: str):
|
||||
z_woerter = [w for w, _, _ in _woerter_mit_spans(zitat)]
|
||||
if len(z_woerter) < config.WORT_MIN:
|
||||
return None
|
||||
t_woerter = _woerter_mit_spans(text)
|
||||
if not t_woerter:
|
||||
return None
|
||||
limit = int(len(z_woerter) * config.WORT_FEHLERQUOTE)
|
||||
# Kandidaten über das seltenste Zitat-Wort (kein Voll-Scan)
|
||||
haeufigkeit = {}
|
||||
for w, _, _ in t_woerter:
|
||||
haeufigkeit[w] = haeufigkeit.get(w, 0) + 1
|
||||
selten = min((w for w in z_woerter if w in haeufigkeit),
|
||||
key=lambda w: haeufigkeit[w], default=None)
|
||||
if selten is None:
|
||||
return None
|
||||
beste, beste_d = None, limit + 1
|
||||
for i, (w, _, _) in enumerate(t_woerter):
|
||||
if w != selten:
|
||||
continue
|
||||
von = max(0, i - len(z_woerter))
|
||||
bis = min(len(t_woerter), i + len(z_woerter))
|
||||
fenster = [x[0] for x in t_woerter[von:bis]]
|
||||
d = _wort_distanz(fenster, z_woerter, limit)
|
||||
if d is not None and d < beste_d:
|
||||
beste, beste_d = (von, bis), d
|
||||
elif d is not None and d == beste_d and beste and von != beste[0]:
|
||||
return None # mehrdeutig
|
||||
if beste is None:
|
||||
return None
|
||||
return t_woerter[beste[0]][1], t_woerter[beste[1] - 1][2]
|
||||
|
||||
|
||||
def _wort_distanz(a: list[str], b: list[str], limit: int) -> int | None:
|
||||
zeile = list(range(len(b) + 1))
|
||||
for i, wa in enumerate(a, 1):
|
||||
neu = [i]
|
||||
for j, wb in enumerate(b, 1):
|
||||
neu.append(min(zeile[j] + 1, neu[-1] + 1,
|
||||
zeile[j - 1] + (wa != wb)))
|
||||
if min(neu) > limit:
|
||||
return None
|
||||
zeile = neu
|
||||
return zeile[-1] if zeile[-1] <= limit else None
|
||||
|
||||
|
||||
# ── DELIMITED-Parser (===BLOCK===-Format, R5/R9: kein JSON) ──
|
||||
|
||||
def bloecke(text: str, marke: str) -> list[dict]:
|
||||
"""'===MARKE===\nKEY: wert\n…' → [{key: wert}]. Mehrzeilige Werte bis zum
|
||||
nächsten GROSS-KEY. Unbekannte Zeilen hängen am letzten Key."""
|
||||
ergebnis = []
|
||||
teile = re.split(rf"^===\s*{re.escape(marke)}\s*===\s*$", text,
|
||||
flags=re.MULTILINE)
|
||||
for teil in teile[1:]:
|
||||
felder: dict[str, str] = {}
|
||||
key = None
|
||||
for zeile in teil.splitlines():
|
||||
if zeile.strip().startswith("==="):
|
||||
break
|
||||
m = re.match(r"^([A-ZÄÖÜ_]{2,20}):\s?(.*)$", zeile)
|
||||
if m:
|
||||
key = m.group(1).lower()
|
||||
felder[key] = m.group(2)
|
||||
elif key and zeile.strip():
|
||||
felder[key] += "\n" + zeile
|
||||
if felder:
|
||||
ergebnis.append({k: v.strip() for k, v in felder.items()})
|
||||
return ergebnis
|
||||
|
||||
|
||||
def block_text(text: str, marke: str) -> str:
|
||||
"""Inhalt zwischen ===MARKE=== und dem nächsten ===…=== (oder Ende)."""
|
||||
m = re.search(rf"^===\s*{re.escape(marke)}\s*===\s*$(.*?)(?=^===|\Z)", text,
|
||||
flags=re.MULTILINE | re.DOTALL)
|
||||
return m.group(1).strip() if m else ""
|
||||
46
backend/ws.py
Normal file
46
backend/ws.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""WS-Hub (R1): EINE Queue + EIN Sender-Task — Broadcasts bleiben geordnet.
|
||||
Payload ist nur ein dirty-Signal {typ, topic, bereich}; der Client lädt selbst."""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
|
||||
class Hub:
|
||||
def __init__(self) -> None:
|
||||
self._clients: set = set()
|
||||
self._queue: asyncio.Queue | None = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
def bind_loop(self, loop: asyncio.AbstractEventLoop) -> None:
|
||||
self._loop = loop
|
||||
self._queue = asyncio.Queue()
|
||||
loop.create_task(self._sende_schleife())
|
||||
|
||||
def push(self, tabelle: str, row: dict) -> None:
|
||||
"""Thread-sicher aus der DB-Schicht aufrufbar."""
|
||||
if self._loop is None or self._queue is None:
|
||||
return
|
||||
bereich = {"tasks": "graph", "gate_laeufe": "graph", "befunde": "graph",
|
||||
"runs": "run", "topics": "run"}.get(tabelle, "graph")
|
||||
msg = json.dumps({"typ": "dirty", "bereich": bereich}, ensure_ascii=False)
|
||||
self._loop.call_soon_threadsafe(self._queue.put_nowait, msg)
|
||||
|
||||
async def _sende_schleife(self) -> None:
|
||||
while True:
|
||||
msg = await self._queue.get()
|
||||
tot = []
|
||||
for ws in list(self._clients):
|
||||
try:
|
||||
await ws.send_text(msg)
|
||||
except Exception:
|
||||
tot.append(ws)
|
||||
for ws in tot:
|
||||
self._clients.discard(ws)
|
||||
|
||||
def connect(self, ws) -> None:
|
||||
self._clients.add(ws)
|
||||
|
||||
def disconnect(self, ws) -> None:
|
||||
self._clients.discard(ws)
|
||||
|
||||
|
||||
hub = Hub()
|
||||
Reference in New Issue
Block a user