init

2026-07-22 16:12:23 +02:00
commit f07ef9653d
851 changed files with 501480 additions and 0 deletions
--- a/backend/database.py
+++ b/backend/database.py
@@ -0,0 +1,327 @@
+"""Planer-DB: NUR Lauf-Zustand (Kanban-Karten) + Telemetrie (Events) — wegwerfbar.
+Die Wahrheit über Projekte liegt als Dateien in deren Repos (.planer/), nie hier.
+
+Gekürzte Übernahme aus creator/backend/database.py: eine geteilte aiosqlite-Connection,
+Schreib-Serialisierung über einen loop-gebundenen Lock (_tx), WAL + busy_timeout.
+"""
+import asyncio
+import json
+from contextlib import asynccontextmanager
+from datetime import datetime, timedelta, timezone
+
+import aiosqlite
+
+import config
+
+CREATE_KANBAN_CARDS = """
+CREATE TABLE IF NOT EXISTS kanban_cards (
+    topic TEXT NOT NULL,
+    board TEXT NOT NULL,
+    card_id TEXT NOT NULL,
+    kind TEXT NOT NULL,
+    stage TEXT NOT NULL,
+    payload TEXT NOT NULL DEFAULT '{}',
+    retries INTEGER NOT NULL DEFAULT 0,
+    not_before TEXT NOT NULL DEFAULT '',
+    last_error TEXT,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, board, card_id)
+)
+"""
+
+CREATE_KANBAN_PULL_INDEX = """
+CREATE INDEX IF NOT EXISTS idx_kanban_pull ON kanban_cards(topic, board, stage, not_before, updated_at)
+"""
+
+CREATE_EVENTS = """
+CREATE TABLE IF NOT EXISTS events (
+    id INTEGER PRIMARY KEY,
+    topic TEXT NOT NULL,
+    ts TEXT NOT NULL,
+    kind TEXT NOT NULL,
+    key TEXT NOT NULL DEFAULT '',
+    label TEXT NOT NULL DEFAULT '',
+    status TEXT NOT NULL DEFAULT '',
+    dur_ms INTEGER,
+    wait_ms INTEGER,
+    meta TEXT NOT NULL DEFAULT '{}',
+    run_id TEXT NOT NULL DEFAULT ''
+)
+"""
+
+CREATE_EVENTS_INDEX = "CREATE INDEX IF NOT EXISTS idx_events ON events(topic, ts)"
+CREATE_EVENTS_RUN_INDEX = "CREATE INDEX IF NOT EXISTS idx_events_run ON events(topic, run_id)"
+
+_db: aiosqlite.Connection | None = None
+
+
+async def get_db() -> aiosqlite.Connection:
+    global _db
+    if _db is None:
+        config.DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+        _db = await aiosqlite.connect(config.DB_PATH)
+        _db.row_factory = None
+    return _db
+
+
+async def close_db() -> None:
+    global _db
+    if _db is not None:
+        await _db.close()
+        _db = None
+
+
+_write_lock: asyncio.Lock | None = None
+_write_lock_loop: asyncio.AbstractEventLoop | None = None
+
+
+def _get_write_lock() -> asyncio.Lock:
+    """Lock lazy an den AKTUELLEN Loop gebunden — ein modul-globaler Lock bricht,
+    wenn Tests jeden Fall in einem frischen Loop fahren."""
+    global _write_lock, _write_lock_loop
+    loop = asyncio.get_running_loop()
+    if _write_lock is None or _write_lock_loop is not loop:
+        _write_lock = asyncio.Lock()
+        _write_lock_loop = loop
+    return _write_lock
+
+
+@asynccontextmanager
+async def _tx():
+    """Atomarer Schreibblock: Lock + commit. Nur in Blatt-Funktionen verwenden,
+    die selbst committen — nie in Funktionen, die andere schreibende rufen."""
+    async with _get_write_lock():
+        db = await get_db()
+        try:
+            yield db
+            await db.commit()
+        except BaseException:
+            try:
+                await db.rollback()
+            except Exception:
+                pass
+            raise
+
+
+async def init_db() -> None:
+    db = await get_db()
+    await db.execute("PRAGMA journal_mode=WAL")
+    await db.execute("PRAGMA synchronous=NORMAL")
+    await db.execute("PRAGMA busy_timeout=5000")
+    await db.execute(CREATE_KANBAN_CARDS)
+    await db.execute(CREATE_KANBAN_PULL_INDEX)
+    await db.execute(CREATE_EVENTS)
+    await db.execute(CREATE_EVENTS_INDEX)
+    await db.execute(CREATE_EVENTS_RUN_INDEX)
+    await db.commit()
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _now_plus(seconds: float) -> str:
+    return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat()
+
+
+def _row_to_dict(row, cursor) -> dict:
+    return {d[0]: row[i] for i, d in enumerate(cursor.description)}
+
+
+def _card(row, cursor) -> dict:
+    """Row → dict mit dekodiertem Payload-JSON (bleibt unter 'payload')."""
+    d = _row_to_dict(row, cursor)
+    try:
+        d["payload"] = json.loads(d.get("payload") or "{}")
+    except (TypeError, ValueError):
+        d["payload"] = {}
+    return d
+
+
+# --- Karten ------------------------------------------------------------------------
+
+async def kanban_pull(topic: str, board: str, stage: str, limit: int) -> list[dict]:
+    """`limit` bereite Karten der Stufe (Backoff abgelaufen). LPT: Karten mit
+    `groesse`-Payload zuerst (größter Chunk startet früh), sonst FIFO."""
+    db = await get_db()
+    cursor = await db.execute(
+        """SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? AND not_before <= ?
+           ORDER BY COALESCE(json_extract(payload, '$.groesse'), 0) DESC, updated_at LIMIT ?""",
+        (topic, board, stage, _now(), limit))
+    return [_card(row, cursor) for row in await cursor.fetchall()]
+
+
+async def kanban_count(topic: str, stages, board: str | None = None) -> int:
+    """Karten in einer der `stages` (str oder Liste) — Queue-Länge/Quieszenz.
+    Karten im Backoff zählen mit: ihre Arbeit ist nicht getan."""
+    if isinstance(stages, str):
+        stages = [stages]
+    if not stages:
+        return 0
+    db = await get_db()
+    ph = ",".join("?" * len(stages))
+    sql = f"SELECT count(*) FROM kanban_cards WHERE topic = ? AND stage IN ({ph})"
+    args: tuple = (topic, *stages)
+    if board:
+        sql += " AND board = ?"
+        args += (board,)
+    cursor = await db.execute(sql, args)
+    return (await cursor.fetchone())[0]
+
+
+async def kanban_advance(topic: str, board: str, card_id: str, stage: str) -> None:
+    await kanban_advance_many(topic, board, [(card_id, stage)])
+
+
+async def kanban_advance_many(topic: str, board: str, moves: list[tuple[str, str]]) -> None:
+    """Batch-Stufenwechsel in EINEM Commit (der Flow advanced ganze Pakete)."""
+    if not moves:
+        return
+    async with _tx() as db:
+        now = _now()
+        await db.executemany(
+            """UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ?
+               WHERE topic = ? AND board = ? AND card_id = ?""",
+            [(stage, now, topic, board, cid) for cid, stage in moves])
+        await _add_events_many(db, topic, [("stage", f"{board}:{cid}", "", stage) for cid, stage in moves])
+
+
+async def kanban_upsert_card(topic: str, board: str, card_id: str, kind: str, stage: str,
+                             payload: dict | None = None) -> None:
+    """Insert oder Overwrite (stabile IDs upserted, nie dupliziert).
+    payload=None behält bei Konflikt das bestehende Payload."""
+    async with _tx() as db:
+        await db.execute(
+            """INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at)
+               VALUES (?, ?, ?, ?, ?, COALESCE(?, '{}'), ?)
+               ON CONFLICT(topic, board, card_id) DO UPDATE SET
+                   kind = excluded.kind, stage = excluded.stage,
+                   payload = COALESCE(?, kanban_cards.payload),
+                   retries = 0, not_before = '', last_error = NULL, updated_at = excluded.updated_at""",
+            (topic, board, card_id, kind, stage,
+             json.dumps(payload, ensure_ascii=False) if payload is not None else None, _now(),
+             json.dumps(payload, ensure_ascii=False) if payload is not None else None))
+
+
+async def kanban_set_payload(topic: str, board: str, card_id: str, payload: dict) -> None:
+    async with _tx() as db:
+        await db.execute(
+            "UPDATE kanban_cards SET payload = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?",
+            (json.dumps(payload, ensure_ascii=False), _now(), topic, board, card_id))
+
+
+async def kanban_get_card(topic: str, board: str, card_id: str) -> dict | None:
+    db = await get_db()
+    cursor = await db.execute(
+        "SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?", (topic, board, card_id))
+    row = await cursor.fetchone()
+    return _card(row, cursor) if row else None
+
+
+async def kanban_cards(topic: str, board: str | None = None, stage: str | None = None,
+                       kind: str | None = None) -> list[dict]:
+    db = await get_db()
+    sql, args = "SELECT * FROM kanban_cards WHERE topic = ?", [topic]
+    for col, val in (("board", board), ("stage", stage), ("kind", kind)):
+        if val is not None:
+            sql += f" AND {col} = ?"
+            args.append(val)
+    cursor = await db.execute(sql, args)
+    return [_card(row, cursor) for row in await cursor.fetchall()]
+
+
+async def kanban_fail_card(topic: str, board: str, card_id: str, error: str,
+                           max_retries: int, backoff_base: float = 30.0) -> bool:
+    """Verarbeitungsfehler: retries++, exponentieller Backoff (not_before), nach
+    `max_retries` → stage 'dead' (Dead-Letter, requeue-bar). → True wenn dead."""
+    async with _tx() as db:
+        cursor = await db.execute(
+            "SELECT retries FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?",
+            (topic, board, card_id))
+        row = await cursor.fetchone()
+        if row is None:
+            return False
+        retries = (row[0] or 0) + 1
+        dead = retries >= max_retries
+        if dead:
+            await db.execute(
+                """UPDATE kanban_cards SET stage = 'dead', retries = ?, not_before = '',
+                   last_error = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?""",
+                (retries, error[:500], _now(), topic, board, card_id))
+        else:
+            await db.execute(
+                """UPDATE kanban_cards SET retries = ?, not_before = ?, last_error = ?, updated_at = ?
+                   WHERE topic = ? AND board = ? AND card_id = ?""",
+                (retries, _now_plus(backoff_base * (2 ** (retries - 1))), error[:500], _now(),
+                 topic, board, card_id))
+        await db.execute(
+            "INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta, run_id) VALUES (?,?,?,?,?,?,?,?,?,?)",
+            (topic, _now(), "fail", f"{board}:{card_id}", "", "dead" if dead else f"retry{retries}",
+             None, None, json.dumps({"error": error[:200]}, ensure_ascii=False), _current_run.get(topic, "")))
+        return dead
+
+
+async def kanban_dead(topic: str) -> list[dict]:
+    return await kanban_cards(topic, stage="dead")
+
+
+async def kanban_requeue_dead(topic: str, board: str, stage: str) -> int:
+    """dead → `stage` (frische Retries). → Anzahl requeueter Karten."""
+    async with _tx() as db:
+        cursor = await db.execute(
+            """UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ?
+               WHERE topic = ? AND board = ? AND stage = 'dead'""",
+            (stage, _now(), topic, board))
+        await _add_events_many(db, topic, [("reset", f"{board}:requeue-dead", "", stage)])
+        return cursor.rowcount
+
+
+async def kanban_stage_counts(topic: str) -> dict[str, dict[str, int]]:
+    """{board: {stage: count}} — das Live-Board."""
+    db = await get_db()
+    cursor = await db.execute(
+        "SELECT board, stage, count(*) FROM kanban_cards WHERE topic = ? GROUP BY board, stage", (topic,))
+    out: dict[str, dict[str, int]] = {}
+    for board, stage, n in await cursor.fetchall():
+        out.setdefault(board, {})[stage] = n
+    return out
+
+
+async def letzter_agent_ts(topic: str) -> str | None:
+    """Zeitstempel des letzten Agent-Events (auch aus CLI-Läufen — gleiche DB)."""
+    db = await get_db()
+    cursor = await db.execute(
+        "SELECT MAX(ts) FROM events WHERE topic = ? AND kind = 'agent'", (topic,))
+    return (await cursor.fetchone())[0]
+
+
+# --- Telemetrie ---------------------------------------------------------------------
+
+_current_run: dict[str, str] = {}
+
+
+def set_current_run(topic: str, run_id: str | None) -> None:
+    if run_id:
+        _current_run[topic] = run_id
+    else:
+        _current_run.pop(topic, None)
+
+
+async def add_event(topic: str, kind: str, key: str = "", label: str = "", status: str = "",
+                    dur_ms: int | None = None, wait_ms: int | None = None, meta: dict | None = None) -> None:
+    """Eine Telemetrie-Zeile, eigener Commit. Aufrufer behandeln das fire-and-forget."""
+    async with _tx() as db:
+        await db.execute(
+            "INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta, run_id) VALUES (?,?,?,?,?,?,?,?,?,?)",
+            (topic, _now(), kind, key, label, status, dur_ms, wait_ms,
+             json.dumps(meta or {}, ensure_ascii=False), _current_run.get(topic, "")))
+
+
+async def _add_events_many(db, topic: str, rows: list[tuple]) -> None:
+    """Batch-Insert OHNE Commit — muss in der Transaktion des Aufrufers laufen."""
+    now = _now()
+    rid = _current_run.get(topic, "")
+    await db.executemany(
+        "INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta, run_id) VALUES (?,?,?,?,?,?,?,?,?,?)",
+        [(topic, now, kind, key, label, status, None, None, "{}", rid)
+         for kind, key, label, status in rows])