import asyncio import json from contextlib import asynccontextmanager import aiosqlite from config import DB_PATH CREATE_GUIDES = """ CREATE TABLE IF NOT EXISTS guides ( id TEXT PRIMARY KEY, topic TEXT NOT NULL, format TEXT NOT NULL, instructions TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'queued', progress TEXT, step INTEGER, error_msg TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL ) """ CREATE_TOPICS = """ CREATE TABLE IF NOT EXISTS topics ( name TEXT PRIMARY KEY, created_at TEXT NOT NULL ) """ CREATE_BLOCK_PROGRESS = """ CREATE TABLE IF NOT EXISTS block_progress ( topic TEXT NOT NULL, block TEXT NOT NULL, good_answers INTEGER NOT NULL DEFAULT 0, streak INTEGER NOT NULL DEFAULT 0, completed TEXT, understood TEXT, mastered TEXT, updated_at TEXT NOT NULL, PRIMARY KEY (topic, block) ) """ # --- Blocks pipeline content (replaces file sidecars) --- # Inventory: one block per (topic, title_norm). mentions = number of agents/rounds # that named it (≥2 = consensus). status: candidate/consensus/rest/discarded. CREATE_BLOCKS = """ CREATE TABLE IF NOT EXISTS blocks ( topic TEXT NOT NULL, title_norm TEXT NOT NULL, title TEXT NOT NULL, description TEXT NOT NULL DEFAULT '', mentions INTEGER NOT NULL DEFAULT 1, status TEXT NOT NULL DEFAULT 'candidate', sources TEXT NOT NULL DEFAULT '[]', reader TEXT NOT NULL DEFAULT '[]', updated_at TEXT NOT NULL, PRIMARY KEY (topic, title_norm) ) """ # Subblocks per block. level (einfach/mittel/schwer) + relevance (relevant/rand) # are set later. mentions analogous to the inventory. CREATE_SUBBLOCKS = """ CREATE TABLE IF NOT EXISTS subblocks ( topic TEXT NOT NULL, block_norm TEXT NOT NULL, sub_norm TEXT NOT NULL, block TEXT NOT NULL, sub_title TEXT NOT NULL, mentions INTEGER NOT NULL DEFAULT 1, level TEXT, relevance TEXT, facts TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'candidate', updated_at TEXT NOT NULL, PRIMARY KEY (topic, block_norm, sub_norm) ) """ # One question pattern per (block, subblock). Difficulty comes only at exam time # from the learner tier, not from the pattern — hence no more type cross-product. CREATE_QUESTION_PATTERN = """ CREATE TABLE IF NOT EXISTS question_pattern ( topic TEXT NOT NULL, block_norm TEXT NOT NULL, sub_norm TEXT NOT NULL, block TEXT NOT NULL, sub_title TEXT NOT NULL, question TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (topic, block_norm, sub_norm) ) """ # Crawl pages per topic: content (1=content/0=noise, from the triage) + read_done (from the research). CREATE_RESEARCH_COVERAGE = """ CREATE TABLE IF NOT EXISTS research_coverage ( topic TEXT NOT NULL, source TEXT NOT NULL, read_done INTEGER NOT NULL DEFAULT 0, content INTEGER, updated_at TEXT NOT NULL, PRIMARY KEY (topic, source) ) """ # Step status of the pipeline (replaces file-existence resume + reset globs). CREATE_PIPELINE_STATE = """ CREATE TABLE IF NOT EXISTS pipeline_state ( topic TEXT NOT NULL, step TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'open', updated_at TEXT NOT NULL, PRIMARY KEY (topic, step) ) """ # Finished guide content per (topic, format) as a JSON blob (replaces the guide JSON file). # Shared across all guide runs of the same topic+format (as the content file was before). CREATE_GUIDE_CONTENT = """ CREATE TABLE IF NOT EXISTS guide_content ( topic TEXT NOT NULL, format TEXT NOT NULL, json TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (topic, format) ) """ # Source choice per topic (replaces source.json). CREATE_SOURCE = """ CREATE TABLE IF NOT EXISTS source ( topic TEXT PRIMARY KEY, type TEXT NOT NULL, location TEXT NOT NULL DEFAULT '', spec TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL ) """ # Outline (chapter → block numbers) per topic as JSON. Produced in the blocks phase, # the guide only reads it + filters per format. Format-agnostic (all blocks). CREATE_GUIDE_OUTLINE = """ CREATE TABLE IF NOT EXISTS guide_outline ( topic TEXT PRIMARY KEY, json TEXT NOT NULL, updated_at TEXT NOT NULL ) """ # Generic learning artifact layer: one artifact per (block, subblock, type) (JSON in `data`). # type: flashcard | example. sub_norm='' = block level (for future block-wide artifacts). # Produced from the facts in the blocks phase, presented by the frontend (artifact/display split). CREATE_SUB_ARTEFAKTE = """ CREATE TABLE IF NOT EXISTS sub_artefakte ( topic TEXT NOT NULL, block_norm TEXT NOT NULL, sub_norm TEXT NOT NULL DEFAULT '', type TEXT NOT NULL, block TEXT NOT NULL DEFAULT '', sub_title TEXT NOT NULL DEFAULT '', data TEXT NOT NULL DEFAULT '{}', updated_at TEXT NOT NULL, PRIMARY KEY (topic, block_norm, sub_norm, type) ) """ # Leitner learning state per flashcard — IDENTITY-keyed (no content): survives the # sub_artefakte wipe on regeneration; orphaned rows simply never match in the deck join. CREATE_PRACTICE_PROGRESS = """ CREATE TABLE IF NOT EXISTS practice_progress ( topic TEXT NOT NULL, block_norm TEXT NOT NULL, sub_norm TEXT NOT NULL, box INTEGER NOT NULL DEFAULT 1, due_at TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (topic, block_norm, sub_norm) ) """ # Kanban dataflow (boards 'inventory' + 'artefacts'): ONE generic card table for all card kinds # (title/cluster/block). `stage` is the queue key — a worker pulls WHERE stage = . # `payload` is a JSON blob (title, description, sources, readers, mentions, parent_norm, journal …); # retries/not_before/last_error implement backoff + dead-letter (stage 'dead' after MAX_CARD_RETRIES). CREATE_KANBAN_CARDS = """ CREATE TABLE IF NOT EXISTS kanban_cards ( topic TEXT NOT NULL, board TEXT NOT NULL, card_id TEXT NOT NULL, kind TEXT NOT NULL, stage TEXT NOT NULL, payload TEXT NOT NULL DEFAULT '{}', retries INTEGER NOT NULL DEFAULT 0, not_before TEXT NOT NULL DEFAULT '', last_error TEXT, updated_at TEXT NOT NULL, PRIMARY KEY (topic, board, card_id) ) """ # Pipeline history (agents, stage moves, failures) — the raw data for quality/perf # analysis. Written fire-and-forget from the hooks in kanban_advance_many / kanban_fail_card / # set_guide_card and agents.run_agent (injected via agents.on_event); never load-bearing. CREATE_EVENTS = """ CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY, topic TEXT NOT NULL, ts TEXT NOT NULL, kind TEXT NOT NULL, key TEXT NOT NULL DEFAULT '', label TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT '', dur_ms INTEGER, wait_ms INTEGER, meta TEXT NOT NULL DEFAULT '{}', run_id TEXT NOT NULL DEFAULT '' ) """ CREATE_EVENTS_INDEX = """ CREATE INDEX IF NOT EXISTS idx_events ON events(topic, ts) """ # run-Summaries filtern topic + run_id — ohne den Index wird jeder Aufruf ein Topic-Scan CREATE_EVENTS_RUN_INDEX = """ CREATE INDEX IF NOT EXISTS idx_events_run ON events(topic, run_id) """ CREATE_KANBAN_PULL_INDEX = """ CREATE INDEX IF NOT EXISTS idx_kanban_pull ON kanban_cards(topic, board, stage, not_before, updated_at) """ # title_norm → cluster membership (one title belongs to exactly one cluster). CREATE_KANBAN_MEMBERS = """ CREATE TABLE IF NOT EXISTS kanban_members ( topic TEXT NOT NULL, member_id TEXT NOT NULL, group_id TEXT NOT NULL, PRIMARY KEY (topic, member_id) ) """ # Guide board: one card per block, linear stages (lernziele … lesbarkeit → done). # `md` carries the writer fragment (with kapitel/section/sub markers) between the gates. CREATE_GUIDE_CARDS = """ CREATE TABLE IF NOT EXISTS guide_cards ( topic TEXT NOT NULL, format TEXT NOT NULL DEFAULT 'Guide', block_norm TEXT NOT NULL, block TEXT NOT NULL, chapter TEXT NOT NULL DEFAULT '', ord INTEGER NOT NULL DEFAULT 0, stage TEXT NOT NULL DEFAULT 'lernziele', status TEXT NOT NULL DEFAULT 'open', writer_rounds INTEGER NOT NULL DEFAULT 0, gate_info TEXT NOT NULL DEFAULT '', md TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL, PRIMARY KEY (topic, format, block_norm) ) """ CREATE_GUIDE_LERNZIELE = """ CREATE TABLE IF NOT EXISTS guide_lernziele ( topic TEXT NOT NULL, block_norm TEXT NOT NULL, ziel_id TEXT NOT NULL, text TEXT NOT NULL, sub_norm TEXT NOT NULL DEFAULT '', covered INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL, PRIMARY KEY (topic, block_norm, ziel_id) ) """ _db: aiosqlite.Connection | None = None async def get_db() -> aiosqlite.Connection: global _db if _db is None: _db = await aiosqlite.connect(DB_PATH) _db.row_factory = None return _db _write_lock: asyncio.Lock | None = None _write_lock_loop: asyncio.AbstractEventLoop | None = None def _get_write_lock() -> asyncio.Lock: """Lock lazily bound to the CURRENT loop: an asyncio.Lock is loop-bound, and a module-global one breaks when tests run each case in a fresh loop.""" global _write_lock, _write_lock_loop loop = asyncio.get_running_loop() if _write_lock is None or _write_lock_loop is not loop: _write_lock = asyncio.Lock() _write_lock_loop = loop return _write_lock @asynccontextmanager async def _tx(): """Atomarer Schreibblock: Lock + commit. Die eine geteilte Connection interleavt sonst fremde Commits zwischen execute und commit eines Batches. Lock NUR in Blatt-Funktionen, die selbst committen — nie in Funktionen, die andere schreibende DB-Funktionen rufen.""" async with _get_write_lock(): db = await get_db() try: yield db await db.commit() except BaseException: try: await db.rollback() except Exception: pass raise async def init_db(): db = await get_db() # WAL survives crashes much better; busy_timeout absorbs short locks. # synchronous=NORMAL: safe under WAL, much less fsync — the kanban flow commits often. await db.execute("PRAGMA journal_mode=WAL") await db.execute("PRAGMA synchronous=NORMAL") await db.execute("PRAGMA busy_timeout=5000") await db.execute(CREATE_GUIDES) await db.execute(CREATE_TOPICS) await db.execute(CREATE_BLOCK_PROGRESS) await db.execute(CREATE_BLOCKS) await db.execute(CREATE_SUBBLOCKS) await db.execute(CREATE_QUESTION_PATTERN) await db.execute(CREATE_RESEARCH_COVERAGE) await db.execute(CREATE_PIPELINE_STATE) await db.execute(CREATE_GUIDE_CONTENT) await db.execute(CREATE_SOURCE) await db.execute(CREATE_GUIDE_OUTLINE) await db.execute(CREATE_SUB_ARTEFAKTE) await db.execute(CREATE_PRACTICE_PROGRESS) await db.execute(CREATE_KANBAN_CARDS) await db.execute(CREATE_EVENTS) await db.execute(CREATE_EVENTS_INDEX) await db.execute(CREATE_KANBAN_PULL_INDEX) await db.execute(CREATE_KANBAN_MEMBERS) await db.execute(CREATE_GUIDE_CARDS) await db.execute(CREATE_GUIDE_LERNZIELE) try: # migration for existing DBs without the step column await db.execute("ALTER TABLE guides ADD COLUMN step INTEGER") except aiosqlite.OperationalError: pass try: # migration: research_coverage.content (content/noise from the triage) await db.execute("ALTER TABLE research_coverage ADD COLUMN content INTEGER") except aiosqlite.OperationalError: pass try: # migration for existing DBs without the understood column (mastery level) await db.execute("ALTER TABLE block_progress ADD COLUMN understood TEXT") except aiosqlite.OperationalError: pass try: # migration for existing DBs without the mastered column (master path 25) await db.execute("ALTER TABLE block_progress ADD COLUMN mastered TEXT") except aiosqlite.OperationalError: pass try: # migration for existing DBs without the streak column (persistent streak-bonus run) await db.execute("ALTER TABLE block_progress ADD COLUMN streak INTEGER NOT NULL DEFAULT 0") except aiosqlite.OperationalError: pass # Open-question anchor: base/streak BEFORE the currently open question — makes the rating # idempotent server-side (re-rating) and drift-free across questions. for _col, _type in (("offene_question", "TEXT"), ("offene_basis", "INTEGER"), ("offene_streak", "INTEGER")): try: await db.execute(f"ALTER TABLE block_progress ADD COLUMN {_col} {_type}") except aiosqlite.OperationalError: pass # Migration: question_pattern without a type column (1 pattern per sub instead of a sub×type cross-product). # PK change → rebuild the table once. Existing patterns are lost (intentional, no mapping). cursor = await db.execute("PRAGMA table_info(question_pattern)") if any(_r[1] == "type" for _r in await cursor.fetchall()): await db.execute("DROP TABLE question_pattern") await db.execute(CREATE_QUESTION_PATTERN) try: # migration: subblocks.facts (source facts per sub, JSON blob) — extract-once grounding. await db.execute("ALTER TABLE subblocks ADD COLUMN facts TEXT NOT NULL DEFAULT ''") # DEFAULT '' needed for NOT NULL on ADD COLUMN except aiosqlite.OperationalError: pass try: # migration: blocks.reader (reader set per candidate, JSON) — exact consensus count (≥2 readers) await db.execute("ALTER TABLE blocks ADD COLUMN reader TEXT NOT NULL DEFAULT '[]'") except aiosqlite.OperationalError: pass # Migration: removed features leave orphaned tables behind — drop them. # elements (feature removed), vertiefungen/block_texte (never read), guide_progress # (chapter progress had no frontend and its only reader ignored the value). await db.execute("DROP TABLE IF EXISTS elements") await db.execute("DROP TABLE IF EXISTS vertiefungen") await db.execute("DROP TABLE IF EXISTS block_texte") await db.execute("DROP TABLE IF EXISTS guide_progress") try: # migration: run_id per generation run (QA groups events by it) await db.execute("ALTER TABLE events ADD COLUMN run_id TEXT NOT NULL DEFAULT ''") except aiosqlite.OperationalError: pass await db.execute(CREATE_EVENTS_RUN_INDEX) # nach der run_id-Migration — Spalte muss existieren # Retention: events wachsen sonst unbegrenzt (gelöscht wurde nur per Topic-Delete) from config import EVENTS_RETENTION_TAGE await db.execute("DELETE FROM events WHERE ts < datetime('now', ?)", (f"-{EVENTS_RETENTION_TAGE} days",)) await db.execute( "UPDATE guides SET status = 'error', progress = NULL, error_msg = 'Server restart' " "WHERE status IN ('queued', 'generating')" ) await db.commit() async def close_db(): global _db if _db is not None: await _db.close() _db = None def _row_to_dict(row, cursor): columns = [d[0] for d in cursor.description] return dict(zip(columns, row)) async def create_guide(guide: dict) -> dict: async with _tx() as db: await db.execute( """INSERT INTO guides (id, topic, format, instructions, status, progress, created_at, updated_at) VALUES (:id, :topic, :format, :instructions, :status, :progress, :created_at, :updated_at)""", guide, ) return guide async def get_guide(guide_id: str) -> dict | None: db = await get_db() cursor = await db.execute("SELECT * FROM guides WHERE id = ?", (guide_id,)) row = await cursor.fetchone() if row is None: return None return _row_to_dict(row, cursor) async def list_guides() -> list[dict]: db = await get_db() cursor = await db.execute("SELECT * FROM guides ORDER BY created_at DESC") rows = await cursor.fetchall() return [_row_to_dict(row, cursor) for row in rows] async def _update(table: str, fields: dict, where: dict) -> None: """UPDATE SET WHERE (+ commit). WHERE params are aliased (`w_`) so a field and a WHERE key of the same name don't collide — needed e.g. for a `title_norm` rename (SET new norm WHERE old norm).""" sets = ", ".join(f"{k} = :{k}" for k in fields) cond = " AND ".join(f"{k} = :w_{k}" for k in where) async with _tx() as db: await db.execute(f"UPDATE {table} SET {sets} WHERE {cond}", {**fields, **{f"w_{k}": v for k, v in where.items()}}) async def update_guide(guide_id: str, **fields) -> None: await _update("guides", fields, {"id": guide_id}) async def delete_guide(guide_id: str) -> bool: async with _tx() as db: cursor = await db.execute("DELETE FROM guides WHERE id = ?", (guide_id,)) return cursor.rowcount > 0 # --- Topics --- async def create_topic(name: str) -> None: from datetime import datetime, timezone async with _tx() as db: await db.execute( "INSERT OR IGNORE INTO topics (name, created_at) VALUES (?, ?)", (name, datetime.now(timezone.utc).isoformat()), ) async def list_topics() -> list[str]: db = await get_db() cursor = await db.execute("SELECT name FROM topics ORDER BY created_at DESC") rows = await cursor.fetchall() return [row[0] for row in rows] async def delete_topic(name: str) -> None: async with _tx() as db: await db.execute("DELETE FROM topics WHERE name = ?", (name,)) # --- Block learning: deep-dives + exam progress --- def _now() -> str: from datetime import datetime, timezone return datetime.now(timezone.utc).isoformat() async def list_block_progress(topic: str) -> list[dict]: db = await get_db() cursor = await db.execute( "SELECT block, good_answers, streak, completed, understood, mastered FROM block_progress WHERE topic = ?", (topic,) ) rows = await cursor.fetchall() return [{"block": b, "good_answers": n, "streak": s, "completed": a, "understood": v, "mastered": m} for b, n, s, a, v, m in rows] async def get_block_progress(topic: str, block: str) -> dict: """One block row incl. open-question anchor. Defaults if none exists yet.""" db = await get_db() cursor = await db.execute( "SELECT good_answers, streak, completed, understood, mastered, " "offene_question, offene_basis, offene_streak FROM block_progress " "WHERE topic = ? AND block = ?", (topic, block), ) row = await cursor.fetchone() if row is None: return {"good_answers": 0, "streak": 0, "completed": None, "understood": None, "mastered": None, "offene_question": None, "offene_basis": None, "offene_streak": None} return {"good_answers": row[0], "streak": row[1], "completed": row[2], "understood": row[3], "mastered": row[4], "offene_question": row[5], "offene_basis": row[6], "offene_streak": row[7]} async def set_open_question(topic: str, block: str, question: str, basis: int, streak: int) -> None: """Freeze base + streak BEFORE the now-open question (anchor for idempotent re-rating).""" async with _tx() as db: now = _now() await db.execute( """INSERT INTO block_progress (topic, block, offene_question, offene_basis, offene_streak, updated_at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(topic, block) DO UPDATE SET offene_question = excluded.offene_question, offene_basis = excluded.offene_basis, offene_streak = excluded.offene_streak, updated_at = excluded.updated_at""", (topic, block, question, basis, streak, now), ) async def set_block_score_and_streak(topic: str, block: str, score: int, streak: int) -> tuple[int, int]: """Set score + streak atomically (clamped by the caller). Returns (score, streak).""" async with _tx() as db: await db.execute( """INSERT INTO block_progress (topic, block, good_answers, streak, updated_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(topic, block) DO UPDATE SET good_answers = excluded.good_answers, streak = excluded.streak, updated_at = excluded.updated_at""", (topic, block, score, streak, _now()), ) return score, streak async def delete_block_progress(topic: str, block: str) -> None: """Reset the progress of ONE block: delete the row (score/streak/flags/open question). If the row is missing, get_block_progress returns defaults (0) — i.e. a full reset.""" async with _tx() as db: await db.execute("DELETE FROM block_progress WHERE topic = ? AND block = ?", (topic, block)) # Sub-level from the two orthogonal columns: peripheral → 4 (V), otherwise level (learning-path position): # beginner/NULL → 1, advanced → 2, expert → 3. Old values (einfach/mittel/schwer) are # mapped in backward-compatibly. Drives the guide view A/F/E/V + cap (unlocked subs × 25). _LEVEL_CASE = """CASE WHEN relevance = 'peripheral' THEN 4 WHEN level IN ('advanced', 'medium') THEN 2 WHEN level IN ('expert', 'hard') THEN 3 ELSE 1 END""" def _empty_levels() -> dict[int, int]: return {1: 0, 2: 0, 3: 0, 4: 0} async def subs_per_level(topic: str, block: str) -> dict[int, int]: """Consensus subblocks of a block per level 1–4. Pass in the raw block title.""" from textkit import _norm_title db = await get_db() cursor = await db.execute( # alias must NOT be named `level`: SQLite resolves an ambiguous GROUP BY name to # the source COLUMN, which silently miscounts peripheral subs f"SELECT {_LEVEL_CASE} AS lv, COUNT(*) FROM subblocks " "WHERE topic = ? AND block_norm = ? AND status = 'consensus' GROUP BY lv", (topic, _norm_title(block)), ) out = _empty_levels() for level, n in await cursor.fetchall(): out[level] = n return out async def subs_per_level_raw(topic: str) -> dict[str, dict[int, int]]: """Subblocks per level, grouped by RAW block title (= guide section title).""" db = await get_db() cursor = await db.execute( f"SELECT block, {_LEVEL_CASE} AS lv, COUNT(*) FROM subblocks " "WHERE topic = ? AND status = 'consensus' GROUP BY block, lv", (topic,), ) out: dict[str, dict[int, int]] = {} for b, level, n in await cursor.fetchall(): out.setdefault(b, _empty_levels())[level] = n return out async def subs_per_level_all() -> dict[tuple[str, str], dict[int, int]]: """Subblocks per level per (topic, block_norm) — for the topic-wide levels derivation.""" db = await get_db() cursor = await db.execute( f"SELECT topic, block_norm, {_LEVEL_CASE} AS lv, COUNT(*) FROM subblocks " "WHERE status = 'consensus' GROUP BY topic, block_norm, lv" ) out: dict[tuple[str, str], dict[int, int]] = {} for t, bn, level, n in await cursor.fetchall(): out.setdefault((t, bn), _empty_levels())[level] = n return out async def subs_with_level(topic: str, block: str) -> list[dict]: """Consensus subblocks of a block with title, sub_norm and level 1–4.""" from textkit import _norm_title db = await get_db() cursor = await db.execute( f"SELECT sub_title, sub_norm, {_LEVEL_CASE} AS level FROM subblocks " "WHERE topic = ? AND block_norm = ? AND status = 'consensus'", (topic, _norm_title(block)), ) return [{"title": t, "norm": sn, "level": e} for t, sn, e in await cursor.fetchall()] async def list_block_scores_all() -> list[tuple[str, str, int]]: """(topic, block, good_answers) per block — raw data for the levels derivation.""" db = await get_db() cursor = await db.execute("SELECT topic, block, good_answers FROM block_progress") return [(t, b, n) for t, b, n in await cursor.fetchall()] async def delete_block_data(topic: str) -> None: async with _tx() as db: await db.execute("DELETE FROM block_progress WHERE topic = ?", (topic,)) # --- Blocks pipeline content: inventory / subblocks / question pattern / coverage / state / source --- async def upsert_block(topic: str, title_norm: str, title: str, description: str = "", sources: list | None = None, reader: str | None = None) -> None: """Insert a candidate or union the reader set. The first description is kept. `reader` = ID of the research reader (e.g. "a5-1"). The union runs race-free in ONE statement (json1), because several reader coroutines upsert concurrently — a read-modify-write across `await` would lose members. `mentions` stays in sync with `len(reader)`. `reader=None` (e.g. from `_set_inventar`) leaves the set unchanged.""" async with _tx() as db: rid = reader if isinstance(reader, str) and reader else None await db.execute( """INSERT INTO blocks (topic, title_norm, title, description, mentions, status, sources, reader, updated_at) VALUES (?, ?, ?, ?, 1, 'candidate', ?, ?, ?) ON CONFLICT(topic, title_norm) DO UPDATE SET reader = (SELECT json_group_array(v) FROM ( SELECT value AS v FROM json_each(blocks.reader) UNION SELECT ? WHERE ? IS NOT NULL)), mentions = (SELECT count(*) FROM ( SELECT value AS v FROM json_each(blocks.reader) UNION SELECT ? WHERE ? IS NOT NULL)), sources = excluded.sources, updated_at = excluded.updated_at""", (topic, title_norm, title, description, json.dumps(sources or [], ensure_ascii=False), json.dumps([rid] if rid else [], ensure_ascii=False), _now(), rid, rid, rid, rid), ) async def list_blocks(topic: str, status: str | None = None) -> list[dict]: db = await get_db() if status is None: cursor = await db.execute("SELECT * FROM blocks WHERE topic = ? ORDER BY rowid", (topic,)) else: cursor = await db.execute("SELECT * FROM blocks WHERE topic = ? AND status = ? ORDER BY rowid", (topic, status)) rows = await cursor.fetchall() out = [] for row in rows: d = _row_to_dict(row, cursor) d["sources"] = json.loads(d.get("sources") or "[]") d["reader"] = json.loads(d.get("reader") or "[]") out.append(d) return out async def set_block_status(topic: str, title_norm: str, status: str, title: str | None = None, description: str | None = None, neu_norm: str | None = None) -> None: """Set status; optionally update title/description (e.g. after a semantic merge). `neu_norm` renames the norm key (clarification: reference title → meaningful name). Only safe while no subblocks/facts are attached to the old `title_norm` yet.""" fields = {"status": status, "updated_at": _now()} if title is not None: fields["title"] = title if description is not None: fields["description"] = description if neu_norm is not None and neu_norm != title_norm: fields["title_norm"] = neu_norm await _update("blocks", fields, {"topic": topic, "title_norm": title_norm}) async def rename_block_norm(topic: str, alt_norm: str, neu_norm: str, neu_titel: str) -> None: """Block-Norm vollständig um-keyen (Repair: `(n)`-Kollisionssuffix ablegen): blocks- Spiegel PLUS alle angehängten Tabellen in einer Transaktion — anders als set_block_status(neu_norm=…) auch mit vorhandenen Subblocks/Fragen/Artefakten sicher.""" async with _tx() as db: for table in ("subblocks", "question_pattern", "sub_artefakte"): await db.execute( f"UPDATE {table} SET block_norm = ?, block = ?, updated_at = ? " "WHERE topic = ? AND block_norm = ?", (neu_norm, neu_titel, _now(), topic, alt_norm)) await db.execute( "UPDATE blocks SET title_norm = ?, title = ?, updated_at = ? " "WHERE topic = ? AND title_norm = ?", (neu_norm, neu_titel, _now(), topic, alt_norm)) async def delete_blocks(topic: str) -> None: async with _tx() as db: await db.execute("DELETE FROM blocks WHERE topic = ?", (topic,)) # ── Kanban dataflow (generic card layer, boards 'inventory' + 'artefacts') ──────── def _now_plus(seconds: float) -> str: from datetime import datetime, timedelta, timezone return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat() def _card(row, cursor) -> dict: """Row → dict with the JSON payload decoded (payload keys stay under 'payload').""" d = _row_to_dict(row, cursor) try: d["payload"] = json.loads(d.get("payload") or "{}") except (TypeError, ValueError): d["payload"] = {} return d async def kanban_pull(topic: str, board: str, stage: str, limit: int) -> list[dict]: """`limit` ready cards of `stage` (backoff expired). LPT: cards carrying a `subs_n` payload field (board 2, set after subblocks) are pulled BIGGEST first — the longest block starts earliest and stops dominating the makespan tail. `n_size` (board 1, reader count) is the coarser fallback estimate. Others stay FIFO.""" db = await get_db() cursor = await db.execute( """SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? AND not_before <= ? ORDER BY COALESCE(json_extract(payload, '$.subs_n'), json_extract(payload, '$.n_size'), 0) DESC, updated_at LIMIT ?""", (topic, board, stage, _now(), limit)) return [_card(row, cursor) for row in await cursor.fetchall()] async def kanban_count(topic: str, stages, board: str | None = None) -> int: """Cards sitting in any of `stages` (str or list) — queue length / quiescence. Cards in backoff still count: their work is not done. `board=None` = across boards.""" if isinstance(stages, str): stages = [stages] if not stages: return 0 db = await get_db() ph = ",".join("?" * len(stages)) sql = f"SELECT count(*) FROM kanban_cards WHERE topic = ? AND stage IN ({ph})" args: tuple = (topic, *stages) if board: sql += " AND board = ?" args += (board,) cursor = await db.execute(sql, args) return (await cursor.fetchone())[0] async def kanban_advance(topic: str, board: str, card_id: str, stage: str) -> None: """Move a card to `stage` (next column, or back for rework). Clears backoff/error.""" await kanban_advance_many(topic, board, [(card_id, stage)]) # Current generation run per topic — every event writer stamps run_id from here, so no # signature threading through agents/kanban is needed. Set/cleared by the flow entries # (board_inventory.run_boards, guide_board.run_guide_board). _current_run: dict[str, str] = {} def set_current_run(topic: str, run_id: str | None) -> None: if run_id: _current_run[topic] = run_id else: _current_run.pop(topic, None) async def add_event(topic: str, kind: str, key: str = "", label: str = "", status: str = "", dur_ms: int | None = None, wait_ms: int | None = None, meta: dict | None = None) -> None: """One pipeline-history row, own commit. Callers treat this as fire-and-forget.""" async with _tx() as db: await db.execute( "INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta, run_id) VALUES (?,?,?,?,?,?,?,?,?,?)", (topic, _now(), kind, key, label, status, dur_ms, wait_ms, json.dumps(meta or {}, ensure_ascii=False), _current_run.get(topic, ""))) async def _add_events_many(db, topic: str, rows: list[tuple]) -> None: """Batch insert WITHOUT commit — must run inside the caller's transaction (kanban_advance_many) so the event batch stays atomic with the moves.""" now = _now() rid = _current_run.get(topic, "") await db.executemany( "INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta, run_id) VALUES (?,?,?,?,?,?,?,?,?,?)", [(topic, now, kind, key, label, status, None, None, "{}", rid) for kind, key, label, status in rows]) async def kanban_advance_many(topic: str, board: str, moves: list[tuple[str, str]]) -> None: """Batch stage moves in ONE commit (the flow advances whole packages).""" if not moves: return async with _tx() as db: now = _now() await db.executemany( """UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?""", [(stage, now, topic, board, cid) for cid, stage in moves]) await _add_events_many(db, topic, [("stage", f"{board}:{cid}", "", stage) for cid, stage in moves]) async def kanban_upsert_card(topic: str, board: str, card_id: str, kind: str, stage: str, payload: dict | None = None) -> None: """Insert or overwrite a card (stable ids → growing clusters upsert, never duplicate). payload=None keeps the existing payload on conflict.""" async with _tx() as db: await db.execute( """INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at) VALUES (?, ?, ?, ?, ?, COALESCE(?, '{}'), ?) ON CONFLICT(topic, board, card_id) DO UPDATE SET kind = excluded.kind, stage = excluded.stage, payload = COALESCE(?, kanban_cards.payload), retries = 0, not_before = '', last_error = NULL, updated_at = excluded.updated_at""", (topic, board, card_id, kind, stage, json.dumps(payload, ensure_ascii=False) if payload is not None else None, _now(), json.dumps(payload, ensure_ascii=False) if payload is not None else None)) async def kanban_set_payload(topic: str, board: str, card_id: str, payload: dict) -> None: async with _tx() as db: await db.execute( "UPDATE kanban_cards SET payload = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?", (json.dumps(payload, ensure_ascii=False), _now(), topic, board, card_id)) async def kanban_get_card(topic: str, board: str, card_id: str) -> dict | None: db = await get_db() cursor = await db.execute( "SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?", (topic, board, card_id)) row = await cursor.fetchone() return _card(row, cursor) if row else None async def kanban_cards(topic: str, board: str | None = None, stage: str | None = None, kind: str | None = None) -> list[dict]: db = await get_db() sql, args = "SELECT * FROM kanban_cards WHERE topic = ?", [topic] for col, val in (("board", board), ("stage", stage), ("kind", kind)): if val is not None: sql += f" AND {col} = ?" args.append(val) cursor = await db.execute(sql, args) return [_card(row, cursor) for row in await cursor.fetchall()] async def kanban_fail_card(topic: str, board: str, card_id: str, error: str, max_retries: int, backoff_base: float = 30.0) -> bool: """Register a processing failure: retries++, exponential backoff (not_before), and after `max_retries` → stage 'dead' (dead-letter, requeue-able). → True if the card went dead.""" async with _tx() as db: cursor = await db.execute( "SELECT retries FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?", (topic, board, card_id)) row = await cursor.fetchone() if row is None: return False retries = (row[0] or 0) + 1 dead = retries >= max_retries if dead: await db.execute( """UPDATE kanban_cards SET stage = 'dead', retries = ?, not_before = '', last_error = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?""", (retries, error[:500], _now(), topic, board, card_id)) else: await db.execute( """UPDATE kanban_cards SET retries = ?, not_before = ?, last_error = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?""", (retries, _now_plus(backoff_base * (2 ** (retries - 1))), error[:500], _now(), topic, board, card_id)) await db.execute( "INSERT INTO events (topic, ts, kind, key, label, status, dur_ms, wait_ms, meta, run_id) VALUES (?,?,?,?,?,?,?,?,?,?)", (topic, _now(), "fail", f"{board}:{card_id}", "", "dead" if dead else f"retry{retries}", None, None, json.dumps({"error": error[:200]}, ensure_ascii=False), _current_run.get(topic, ""))) return dead async def events_run_summary(topic: str, run_id: str, board: str | None = None) -> dict: """Agent/token aggregate of ONE run — the numbers block of lauf-summary.json. board="inventory"|"artefacts" scopes to that level (via meta.board); None = whole run. Returns start/ende (MIN/MAX ts of the scoped events) so the caller can show a per-board span.""" db = await get_db() where = "topic = ? AND run_id = ? AND kind = 'agent'" params = [topic, run_id] if board: where += " AND json_extract(meta,'$.board') = ?" params.append(board) cursor = await db.execute( f"""SELECT status, COUNT(*), SUM(dur_ms), SUM(json_extract(meta,'$.tokens.input')), SUM(json_extract(meta,'$.tokens.output')), SUM(json_extract(meta,'$.tokens.cache_read')), SUM(json_extract(meta,'$.tokens.cache_write')), MIN(ts), MAX(ts) FROM events WHERE {where} GROUP BY status""", params) agents = {"gesamt": 0, "ok": 0, "timeout": 0, "cancelled": 0, "sonstige": 0, "verlorene_min": 0} tokens = {"input": 0, "output": 0, "cache_read": 0, "cache_write": 0} start = ende = None for status, n, dur, ti, to, cr, cw, mn, mx in await cursor.fetchall(): agents["gesamt"] += n if status in ("ok", "timeout", "cancelled"): agents[status] += n else: agents["sonstige"] += n if status != "ok": agents["verlorene_min"] += round((dur or 0) / 60000) tokens["input"] += ti or 0 tokens["output"] += to or 0 tokens["cache_read"] += cr or 0 tokens["cache_write"] += cw or 0 if mn and (start is None or mn < start): start = mn if mx and (ende is None or mx > ende): ende = mx return {"agents": agents, "tokens": tokens, "start": start, "ende": ende} async def list_runs(topic: str, limit: int = 10) -> list[dict]: """Läufe eines Topics (Blocks UND Guide — beide setzen run_id), jüngster zuerst: Zeitspanne, Agent-/Token-Bilanz, letzte Fehler. Datenquelle für GET /api/runs.""" db = await get_db() cursor = await db.execute( "SELECT run_id, MIN(ts), MAX(ts) FROM events WHERE topic = ? AND run_id != '' " "GROUP BY run_id ORDER BY 3 DESC LIMIT ?", (topic, limit)) rows = await cursor.fetchall() out = [] for run_id, start, ende in rows: summary = await events_run_summary(topic, run_id) # Per-Board-Aufschlüsselung (Zeit/Tokens getrennt für Inventar vs. Artefakte) boards = {b: await events_run_summary(topic, run_id, board=b) for b in ("inventory", "artefacts")} cur = await db.execute( "SELECT key, status, meta, ts FROM events WHERE topic = ? AND run_id = ? AND kind = 'fail' " "ORDER BY ts DESC LIMIT 10", (topic, run_id)) fails = [{"key": k, "status": s, "error": json.loads(m or "{}").get("error", ""), "ts": ts} for k, s, m, ts in await cur.fetchall()] out.append({"run_id": run_id, "aktiv": _current_run.get(topic) == run_id, "start": start, "ende": ende, **summary, "boards": boards, "fails": fails}) return out async def latest_board_runs(topic: str) -> dict: """Jüngster Lauf MIT Daten je Ebene — Anzeige-Quelle der Board-Kopfzeilen im Frontend. „Nur Artefakte"-/Guide-Läufe bekommen eine frische run_id und lassen die anderen Ebenen leer; der jüngste Lauf allein zeigt dann nichts. Hier zählt je Ebene der letzte Lauf, der sie wirklich enthielt. inventory/artefacts via meta.$.board (agents.agent_ebene); guide via run_id-Suffix „-g"+4hex (guide_board.run_guide_board setzt den Marker, Blocks-Suffixe sind reines Hex — enthalten nie „g"). → {ebene: {run_id, aktiv, agents, tokens, start, ende} | None}""" db = await get_db() out: dict = {} for b in ("inventory", "artefacts"): cur = await db.execute( "SELECT run_id FROM events WHERE topic = ? AND kind = 'agent' AND run_id != '' " "AND json_extract(meta,'$.board') = ? ORDER BY ts DESC LIMIT 1", (topic, b)) row = await cur.fetchone() out[b] = None if row is None else { "run_id": row[0], "aktiv": _current_run.get(topic) == row[0], **await events_run_summary(topic, row[0], board=b)} cur = await db.execute( "SELECT run_id FROM events WHERE topic = ? AND kind = 'agent' AND run_id != '' " "AND run_id GLOB '*-g[0-9a-f][0-9a-f][0-9a-f][0-9a-f]' ORDER BY ts DESC LIMIT 1", (topic,)) row = await cur.fetchone() out["guide"] = None if row is None else { "run_id": row[0], "aktiv": _current_run.get(topic) == row[0], **await events_run_summary(topic, row[0])} # Guide: ganzer Lauf (Events sind untagged) return out async def kanban_dead(topic: str) -> list[dict]: """Dead-letter cards across boards (for the board UI + requeue).""" return await kanban_cards(topic, stage="dead") async def kanban_requeue_dead(topic: str, board: str, stage: str) -> int: """dead → `stage` (fresh retries). → number of requeued cards.""" async with _tx() as db: cursor = await db.execute( """UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ? WHERE topic = ? AND board = ? AND stage = 'dead'""", (stage, _now(), topic, board)) await _add_events_many(db, topic, [("reset", f"{board}:requeue-dead", "", stage)]) return cursor.rowcount async def kanban_stage_counts(topic: str) -> dict[str, dict[str, int]]: """{board: {stage: count}} — the live board.""" db = await get_db() cursor = await db.execute( "SELECT board, stage, count(*) FROM kanban_cards WHERE topic = ? GROUP BY board, stage", (topic,)) out: dict[str, dict[str, int]] = {} for board, stage, n in await cursor.fetchall(): out.setdefault(board, {})[stage] = n return out async def kanban_stage_cards(topic: str, board: str, stage: str, limit: int = 20) -> list[dict]: """Newest `limit` cards of one column (for the live card display).""" db = await get_db() cursor = await db.execute( """SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? ORDER BY updated_at DESC LIMIT ?""", (topic, board, stage, limit)) return [_card(row, cursor) for row in await cursor.fetchall()] async def kanban_delete_card(topic: str, board: str, card_id: str) -> None: """Delete ONE card (repair: the merged-away/removed block's board-2 card).""" async with _tx() as db: await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?", (topic, board, card_id)) async def kanban_delete_cards(topic: str, board: str, kind: str | None = None) -> None: """Delete derived cards (board reset) — kind=None wipes the whole board.""" async with _tx() as db: if kind: await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ? AND kind = ?", (topic, board, kind)) else: await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ?", (topic, board)) async def kanban_reset(topic: str, board: str | None = None) -> None: async with _tx() as db: if board: await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ?", (topic, board)) if board == "inventory": await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,)) else: await db.execute("DELETE FROM kanban_cards WHERE topic = ?", (topic,)) await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,)) async def kanban_set_members(topic: str, group_id: str, members: list[str]) -> None: """Replace the member set of a cluster (one member belongs to exactly one cluster).""" async with _tx() as db: await db.execute("DELETE FROM kanban_members WHERE topic = ? AND group_id = ?", (topic, group_id)) await db.executemany( """INSERT INTO kanban_members (topic, member_id, group_id) VALUES (?, ?, ?) ON CONFLICT(topic, member_id) DO UPDATE SET group_id = excluded.group_id""", [(topic, m, group_id) for m in members]) async def kanban_delete_members(topic: str) -> None: async with _tx() as db: await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,)) async def kanban_members_of(topic: str, group_id: str) -> list[str]: db = await get_db() cursor = await db.execute( "SELECT member_id FROM kanban_members WHERE topic = ? AND group_id = ?", (topic, group_id)) return [r[0] for r in await cursor.fetchall()] # ── Guide board (one card per block, linear stages) ────────────────────────────── async def upsert_guide_card(topic: str, format: str, block_norm: str, block: str, stage: str = "lernziele") -> None: """Insert a card; an existing one keeps its stage/progress (resume).""" async with _tx() as db: await db.execute( """INSERT INTO guide_cards (topic, format, block_norm, block, stage, updated_at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(topic, format, block_norm) DO UPDATE SET block = excluded.block, updated_at = excluded.updated_at""", (topic, format, block_norm, block, stage, _now())) async def upsert_guide_cards_many(topic: str, format: str, cards: list[tuple[str, str]]) -> None: """Batch-Seed (block_norm, block); ein Commit statt einem pro Karte.""" if not cards: return async with _tx() as db: await db.executemany( """INSERT INTO guide_cards (topic, format, block_norm, block, stage, updated_at) VALUES (?, ?, ?, ?, 'lernziele', ?) ON CONFLICT(topic, format, block_norm) DO UPDATE SET block = excluded.block, updated_at = excluded.updated_at""", [(topic, format, bn, b, _now()) for bn, b in cards]) async def delete_lernziele_all(topic: str) -> None: """Alle Lernziele eines Topics in einem Statement (statt Karte für Karte).""" async with _tx() as db: await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,)) async def list_guide_cards(topic: str, format: str | None = None) -> list[dict]: """format=None: alle Formate — das Guide-QA misst den Bestand topic-weit.""" db = await get_db() if format is None: cursor = await db.execute( "SELECT * FROM guide_cards WHERE topic = ? ORDER BY format, ord, block_norm", (topic,)) else: cursor = await db.execute( "SELECT * FROM guide_cards WHERE topic = ? AND format = ? ORDER BY ord, block_norm", (topic, format)) return [_row_to_dict(row, cursor) for row in await cursor.fetchall()] async def set_guide_card(topic: str, format: str, block_norm: str, **fields) -> None: if not fields: return async with _tx() as db: cols = ", ".join(f"{k} = ?" for k in fields) await db.execute( f"UPDATE guide_cards SET {cols}, updated_at = ? WHERE topic = ? AND format = ? AND block_norm = ?", (*fields.values(), _now(), topic, format, block_norm)) if "stage" in fields: # the guide board moves stages here, not via kanban_advance_many await _add_events_many(db, topic, [("stage", f"guide:{format}:{block_norm}", "", fields["stage"])]) elif fields.get("status") == "error": # guide cards fail here, not via kanban_fail_card await _add_events_many(db, topic, [("fail", f"guide:{format}:{block_norm}", "", str(fields.get("gate_info", ""))[:200])]) async def guide_stage_counts(topic: str, format: str) -> dict[str, int]: db = await get_db() cursor = await db.execute( "SELECT stage, count(*) FROM guide_cards WHERE topic = ? AND format = ? GROUP BY stage", (topic, format)) return {stage: n for stage, n in await cursor.fetchall()} async def reset_guide_cards_from_stage(topic: str, format: str, stages: list[str], to_stage: str, clear_md: bool = False) -> int: """Cards sitting in any of `stages` → back to `to_stage` (fresh rounds/gate info).""" if not stages: return 0 async with _tx() as db: ph = ",".join("?" * len(stages)) md = ", md = ''" if clear_md else "" cursor = await db.execute( f"""UPDATE guide_cards SET stage = ?, status = 'open', writer_rounds = 0, gate_info = ''{md}, updated_at = ? WHERE topic = ? AND format = ? AND stage IN ({ph})""", (to_stage, _now(), topic, format, *stages)) return cursor.rowcount async def delete_guide_board(topic: str, format: str | None = None) -> None: async with _tx() as db: if format: await db.execute("DELETE FROM guide_cards WHERE topic = ? AND format = ?", (topic, format)) cursor = await db.execute("SELECT count(*) FROM guide_cards WHERE topic = ?", (topic,)) if (await cursor.fetchone())[0] == 0: await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,)) else: await db.execute("DELETE FROM guide_cards WHERE topic = ?", (topic,)) await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,)) async def put_lernziel(topic: str, block_norm: str, ziel_id: str, text: str, sub_norm: str = "") -> None: async with _tx() as db: await db.execute( """INSERT INTO guide_lernziele (topic, block_norm, ziel_id, text, sub_norm, covered, updated_at) VALUES (?, ?, ?, ?, ?, 0, ?) ON CONFLICT(topic, block_norm, ziel_id) DO UPDATE SET text = excluded.text, sub_norm = excluded.sub_norm, updated_at = excluded.updated_at""", (topic, block_norm, ziel_id, text, sub_norm, _now())) async def list_lernziele(topic: str, block_norm: str | None = None) -> list[dict]: db = await get_db() if block_norm is None: cursor = await db.execute( "SELECT * FROM guide_lernziele WHERE topic = ? ORDER BY block_norm, ziel_id", (topic,)) else: cursor = await db.execute( "SELECT * FROM guide_lernziele WHERE topic = ? AND block_norm = ? ORDER BY ziel_id", (topic, block_norm)) return [_row_to_dict(row, cursor) for row in await cursor.fetchall()] async def set_ziel_covered(topic: str, block_norm: str, ziel_id: str, covered: bool) -> None: async with _tx() as db: await db.execute( "UPDATE guide_lernziele SET covered = ?, updated_at = ? WHERE topic = ? AND block_norm = ? AND ziel_id = ?", (1 if covered else 0, _now(), topic, block_norm, ziel_id)) async def delete_lernziele(topic: str, block_norm: str) -> None: async with _tx() as db: await db.execute("DELETE FROM guide_lernziele WHERE topic = ? AND block_norm = ?", (topic, block_norm)) async def kanban_membership(topic: str) -> dict[str, str]: """{member_id: group_id} for the whole topic (the cluster worker's working map).""" db = await get_db() cursor = await db.execute("SELECT member_id, group_id FROM kanban_members WHERE topic = ?", (topic,)) return {m: g for m, g in await cursor.fetchall()} async def kanban_set_member(topic: str, member_id: str, group_id: str) -> None: async with _tx() as db: await db.execute( """INSERT INTO kanban_members (topic, member_id, group_id) VALUES (?, ?, ?) ON CONFLICT(topic, member_id) DO UPDATE SET group_id = excluded.group_id""", (topic, member_id, group_id)) async def kanban_add_title(topic: str, board: str, card_id: str, title: str, description: str, source: str, reader: str) -> bool: """Ingest one research title (stage 'ingest'). An exact dupe folds instead of duplicating: reader-set union, source union, longer description wins; stage stays untouched (a title already consumed into a cluster is not re-queued). NOT concurrency-safe across awaits — callers serialize through one ingest lock. → True if the card is new.""" row = await kanban_get_card(topic, board, card_id) if row is None: payload = {"title": title, "description": description, "sources": [source] if source else [], "readers": [reader]} async with _tx() as db: await db.execute( """INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at) VALUES (?, ?, ?, 'title', 'ingest', ?, ?)""", (topic, board, card_id, json.dumps(payload, ensure_ascii=False), _now())) return True p = row["payload"] p["readers"] = list(dict.fromkeys((p.get("readers") or []) + [reader])) p["sources"] = list(dict.fromkeys((p.get("sources") or []) + ([source] if source else []))) if len(description or "") > len(p.get("description") or ""): p["description"] = description await kanban_set_payload(topic, board, card_id, p) return False async def upsert_subblock(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str) -> None: async with _tx() as db: await db.execute( """INSERT INTO subblocks (topic, block_norm, sub_norm, block, sub_title, mentions, status, updated_at) VALUES (?, ?, ?, ?, ?, 1, 'candidate', ?) ON CONFLICT(topic, block_norm, sub_norm) DO UPDATE SET mentions = mentions + 1, updated_at = excluded.updated_at""", (topic, block_norm, sub_norm, block, sub_title, _now()), ) async def put_subblock(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str, level: str | None = None, relevance: str | None = None, facts: str | None = None, status: str = "consensus") -> None: """Insert/update WITHOUT a mention counter (mirror from the sidecar). Overwrite level/relevance/facts only when a new value is passed (COALESCE protects existing data).""" async with _tx() as db: await db.execute( """INSERT INTO subblocks (topic, block_norm, sub_norm, block, sub_title, mentions, level, relevance, facts, status, updated_at) VALUES (?, ?, ?, ?, ?, 1, ?, ?, COALESCE(?, ''), ?, ?) ON CONFLICT(topic, block_norm, sub_norm) DO UPDATE SET block = excluded.block, sub_title = excluded.sub_title, level = COALESCE(excluded.level, subblocks.level), relevance = COALESCE(excluded.relevance, subblocks.relevance), facts = COALESCE(NULLIF(excluded.facts, ''), subblocks.facts), status = excluded.status, updated_at = excluded.updated_at""", (topic, block_norm, sub_norm, block, sub_title, level, relevance, facts, status, _now()), ) async def list_subblocks(topic: str, block_norm: str | None = None) -> list[dict]: db = await get_db() if block_norm is None: cursor = await db.execute("SELECT * FROM subblocks WHERE topic = ? ORDER BY rowid", (topic,)) else: cursor = await db.execute( "SELECT * FROM subblocks WHERE topic = ? AND block_norm = ? ORDER BY rowid", (topic, block_norm) ) rows = await cursor.fetchall() return [_row_to_dict(row, cursor) for row in rows] async def default_subblock_levels(topic: str, block_norm: str) -> None: """Classify stragglers after finalize: consensus rows without a valid level fall out of the guide/practice/level queries (re-run resume left 25 such rows — invisible content).""" async with _tx() as db: await db.execute( """UPDATE subblocks SET level = 'advanced' WHERE topic = ? AND block_norm = ? AND status = 'consensus' AND (level IS NULL OR level NOT IN ('beginner', 'advanced', 'expert'))""", (topic, block_norm)) await db.execute( """UPDATE subblocks SET relevance = 'relevant' WHERE topic = ? AND block_norm = ? AND status = 'consensus' AND relevance IS NULL""", (topic, block_norm)) async def copy_topic(quelle: str, ziel: str) -> None: """Trainings-Helfer: Kanban-Karten + Block-Rows der Quelle unter neuem Topic duplizieren (Frozen-Inventar-Trials — Board 2 läuft auf identischem Board-1-Stand neu). Nur DB; Dateien (source.json/blocks.md) kopiert der Runner.""" async with _tx() as db: await db.execute("DELETE FROM kanban_cards WHERE topic = ?", (ziel,)) await db.execute("DELETE FROM blocks WHERE topic = ?", (ziel,)) await db.execute( """INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, retries, not_before, last_error, updated_at) SELECT ?, board, card_id, kind, stage, payload, 0, 0, '', updated_at FROM kanban_cards WHERE topic = ?""", (ziel, quelle)) await db.execute( """INSERT INTO blocks (topic, title_norm, title, description, mentions, status, sources, reader, updated_at) SELECT ?, title_norm, title, description, mentions, status, sources, reader, updated_at FROM blocks WHERE topic = ?""", (ziel, quelle)) async def delete_stale_consensus(topic: str, block_norm: str, keep: set[str]) -> None: """Drop consensus rows of a block that are NOT in this run's sidecar (`keep`): finalize only upserts, so re-runs piled up orphan rows (measured: 25 subs without any board-2 output). variant/discarded rows stay — QA reads those statuses.""" async with _tx() as db: cursor = await db.execute( "SELECT sub_norm FROM subblocks WHERE topic = ? AND block_norm = ? AND status = 'consensus'", (topic, block_norm)) rows = await cursor.fetchall() stale = [r[0] for r in rows if r[0] not in keep] for sn in stale: await db.execute("DELETE FROM subblocks WHERE topic = ? AND block_norm = ? AND sub_norm = ?", (topic, block_norm, sn)) async def set_subblock_fields(topic: str, block_norm: str, sub_norm: str, **fields) -> None: """Set fields (level/relevance/status/sub_title) of a subblock row.""" fields["updated_at"] = _now() await _update("subblocks", fields, {"topic": topic, "block_norm": block_norm, "sub_norm": sub_norm}) async def delete_subblocks(topic: str, block_norm: str | None = None) -> None: async with _tx() as db: if block_norm is None: await db.execute("DELETE FROM subblocks WHERE topic = ?", (topic,)) else: await db.execute("DELETE FROM subblocks WHERE topic = ? AND block_norm = ?", (topic, block_norm)) async def upsert_question_pattern(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str, question: str) -> None: async with _tx() as db: await db.execute( """INSERT INTO question_pattern (topic, block_norm, sub_norm, block, sub_title, question, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(topic, block_norm, sub_norm) DO UPDATE SET sub_title = excluded.sub_title, question = excluded.question, updated_at = excluded.updated_at""", (topic, block_norm, sub_norm, block, sub_title, question, _now()), ) async def list_question_pattern(topic: str, block_norm: str | None = None) -> list[dict]: db = await get_db() if block_norm is None: cursor = await db.execute("SELECT * FROM question_pattern WHERE topic = ? ORDER BY rowid", (topic,)) else: cursor = await db.execute( "SELECT * FROM question_pattern WHERE topic = ? AND block_norm = ? ORDER BY rowid", (topic, block_norm) ) rows = await cursor.fetchall() return [_row_to_dict(row, cursor) for row in rows] async def count_question_pattern_blocks(topic: str) -> int: """Blocks that have at least one exam question pattern.""" db = await get_db() cur = await db.execute("SELECT COUNT(DISTINCT block_norm) FROM question_pattern WHERE topic = ?", (topic,)) return (await cur.fetchone())[0] async def count_sub_artefakte(topic: str) -> int: db = await get_db() cur = await db.execute("SELECT COUNT(*) FROM sub_artefakte WHERE topic = ?", (topic,)) return (await cur.fetchone())[0] async def event_span(topic: str) -> int: """Minutes between first and last pipeline event of the topic (0 if none).""" db = await get_db() cur = await db.execute("SELECT MIN(ts), MAX(ts) FROM events WHERE topic = ?", (topic,)) lo, hi = await cur.fetchone() if not lo or not hi: return 0 from datetime import datetime return int((datetime.fromisoformat(hi) - datetime.fromisoformat(lo)).total_seconds() // 60) async def delete_question_pattern(topic: str, block_norm: str | None = None) -> None: async with _tx() as db: if block_norm is None: await db.execute("DELETE FROM question_pattern WHERE topic = ?", (topic,)) else: await db.execute("DELETE FROM question_pattern WHERE topic = ? AND block_norm = ?", (topic, block_norm)) async def mark_sources_read_done(topic: str, sources: list[str]) -> None: """Mark the cited crawl pages as read_done (research-loop coverage).""" if not sources: return async with _tx() as db: now = _now() await db.executemany( """INSERT INTO research_coverage (topic, source, read_done, updated_at) VALUES (?, ?, 1, ?) ON CONFLICT(topic, source) DO UPDATE SET read_done = 1, updated_at = excluded.updated_at""", [(topic, q, now) for q in sources], ) async def mark_content(topic: str, content: list[str], noise: list[str]) -> None: """Store the triage result per crawl page: content=1 (content) or 0 (noise).""" async with _tx() as db: now = _now() rows = [(topic, q, 1, now) for q in content] + [(topic, q, 0, now) for q in noise] if not rows: return await db.executemany( """INSERT INTO research_coverage (topic, source, content, updated_at) VALUES (?, ?, ?, ?) ON CONFLICT(topic, source) DO UPDATE SET content = excluded.content, updated_at = excluded.updated_at""", rows, ) async def list_content(topic: str) -> list[str]: """Crawl pages the triage marked as content (content=1).""" db = await get_db() cursor = await db.execute( "SELECT source FROM research_coverage WHERE topic = ? AND content = 1 ORDER BY source", (topic,) ) return [r[0] for r in await cursor.fetchall()] async def delete_coverage(topic: str) -> None: async with _tx() as db: await db.execute("DELETE FROM research_coverage WHERE topic = ?", (topic,)) async def set_step_status(topic: str, step: str, status: str) -> None: async with _tx() as db: await db.execute( """INSERT INTO pipeline_state (topic, step, status, updated_at) VALUES (?, ?, ?, ?) ON CONFLICT(topic, step) DO UPDATE SET status = excluded.status, updated_at = excluded.updated_at""", (topic, step, status, _now()), ) async def get_step_status(topic: str, step: str) -> str: db = await get_db() cursor = await db.execute( "SELECT status FROM pipeline_state WHERE topic = ? AND step = ?", (topic, step) ) row = await cursor.fetchone() return row[0] if row else "open" async def delete_source(topic: str) -> None: async with _tx() as db: await db.execute("DELETE FROM source WHERE topic = ?", (topic,)) async def set_guide_content(topic: str, format: str, content_json: str) -> None: """Store finished guide content (JSON blob) per topic+format.""" async with _tx() as db: await db.execute( """INSERT INTO guide_content (topic, format, json, updated_at) VALUES (?, ?, ?, ?) ON CONFLICT(topic, format) DO UPDATE SET json = excluded.json, updated_at = excluded.updated_at""", (topic, format, content_json, _now()), ) async def get_guide_content(topic: str, format: str) -> str | None: db = await get_db() cursor = await db.execute("SELECT json FROM guide_content WHERE topic = ? AND format = ?", (topic, format)) row = await cursor.fetchone() return row[0] if row else None async def delete_guide_content(topic: str, format: str | None = None) -> None: async with _tx() as db: if format is None: await db.execute("DELETE FROM guide_content WHERE topic = ?", (topic,)) else: await db.execute("DELETE FROM guide_content WHERE topic = ? AND format = ?", (topic, format)) async def set_outline(topic: str, outline_json: str) -> None: """Store the outline (chapter→numbers, JSON) per topic — blocks artifact for the guide.""" async with _tx() as db: await db.execute( """INSERT INTO guide_outline (topic, json, updated_at) VALUES (?, ?, ?) ON CONFLICT(topic) DO UPDATE SET json = excluded.json, updated_at = excluded.updated_at""", (topic, outline_json, _now()), ) async def get_outline(topic: str) -> str | None: db = await get_db() cursor = await db.execute("SELECT json FROM guide_outline WHERE topic = ?", (topic,)) row = await cursor.fetchone() return row[0] if row else None async def delete_outline(topic: str) -> None: async with _tx() as db: await db.execute("DELETE FROM guide_outline WHERE topic = ?", (topic,)) async def put_sub_artifact(topic: str, block_norm: str, sub_norm: str, type: str, data: str, block: str = "", sub_title: str = "") -> None: """Store one learning artifact (flashcard/example) as JSON in `data`.""" async with _tx() as db: await db.execute( """INSERT INTO sub_artefakte (topic, block_norm, sub_norm, type, block, sub_title, data, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(topic, block_norm, sub_norm, type) DO UPDATE SET block = excluded.block, sub_title = excluded.sub_title, data = excluded.data, updated_at = excluded.updated_at""", (topic, block_norm, sub_norm, type, block, sub_title, data, _now()), ) async def get_sub_artefakte(topic: str, type: str | None = None, block_norm: str | None = None) -> list[dict]: db = await get_db() sql = "SELECT * FROM sub_artefakte WHERE topic = ?" args: list = [topic] if type is not None: sql += " AND type = ?" args.append(type) if block_norm is not None: sql += " AND block_norm = ?" args.append(block_norm) cursor = await db.execute(sql + " ORDER BY rowid", args) rows = await cursor.fetchall() return [_row_to_dict(row, cursor) for row in rows] async def get_practice_progress(topic: str) -> list[dict]: db = await get_db() cursor = await db.execute( "SELECT block_norm, sub_norm, box, due_at FROM practice_progress WHERE topic = ?", (topic,)) rows = await cursor.fetchall() return [_row_to_dict(row, cursor) for row in rows] async def upsert_practice_progress(topic: str, block_norm: str, sub_norm: str, box: int, due_at: str) -> None: async with _tx() as db: await db.execute( """INSERT INTO practice_progress (topic, block_norm, sub_norm, box, due_at, updated_at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(topic, block_norm, sub_norm) DO UPDATE SET box = excluded.box, due_at = excluded.due_at, updated_at = excluded.updated_at""", (topic, block_norm, sub_norm, box, due_at, _now())) async def sub_levels_norm(topic: str) -> dict[tuple[str, str], int]: """(block_norm, sub_norm) → level 1-4 for the consensus subs — the practice deck's gate.""" db = await get_db() cursor = await db.execute( f"SELECT block_norm, sub_norm, {_LEVEL_CASE} AS level FROM subblocks " "WHERE topic = ? AND status = 'consensus'", (topic,)) return {(bn, sn): lv for bn, sn, lv in await cursor.fetchall()} async def subs_per_level_norm(topic: str) -> dict[str, dict[int, int]]: """Subblocks per level, grouped by block_norm (sub_artefakte is norm-keyed).""" db = await get_db() cursor = await db.execute( f"SELECT block_norm, {_LEVEL_CASE} AS lv, COUNT(*) FROM subblocks " "WHERE topic = ? AND status = 'consensus' GROUP BY block_norm, lv", (topic,)) out: dict[str, dict[int, int]] = {} for bn, level, n in await cursor.fetchall(): out.setdefault(bn, _empty_levels())[level] = n return out async def delete_artefakt_row(topic: str, block_norm: str, sub_norm: str, type: str) -> None: """Remove ONE artefact row (repair: dead target — sub discarded or gone).""" async with _tx() as db: await db.execute("DELETE FROM sub_artefakte WHERE topic = ? AND block_norm = ? AND sub_norm = ? AND type = ?", (topic, block_norm, sub_norm, type)) async def delete_frage_row(topic: str, block_norm: str, sub_norm: str) -> None: """Remove ONE question_pattern row (repair: dead target).""" async with _tx() as db: await db.execute("DELETE FROM question_pattern WHERE topic = ? AND block_norm = ? AND sub_norm = ?", (topic, block_norm, sub_norm)) async def delete_sub_artefakte(topic: str, block_norm: str | None = None) -> None: async with _tx() as db: if block_norm is None: await db.execute("DELETE FROM sub_artefakte WHERE topic = ?", (topic,)) else: await db.execute("DELETE FROM sub_artefakte WHERE topic = ? AND block_norm = ?", (topic, block_norm)) async def get_block_hurdles(topic: str, block_norm: str) -> list[str]: """Typical misconceptions (hurdles) of all subs of a block — as a distractor pool for the quiz. Reads from subblocks.facts (JSON blob); empty/missing ones are skipped.""" db = await get_db() cursor = await db.execute( "SELECT facts FROM subblocks WHERE topic = ? AND block_norm = ?", (topic, block_norm) ) rows = await cursor.fetchall() hurdles = [] for (facts,) in rows: if not facts: continue try: fk = json.loads(facts) except (ValueError, TypeError): continue h = (fk.get("hurdles") or "").strip() if isinstance(fk, dict) else "" if h: hurdles.append(h) return hurdles async def delete_topic_pipeline(topic: str) -> None: """Discard the blocks area of a topic (inventory/subs/pattern/coverage/state/artifacts). NOT the topic config `source` — that is managed separately (delete_source).""" async with _tx() as db: for tab in ("blocks", "subblocks", "question_pattern", "research_coverage", "pipeline_state", "guide_outline", "sub_artefakte", "practice_progress", "events"): await db.execute(f"DELETE FROM {tab} WHERE topic = ?", (topic,))