update

2026-07-01 22:01:32 +02:00
parent fa718b7d6c
commit b5398f73d2
17 changed files with 1580 additions and 522 deletions
--- a/backend/database.py
+++ b/backend/database.py
@@ -199,6 +199,59 @@ CREATE TABLE IF NOT EXISTS sub_artefakte (
 )
 """

+# Kanban streaming dataflow for the inventory phase. Cards (titles → chains → blocks) flow through
+# columns; `stage` is the current/next column (the queue of a worker = WHERE stage = <predecessor>).
+# `stage` is used instead of the reserved word `column`. chain_id/block_id are stable → upsert, not dup.
+CREATE_KANBAN_TITLES = """
+CREATE TABLE IF NOT EXISTS kanban_titles (
+    topic TEXT NOT NULL,
+    title_norm TEXT NOT NULL,
+    title TEXT NOT NULL,
+    source TEXT NOT NULL DEFAULT '',
+    content TEXT NOT NULL DEFAULT '',
+    stage TEXT NOT NULL DEFAULT 'merge',
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, title_norm)
+)
+"""
+
+CREATE_KANBAN_CHAINS = """
+CREATE TABLE IF NOT EXISTS kanban_chains (
+    topic TEXT NOT NULL,
+    chain_id TEXT NOT NULL,
+    stage TEXT NOT NULL DEFAULT 'chain_verify',
+    main_title_norm TEXT,
+    dirty INTEGER NOT NULL DEFAULT 0,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, chain_id)
+)
+"""
+
+CREATE_KANBAN_CHAIN_MEMBERS = """
+CREATE TABLE IF NOT EXISTS kanban_chain_members (
+    topic TEXT NOT NULL,
+    chain_id TEXT NOT NULL,
+    title_norm TEXT NOT NULL,
+    PRIMARY KEY (topic, title_norm)
+)
+"""
+
+CREATE_KANBAN_BLOCKS = """
+CREATE TABLE IF NOT EXISTS kanban_blocks (
+    topic TEXT NOT NULL,
+    block_id TEXT NOT NULL,
+    chain_id TEXT,
+    title TEXT NOT NULL,
+    source TEXT NOT NULL DEFAULT '',
+    content TEXT NOT NULL DEFAULT '',
+    stage TEXT NOT NULL DEFAULT 'small_blocks',
+    is_small INTEGER NOT NULL DEFAULT 0,
+    parent_block_id TEXT,
+    updated_at TEXT NOT NULL,
+    PRIMARY KEY (topic, block_id)
+)
+"""
+
 _db: aiosqlite.Connection | None = None


@@ -230,6 +283,10 @@ async def init_db():
    await db.execute(CREATE_SOURCE)
    await db.execute(CREATE_GUIDE_OUTLINE)
    await db.execute(CREATE_SUB_ARTEFAKTE)
+    await db.execute(CREATE_KANBAN_TITLES)
+    await db.execute(CREATE_KANBAN_CHAINS)
+    await db.execute(CREATE_KANBAN_CHAIN_MEMBERS)
+    await db.execute(CREATE_KANBAN_BLOCKS)
    try:  # migration for existing DBs without the step column
        await db.execute("ALTER TABLE guides ADD COLUMN step INTEGER")
    except aiosqlite.OperationalError:
@@ -706,6 +763,144 @@ async def delete_blocks(topic: str) -> None:
    await db.commit()


+# ── Kanban streaming dataflow (inventory) ───────────────────────────────────────
+# Generic stage helpers. `stage` is the queue key: a worker pulls WHERE stage = <its input stage>.
+_KANBAN_ID = {"kanban_titles": "title_norm", "kanban_chains": "chain_id", "kanban_blocks": "block_id"}
+
+
+async def kanban_pull(topic: str, table: str, stage: str, limit: int) -> list[dict]:
+    """Oldest `limit` cards sitting in `stage` (FIFO via updated_at)."""
+    idc = _KANBAN_ID[table]  # validates table name
+    db = await get_db()
+    cursor = await db.execute(
+        f"SELECT * FROM {table} WHERE topic = ? AND stage = ? ORDER BY updated_at LIMIT ?", (topic, stage, limit))
+    rows = await cursor.fetchall()
+    return [_row_to_dict(row, cursor) for row in rows]
+
+
+async def kanban_count(topic: str, table: str, stages) -> int:
+    """How many cards sit in any of `stages` (str or list) — for queue length / quiescence."""
+    _ = _KANBAN_ID[table]
+    if isinstance(stages, str):
+        stages = [stages]
+    if not stages:
+        return 0
+    db = await get_db()
+    ph = ",".join("?" * len(stages))
+    cursor = await db.execute(f"SELECT count(*) FROM {table} WHERE topic = ? AND stage IN ({ph})", (topic, *stages))
+    return (await cursor.fetchone())[0]
+
+
+async def kanban_advance(topic: str, table: str, id_val: str, stage: str) -> None:
+    """Move a card to `stage` (advance to next column, or back for rework/retraction)."""
+    idc = _KANBAN_ID[table]
+    db = await get_db()
+    await db.execute(f"UPDATE {table} SET stage = ?, updated_at = ? WHERE topic = ? AND {idc} = ?",
+                     (stage, _now(), topic, id_val))
+    await db.commit()
+
+
+async def kanban_add_title(topic: str, title_norm: str, title: str, source: str = "", content: str = "") -> bool:
+    """Research → titles queue (stage 'merge'). Exact dupes are dropped (PK conflict). → True if new."""
+    db = await get_db()
+    cursor = await db.execute(
+        """INSERT INTO kanban_titles (topic, title_norm, title, source, content, stage, updated_at)
+           VALUES (?, ?, ?, ?, ?, 'merge', ?) ON CONFLICT(topic, title_norm) DO NOTHING""",
+        (topic, title_norm, title, source, content, _now()))
+    await db.commit()
+    return cursor.rowcount > 0
+
+
+async def kanban_upsert_chain(topic: str, chain_id: str, stage: str, main_title_norm: str | None = None,
+                                 dirty: int = 0) -> None:
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO kanban_chains (topic, chain_id, stage, main_title_norm, dirty, updated_at)
+           VALUES (?, ?, ?, ?, ?, ?)
+           ON CONFLICT(topic, chain_id) DO UPDATE SET
+               stage = excluded.stage, main_title_norm = COALESCE(excluded.main_title_norm, kanban_chains.main_title_norm),
+               dirty = excluded.dirty, updated_at = excluded.updated_at""",
+        (topic, chain_id, stage, main_title_norm, dirty, _now()))
+    await db.commit()
+
+
+async def kanban_set_chain_members(topic: str, chain_id: str, members: list[str]) -> None:
+    """Replace the member set of a chain (one title belongs to exactly one chain)."""
+    db = await get_db()
+    await db.execute("DELETE FROM kanban_chain_members WHERE topic = ? AND chain_id = ?", (topic, chain_id))
+    for nm in members:
+        await db.execute(
+            """INSERT INTO kanban_chain_members (topic, chain_id, title_norm) VALUES (?, ?, ?)
+               ON CONFLICT(topic, title_norm) DO UPDATE SET chain_id = excluded.chain_id""",
+            (topic, chain_id, nm))
+    await db.commit()
+
+
+async def kanban_chain_members(topic: str, chain_id: str) -> list[str]:
+    db = await get_db()
+    cursor = await db.execute(
+        "SELECT title_norm FROM kanban_chain_members WHERE topic = ? AND chain_id = ?", (topic, chain_id))
+    return [r[0] for r in await cursor.fetchall()]
+
+
+async def kanban_member_chain(topic: str, title_norm: str) -> str | None:
+    """Which chain a title currently belongs to (or None)."""
+    db = await get_db()
+    cursor = await db.execute(
+        "SELECT chain_id FROM kanban_chain_members WHERE topic = ? AND title_norm = ?", (topic, title_norm))
+    row = await cursor.fetchone()
+    return row[0] if row else None
+
+
+async def kanban_upsert_block(topic: str, block_id: str, chain_id: str | None, title: str, source: str = "",
+                                 content: str = "", stage: str = "small_blocks", is_small: int = 0,
+                                 parent_block_id: str | None = None) -> None:
+    """Chain-id-stable block (Filter/Block). Upsert → growing chains overwrite, never duplicate."""
+    db = await get_db()
+    await db.execute(
+        """INSERT INTO kanban_blocks (topic, block_id, chain_id, title, source, content, stage, is_small, parent_block_id, updated_at)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+           ON CONFLICT(topic, block_id) DO UPDATE SET
+               chain_id = excluded.chain_id, title = excluded.title, source = excluded.source,
+               content = excluded.content, stage = excluded.stage, is_small = excluded.is_small,
+               parent_block_id = excluded.parent_block_id, updated_at = excluded.updated_at""",
+        (topic, block_id, chain_id, title, source, content, stage, is_small, parent_block_id, _now()))
+    await db.commit()
+
+
+async def kanban_titles_by_norm(topic: str) -> dict[str, dict]:
+    """All titles of a topic keyed by title_norm (the candidate universe for chaining)."""
+    db = await get_db()
+    cursor = await db.execute("SELECT * FROM kanban_titles WHERE topic = ?", (topic,))
+    rows = await cursor.fetchall()
+    return {(d := _row_to_dict(row, cursor))["title_norm"]: d for row in rows}
+
+
+async def kanban_all_blocks(topic: str) -> list[dict]:
+    db = await get_db()
+    cursor = await db.execute("SELECT * FROM kanban_blocks WHERE topic = ?", (topic,))
+    rows = await cursor.fetchall()
+    return [_row_to_dict(row, cursor) for row in rows]
+
+
+async def kanban_stage_counts(topic: str) -> dict[str, int]:
+    """{stage: count} across all kanban tables — for the live board / quiescence."""
+    db = await get_db()
+    out: dict[str, int] = {}
+    for table in ("kanban_titles", "kanban_chains", "kanban_blocks"):
+        cursor = await db.execute(f"SELECT stage, count(*) FROM {table} WHERE topic = ? GROUP BY stage", (topic,))
+        for stage, n in await cursor.fetchall():
+            out[stage] = out.get(stage, 0) + n
+    return out
+
+
+async def kanban_reset(topic: str) -> None:
+    db = await get_db()
+    for table in ("kanban_titles", "kanban_chains", "kanban_chain_members", "kanban_blocks"):
+        await db.execute(f"DELETE FROM {table} WHERE topic = ?", (topic,))
+    await db.commit()
+
+
 async def upsert_subblock(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str) -> None:
    db = await get_db()
    await db.execute(