creator/backend/kanban.py

"""Streaming kanban dataflow for the inventory phase.

Each column is a worker that pulls cards from its input `stage` (the queue), processes up to
KANBAN_BATCH at a time, and advances them to the next stage. Cards: titles → chains → blocks.

Streaming columns run continuously; barrier columns start only at QUIESCENCE of everything before
them (no active worker + empty queues). Verify columns push failures back (rework). The Chain column
re-clusters live: a chain that gains a member is marked dirty and flows back to chain_verify.

Reused from blocks.py (imported lazily-safe — kanban is only imported after blocks is loaded):
embedding clustering, `_pairs_schema`/`_cliques`, `_canonical`, research prompt + file payload.
"""

import asyncio
import json
import uuid

import database as db
import embedding
import blocks
from config import RESEARCH_GRACE, MAX_CONCURRENT_AGENTS_PER_TOPIC
from pipeline import GenContext, run_single_slot, _prompt, _timeout, _log, OK
from textkit import _norm_title, _title, _parse_selection
from jsonio import read_json_file as _json_file

KANBAN_BATCH = 5          # cards a worker pulls per package (micro-batching)
# How many packages ONE worker keeps in flight at once. A worker no longer blocks on a single
# package — it keeps pulling and dispatching until this many run concurrently, so a busy column
# fills the agent slots (the per-topic semaphore is the real cap; over-dispatch just queues cheaply).
WORKER_INFLIGHT = MAX_CONCURRENT_AGENTS_PER_TOPIC
# Stages whose processor mutates shared cross-card state and MUST run one package at a time.
# Online chain-clustering reads the whole universe + membership; parallel packages would race.
_SERIAL_STAGES = {"chain"}
_ID_COL = {"kanban_titles": "title_norm", "kanban_chains": "chain_id", "kanban_blocks": "block_id"}
CHAIN_CAP = 12            # max members per chain — caps the O(n²) pair-verification blow-up
_POLL = 0.3               # seconds between empty-queue polls

# Stage order. A card's `stage` = the column it waits in (its worker's input).
TITLE_STAGES = ["merge", "chain", "chained"]                 # 'chained' = consumed into a chain
CHAIN_STAGES = ["chain_verify", "naming", "naming_verify", "chain_filter", "filter_verify", "block_assemble"]
BLOCK_STAGES = ["small_blocks", "small_verify", "dependency", "dependency_verify", "main"]
DONE_CHAIN = "done_chain"
DONE_BLOCK = "done_block"
REJECTED = "rejected"     # block dropped by filter_verify (off-topic / noise) — terminal, never mirrored

# Predecessor stages for each barrier (must ALL be quiescent before the barrier worker runs).
_BEFORE_CHAIN_FILTER = ["merge", "chain", "chain_verify", "naming", "naming_verify"]
_BEFORE_BLOCK = _BEFORE_CHAIN_FILTER + ["chain_filter", "filter_verify"]
_BEFORE_MAIN = ["small_blocks", "small_verify", "dependency", "dependency_verify"]


class _Flow:
    """Shared runtime state: active-task counters per stage + a wakeup event. `producers` counts the
    running research agents (initial + any added live via the generate button); research counts as
    done only when ALL producers have finished, so the flow stays awake while extras still search."""
    def __init__(self, topic: str, work_dir):
        self.topic = topic
        self.work_dir = work_dir
        self.active: dict[str, int] = {}
        self.producers = 1          # the initial research agent
        self.research_tag = 0
        self.stop = False
        self.wake = asyncio.Event()
        self.spawn_research = None  # set by run_kanban: () → coroutine that adds one more research agent

    @property
    def research_done(self) -> bool:
        return self.producers <= 0

    def add_producer(self):
        self.producers += 1
        self.wake.set()

    def done_producer(self):
        self.producers -= 1
        self.wake.set()

    def next_tag(self) -> int:
        self.research_tag += 1
        return self.research_tag

    def enter(self, stage: str):
        self.active[stage] = self.active.get(stage, 0) + 1

    def leave(self, stage: str):
        self.active[stage] = max(0, self.active.get(stage, 0) - 1)
        self.wake.set()

    def active_in(self, stages) -> bool:
        return any(self.active.get(s, 0) > 0 for s in stages)

    async def queued_in(self, table: str, stages) -> bool:
        return await db.kanban_count(self.topic, table, list(stages)) > 0


# ── Research producer ────────────────────────────────────────────────────────────
async def _ingest_titles(topic: str, text: str) -> int:
    """Parse a reader file into kanban_titles (stage 'merge'). Exact dupes drop on the PK. → new count."""
    n, seen = 0, set()
    for record in _parse_selection(text).values():
        title = _title(record)
        norm = _norm_title(title)
        if not norm or norm in seen:
            continue
        seen.add(norm)
        parts = [t.strip() for t in record.split(" — ")]
        source = parts[2] if len(parts) >= 3 else ""
        desc = parts[1] if len(parts) >= 2 else ""
        if await db.kanban_add_title(topic, norm, title, source, desc):
            n += 1
    return n


RESEARCH_RUNTIME = 900  # one research agent, one round, ~15 min hard cap — the tail ingests live while it writes
_POLL_RESEARCH = 3      # seconds between live reads of a running research file

# Live registry of running flows, so the "+ research" button can attach another agent to a live run.
_active_flows: dict[str, "_Flow"] = {}


def _extract_text(raw_line: str) -> str:
    """Best-effort: pull assistant/tool text out of ONE opencode `--format json` event line.
    Recursively collects every `text`/`content` string — robust to the exact event schema."""
    try:
        obj = json.loads(raw_line)
    except Exception:
        return ""
    parts: list[str] = []
    def _walk(o):
        if isinstance(o, dict):
            for k, v in o.items():
                if k in ("text", "content") and isinstance(v, str):
                    parts.append(v)
                else:
                    _walk(v)
        elif isinstance(o, list):
            for v in o:
                _walk(v)
    _walk(obj)
    return "".join(parts)


async def _research_once(ctx: GenContext, files: dict, q: dict, folder, instructions: str, tag: str, flow: "_Flow"):
    """ONE agent searches the topic; its titles go into the merge queue LIVE. Two sources feed the
    ingest: the JSON event stream (on_line → text buffer) AND the file the agent writes — whichever
    the agent uses, cards stream in immediately (not only after it finishes)."""
    work_dir = files["arbeit"]
    caps = "files" if folder else "full"
    p = work_dir / f"research-{tag}.md"
    p.unlink(missing_ok=True)
    stop = asyncio.Event()
    buf: list[str] = []  # assistant text streamed live from the JSON events

    def _on_line(raw: str):  # sync, called per stdout line by the agent runner
        if (t := _extract_text(raw)):
            buf.append(t)

    async def _drain() -> bool:  # ingest from BOTH event buffer and file (idempotent, dupes drop on PK)
        text = "".join(buf)
        if (ft := blocks._file_payload(p)):
            text += "\n" + ft
        return bool(text) and await _ingest_titles(ctx.topic, text)

    async def _tail():  # live-ingest loop while the agent runs
        while not stop.is_set():
            try:
                await asyncio.wait_for(stop.wait(), timeout=_POLL_RESEARCH)
            except asyncio.TimeoutError:
                pass
            if await _drain():
                flow.wake.set()  # new cards → wake the workers

    tail = asyncio.create_task(_tail())
    try:
        await run_single_slot(
            ctx, f"research-{tag}", key=f"blocks-{ctx.topic}-research-{tag}",
            prompt=blocks._build_research_prompt(ctx.topic, p, instructions, q["type"], folder),
            role="quick", capabilities=caps,
            payload=(lambda result, p=p: blocks._file_payload(p)),
            timeout=RESEARCH_RUNTIME, on_line=_on_line,
        )
    finally:
        stop.set()
        await tail
    if await _drain():  # final catch-up
        flow.wake.set()
    _log(ctx.topic, f"Research {tag}: titles → merge queue")


async def _research(ctx: GenContext, files: dict, q: dict, folder, instructions: str, flow: _Flow):
    """The initial research producer (already counted in flow.producers=1)."""
    try:
        await _research_once(ctx, files, q, folder, instructions, "1", flow)
    finally:
        flow.done_producer()


async def _extra_research(ctx: GenContext, files: dict, q: dict, folder, instructions: str, flow: _Flow):
    """One more research agent, added live via the generate button. Keeps the flow awake until done."""
    flow.add_producer()
    try:
        await _research_once(ctx, files, q, folder, instructions, f"x{flow.next_tag()}", flow)
    finally:
        flow.done_producer()


def add_research_agent(topic: str) -> bool:
    """Attach one more research agent to a running flow. → True if a run was live to attach to."""
    flow = _active_flows.get(topic)
    if flow is None or flow.stop or flow.spawn_research is None:
        return False
    asyncio.create_task(flow.spawn_research())
    return True


# ── Generic worker loop ────────────────────────────────────────────────────────────
async def _quiescent(flow: _Flow, stages) -> bool:
    """True iff no worker is active in `stages` AND no card is queued in any of them (all tables).
    The barrier/exit condition — must include QUEUED cards, not just active workers, or a worker
    could exit in a momentary lull while an upstream worker still has work to push down."""
    if not stages:
        return True
    if flow.active_in(stages):
        return False
    for tb in ("kanban_titles", "kanban_chains", "kanban_blocks"):
        if await db.kanban_count(flow.topic, tb, list(stages)):
            return False
    return True


async def _worker(flow: _Flow, table: str, in_stage: str, process, upstream, *, barrier=False, inflight=WORKER_INFLIGHT):
    """Pull cards from `in_stage`, run `process` — keeping up to `inflight` packages running CONCURRENTLY
    so a busy column fills the agent slots instead of doing one package at a time. `upstream` = all stages
    before this one. A barrier worker only pulls when `upstream` is fully quiescent. ANY worker exits only
    when research is done, its own queue is empty, AND `upstream` is quiescent (nothing can still arrive).

    Double-pull safety: each stage has exactly ONE worker, so an in-memory `claimed` set of card-ids (held
    while a package runs) is enough to keep concurrent pulls from grabbing the same cards."""
    topic = flow.topic
    idc = _ID_COL[table]
    claimed: set[str] = set()
    tasks: set[asyncio.Task] = set()

    async def _run(cards):
        ids = [c[idc] for c in cards]
        flow.enter(in_stage)
        try:
            await process(cards)
        except Exception as e:  # one bad package must not kill the worker
            _log(topic, f"worker {in_stage}: {type(e).__name__}: {e}")
        finally:
            flow.leave(in_stage)
            for i in ids:
                claimed.discard(i)
            flow.wake.set()

    try:
        while not flow.stop:
            tasks = {t for t in tasks if not t.done()}
            # Fill the pipeline: pull fresh cards and dispatch until `inflight` packages run.
            if not barrier or await _quiescent(flow, upstream):
                while len(tasks) < inflight:
                    rows = await db.kanban_pull(topic, table, in_stage, KANBAN_BATCH + len(claimed))
                    fresh = [r for r in rows if r[idc] not in claimed][:KANBAN_BATCH]
                    if not fresh:
                        break
                    for r in fresh:
                        claimed.add(r[idc])
                    tasks.add(asyncio.create_task(_run(list(fresh))))
            if tasks:  # busy → wait for a package to finish, then refill
                await asyncio.wait(tasks, timeout=_POLL, return_when=asyncio.FIRST_COMPLETED)
                continue
            # idle: nothing in flight and nothing pulled
            up_quiet = await _quiescent(flow, upstream)
            if (flow.research_done and up_quiet and not flow.active_in([in_stage])
                    and await db.kanban_count(topic, table, in_stage) == 0):
                return  # nothing left and nothing upstream can produce
            await _sleep_wake(flow)
    finally:
        for t in tasks:
            t.cancel()
        if tasks:
            await asyncio.gather(*tasks, return_exceptions=True)


async def _sleep_wake(flow: _Flow):
    try:
        await asyncio.wait_for(flow.wake.wait(), timeout=_POLL)
    except asyncio.TimeoutError:
        pass
    flow.wake.clear()


# ── Column processors ──────────────────────────────────────────────────────────────
async def _proc_merge(flow: _Flow, cards):
    """Exact dedup happened at ingest (PK). Merge just advances titles to the chain column."""
    for c in cards:
        await db.kanban_advance(flow.topic, "kanban_titles", c["title_norm"], "chain")
    flow.wake.set()


async def _proc_chain(flow: _Flow, cards):
    """Embedding blocking: for each new title, find the most similar existing title (cosine ≥ floor).
    Join its chain (or open a new one), mark the chain dirty → chain_verify. Live-growing clusters."""
    topic = flow.topic
    by_norm = await db.kanban_titles_by_norm(topic)
    # Universe = titles already chained + the new batch (for nearest-neighbour search).
    universe = [nm for nm, r in by_norm.items() if r["stage"] in ("chain", "chained")]
    if len(universe) < 1:
        return
    texts = [f"{by_norm[nm]['title']} — {by_norm[nm]['content']}" if by_norm[nm]["content"] else by_norm[nm]["title"]
             for nm in universe]
    sims = await asyncio.to_thread(embedding.embed_sims, texts) if (
        embedding and await asyncio.to_thread(embedding.available)) else None
    idx = {nm: i for i, nm in enumerate(universe)}
    # existing membership
    member_chain = {}
    for nm in universe:
        cid = await _chain_of(topic, nm)
        if cid:
            member_chain[nm] = cid
    touched = set()
    for c in cards:
        nm = c["title_norm"]
        target = None
        if sims is not None and nm in idx:
            best, bestcos = None, blocks.DEDUP_PAIR_FLOOR
            for other in universe:
                if other == nm or other not in member_chain and other not in idx:
                    continue
                cos = float(sims[idx[nm]][idx[other]]) if other in idx else -1
                if cos >= bestcos and other != nm:
                    best, bestcos = other, cos
            if best is not None:
                target = member_chain.get(best)
        cid = target or f"c-{uuid.uuid4().hex[:12]}"
        members = set(await db.kanban_chain_members(topic, cid))
        if target and len(members) >= CHAIN_CAP:   # neighbour's chain is full → start a fresh chain
            cid = f"c-{uuid.uuid4().hex[:12]}"
            members = set()
        members.add(nm)
        await db.kanban_set_chain_members(topic, cid, sorted(members))
        await db.kanban_upsert_chain(topic, cid, "chain_verify", dirty=1)
        member_chain[nm] = cid
        await db.kanban_advance(topic, "kanban_titles", nm, "chained")
        touched.add(cid)
    flow.wake.set()


async def _chain_of(topic: str, title_norm: str) -> str | None:
    return await db.kanban_member_chain(topic, title_norm)


async def _members_dicts(topic: str, members: list[str]) -> list[dict]:
    by = await db.kanban_titles_by_norm(topic)
    return [by[m] for m in members if m in by]


async def _proc_chain_verify(ctx: GenContext, flow: _Flow, cards):
    """Pairwise-verify the batch's chains IN PARALLEL (one agent per chain). Failures split off."""
    await asyncio.gather(*[_verify_one(ctx, flow, c) for c in cards], return_exceptions=True)
    flow.wake.set()


async def _verify_one(ctx: GenContext, flow: _Flow, c):
    topic = flow.topic
    cid = c["chain_id"]
    members = await db.kanban_chain_members(topic, cid)
    dicts = await _members_dicts(topic, members)
    if len(dicts) <= 1:
        await db.kanban_upsert_chain(topic, cid, "naming", dirty=0)
        return
    # Only the embedding-NEAR candidate pairs (cosine ≥ floor) — NOT all O(n²) pairs. A 12-member
    # chain shrinks from 66 pairs to a handful. Transitivity (connected components) does the rest.
    nm = [d["title_norm"] for d in dicts]
    texts = [f"{d['title']} — {d['content']}" if d["content"] else d["title"] for d in dicts]
    sims = await asyncio.to_thread(embedding.embed_sims, texts) if (
        embedding and await asyncio.to_thread(embedding.available)) else None
    if sims is not None:
        pairs = [(nm[i], nm[j]) for i in range(len(nm)) for j in range(i + 1, len(nm))
                 if float(sims[i][j]) >= blocks.DEDUP_PAIR_FLOOR]
    else:
        pairs = [(a, b) for x, a in enumerate(members) for b in members[x + 1:]]
    keep_edges = await _verify_pairs(ctx, flow.work_dir, topic, cid, dicts, pairs)
    groups = _components(members, keep_edges)  # transitive groups over confirmed near-pairs
    groups.sort(key=len, reverse=True)
    main = groups[0] if groups else members
    await db.kanban_set_chain_members(topic, cid, sorted(main))
    await db.kanban_upsert_chain(topic, cid, "naming", dirty=0)
    for g in groups[1:]:  # the rest split into fresh chains
        ncid = f"c-{uuid.uuid4().hex[:12]}"
        await db.kanban_set_chain_members(topic, ncid, sorted(g))
        await db.kanban_upsert_chain(topic, ncid, "naming", dirty=0)


def _components(members: list[str], edges) -> list[list[str]]:
    """Connected components (union-find) over confirmed pairs. Members without an edge stay alone."""
    parent = {m: m for m in members}

    def find(x):
        while parent[x] != x:
            parent[x] = parent[parent[x]]
            x = parent[x]
        return x

    for a, b in edges:
        if a in parent and b in parent:
            parent[find(a)] = find(b)
    comp: dict[str, list[str]] = {}
    for m in members:
        comp.setdefault(find(m), []).append(m)
    return list(comp.values())


async def _verify_pairs(ctx, work_dir, topic, cid, dicts, pairs):
    """Judge the candidate pairs in DEDUP_PAIRS_CHUNK packages, all packages IN PARALLEL → confirmed edges."""
    by = {d["title_norm"]: d for d in dicts}
    chunks = [pairs[k:k + blocks.DEDUP_PAIRS_CHUNK] for k in range(0, len(pairs), blocks.DEDUP_PAIRS_CHUNK)]

    async def _chunk(ci, chunk):
        path = work_dir / f"verify-{cid}-{ci}.json"
        lines = "\n\n".join(
            f"{j + 1}.\nA: {by[a]['title']} — {by[a]['content']}\nB: {by[b]['title']} — {by[b]['content']}"
            for j, (a, b) in enumerate(chunk))
        await run_single_slot(
            ctx, f"Chain verify {cid}", key=f"blocks-{topic}-verify-{cid}-{ci}",
            prompt=_prompt("Blocks-Paar-Filter", topic=topic, pairs=lines, out_path=path),
            role="judge", capabilities="files",
            payload=lambda result, p=path: blocks._pairs_schema(_json_file(p)),
            timeout=_timeout("selection_mapping", len(chunk)))
        verdict = blocks._pairs_schema(_json_file(path)) or {}
        return [(a, b) for j, (a, b) in enumerate(chunk) if verdict.get(j + 1)]

    results = await asyncio.gather(*[_chunk(ci, ch) for ci, ch in enumerate(chunks)], return_exceptions=True)
    return [e for r in results if isinstance(r, list) for e in r]


async def _proc_naming(ctx: GenContext, flow: _Flow, cards):
    """Pick the best member title per chain — batch runs IN PARALLEL (one agent per chain)."""
    await asyncio.gather(*[_name_one(ctx, flow, c) for c in cards], return_exceptions=True)
    flow.wake.set()


async def _name_one(ctx: GenContext, flow: _Flow, c):
    topic = flow.topic
    cid = c["chain_id"]
    members = await db.kanban_chain_members(topic, cid)
    dicts = await _members_dicts(topic, members)
    if len(dicts) <= 1:
        await db.kanban_upsert_chain(topic, cid, "naming_verify",
                                     main_title_norm=(members[0] if members else None), dirty=0)
        return
    winner = await _choose_title(ctx, flow.work_dir, topic, cid, members, dicts, "Blocks-Naming")
    await db.kanban_upsert_chain(topic, cid, "naming_verify", main_title_norm=winner, dirty=0)


async def _proc_naming_verify(ctx: GenContext, flow: _Flow, cards):
    """Second judge checks each title — batch runs IN PARALLEL."""
    await asyncio.gather(*[_namecheck_one(ctx, flow, c) for c in cards], return_exceptions=True)
    flow.wake.set()


async def _namecheck_one(ctx: GenContext, flow: _Flow, c):
    topic = flow.topic
    cid = c["chain_id"]
    members = await db.kanban_chain_members(topic, cid)
    dicts = await _members_dicts(topic, members)
    winner = c.get("main_title_norm") or (members[0] if members else None)
    if len(dicts) > 1:
        winner = await _choose_title(ctx, flow.work_dir, topic, cid, members, dicts, "Blocks-Naming-Check",
                                     current=(members.index(winner) + 1 if winner in members else 1))
    await db.kanban_upsert_chain(topic, cid, "chain_filter", main_title_norm=winner, dirty=0)


async def _choose_title(ctx, work_dir, topic, cid, members, dicts, template, current=None):
    by = {d["title_norm"]: d for d in dicts}
    path = work_dir / f"naming-{cid}.json"
    lines = "\n".join(f"{k + 1}. {by[m]['title']} — {by[m]['content']}" for k, m in enumerate(members) if m in by)
    kw = dict(topic=topic, members=lines, out_path=path)
    if current is not None:
        kw["current"] = current
    await run_single_slot(
        ctx, f"Naming {cid}", key=f"blocks-{topic}-naming-{cid}",
        prompt=_prompt(template, **kw), role="judge", capabilities="files",
        payload=lambda result, p=path: blocks._naming_schema(_json_file(p), len(members)),
        timeout=_timeout("selection_mapping", len(members)))
    best = blocks._naming_schema(_json_file(path), len(members))
    if best is None:
        rep = blocks._canonical(dicts, list(range(len(dicts))), set())
        w = _norm_title(rep["title"])
        return w if w in members else members[0]
    return members[best - 1]


async def _proc_chain_filter(flow: _Flow, cards):
    """BARRIER. Reduce each chain to its winner → upsert a block (chain_id stable). → filter_verify."""
    topic = flow.topic
    by = await db.kanban_titles_by_norm(topic)
    for c in cards:
        cid = c["chain_id"]
        members = await db.kanban_chain_members(topic, cid)
        winner = c.get("main_title_norm") if c.get("main_title_norm") in members else (members[0] if members else None)
        if not winner or winner not in by:
            await db.kanban_upsert_chain(topic, cid, DONE_CHAIN, dirty=0)
            continue
        w = by[winner]
        await db.kanban_upsert_block(topic, f"b-{cid}", cid, w["title"], w["source"], w["content"], stage="filter_verify_b")
        await db.kanban_upsert_chain(topic, cid, "filter_verify", dirty=0)
    flow.wake.set()


async def _proc_filter_verify(ctx: GenContext, flow: _Flow, cards):
    """An agent confirms each reduced block is a valid, on-topic, self-contained concept. Off-topic /
    noise / empty blocks are dropped (→ REJECTED, chain done). IN PARALLEL (one agent per block)."""
    await asyncio.gather(*[_filtercheck_one(ctx, flow, c) for c in cards], return_exceptions=True)
    flow.wake.set()


async def _filtercheck_one(ctx: GenContext, flow: _Flow, c):
    topic = flow.topic
    cid = c["chain_id"]
    bid = f"b-{cid}"
    by = await db.kanban_titles_by_norm(topic)
    members = await db.kanban_chain_members(topic, cid)
    winner = c.get("main_title_norm") if c.get("main_title_norm") in by else (members[0] if members else None)

    async def _drop():
        await db.kanban_advance(topic, "kanban_blocks", bid, REJECTED)
        await db.kanban_upsert_chain(topic, cid, DONE_CHAIN, dirty=0)

    async def _pass():
        await db.kanban_advance(topic, "kanban_blocks", bid, "block_assemble_b")
        await db.kanban_upsert_chain(topic, cid, "block_assemble", dirty=0)

    if not winner or winner not in by:  # nothing to verify → drop the empty chain
        await _drop()
        return
    w = by[winner]
    path = flow.work_dir / f"filtercheck-{cid}.json"
    await run_single_slot(
        ctx, f"Filter verify {cid}", key=f"blocks-{topic}-verify-filter-{cid}",
        prompt=_prompt("Blocks-Filter-Check", topic=topic, title=w["title"], content=w["content"], out_path=path),
        role="judge", capabilities="files",
        payload=lambda result, p=path: _keep_schema(_json_file(p)),
        timeout=_timeout("selection_mapping", 1))
    keep = _keep_schema(_json_file(path))
    await (_drop() if keep is False else _pass())  # None (parse fail) → keep, conservative


async def _proc_block(flow: _Flow, cards):
    """BARRIER. Assemble the final block row → small_blocks. (chain card consumed → done.)"""
    topic = flow.topic
    for c in cards:
        cid = c["chain_id"]
        await db.kanban_advance(topic, "kanban_blocks", f"b-{cid}", "small_blocks")
        await db.kanban_upsert_chain(topic, cid, DONE_CHAIN, dirty=0)
    flow.wake.set()


def _small_schema(data, count):
    """{"small": {"1": true, ...}} → {block_index: bool} · else None."""
    if not isinstance(data, dict) or not isinstance(data.get("small"), dict):
        return None
    out = {}
    for k, v in data["small"].items():
        try:
            n = int(k)
        except (ValueError, TypeError):
            continue
        if 1 <= n <= count:
            out[n] = str(v).strip().casefold() in ("true", "ja", "yes", "1")
    return out or None


def _keep_schema(data):
    """{"keep": true/false} → bool · None when absent/unparseable (caller keeps on None, conservative)."""
    if not isinstance(data, dict) or "keep" not in data:
        return None
    return str(data["keep"]).strip().casefold() in ("true", "ja", "yes", "1")


def _dep_schema(data, count):
    """{"parent": N} → 0..count (0 = standalone) · else None."""
    if not isinstance(data, dict):
        return None
    try:
        n = int(data.get("parent"))
    except (ValueError, TypeError):
        return None
    return n if 0 <= n <= count else None


async def _proc_small(ctx: GenContext, flow: _Flow, cards):
    """Judge marks fragment-like blocks (batch). → small_verify."""
    topic = flow.topic
    path = flow.work_dir / f"small-{cards[0]['block_id']}.json"
    lines = "\n".join(f"{i + 1}. {c['title']} — {c['content']}" for i, c in enumerate(cards))
    await run_single_slot(
        ctx, "Small blocks", key=f"blocks-{topic}-small-{cards[0]['block_id']}",
        prompt=_prompt("Blocks-Small", topic=topic, blocks=lines, out_path=path),
        role="judge", capabilities="files",
        payload=lambda result, p=path: _small_schema(_json_file(p), len(cards)),
        timeout=_timeout("selection_mapping", len(cards)))
    verdict = _small_schema(_json_file(path), len(cards)) or {}
    for i, c in enumerate(cards):
        is_small = 1 if verdict.get(i + 1) else 0
        await db.kanban_upsert_block(topic, c["block_id"], c["chain_id"], c["title"], c["source"], c["content"],
                                     stage="small_verify", is_small=is_small, parent_block_id=c.get("parent_block_id"))
    flow.wake.set()


async def _proc_small_verify(ctx: GenContext, flow: _Flow, cards):
    """Second judge re-checks the small flag (consensus): a block stays `small` only if it was marked
    small AND this judge also calls it a fragment. Disagreement → keep as a main block (conservative).
    → dependency."""
    topic = flow.topic
    path = flow.work_dir / f"smallcheck-{cards[0]['block_id']}.json"
    lines = "\n".join(f"{i + 1}. {c['title']} — {c['content']}" for i, c in enumerate(cards))
    await run_single_slot(
        ctx, "Small verify", key=f"blocks-{topic}-verify-small-{cards[0]['block_id']}",
        prompt=_prompt("Blocks-Small", topic=topic, blocks=lines, out_path=path),
        role="judge", capabilities="files",
        payload=lambda result, p=path: _small_schema(_json_file(p), len(cards)),
        timeout=_timeout("selection_mapping", len(cards)))
    verdict = _small_schema(_json_file(path), len(cards)) or {}
    for i, c in enumerate(cards):
        is_small = 1 if (c["is_small"] and verdict.get(i + 1)) else 0
        await db.kanban_upsert_block(topic, c["block_id"], c["chain_id"], c["title"], c["source"], c["content"],
                                     stage="dependency", is_small=is_small, parent_block_id=c.get("parent_block_id"))
    flow.wake.set()


async def _proc_dependency(ctx: GenContext, flow: _Flow, cards):
    """For each SMALL block, a judge picks its parent from the full list — batch runs IN PARALLEL."""
    topic = flow.topic
    parents = [b for b in await db.kanban_all_blocks(topic) if not b["is_small"] and b["stage"] != REJECTED]
    plist = "\n".join(f"{i + 1}. {b['title']}" for i, b in enumerate(parents))
    await asyncio.gather(*[_dep_one(ctx, flow, c, parents, plist) for c in cards], return_exceptions=True)
    flow.wake.set()


async def _dep_one(ctx: GenContext, flow: _Flow, c, parents, plist):
    topic = flow.topic
    if not c["is_small"] or not parents:
        await db.kanban_advance(topic, "kanban_blocks", c["block_id"], "dependency_verify")
        return
    path = flow.work_dir / f"dep-{c['block_id']}.json"
    await run_single_slot(
        ctx, "Dependency", key=f"blocks-{topic}-dep-{c['block_id']}",
        prompt=_prompt("Blocks-Dependency", topic=topic,
                       small=f"{c['title']} — {c['content']}", parents=plist, out_path=path),
        role="judge", capabilities="files",
        payload=lambda result, p=path: _dep_schema(_json_file(p), len(parents)),
        timeout=_timeout("selection_mapping", len(parents)))
    pick = _dep_schema(_json_file(path), len(parents))
    parent_id = parents[pick - 1]["block_id"] if pick else None
    await db.kanban_upsert_block(topic, c["block_id"], c["chain_id"], c["title"], c["source"], c["content"],
                                 stage="dependency_verify", is_small=c["is_small"], parent_block_id=parent_id)


async def _proc_dependency_verify(flow: _Flow, cards):
    """A small block without a parent is demarked → becomes a main block. → main."""
    topic = flow.topic
    for c in cards:
        is_small = c["is_small"]
        if is_small and not c.get("parent_block_id"):
            is_small = 0
        await db.kanban_upsert_block(topic, c["block_id"], c["chain_id"], c["title"], c["source"], c["content"],
                                     stage="main", is_small=is_small, parent_block_id=c.get("parent_block_id"))
    flow.wake.set()


async def _proc_main(flow: _Flow, cards):
    """BARRIER. Finalize non-small blocks → mirror into the legacy `blocks` table as consensus."""
    topic = flow.topic
    for c in cards:
        if not c["is_small"]:
            norm = _norm_title(c["title"])
            await db.upsert_block(topic, norm, c["title"], c["content"], [c["source"]] if c["source"] else [])
            await db.set_block_status(topic, norm, "consensus")
        await db.kanban_advance(topic, "kanban_blocks", c["block_id"], DONE_BLOCK)
    flow.wake.set()


# ── Orchestration ──────────────────────────────────────────────────────────────────
async def run_kanban(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str,
                     research: bool = True) -> bool:
    """Run the streaming inventory. Returns True when the whole flow reaches quiescence at 'main'.

    research=False ("Continue"): process the EXISTING queue without searching new titles. No initial
    research producer, producers=0 → research_done is true at once; workers drain the queue and exit.
    The +Research button can still attach an agent later via flow.spawn_research."""
    topic = ctx.topic
    flow = _Flow(topic, files["arbeit"])
    flow.spawn_research = lambda: _extra_research(ctx, files, q, folder, instructions, flow)
    if not research:
        flow.producers = 0  # continue the existing queue, search no new titles
    _active_flows[topic] = flow
    set_p("Kanban inventory…")

    ORDER = ["merge", "chain", "chain_verify", "naming", "naming_verify", "chain_filter",
             "filter_verify", "block_assemble", "small_blocks", "small_verify",
             "dependency", "dependency_verify", "main"]
    up = {s: ORDER[:i] for i, s in enumerate(ORDER)}   # upstream = all stages before this one
    barriers = {"chain_filter", "block_assemble", "main"}
    specs = [
        ("kanban_titles", "merge", lambda cs: _proc_merge(flow, cs)),
        ("kanban_titles", "chain", lambda cs: _proc_chain(flow, cs)),
        ("kanban_chains", "chain_verify", lambda cs: _proc_chain_verify(ctx, flow, cs)),
        ("kanban_chains", "naming", lambda cs: _proc_naming(ctx, flow, cs)),
        ("kanban_chains", "naming_verify", lambda cs: _proc_naming_verify(ctx, flow, cs)),
        ("kanban_chains", "chain_filter", lambda cs: _proc_chain_filter(flow, cs)),
        ("kanban_chains", "filter_verify", lambda cs: _proc_filter_verify(ctx, flow, cs)),
        ("kanban_chains", "block_assemble", lambda cs: _proc_block(flow, cs)),
        ("kanban_blocks", "small_blocks", lambda cs: _proc_small(ctx, flow, cs)),
        ("kanban_blocks", "small_verify", lambda cs: _proc_small_verify(ctx, flow, cs)),
        ("kanban_blocks", "dependency", lambda cs: _proc_dependency(ctx, flow, cs)),
        ("kanban_blocks", "dependency_verify", lambda cs: _proc_dependency_verify(flow, cs)),
        ("kanban_blocks", "main", lambda cs: _proc_main(flow, cs)),
    ]
    workers = [_research(ctx, files, q, folder, instructions, flow)] if research else []
    for table, stage, proc in specs:
        workers.append(_worker(flow, table, stage, proc, up[stage], barrier=(stage in barriers),
                               inflight=(1 if stage in _SERIAL_STAGES else WORKER_INFLIGHT)))

    progress = asyncio.create_task(_progress(flow, set_p))
    try:
        await asyncio.gather(*workers, return_exceptions=True)
    finally:
        flow.stop = True
        progress.cancel()
        _active_flows.pop(topic, None)
    if ctx.is_cancelled():
        return False
    n = await db.kanban_count(topic, "kanban_blocks", DONE_BLOCK)
    _log(topic, f"Kanban: done — {n} blocks finalized")
    return True


async def _progress(flow: _Flow, set_p):
    while not flow.stop:
        try:
            counts = await db.kanban_stage_counts(flow.topic)
            total = sum(counts.values())
            set_p(f"Kanban: {total} cards in flow")
        except Exception:
            pass
        await asyncio.sleep(1.0)