creator/backend/board_artefacts.py

"""Board 2 „Artefakte": per finished block, prepare the learning artefacts the guide presents.

A card is spawned by board 1's `done` column per mirrored block and runs through:
  subblocks → facts → levels → relevance → question_pattern → artefacts → finalize
finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte
files + the DB tables. `outline` is a topic-wide BARRIER singleton at the very end
(prerequisite graph → chapter order), re-run once per generation run.

The heavy lifting is the existing per-block functions in blocks.py — each card gets its own
work subdirectory + facts/artefakte paths, so their slot files never collide across blocks."""

import asyncio
import hashlib
import json
import logging
import re

import database as db
import blocks
import embedding
from blocks import (
    ARTEFACT_TYPES, _artefacts_block, _facts_block, _konsolidiere_subblocks, _levels_block,
    _luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
    _sink_json, _subblocks_block, _outline_block,
)
from config import EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
from fsutil import atomic_write_json
from jsonio import read_json_file as _json_file
from kanban import Flow, Stage
from pipeline import FAILED, GenContext, _extra, _log, _prompt, _timeout, run_single_slot
from textkit import _norm_title, _title

log = logging.getLogger("creator.board_artefacts")

BOARD = "artefacts"
DONE = "done_artefact"


def _nset(msg: str, step: int | None = None) -> None:
    """Progress no-op — the kanban board itself is the progress display."""


def _safe(norm: str) -> str:
    return re.sub(r"\W+", "-", norm).strip("-")[:24] or "block"


def _sub_key(existing: set[str], sn: str) -> str:
    """Agents echo the short sub title while the sub row is keyed 'kurztitel: beschreibung' —
    resolve to the stored key: exact, unambiguous prefix, then unambiguous substring
    containment either way (agents paraphrase/truncate, measured 23 orphans of ~560 rows).
    Ambiguous or unresolvable echoes stay unchanged (visible as QA orphan)."""
    if sn in existing:
        return sn
    hits = [s for s in existing if s.startswith(sn + ":")]
    if len(hits) == 1:
        return hits[0]
    if not hits:
        hits = [s for s in sorted(existing) if sn in s or s in sn]
        if len(hits) == 1:
            return hits[0]
    return sn


def _card_set_p(flow: Flow, norm: str):
    """Per-card progress: the inner step messages land in-memory on the flow —
    board_snapshot shows them as the card's info line + phase stepper while active.
    The step INDEX is resolved to its NAME at write time (indices shift with the
    source type, names are stable)."""
    info = flow.state.setdefault("card_info", {})

    def set_p(msg: str, step: int | None = None) -> None:
        name = ""
        if step is not None:
            steps = flow.state.get("blocks_steps")
            if steps is None:
                steps = flow.state["blocks_steps"] = blocks._blocks_steps(flow.topic)
            if 0 <= step < len(steps):
                name = steps[step]
        info[f"{BOARD}:{norm}"] = {"msg": msg, "step": name}
    return set_p


def _pfiles(files: dict, norm: str) -> dict:
    """Per-block file namespace: own work dir + facts/artefakte paths, global rest."""
    sub = files["arbeit"] / f"ab-{_norm_title(norm).replace(' ', '_')[:60]}"
    sub.mkdir(parents=True, exist_ok=True)
    return {**files, "arbeit": sub, "facts": sub / "facts.json", "artefakte": sub / "artefakte.json"}


def _entry_line(p: dict) -> str:
    d = p.get("description")
    return f"{p['title']} — {d}" if d else p["title"]


def make_spawner(topic: str, files: dict):
    """Hook for board 1's `done` column: one artefact card per mirrored block."""

    async def spawn(block_card_id: str, payload: dict):
        norm = payload.get("mirrored_norm")
        if not norm:
            return
        await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "subblocks", {
            "title": payload.get("title", ""),
            "description": payload.get("description", ""),
            "n_size": payload.get("n_size", 0),  # LPT estimate until subs_n exists
        })
    return spawn


async def _gather_cards(ctx: GenContext, flow: Flow, cards, one):
    results = await asyncio.gather(*[one(c) for c in cards], return_exceptions=True)
    errs = [r for r in results if isinstance(r, Exception)]
    if errs:
        raise errs[0]
    flow.wake.set()


def _fail_or_cancel(ctx: GenContext, what: str):
    # A per-card failure belongs on the card (last_error/dead-letter), never in the
    # topic banner — the inner block functions may have set it there.
    blocks._blocks_errors.pop(ctx.topic, None)
    if ctx.is_cancelled():
        return None  # leave the card where it is
    raise RuntimeError(f"{what} ohne Ergebnis")


# ── Stage processors (one call per card, all parallel) ─────────────────────────────
async def _seed_map(topic: str) -> dict[str, list[str]]:
    """Demoted fragments become seed candidates of their SURVIVING parent block.
    parent_norm may point at a block that itself got grouped/merged/renamed — follow the
    redirect chain (grouped → merged_into, rejected → parent_norm, done → mirrored_norm)
    to the living board-2 card id (= mirrored_norm). A dead end drops the seed (as before)."""
    alive: set[str] = set()
    redirect: dict[str, str] = {}
    rejected: list[dict] = []
    for r in await db.kanban_cards(topic, board="inventory", kind="block"):
        p = r["payload"]
        tn = _norm_title(p.get("title", ""))
        if not tn:
            continue
        if r["stage"] in ("done", "done_block"):
            mn = p.get("mirrored_norm") or tn
            alive.add(mn)
            if tn != mn:
                redirect.setdefault(tn, mn)
        elif r["stage"] == "grouped" and p.get("merged_into"):
            redirect.setdefault(tn, _norm_title(p["merged_into"]))
            # umbrella members are absorbed WHOLE topics ("Aufgabenlisten" → "Listen") —
            # without a seed the umbrella's finders may simply miss them (measured).
            rejected.append({"title": p.get("title", ""), "parent_norm": _norm_title(p["merged_into"])})
        elif r["stage"] == "rejected":
            if p.get("parent_norm"):
                redirect.setdefault(tn, p["parent_norm"])
            rejected.append(p)

    def _resolve(norm: str) -> str | None:
        seen: set[str] = set()
        cur = norm
        while cur and cur not in seen:
            if cur in alive:  # alive check BEFORE following (self-edges like „Listen"→„Listen")
                return cur
            seen.add(cur)
            cur = redirect.get(cur, "")
        return None

    seeds: dict[str, list[str]] = {}
    for p in rejected:
        pn = p.get("parent_norm")
        if pn and (target := _resolve(pn)):
            seeds.setdefault(target, []).append(p.get("title", ""))
    return seeds


async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
    topic = flow.topic
    # Fix 4: fragments demoted to a parent become seed candidates of the parent's subblocks.
    seeds = await _seed_map(topic)

    async def one(c):
        p = c["payload"]
        norm = c["card_id"]
        instr = instructions
        sd = [s for s in seeds.get(norm, []) if s]
        if sd:
            instr = (instructions + "\n\nBereits identifizierte Unterpunkt-Kandidaten dieses "
                     "Blocks (prüfen; wenn belegt UND noch nicht durch einen anderen Eintrag "
                     "abgedeckt, aufnehmen — nicht wörtlich übernehmen, sondern als eigenständige "
                     "Aussage formulieren):\n"
                     + "\n".join(f"- {s}" for s in sd))
        raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
                                     {1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-",
                                     seeds=sd or None, lbl=f"{p.get('title', norm)} · ",
                                     sources=p.get("sources"))
        if raw is None:
            return _fail_or_cancel(ctx, f"Subblocks {p.get('title', norm)}")
        p["raw"] = raw
        p["subs_n"] = sum(len(v) for v in raw.values())  # LPT: bigger blocks pull first
        await db.kanban_set_payload(topic, BOARD, norm, p)
        await db.kanban_advance(topic, BOARD, norm, "facts")

    await _gather_cards(ctx, flow, cards, one)


async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
                      instructions: str, cards):
    topic = flow.topic

    async def one(c):
        p = c["payload"]
        norm = c["card_id"]
        raw = p.get("raw") or {}
        res = await _facts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), raw, q,
                                 folder, instructions, ns=f"{_safe(norm)}-",
                                 lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
        if res is None:
            return _fail_or_cancel(ctx, f"Facts {p.get('title', norm)}")
        facts_map, discarded = res
        if discarded:  # unsupportable subs vanish from raw too (guide never sees them)
            for bt, sns in discarded.items():
                if bt in raw:
                    raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns]
            raw = {bt: subs for bt, subs in raw.items() if subs}
        # In-block consolidation: two-judge panel folds same-statement/subset subs, bundles
        # catalogs, drops off-topic ones — the facts are in hand (key points as evidence),
        # questions/artefacts not yet built. Reported gaps get ONE follow-up finder round.
        luecken = await _konsolidiere_subblocks(ctx, _pfiles(files, norm), raw, facts_map,
                                                instructions, ns=f"{_safe(norm)}-",
                                                lbl=f"{p.get('title', norm)} · ")
        if ctx.is_cancelled():
            return None
        nachgefasst = 0
        for bt, lk in (luecken or {}).items():
            nachgefasst += await _luecken_runde(ctx, _pfiles(files, norm), bt, lk, raw, facts_map,
                                                q, folder, instructions, ns=f"{_safe(norm)}-",
                                                lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
        if ctx.is_cancelled():
            return None
        if nachgefasst:  # close the loop: follow-up finds get the SAME duplicate test as the
            # rest (new subs-hash → fresh judge files); their gap report is deliberately ignored
            await _konsolidiere_subblocks(ctx, _pfiles(files, norm), raw, facts_map,
                                          instructions, ns=f"{_safe(norm)}-",
                                          lbl=f"{p.get('title', norm)} · ")
        if ctx.is_cancelled():
            return None
        raw = {bt: subs for bt, subs in raw.items() if subs}
        p["raw"], p["facts"] = raw, facts_map
        await db.kanban_set_payload(topic, BOARD, norm, p)
        await db.kanban_advance(topic, BOARD, norm, "levels")

    await _gather_cards(ctx, flow, cards, one)


def _cross_schema(data) -> dict[int, str] | None:
    """{"pairs": {"1": "a"|"b"|"nein"}} → {pair_nr: verdict} · otherwise None."""
    if not isinstance(data, dict) or not isinstance(data.get("pairs"), dict):
        return None
    out: dict[int, str] = {}
    for k, v in data["pairs"].items():
        try:
            nr = int(k)
        except (ValueError, TypeError):
            continue
        s = str(v).strip().casefold()
        if s in ("a", "b", "nein"):
            out[nr] = s
    return out or None


async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
    """BARRIER/drain — cross-block sub dedup: the SAME statement carried by two blocks
    (measured on Markdown: tab handling in 3 blocks, HTML blocks, backslash escapes — the
    in-block paths never see these). Embedding candidates (block≠block, cos ≥
    SUB_DUP_KANDIDAT_COS) go to a two-judge panel; UNANIMITY decides which block keeps the
    statement. The loser leaves its card's raw/facts and turns DB `variant` — before
    questions/artefacts exist, so no orphans. Fail-open on judge failure/dissent."""
    topic = flow.topic
    work_dir = flow.work_dir
    package_norms = {c["card_id"] for c in cards}
    entries: list[tuple[int, str, str]] = []  # (card idx, block title, sub title); idx -1 = context
    for ci, c in enumerate(cards):
        for bt, subs in (c["payload"].get("raw") or {}).items():
            for s in subs:
                entries.append((ci, bt, s))
    n_pkg = len(entries)
    # Context: consensus subs of blocks already PAST this barrier (late spawns via the
    # gap-check feedback would otherwise never be compared). Context never folds —
    # its card payload lives downstream (board-1 rule: confirmed context always wins).
    for r in await db.list_subblocks(topic):
        if r["status"] == "consensus" and r["block_norm"] not in package_norms:
            entries.append((-1, r["block"], r["sub_title"]))
    ctx_facts: dict[str, dict] = {}  # facts of downstream cards (DB rows carry none yet)
    for bc in await db.kanban_cards(topic, board=BOARD, kind="ablock"):
        if bc["card_id"] not in package_norms:
            for bt, fm in (bc["payload"].get("facts") or {}).items():
                ctx_facts[_norm_title(bt)] = fm

    async def _advance_all():
        await db.kanban_advance_many(topic, BOARD, [(c["card_id"], "question_pattern") for c in cards])
        flow.wake.set()

    if n_pkg < 1 or len(entries) < 2 or not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
        await _advance_all()
        return
    sims = await asyncio.to_thread(embedding.embed_sims, [s for _, _, s in entries])
    if sims is None:
        await _advance_all()
        return
    negs = [_neg_set(s) for _, _, s in entries]
    pairs = [(i, j) for i in range(len(entries)) for j in range(i + 1, len(entries))
             if entries[i][0] != entries[j][0] and negs[i] == negs[j]
             and float(sims[i][j]) >= SUB_DUP_KANDIDAT_COS]
    if not pairs:
        await _advance_all()
        return

    def _kp(ci: int, bt: str, s: str) -> list:
        if ci < 0:
            f = ctx_facts.get(_norm_title(bt)) or {}
        else:
            f = (cards[ci]["payload"].get("facts") or {}).get(bt) or {}
        return (f.get(_norm_title(s)) or {}).get("key_points") or []

    def _side(tag: str, ci: int, bt: str, s: str) -> str:
        return f"{tag}: [Block: {bt}] {s}" + "".join(f"\n   - {p}" for p in _kp(ci, bt, s))

    lines = "\n\n".join(
        f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
        for k, (i, j) in enumerate(pairs, 1))
    h = hashlib.md5(lines.encode()).hexdigest()[:8]
    paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]

    async def _judge(j, path):
        if _cross_schema(_json_file(path)) is not None:
            return  # resume
        status, _v = await run_single_slot(
            ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
            prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=lines, extra=_extra(instructions)),
            role="judge", capabilities="none",
            payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
            timeout=_timeout("subblock_check", len(pairs)))
        if status == FAILED:
            _log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")

    await asyncio.gather(*[_judge(j, p) for j, p in zip((1, 2), paths)])
    if ctx.is_cancelled():
        return
    outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
    if len(outs) == 1:  # Ersatz-Richter statt fail-open bei EINEM Ausfall
        ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
        await _judge("E", ersatz)
        if ctx.is_cancelled():
            return
        outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
    journal = {"paare": len(pairs), "richter": len(outs), "gefaltet": [], "verdicts": []}
    gone: set[int] = set()
    touched: set[int] = set()
    if len(outs) == 2:
        final = {k: (outs[0].get(k, "nein") if outs[0].get(k, "nein") == outs[1].get(k, "nein")
                     else "uneinig") for k in range(1, len(pairs) + 1)}
        disputed = [k for k, v in final.items() if v == "uneinig"]
        if disputed:  # tie-breaker: a third judge sees ONLY the disputed pairs, majority 2/3
            d_lines = "\n\n".join(
                f"{x}.\n{_side('A', *entries[pairs[k - 1][0]])}\n{_side('B', *entries[pairs[k - 1][1]])}"
                for x, k in enumerate(disputed, 1))
            p3 = work_dir / f"sub-crossblock-{h}-j3.json"
            if _cross_schema(_json_file(p3)) is None:
                status, _v = await run_single_slot(
                    ctx, "Sub-Crossblock j3", key=f"blocks-{topic}-sub-crossblock-{h}-j3",
                    prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=d_lines, extra=_extra(instructions)),
                    role="judge", capabilities="none",
                    payload=lambda result, p=p3: _sink_json(result, p, _cross_schema),
                    timeout=_timeout("subblock_check", len(disputed)))
                if status == FAILED:
                    _log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
            if ctx.is_cancelled():
                return
            v3 = _cross_schema(_json_file(p3)) or {}
            for x, k in enumerate(disputed, 1):
                t = v3.get(x, "nein")
                if t in (outs[0].get(k, "nein"), outs[1].get(k, "nein")):
                    final[k] = t  # majority 2/3; anything else stays disputed → no fold
        for k, (i, j) in enumerate(pairs, 1):
            verdict = final[k]
            journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
                                        "b": f"{entries[j][1]} · {entries[j][2]}",
                                        "verdict": verdict})
            if verdict not in ("a", "b"):
                continue
            lose = j if verdict == "a" else i
            if entries[lose][0] < 0:  # context never folds — the package side goes instead
                lose = i if lose == j else j
            keep = i if lose == j else j
            if lose in gone or keep in gone:  # keeper already folded → don't chain away the content
                continue
            ci, bt, s = entries[lose]
            p = cards[ci]["payload"]
            if s in (p.get("raw") or {}).get(bt, []):
                p["raw"][bt].remove(s)
                (p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
                sc = (p.get("sidecar") or {}).get(bt)
                if isinstance(sc, list):  # questions/artefacts consume the sidecar downstream
                    p["sidecar"][bt] = [e for e in sc
                                        if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
                await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
                gone.add(lose)
                touched.add(ci)
                journal["gefaltet"].append({"weg": f"{bt} · {s}",
                                            "bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
    elif outs:
        _log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
    for ci in touched:
        p = cards[ci]["payload"]
        p["raw"] = {bt: subs for bt, subs in (p.get("raw") or {}).items() if subs}
        await db.kanban_set_payload(topic, BOARD, cards[ci]["card_id"], p)
    if journal["gefaltet"]:
        _log(topic, f"Sub-Crossblock: {len(journal['gefaltet'])} blockübergreifende Dublette(n) gefaltet")
    atomic_write_json(work_dir / f"sub-crossblock-{h}.json", journal, indent=1)
    await _advance_all()


async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
    topic = flow.topic

    async def one(c):
        p = c["payload"]
        norm = c["card_id"]
        sidecar = await _levels_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
                                      p.get("raw") or {}, instructions, ns=f"{_safe(norm)}-",
                                      lbl=f"{p.get('title', norm)} · ")
        if sidecar is None:
            return _fail_or_cancel(ctx, f"Levels {p.get('title', norm)}")
        facts_map = p.get("facts") or {}
        for btitle, subs in sidecar.items():  # merge facts into the subs (mirror + guide)
            fm = facts_map.get(btitle, {})
            for sub in subs:
                if (fk := fm.get(_norm_title(sub["title"]))):
                    sub["facts"] = fk
        p["sidecar"] = sidecar
        await db.kanban_set_payload(topic, BOARD, norm, p)
        await db.kanban_advance(topic, BOARD, norm, "relevance")

    await _gather_cards(ctx, flow, cards, one)


async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
    topic = flow.topic

    async def one(c):
        p = c["payload"]
        norm = c["card_id"]
        sidecar = p.get("sidecar") or {}
        rel = await _relevance_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
                                     sidecar, instructions, ns=f"{_safe(norm)}-",
                                     lbl=f"{p.get('title', norm)} · ")
        if rel is None:
            return _fail_or_cancel(ctx, f"Relevance {p.get('title', norm)}")
        gid = 0
        for subs in sidecar.values():
            for sub in subs:
                gid += 1
                sub["relevance"] = rel.get(gid, "relevant")
        p["sidecar"] = sidecar
        await db.kanban_set_payload(topic, BOARD, norm, p)
        await db.kanban_advance(topic, BOARD, norm, "konsolidierung")

    await _gather_cards(ctx, flow, cards, one)


async def _proc_question_pattern(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
    topic = flow.topic

    async def one(c):
        p = c["payload"]
        norm = c["card_id"]
        pattern = await _question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
                                                p.get("sidecar") or {}, instructions,
                                                ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ")
        if pattern is None:
            return _fail_or_cancel(ctx, f"Fragen {p.get('title', norm)}")
        p["pattern"] = pattern
        await db.kanban_set_payload(topic, BOARD, norm, p)
        await db.kanban_advance(topic, BOARD, norm, "artefacts")

    await _gather_cards(ctx, flow, cards, one)


async def _proc_artefacts(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
    topic = flow.topic

    async def one(c):
        p = c["payload"]
        norm = c["card_id"]
        artefacts = await _artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
                                           p.get("sidecar") or {}, instructions,
                                           ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ")
        if artefacts is None and ctx.is_cancelled():
            return None
        p["artefacts"] = artefacts or {}  # artefacts are optional — never fatal
        await db.kanban_set_payload(topic, BOARD, norm, p)
        await db.kanban_advance(topic, BOARD, norm, "finalize")

    await _gather_cards(ctx, flow, cards, one)


# ── Finalize (SERIAL): merge into the global files + DB tables ─────────────────────
def _merge_json(path, block_keys: dict) -> None:
    data = _json_file(path)
    if not isinstance(data, dict):
        data = {}
    data.update(block_keys)
    atomic_write_json(path, data, indent=1)


async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
    topic = flow.topic
    for c in cards:
        p = c["payload"]
        title = p.get("title", "")
        sidecar = p.get("sidecar") or {}
        pattern = p.get("pattern") or {}
        artefacts = p.get("artefacts") or {}
        # global sidecar files (the legacy read path of guide/frontend/resume)
        _merge_json(files["sub_roh"], {t: subs for t, subs in (p.get("raw") or {}).items()})
        _merge_json(files["facts"], p.get("facts") or {})
        _merge_json(files["sidecar"], sidecar)
        _merge_json(files["question_pattern"], pattern)
        art_global = _json_file(files["artefakte"])
        if not isinstance(art_global, dict):
            art_global = {}
        for typ in ARTEFACT_TYPES:
            kept = [e for e in art_global.get(typ, [])
                    if _norm_title(_title(str(e.get("block", "")))) != _norm_title(title)]
            art_global[typ] = kept + list(artefacts.get(typ, []))
        atomic_write_json(files["artefakte"], art_global, indent=1)
        # DB mirrors — per block only (no global deletes)
        await blocks._mirror_sidecar_db(topic, sidecar)
        # stale question/artefact rows of a PREVIOUS run keyed to gone subs: finalize only
        # upserts, so re-runs left orphans (measured: 28). subblocks rows stay — QA needs
        # the variant/discarded statuses, and the sidecar mirror re-writes only consensus.
        await db.delete_question_pattern(topic, _norm_title(title))
        await db.delete_sub_artefakte(topic, _norm_title(title))
        # stragglers the mirror didn't level (row not in this run's sidecar): without a
        # valid level they vanish from guide/practice/level views while QA still counts them
        await db.default_subblock_levels(topic, _norm_title(title))
        sub_keys: dict[str, set[str]] = {}

        async def _keys(bnorm: str) -> set[str]:
            if bnorm not in sub_keys:
                sub_keys[bnorm] = {r["sub_norm"] for r in await db.list_subblocks(topic, bnorm)}
            return sub_keys[bnorm]

        for btitle, entries in pattern.items():
            bnorm = _norm_title(btitle)
            for e in entries if isinstance(entries, list) else []:
                sub = str(e.get("subblock", "")).strip()
                sn = _norm_title(sub)
                question = str(e.get("question", "")).strip()
                if bnorm and sn and question:
                    sn = _sub_key(await _keys(bnorm), sn)
                    await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question)
        btitles = list(sidecar.keys())
        for typ in ARTEFACT_TYPES:
            for e in artefacts.get(typ, []):
                bt = _match_sub(str(e.get("block", "")), btitles)
                bnorm, sn = _norm_title(bt), _norm_title(str(e.get("subblock", "")))
                if not bnorm or not sn:
                    continue
                sn = _sub_key(await _keys(bnorm), sn)
                data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")},
                                  ensure_ascii=False)
                await db.put_sub_artifact(topic, bnorm, sn, typ, data, bt, str(e.get("subblock", "")))
        await db.kanban_advance(topic, BOARD, c["card_id"], DONE)
        _log(topic, f"Artefakte fertig: {title}")
    flow.wake.set()


# ── Outline (topic-wide barrier singleton) ─────────────────────────────────────────
OUTLINE_CARD = "outline"


async def ensure_outline_card(topic: str) -> None:
    """(Re-)queue the outline singleton — run once per generation run, after everything."""
    await db.kanban_upsert_card(topic, BOARD, OUTLINE_CARD, "outline", "outline",
                                {"title": "Gliederung"})


async def _proc_outline(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
    topic = flow.topic
    done = await db.kanban_cards(topic, board="inventory", stage="done_block")
    done.sort(key=lambda c: c["updated_at"])
    entries = {i: _entry_line(c["payload"]) for i, c in enumerate(done, 1)
               if c["payload"].get("title")}
    if entries:
        # The outline may run BEFORE finalize has merged the global facts.json — feed the
        # prereq hints of _learning_order from the card payloads instead (complete as soon
        # as every block passed the facts stage, which the trimmed barrier guarantees).
        facts_map: dict = {}
        for bc in await db.kanban_cards(topic, board=BOARD, kind="ablock"):
            facts_map.update(bc["payload"].get("facts") or {})
        fp = flow.work_dir / "outline-facts.json"
        atomic_write_json(fp, facts_map, indent=1)
        plan = await _outline_block(ctx, _nset, {**files, "facts": fp}, entries, instructions)
        if ctx.is_cancelled():
            return
        if isinstance(plan, dict) and plan.get("chapters"):
            chapters = [
                {"title": ch.get("title", "Kapitel"),
                 "blocks": [_title(entries[n]) for n in ch.get("numbers", []) if n in entries]}
                for ch in plan["chapters"]
            ]
            await db.set_outline(topic, json.dumps({"chapters": chapters}, ensure_ascii=False))
    await db.kanban_advance_many(topic, BOARD, [(c["card_id"], DONE) for c in cards])
    flow.wake.set()


# ── Stage list (appended after board 1 in chain order) ─────────────────────────────
def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
                    instructions: str) -> list[Stage]:
    research_done = lambda: flow.research_done  # noqa: E731
    return [
        Stage(BOARD, "subblocks", lambda cs: _proc_subblocks(ctx, flow, files, instructions, cs)),
        Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)),
        Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)),
        Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)),
        # Barrier sits AFTER the sub-local stages: cards used to idle here median 36 min
        # while levels/relevance work was still ahead of them
        Stage(BOARD, "konsolidierung",
              lambda cs: _proc_konsolidierung(ctx, flow, files, instructions, cs),
              barrier=True, drain=True),
        Stage(BOARD, "question_pattern",
              lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)),
        Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)),
        Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True),
        Stage(BOARD, "outline", lambda cs: _proc_outline(ctx, flow, files, instructions, cs),
              barrier=True, drain=True, gate=research_done),
    ]