update

backend/board_artefacts.py

@@ -0,0 +1,333 @@
+"""Board 2 „Artefakte": per finished block, prepare the learning artefacts the guide presents.
+
+A card is spawned by board 1's `done` column per mirrored block and runs through:
+  subblocks → facts → levels → relevance → question_pattern → artefacts → finalize
+finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte
+files + the DB tables. `outline` is a topic-wide BARRIER singleton at the very end
+(prerequisite graph → chapter order), re-run once per generation run.
+
+The heavy lifting is the existing per-block functions in blocks.py — each card gets its own
+work subdirectory + facts/artefakte paths, so their slot files never collide across blocks."""
+
+import asyncio
+import json
+import logging
+import re
+
+import database as db
+import blocks
+from blocks import (
+    ARTEFACT_TYPES, _artefacts_block, _facts_block, _levels_block, _match_sub,
+    _question_pattern_block, _relevance_block, _subblocks_block, _outline_block,
+)
+from fsutil import atomic_write_json
+from jsonio import read_json_file as _json_file
+from kanban import Flow, Stage
+from pipeline import GenContext, _log
+from textkit import _norm_title, _title
+
+log = logging.getLogger("creator.board_artefacts")
+
+BOARD = "artefacts"
+DONE = "done_artefact"
+
+
+def _nset(msg: str, step: int | None = None) -> None:
+    """Progress no-op — the kanban board itself is the progress display."""
+
+
+def _safe(norm: str) -> str:
+    return re.sub(r"\W+", "-", norm).strip("-")[:24] or "block"
+
+
+def _card_set_p(flow: Flow, norm: str):
+    """Per-card progress: the inner step messages land in-memory on the flow —
+    board_snapshot shows them as the card's info line while it is active."""
+    info = flow.state.setdefault("card_info", {})
+
+    def set_p(msg: str, step: int | None = None) -> None:
+        info[f"{BOARD}:{norm}"] = msg
+    return set_p
+
+
+def _pfiles(files: dict, norm: str) -> dict:
+    """Per-block file namespace: own work dir + facts/artefakte paths, global rest."""
+    sub = files["arbeit"] / f"ab-{_norm_title(norm).replace(' ', '_')[:60]}"
+    sub.mkdir(parents=True, exist_ok=True)
+    return {**files, "arbeit": sub, "facts": sub / "facts.json", "artefakte": sub / "artefakte.json"}
+
+
+def _entry_line(p: dict) -> str:
+    d = p.get("description")
+    return f"{p['title']} — {d}" if d else p["title"]
+
+
+def make_spawner(topic: str, files: dict):
+    """Hook for board 1's `done` column: one artefact card per mirrored block."""
+
+    async def spawn(block_card_id: str, payload: dict):
+        norm = payload.get("mirrored_norm")
+        if not norm:
+            return
+        await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "subblocks", {
+            "title": payload.get("title", ""),
+            "description": payload.get("description", ""),
+        })
+    return spawn
+
+
+async def _gather_cards(ctx: GenContext, flow: Flow, cards, one):
+    results = await asyncio.gather(*[one(c) for c in cards], return_exceptions=True)
+    errs = [r for r in results if isinstance(r, Exception)]
+    if errs:
+        raise errs[0]
+    flow.wake.set()
+
+
+def _fail_or_cancel(ctx: GenContext, what: str):
+    # A per-card failure belongs on the card (last_error/dead-letter), never in the
+    # topic banner — the inner block functions may have set it there.
+    blocks._blocks_errors.pop(ctx.topic, None)
+    if ctx.is_cancelled():
+        return None  # leave the card where it is
+    raise RuntimeError(f"{what} ohne Ergebnis")
+
+
+# ── Stage processors (one call per card, all parallel) ─────────────────────────────
+async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
+    topic = flow.topic
+    # Fix 4: fragments demoted to a parent become seed candidates of the parent's subblocks.
+    seeds: dict[str, list[str]] = {}
+    for r in await db.kanban_cards(topic, board="inventory", stage="rejected"):
+        pn = r["payload"].get("parent_norm")
+        if pn:
+            seeds.setdefault(pn, []).append(r["payload"].get("title", ""))
+
+    async def one(c):
+        p = c["payload"]
+        norm = c["card_id"]
+        instr = instructions
+        if (sd := [s for s in seeds.get(norm, []) if s]):
+            instr = (instructions + "\n\nBereits identifizierte Unterpunkt-Kandidaten dieses "
+                     "Blocks (unbedingt prüfen und, wenn belegt, aufnehmen):\n"
+                     + "\n".join(f"- {s}" for s in sd))
+        raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
+                                     {1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-")
+        if raw is None:
+            return _fail_or_cancel(ctx, f"Subblocks {p.get('title', norm)}")
+        p["raw"] = raw
+        await db.kanban_set_payload(topic, BOARD, norm, p)
+        await db.kanban_advance(topic, BOARD, norm, "facts")
+
+    await _gather_cards(ctx, flow, cards, one)
+
+
+async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
+                      instructions: str, cards):
+    topic = flow.topic
+
+    async def one(c):
+        p = c["payload"]
+        norm = c["card_id"]
+        raw = p.get("raw") or {}
+        res = await _facts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), raw, q,
+                                 folder, instructions, ns=f"{_safe(norm)}-")
+        if res is None:
+            return _fail_or_cancel(ctx, f"Facts {p.get('title', norm)}")
+        facts_map, discarded = res
+        if discarded:  # unsupportable subs vanish from raw too (guide never sees them)
+            for bt, sns in discarded.items():
+                if bt in raw:
+                    raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns]
+            raw = {bt: subs for bt, subs in raw.items() if subs}
+        p["raw"], p["facts"] = raw, facts_map
+        await db.kanban_set_payload(topic, BOARD, norm, p)
+        await db.kanban_advance(topic, BOARD, norm, "levels")
+
+    await _gather_cards(ctx, flow, cards, one)
+
+
+async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
+    topic = flow.topic
+
+    async def one(c):
+        p = c["payload"]
+        norm = c["card_id"]
+        sidecar = await _levels_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
+                                      p.get("raw") or {}, instructions, ns=f"{_safe(norm)}-")
+        if sidecar is None:
+            return _fail_or_cancel(ctx, f"Levels {p.get('title', norm)}")
+        facts_map = p.get("facts") or {}
+        for btitle, subs in sidecar.items():  # merge facts into the subs (mirror + guide)
+            fm = facts_map.get(btitle, {})
+            for sub in subs:
+                if (fk := fm.get(_norm_title(sub["title"]))):
+                    sub["facts"] = fk
+        p["sidecar"] = sidecar
+        await db.kanban_set_payload(topic, BOARD, norm, p)
+        await db.kanban_advance(topic, BOARD, norm, "relevance")
+
+    await _gather_cards(ctx, flow, cards, one)
+
+
+async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
+    topic = flow.topic
+
+    async def one(c):
+        p = c["payload"]
+        norm = c["card_id"]
+        sidecar = p.get("sidecar") or {}
+        rel = await _relevance_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
+                                     sidecar, instructions, ns=f"{_safe(norm)}-")
+        if rel is None:
+            return _fail_or_cancel(ctx, f"Relevance {p.get('title', norm)}")
+        gid = 0
+        for subs in sidecar.values():
+            for sub in subs:
+                gid += 1
+                sub["relevance"] = rel.get(gid, "relevant")
+        p["sidecar"] = sidecar
+        await db.kanban_set_payload(topic, BOARD, norm, p)
+        await db.kanban_advance(topic, BOARD, norm, "question_pattern")
+
+    await _gather_cards(ctx, flow, cards, one)
+
+
+async def _proc_question_pattern(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
+    topic = flow.topic
+
+    async def one(c):
+        p = c["payload"]
+        norm = c["card_id"]
+        pattern = await _question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
+                                                p.get("sidecar") or {}, instructions,
+                                                ns=f"{_safe(norm)}-")
+        if pattern is None:
+            return _fail_or_cancel(ctx, f"Fragen {p.get('title', norm)}")
+        p["pattern"] = pattern
+        await db.kanban_set_payload(topic, BOARD, norm, p)
+        await db.kanban_advance(topic, BOARD, norm, "artefacts")
+
+    await _gather_cards(ctx, flow, cards, one)
+
+
+async def _proc_artefacts(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
+    topic = flow.topic
+
+    async def one(c):
+        p = c["payload"]
+        norm = c["card_id"]
+        artefacts = await _artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
+                                           p.get("sidecar") or {}, instructions,
+                                           ns=f"{_safe(norm)}-")
+        if artefacts is None and ctx.is_cancelled():
+            return None
+        p["artefacts"] = artefacts or {}  # artefacts are optional — never fatal
+        await db.kanban_set_payload(topic, BOARD, norm, p)
+        await db.kanban_advance(topic, BOARD, norm, "finalize")
+
+    await _gather_cards(ctx, flow, cards, one)
+
+
+# ── Finalize (SERIAL): merge into the global files + DB tables ─────────────────────
+def _merge_json(path, block_keys: dict) -> None:
+    data = _json_file(path)
+    if not isinstance(data, dict):
+        data = {}
+    data.update(block_keys)
+    atomic_write_json(path, data, indent=1)
+
+
+async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
+    topic = flow.topic
+    for c in cards:
+        p = c["payload"]
+        title = p.get("title", "")
+        sidecar = p.get("sidecar") or {}
+        pattern = p.get("pattern") or {}
+        artefacts = p.get("artefacts") or {}
+        # global sidecar files (the legacy read path of guide/frontend/resume)
+        _merge_json(files["sub_roh"], {t: subs for t, subs in (p.get("raw") or {}).items()})
+        _merge_json(files["facts"], p.get("facts") or {})
+        _merge_json(files["sidecar"], sidecar)
+        _merge_json(files["question_pattern"], pattern)
+        art_global = _json_file(files["artefakte"])
+        if not isinstance(art_global, dict):
+            art_global = {}
+        for typ in ARTEFACT_TYPES:
+            kept = [e for e in art_global.get(typ, [])
+                    if _norm_title(_title(str(e.get("block", "")))) != _norm_title(title)]
+            art_global[typ] = kept + list(artefacts.get(typ, []))
+        atomic_write_json(files["artefakte"], art_global, indent=1)
+        # DB mirrors — per block only (no global deletes)
+        await blocks._mirror_sidecar_db(topic, sidecar)
+        for btitle, entries in pattern.items():
+            bnorm = _norm_title(btitle)
+            for e in entries if isinstance(entries, list) else []:
+                sub = str(e.get("subblock", "")).strip()
+                sn = _norm_title(sub)
+                question = str(e.get("question", "")).strip()
+                if bnorm and sn and question:
+                    await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question)
+        btitles = list(sidecar.keys())
+        for typ in ARTEFACT_TYPES:
+            for e in artefacts.get(typ, []):
+                bt = _match_sub(str(e.get("block", "")), btitles)
+                bnorm, sn = _norm_title(bt), _norm_title(str(e.get("subblock", "")))
+                if not bnorm or not sn:
+                    continue
+                data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")},
+                                  ensure_ascii=False)
+                await db.put_sub_artifact(topic, bnorm, sn, typ, data, bt, str(e.get("subblock", "")))
+        await db.kanban_advance(topic, BOARD, c["card_id"], DONE)
+        _log(topic, f"Artefakte fertig: {title}")
+    flow.wake.set()
+
+
+# ── Outline (topic-wide barrier singleton) ─────────────────────────────────────────
+OUTLINE_CARD = "outline"
+
+
+async def ensure_outline_card(topic: str) -> None:
+    """(Re-)queue the outline singleton — run once per generation run, after everything."""
+    await db.kanban_upsert_card(topic, BOARD, OUTLINE_CARD, "outline", "outline",
+                                {"title": "Gliederung"})
+
+
+async def _proc_outline(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
+    topic = flow.topic
+    done = await db.kanban_cards(topic, board="inventory", stage="done_block")
+    done.sort(key=lambda c: c["updated_at"])
+    entries = {i: _entry_line(c["payload"]) for i, c in enumerate(done, 1)
+               if c["payload"].get("title")}
+    if entries:
+        plan = await _outline_block(ctx, _nset, files, entries, instructions)
+        if ctx.is_cancelled():
+            return
+        if isinstance(plan, dict) and plan.get("chapters"):
+            chapters = [
+                {"title": ch.get("title", "Kapitel"),
+                 "blocks": [_title(entries[n]) for n in ch.get("numbers", []) if n in entries]}
+                for ch in plan["chapters"]
+            ]
+            await db.set_outline(topic, json.dumps({"chapters": chapters}, ensure_ascii=False))
+    await db.kanban_advance_many(topic, BOARD, [(c["card_id"], DONE) for c in cards])
+    flow.wake.set()
+
+
+# ── Stage list (appended after board 1 in chain order) ─────────────────────────────
+def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
+                    instructions: str) -> list[Stage]:
+    research_done = lambda: flow.research_done  # noqa: E731
+    return [
+        Stage(BOARD, "subblocks", lambda cs: _proc_subblocks(ctx, flow, files, instructions, cs)),
+        Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)),
+        Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)),
+        Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)),
+        Stage(BOARD, "question_pattern",
+              lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)),
+        Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)),
+        Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True),
+        Stage(BOARD, "outline", lambda cs: _proc_outline(ctx, flow, files, instructions, cs),
+              barrier=True, drain=True, gate=research_done),
+    ]