"""Board 2 „Artefakte": per finished block, prepare the learning artefacts the guide presents. A card is spawned by board 1's `done` column per mirrored block and runs through: subblocks → facts → levels → relevance → question_pattern → artefacts → finalize finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte files + the DB tables. `outline` is a topic-wide BARRIER singleton at the very end (prerequisite graph → chapter order), re-run once per generation run. The heavy lifting is the existing per-block functions in blocks.py — each card gets its own work subdirectory + facts/artefakte paths, so their slot files never collide across blocks.""" import asyncio import json import logging import re import database as db import blocks from blocks import ( ARTEFACT_TYPES, _artefacts_block, _facts_block, _levels_block, _match_sub, _question_pattern_block, _relevance_block, _subblocks_block, _outline_block, ) from fsutil import atomic_write_json from jsonio import read_json_file as _json_file from kanban import Flow, Stage from pipeline import GenContext, _log from textkit import _norm_title, _title log = logging.getLogger("creator.board_artefacts") BOARD = "artefacts" DONE = "done_artefact" def _nset(msg: str, step: int | None = None) -> None: """Progress no-op — the kanban board itself is the progress display.""" def _safe(norm: str) -> str: return re.sub(r"\W+", "-", norm).strip("-")[:24] or "block" def _card_set_p(flow: Flow, norm: str): """Per-card progress: the inner step messages land in-memory on the flow — board_snapshot shows them as the card's info line + phase stepper while active. The step INDEX is resolved to its NAME at write time (indices shift with the source type, names are stable).""" info = flow.state.setdefault("card_info", {}) def set_p(msg: str, step: int | None = None) -> None: name = "" if step is not None: steps = flow.state.get("blocks_steps") if steps is None: steps = flow.state["blocks_steps"] = blocks._blocks_steps(flow.topic) if 0 <= step < len(steps): name = steps[step] info[f"{BOARD}:{norm}"] = {"msg": msg, "step": name} return set_p def _pfiles(files: dict, norm: str) -> dict: """Per-block file namespace: own work dir + facts/artefakte paths, global rest.""" sub = files["arbeit"] / f"ab-{_norm_title(norm).replace(' ', '_')[:60]}" sub.mkdir(parents=True, exist_ok=True) return {**files, "arbeit": sub, "facts": sub / "facts.json", "artefakte": sub / "artefakte.json"} def _entry_line(p: dict) -> str: d = p.get("description") return f"{p['title']} — {d}" if d else p["title"] def make_spawner(topic: str, files: dict): """Hook for board 1's `done` column: one artefact card per mirrored block.""" async def spawn(block_card_id: str, payload: dict): norm = payload.get("mirrored_norm") if not norm: return await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "subblocks", { "title": payload.get("title", ""), "description": payload.get("description", ""), "n_size": payload.get("n_size", 0), # LPT estimate until subs_n exists }) return spawn async def _gather_cards(ctx: GenContext, flow: Flow, cards, one): results = await asyncio.gather(*[one(c) for c in cards], return_exceptions=True) errs = [r for r in results if isinstance(r, Exception)] if errs: raise errs[0] flow.wake.set() def _fail_or_cancel(ctx: GenContext, what: str): # A per-card failure belongs on the card (last_error/dead-letter), never in the # topic banner — the inner block functions may have set it there. blocks._blocks_errors.pop(ctx.topic, None) if ctx.is_cancelled(): return None # leave the card where it is raise RuntimeError(f"{what} ohne Ergebnis") # ── Stage processors (one call per card, all parallel) ───────────────────────────── async def _seed_map(topic: str) -> dict[str, list[str]]: """Demoted fragments become seed candidates of their SURVIVING parent block. parent_norm may point at a block that itself got grouped/merged/renamed — follow the redirect chain (grouped → merged_into, rejected → parent_norm, done → mirrored_norm) to the living board-2 card id (= mirrored_norm). A dead end drops the seed (as before).""" alive: set[str] = set() redirect: dict[str, str] = {} rejected: list[dict] = [] for r in await db.kanban_cards(topic, board="inventory", kind="block"): p = r["payload"] tn = _norm_title(p.get("title", "")) if not tn: continue if r["stage"] in ("done", "done_block"): mn = p.get("mirrored_norm") or tn alive.add(mn) if tn != mn: redirect.setdefault(tn, mn) elif r["stage"] == "grouped" and p.get("merged_into"): redirect.setdefault(tn, _norm_title(p["merged_into"])) # umbrella members are absorbed WHOLE topics ("Aufgabenlisten" → "Listen") — # without a seed the umbrella's finders may simply miss them (measured). rejected.append({"title": p.get("title", ""), "parent_norm": _norm_title(p["merged_into"])}) elif r["stage"] == "rejected": if p.get("parent_norm"): redirect.setdefault(tn, p["parent_norm"]) rejected.append(p) def _resolve(norm: str) -> str | None: seen: set[str] = set() cur = norm while cur and cur not in seen: if cur in alive: # alive check BEFORE following (self-edges like „Listen"→„Listen") return cur seen.add(cur) cur = redirect.get(cur, "") return None seeds: dict[str, list[str]] = {} for p in rejected: pn = p.get("parent_norm") if pn and (target := _resolve(pn)): seeds.setdefault(target, []).append(p.get("title", "")) return seeds async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): topic = flow.topic # Fix 4: fragments demoted to a parent become seed candidates of the parent's subblocks. seeds = await _seed_map(topic) async def one(c): p = c["payload"] norm = c["card_id"] instr = instructions sd = [s for s in seeds.get(norm, []) if s] if sd: instr = (instructions + "\n\nBereits identifizierte Unterpunkt-Kandidaten dieses " "Blocks (unbedingt prüfen und, wenn belegt, aufnehmen):\n" + "\n".join(f"- {s}" for s in sd)) raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), {1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-", seeds=sd or None, lbl=f"{p.get('title', norm)} · ", sources=p.get("sources")) if raw is None: return _fail_or_cancel(ctx, f"Subblocks {p.get('title', norm)}") p["raw"] = raw p["subs_n"] = sum(len(v) for v in raw.values()) # LPT: bigger blocks pull first await db.kanban_set_payload(topic, BOARD, norm, p) await db.kanban_advance(topic, BOARD, norm, "facts") await _gather_cards(ctx, flow, cards, one) async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder, instructions: str, cards): topic = flow.topic async def one(c): p = c["payload"] norm = c["card_id"] raw = p.get("raw") or {} res = await _facts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), raw, q, folder, instructions, ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ", sources=p.get("sources")) if res is None: return _fail_or_cancel(ctx, f"Facts {p.get('title', norm)}") facts_map, discarded = res if discarded: # unsupportable subs vanish from raw too (guide never sees them) for bt, sns in discarded.items(): if bt in raw: raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns] raw = {bt: subs for bt, subs in raw.items() if subs} p["raw"], p["facts"] = raw, facts_map await db.kanban_set_payload(topic, BOARD, norm, p) await db.kanban_advance(topic, BOARD, norm, "levels") await _gather_cards(ctx, flow, cards, one) async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): topic = flow.topic async def one(c): p = c["payload"] norm = c["card_id"] sidecar = await _levels_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), p.get("raw") or {}, instructions, ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ") if sidecar is None: return _fail_or_cancel(ctx, f"Levels {p.get('title', norm)}") facts_map = p.get("facts") or {} for btitle, subs in sidecar.items(): # merge facts into the subs (mirror + guide) fm = facts_map.get(btitle, {}) for sub in subs: if (fk := fm.get(_norm_title(sub["title"]))): sub["facts"] = fk p["sidecar"] = sidecar await db.kanban_set_payload(topic, BOARD, norm, p) await db.kanban_advance(topic, BOARD, norm, "relevance") await _gather_cards(ctx, flow, cards, one) async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): topic = flow.topic async def one(c): p = c["payload"] norm = c["card_id"] sidecar = p.get("sidecar") or {} rel = await _relevance_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), sidecar, instructions, ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ") if rel is None: return _fail_or_cancel(ctx, f"Relevance {p.get('title', norm)}") gid = 0 for subs in sidecar.values(): for sub in subs: gid += 1 sub["relevance"] = rel.get(gid, "relevant") p["sidecar"] = sidecar await db.kanban_set_payload(topic, BOARD, norm, p) await db.kanban_advance(topic, BOARD, norm, "question_pattern") await _gather_cards(ctx, flow, cards, one) async def _proc_question_pattern(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): topic = flow.topic async def one(c): p = c["payload"] norm = c["card_id"] pattern = await _question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), p.get("sidecar") or {}, instructions, ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ") if pattern is None: return _fail_or_cancel(ctx, f"Fragen {p.get('title', norm)}") p["pattern"] = pattern await db.kanban_set_payload(topic, BOARD, norm, p) await db.kanban_advance(topic, BOARD, norm, "artefacts") await _gather_cards(ctx, flow, cards, one) async def _proc_artefacts(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): topic = flow.topic async def one(c): p = c["payload"] norm = c["card_id"] artefacts = await _artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), p.get("sidecar") or {}, instructions, ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ") if artefacts is None and ctx.is_cancelled(): return None p["artefacts"] = artefacts or {} # artefacts are optional — never fatal await db.kanban_set_payload(topic, BOARD, norm, p) await db.kanban_advance(topic, BOARD, norm, "finalize") await _gather_cards(ctx, flow, cards, one) # ── Finalize (SERIAL): merge into the global files + DB tables ───────────────────── def _merge_json(path, block_keys: dict) -> None: data = _json_file(path) if not isinstance(data, dict): data = {} data.update(block_keys) atomic_write_json(path, data, indent=1) async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards): topic = flow.topic for c in cards: p = c["payload"] title = p.get("title", "") sidecar = p.get("sidecar") or {} pattern = p.get("pattern") or {} artefacts = p.get("artefacts") or {} # global sidecar files (the legacy read path of guide/frontend/resume) _merge_json(files["sub_roh"], {t: subs for t, subs in (p.get("raw") or {}).items()}) _merge_json(files["facts"], p.get("facts") or {}) _merge_json(files["sidecar"], sidecar) _merge_json(files["question_pattern"], pattern) art_global = _json_file(files["artefakte"]) if not isinstance(art_global, dict): art_global = {} for typ in ARTEFACT_TYPES: kept = [e for e in art_global.get(typ, []) if _norm_title(_title(str(e.get("block", "")))) != _norm_title(title)] art_global[typ] = kept + list(artefacts.get(typ, [])) atomic_write_json(files["artefakte"], art_global, indent=1) # DB mirrors — per block only (no global deletes) await blocks._mirror_sidecar_db(topic, sidecar) for btitle, entries in pattern.items(): bnorm = _norm_title(btitle) for e in entries if isinstance(entries, list) else []: sub = str(e.get("subblock", "")).strip() sn = _norm_title(sub) question = str(e.get("question", "")).strip() if bnorm and sn and question: await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question) btitles = list(sidecar.keys()) for typ in ARTEFACT_TYPES: for e in artefacts.get(typ, []): bt = _match_sub(str(e.get("block", "")), btitles) bnorm, sn = _norm_title(bt), _norm_title(str(e.get("subblock", ""))) if not bnorm or not sn: continue data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")}, ensure_ascii=False) await db.put_sub_artifact(topic, bnorm, sn, typ, data, bt, str(e.get("subblock", ""))) await db.kanban_advance(topic, BOARD, c["card_id"], DONE) _log(topic, f"Artefakte fertig: {title}") flow.wake.set() # ── Outline (topic-wide barrier singleton) ───────────────────────────────────────── OUTLINE_CARD = "outline" async def ensure_outline_card(topic: str) -> None: """(Re-)queue the outline singleton — run once per generation run, after everything.""" await db.kanban_upsert_card(topic, BOARD, OUTLINE_CARD, "outline", "outline", {"title": "Gliederung"}) async def _proc_outline(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): topic = flow.topic done = await db.kanban_cards(topic, board="inventory", stage="done_block") done.sort(key=lambda c: c["updated_at"]) entries = {i: _entry_line(c["payload"]) for i, c in enumerate(done, 1) if c["payload"].get("title")} if entries: # The outline may run BEFORE finalize has merged the global facts.json — feed the # prereq hints of _learning_order from the card payloads instead (complete as soon # as every block passed the facts stage, which the trimmed barrier guarantees). facts_map: dict = {} for bc in await db.kanban_cards(topic, board=BOARD, kind="ablock"): facts_map.update(bc["payload"].get("facts") or {}) fp = flow.work_dir / "outline-facts.json" atomic_write_json(fp, facts_map, indent=1) plan = await _outline_block(ctx, _nset, {**files, "facts": fp}, entries, instructions) if ctx.is_cancelled(): return if isinstance(plan, dict) and plan.get("chapters"): chapters = [ {"title": ch.get("title", "Kapitel"), "blocks": [_title(entries[n]) for n in ch.get("numbers", []) if n in entries]} for ch in plan["chapters"] ] await db.set_outline(topic, json.dumps({"chapters": chapters}, ensure_ascii=False)) await db.kanban_advance_many(topic, BOARD, [(c["card_id"], DONE) for c in cards]) flow.wake.set() # ── Stage list (appended after board 1 in chain order) ───────────────────────────── def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder, instructions: str) -> list[Stage]: research_done = lambda: flow.research_done # noqa: E731 return [ Stage(BOARD, "subblocks", lambda cs: _proc_subblocks(ctx, flow, files, instructions, cs)), Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)), Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)), Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)), Stage(BOARD, "question_pattern", lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)), Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)), Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True), Stage(BOARD, "outline", lambda cs: _proc_outline(ctx, flow, files, instructions, cs), barrier=True, drain=True, gate=research_done), ]