399 lines
18 KiB
Python
399 lines
18 KiB
Python
"""Board 2 „Artefakte": per finished block, prepare the learning artefacts the guide presents.
|
|
|
|
A card is spawned by board 1's `done` column per mirrored block and runs through:
|
|
subblocks → facts → levels → relevance → question_pattern → artefacts → finalize
|
|
finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte
|
|
files + the DB tables. `outline` is a topic-wide BARRIER singleton at the very end
|
|
(prerequisite graph → chapter order), re-run once per generation run.
|
|
|
|
The heavy lifting is the existing per-block functions in blocks.py — each card gets its own
|
|
work subdirectory + facts/artefakte paths, so their slot files never collide across blocks."""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import re
|
|
|
|
import database as db
|
|
import blocks
|
|
from blocks import (
|
|
ARTEFACT_TYPES, _artefacts_block, _facts_block, _levels_block, _match_sub,
|
|
_question_pattern_block, _relevance_block, _subblocks_block, _outline_block,
|
|
)
|
|
from fsutil import atomic_write_json
|
|
from jsonio import read_json_file as _json_file
|
|
from kanban import Flow, Stage
|
|
from pipeline import GenContext, _log
|
|
from textkit import _norm_title, _title
|
|
|
|
log = logging.getLogger("creator.board_artefacts")
|
|
|
|
BOARD = "artefacts"
|
|
DONE = "done_artefact"
|
|
|
|
|
|
def _nset(msg: str, step: int | None = None) -> None:
|
|
"""Progress no-op — the kanban board itself is the progress display."""
|
|
|
|
|
|
def _safe(norm: str) -> str:
|
|
return re.sub(r"\W+", "-", norm).strip("-")[:24] or "block"
|
|
|
|
|
|
def _card_set_p(flow: Flow, norm: str):
|
|
"""Per-card progress: the inner step messages land in-memory on the flow —
|
|
board_snapshot shows them as the card's info line + phase stepper while active.
|
|
The step INDEX is resolved to its NAME at write time (indices shift with the
|
|
source type, names are stable)."""
|
|
info = flow.state.setdefault("card_info", {})
|
|
|
|
def set_p(msg: str, step: int | None = None) -> None:
|
|
name = ""
|
|
if step is not None:
|
|
steps = flow.state.get("blocks_steps")
|
|
if steps is None:
|
|
steps = flow.state["blocks_steps"] = blocks._blocks_steps(flow.topic)
|
|
if 0 <= step < len(steps):
|
|
name = steps[step]
|
|
info[f"{BOARD}:{norm}"] = {"msg": msg, "step": name}
|
|
return set_p
|
|
|
|
|
|
def _pfiles(files: dict, norm: str) -> dict:
|
|
"""Per-block file namespace: own work dir + facts/artefakte paths, global rest."""
|
|
sub = files["arbeit"] / f"ab-{_norm_title(norm).replace(' ', '_')[:60]}"
|
|
sub.mkdir(parents=True, exist_ok=True)
|
|
return {**files, "arbeit": sub, "facts": sub / "facts.json", "artefakte": sub / "artefakte.json"}
|
|
|
|
|
|
def _entry_line(p: dict) -> str:
|
|
d = p.get("description")
|
|
return f"{p['title']} — {d}" if d else p["title"]
|
|
|
|
|
|
def make_spawner(topic: str, files: dict):
|
|
"""Hook for board 1's `done` column: one artefact card per mirrored block."""
|
|
|
|
async def spawn(block_card_id: str, payload: dict):
|
|
norm = payload.get("mirrored_norm")
|
|
if not norm:
|
|
return
|
|
await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "subblocks", {
|
|
"title": payload.get("title", ""),
|
|
"description": payload.get("description", ""),
|
|
})
|
|
return spawn
|
|
|
|
|
|
async def _gather_cards(ctx: GenContext, flow: Flow, cards, one):
|
|
results = await asyncio.gather(*[one(c) for c in cards], return_exceptions=True)
|
|
errs = [r for r in results if isinstance(r, Exception)]
|
|
if errs:
|
|
raise errs[0]
|
|
flow.wake.set()
|
|
|
|
|
|
def _fail_or_cancel(ctx: GenContext, what: str):
|
|
# A per-card failure belongs on the card (last_error/dead-letter), never in the
|
|
# topic banner — the inner block functions may have set it there.
|
|
blocks._blocks_errors.pop(ctx.topic, None)
|
|
if ctx.is_cancelled():
|
|
return None # leave the card where it is
|
|
raise RuntimeError(f"{what} ohne Ergebnis")
|
|
|
|
|
|
# ── Stage processors (one call per card, all parallel) ─────────────────────────────
|
|
async def _seed_map(topic: str) -> dict[str, list[str]]:
|
|
"""Demoted fragments become seed candidates of their SURVIVING parent block.
|
|
parent_norm may point at a block that itself got grouped/merged/renamed — follow the
|
|
redirect chain (grouped → merged_into, rejected → parent_norm, done → mirrored_norm)
|
|
to the living board-2 card id (= mirrored_norm). A dead end drops the seed (as before)."""
|
|
alive: set[str] = set()
|
|
redirect: dict[str, str] = {}
|
|
rejected: list[dict] = []
|
|
for r in await db.kanban_cards(topic, board="inventory", kind="block"):
|
|
p = r["payload"]
|
|
tn = _norm_title(p.get("title", ""))
|
|
if not tn:
|
|
continue
|
|
if r["stage"] in ("done", "done_block"):
|
|
mn = p.get("mirrored_norm") or tn
|
|
alive.add(mn)
|
|
if tn != mn:
|
|
redirect.setdefault(tn, mn)
|
|
elif r["stage"] == "grouped" and p.get("merged_into"):
|
|
redirect.setdefault(tn, _norm_title(p["merged_into"]))
|
|
# umbrella members are absorbed WHOLE topics ("Aufgabenlisten" → "Listen") —
|
|
# without a seed the umbrella's finders may simply miss them (measured).
|
|
rejected.append({"title": p.get("title", ""), "parent_norm": _norm_title(p["merged_into"])})
|
|
elif r["stage"] == "rejected":
|
|
if p.get("parent_norm"):
|
|
redirect.setdefault(tn, p["parent_norm"])
|
|
rejected.append(p)
|
|
|
|
def _resolve(norm: str) -> str | None:
|
|
seen: set[str] = set()
|
|
cur = norm
|
|
while cur and cur not in seen:
|
|
if cur in alive: # alive check BEFORE following (self-edges like „Listen"→„Listen")
|
|
return cur
|
|
seen.add(cur)
|
|
cur = redirect.get(cur, "")
|
|
return None
|
|
|
|
seeds: dict[str, list[str]] = {}
|
|
for p in rejected:
|
|
pn = p.get("parent_norm")
|
|
if pn and (target := _resolve(pn)):
|
|
seeds.setdefault(target, []).append(p.get("title", ""))
|
|
return seeds
|
|
|
|
|
|
async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
|
topic = flow.topic
|
|
# Fix 4: fragments demoted to a parent become seed candidates of the parent's subblocks.
|
|
seeds = await _seed_map(topic)
|
|
|
|
async def one(c):
|
|
p = c["payload"]
|
|
norm = c["card_id"]
|
|
instr = instructions
|
|
sd = [s for s in seeds.get(norm, []) if s]
|
|
if sd:
|
|
instr = (instructions + "\n\nBereits identifizierte Unterpunkt-Kandidaten dieses "
|
|
"Blocks (unbedingt prüfen und, wenn belegt, aufnehmen):\n"
|
|
+ "\n".join(f"- {s}" for s in sd))
|
|
raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
|
{1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-",
|
|
seeds=sd or None, lbl=f"{p.get('title', norm)} · ")
|
|
if raw is None:
|
|
return _fail_or_cancel(ctx, f"Subblocks {p.get('title', norm)}")
|
|
p["raw"] = raw
|
|
p["subs_n"] = sum(len(v) for v in raw.values()) # LPT: bigger blocks pull first
|
|
await db.kanban_set_payload(topic, BOARD, norm, p)
|
|
await db.kanban_advance(topic, BOARD, norm, "facts")
|
|
|
|
await _gather_cards(ctx, flow, cards, one)
|
|
|
|
|
|
async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
|
|
instructions: str, cards):
|
|
topic = flow.topic
|
|
|
|
async def one(c):
|
|
p = c["payload"]
|
|
norm = c["card_id"]
|
|
raw = p.get("raw") or {}
|
|
res = await _facts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), raw, q,
|
|
folder, instructions, ns=f"{_safe(norm)}-",
|
|
lbl=f"{p.get('title', norm)} · ")
|
|
if res is None:
|
|
return _fail_or_cancel(ctx, f"Facts {p.get('title', norm)}")
|
|
facts_map, discarded = res
|
|
if discarded: # unsupportable subs vanish from raw too (guide never sees them)
|
|
for bt, sns in discarded.items():
|
|
if bt in raw:
|
|
raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns]
|
|
raw = {bt: subs for bt, subs in raw.items() if subs}
|
|
p["raw"], p["facts"] = raw, facts_map
|
|
await db.kanban_set_payload(topic, BOARD, norm, p)
|
|
await db.kanban_advance(topic, BOARD, norm, "levels")
|
|
|
|
await _gather_cards(ctx, flow, cards, one)
|
|
|
|
|
|
async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
|
topic = flow.topic
|
|
|
|
async def one(c):
|
|
p = c["payload"]
|
|
norm = c["card_id"]
|
|
sidecar = await _levels_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
|
p.get("raw") or {}, instructions, ns=f"{_safe(norm)}-",
|
|
lbl=f"{p.get('title', norm)} · ")
|
|
if sidecar is None:
|
|
return _fail_or_cancel(ctx, f"Levels {p.get('title', norm)}")
|
|
facts_map = p.get("facts") or {}
|
|
for btitle, subs in sidecar.items(): # merge facts into the subs (mirror + guide)
|
|
fm = facts_map.get(btitle, {})
|
|
for sub in subs:
|
|
if (fk := fm.get(_norm_title(sub["title"]))):
|
|
sub["facts"] = fk
|
|
p["sidecar"] = sidecar
|
|
await db.kanban_set_payload(topic, BOARD, norm, p)
|
|
await db.kanban_advance(topic, BOARD, norm, "relevance")
|
|
|
|
await _gather_cards(ctx, flow, cards, one)
|
|
|
|
|
|
async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
|
topic = flow.topic
|
|
|
|
async def one(c):
|
|
p = c["payload"]
|
|
norm = c["card_id"]
|
|
sidecar = p.get("sidecar") or {}
|
|
rel = await _relevance_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
|
sidecar, instructions, ns=f"{_safe(norm)}-",
|
|
lbl=f"{p.get('title', norm)} · ")
|
|
if rel is None:
|
|
return _fail_or_cancel(ctx, f"Relevance {p.get('title', norm)}")
|
|
gid = 0
|
|
for subs in sidecar.values():
|
|
for sub in subs:
|
|
gid += 1
|
|
sub["relevance"] = rel.get(gid, "relevant")
|
|
p["sidecar"] = sidecar
|
|
await db.kanban_set_payload(topic, BOARD, norm, p)
|
|
await db.kanban_advance(topic, BOARD, norm, "question_pattern")
|
|
|
|
await _gather_cards(ctx, flow, cards, one)
|
|
|
|
|
|
async def _proc_question_pattern(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
|
topic = flow.topic
|
|
|
|
async def one(c):
|
|
p = c["payload"]
|
|
norm = c["card_id"]
|
|
pattern = await _question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
|
p.get("sidecar") or {}, instructions,
|
|
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ")
|
|
if pattern is None:
|
|
return _fail_or_cancel(ctx, f"Fragen {p.get('title', norm)}")
|
|
p["pattern"] = pattern
|
|
await db.kanban_set_payload(topic, BOARD, norm, p)
|
|
await db.kanban_advance(topic, BOARD, norm, "artefacts")
|
|
|
|
await _gather_cards(ctx, flow, cards, one)
|
|
|
|
|
|
async def _proc_artefacts(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
|
topic = flow.topic
|
|
|
|
async def one(c):
|
|
p = c["payload"]
|
|
norm = c["card_id"]
|
|
artefacts = await _artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
|
p.get("sidecar") or {}, instructions,
|
|
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ")
|
|
if artefacts is None and ctx.is_cancelled():
|
|
return None
|
|
p["artefacts"] = artefacts or {} # artefacts are optional — never fatal
|
|
await db.kanban_set_payload(topic, BOARD, norm, p)
|
|
await db.kanban_advance(topic, BOARD, norm, "finalize")
|
|
|
|
await _gather_cards(ctx, flow, cards, one)
|
|
|
|
|
|
# ── Finalize (SERIAL): merge into the global files + DB tables ─────────────────────
|
|
def _merge_json(path, block_keys: dict) -> None:
|
|
data = _json_file(path)
|
|
if not isinstance(data, dict):
|
|
data = {}
|
|
data.update(block_keys)
|
|
atomic_write_json(path, data, indent=1)
|
|
|
|
|
|
async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
|
|
topic = flow.topic
|
|
for c in cards:
|
|
p = c["payload"]
|
|
title = p.get("title", "")
|
|
sidecar = p.get("sidecar") or {}
|
|
pattern = p.get("pattern") or {}
|
|
artefacts = p.get("artefacts") or {}
|
|
# global sidecar files (the legacy read path of guide/frontend/resume)
|
|
_merge_json(files["sub_roh"], {t: subs for t, subs in (p.get("raw") or {}).items()})
|
|
_merge_json(files["facts"], p.get("facts") or {})
|
|
_merge_json(files["sidecar"], sidecar)
|
|
_merge_json(files["question_pattern"], pattern)
|
|
art_global = _json_file(files["artefakte"])
|
|
if not isinstance(art_global, dict):
|
|
art_global = {}
|
|
for typ in ARTEFACT_TYPES:
|
|
kept = [e for e in art_global.get(typ, [])
|
|
if _norm_title(_title(str(e.get("block", "")))) != _norm_title(title)]
|
|
art_global[typ] = kept + list(artefacts.get(typ, []))
|
|
atomic_write_json(files["artefakte"], art_global, indent=1)
|
|
# DB mirrors — per block only (no global deletes)
|
|
await blocks._mirror_sidecar_db(topic, sidecar)
|
|
for btitle, entries in pattern.items():
|
|
bnorm = _norm_title(btitle)
|
|
for e in entries if isinstance(entries, list) else []:
|
|
sub = str(e.get("subblock", "")).strip()
|
|
sn = _norm_title(sub)
|
|
question = str(e.get("question", "")).strip()
|
|
if bnorm and sn and question:
|
|
await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question)
|
|
btitles = list(sidecar.keys())
|
|
for typ in ARTEFACT_TYPES:
|
|
for e in artefacts.get(typ, []):
|
|
bt = _match_sub(str(e.get("block", "")), btitles)
|
|
bnorm, sn = _norm_title(bt), _norm_title(str(e.get("subblock", "")))
|
|
if not bnorm or not sn:
|
|
continue
|
|
data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")},
|
|
ensure_ascii=False)
|
|
await db.put_sub_artifact(topic, bnorm, sn, typ, data, bt, str(e.get("subblock", "")))
|
|
await db.kanban_advance(topic, BOARD, c["card_id"], DONE)
|
|
_log(topic, f"Artefakte fertig: {title}")
|
|
flow.wake.set()
|
|
|
|
|
|
# ── Outline (topic-wide barrier singleton) ─────────────────────────────────────────
|
|
OUTLINE_CARD = "outline"
|
|
|
|
|
|
async def ensure_outline_card(topic: str) -> None:
|
|
"""(Re-)queue the outline singleton — run once per generation run, after everything."""
|
|
await db.kanban_upsert_card(topic, BOARD, OUTLINE_CARD, "outline", "outline",
|
|
{"title": "Gliederung"})
|
|
|
|
|
|
async def _proc_outline(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
|
topic = flow.topic
|
|
done = await db.kanban_cards(topic, board="inventory", stage="done_block")
|
|
done.sort(key=lambda c: c["updated_at"])
|
|
entries = {i: _entry_line(c["payload"]) for i, c in enumerate(done, 1)
|
|
if c["payload"].get("title")}
|
|
if entries:
|
|
# The outline may run BEFORE finalize has merged the global facts.json — feed the
|
|
# prereq hints of _learning_order from the card payloads instead (complete as soon
|
|
# as every block passed the facts stage, which the trimmed barrier guarantees).
|
|
facts_map: dict = {}
|
|
for bc in await db.kanban_cards(topic, board=BOARD, kind="ablock"):
|
|
facts_map.update(bc["payload"].get("facts") or {})
|
|
fp = flow.work_dir / "outline-facts.json"
|
|
atomic_write_json(fp, facts_map, indent=1)
|
|
plan = await _outline_block(ctx, _nset, {**files, "facts": fp}, entries, instructions)
|
|
if ctx.is_cancelled():
|
|
return
|
|
if isinstance(plan, dict) and plan.get("chapters"):
|
|
chapters = [
|
|
{"title": ch.get("title", "Kapitel"),
|
|
"blocks": [_title(entries[n]) for n in ch.get("numbers", []) if n in entries]}
|
|
for ch in plan["chapters"]
|
|
]
|
|
await db.set_outline(topic, json.dumps({"chapters": chapters}, ensure_ascii=False))
|
|
await db.kanban_advance_many(topic, BOARD, [(c["card_id"], DONE) for c in cards])
|
|
flow.wake.set()
|
|
|
|
|
|
# ── Stage list (appended after board 1 in chain order) ─────────────────────────────
|
|
def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
|
|
instructions: str) -> list[Stage]:
|
|
research_done = lambda: flow.research_done # noqa: E731
|
|
return [
|
|
Stage(BOARD, "subblocks", lambda cs: _proc_subblocks(ctx, flow, files, instructions, cs)),
|
|
Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)),
|
|
Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)),
|
|
Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)),
|
|
Stage(BOARD, "question_pattern",
|
|
lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)),
|
|
Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)),
|
|
Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True),
|
|
Stage(BOARD, "outline", lambda cs: _proc_outline(ctx, flow, files, instructions, cs),
|
|
barrier=True, drain=True, gate=research_done),
|
|
]
|