marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Team3
2026-07-04 19:20:48 +02:00
parent 92c69c1561
commit c05421a8c1
14 changed files with 695 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
backend/board_artefacts.py
View File

@@ -1,10 +1,11 @@
"""Board 2 „Artefakte": per finished block, prepare the learning artefacts the guide presents.
A card is spawned by board 1's `done` column per mirrored block and runs through:
subblocks → facts → levels → relevance → question_pattern → artefacts → finalize
subblocks → facts → levels → relevance → question_pattern (+artefacts parallel) → finalize
finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte
files + the DB tables. `outline` is a topic-wide BARRIER singleton at the very end
(prerequisite graph → chapter order), re-run once per generation run.
files + the DB tables. Danach zwei topic-weite BARRIEREN: `konsolidierung` (cross-block
sub dedup, faltet per repair.falte_sub) und `outline` (prerequisite graph → chapter order),
re-run once per generation run.
The heavy lifting is the existing per-block functions in blocks.py — each card gets its own
work subdirectory + facts/artefakte paths, so their slot files never collide across blocks."""
@@ -274,61 +275,56 @@ def _cross_schema(data) -> dict[int, str] | None:
async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
"""BARRIER/drain — cross-block sub dedup: the SAME statement carried by two blocks
(measured on Markdown: tab handling in 3 blocks, HTML blocks, backslash escapes — the
in-block paths never see these). Embedding candidates (block≠block, cos ≥
"""BARRIER/drain am RUN-ENDE — cross-block sub dedup: the SAME statement carried by two
blocks (measured on Markdown: tab handling in 3 blocks, HTML blocks, backslash escapes —
the in-block paths never see these). Embedding candidates (block≠block, cos ≥
SUB_DUP_KANDIDAT_COS) go to a two-judge panel; UNANIMITY decides which block keeps the
statement. The loser leaves its card's raw/facts and turns DB `variant` — before
questions/artefacts exist, so no orphans. Fail-open on judge failure/dissent."""
statement. Sitzt seit dem Umbau NACH finalize: als Mittel-Barriere wartete jede fertige
Karte auf die langsamste (gemessen: 8:46 min Leerlauf pro Block, kanban-smoke). Der
Verlierer wird per repair.falte_sub gefaltet (variant + Fragen/Artefakte umhängen) —
die wenigen Cross-Dubletten kosten so ein paar umsonst generierte Artefakte statt
Minuten Wandzeit für alle. Fail-open on judge failure/dissent."""
from repair import falte_sub
topic = flow.topic
work_dir = flow.work_dir
package_norms = {c["card_id"] for c in cards}
entries: list[tuple[int, str, str]] = [] # (card idx, block title, sub title); idx -1 = context
for ci, c in enumerate(cards):
for bt, subs in (c["payload"].get("raw") or {}).items():
for s in subs:
entries.append((ci, bt, s))
n_pkg = len(entries)
# Context: consensus subs of blocks already PAST this barrier (late spawns via the
# gap-check feedback would otherwise never be compared). Context never folds —
# its card payload lives downstream (board-1 rule: confirmed context always wins).
for r in await db.list_subblocks(topic):
if r["status"] == "consensus" and r["block_norm"] not in package_norms:
entries.append((-1, r["block"], r["sub_title"]))
ctx_facts: dict[str, dict] = {} # facts of downstream cards (DB rows carry none yet)
for bc in await db.kanban_cards(topic, board=BOARD, kind="ablock"):
if bc["card_id"] not in package_norms:
for bt, fm in (bc["payload"].get("facts") or {}).items():
ctx_facts[_norm_title(bt)] = fm
# Resume-Karten aus der alten Stage-Position (Barriere lag vor den Fragen): erst fertig
# generieren — die Barriere feuert erneut, wenn alle wieder hier sind. Direkt dedupen
# ginge schief: finalize würde den gefalteten Sub aus dem Karten-Sidecar re-spiegeln.
nachzuegler = [(c["card_id"], "question_pattern") for c in cards
if "pattern" not in c["payload"]]
if nachzuegler:
await db.kanban_advance_many(topic, BOARD, nachzuegler)
flow.wake.set()
return
async def _advance_all():
await db.kanban_advance_many(topic, BOARD, [(c["card_id"], "question_pattern") for c in cards])
await db.kanban_advance_many(topic, BOARD, [(c["card_id"], DONE) for c in cards])
flow.wake.set()
if n_pkg < 1 or len(entries) < 2 or not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
rows = [r for r in await db.list_subblocks(topic) if r["status"] == "consensus"]
if len(rows) < 2 or not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
await _advance_all()
return
sims = await asyncio.to_thread(embedding.embed_sims, [s for _, _, s in entries])
sims = await asyncio.to_thread(embedding.embed_sims, [r["sub_title"] for r in rows])
if sims is None:
await _advance_all()
return
negs = [_neg_set(s) for _, _, s in entries]
pairs = [(i, j) for i in range(len(entries)) for j in range(i + 1, len(entries))
if entries[i][0] != entries[j][0] and negs[i] == negs[j]
negs = [_neg_set(r["sub_title"]) for r in rows]
pairs = [(i, j) for i in range(len(rows)) for j in range(i + 1, len(rows))
if rows[i]["block_norm"] != rows[j]["block_norm"] and negs[i] == negs[j]
and float(sims[i][j]) >= SUB_DUP_KANDIDAT_COS]
if not pairs:
await _advance_all()
return
def _kp(ci: int, bt: str, s: str) -> list:
if ci < 0:
f = ctx_facts.get(_norm_title(bt)) or {}
else:
f = (cards[ci]["payload"].get("facts") or {}).get(bt) or {}
return (f.get(_norm_title(s)) or {}).get("key_points") or []
def _kp(r: dict) -> list:
try:
return (json.loads(r.get("facts") or "{}")).get("key_points") or []
except ValueError:
return []
def _side(tag: str, ci: int, bt: str, s: str) -> str:
return f"{tag}: [Block: {bt}] {s}" + "".join(f"\n - {p}" for p in _kp(ci, bt, s))
def _side(tag: str, r: dict) -> str:
return f"{tag}: [Block: {r['block']}] {r['sub_title']}" + "".join(f"\n - {p}" for p in _kp(r))
# chunked judging: ONE call over all pairs scaled its timeout past 50 min, and a hung
# call blocked the barrier for the full window (measured on aak: 196 pairs, 2×54 min)
@@ -338,7 +334,7 @@ async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instruc
"""Two judges (+ substitute, + tie-breaker) over one pair chunk → {local_k: verdict};
empty dict = fail-open (pairs stay)."""
lines = "\n\n".join(
f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
f"{k}.\n{_side('A', rows[i])}\n{_side('B', rows[j])}"
for k, (i, j) in enumerate(chunk, 1))
h = hashlib.md5(lines.encode()).hexdigest()[:8]
paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
@@ -374,7 +370,7 @@ async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instruc
disputed = [k for k, v in final.items() if v == "uneinig"]
if disputed: # tie-breaker: a third judge sees ONLY the disputed pairs, majority 2/3
d_lines = "\n\n".join(
f"{x}.\n{_side('A', *entries[chunk[k - 1][0]])}\n{_side('B', *entries[chunk[k - 1][1]])}"
f"{x}.\n{_side('A', rows[chunk[k - 1][0]])}\n{_side('B', rows[chunk[k - 1][1]])}"
for x, k in enumerate(disputed, 1))
p3 = work_dir / f"sub-crossblock-{h}-j3.json"
await _judge(3, p3, d_lines, len(disputed))
@@ -397,39 +393,23 @@ async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instruc
for k, v in fin.items():
final_all[cnr * CROSS_CHUNK_PAARE + k] = v
journal = {"paare": len(pairs), "chunks": len(chunks), "gefaltet": [], "verdicts": []}
gone: set[int] = set()
touched: set[int] = set()
gone: set[tuple] = set()
for k, (i, j) in enumerate(pairs, 1):
verdict = final_all.get(k, "nein")
journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
"b": f"{entries[j][1]} · {entries[j][2]}",
journal["verdicts"].append({"a": f"{rows[i]['block']} · {rows[i]['sub_title']}",
"b": f"{rows[j]['block']} · {rows[j]['sub_title']}",
"verdict": verdict})
if verdict not in ("a", "b"):
continue
lose = j if verdict == "a" else i
if entries[lose][0] < 0: # context never folds — the package side goes instead
lose = i if lose == j else j
keep = i if lose == j else j
if lose in gone or keep in gone: # keeper already folded → don't chain away the content
win, lose = (rows[i], rows[j]) if verdict == "a" else (rows[j], rows[i])
wk = (win["block_norm"], win["sub_norm"])
lk = (lose["block_norm"], lose["sub_norm"])
if lk in gone or wk in gone: # keeper already folded → don't chain away the content
continue
ci, bt, s = entries[lose]
p = cards[ci]["payload"]
if s in (p.get("raw") or {}).get(bt, []):
p["raw"][bt].remove(s)
(p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
sc = (p.get("sidecar") or {}).get(bt)
if isinstance(sc, list): # questions/artefacts consume the sidecar downstream
p["sidecar"][bt] = [e for e in sc
if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
gone.add(lose)
touched.add(ci)
journal["gefaltet"].append({"weg": f"{bt} · {s}",
"bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
for ci in touched:
p = cards[ci]["payload"]
p["raw"] = {bt: subs for bt, subs in (p.get("raw") or {}).items() if subs}
await db.kanban_set_payload(topic, BOARD, cards[ci]["card_id"], p)
await falte_sub(topic, files, win, lose)
gone.add(lk)
journal["gefaltet"].append({"weg": f"{lose['block']} · {lose['sub_title']}",
"bleibt": f"{win['block']} · {win['sub_title']}"})
if journal["gefaltet"]:
_log(topic, f"Sub-Crossblock: {len(journal['gefaltet'])} blockübergreifende Dublette(n) gefaltet")
hg = hashlib.md5("\n".join(f"{i}:{j}" for i, j in pairs).encode()).hexdigest()[:8]
@@ -483,25 +463,33 @@ async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions
sub["relevance"] = rel.get(gid, "relevant")
p["sidecar"] = sidecar
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "konsolidierung")
await db.kanban_advance(topic, BOARD, norm, "question_pattern")
await _gather_cards(ctx, flow, cards, one)
async def _proc_question_pattern(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
"""Fragen UND Artefakte im Fächer: beide brauchen nur den sidecar, nichts voneinander —
als Stage-Treppe kosteten sie zwei serielle Call-Segmente auf dem kritischen Pfad.
Die artefacts-Stage bleibt für Resume-Karten alter Läufe registriert."""
topic = flow.topic
async def one(c):
p = c["payload"]
norm = c["card_id"]
pattern = await _question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
p.get("sidecar") or {}, instructions,
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ")
pattern, artefacts = await asyncio.gather(
_question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
p.get("sidecar") or {}, instructions,
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · "),
_artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
p.get("sidecar") or {}, instructions,
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · "))
if pattern is None:
return _fail_or_cancel(ctx, f"Fragen {p.get('title', norm)}")
p["pattern"] = pattern
p["artefacts"] = artefacts or {} # artefacts are optional — never fatal
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "artefacts")
await db.kanban_advance(topic, BOARD, norm, "finalize")
await _gather_cards(ctx, flow, cards, one)
@@ -594,7 +582,7 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")},
ensure_ascii=False)
await db.put_sub_artifact(topic, bnorm, sn, typ, data, bt, str(e.get("subblock", "")))
await db.kanban_advance(topic, BOARD, c["card_id"], DONE)
await db.kanban_advance(topic, BOARD, c["card_id"], "konsolidierung")
_log(topic, f"Artefakte fertig: {title}")
flow.wake.set()
@@ -647,15 +635,17 @@ def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)),
Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)),
Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)),
# Barrier sits AFTER the sub-local stages: cards used to idle here median 36 min
# while levels/relevance work was still ahead of them
Stage(BOARD, "question_pattern",
lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)),
# Resume-Pfad: Karten alter Läufe, die noch in artefacts stehen
Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)),
Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True),
# Cross-Block-Dedup als END-Barriere: als Mittel-Barriere idelte jede fertige Karte
# auf die langsamste (8:46 min/Block gemessen); jetzt faltet sie nach finalize
# per repair.falte_sub — spät gefundene Dubletten kosten Artefakt-Tokens, keine Wandzeit
Stage(BOARD, "konsolidierung",
lambda cs: _proc_konsolidierung(ctx, flow, files, instructions, cs),
barrier=True, drain=True),
Stage(BOARD, "question_pattern",
lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)),
Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)),
Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True),
Stage(BOARD, "outline", lambda cs: _proc_outline(ctx, flow, files, instructions, cs),
barrier=True, drain=True, gate=research_done),
]