update
This commit is contained in:
Team3
@@ -1,14 +1,12 @@
|
||||
"""Board 2 „Artefakte": per finished block, prepare the learning artefacts the guide presents.
|
||||
|
||||
A card is spawned by board 1's `done` column per mirrored block and runs through:
|
||||
subblocks → facts → levels → relevance → question_pattern (+artefacts parallel) → finalize
|
||||
generate → verify (inkl. Fix-Tail) → artefakte (Gen + Prüfer) → finalize
|
||||
(die verschmolzenen Calls liegen in block_calls.py — 4–5 serielle Segmente statt ~20).
|
||||
finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte
|
||||
files + the DB tables. Danach zwei topic-weite BARRIEREN: `konsolidierung` (cross-block
|
||||
sub dedup, faltet per repair.falte_sub) und `outline` (prerequisite graph → chapter order),
|
||||
re-run once per generation run.
|
||||
|
||||
The heavy lifting is the existing per-block functions in blocks.py — each card gets its own
|
||||
work subdirectory + facts/artefakte paths, so their slot files never collide across blocks."""
|
||||
re-run once per generation run — outline läuft parallel zur Dedup-Barriere."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
@@ -19,11 +17,8 @@ import re
|
||||
import database as db
|
||||
import blocks
|
||||
import embedding
|
||||
from blocks import (
|
||||
ARTEFACT_TYPES, _artefacts_block, _facts_block, _facts_nachfass, _konsolidiere_subblocks,
|
||||
_levels_block, _luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
|
||||
_sink_json, _subblocks_block, _outline_block,
|
||||
)
|
||||
from block_calls import _artefakte_block, _generate_block, _verify_block
|
||||
from blocks import ARTEFACT_TYPES, _match_sub, _neg_set, _sink_json, _outline_block
|
||||
from config import CROSS_CHUNK_PAARE, EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
|
||||
from fsutil import atomic_write_json
|
||||
from jsonio import read_json_file as _json_file
|
||||
@@ -100,7 +95,7 @@ def make_spawner(topic: str, files: dict):
|
||||
norm = payload.get("mirrored_norm")
|
||||
if not norm:
|
||||
return
|
||||
await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "subblocks", {
|
||||
await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "generate", {
|
||||
"title": payload.get("title", ""),
|
||||
"description": payload.get("description", ""),
|
||||
"n_size": payload.get("n_size", 0), # LPT estimate until subs_n exists
|
||||
@@ -172,88 +167,76 @@ async def _seed_map(topic: str) -> dict[str, list[str]]:
|
||||
return seeds
|
||||
|
||||
|
||||
async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
||||
async def _proc_generate(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
||||
"""Verschmolzener Erzeuger: 2 unabhängige Generatoren liefern Subs+Facts+Einstufung
|
||||
in EINEM Call, Konsens im Code (block_calls._generate_block)."""
|
||||
topic = flow.topic
|
||||
# Fix 4: fragments demoted to a parent become seed candidates of the parent's subblocks.
|
||||
# fragments demoted to a parent become seed candidates of the parent's subblocks
|
||||
seeds = await _seed_map(topic)
|
||||
|
||||
async def one(c):
|
||||
p = c["payload"]
|
||||
norm = c["card_id"]
|
||||
instr = instructions
|
||||
sd = [s for s in seeds.get(norm, []) if s]
|
||||
if sd:
|
||||
instr = (instructions + "\n\nBereits identifizierte Unterpunkt-Kandidaten dieses "
|
||||
"Blocks (prüfen; wenn belegt UND noch nicht durch einen anderen Eintrag "
|
||||
"abgedeckt, aufnehmen — nicht wörtlich übernehmen, sondern als eigenständige "
|
||||
"Aussage formulieren):\n"
|
||||
+ "\n".join(f"- {s}" for s in sd))
|
||||
raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
||||
{1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-",
|
||||
seeds=sd or None, lbl=f"{p.get('title', norm)} · ",
|
||||
sources=p.get("sources"))
|
||||
if raw is None:
|
||||
return _fail_or_cancel(ctx, f"Subblocks {p.get('title', norm)}")
|
||||
p["raw"] = raw
|
||||
p["subs_n"] = sum(len(v) for v in raw.values()) # LPT: bigger blocks pull first
|
||||
gen = await _generate_block(ctx, _pfiles(files, norm), p.get("title", ""),
|
||||
p.get("description", ""), instructions,
|
||||
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ",
|
||||
sources=p.get("sources"),
|
||||
seeds=[s for s in seeds.get(norm, []) if s] or None)
|
||||
if gen is None:
|
||||
return _fail_or_cancel(ctx, f"Generate {p.get('title', norm)}")
|
||||
p["raw"], p["facts"] = gen["raw"], gen["facts"]
|
||||
p["unsicher"], p["votes"] = gen["unsicher"], gen["votes"]
|
||||
p["subs_n"] = sum(len(v) for v in gen["raw"].values()) + len(gen["unsicher"])
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "facts")
|
||||
await db.kanban_advance(topic, BOARD, norm, "verify")
|
||||
|
||||
await _gather_cards(ctx, flow, cards, one)
|
||||
|
||||
|
||||
async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
|
||||
instructions: str, cards):
|
||||
async def _proc_verify(ctx: GenContext, flow: Flow, files: dict, q: dict, instructions: str, cards):
|
||||
"""Verschmolzener Prüfer: MECE + Facts + Einstufung in einem Panel-Call, Fix-Tail
|
||||
inklusive (block_calls._verify_block) — ersetzt facts-check/konsolidierung/levels/
|
||||
relevance als eigene Stages."""
|
||||
topic = flow.topic
|
||||
|
||||
async def one(c):
|
||||
p = c["payload"]
|
||||
norm = c["card_id"]
|
||||
raw = p.get("raw") or {}
|
||||
res = await _facts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), raw, q,
|
||||
folder, instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
|
||||
title = p.get("title", "")
|
||||
gen = {"raw": p.get("raw") or {}, "facts": p.get("facts") or {},
|
||||
"unsicher": p.get("unsicher") or [], "votes": p.get("votes") or {}}
|
||||
res = await _verify_block(ctx, _pfiles(files, norm), title, gen, q, instructions,
|
||||
ns=f"{_safe(norm)}-", lbl=f"{title or norm} · ",
|
||||
sources=p.get("sources"))
|
||||
if res is None:
|
||||
return _fail_or_cancel(ctx, f"Facts {p.get('title', norm)}")
|
||||
facts_map, discarded = res
|
||||
if discarded: # unsupportable subs vanish from raw too (guide never sees them)
|
||||
for bt, sns in discarded.items():
|
||||
if bt in raw:
|
||||
raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns]
|
||||
raw = {bt: subs for bt, subs in raw.items() if subs}
|
||||
# In-block consolidation: two-judge panel folds same-statement/subset subs, bundles
|
||||
# catalogs, drops off-topic ones — the facts are in hand (key points as evidence),
|
||||
# questions/artefacts not yet built. Reported gaps get ONE follow-up finder round.
|
||||
luecken = await _konsolidiere_subblocks(ctx, _pfiles(files, norm), raw, facts_map,
|
||||
instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ")
|
||||
if ctx.is_cancelled():
|
||||
return None
|
||||
nachgefasst = 0
|
||||
for bt, lk in (luecken or {}).items():
|
||||
nachgefasst += await _luecken_runde(ctx, _pfiles(files, norm), bt, lk, raw, facts_map,
|
||||
q, folder, instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
|
||||
if ctx.is_cancelled():
|
||||
return None
|
||||
if nachgefasst: # close the loop: follow-up finds get the SAME duplicate test as the
|
||||
# rest (new subs-hash → fresh judge files); their gap report is deliberately ignored
|
||||
await _konsolidiere_subblocks(ctx, _pfiles(files, norm), raw, facts_map,
|
||||
instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ")
|
||||
if ctx.is_cancelled():
|
||||
return None
|
||||
raw = {bt: subs for bt, subs in raw.items() if subs}
|
||||
# consolidation renames/catalogs can leave consensus subs without grounding — the
|
||||
# guide fact gate then flags their correct statements wholesale (measured: 74/210)
|
||||
await _facts_nachfass(ctx, _pfiles(files, norm), raw, facts_map, q, folder,
|
||||
instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
|
||||
if ctx.is_cancelled():
|
||||
return None
|
||||
p["raw"], p["facts"] = raw, facts_map
|
||||
return None # nur Cancel — Karte bleibt liegen
|
||||
p["raw"], p["facts"], p["sidecar"] = res["raw"], res["facts"], res["sidecar"]
|
||||
p.pop("unsicher", None)
|
||||
p.pop("votes", None)
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "levels")
|
||||
await db.kanban_advance(topic, BOARD, norm, "artefakte")
|
||||
|
||||
await _gather_cards(ctx, flow, cards, one)
|
||||
|
||||
|
||||
async def _proc_artefakte(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
||||
"""Fragen + Flashcards + Beispiele in einem Generator-Call, ein Prüfer-Call dahinter
|
||||
(block_calls._artefakte_block)."""
|
||||
topic = flow.topic
|
||||
|
||||
async def one(c):
|
||||
p = c["payload"]
|
||||
norm = c["card_id"]
|
||||
title = p.get("title", "")
|
||||
res = await _artefakte_block(ctx, _pfiles(files, norm), title,
|
||||
(p.get("sidecar") or {}).get(title) or [],
|
||||
instructions, ns=f"{_safe(norm)}-", lbl=f"{title or norm} · ")
|
||||
if res is None:
|
||||
return None # nur Cancel
|
||||
p["pattern"] = res["pattern"]
|
||||
p["artefacts"] = res["artefacts"]
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "finalize")
|
||||
|
||||
await _gather_cards(ctx, flow, cards, one)
|
||||
|
||||
@@ -290,8 +273,8 @@ async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instruc
|
||||
# Resume-Karten aus der alten Stage-Position (Barriere lag vor den Fragen): erst fertig
|
||||
# generieren — die Barriere feuert erneut, wenn alle wieder hier sind. Direkt dedupen
|
||||
# ginge schief: finalize würde den gefalteten Sub aus dem Karten-Sidecar re-spiegeln.
|
||||
nachzuegler = [(c["card_id"], "question_pattern") for c in cards
|
||||
if "pattern" not in c["payload"]]
|
||||
nachzuegler = [(c["card_id"], "artefakte" if "sidecar" in c["payload"] else "generate")
|
||||
for c in cards if "pattern" not in c["payload"]]
|
||||
if nachzuegler:
|
||||
await db.kanban_advance_many(topic, BOARD, nachzuegler)
|
||||
flow.wake.set()
|
||||
@@ -417,101 +400,6 @@ async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instruc
|
||||
await _advance_all()
|
||||
|
||||
|
||||
async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
||||
topic = flow.topic
|
||||
|
||||
async def one(c):
|
||||
p = c["payload"]
|
||||
norm = c["card_id"]
|
||||
sidecar = await _levels_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
||||
p.get("raw") or {}, instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ")
|
||||
if sidecar is None:
|
||||
return _fail_or_cancel(ctx, f"Levels {p.get('title', norm)}")
|
||||
facts_map = p.get("facts") or {}
|
||||
for btitle, subs in sidecar.items(): # merge facts into the subs (mirror + guide)
|
||||
fm = facts_map.get(btitle, {})
|
||||
for sub in subs:
|
||||
# level agents paraphrase titles — exact miss falls back to the unique
|
||||
# prefix/containment match, else the sub silently loses its grounding
|
||||
sn = _sub_key(set(fm), _norm_title(sub["title"]))
|
||||
if (fk := fm.get(sn)):
|
||||
sub["facts"] = fk
|
||||
p["sidecar"] = sidecar
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "relevance")
|
||||
|
||||
await _gather_cards(ctx, flow, cards, one)
|
||||
|
||||
|
||||
async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
||||
topic = flow.topic
|
||||
|
||||
async def one(c):
|
||||
p = c["payload"]
|
||||
norm = c["card_id"]
|
||||
sidecar = p.get("sidecar") or {}
|
||||
rel = await _relevance_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
||||
sidecar, instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ")
|
||||
if rel is None:
|
||||
return _fail_or_cancel(ctx, f"Relevance {p.get('title', norm)}")
|
||||
gid = 0
|
||||
for subs in sidecar.values():
|
||||
for sub in subs:
|
||||
gid += 1
|
||||
sub["relevance"] = rel.get(gid, "relevant")
|
||||
p["sidecar"] = sidecar
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "question_pattern")
|
||||
|
||||
await _gather_cards(ctx, flow, cards, one)
|
||||
|
||||
|
||||
async def _proc_question_pattern(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
||||
"""Fragen UND Artefakte im Fächer: beide brauchen nur den sidecar, nichts voneinander —
|
||||
als Stage-Treppe kosteten sie zwei serielle Call-Segmente auf dem kritischen Pfad.
|
||||
Die artefacts-Stage bleibt für Resume-Karten alter Läufe registriert."""
|
||||
topic = flow.topic
|
||||
|
||||
async def one(c):
|
||||
p = c["payload"]
|
||||
norm = c["card_id"]
|
||||
pattern, artefacts = await asyncio.gather(
|
||||
_question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
||||
p.get("sidecar") or {}, instructions,
|
||||
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · "),
|
||||
_artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
||||
p.get("sidecar") or {}, instructions,
|
||||
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · "))
|
||||
if pattern is None:
|
||||
return _fail_or_cancel(ctx, f"Fragen {p.get('title', norm)}")
|
||||
p["pattern"] = pattern
|
||||
p["artefacts"] = artefacts or {} # artefacts are optional — never fatal
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "finalize")
|
||||
|
||||
await _gather_cards(ctx, flow, cards, one)
|
||||
|
||||
|
||||
async def _proc_artefacts(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
||||
topic = flow.topic
|
||||
|
||||
async def one(c):
|
||||
p = c["payload"]
|
||||
norm = c["card_id"]
|
||||
artefacts = await _artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
|
||||
p.get("sidecar") or {}, instructions,
|
||||
ns=f"{_safe(norm)}-", lbl=f"{p.get('title', norm)} · ")
|
||||
if artefacts is None and ctx.is_cancelled():
|
||||
return None
|
||||
p["artefacts"] = artefacts or {} # artefacts are optional — never fatal
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "finalize")
|
||||
|
||||
await _gather_cards(ctx, flow, cards, one)
|
||||
|
||||
|
||||
# ── Finalize (SERIAL): merge into the global files + DB tables ─────────────────────
|
||||
def _merge_json(path, block_keys: dict) -> None:
|
||||
data = _json_file(path)
|
||||
@@ -627,18 +515,33 @@ async def _proc_outline(ctx: GenContext, flow: Flow, files: dict, instructions:
|
||||
|
||||
|
||||
# ── Stage list (appended after board 1 in chain order) ─────────────────────────────
|
||||
_ALT_STAGES = ("subblocks", "facts", "levels", "relevance", "question_pattern", "artefacts")
|
||||
|
||||
|
||||
async def migriere_alt_karten(topic: str) -> int:
|
||||
"""Harter Schnitt: Karten der alten Stage-Treppe beim Flow-Start auf `generate`
|
||||
zurücksetzen (Payload auf die Spawn-Felder reduziert — Zwischenstände der alten
|
||||
Struktur sind für die verschmolzenen Calls wertlos). → Anzahl migrierter Karten."""
|
||||
moves = []
|
||||
for c in await db.kanban_cards(topic, board=BOARD, kind="ablock"):
|
||||
if c["stage"] in _ALT_STAGES:
|
||||
p = c["payload"]
|
||||
await db.kanban_set_payload(topic, BOARD, c["card_id"], {
|
||||
"title": p.get("title", ""), "description": p.get("description", ""),
|
||||
"n_size": p.get("n_size", 0), "sources": p.get("sources")})
|
||||
moves.append((c["card_id"], "generate"))
|
||||
if moves:
|
||||
await db.kanban_advance_many(topic, BOARD, moves)
|
||||
return len(moves)
|
||||
|
||||
|
||||
def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
|
||||
instructions: str) -> list[Stage]:
|
||||
research_done = lambda: flow.research_done # noqa: E731
|
||||
return [
|
||||
Stage(BOARD, "subblocks", lambda cs: _proc_subblocks(ctx, flow, files, instructions, cs)),
|
||||
Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)),
|
||||
Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)),
|
||||
Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)),
|
||||
Stage(BOARD, "question_pattern",
|
||||
lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)),
|
||||
# Resume-Pfad: Karten alter Läufe, die noch in artefacts stehen
|
||||
Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)),
|
||||
Stage(BOARD, "generate", lambda cs: _proc_generate(ctx, flow, files, instructions, cs)),
|
||||
Stage(BOARD, "verify", lambda cs: _proc_verify(ctx, flow, files, q, instructions, cs)),
|
||||
Stage(BOARD, "artefakte", lambda cs: _proc_artefakte(ctx, flow, files, instructions, cs)),
|
||||
Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True),
|
||||
# Cross-Block-Dedup als END-Barriere: als Mittel-Barriere idelte jede fertige Karte
|
||||
# auf die langsamste (8:46 min/Block gemessen); jetzt faltet sie nach finalize
|
||||
|
||||
Reference in New Issue
Block a user