update
This commit is contained in:
team3
@@ -4,27 +4,23 @@ A card is spawned by board 1's `done` column per mirrored block and runs through
|
||||
generate → verify (inkl. Fix-Tail) → artefakte (Gen + Prüfer) → finalize
|
||||
(die verschmolzenen Calls liegen in block_calls.py — 4–5 serielle Segmente statt ~20).
|
||||
finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte
|
||||
files + the DB tables. Danach zwei topic-weite BARRIEREN: `konsolidierung` (cross-block
|
||||
sub dedup, faltet per repair.falte_sub) und `outline` (prerequisite graph → chapter order),
|
||||
re-run once per generation run — outline läuft parallel zur Dedup-Barriere."""
|
||||
files + the DB tables. Danach eine topic-weite BARRIERE: `outline` (prerequisite graph →
|
||||
chapter order), re-run once per generation run."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
import database as db
|
||||
import blocks
|
||||
import embedding
|
||||
from block_calls import _artefakte_block, _generate_block, _verify_block
|
||||
from blocks import ARTEFACT_TYPES, _match_sub, _neg_set, _sink_json, _outline_block
|
||||
from config import CROSS_CHUNK_PAARE, EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
|
||||
from blocks import ARTEFACT_TYPES, _match_sub, _outline_block
|
||||
from fsutil import atomic_write_json
|
||||
from jsonio import read_json_file as _json_file
|
||||
from kanban import Flow, Stage
|
||||
from pipeline import FAILED, GenContext, _extra, _log, _prompt, _timeout, run_single_slot
|
||||
from textkit import _norm_title, _title, parse_facts
|
||||
from pipeline import GenContext, _log
|
||||
from textkit import _norm_title, _title
|
||||
|
||||
log = logging.getLogger("creator.board_artefacts")
|
||||
|
||||
@@ -274,162 +270,6 @@ async def _proc_artefakte(ctx: GenContext, flow: Flow, files: dict, instructions
|
||||
await _gather_cards(ctx, flow, cards, one)
|
||||
|
||||
|
||||
def _cross_schema(data) -> dict[int, str] | None:
|
||||
"""{"pairs": {"1": "a"|"b"|"nein"}} → {pair_nr: verdict} · otherwise None."""
|
||||
if not isinstance(data, dict) or not isinstance(data.get("pairs"), dict):
|
||||
return None
|
||||
out: dict[int, str] = {}
|
||||
for k, v in data["pairs"].items():
|
||||
try:
|
||||
nr = int(k)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
s = str(v).strip().casefold()
|
||||
if s in ("a", "b", "nein"):
|
||||
out[nr] = s
|
||||
return out or None
|
||||
|
||||
|
||||
async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
|
||||
"""BARRIER/drain am RUN-ENDE — cross-block sub dedup: the SAME statement carried by two
|
||||
blocks (measured on Markdown: tab handling in 3 blocks, HTML blocks, backslash escapes —
|
||||
the in-block paths never see these). Embedding candidates (block≠block, cos ≥
|
||||
SUB_DUP_KANDIDAT_COS) go to a two-judge panel; UNANIMITY decides which block keeps the
|
||||
statement. Sitzt seit dem Umbau NACH finalize: als Mittel-Barriere wartete jede fertige
|
||||
Karte auf die langsamste (gemessen: 8:46 min Leerlauf pro Block, kanban-smoke). Der
|
||||
Verlierer wird per repair.falte_sub gefaltet (variant + Fragen/Artefakte umhängen) —
|
||||
die wenigen Cross-Dubletten kosten so ein paar umsonst generierte Artefakte statt
|
||||
Minuten Wandzeit für alle. Fail-open on judge failure/dissent."""
|
||||
from repair import falte_sub
|
||||
topic = flow.topic
|
||||
work_dir = flow.work_dir
|
||||
# Resume-Karten aus der alten Stage-Position (Barriere lag vor den Fragen): erst fertig
|
||||
# generieren — die Barriere feuert erneut, wenn alle wieder hier sind. Direkt dedupen
|
||||
# ginge schief: finalize würde den gefalteten Sub aus dem Karten-Sidecar re-spiegeln.
|
||||
nachzuegler = [(c["card_id"], "artefakte" if "sidecar" in c["payload"] else "generate")
|
||||
for c in cards if "pattern" not in c["payload"]]
|
||||
if nachzuegler:
|
||||
await db.kanban_advance_many(topic, BOARD, nachzuegler)
|
||||
flow.wake.set()
|
||||
return
|
||||
|
||||
async def _advance_all():
|
||||
await db.kanban_advance_many(topic, BOARD, [(c["card_id"], DONE) for c in cards])
|
||||
flow.wake.set()
|
||||
|
||||
rows = [r for r in await db.list_subblocks(topic) if r["status"] == "consensus"]
|
||||
if len(rows) < 2 or not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
|
||||
await _advance_all()
|
||||
return
|
||||
sims = await asyncio.to_thread(embedding.embed_sims, [r["sub_title"] for r in rows])
|
||||
if sims is None:
|
||||
await _advance_all()
|
||||
return
|
||||
negs = [_neg_set(r["sub_title"]) for r in rows]
|
||||
pairs = [(i, j) for i in range(len(rows)) for j in range(i + 1, len(rows))
|
||||
if rows[i]["block_norm"] != rows[j]["block_norm"] and negs[i] == negs[j]
|
||||
and float(sims[i][j]) >= SUB_DUP_KANDIDAT_COS]
|
||||
if not pairs:
|
||||
await _advance_all()
|
||||
return
|
||||
|
||||
def _kp(r: dict) -> list:
|
||||
return parse_facts(r.get("facts")).get("key_points") or []
|
||||
|
||||
def _side(tag: str, r: dict) -> str:
|
||||
return f"{tag}: [Block: {r['block']}] {r['sub_title']}" + "".join(f"\n - {p}" for p in _kp(r))
|
||||
|
||||
# chunked judging: ONE call over all pairs scaled its timeout past 50 min, and a hung
|
||||
# call blocked the barrier for the full window (measured on aak: 196 pairs, 2×54 min)
|
||||
chunks = [pairs[lo:lo + CROSS_CHUNK_PAARE] for lo in range(0, len(pairs), CROSS_CHUNK_PAARE)]
|
||||
|
||||
async def _urteile_chunk(chunk: list[tuple[int, int]]) -> dict[int, str]:
|
||||
"""Two judges (+ substitute, + tie-breaker) over one pair chunk → {local_k: verdict};
|
||||
empty dict = fail-open (pairs stay)."""
|
||||
lines = "\n\n".join(
|
||||
f"{k}.\n{_side('A', rows[i])}\n{_side('B', rows[j])}"
|
||||
for k, (i, j) in enumerate(chunk, 1))
|
||||
h = hashlib.md5(lines.encode()).hexdigest()[:8]
|
||||
paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
|
||||
|
||||
async def _judge(j, path, plines, n):
|
||||
if _cross_schema(_json_file(path)) is not None:
|
||||
return # resume
|
||||
status, _v = await run_single_slot(
|
||||
ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
|
||||
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=plines, extra=_extra(instructions)),
|
||||
role="judge", capabilities="none",
|
||||
payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
|
||||
timeout=_timeout("subblock_check", n))
|
||||
if status == FAILED:
|
||||
_log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")
|
||||
|
||||
await asyncio.gather(*[_judge(j, p, lines, len(chunk)) for j, p in zip((1, 2), paths)])
|
||||
if ctx.is_cancelled():
|
||||
return {}
|
||||
outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
|
||||
if len(outs) == 1: # Ersatz-Richter statt fail-open bei EINEM Ausfall
|
||||
ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
|
||||
await _judge("E", ersatz, lines, len(chunk))
|
||||
if ctx.is_cancelled():
|
||||
return {}
|
||||
outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
|
||||
if len(outs) != 2:
|
||||
if outs:
|
||||
_log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
|
||||
return {}
|
||||
final = {k: (outs[0].get(k, "nein") if outs[0].get(k, "nein") == outs[1].get(k, "nein")
|
||||
else "uneinig") for k in range(1, len(chunk) + 1)}
|
||||
disputed = [k for k, v in final.items() if v == "uneinig"]
|
||||
if disputed: # tie-breaker: a third judge sees ONLY the disputed pairs, majority 2/3
|
||||
d_lines = "\n\n".join(
|
||||
f"{x}.\n{_side('A', rows[chunk[k - 1][0]])}\n{_side('B', rows[chunk[k - 1][1]])}"
|
||||
for x, k in enumerate(disputed, 1))
|
||||
p3 = work_dir / f"sub-crossblock-{h}-j3.json"
|
||||
await _judge(3, p3, d_lines, len(disputed))
|
||||
if ctx.is_cancelled():
|
||||
return {}
|
||||
v3 = _cross_schema(_json_file(p3)) or {}
|
||||
if not v3:
|
||||
_log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
|
||||
for x, k in enumerate(disputed, 1):
|
||||
t = v3.get(x, "nein")
|
||||
if t in (outs[0].get(k, "nein"), outs[1].get(k, "nein")):
|
||||
final[k] = t # majority 2/3; anything else stays disputed → no fold
|
||||
return final
|
||||
|
||||
chunk_finals = await asyncio.gather(*[_urteile_chunk(c) for c in chunks])
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
final_all: dict[int, str] = {} # global pair index (1-based over `pairs`) → verdict
|
||||
for cnr, fin in enumerate(chunk_finals):
|
||||
for k, v in fin.items():
|
||||
final_all[cnr * CROSS_CHUNK_PAARE + k] = v
|
||||
journal = {"paare": len(pairs), "chunks": len(chunks), "gefaltet": [], "verdicts": []}
|
||||
gone: set[tuple] = set()
|
||||
for k, (i, j) in enumerate(pairs, 1):
|
||||
verdict = final_all.get(k, "nein")
|
||||
journal["verdicts"].append({"a": f"{rows[i]['block']} · {rows[i]['sub_title']}",
|
||||
"b": f"{rows[j]['block']} · {rows[j]['sub_title']}",
|
||||
"verdict": verdict})
|
||||
if verdict not in ("a", "b"):
|
||||
continue
|
||||
win, lose = (rows[i], rows[j]) if verdict == "a" else (rows[j], rows[i])
|
||||
wk = (win["block_norm"], win["sub_norm"])
|
||||
lk = (lose["block_norm"], lose["sub_norm"])
|
||||
if lk in gone or wk in gone: # keeper already folded → don't chain away the content
|
||||
continue
|
||||
await falte_sub(topic, files, win, lose)
|
||||
gone.add(lk)
|
||||
journal["gefaltet"].append({"weg": f"{lose['block']} · {lose['sub_title']}",
|
||||
"bleibt": f"{win['block']} · {win['sub_title']}"})
|
||||
if journal["gefaltet"]:
|
||||
_log(topic, f"Sub-Crossblock: {len(journal['gefaltet'])} blockübergreifende Dublette(n) gefaltet")
|
||||
hg = hashlib.md5("\n".join(f"{i}:{j}" for i, j in pairs).encode()).hexdigest()[:8]
|
||||
atomic_write_json(work_dir / f"sub-crossblock-{hg}.json", journal, indent=1)
|
||||
await _advance_all()
|
||||
|
||||
|
||||
# ── Finalize (SERIAL): merge into the global files + DB tables ─────────────────────
|
||||
def _merge_json(path, block_keys: dict) -> None:
|
||||
data = _json_file(path)
|
||||
|
||||
Reference in New Issue
Block a user