marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-04 12:21:45 +02:00
parent 2f5d5b9ca1
commit 8d8f6c8e51
43 changed files with 1920 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
backend/board_artefacts.py
View File

@@ -19,11 +19,11 @@ import database as db
import blocks
import embedding
from blocks import (
ARTEFACT_TYPES, _artefacts_block, _facts_block, _konsolidiere_subblocks, _levels_block,
_luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
ARTEFACT_TYPES, _artefacts_block, _facts_block, _facts_nachfass, _konsolidiere_subblocks,
_levels_block, _luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
_sink_json, _subblocks_block, _outline_block,
)
from config import EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
from config import CROSS_CHUNK_PAARE, EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
from fsutil import atomic_write_json
from jsonio import read_json_file as _json_file
from kanban import Flow, Stage
@@ -243,6 +243,13 @@ async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
if ctx.is_cancelled():
return None
raw = {bt: subs for bt, subs in raw.items() if subs}
# consolidation renames/catalogs can leave consensus subs without grounding — the
# guide fact gate then flags their correct statements wholesale (measured: 74/210)
await _facts_nachfass(ctx, _pfiles(files, norm), raw, facts_map, q, folder,
instructions, ns=f"{_safe(norm)}-",
lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
if ctx.is_cancelled():
return None
p["raw"], p["facts"] = raw, facts_map
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "levels")
@@ -323,98 +330,110 @@ async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instruc
def _side(tag: str, ci: int, bt: str, s: str) -> str:
return f"{tag}: [Block: {bt}] {s}" + "".join(f"\n - {p}" for p in _kp(ci, bt, s))
lines = "\n\n".join(
f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
for k, (i, j) in enumerate(pairs, 1))
h = hashlib.md5(lines.encode()).hexdigest()[:8]
paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
# chunked judging: ONE call over all pairs scaled its timeout past 50 min, and a hung
# call blocked the barrier for the full window (measured on aak: 196 pairs, 2×54 min)
chunks = [pairs[lo:lo + CROSS_CHUNK_PAARE] for lo in range(0, len(pairs), CROSS_CHUNK_PAARE)]
async def _judge(j, path):
if _cross_schema(_json_file(path)) is not None:
return # resume
status, _v = await run_single_slot(
ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=lines, extra=_extra(instructions)),
role="judge", capabilities="none",
payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
timeout=_timeout("subblock_check", len(pairs)))
if status == FAILED:
_log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")
async def _urteile_chunk(chunk: list[tuple[int, int]]) -> dict[int, str]:
"""Two judges (+ substitute, + tie-breaker) over one pair chunk → {local_k: verdict};
empty dict = fail-open (pairs stay)."""
lines = "\n\n".join(
f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
for k, (i, j) in enumerate(chunk, 1))
h = hashlib.md5(lines.encode()).hexdigest()[:8]
paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
await asyncio.gather(*[_judge(j, p) for j, p in zip((1, 2), paths)])
if ctx.is_cancelled():
return
outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
if len(outs) == 1: # Ersatz-Richter statt fail-open bei EINEM Ausfall
ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
await _judge("E", ersatz)
async def _judge(j, path, plines, n):
if _cross_schema(_json_file(path)) is not None:
return # resume
status, _v = await run_single_slot(
ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=plines, extra=_extra(instructions)),
role="judge", capabilities="none",
payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
timeout=_timeout("subblock_check", n))
if status == FAILED:
_log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")
await asyncio.gather(*[_judge(j, p, lines, len(chunk)) for j, p in zip((1, 2), paths)])
if ctx.is_cancelled():
return
outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
journal = {"paare": len(pairs), "richter": len(outs), "gefaltet": [], "verdicts": []}
gone: set[int] = set()
touched: set[int] = set()
if len(outs) == 2:
return {}
outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
if len(outs) == 1: # Ersatz-Richter statt fail-open bei EINEM Ausfall
ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
await _judge("E", ersatz, lines, len(chunk))
if ctx.is_cancelled():
return {}
outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
if len(outs) != 2:
if outs:
_log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
return {}
final = {k: (outs[0].get(k, "nein") if outs[0].get(k, "nein") == outs[1].get(k, "nein")
else "uneinig") for k in range(1, len(pairs) + 1)}
else "uneinig") for k in range(1, len(chunk) + 1)}
disputed = [k for k, v in final.items() if v == "uneinig"]
if disputed: # tie-breaker: a third judge sees ONLY the disputed pairs, majority 2/3
d_lines = "\n\n".join(
f"{x}.\n{_side('A', *entries[pairs[k - 1][0]])}\n{_side('B', *entries[pairs[k - 1][1]])}"
f"{x}.\n{_side('A', *entries[chunk[k - 1][0]])}\n{_side('B', *entries[chunk[k - 1][1]])}"
for x, k in enumerate(disputed, 1))
p3 = work_dir / f"sub-crossblock-{h}-j3.json"
if _cross_schema(_json_file(p3)) is None:
status, _v = await run_single_slot(
ctx, "Sub-Crossblock j3", key=f"blocks-{topic}-sub-crossblock-{h}-j3",
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=d_lines, extra=_extra(instructions)),
role="judge", capabilities="none",
payload=lambda result, p=p3: _sink_json(result, p, _cross_schema),
timeout=_timeout("subblock_check", len(disputed)))
if status == FAILED:
_log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
await _judge(3, p3, d_lines, len(disputed))
if ctx.is_cancelled():
return
return {}
v3 = _cross_schema(_json_file(p3)) or {}
if not v3:
_log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
for x, k in enumerate(disputed, 1):
t = v3.get(x, "nein")
if t in (outs[0].get(k, "nein"), outs[1].get(k, "nein")):
final[k] = t # majority 2/3; anything else stays disputed → no fold
for k, (i, j) in enumerate(pairs, 1):
verdict = final[k]
journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
"b": f"{entries[j][1]} · {entries[j][2]}",
"verdict": verdict})
if verdict not in ("a", "b"):
continue
lose = j if verdict == "a" else i
if entries[lose][0] < 0: # context never folds — the package side goes instead
lose = i if lose == j else j
keep = i if lose == j else j
if lose in gone or keep in gone: # keeper already folded → don't chain away the content
continue
ci, bt, s = entries[lose]
p = cards[ci]["payload"]
if s in (p.get("raw") or {}).get(bt, []):
p["raw"][bt].remove(s)
(p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
sc = (p.get("sidecar") or {}).get(bt)
if isinstance(sc, list): # questions/artefacts consume the sidecar downstream
p["sidecar"][bt] = [e for e in sc
if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
gone.add(lose)
touched.add(ci)
journal["gefaltet"].append({"weg": f"{bt} · {s}",
"bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
elif outs:
_log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
return final
chunk_finals = await asyncio.gather(*[_urteile_chunk(c) for c in chunks])
if ctx.is_cancelled():
return
final_all: dict[int, str] = {} # global pair index (1-based over `pairs`) → verdict
for cnr, fin in enumerate(chunk_finals):
for k, v in fin.items():
final_all[cnr * CROSS_CHUNK_PAARE + k] = v
journal = {"paare": len(pairs), "chunks": len(chunks), "gefaltet": [], "verdicts": []}
gone: set[int] = set()
touched: set[int] = set()
for k, (i, j) in enumerate(pairs, 1):
verdict = final_all.get(k, "nein")
journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
"b": f"{entries[j][1]} · {entries[j][2]}",
"verdict": verdict})
if verdict not in ("a", "b"):
continue
lose = j if verdict == "a" else i
if entries[lose][0] < 0: # context never folds — the package side goes instead
lose = i if lose == j else j
keep = i if lose == j else j
if lose in gone or keep in gone: # keeper already folded → don't chain away the content
continue
ci, bt, s = entries[lose]
p = cards[ci]["payload"]
if s in (p.get("raw") or {}).get(bt, []):
p["raw"][bt].remove(s)
(p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
sc = (p.get("sidecar") or {}).get(bt)
if isinstance(sc, list): # questions/artefacts consume the sidecar downstream
p["sidecar"][bt] = [e for e in sc
if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
gone.add(lose)
touched.add(ci)
journal["gefaltet"].append({"weg": f"{bt} · {s}",
"bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
for ci in touched:
p = cards[ci]["payload"]
p["raw"] = {bt: subs for bt, subs in (p.get("raw") or {}).items() if subs}
await db.kanban_set_payload(topic, BOARD, cards[ci]["card_id"], p)
if journal["gefaltet"]:
_log(topic, f"Sub-Crossblock: {len(journal['gefaltet'])} blockübergreifende Dublette(n) gefaltet")
atomic_write_json(work_dir / f"sub-crossblock-{h}.json", journal, indent=1)
hg = hashlib.md5("\n".join(f"{i}:{j}" for i, j in pairs).encode()).hexdigest()[:8]
atomic_write_json(work_dir / f"sub-crossblock-{hg}.json", journal, indent=1)
await _advance_all()
@@ -433,7 +452,10 @@ async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: s
for btitle, subs in sidecar.items(): # merge facts into the subs (mirror + guide)
fm = facts_map.get(btitle, {})
for sub in subs:
if (fk := fm.get(_norm_title(sub["title"]))):
# level agents paraphrase titles — exact miss falls back to the unique
# prefix/containment match, else the sub silently loses its grounding
sn = _sub_key(set(fm), _norm_title(sub["title"]))
if (fk := fm.get(sn)):
sub["facts"] = fk
p["sidecar"] = sidecar
await db.kanban_set_payload(topic, BOARD, norm, p)
@@ -535,12 +557,15 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
# DB mirrors — per block only (no global deletes)
await blocks._mirror_sidecar_db(topic, sidecar)
# stale question/artefact rows of a PREVIOUS run keyed to gone subs: finalize only
# upserts, so re-runs left orphans (measured: 28). subblocks rows stay — QA needs
# the variant/discarded statuses, and the sidecar mirror re-writes only consensus.
# upserts, so re-runs left orphans (measured: 28).
await db.delete_question_pattern(topic, _norm_title(title))
await db.delete_sub_artefakte(topic, _norm_title(title))
# stragglers the mirror didn't level (row not in this run's sidecar): without a
# valid level they vanish from guide/practice/level views while QA still counts them
# consensus rows of a PREVIOUS run that this run's sidecar no longer carries would
# linger without facts/questions/artefacts (measured: 25) — drop them per block;
# variant/discarded stay for QA. Then default-level the mirror's own stragglers.
for btitle, subs in sidecar.items():
keep = {_norm_title(str(s.get("title", ""))) for s in subs if isinstance(s, dict)}
await db.delete_stale_consensus(topic, _norm_title(btitle), keep - {""})
await db.default_subblock_levels(topic, _norm_title(title))
sub_keys: dict[str, set[str]] = {}