update

backend/board_artefacts.py

@@ -10,20 +10,24 @@ The heavy lifting is the existing per-block functions in blocks.py — each card
 work subdirectory + facts/artefakte paths, so their slot files never collide across blocks."""
 
 import asyncio
+import hashlib
 import json
 import logging
 import re
 
 import database as db
 import blocks
+import embedding
 from blocks import (
-    ARTEFACT_TYPES, _artefacts_block, _facts_block, _levels_block, _match_sub,
-    _question_pattern_block, _relevance_block, _subblocks_block, _outline_block,
+    ARTEFACT_TYPES, _artefacts_block, _facts_block, _konsolidiere_subblocks, _levels_block,
+    _luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
+    _sink_json, _subblocks_block, _outline_block,
 )
+from config import EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
 from fsutil import atomic_write_json
 from jsonio import read_json_file as _json_file
 from kanban import Flow, Stage
-from pipeline import GenContext, _log
+from pipeline import FAILED, GenContext, _extra, _log, _prompt, _timeout, run_single_slot
 from textkit import _norm_title, _title
 
 log = logging.getLogger("creator.board_artefacts")
@@ -40,6 +44,23 @@ def _safe(norm: str) -> str:
     return re.sub(r"\W+", "-", norm).strip("-")[:24] or "block"
 
 
+def _sub_key(existing: set[str], sn: str) -> str:
+    """Agents echo the short sub title while the sub row is keyed 'kurztitel: beschreibung' —
+    resolve to the stored key: exact, unambiguous prefix, then unambiguous substring
+    containment either way (agents paraphrase/truncate, measured 23 orphans of ~560 rows).
+    Ambiguous or unresolvable echoes stay unchanged (visible as QA orphan)."""
+    if sn in existing:
+        return sn
+    hits = [s for s in existing if s.startswith(sn + ":")]
+    if len(hits) == 1:
+        return hits[0]
+    if not hits:
+        hits = [s for s in sorted(existing) if sn in s or s in sn]
+        if len(hits) == 1:
+            return hits[0]
+    return sn
+
+
 def _card_set_p(flow: Flow, norm: str):
     """Per-card progress: the inner step messages land in-memory on the flow —
     board_snapshot shows them as the card's info line + phase stepper while active.
@@ -162,7 +183,9 @@ async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions
         sd = [s for s in seeds.get(norm, []) if s]
         if sd:
             instr = (instructions + "\n\nBereits identifizierte Unterpunkt-Kandidaten dieses "
-                     "Blocks (unbedingt prüfen und, wenn belegt, aufnehmen):\n"
+                     "Blocks (prüfen; wenn belegt UND noch nicht durch einen anderen Eintrag "
+                     "abgedeckt, aufnehmen — nicht wörtlich übernehmen, sondern als eigenständige "
+                     "Aussage formulieren):\n"
                      + "\n".join(f"- {s}" for s in sd))
         raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
                                      {1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-",
@@ -197,6 +220,29 @@ async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
                 if bt in raw:
                     raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns]
             raw = {bt: subs for bt, subs in raw.items() if subs}
+        # In-block consolidation: two-judge panel folds same-statement/subset subs, bundles
+        # catalogs, drops off-topic ones — the facts are in hand (key points as evidence),
+        # questions/artefacts not yet built. Reported gaps get ONE follow-up finder round.
+        luecken = await _konsolidiere_subblocks(ctx, _pfiles(files, norm), raw, facts_map,
+                                                instructions, ns=f"{_safe(norm)}-",
+                                                lbl=f"{p.get('title', norm)} · ")
+        if ctx.is_cancelled():
+            return None
+        nachgefasst = 0
+        for bt, lk in (luecken or {}).items():
+            nachgefasst += await _luecken_runde(ctx, _pfiles(files, norm), bt, lk, raw, facts_map,
+                                                q, folder, instructions, ns=f"{_safe(norm)}-",
+                                                lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
+        if ctx.is_cancelled():
+            return None
+        if nachgefasst:  # close the loop: follow-up finds get the SAME duplicate test as the
+            # rest (new subs-hash → fresh judge files); their gap report is deliberately ignored
+            await _konsolidiere_subblocks(ctx, _pfiles(files, norm), raw, facts_map,
+                                          instructions, ns=f"{_safe(norm)}-",
+                                          lbl=f"{p.get('title', norm)} · ")
+        if ctx.is_cancelled():
+            return None
+        raw = {bt: subs for bt, subs in raw.items() if subs}
         p["raw"], p["facts"] = raw, facts_map
         await db.kanban_set_payload(topic, BOARD, norm, p)
         await db.kanban_advance(topic, BOARD, norm, "levels")
@@ -204,6 +250,174 @@ async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
     await _gather_cards(ctx, flow, cards, one)
 
 
+def _cross_schema(data) -> dict[int, str] | None:
+    """{"pairs": {"1": "a"|"b"|"nein"}} → {pair_nr: verdict} · otherwise None."""
+    if not isinstance(data, dict) or not isinstance(data.get("pairs"), dict):
+        return None
+    out: dict[int, str] = {}
+    for k, v in data["pairs"].items():
+        try:
+            nr = int(k)
+        except (ValueError, TypeError):
+            continue
+        s = str(v).strip().casefold()
+        if s in ("a", "b", "nein"):
+            out[nr] = s
+    return out or None
+
+
+async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
+    """BARRIER/drain — cross-block sub dedup: the SAME statement carried by two blocks
+    (measured on Markdown: tab handling in 3 blocks, HTML blocks, backslash escapes — the
+    in-block paths never see these). Embedding candidates (block≠block, cos ≥
+    SUB_DUP_KANDIDAT_COS) go to a two-judge panel; UNANIMITY decides which block keeps the
+    statement. The loser leaves its card's raw/facts and turns DB `variant` — before
+    questions/artefacts exist, so no orphans. Fail-open on judge failure/dissent."""
+    topic = flow.topic
+    work_dir = flow.work_dir
+    package_norms = {c["card_id"] for c in cards}
+    entries: list[tuple[int, str, str]] = []  # (card idx, block title, sub title); idx -1 = context
+    for ci, c in enumerate(cards):
+        for bt, subs in (c["payload"].get("raw") or {}).items():
+            for s in subs:
+                entries.append((ci, bt, s))
+    n_pkg = len(entries)
+    # Context: consensus subs of blocks already PAST this barrier (late spawns via the
+    # gap-check feedback would otherwise never be compared). Context never folds —
+    # its card payload lives downstream (board-1 rule: confirmed context always wins).
+    for r in await db.list_subblocks(topic):
+        if r["status"] == "consensus" and r["block_norm"] not in package_norms:
+            entries.append((-1, r["block"], r["sub_title"]))
+    ctx_facts: dict[str, dict] = {}  # facts of downstream cards (DB rows carry none yet)
+    for bc in await db.kanban_cards(topic, board=BOARD, kind="ablock"):
+        if bc["card_id"] not in package_norms:
+            for bt, fm in (bc["payload"].get("facts") or {}).items():
+                ctx_facts[_norm_title(bt)] = fm
+
+    async def _advance_all():
+        await db.kanban_advance_many(topic, BOARD, [(c["card_id"], "question_pattern") for c in cards])
+        flow.wake.set()
+
+    if n_pkg < 1 or len(entries) < 2 or not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
+        await _advance_all()
+        return
+    sims = await asyncio.to_thread(embedding.embed_sims, [s for _, _, s in entries])
+    if sims is None:
+        await _advance_all()
+        return
+    negs = [_neg_set(s) for _, _, s in entries]
+    pairs = [(i, j) for i in range(len(entries)) for j in range(i + 1, len(entries))
+             if entries[i][0] != entries[j][0] and negs[i] == negs[j]
+             and float(sims[i][j]) >= SUB_DUP_KANDIDAT_COS]
+    if not pairs:
+        await _advance_all()
+        return
+
+    def _kp(ci: int, bt: str, s: str) -> list:
+        if ci < 0:
+            f = ctx_facts.get(_norm_title(bt)) or {}
+        else:
+            f = (cards[ci]["payload"].get("facts") or {}).get(bt) or {}
+        return (f.get(_norm_title(s)) or {}).get("key_points") or []
+
+    def _side(tag: str, ci: int, bt: str, s: str) -> str:
+        return f"{tag}: [Block: {bt}] {s}" + "".join(f"\n   - {p}" for p in _kp(ci, bt, s))
+
+    lines = "\n\n".join(
+        f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
+        for k, (i, j) in enumerate(pairs, 1))
+    h = hashlib.md5(lines.encode()).hexdigest()[:8]
+    paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
+
+    async def _judge(j, path):
+        if _cross_schema(_json_file(path)) is not None:
+            return  # resume
+        status, _v = await run_single_slot(
+            ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
+            prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=lines, extra=_extra(instructions)),
+            role="judge", capabilities="none",
+            payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
+            timeout=_timeout("subblock_check", len(pairs)))
+        if status == FAILED:
+            _log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")
+
+    await asyncio.gather(*[_judge(j, p) for j, p in zip((1, 2), paths)])
+    if ctx.is_cancelled():
+        return
+    outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
+    if len(outs) == 1:  # Ersatz-Richter statt fail-open bei EINEM Ausfall
+        ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
+        await _judge("E", ersatz)
+        if ctx.is_cancelled():
+            return
+        outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
+    journal = {"paare": len(pairs), "richter": len(outs), "gefaltet": [], "verdicts": []}
+    gone: set[int] = set()
+    touched: set[int] = set()
+    if len(outs) == 2:
+        final = {k: (outs[0].get(k, "nein") if outs[0].get(k, "nein") == outs[1].get(k, "nein")
+                     else "uneinig") for k in range(1, len(pairs) + 1)}
+        disputed = [k for k, v in final.items() if v == "uneinig"]
+        if disputed:  # tie-breaker: a third judge sees ONLY the disputed pairs, majority 2/3
+            d_lines = "\n\n".join(
+                f"{x}.\n{_side('A', *entries[pairs[k - 1][0]])}\n{_side('B', *entries[pairs[k - 1][1]])}"
+                for x, k in enumerate(disputed, 1))
+            p3 = work_dir / f"sub-crossblock-{h}-j3.json"
+            if _cross_schema(_json_file(p3)) is None:
+                status, _v = await run_single_slot(
+                    ctx, "Sub-Crossblock j3", key=f"blocks-{topic}-sub-crossblock-{h}-j3",
+                    prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=d_lines, extra=_extra(instructions)),
+                    role="judge", capabilities="none",
+                    payload=lambda result, p=p3: _sink_json(result, p, _cross_schema),
+                    timeout=_timeout("subblock_check", len(disputed)))
+                if status == FAILED:
+                    _log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
+            if ctx.is_cancelled():
+                return
+            v3 = _cross_schema(_json_file(p3)) or {}
+            for x, k in enumerate(disputed, 1):
+                t = v3.get(x, "nein")
+                if t in (outs[0].get(k, "nein"), outs[1].get(k, "nein")):
+                    final[k] = t  # majority 2/3; anything else stays disputed → no fold
+        for k, (i, j) in enumerate(pairs, 1):
+            verdict = final[k]
+            journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
+                                        "b": f"{entries[j][1]} · {entries[j][2]}",
+                                        "verdict": verdict})
+            if verdict not in ("a", "b"):
+                continue
+            lose = j if verdict == "a" else i
+            if entries[lose][0] < 0:  # context never folds — the package side goes instead
+                lose = i if lose == j else j
+            keep = i if lose == j else j
+            if lose in gone or keep in gone:  # keeper already folded → don't chain away the content
+                continue
+            ci, bt, s = entries[lose]
+            p = cards[ci]["payload"]
+            if s in (p.get("raw") or {}).get(bt, []):
+                p["raw"][bt].remove(s)
+                (p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
+                sc = (p.get("sidecar") or {}).get(bt)
+                if isinstance(sc, list):  # questions/artefacts consume the sidecar downstream
+                    p["sidecar"][bt] = [e for e in sc
+                                        if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
+                await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
+                gone.add(lose)
+                touched.add(ci)
+                journal["gefaltet"].append({"weg": f"{bt} · {s}",
+                                            "bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
+    elif outs:
+        _log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
+    for ci in touched:
+        p = cards[ci]["payload"]
+        p["raw"] = {bt: subs for bt, subs in (p.get("raw") or {}).items() if subs}
+        await db.kanban_set_payload(topic, BOARD, cards[ci]["card_id"], p)
+    if journal["gefaltet"]:
+        _log(topic, f"Sub-Crossblock: {len(journal['gefaltet'])} blockübergreifende Dublette(n) gefaltet")
+    atomic_write_json(work_dir / f"sub-crossblock-{h}.json", journal, indent=1)
+    await _advance_all()
+
+
 async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
     topic = flow.topic
 
@@ -247,7 +461,7 @@ async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions
                 sub["relevance"] = rel.get(gid, "relevant")
         p["sidecar"] = sidecar
         await db.kanban_set_payload(topic, BOARD, norm, p)
-        await db.kanban_advance(topic, BOARD, norm, "question_pattern")
+        await db.kanban_advance(topic, BOARD, norm, "konsolidierung")
 
     await _gather_cards(ctx, flow, cards, one)
 
@@ -320,6 +534,18 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
         atomic_write_json(files["artefakte"], art_global, indent=1)
         # DB mirrors — per block only (no global deletes)
         await blocks._mirror_sidecar_db(topic, sidecar)
+        # stale question/artefact rows of a PREVIOUS run keyed to gone subs: finalize only
+        # upserts, so re-runs left orphans (measured: 28). subblocks rows stay — QA needs
+        # the variant/discarded statuses, and the sidecar mirror re-writes only consensus.
+        await db.delete_question_pattern(topic, _norm_title(title))
+        await db.delete_sub_artefakte(topic, _norm_title(title))
+        sub_keys: dict[str, set[str]] = {}
+
+        async def _keys(bnorm: str) -> set[str]:
+            if bnorm not in sub_keys:
+                sub_keys[bnorm] = {r["sub_norm"] for r in await db.list_subblocks(topic, bnorm)}
+            return sub_keys[bnorm]
+
         for btitle, entries in pattern.items():
             bnorm = _norm_title(btitle)
             for e in entries if isinstance(entries, list) else []:
@@ -327,6 +553,7 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
                 sn = _norm_title(sub)
                 question = str(e.get("question", "")).strip()
                 if bnorm and sn and question:
+                    sn = _sub_key(await _keys(bnorm), sn)
                     await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question)
         btitles = list(sidecar.keys())
         for typ in ARTEFACT_TYPES:
@@ -335,6 +562,7 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
                 bnorm, sn = _norm_title(bt), _norm_title(str(e.get("subblock", "")))
                 if not bnorm or not sn:
                     continue
+                sn = _sub_key(await _keys(bnorm), sn)
                 data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")},
                                   ensure_ascii=False)
                 await db.put_sub_artifact(topic, bnorm, sn, typ, data, bt, str(e.get("subblock", "")))
@@ -391,6 +619,11 @@ def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
         Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)),
         Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)),
         Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)),
+        # Barrier sits AFTER the sub-local stages: cards used to idle here median 36 min
+        # while levels/relevance work was still ahead of them
+        Stage(BOARD, "konsolidierung",
+              lambda cs: _proc_konsolidierung(ctx, flow, files, instructions, cs),
+              barrier=True, drain=True),
         Stage(BOARD, "question_pattern",
               lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)),
         Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)),