667 lines
33 KiB
Python
667 lines
33 KiB
Python
"""Board 2, verschmolzene Call-Struktur: 3 Bausteine pro Block statt ~20 serieller Segmente.
|
||
|
||
Die alte Stage-Treppe (Finder-Runden → Facts find/erg/check → Konsolidierung → Lücken →
|
||
Nachfass → Levels → Relevanz → Fragen → Kritik → Flashcards → Beispiele → Check) kostete
|
||
pro Block 35–55 Calls und ~14 min Wandzeit — bei p50 20–50 s pro Call zählt NUR die Zahl
|
||
der seriellen Segmente. Hier: Generate(∥2) → Verify(∥2, + Fix-Tail) → Artefakte(Gen+Check)
|
||
= 4–5 Segmente, 6–9 Calls. Unabhängigkeit bleibt: Generatoren und Prüfer sind getrennte
|
||
Agenten, Konsens (≥2 unabhängige Nennungen) und Einstimmigkeits-Faltung wie zuvor.
|
||
|
||
Output-Kontrakt unverändert (finalize/QA/Guide/Übungssystem lesen dieselben Strukturen):
|
||
raw {block: [sub]}, facts {block: {sub_norm: 5-Felder}}, sidecar {block: [{title, level,
|
||
relevance, facts}]}, pattern {block: [{subblock, question}]}, artefacts {flashcard/example}."""
|
||
|
||
import asyncio
|
||
import hashlib
|
||
import logging
|
||
|
||
import database as db
|
||
import embedding
|
||
from blocks import (
|
||
_SOURCE_TEMPLATE, _FACTS_FIELDS, _agreed_cliques, _cited_evidence, _dedup_subblocks,
|
||
_evidence_pack, _facts_lines, _facts_union, _luecken_schnitt, _neg_set, _pairs_of,
|
||
_sink_json, _sub_tokens, _subs_hash, _variant_clusters, load_source, material_folder,
|
||
source_folder,
|
||
)
|
||
from config import (ART_SPLIT_SUBS, EMBEDDING_AKTIV, GEN_PANEL, SEED_COVER_COS,
|
||
VERIFY_PANEL)
|
||
from jsonio import read_json_file as _json_file
|
||
from pipeline import FAILED, GenContext, _extra, _log, _prompt, _race, _timeout, run_single_slot
|
||
from textkit import _norm_title, clean_title
|
||
|
||
log = logging.getLogger("creator.block_calls")
|
||
|
||
_STUFEN = ("beginner", "advanced", "expert")
|
||
_RELEVANZ = ("relevant", "peripheral")
|
||
|
||
|
||
def _h8(*parts: str) -> str:
|
||
return hashlib.md5("|".join(parts).encode("utf-8")).hexdigest()[:8]
|
||
|
||
|
||
# ── Schemas ─────────────────────────────────────────────────────────────────────────
|
||
|
||
def _gen_schema(data) -> list[dict] | None:
|
||
"""{"subs": [{title, level, relevance, …facts}]} → normalisierte Liste · sonst None.
|
||
Feld-Normalisierung wie _facts_schema; ungültiges level/relevance fällt auf ""
|
||
(die Stimme entfällt im Vote, der Sub bleibt)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("subs"), list):
|
||
return None
|
||
out = []
|
||
for e in data["subs"]:
|
||
if not isinstance(e, dict) or not str(e.get("title", "")).strip():
|
||
continue
|
||
bf = [{"text": t, "source": str(f.get("source", "")).strip()}
|
||
for f in (e.get("cited_facts") or []) if isinstance(f, dict) and (t := str(f.get("text", "")).strip())]
|
||
lv = str(e.get("level", "")).strip().casefold()
|
||
rv = str(e.get("relevance", "")).strip().casefold()
|
||
out.append({
|
||
"title": clean_title(str(e["title"]).strip()),
|
||
"level": lv if lv in _STUFEN else "",
|
||
"relevance": rv if rv in _RELEVANZ else "",
|
||
"key_points": [k for x in (e.get("key_points") or []) if (k := str(x).strip())],
|
||
"prerequisites": str(e.get("prerequisites", "")).strip(),
|
||
"hurdles": str(e.get("hurdles", "")).strip(),
|
||
"cited_facts": bf,
|
||
"example_idea": str(e.get("example_idea", "")).strip(),
|
||
})
|
||
return out or None
|
||
|
||
|
||
def _vid(x, n: int) -> int | None:
|
||
"""Prüfer-Nummer → int in 1..n, else None (bools sind keine ids)."""
|
||
if isinstance(x, bool):
|
||
return None
|
||
if isinstance(x, str) and x.isdigit():
|
||
x = int(x)
|
||
return x if isinstance(x, int) and 1 <= x <= n else None
|
||
|
||
|
||
def _verify_schema(data, n: int) -> dict | None:
|
||
"""Prüfer-Output → normalisiertes Verdikt · None wenn kaputt. Alle Felder optional
|
||
außer der Grundform (dict) — ein leeres Verdikt {"gruppen": []} heißt „alles ok"."""
|
||
if not isinstance(data, dict):
|
||
return None
|
||
pflicht = ("gruppen", "kataloge", "fremd", "luecken", "uebernehmen", "facts_probleme",
|
||
"levels", "relevanz")
|
||
if not any(k in data for k in pflicht):
|
||
return None
|
||
|
||
def _ids(lst):
|
||
return sorted({i for x in (lst or []) if (i := _vid(x, n)) is not None})
|
||
|
||
gruppen = []
|
||
for g in data.get("gruppen") or []:
|
||
if not isinstance(g, dict):
|
||
continue
|
||
haupt = _vid(g.get("haupt"), n)
|
||
ids = _ids(([haupt] if haupt else []) + list(g.get("weitere") or []))
|
||
if len(ids) >= 2:
|
||
gruppen.append({"haupt": haupt if haupt in ids else None, "ids": ids})
|
||
kataloge = []
|
||
for k in data.get("kataloge") or []:
|
||
if not isinstance(k, dict):
|
||
continue
|
||
ids = _ids(k.get("mitglieder"))
|
||
titel = str(k.get("titel") or "").strip()
|
||
if len(ids) >= 2 and titel:
|
||
kataloge.append({"titel": titel, "ids": ids})
|
||
uebernehmen = {}
|
||
for k, v in (data.get("uebernehmen") or {}).items() if isinstance(data.get("uebernehmen"), dict) else []:
|
||
if (i := _vid(k, n)) is not None:
|
||
uebernehmen[i] = str(v).strip().casefold()
|
||
probleme = []
|
||
for p in data.get("facts_probleme") or []:
|
||
if isinstance(p, dict) and (i := _vid(p.get("nr"), n)) is not None:
|
||
probleme.append({"nr": i, "discard": bool(p.get("discard")),
|
||
"hinweis": str(p.get("hinweis", "")).strip()})
|
||
def _enum_map(key, allowed):
|
||
out = {}
|
||
raw = data.get(key)
|
||
for k, v in (raw.items() if isinstance(raw, dict) else []):
|
||
if (i := _vid(k, n)) is not None and str(v).strip().casefold() in allowed:
|
||
out[i] = str(v).strip().casefold()
|
||
return out
|
||
return {"gruppen": gruppen, "kataloge": kataloge, "fremd": set(_ids(data.get("fremd"))),
|
||
"luecken": [s.strip() for s in data.get("luecken") or [] if isinstance(s, str) and s.strip()],
|
||
"uebernehmen": uebernehmen, "facts_probleme": probleme,
|
||
"levels": _enum_map("levels", _STUFEN), "relevanz": _enum_map("relevanz", _RELEVANZ)}
|
||
|
||
|
||
def _pattern_liste(lst) -> list[dict]:
|
||
out = []
|
||
for e in lst or []:
|
||
if isinstance(e, dict):
|
||
blk, sub, q = (str(e.get(k, "")).strip() for k in ("block", "subblock", "question"))
|
||
if blk and sub and q:
|
||
out.append({"block": blk, "subblock": sub, "question": q})
|
||
return out
|
||
|
||
|
||
def _art_gen_schema(data) -> dict | None:
|
||
"""{"pattern": […], "cards": […], "examples": […]} → normalisiert · None wenn kaputt.
|
||
pattern ist Pflicht (Leitner hängt an Fragen), cards/examples best-effort."""
|
||
if not isinstance(data, dict):
|
||
return None
|
||
pattern = _pattern_liste(data.get("pattern"))
|
||
if not pattern:
|
||
return None
|
||
cards = []
|
||
for e in data.get("cards") or []:
|
||
if isinstance(e, dict):
|
||
blk, sub, q, a = (str(e.get(k, "")).strip() for k in ("block", "subblock", "question", "answer"))
|
||
if blk and sub and q and a:
|
||
cards.append({"block": blk, "subblock": sub, "question": q, "answer": a})
|
||
examples = []
|
||
for e in data.get("examples") or []:
|
||
if isinstance(e, dict):
|
||
blk, sub, pr, res = (str(e.get(k, "")).strip() for k in ("block", "subblock", "problem", "result"))
|
||
steps = [s for x in (e.get("steps") or []) if (s := str(x).strip())]
|
||
if blk and sub and pr and steps:
|
||
examples.append({"block": blk, "subblock": sub, "problem": pr, "steps": steps, "result": res})
|
||
return {"pattern": pattern, "cards": cards, "examples": examples}
|
||
|
||
|
||
def _art_check_schema(data) -> dict | None:
|
||
"""{"ok": true} → leeres Verdikt · sonst pattern (bereinigt) + pattern_ergaenzt +
|
||
examples_probleme (1-basierte Indizes)."""
|
||
if not isinstance(data, dict):
|
||
return None
|
||
if data.get("ok") is True:
|
||
return {"pattern": [], "pattern_ergaenzt": [], "examples_probleme": set()}
|
||
if not any(k in data for k in ("pattern", "pattern_ergaenzt", "examples_probleme")):
|
||
return None
|
||
probleme = set()
|
||
for p in data.get("examples_probleme") or []:
|
||
i = p.get("index") if isinstance(p, dict) else p
|
||
if isinstance(i, str) and i.isdigit():
|
||
i = int(i)
|
||
if isinstance(i, int) and not isinstance(i, bool) and i >= 1:
|
||
probleme.add(i)
|
||
return {"pattern": _pattern_liste(data.get("pattern")),
|
||
"pattern_ergaenzt": _pattern_liste(data.get("pattern_ergaenzt")),
|
||
"examples_probleme": probleme}
|
||
|
||
|
||
# ── Gemeinsames ─────────────────────────────────────────────────────────────────────
|
||
|
||
def _inline_source(topic: str, sources: list[str] | None, queries: list[str]) -> tuple[str, str]:
|
||
"""→ (source-Slot, capabilities). Korpus-Auszüge inline (uni/projekt/link oder
|
||
thema-Research-Material); ohne Treffer fail-open auf die alte Selbst-Recherche."""
|
||
mat = material_folder(topic)
|
||
ev = _evidence_pack(mat, sources, queries) if mat else ""
|
||
if ev:
|
||
return _prompt("Blocks-Source-Inline", excerpts=ev), "none"
|
||
_type = load_source(topic).get("type", "thema")
|
||
folder = source_folder(topic)
|
||
if _type in _SOURCE_TEMPLATE:
|
||
return _prompt(_SOURCE_TEMPLATE[_type], project=folder), ("files" if folder else "full")
|
||
return _prompt("Blocks-Source-Thema", topic=topic), "full"
|
||
|
||
|
||
async def _sims_of(titles: list[str]):
|
||
"""Ähnlichkeitsmatrix fürs Variant-Clustering; ohne Modell exakte Norm-Gleichheit."""
|
||
if EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available):
|
||
sims = await asyncio.to_thread(embedding.embed_sims, titles)
|
||
if sims is not None:
|
||
return sims
|
||
norms = [_norm_title(t) for t in titles]
|
||
return [[1.0 if norms[i] == norms[j] else 0.0 for j in range(len(titles))]
|
||
for i in range(len(titles))]
|
||
|
||
|
||
def _fk_of(e: dict) -> dict:
|
||
return {k: e.get(k) for k in _FACTS_FIELDS}
|
||
|
||
|
||
# ── Generate ────────────────────────────────────────────────────────────────────────
|
||
|
||
async def _generate_block(ctx: GenContext, files: dict, title: str, description: str,
|
||
instructions: str = "", ns: str = "", lbl: str = "",
|
||
sources: list[str] | None = None,
|
||
seeds: list[str] | None = None, melde=None) -> dict | None:
|
||
"""Bottom-up-Anreicherung: die Subs stehen FEST — Board 1 hat die Atome geclustert und
|
||
als Subs persistiert. EIN Call liefert pro vorgegebenem Sub die Lern-Facts + Level +
|
||
Relevanz aus dem Material. Er entdeckt und entfernt KEINE Subs (Discovery/Konsens/Seeds
|
||
entfallen; die Vollständigkeit kommt aus Board 1). → {raw, facts, unsicher, votes} | None."""
|
||
topic = ctx.topic
|
||
work_dir = files["arbeit"]
|
||
bnorm = _norm_title(title)
|
||
# feste Subs aus Board 1 (Reihenfolge erhalten, dublettenfrei)
|
||
subs: list[str] = []
|
||
seen: set[str] = set()
|
||
for r in await db.list_subblocks(topic, bnorm):
|
||
st = str(r.get("sub_title") or "").strip()
|
||
sn = _norm_title(st)
|
||
if st and sn and sn not in seen and r.get("status") in ("consensus", "candidate"):
|
||
seen.add(sn)
|
||
subs.append(st)
|
||
if not subs:
|
||
return {"raw": {title: []}, "facts": {title: {}}, "unsicher": [], "votes": {}}
|
||
source, caps = await asyncio.to_thread(_inline_source, topic, sources, [title] + subs)
|
||
if melde:
|
||
melde("Anreichern")
|
||
h = _h8(title, description, "enrich")
|
||
pfad = work_dir / f"enrich-{h}.json"
|
||
sub_liste = "\n".join(f"{i}. {t}" for i, t in enumerate(subs, 1))
|
||
prompt = _prompt("Subblock-Anreichern", topic=topic,
|
||
block=f"{title} — {description}" if description else title,
|
||
subs=sub_liste, source=source, extra=_extra(instructions))
|
||
if _gen_schema(_json_file(pfad)) is None:
|
||
status, _v = await run_single_slot(
|
||
ctx, f"{lbl}Anreichern", key=f"blocks-{topic}-{ns}sb-enrich-{h}",
|
||
prompt=prompt, role="quick", capabilities=caps,
|
||
payload=lambda result, p=pfad: _sink_json(result, p, _gen_schema),
|
||
timeout=_timeout("generate", len(subs)))
|
||
if status == FAILED:
|
||
_log(topic, f"Anreichern {title} ohne Ergebnis — Facts bleiben leer, Prüfer misst")
|
||
if ctx.is_cancelled():
|
||
return None
|
||
out = _gen_schema(_json_file(pfad)) or []
|
||
by_norm = {_norm_title(e["title"]): e for e in out}
|
||
facts: dict[str, dict] = {}
|
||
votes: dict[str, dict] = {}
|
||
for st in subs:
|
||
sn = _norm_title(st)
|
||
e = by_norm.get(sn)
|
||
if e:
|
||
fk = _fk_of(e)
|
||
votes[sn] = {"level": [e["level"]] if e.get("level") else [],
|
||
"relevance": [e["relevance"]] if e.get("relevance") else []}
|
||
else: # der Call ließ diesen Sub aus — leere Facts, der Prüfer/QA sieht die Lücke
|
||
fk = {k: ([] if k in ("key_points", "cited_facts") else "") for k in _FACTS_FIELDS}
|
||
votes[sn] = {"level": [], "relevance": []}
|
||
facts[sn] = fk
|
||
await db.put_subblock(topic, bnorm, sn, title, st, status="consensus")
|
||
return {"raw": {title: subs}, "facts": {title: facts}, "unsicher": [], "votes": votes}
|
||
|
||
|
||
# ── Verify (+ Fix-Tail) ─────────────────────────────────────────────────────────────
|
||
|
||
def _default_vote(stimmen: list[str], default: str) -> str:
|
||
"""Mehrheit über die Stimmen (Gen-Vorschläge + implizite/explizite Prüfer-Stimmen);
|
||
Patt oder leer → default."""
|
||
counter: dict[str, int] = {}
|
||
for s in stimmen:
|
||
if s:
|
||
counter[s] = counter.get(s, 0) + 1
|
||
best = max(counter.values(), default=0)
|
||
winners = [s for s, v in counter.items() if v == best]
|
||
return winners[0] if len(winners) == 1 and best else default
|
||
|
||
|
||
async def _verify_block(ctx: GenContext, files: dict, title: str, gen: dict, q: dict,
|
||
instructions: str = "", ns: str = "", lbl: str = "",
|
||
sources: list[str] | None = None, melde=None,
|
||
keep_all: bool = False) -> dict | None:
|
||
"""VERIFY_PANEL unabhängige Prüfer auditieren den Block in EINEM Call (MECE-Faltung,
|
||
Fremd, Lücken, Unsicher-Übernahme, Facts-Korrektheit, Level/Relevanz). Auswertung mit
|
||
Schnittmengen-Semantik pro Befundklasse (Faltung/Fremd/Übernahme einstimmig, Discard
|
||
2/2, Korrektur ≥1 Stimme); Fix-Tail ist EIN Call für Korrekturen + belegte Lücken.
|
||
keep_all=True (Bottom-up): kein Atom wird entfernt/gefaltet/als neu erfunden — nur
|
||
Facts-Korrektur + Level/Relevanz bleiben (Board 1 hat schon MECE geclustert).
|
||
→ {raw, facts, sidecar} | None (nur bei Cancel)."""
|
||
topic = ctx.topic
|
||
work_dir = files["arbeit"]
|
||
bnorm = _norm_title(title)
|
||
subs = list(gen["raw"].get(title) or [])
|
||
bfacts: dict[str, dict] = dict(gen["facts"].get(title) or {})
|
||
unsicher: list[dict] = list(gen.get("unsicher") or [])
|
||
votes: dict[str, dict] = gen.get("votes") or {}
|
||
nummern = subs + [u["title"] for u in unsicher] # 1-basiert: consensus, dann unsicher
|
||
n = len(nummern)
|
||
|
||
def _kp(t: str) -> list:
|
||
fk = bfacts.get(_norm_title(t)) or next(
|
||
(u for u in unsicher if u["title"] == t), {})
|
||
return (fk.get("key_points") or [])[:3]
|
||
|
||
if melde:
|
||
melde("Verify")
|
||
verdicts: list[dict] = []
|
||
if n:
|
||
zeilen = "\n".join(f"{k}. {t}" + "".join(f"\n - {p}" for p in _kp(t))
|
||
for k, t in enumerate(nummern, 1))
|
||
u_txt = ""
|
||
if unsicher:
|
||
erste = len(subs) + 1
|
||
u_txt = (f"\nUNSICHER — entries {erste}–{n} were named by only ONE generator "
|
||
"(or are seed candidates). Judge their adoption under `uebernehmen`.\n")
|
||
cites = [bf.get("source", "") for fk in bfacts.values() for bf in fk.get("cited_facts", [])]
|
||
mat = material_folder(topic)
|
||
ev = _cited_evidence(mat, sources, cites, [title] + nummern) if mat else ""
|
||
if ev:
|
||
source = _prompt("Blocks-Source-Inline", excerpts=ev)
|
||
else:
|
||
source, _caps = await asyncio.to_thread(_inline_source, topic, sources, [title] + nummern)
|
||
sh = _subs_hash({title: nummern})
|
||
pfade = {j: work_dir / f"verify-{sh}-j{j}.json" for j in (*range(1, VERIFY_PANEL + 1), "E")}
|
||
|
||
async def _judge(j):
|
||
if _verify_schema(_json_file(pfade[j]), n) is not None:
|
||
return # resume
|
||
status, _v = await run_single_slot(
|
||
ctx, f"{lbl}Verify j{j}", key=f"blocks-{topic}-{ns}sb-verify-{sh}-j{j}",
|
||
prompt=_prompt("Subblock-Verify", topic=topic, block=title, subs=zeilen,
|
||
unsicher=u_txt, source=source, extra=_extra(instructions)),
|
||
role="judge", capabilities="none",
|
||
payload=lambda result, p=pfade[j]: _sink_json(result, p, lambda d: _verify_schema(d, n)),
|
||
timeout=_timeout("verify", n))
|
||
if status == FAILED:
|
||
_log(topic, f"Verify {title} j{j} ohne Ergebnis — fail-open")
|
||
|
||
await asyncio.gather(*[_judge(j) for j in range(1, VERIFY_PANEL + 1)])
|
||
if ctx.is_cancelled():
|
||
return None
|
||
verdicts = [v for j in range(1, VERIFY_PANEL + 1)
|
||
if (v := _verify_schema(_json_file(pfade[j]), n)) is not None]
|
||
if len(verdicts) == 1 and VERIFY_PANEL >= 2: # Ersatz-Richter statt fail-open
|
||
await _judge("E")
|
||
if ctx.is_cancelled():
|
||
return None
|
||
verdicts = [v for j in (*range(1, VERIFY_PANEL + 1), "E")
|
||
if (v := _verify_schema(_json_file(pfade[j]), n)) is not None][:2]
|
||
|
||
einstimmig = len(verdicts) >= 2
|
||
if n and not einstimmig:
|
||
_log(topic, f"Verify {title}: nur {len(verdicts)}/2 Prüfer — fail-open, unsicher verworfen")
|
||
negs = [_neg_set(t) for t in nummern]
|
||
gone: set[int] = set()
|
||
keep = list(subs)
|
||
korrekturen: list[dict] = [] # {titel, hinweis}
|
||
luecken: list[str] = []
|
||
|
||
def _titel(k: int) -> str:
|
||
return nummern[k - 1]
|
||
|
||
async def _fold(k: int, wf: dict | None):
|
||
t = _titel(k)
|
||
lf = bfacts.pop(_norm_title(t), None) or {}
|
||
if wf is not None:
|
||
_facts_union(wf, lf)
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(t), status="variant")
|
||
if t in keep:
|
||
keep.remove(t)
|
||
gone.add(k)
|
||
|
||
if einstimmig:
|
||
v1, v2 = verdicts[0], verdicts[1]
|
||
if keep_all:
|
||
# Bottom-up: jedes Board-1-Atom bleibt. Destruktive/erzeugende Befunde
|
||
# neutralisieren — die Loops unten laufen dann leer. Nur Facts-Korrektur
|
||
# (ohne discard) + Level/Relevanz überleben.
|
||
for v in (v1, v2):
|
||
v["fremd"] = set()
|
||
v["gruppen"] = []
|
||
v["kataloge"] = []
|
||
v["uebernehmen"] = {}
|
||
v["luecken"] = []
|
||
# discard abschalten (kein Atom entfernen), aber den Korrektur-Hinweis behalten
|
||
v["facts_probleme"] = [{**p, "discard": False} for p in v["facts_probleme"]]
|
||
# 1. Fremd (einstimmig): fürs THEMA fremde Aussagen → discarded
|
||
for k in sorted(v1["fremd"] & v2["fremd"]):
|
||
t = _titel(k)
|
||
bfacts.pop(_norm_title(t), None)
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(t), status="discarded")
|
||
if t in keep:
|
||
keep.remove(t)
|
||
gone.add(k)
|
||
# 2. Unsicher-Übernahme (2/2 „ja"): wird consensus samt Generator-Facts
|
||
for k in range(len(subs) + 1, n + 1):
|
||
if k in gone:
|
||
continue
|
||
u = unsicher[k - len(subs) - 1]
|
||
if v1["uebernehmen"].get(k) == "ja" and v2["uebernehmen"].get(k) == "ja":
|
||
sn = _norm_title(u["title"])
|
||
bfacts[sn] = _fk_of(u)
|
||
keep.append(u["title"])
|
||
await db.set_subblock_fields(topic, bnorm, sn, status="consensus")
|
||
else:
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(u["title"]), status="discarded")
|
||
gone.add(k)
|
||
# 3. Gruppen (einstimmige Paare): haupt-Votum, sonst key_points/Länge
|
||
haupt_votes: dict[int, int] = {}
|
||
for v in (v1, v2):
|
||
for g in v["gruppen"]:
|
||
if g["haupt"]:
|
||
haupt_votes[g["haupt"]] = haupt_votes.get(g["haupt"], 0) + 1
|
||
for g in _agreed_cliques([_pairs_of([x["ids"] for x in v["gruppen"]]) for v in (v1, v2)], negs, n):
|
||
g = [k for k in g if k not in gone]
|
||
if len(g) < 2:
|
||
continue
|
||
win = max(g, key=lambda k: (haupt_votes.get(k, 0), len(_kp(_titel(k))), len(_titel(k)), -k))
|
||
wf = bfacts.setdefault(_norm_title(_titel(win)), {})
|
||
for k in g:
|
||
if k != win:
|
||
await _fold(k, wf)
|
||
# 4. Kataloge: Aufzählungszeilen → EIN neuer benannter Sub (Facts-Union)
|
||
for g in _agreed_cliques([_pairs_of([x["ids"] for x in v["kataloge"]]) for v in (v1, v2)], negs, n):
|
||
g = [k for k in g if k not in gone]
|
||
if len(g) < 2:
|
||
continue
|
||
titel = next((clean_title(x["titel"]) for x in v1["kataloge"] + v2["kataloge"]
|
||
if set(x["ids"]) & set(g) and clean_title(x["titel"])), "")
|
||
kn = _norm_title(titel)
|
||
if not kn or kn in {_norm_title(s) for s in keep}:
|
||
continue
|
||
kf: dict = {}
|
||
for k in g:
|
||
await _fold(k, kf)
|
||
bfacts[kn] = kf
|
||
keep.append(titel)
|
||
await db.put_subblock(topic, bnorm, kn, title, titel, status="consensus")
|
||
# 5. Facts-Probleme: discard nur 2/2 (irreversibel), Korrektur ab 1 Stimme
|
||
d1 = {p["nr"] for p in v1["facts_probleme"] if p["discard"]}
|
||
d2 = {p["nr"] for p in v2["facts_probleme"] if p["discard"]}
|
||
for k in sorted(d1 & d2):
|
||
if k in gone:
|
||
continue
|
||
t = _titel(k)
|
||
bfacts.pop(_norm_title(t), None)
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(t), status="discarded")
|
||
if t in keep:
|
||
keep.remove(t)
|
||
gone.add(k)
|
||
for p in v1["facts_probleme"] + v2["facts_probleme"]:
|
||
k = p["nr"]
|
||
if k in gone or not p["hinweis"]:
|
||
continue
|
||
t = _titel(k)
|
||
if t in keep and all(x["titel"] != t for x in korrekturen):
|
||
korrekturen.append({"titel": t, "hinweis": p["hinweis"]})
|
||
# 6. Lücken (Schnitt beider Prüfer, Cap)
|
||
luecken = _luecken_schnitt(v1["luecken"], v2["luecken"])
|
||
else:
|
||
# fail-open: consensus bleibt, unsicher wird verworfen (wie heutiges Clarify-Aus)
|
||
for u in unsicher:
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(u["title"]), status="discarded")
|
||
|
||
# 7. Level/Relevanz: Stimmen = Generatoren + Prüfer (explizite Korrektur schlägt
|
||
# die implizite Zustimmung); Patt → advanced/relevant (heutige Defaults)
|
||
sidecar_subs = []
|
||
for t in keep:
|
||
sn = _norm_title(t)
|
||
try:
|
||
k = nummern.index(t) + 1
|
||
except ValueError:
|
||
k = 0 # Katalog-/Fix-Neuzugänge haben keine Nummer
|
||
stimmen_l = list((votes.get(sn) or {}).get("level") or [])
|
||
stimmen_r = list((votes.get(sn) or {}).get("relevance") or [])
|
||
for v in verdicts[:2]:
|
||
if k and k in v["levels"]:
|
||
stimmen_l += [v["levels"][k]] * 2 # explizite Korrektur wiegt doppelt
|
||
if k and k in v["relevanz"]:
|
||
stimmen_r += [v["relevanz"][k]] * 2
|
||
fk = bfacts.get(sn) or {}
|
||
sidecar_subs.append({"title": t, "level": _default_vote(stimmen_l, "advanced"),
|
||
"relevance": _default_vote(stimmen_r, "relevant"), "facts": fk})
|
||
|
||
# 8. Fix-Tail (0–1 Call): Korrekturen + belegte Lücken
|
||
if (korrekturen or luecken) and not ctx.is_cancelled():
|
||
if melde:
|
||
melde("Fix")
|
||
neu = await _fix_befunde(ctx, files, title, korrekturen, luecken,
|
||
[s["title"] for s in sidecar_subs], instructions, ns, lbl, sources)
|
||
for e in neu or []:
|
||
sn = _norm_title(e["title"])
|
||
vorhanden = next((s for s in sidecar_subs if _norm_title(s["title"]) == sn), None)
|
||
if vorhanden is not None: # Korrektur: Facts ersetzen, Einstufung bleibt
|
||
vorhanden["facts"] = _fk_of(e)
|
||
bfacts[sn] = vorhanden["facts"]
|
||
else: # Lücken-Fund: neuer consensus-Sub
|
||
bfacts[sn] = _fk_of(e)
|
||
keep.append(e["title"])
|
||
sidecar_subs.append({"title": e["title"],
|
||
"level": e["level"] or "advanced",
|
||
"relevance": e["relevance"] or "relevant",
|
||
"facts": bfacts[sn]})
|
||
await db.put_subblock(topic, bnorm, sn, title, e["title"], status="consensus")
|
||
|
||
if len(keep) != len(subs):
|
||
_log(topic, f"Verify {title}: {len(subs)} consensus + {len(unsicher)} unsicher → {len(keep)}")
|
||
return {"raw": {title: [s["title"] for s in sidecar_subs]},
|
||
"facts": {title: bfacts},
|
||
"sidecar": {title: sidecar_subs}}
|
||
|
||
|
||
async def _fix_befunde(ctx: GenContext, files: dict, title: str, korrekturen: list[dict],
|
||
luecken: list[str], vorhanden: list[str], instructions: str,
|
||
ns: str, lbl: str, sources: list[str] | None) -> list[dict]:
|
||
"""EIN Call korrigiert beanstandete Facts und füllt gemeldete Lücken. Hartes Beleg-Gate
|
||
für Neuzugänge (key_points/cited_facts nicht leer) + Dedup gegen den Bestand — ein
|
||
unbelegter Lücken-„Fund" flutet sonst das Fakten-Gate des Guides."""
|
||
topic = ctx.topic
|
||
work_dir = files["arbeit"]
|
||
auftraege = [f"- KORRIGIEREN: „{k['titel']}“ — {k['hinweis']}" for k in korrekturen]
|
||
auftraege += [f"- LÜCKE (neuer Subbaustein, nur wenn belegbar): {l}" for l in luecken]
|
||
source, caps = await asyncio.to_thread(
|
||
_inline_source, topic, sources,
|
||
[title] + [k["titel"] for k in korrekturen] + list(luecken))
|
||
sh = _h8(title, *sorted(a for a in auftraege))
|
||
pfad = work_dir / f"fix-{sh}.json"
|
||
if _gen_schema(_json_file(pfad)) is None:
|
||
status, _v = await run_single_slot(
|
||
ctx, f"{lbl}Fix", key=f"blocks-{topic}-{ns}sb-fix-{sh}",
|
||
prompt=_prompt("Subblock-Fix", topic=topic, block=title, source=source,
|
||
auftraege="\n".join(auftraege), extra=_extra(instructions)),
|
||
role="quick", capabilities=caps,
|
||
payload=lambda result, p=pfad: _sink_json(result, p, _gen_schema),
|
||
timeout=_timeout("fix", len(auftraege)))
|
||
if status == FAILED:
|
||
_log(topic, f"Fix {title} ohne Ergebnis — Befunde bleiben offen")
|
||
return []
|
||
out = _gen_schema(_json_file(pfad)) or []
|
||
korrektur_norms = {_norm_title(k["titel"]) for k in korrekturen}
|
||
have_norms = {_norm_title(t) for t in vorhanden}
|
||
angenommen = []
|
||
for e in out:
|
||
sn = _norm_title(e["title"])
|
||
if sn in korrektur_norms:
|
||
angenommen.append(e)
|
||
continue
|
||
if sn in have_norms or not (e["key_points"] or e["cited_facts"]):
|
||
continue # unbelegt oder Dublette → verfällt
|
||
st = _sub_tokens(e["title"])
|
||
if any(st <= _sub_tokens(t) or _sub_tokens(t) <= st for t in vorhanden):
|
||
continue
|
||
angenommen.append(e)
|
||
have_norms.add(sn)
|
||
return angenommen
|
||
|
||
|
||
# ── Artefakte ───────────────────────────────────────────────────────────────────────
|
||
|
||
def _subs_text(title: str, sidecar_subs: list[dict]) -> str:
|
||
return f"BLOCK: {title}\n" + "\n".join(
|
||
f"- {s['title']}\n" + "\n".join(f" {z}" for z in _facts_lines(s.get("facts") or {}).splitlines())
|
||
for s in sidecar_subs)
|
||
|
||
|
||
async def _artefakte_block(ctx: GenContext, files: dict, title: str,
|
||
sidecar_subs: list[dict], instructions: str = "",
|
||
ns: str = "", lbl: str = "", melde=None) -> dict | None:
|
||
"""EIN Generator-Call liefert Fragen + Flashcards + Beispiele (Split in 2 parallele
|
||
Calls bei > ART_SPLIT_SUBS Subs), EIN Prüfer-Call verifiziert Beispiele, bereinigt
|
||
die Fragen und ergänzt fehlende. → {pattern, artefacts} | None (nur Cancel)."""
|
||
topic = ctx.topic
|
||
work_dir = files["arbeit"]
|
||
if not sidecar_subs:
|
||
return {"pattern": {title: []}, "artefacts": {"flashcard": [], "example": []}}
|
||
if melde:
|
||
melde("Artefakte gen")
|
||
sh = _subs_hash({title: sidecar_subs})
|
||
haelften = ([sidecar_subs] if len(sidecar_subs) <= ART_SPLIT_SUBS
|
||
else [sidecar_subs[:len(sidecar_subs) // 2], sidecar_subs[len(sidecar_subs) // 2:]])
|
||
|
||
async def _gen(gi: int, teil: list[dict]):
|
||
pfad = work_dir / f"art-{sh}-t{gi}.json"
|
||
if _art_gen_schema(_json_file(pfad)) is not None:
|
||
return
|
||
status, _v = await run_single_slot(
|
||
ctx, f"{lbl}Artefakte {gi}", key=f"blocks-{topic}-{ns}art-gen-{sh}-t{gi}",
|
||
prompt=_prompt("Artefakt-Generate", topic=topic, blocks=_subs_text(title, teil),
|
||
extra=_extra(instructions)),
|
||
role="quick", capabilities="none",
|
||
payload=lambda result, p=pfad: _sink_json(result, p, _art_gen_schema),
|
||
timeout=_timeout("artefakt", len(teil)))
|
||
if status == FAILED:
|
||
_log(topic, f"Artefakte {title} Teil {gi} ohne Ergebnis")
|
||
|
||
await asyncio.gather(*[_gen(gi, teil) for gi, teil in enumerate(haelften, 1)])
|
||
if ctx.is_cancelled():
|
||
return None
|
||
pattern: list[dict] = []
|
||
cards: list[dict] = []
|
||
examples: list[dict] = []
|
||
for gi in range(1, len(haelften) + 1):
|
||
o = _art_gen_schema(_json_file(work_dir / f"art-{sh}-t{gi}.json"))
|
||
if o:
|
||
pattern += o["pattern"]
|
||
cards += o["cards"]
|
||
examples += o["examples"]
|
||
|
||
# Prüfer: Beispiele verifizieren, Fragen bereinigen + fehlende ergänzen
|
||
sub_titles = [s["title"] for s in sidecar_subs]
|
||
fehlend = [t for t in sub_titles
|
||
if _norm_title(t) not in {_norm_title(p["subblock"]) for p in pattern}]
|
||
if pattern or examples:
|
||
if melde:
|
||
melde("Artefakte check")
|
||
tabelle = "\n".join(f"({p['subblock']}) {p['question']}" for p in pattern) or "(keine)"
|
||
beisp = "\n\n".join(
|
||
f"{k}. PROBLEM: {e['problem']}\n SCHRITTE: " + " | ".join(e["steps"])
|
||
+ (f"\n ERGEBNIS: {e['result']}" if e.get("result") else "")
|
||
for k, e in enumerate(examples, 1)) or "(keine)"
|
||
fehlend_txt = ("\nSUBBLOCKS STILL MISSING A QUESTION:\n"
|
||
+ "\n".join(f"- {t}" for t in fehlend) + "\n") if fehlend else "\n"
|
||
pfad = work_dir / f"artcheck-{sh}.json"
|
||
if _art_check_schema(_json_file(pfad)) is None:
|
||
status, _v = await run_single_slot(
|
||
ctx, f"{lbl}Artefakt-Check", key=f"blocks-{topic}-{ns}art-check-{sh}",
|
||
prompt=_prompt("Artefakt-Check", topic=topic, facts=_subs_text(title, sidecar_subs),
|
||
table=tabelle, fehlend=fehlend_txt, examples=beisp,
|
||
extra=_extra(instructions)),
|
||
role="judge", capabilities="none",
|
||
payload=lambda result, p=pfad: _sink_json(result, p, _art_check_schema),
|
||
timeout=_timeout("artefakt_check", len(sidecar_subs)))
|
||
if status == FAILED:
|
||
_log(topic, f"Artefakt-Check {title} ohne Ergebnis — Rohfassung übernommen")
|
||
check = _art_check_schema(_json_file(pfad))
|
||
if check:
|
||
if check["examples_probleme"]:
|
||
examples = [e for k, e in enumerate(examples, 1)
|
||
if k not in check["examples_probleme"]]
|
||
_log(topic, f"Artefakt-Check {title}: {len(check['examples_probleme'])} Beispiel(e) verworfen")
|
||
if check["pattern"]: # bereinigte Fassung ersetzt die Rohfassung
|
||
pattern = check["pattern"]
|
||
pattern += check["pattern_ergaenzt"]
|
||
|
||
pattern_map = {title: [{"subblock": p["subblock"], "question": p["question"]}
|
||
for p in pattern]}
|
||
# block-Feld auf den Karten-Block normieren (Ein-Block-Call — Agent-Echos abfangen)
|
||
for e in cards + examples:
|
||
e["block"] = title
|
||
return {"pattern": pattern_map,
|
||
"artefacts": {"flashcard": [{k: e[k] for k in ("block", "subblock", "question", "answer")} for e in cards],
|
||
"example": [{k: e[k] for k in ("block", "subblock", "problem", "steps", "result")} for e in examples]}}
|