creator/backend/blocks.py

"""Blocks pipeline: research consensus + clarification loop — pure inventory, unsorted.

5x research (min. 3, grace) → mapping (consensus/rest) → clarification loop (max.
CONSENSUS_MAX_ROUNDS rounds): 3 selection agents (min. 2, grace) decide
on the disputed rest, a mapping agent sorts into accept/discard/
still disputed. An empty rest ends the loop; the last round must decide
everything. Races use a grace window instead of "first N win": after the
first valid result, the remaining agents get CONSENSUS_GRACE seconds to
finish. The consensus is accumulated in code — no agent re-emits
the full list.
"""

import asyncio
import hashlib
import json
import logging
import math
import re
import shutil
import subprocess
import time
import unicodedata
from pathlib import Path

import database as db
import embedding
from agents import kill_process, cancel_scope, clear_scope, run_agent
from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP, GROUP_RECONCILE_FLOOR, GROUP_MIN_COS_FLOOR, SUB_VARIANT_COS, SEED_COVER_COS, EVIDENCE_BUDGET_CHARS, EVIDENCE_CTX_LINES
from fsutil import atomic_write_text, atomic_write_json
from jsonio import parse_json_text, read_json_file as _json_file
from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder
from crawl import crawl
from pipeline import (
    CANCELLED, FAILED, OK, GenContext, _extra, _gather_progress, _yesno_schema, _log, _prompt, _race,
    _relevance_schema, _runde_schema, _semaphore, _str_list, _levels_schema, _timeout, run_single_slot,
)
from textkit import (
    _unique_title, _load_blocks, _norm_title, _parse_selection, _parse_subblocks, _title,
    _resolve_title, _title_index, clean_title,
)

# Chunk the subblocks (web search per block): 1 agent per ~10 blocks, capped.
SUBBLOCK_CHUNK = 10
SUBBLOCK_MAX = 40
# Classifying is cheap (short verdict, no web search) → larger packages, fewer files/agents.
LEVEL_CHUNK = 100

# Research: fixed file batches instead of a search loop → each crawl page is assigned exactly once.
RESEARCH_BATCH = 20        # crawl pages per batch
RESEARCH_READERS = 2       # reader agents per batch (consensus ≥2 within the batch)
RESEARCH_THEMA_AGENTS = 5  # web mode (source "thema", no crawl folder)
# uni/projekt: chunk the script text into sections of ~this size (against lost-in-the-middle on
# large documents). ~12k chars ≈ 3k tokens → safely below the recall-drop threshold.
RESEARCH_SECTION_CHARS = 12000
# Triage (content/noise) is now a deterministic rule filter (config.CRAWL_*).
SUBBLOCK_CAP = 900     # subblock find loop per chunk (15 min)
SUBBLOCK_MIN = 5           # below this consensus count a block gets focused catch-up rounds
SUBBLOCK_EXTRA_ROUNDS = 2  # max catch-up rounds (saturation stop still applies — thin stays thin)
SUBBLOCK_MAX_ROUNDS = 3    # hard round cap: measured, rounds 4–5 burned 29 % of the finder agents
                           # for ~zero consensus gain (fringe ideas never saturate) — thin blocks
                           # are caught by the SUBBLOCK_MIN catch-up plus the gap follow-up round
CONSOLIDATION_CHUNK = 600  # up to here ONE global judge (dedups everything); above that chunked + merge pass — fallback path only
DEDUP_PAIR_FLOOR = 0.6      # min cosine for a candidate pair (complete-link aggregates → no chaining)
DEDUP_PAIRS_CHUNK = 40      # pairs per judge package (pairwise verification instead of a block mixer)
DEDUP_TITLE_AUTO = 0.95     # near-identical TITLE cosine ⇒ same entity → merge without the judge (recall net)
DEDUP_GLOBAL_FLOOR = 0.65   # global post-naming dedup stage: candidate floor above the 0.5-0.65
                            # same-domain noise band, below the sibling zone (~0.85) — the judge decides there
FILTER_CHUNK = 35           # blocks to assess per judge in the degrade pass (full list as context)
# Balance question-pattern chunks by sub load via LPT (makespan), not by block count.
QUESTION_CHUNK_SUBS = 25       # target sum of relevant subs per chunk — at 50 the generator
                               # skipped so many subs that 60 % of all question calls were catch-up
QUESTION_MAX_ROUNDS = 3        # catch-up rounds for subs without a pattern (the LLM omits ~18 % per chunk)
FACTS_CHUNK_SUBS = 10      # facts extraction: small chunks — the 4 phases (find/erg/check/fix) are
                           # serial PER CHUNK, so chunk count = parallelism; the makespan tail of a
                           # late block is bounded by ONE chunk's phase chain, not the whole block
ARTEFACT_CHUNK_SUBS = 25   # flashcards/examples: bulk generation, phases are cheap → bigger packages
FACTS_CHECK_PANEL = 3      # judges per chunk in the facts check (majority objects)
CONSOLIDATION_PANEL = 3    # mapping judges per chunk (panel → reconcile instead of a single judge)
SUBBLOCK_PANEL = 3       # source judges in the subblock clarification (majority instead of a single judge)
FILTER_RECHECK_PANEL = 3   # judges in the survivor-recheck (rare-positive "fragment" recall; majority ≥2)

log = logging.getLogger("creator.blocks")

_blocks_progress: dict[str, str] = {}
_blocks_errors: dict[str, str] = {}
_blocks_cancelled: set[str] = set()
_blocks_step: dict[str, int] = {}

ARTEFACT_TYPES = ("flashcard", "example")


def load_source(topic: str) -> dict:
    """Read the persisted source choice. Fallback (legacy topics without source.json):
    if projects/<topic> exists → projekt, otherwise thema."""
    q = _json_file(source_path(topic))
    if isinstance(q, dict) and q.get("type") in ("thema", "projekt", "uni", "link"):
        return q
    if project_dir(topic).is_dir():
        return {"type": "projekt", "location": f"projects/{topic}", "spec": ""}
    return {"type": "thema", "location": "", "spec": ""}


def source_folder(topic: str) -> Path | None:
    """Folder source (projekt/uni → path, link → crawl folder) — otherwise None (thema)."""
    q = load_source(topic)
    if q["type"] == "link":
        return source_crawl_dir(topic)
    if q["type"] in ("projekt", "uni"):
        return safe_folder(q.get("location", ""))
    return None


def _crawl_done(topic: str) -> bool:
    return (source_crawl_dir(topic) / ".done").exists()  # marker only on clean completion


# Learning-path levels (beginner/advanced/expert); old difficulty values are backward-compatible.
_LEVELS = ("beginner", "advanced", "expert", "easy", "medium", "hard")


async def subblocks_title(topic: str, block: str) -> list[str]:
    """Subblock titles of a block — DB-first (consensus), fallback to the sidecar file."""
    rows = [s["sub_title"] for s in await db.list_subblocks(topic, _norm_title(block))
            if s["status"] == "consensus" and s["sub_title"]]
    if rows:
        return rows
    sc = _json_file(subblocks_path(topic))
    if not isinstance(sc, dict):
        return []
    return [
        t for s in (sc.get(block) or [])
        if isinstance(s, dict) and (t := str(s.get("title", "")).strip())
    ]


async def load_question_pattern(topic: str, block: str) -> list[dict]:
    """Predefined question patterns of a block — DB-first, fallback to sidecar (empty = live)."""
    rows = await db.list_question_pattern(topic, _norm_title(block))
    if rows:
        return [{"subblock": r["sub_title"], "question": r["question"]} for r in rows if r["question"]]
    fm = _json_file(question_pattern_path(topic))
    if not isinstance(fm, dict):
        return []
    return [
        {"subblock": str(e.get("subblock", "")).strip(), "question": question}
        for e in (fm.get(block) or [])
        if isinstance(e, dict) and (question := str(e.get("question", "")).strip())
    ]


async def subblocks_frei(topic: str, block: str, max_level: int) -> list[str]:
    """Subblock titles up to the unlocked level (≤ max_level). Fallback without
    level knowledge (legacy/sidecar): all subblock titles."""
    rows = await db.subs_with_level(topic, block)
    if not rows:
        return await subblocks_title(topic, block)
    return [s["title"] for s in rows if s["level"] <= max_level and s["title"]]


async def load_question_pattern_free(topic: str, block: str, max_level: int) -> list[dict]:
    """Question patterns, filtered to subblocks up to the unlocked level. Without
    level knowledge (legacy/sidecar), unfiltered."""
    rows = await db.subs_with_level(topic, block)
    if not rows:
        return await load_question_pattern(topic, block)
    unlocked = {s["norm"] for s in rows if s["level"] <= max_level}
    return [m for m in await load_question_pattern(topic, block) if _norm_title(m["subblock"]) in unlocked]


async def load_overview(topic: str) -> list[dict]:
    """Structured block list for the overview — DB-first (consensus + subs/levels/relevance),
    fallback to blocks.md + sidecar (legacy topics)."""
    bs = await db.list_blocks(topic, status="consensus")
    if bs:
        out = []
        for num, b in enumerate(bs, 1):
            subs = [s for s in await db.list_subblocks(topic, b["title_norm"]) if s["status"] == "consensus"]
            out.append({
                "num": num, "title": b["title"], "description": b["description"],
                "subblocks": [
                    {"title": s["sub_title"],
                     "level": s["level"] if s["level"] in _LEVELS else "advanced",
                     "relevance": s["relevance"] if s["relevance"] in ("relevant", "peripheral") else None}
                    for s in subs if s["sub_title"]
                ],
            })
        return out
    entries = _load_blocks(_read(blocks_path(topic)))
    sidecar = _json_file(subblocks_path(topic))
    sidecar = sidecar if isinstance(sidecar, dict) else {}
    out = []
    for num, entry in entries.items():
        title = _title(entry)
        split_parts = entry.split(" — ", 1)
        description = split_parts[1].strip() if len(split_parts) == 2 else ""
        subblocks = [
            {
                "title": t,
                "level": s.get("level") if s.get("level") in _LEVELS else "advanced",
                "relevance": s.get("relevance") if s.get("relevance") in ("relevant", "peripheral") else None,
            }
            for s in (sidecar.get(title) or [])
            if isinstance(s, dict) and (t := str(s.get("title", "")).strip())
        ]
        out.append({"num": num, "title": title, "description": description, "subblocks": subblocks})
    return out


def _blocks_steps(topic: str) -> tuple:
    """Steps per source: link gets "Source laden" up front, projekt additionally "Supplement".

    Subblocks + levels are three phases each (find, select, clarify). Per phase
    all packages run in parallel; the step remains until the last package is done.
    """
    q = load_source(topic)
    base = ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung")
    rest = (
        "Subblocks find", "Subblocks select", "Subblocks clarify",
        "Facts find", "Facts check", "Facts fix",
        "Levels find", "Levels select", "Levels clarify",
        "Relevance find", "Relevance select", "Relevance clarify",
        "Outline",
        "Questions find", "Questions select", "Questions clarify", "Questions check",
        "Flashcards", "Examples",
    )
    middle = base + (("Supplement",) if q["type"] == "projekt" else ()) + rest
    return (("Source prep",) if q["type"] == "link" else ()) + middle


def _step_idx(topic: str, name: str) -> int:
    return _blocks_steps(topic).index(name)


def _report_p(set_p, topic: str, step: str):
    """Async report callback for _gather_progress: sets "<step> d/t…" + step index."""
    idx = _step_idx(topic, step)
    async def report(d, t):
        set_p(f"{step} {d}/{t}…", step=idx)
    return report


# Coarse display phases: bundle the fine steps (internally everything stays fine-grained).
# Special steps (Source laden, Supplement) belong to the "Inventory" phase.
PHASEN = (
    ("Source", ("Source prep",)),
    ("Inventory", ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung", "Supplement")),
    ("Subblocks", ("Subblocks find", "Subblocks select", "Subblocks clarify")),
    ("Facts", ("Facts find", "Facts check", "Facts fix")),
    ("Levels", ("Levels find", "Levels select", "Levels clarify")),
    ("Relevance", ("Relevance find", "Relevance select", "Relevance clarify")),
    ("Outline", ("Outline",)),
    ("Questions", ("Questions find", "Questions select", "Questions clarify", "Questions check")),
    ("Artefacts", ("Flashcards", "Examples")),
)






def _blocks_files(topic: str) -> dict:
    work_dir = arbeit_dir(topic)
    rounds = range(1, CONSENSUS_MAX_ROUNDS + 1)
    return {
        "final": blocks_path(topic),
        "arbeit": work_dir,
        "research": [work_dir / f"research-{i}.md" for i in (1, 2, 3, 4, 5)],
        "research_mapping": work_dir / "research-mapping.json",
        "selection": {n: [work_dir / f"selection-r{n}-{i}.json" for i in (1, 2, 3)] for n in rounds},
        "mapping": {n: work_dir / f"selection-mapping-r{n}.json" for n in rounds},
        "ergaenzung": work_dir / "ergaenzung.json",
        "sub_roh": work_dir / "subblocks-roh.json",
        "facts": work_dir / "subblocks-facts.json",
        "sidecar": subblocks_path(topic),
        "question_pattern": question_pattern_path(topic),
        "outline": work_dir / "outline.json",
        "outline_slots": [work_dir / f"outline-{i}.json" for i in (1, 2, 3)],
        "artefakte": work_dir / "artefakte.json",
    }


def cancel_blocks(topic: str) -> bool:
    if topic not in _blocks_progress:
        return False
    _blocks_cancelled.add(topic)
    cancel_scope(f"blocks-{topic}-")  # waiting agents bail before spawning
    kill_process(f"blocks-{topic}-")  # kill running subprocesses
    return True






async def blocks_status(topic: str) -> dict:
    """Kanban-based status: `generating` from the run registry, progress from the card
    counts. `partial` = cards sit in non-terminal columns while nothing runs (continue-able)."""
    generating = topic in _blocks_progress
    counts = await db.kanban_stage_counts(topic)
    # ready = finished inventory. The DB is the source of truth (a synced topic may lack
    # blocks.md, the file is a legacy mirror) — either signal counts.
    ready = blocks_path(topic).exists() or counts.get("inventory", {}).get("done_block", 0) > 0
    terminal = {"clustered", "done_cluster", "grouped", "rejected", "done_block", "done_artefact", "dead"}
    open_cards = sum(n for stages in counts.values()
                     for stage, n in stages.items() if stage not in terminal)
    return {
        "ready": ready,
        "generating": generating,
        "progress": _blocks_progress.get(topic),
        "error": _blocks_errors.get(topic),
        "partial": not generating and open_cards > 0,
        "steps": [],        # legacy phase pills — replaced by the live board
        "feine_steps": [],
    }


def active_blocks() -> list[dict]:
    return [{"topic": t, "progress": p} for t, p in _blocks_progress.items()]


def reset_blocks(topic: str) -> None:
    """"Remove": deletes the ENTIRE blocks area — crawl, triage, inventory … questions.
    KEEPS only the topic config `source.json` (type/link/spec). Re-generating crawls anew.
    (Crawl/triage belong to the blocks; only the config is the "topic".)"""
    files = _blocks_files(topic)
    files["final"].unlink(missing_ok=True)
    files["sidecar"].unlink(missing_ok=True)
    files["question_pattern"].unlink(missing_ok=True)
    shutil.rmtree(source_crawl_dir(topic), ignore_errors=True)  # crawl belongs to the blocks
    shutil.rmtree(files["arbeit"], ignore_errors=True)
    _blocks_errors.pop(topic, None)
    # source.json intentionally stays — that is the topic config.










def _supplement_schema(data):
    """{"blocks": [{"title", "description"}]} → list (empty allowed) · otherwise None."""
    if not isinstance(data, dict) or not isinstance(data.get("blocks"), list):
        return None
    out = []
    for b in data["blocks"]:
        if not isinstance(b, dict) or not isinstance(b.get("title"), str) or not isinstance(b.get("description"), str):
            return None
        title, description = b["title"].strip(), b["description"].strip()
        if not title:
            return None
        out.append((title, description))
    return out


def _ocr_languages() -> str | None:
    """Installierte Tesseract-Sprachen (deu/eng), None → OCR aus."""
    try:
        import pymupdf
        base = Path(pymupdf.get_tessdata())
    except Exception:
        return None
    langs = [l for l in ("deu", "eng") if (base / f"{l}.traineddata").exists()]
    return "+".join(langs) or None


def _pdf_markdown(pdf: Path) -> str | None:
    """pymupdf4llm → Markdown string (None if the lib is missing or it fails)."""
    try:
        import pymupdf4llm
    except ImportError:
        return None
    try:
        # OCR nur, wenn Tesseract-Sprachdaten wirklich vorhanden sind —
        # sonst wirft der OCR-Pfad und die ganze Datei faellt auf pdftotext.
        langs = _ocr_languages()
        kwargs = {"use_ocr": True, "ocr_language": langs} if langs else {"use_ocr": False}
        return pymupdf4llm.to_markdown(str(pdf), show_progress=False, **kwargs)
    except Exception:
        log.warning("pymupdf4llm failed for %s", pdf.name, exc_info=True)
        return None


def _pdf_plaintext(pdf: Path) -> str | None:
    """pdftotext -layout → plain string (None if missing/fails)."""
    if shutil.which("pdftotext") is None:
        return None
    try:
        out = subprocess.run(["pdftotext", "-layout", str(pdf), "-"],
                             check=True, timeout=120, capture_output=True)
        return out.stdout.decode("utf-8", errors="replace")
    except Exception:
        log.warning("pdftotext failed for %s", pdf.name, exc_info=True)
        return None


# Content-fidelity guard between the two converters (topic-NEUTRAL: measures loss, not domain).
# pymupdf4llm yields structured Markdown but silently DROPS rendered display formulas and can
# splinter combining diacritics (measured on a LaTeX script: „P = {L …}" gone, „h¨aufig").
# pdftotext is structure-poor but faithful. Take the Markdown only when it preserves the bulk
# of the content; otherwise the faithful plaintext wins.
_PDF_FIDELITY_SYMBOLS = "≤≥∈∉⊆∪∧∨¬→Σδα{}=" 


def _pick_conversion(md: str | None, plain: str | None) -> tuple[str, str] | None:
    if md is None and plain is None:
        return None
    if md is None:
        return plain, "pdftotext"
    if plain is None:
        return md, "pymupdf4llm"
    ok_len = len(md) >= 0.7 * len(plain)
    sym_plain = sum(plain.count(c) for c in _PDF_FIDELITY_SYMBOLS)
    ok_sym = sym_plain == 0 or sum(md.count(c) for c in _PDF_FIDELITY_SYMBOLS) >= 0.8 * sym_plain
    ok_diakritik = md.count("\u00a8") <= plain.count("\u00a8") + 2  # standalone ¨ = splintered umlauts
    if ok_len and ok_sym and ok_diakritik:
        return md, "pymupdf4llm"
    return plain, "pdftotext"


def _convert_pdfs(project: Path) -> None:
    """Convert PDFs in the project to .txt — agents read text instead of page images.

    Called before every project generation; converts only if the .txt is missing or
    older than the PDF. Both converters run; the fidelity guard picks the better result
    per file. Neither available → hard error instead of an unreliable direct-read mode
    (MiniMax image limit, vision cost)."""
    pdfs = list(project.rglob("*.pdf"))
    if not pdfs:
        return
    for pdf in pdfs:
        txt = pdf.with_suffix(".txt")
        if txt.exists() and txt.stat().st_mtime >= pdf.stat().st_mtime:
            continue
        picked = _pick_conversion(_pdf_markdown(pdf), _pdf_plaintext(pdf))
        if picked is None:
            raise RuntimeError(f"PDF conversion failed ({pdf.name}): weder pymupdf4llm noch "
                               "pdftotext verfügbar/erfolgreich (pip install pymupdf4llm oder poppler-utils)")
        text, tool = picked
        txt.write_text(text, encoding="utf-8")
        _log(project.name, f"PDF konvertiert ({tool}): {pdf.name} → {txt.name}")


_SOURCE_TEMPLATE = {"projekt": "Blocks-Source-Projekt", "uni": "Blocks-Source-Uni", "link": "Blocks-Source-Link"}


def _text_sections(text: str, goal: int = RESEARCH_SECTION_CHARS) -> list[str]:
    """Split text at paragraph/line boundaries into sections of ~`ziel` chars (against lost-in-the-middle
    on large documents). Small text stays ONE section. Content stays complete — only
    separating whitespace is dropped."""
    text = text.strip()
    if len(text) <= goal:
        return [text] if text else []
    sections: list[str] = []
    buf = ""

    def flush():
        nonlocal buf
        if buf.strip():
            sections.append(buf.strip())
        buf = ""

    for block in re.split(r"\n\s*\n", text):  # at paragraph boundaries
        block = block.strip()
        if not block:
            continue
        if len(block) > goal:  # single huge paragraph → hard-cut at lines
            flush()
            for line in block.split("\n"):
                if buf and len(buf) + len(line) + 1 > goal:
                    flush()
                buf += line + "\n"
            flush()
        elif buf and len(buf) + len(block) + 2 > goal:
            flush()
            buf = block
        else:
            buf = (buf + "\n\n" + block) if buf else block
    flush()
    return sections


# ── Inline evidence for judges ──────────────────────────────────────────────────────
# Judges used to re-search the corpus per session ({source} → "ls/find … read", ~10 tool
# turns each). The corpus excerpts now go INTO the prompt; the agent answers as text.

def _corpus_files(folder: Path | None, sources: list[str] | None) -> list[Path]:
    """The block's named source .txt files; fallback: every .txt in the folder."""
    if folder is None or not folder.is_dir():
        return []
    if sources:
        named = [folder / Path(s).with_suffix(".txt").name for s in sources if s]
        named = [p for p in named if p.is_file()]
        if named:
            return named
    return sorted(p for p in folder.glob("*.txt") if p.is_file())


def _q_tokens(text: str) -> set[str]:
    return set(re.findall(r"\w{3,}", text.casefold()))


def _evidence_pack(folder: Path | None, sources: list[str] | None, queries: list[str],
                   budget: int = EVIDENCE_BUDGET_CHARS) -> str:
    """Keyword-selected corpus excerpts for a judge prompt. Sections are ranked by token
    overlap with `queries` (block title + candidates); every query with any match gets its
    best section (coverage guarantee), the rest of the budget takes the global top. Empty
    string when there is no corpus — the caller keeps the old self-research source then."""
    parts: list[tuple[str, int, str, set[str]]] = []  # (file, idx, text, tokens)
    for f in _corpus_files(folder, sources):
        try:
            text = f.read_text(encoding="utf-8")
        except OSError:
            continue
        for i, sec in enumerate(_text_sections(text), 1):
            parts.append((f.name, i, sec, _q_tokens(sec)))
    if not parts:
        return ""
    qtoks = [(_q_tokens(q)) for q in queries if q]
    score = [sum(len(qt & p[3]) for qt in qtoks) for p in parts]
    chosen: set[int] = set()
    for qt in qtoks:  # coverage guarantee: best section per query
        best = max(range(len(parts)), key=lambda k: len(qt & parts[k][3]), default=None)
        if best is not None and qt & parts[best][3]:
            chosen.add(best)
    used = sum(len(parts[k][2]) for k in chosen)
    for k in sorted(range(len(parts)), key=lambda k: -score[k]):  # top-up to budget
        if k in chosen or score[k] <= 0:
            continue
        if used + len(parts[k][2]) > budget:
            continue
        chosen.add(k)
        used += len(parts[k][2])
    out, total = [], 0
    for k in sorted(chosen):  # document order for readability
        fname, i, sec, _t = parts[k]
        if total + len(sec) > max(budget, used):  # hard cap incl. guarantee overshoot
            break
        out.append(f"── {fname} · Abschnitt {i} ──\n{sec}")
        total += len(sec)
    return "\n\n".join(out)


_CITE_POS = re.compile(r"\b(?:Z(?:eilen?)?|lines?)\.?\s*(\d+)(?:\s*[-–]\s*(\d+))?", re.I)


def _cite_ref(cite: str, files: list[Path]) -> tuple[Path, int, int] | None:
    """(file, line_lo, line_hi) from a cited_facts source string
    („Skript.txt, Übung 6.47, Z.1341-1344") — None when file or position is missing."""
    c = (cite or "").casefold()
    f = next((p for p in files if p.name.casefold() in c or p.stem.casefold() in c), None)
    m = _CITE_POS.search(cite or "")
    if f is None or m is None:
        return None
    lo, hi = int(m.group(1)), int(m.group(2) or m.group(1))
    return (f, min(lo, hi), max(lo, hi))


def _cited_evidence(folder: Path | None, sources: list[str] | None, cites: list[str],
                    fallback_queries: list[str], budget: int = EVIDENCE_BUDGET_CHARS) -> str:
    """Evidence for the facts check: the EXACT cited regions (±EVIDENCE_CTX_LINES, merged,
    line numbers in the header) — precise and tiny. Cites without a parseable position fall
    back to the keyword pack. Empty string without a corpus."""
    files = _corpus_files(folder, sources)
    if not files:
        return ""
    ranges: dict[Path, list[tuple[int, int]]] = {}
    unresolved = False
    for c in cites:
        ref = _cite_ref(c, files)
        if ref is None:
            unresolved = True
            continue
        f, lo, hi = ref
        ranges.setdefault(f, []).append((max(1, lo - EVIDENCE_CTX_LINES), hi + EVIDENCE_CTX_LINES))
    out, total = [], 0
    for f in files:
        if f not in ranges:
            continue
        try:
            lines = f.read_text(encoding="utf-8").splitlines()
        except OSError:
            continue
        merged: list[list[int]] = []
        for lo, hi in sorted(ranges[f]):
            hi = min(hi, len(lines))
            if merged and lo <= merged[-1][1] + 1:
                merged[-1][1] = max(merged[-1][1], hi)
            else:
                merged.append([lo, hi])
        for lo, hi in merged:
            sec = "\n".join(lines[lo - 1:hi])
            if not sec.strip() or total + len(sec) > budget:
                continue
            out.append(f"── {f.name} · Z. {lo}-{hi} ──\n{sec}")
            total += len(sec)
    if unresolved or not out:
        pack = _evidence_pack(folder, sources, fallback_queries, max(0, budget - total))
        if pack:
            out.append(pack)
    return "\n\n".join(out)


def _reply_text(result) -> str:
    """Assistant text of a no-tool agent call ((rc, stdout, stderr) from run_agent)."""
    return (result[1] or "") if result else ""


def _sink_json(result, path: Path, schema):
    """Payload validator for no-tool agents: the JSON comes as reply TEXT; the engine
    persists it to `path`, so resume guards and audit files keep working unchanged."""
    data = parse_json_text(_reply_text(result))
    val = schema(data)
    if val is not None:
        atomic_write_json(path, data)
    return val


def _build_research_prompt(topic: str, out_path: Path, instructions: str, type: str, folder: Path | None, fokus: str = "", section: str = "") -> str:
    if section:
        # Section mode (uni/projekt): text directly in the prompt → small context, no file reading.
        source = section
    elif type in _SOURCE_TEMPLATE:
        source = _prompt(_SOURCE_TEMPLATE[type], project=folder)
    else:
        source = _prompt("Blocks-Source-Thema", topic=topic)
    return _prompt(
        "Blocks-Research",
        topic=topic, source=source, blocks_path=out_path, focus=fokus, extra=_extra(instructions),
    )


def _file_payload(path: Path):
    """Valid if the slot file exists and contains numbered entries."""
    if not path.exists():
        return None
    text = path.read_text(encoding="utf-8")
    return text if _parse_selection(text) else None




def _question_pattern_chunk_schema(data) -> list[dict] | None:
    """{"pattern": [{block, subblock, question}, …]} → list of valid entries · otherwise None.

    One pattern per subblock (no type cross-product — the difficulty only comes at
    exam time from the learner's tier). Invalid individual entries are skipped."""
    if not isinstance(data, dict) or not isinstance(data.get("pattern"), list):
        return None
    out = []
    for e in data["pattern"]:
        if not isinstance(e, dict):
            continue
        blk = str(e.get("block", "")).strip()
        sub = str(e.get("subblock", "")).strip()
        question = str(e.get("question", "")).strip()
        if not blk or not sub or not question:
            continue
        out.append({"block": blk, "subblock": sub, "question": question})
    return out or None


def _read(p: Path) -> str:
    return p.read_text(encoding="utf-8") if p.exists() else ""


def _chunk_nums(items: list, n: int) -> list[list]:
    """Splits a flat list into n chunks as equal in size as possible."""
    n = max(1, n)
    size = max(1, math.ceil(len(items) / n))
    return [items[i:i + size] for i in range(0, len(items), size)]


def _n_chunks(count: int, size: int = SUBBLOCK_CHUNK) -> int:
    return min(SUBBLOCK_MAX, max(1, math.ceil(count / size)))


def _lpt_chunks(weights: list[int], target: int) -> list[list[int]]:
    """Distribute indices across chunks load-balanced (LPT, makespan-minimal). Weight = cost per index.
    K = ceil(total weight/target); heaviest first into the currently lightest bin. → index lists."""
    if not weights:
        return []
    K = max(1, math.ceil(sum(weights) / max(1, target)))
    bins: list[list[int]] = [[] for _ in range(K)]
    last = [0] * K
    for i in sorted(range(len(weights)), key=lambda x: weights[x], reverse=True):
        j = min(range(K), key=lambda b: last[b])
        bins[j].append(i)
        last[j] += weights[i]
    return [b for b in bins if b]








# lemmatized: 'kein Syntaxfehler' vs 'keine Syntax-Fehlermeldung' are the SAME statement —
# unlemmatized token sets ({kein} ≠ {keine}) blocked that fold at cos 0.974 (measured).
_NEG_LEMMA = {"nicht": "nicht", "ohne": "ohne", "nie": "nie", "niemals": "nie",
              "kein": "kein", "keine": "kein", "keinen": "kein", "keiner": "kein",
              "keinem": "kein", "keines": "kein"}


def _neg_set(title: str) -> frozenset:
    """Lemmatized negation tokens of a title — antonym statements measure cos 0.91–0.95
    (above any usable variant threshold), so equal negation sets are a hard merge precondition."""
    return frozenset(l for t in re.findall(r"\w+", _norm_title(title)) if (l := _NEG_LEMMA.get(t)))


def _sub_tokens(title: str) -> set:
    return set(re.findall(r"\w+", _norm_title(title)))


def _variant_clusters(titles: list[str], mentions: list[int], sims) -> list[dict]:
    """Fold phrasing VARIANTS of one concept BEFORE the consensus count: finders rephrase per
    round, so exact-norm counting starves real concepts. Union-find over cos ≥ SUB_VARIANT_COS
    with the negation guard. → [{"rep": idx, "members": [idx…], "mentions": sum}]."""
    n = len(titles)
    parent = list(range(n))

    def find(x):
        while parent[x] != x:
            parent[x] = parent[parent[x]]
            x = parent[x]
        return x

    negs = [_neg_set(t) for t in titles]
    for i in range(n):
        for j in range(i + 1, n):
            if float(sims[i][j]) >= SUB_VARIANT_COS and negs[i] == negs[j]:
                parent[find(i)] = find(j)
    groups: dict[int, list[int]] = {}
    for i in range(n):
        groups.setdefault(find(i), []).append(i)
    return [{"rep": max(g, key=lambda k: (len(titles[k]), -k)), "members": g,
             "mentions": sum(mentions[k] for k in g)} for g in groups.values()]


async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str,
                           wipe: bool = True, ns: str = "", seeds: list[str] | None = None,
                           lbl: str = "", sources: list[str] | None = None) -> dict | None:
    """Block B (DB + loop): per package, find subblocks in rounds (3 finders, until 0 new/cap),
    collect in the DB (variant-clustered mentions ≥2 = consensus), a judge panel cleans up per
    package; blocks below SUBBLOCK_MIN get focused catch-up rounds; `seeds` (demoted fragment
    titles, single-block kanban calls) are guaranteed to reach the facts evidence gate.
    → {block title: [subblock, …]} (consensus) or None. Fills DB table `subblocks`.
    wipe=False (kanban board: one call per block) keeps the other blocks' rows."""
    topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
    work_dir = files["arbeit"]
    folder = source_folder(topic)
    caps = "files" if folder else "full"
    # Source for the evidence exam in the clarify step (discards invented/unsupportable subs).
    _type = load_source(topic).get("type", "thema")
    source = _prompt(_SOURCE_TEMPLATE[_type], project=folder) if _type in _SOURCE_TEMPLATE else _prompt("Blocks-Source-Thema", topic=topic)
    nums = list(entries)
    chunks = _chunk_nums(nums, _n_chunks(len(nums)))
    n = len(chunks)
    title_by_num = {num: _title(entries[num]) for num in nums}
    norm_by_num = {num: _norm_title(title_by_num[num]) for num in nums}
    emb_on = EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available)
    if wipe:
        await db.delete_subblocks(topic)  # fresh start of the block (idempotent counter)
    else:
        for num in nums:  # per-block wipe: a re-spawned card must not accumulate mentions
            await db.delete_subblocks(topic, norm_by_num[num])

    async def _known_block(chunk):
        known = []
        for num in chunk:
            subs = [s["sub_title"] for s in await db.list_subblocks(topic, norm_by_num[num])]
            if subs:
                known.append(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in subs))
        if not known:
            return ""
        # Do NOT list known items again (otherwise re-confirmation inflates the mention count,
        # self-bias/echo) — only add what's missing. This keeps the counter an honest consensus signal.
        return ("\n\nBEREITS ERFASST — liste diese NICHT erneut. Finde nur, was FEHLT:\n" + "\n".join(known))

    # ONE finder round (3 slots, quorum 2) → count of NEW sub norms; None = no result/cancel.
    async def _one_round(label, subset, assignment, paths, keys, known, extra_instr):
        chunk_idx = _title_index({num: title_by_num[num] for num in subset})
        for p in paths:
            p.unlink(missing_ok=True)
        slots = [{
            "key": k,
            "prompt": _prompt("Subblock-Research", topic=topic, assignment=assignment, known=known, out_path=p, extra=_extra(extra_instr)),
            "role": "quick", "capabilities": caps,
            "payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None),
        } for k, p in zip(keys, paths)]
        agent_texts = await _race(topic, label, slots, 2, _timeout("subblock", len(subset)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
        if is_cancelled() or not agent_texts:
            return None
        rows_before = {num: await db.list_subblocks(topic, norm_by_num[num]) for num in subset}
        existing = {num: {s["sub_norm"] for s in rows_before[num]} for num in subset}
        fresh: dict[int, list[str]] = {}
        for d in agent_texts:
            for marker, subs in d.items():
                num = _resolve_title(chunk_idx, marker)
                if num is None:
                    continue
                seen_set = set()
                for sub in subs:
                    sn = _norm_title(sub)
                    if not sn or sn in seen_set:
                        continue
                    seen_set.add(sn)
                    if sn not in existing[num]:
                        existing[num].add(sn)
                        fresh.setdefault(num, []).append(sub)
                    await db.upsert_subblock(topic, norm_by_num[num], sn, title_by_num[num], sub)
        # "New" is variant-robust: a paraphrase of an existing sub (or of another fresh find)
        # still gets stored above (its mention feeds the cluster consensus), but it must not
        # keep the saturation loop spinning — finders rephrase every round (measured: 5–9
        # rounds without this fold). Model off → exact counting (status quo).
        new = 0
        for num, cands in fresh.items():
            sims = None
            base = [s["sub_title"] for s in rows_before[num]]
            if emb_on and base + cands:
                sims = await asyncio.to_thread(embedding.embed_sims, base + cands)
            if sims is None:
                new += len(cands)
                continue
            negs = [_neg_set(t) for t in base + cands]
            nb = len(base)
            kept: list[int] = []
            for i in range(nb, nb + len(cands)):
                dup = any(float(sims[i][j]) >= SUB_VARIANT_COS and negs[i] == negs[j]
                          for j in [*range(nb), *kept])
                if not dup:
                    kept.append(i)
            new += len(kept)
        return new

    # Phase "Subblocks find": per package loop until 0 new subs / time cap.
    async def _find(c, chunk):
        assignment = "\n".join(f"- {entries[num]}" for num in chunk)
        start = time.monotonic()
        round_n = 0
        while not is_cancelled():
            round_n += 1
            bekannt = await _known_block(chunk) if round_n > 1 else ""
            paths = [work_dir / f"subblock-c{c}-r{round_n}-{i}.md" for i in (1, 2, 3)]
            keys = [f"blocks-{topic}-{ns}subblock-c{c}-r{round_n}-{i}" for i in (1, 2, 3)]
            new = await _one_round(f"{lbl}Subblocks package {c} R{round_n}", chunk, assignment, paths, keys, bekannt, instructions)
            if new is None:
                if is_cancelled():
                    return False
                return round_n > 1  # round 1 without result = error; later = simply the end
            if new == 0:
                break
            if round_n >= SUBBLOCK_MAX_ROUNDS:
                _log(topic, f"Subblocks package {c}: round cap reached ({round_n})")
                break
            if time.monotonic() - start > SUBBLOCK_CAP:
                _log(topic, f"Subblocks package {c}: time cap reached (round {round_n})")
                break
        return True

    oks = await _gather_progress([_find(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _report_p(set_p, topic, "Subblocks find"))
    if is_cancelled():
        return None
    if not all(ok is True for ok in oks):
        _blocks_errors[topic] = "Subblocks failed (research)"
        return None

    # Phase "Subblocks select": variant-clustered mentions ≥2 = consensus — the cluster
    # representative carries the status, folded members become `variant` (NOT discarded:
    # the clarify panel's uncertain group must not re-list them). Model off → exact counter.
    async def _select(subset, keep_consensus=False):
        for num in subset:
            rows = await db.list_subblocks(topic, norm_by_num[num])
            clusters = None
            if emb_on and len(rows) >= 2:
                sims = await asyncio.to_thread(embedding.embed_sims, [r["sub_title"] for r in rows])
                if sims is not None:
                    clusters = _variant_clusters([r["sub_title"] for r in rows],
                                                 [r["mentions"] for r in rows], sims)
            if clusters is None:
                for r in rows:
                    if keep_consensus and r["status"] == "consensus":
                        continue
                    await db.set_subblock_fields(topic, norm_by_num[num], r["sub_norm"],
                                                 status=("consensus" if r["mentions"] >= 2 else "discarded"))
                continue
            for cl in clusters:
                # a re-select (catch-up) never demotes panel-confirmed subs — an existing
                # consensus member stays the representative, new variants fold under it.
                kept = [k for k in cl["members"] if keep_consensus and rows[k]["status"] == "consensus"]
                for k in cl["members"]:
                    if kept:
                        st = "consensus" if k in kept else "variant"
                    elif cl["mentions"] >= 2:
                        st = "consensus" if k == cl["rep"] else "variant"
                    else:
                        st = "discarded"
                    if keep_consensus and rows[k]["status"] == "consensus" and st != "consensus":
                        continue
                    await db.set_subblock_fields(topic, norm_by_num[num], rows[k]["sub_norm"], status=st)

    set_p(f"Subblocks select ({n} packages)…", step=_step_idx(topic, "Subblocks select"))
    await _select(nums)

    # Judge formulation → shown candidate (best cos ≥ SUB_VARIANT_COS, negation-guarded).
    # Judges demonstrably paraphrase; without canonicalizing, the exact-norm majority vote
    # splinters across formulations (measured: 0.993-duplicates in a final list).
    async def _canon_map(shown: list[str], judge_titles: list[str]) -> dict[str, tuple[str, str]]:
        if not emb_on or not shown or not judge_titles:
            return {}
        texts = shown + list(judge_titles)
        sims = await asyncio.to_thread(embedding.embed_sims, texts)
        if sims is None:
            return {}
        negs = [_neg_set(t) for t in texts]
        m: dict[str, tuple[str, str]] = {}
        for a in range(len(shown), len(texts)):
            best, bv = None, 0.0
            for b in range(len(shown)):
                v = float(sims[a][b])
                if v >= SUB_VARIANT_COS and v > bv and negs[a] == negs[b]:
                    best, bv = b, v
            if best is not None:
                m[_norm_title(texts[a])] = (_norm_title(shown[best]), shown[best])
        return m

    # Phase "Subblocks clarify": source panel (SUBBAUSTEIN_PANEL judges) checks consensus + uncertain (1×)
    # against the source; code majority per sub. External, multi-voice gate against single-judge bias + echo.
    async def _clarify(c, chunk, tag=""):
        fp = work_dir / f"subblock-final-c{c}{tag}.md"
        if _parse_subblocks(_read(fp)):
            return
        block_texts, has_any = [], False
        consensus_by_num: dict[int, list[str]] = {}
        shown_by_num: dict[int, list[str]] = {}
        for num in chunk:
            rows = await db.list_subblocks(topic, norm_by_num[num])
            consensus_subs = [s["sub_title"] for s in rows if s["status"] == "consensus"]
            # folded variants (status `variant`) are already counted — only true singles are uncertain
            uncertain = [s["sub_title"] for s in rows if s["status"] == "discarded" and s["mentions"] == 1]
            consensus_by_num[num] = consensus_subs
            shown_by_num[num] = consensus_subs + uncertain
            if not consensus_subs and not uncertain:
                continue
            has_any = True
            k_lines = "\n".join(f"- {s}" for s in consensus_subs) if consensus_subs else "- (keiner)"
            u_lines = "\n".join(f"- {s}" for s in uncertain) if uncertain else "- (keiner)"
            band = ""
            shown = shown_by_num[num]
            if emb_on and len(shown) >= 2:  # near-dup pairs BELOW the fold threshold → explicit panel hint
                sims = await asyncio.to_thread(embedding.embed_sims, shown)
                if sims is not None:
                    pairs = [f"- „{shown[i]}“ ↔ „{shown[j]}“"
                             for i in range(len(shown)) for j in range(i + 1, len(shown))
                             if 0.75 <= float(sims[i][j]) < SUB_VARIANT_COS]
                    if pairs:
                        band = ("\nMögliche Duplikate — prüfen und ggf. zu EINEM Eintrag zusammenführen:\n"
                                + "\n".join(pairs[:12]))
            block_texts.append(f"BLOCK: {title_by_num[num]}\nKonsens (≥2 finders):\n{k_lines}\nUnsicher (1× — streng gegen Source check):\n{u_lines}{band}")
        if not has_any:
            return

        chunk_idx = _title_index({num: title_by_num[num] for num in chunk})
        paths = [work_dir / f"subblock-final-c{c}{tag}-j{j}.md" for j in range(1, SUBBLOCK_PANEL + 1)]
        # truthiness, NOT `is None`: a missing file parses to {} — with `is None` the whole
        # panel silently never ran (fallback adopted the raw consensus unchecked).
        pending = [(j, p) for j, p in enumerate(paths, 1) if not _parse_subblocks(_read(p))]
        for _, p in pending:
            p.unlink(missing_ok=True)
        if pending:
            # Inline evidence: corpus excerpts in the prompt (no self-research); the judge
            # answers as TEXT, the engine persists the j-file (resume + majority unchanged).
            ev = _evidence_pack(folder, sources,
                                [title_by_num[num] for num in chunk]
                                + [s for num in chunk for s in shown_by_num.get(num, [])]) if folder else ""
            j_source = _prompt("Blocks-Source-Inline", excerpts=ev) if ev else source

            def _sink(result, p):
                text = _reply_text(result).strip()
                d = _parse_subblocks(text)
                if d:
                    atomic_write_text(p, text)
                return d or None

            slots = [{
                "key": f"blocks-{topic}-{ns}subblock-final-c{c}{tag}-j{j}",
                "prompt": _prompt("Subblock-Mapping", topic=topic, source=j_source, blocks="\n\n".join(block_texts), out_path=p, extra=_extra(instructions)),
                "role": "judge", "capabilities": "none" if ev else caps,
                "payload": (lambda result, p=p: _sink(result, p)),
            } for j, p in pending]
            existing = SUBBLOCK_PANEL - len(pending)
            await _race(topic, f"{lbl}Subblock-Clarification {c}", slots, max(1, 2 - existing),
                        _timeout("subblock_check", len(chunk)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
        if is_cancelled():
            return
        outs = [d for p in paths if (d := _parse_subblocks(_read(p)))]
        if not outs:  # panel fully failed → adopt consensus (the fallback as before)
            _log(topic, f"Subblock clarification package {c} failed — consensus adopted")
            text = "\n\n".join(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in consensus_by_num[num])
                               for num in chunk if consensus_by_num[num])
            atomic_write_text(fp, text)
            return

        # code majority per block/sub-norm: keep if a majority of judges list it (tie → keep).
        # Votes are canonicalized onto the shown candidates first (paraphrase-robust).
        block_texts_out = []
        for num in chunk:
            raw_votes: list[list[str]] = []
            for d in outs:
                subs_of_num: list[str] = []
                for marker, subs in d.items():
                    if _resolve_title(chunk_idx, marker) == num:
                        subs_of_num.extend(subs)
                raw_votes.append(subs_of_num)
            judge_titles = list(dict.fromkeys(s for subs in raw_votes for s in subs
                                              if _norm_title(s) not in {_norm_title(t) for t in shown_by_num[num]}))
            canon = await _canon_map(shown_by_num[num], judge_titles)
            votes: dict[str, int] = {}
            form: dict[str, str] = {}
            for subs_of_num in raw_votes:
                seen = set()
                for sub in subs_of_num:
                    sn = _norm_title(sub)
                    if not sn:
                        continue
                    if sn in canon:
                        sn, sub = canon[sn]
                    if sn in seen:
                        continue
                    seen.add(sn)
                    form.setdefault(sn, sub)
                    votes[sn] = votes.get(sn, 0) + 1
            kept = [form[sn] for sn in form if votes[sn] * 2 >= len(outs)]
            if kept:
                block_texts_out.append(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in kept))
        atomic_write_text(fp, "\n\n".join(block_texts_out))

    await _gather_progress([_clarify(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _report_p(set_p, topic, "Subblocks clarify"))
    if is_cancelled():
        return None

    # Final list per block: judge output, otherwise consensus fallback. Reconcile DB + build raw.
    raw: dict[str, list[str]] = {}

    async def _align(c, chunk, tag=""):
        final = _parse_subblocks(_read(work_dir / f"subblock-final-c{c}{tag}.md")) or {}
        chunk_idx = _title_index({num: title_by_num[num] for num in chunk})
        final_by_num = {_resolve_title(chunk_idx, m): subs for m, subs in final.items() if _resolve_title(chunk_idx, m) is not None}
        for num in chunk:
            title = title_by_num[num]
            consensus = [s["sub_title"] for s in await db.list_subblocks(topic, norm_by_num[num]) if s["status"] == "consensus"]
            subs = final_by_num.get(num) or consensus
            if not subs:
                continue
            raw[title] = subs
            # align DB to the final list: final = consensus, rest discarded, add new ones.
            final_norms = {_norm_title(s) for s in subs}
            have = {s["sub_norm"] for s in await db.list_subblocks(topic, norm_by_num[num])}
            for s in await db.list_subblocks(topic, norm_by_num[num]):
                if s["sub_norm"] in final_norms:
                    st = "consensus"
                elif s["status"] == "variant":
                    st = "variant"  # folded members stay marked — a catch-up clarify must not
                else:               # re-list them as "uncertain singles"
                    st = "discarded"
                await db.set_subblock_fields(topic, norm_by_num[num], s["sub_norm"], status=st)
            for s in subs:
                sn = _norm_title(s)
                if sn and sn not in have:
                    await db.upsert_subblock(topic, norm_by_num[num], sn, title, s)
                    await db.set_subblock_fields(topic, norm_by_num[num], sn, status="consensus")

    for c, chunk in enumerate(chunks, 1):
        await _align(c, chunk)

    # Minimum catch-up: a block below SUBBLOCK_MIN gets up to SUBBLOCK_EXTRA_ROUNDS focused
    # finder rounds. Saturation stop stays — a thin block REMAINS thin if nothing verifiable.
    async def _catchup(c, chunk):
        for k in range(1, SUBBLOCK_EXTRA_ROUNDS + 1):
            lacking = [num for num in chunk if len(raw.get(title_by_num[num]) or []) < SUBBLOCK_MIN]
            if not lacking or is_cancelled():
                return
            assignment = "\n".join(f"- {entries[num]}" for num in lacking)
            known = await _known_block(lacking)  # ALL rows of the block, incl. variants/discarded
            focus = (instructions + "\n\nDieser Block hat bisher nur sehr wenige belegte "
                     "Subbausteine. Suche gezielt nach WEITEREN belegbaren Kernaspekten, die "
                     "oben fehlen. Nimm NUR auf, was die Quellen wirklich hergeben — nicht aufblähen.")
            paths = [work_dir / f"subblock-x{k}-c{c}-{i}.md" for i in (1, 2, 3)]
            keys = [f"blocks-{topic}-{ns}subblock-x{k}-c{c}-{i}" for i in (1, 2, 3)]
            new = await _one_round(f"{lbl}Subblocks catch-up {c} X{k}", lacking, assignment, paths, keys, known, focus)
            if not new:
                return
            await _select(lacking, keep_consensus=True)
            await _clarify(c, chunk, tag=f"-x{k}")
            if is_cancelled():
                return
            await _align(c, chunk, tag=f"-x{k}")

    # progress reuses the clarify step label — catch-up has no own registry entry
    await _gather_progress([_catchup(c, chunk) for c, chunk in enumerate(chunks, 1)], n,
                           _report_p(set_p, topic, "Subblocks clarify"))
    if is_cancelled():
        return None

    # Seed guarantee (single-block kanban calls): every demoted-fragment seed must reach the
    # facts evidence gate — covered by a consensus sub, promoted from a single find, or
    # inserted as its own sub. Unverifiable seeds die at the facts discard, not silently here.
    for num in (nums if seeds else []):
        title = title_by_num[num]
        for seed in dict.fromkeys(s for s in seeds if s):
            st = _sub_tokens(seed)
            have = raw.get(title) or []
            if not st or any(st <= _sub_tokens(s) for s in have):
                continue
            if emb_on and have:  # embedding backup for rephrased covers (lexical is primary)
                sims = await asyncio.to_thread(embedding.embed_sims, [seed] + have)
                if sims is not None and max(float(sims[0][j]) for j in range(1, len(have) + 1)) >= SEED_COVER_COS:
                    continue
            rows = [r for r in await db.list_subblocks(topic, norm_by_num[num]) if r["status"] != "consensus"]
            cand = next((r for r in rows if st <= _sub_tokens(r["sub_title"])), None)
            if cand is None and emb_on and rows:
                sims = await asyncio.to_thread(embedding.embed_sims, [seed] + [r["sub_title"] for r in rows])
                if sims is not None:
                    j = max(range(1, len(rows) + 1), key=lambda x: float(sims[0][x]))
                    if float(sims[0][j]) >= SEED_COVER_COS and _neg_set(seed) == _neg_set(rows[j - 1]["sub_title"]):
                        cand = rows[j - 1]
            if cand is not None:
                await db.set_subblock_fields(topic, norm_by_num[num], cand["sub_norm"], status="consensus")
                raw.setdefault(title, []).append(cand["sub_title"])
                _log(topic, f"Seed „{seed}“: Einzelfund „{cand['sub_title']}“ übernommen ({title})")
            elif (sn := _norm_title(seed)):
                await db.put_subblock(topic, norm_by_num[num], sn, title, seed, status="consensus")
                raw.setdefault(title, []).append(seed)
                _log(topic, f"Seed „{seed}“ als Subbaustein eingefügt ({title}) — Facts-Gate prüft")

    # AFTER the seed guarantee: promoted/inserted seeds must not bypass the near-dup filter
    await _dedup_subblocks(topic, raw)  # near-dup filter per block (deterministic, no LLM)

    if not raw:
        # Finders ran but nothing survived the consensus/evidence gates: a legitimately
        # thin block (e.g. a bare named reduction). {} = done-without-subs — the guide
        # writes it from description+facts. Agent FAILURES return None elsewhere (retry).
        _log(topic, "Subblocks: nichts Belegbares gefunden — Block bleibt ohne Subbausteine")
        return {}
    return raw


async def _dedup_subblocks(topic: str, raw: dict[str, list[str]]) -> None:
    """Deterministic near-duplicate filter per block: subblocks with cosine ≥
    EMBEDDING_SUB_DUP are the same statement (reliable in the narrow block context — no LLM
    needed). Per duplicate group keeps the most informative (longest); rest → DB discarded + out of `roh`.
    Model missing → silently skip (like the rest of the embedding fallback)."""
    if not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
        return
    for title, subs in list(raw.items()):
        if len(subs) < 2:
            continue
        sims = await asyncio.to_thread(embedding.embed_sims, subs)
        if sims is None:
            return
        keepers: list[int] = []
        discarded: list[int] = []
        negs = [_neg_set(s) for s in subs]
        for i in sorted(range(len(subs)), key=lambda x: (-len(subs[x]), x)):  # most informative first
            if any(float(sims[i][j]) >= EMBEDDING_SUB_DUP and negs[i] == negs[j] for j in keepers):
                discarded.append(i)
            else:
                keepers.append(i)
        if not discarded:
            continue
        bnorm = _norm_title(title)
        for i in discarded:
            await db.set_subblock_fields(topic, bnorm, _norm_title(subs[i]), status="discarded")
        raw[title] = [subs[i] for i in sorted(keepers)]  # original order of the kept ones


_KONSOLIDIERUNG_PANEL = 2  # merge needs unanimity — single judges over-merge (blocks-dedup lesson)


def _kons_id(x, n: int) -> int | None:
    """Judge id → int in 1..n, else None (bools are not ids)."""
    if isinstance(x, bool):
        return None
    if isinstance(x, str) and x.isdigit():
        x = int(x)
    return x if isinstance(x, int) and 1 <= x <= n else None


def _konsolidierung_schema(data, n: int) -> dict | None:
    """Judge output → normalized dict, else None. gruppen accepts the {"haupt": 1,
    "weitere": [4]} form AND the legacy plain-list form [1, 4] (resume files of the
    first template version). kataloge/fremd/luecken are optional."""
    if not isinstance(data, dict) or not isinstance(data.get("gruppen"), list):
        return None

    def _ids(lst):
        return sorted({i for x in (lst or []) if (i := _kons_id(x, n)) is not None})

    gruppen = []
    for g in data["gruppen"]:
        if isinstance(g, dict):
            haupt = _kons_id(g.get("haupt"), n)
            ids = _ids(([haupt] if haupt else []) + list(g.get("weitere") or []))
        elif isinstance(g, list):
            haupt, ids = None, _ids(g)
        else:
            return None
        if len(ids) >= 2:
            gruppen.append({"haupt": haupt if haupt in ids else None, "ids": ids})
    kataloge = []
    for k in data.get("kataloge") or []:
        if not isinstance(k, dict):
            continue
        ids = _ids(k.get("mitglieder"))
        titel = str(k.get("titel") or "").strip()
        if len(ids) >= 2 and titel:
            kataloge.append({"titel": titel, "ids": ids})
    return {"gruppen": gruppen, "kataloge": kataloge, "fremd": set(_ids(data.get("fremd"))),
            "luecken": [s.strip() for s in data.get("luecken") or [] if isinstance(s, str) and s.strip()]}


def _facts_union(wf: dict, lf: dict) -> None:
    """Merge a folded sub's facts into the winner's: key_points/cited_facts union
    (exact-duplicate-free), scalar fields only fill gaps."""
    for feld in ("key_points", "cited_facts"):
        have = wf.get(feld) or []
        seen = {json.dumps(e, sort_keys=True, ensure_ascii=False) for e in have}
        fresh = [e for e in (lf.get(feld) or [])
                 if json.dumps(e, sort_keys=True, ensure_ascii=False) not in seen]
        if fresh:
            wf[feld] = have + fresh
    for feld in ("prerequisites", "hurdles", "example_idea"):
        if not wf.get(feld) and lf.get(feld):
            wf[feld] = lf[feld]


def _agreed_cliques(pair_sets: list[set], negs: list, n: int) -> list[list[int]]:
    """Union-find over the UNANIMOUS pairs (both judges grouped them), negation-guarded."""
    agreed = {(a, b) for a, b in pair_sets[0] & pair_sets[1] if negs[a - 1] == negs[b - 1]}
    parent = list(range(n + 1))

    def find(x):
        while parent[x] != x:
            parent[x] = parent[parent[x]]
            x = parent[x]
        return x

    for a, b in agreed:
        parent[find(a)] = find(b)
    groups: dict[int, list[int]] = {}
    for k in range(1, n + 1):
        groups.setdefault(find(k), []).append(k)
    return [g for g in groups.values() if len(g) >= 2]


def _pairs_of(groups) -> set:
    ps: set[tuple[int, int]] = set()
    for ids in groups:
        ps |= {(a, b) for x, a in enumerate(ids) for b in ids[x + 1:]}
    return ps


_LUECKEN_CAP = 3  # the gap list feeds ONE finder round — an uncapped list doubled the decomposition


def _luecken_schnitt(l1: list[str], l2: list[str], cap: int = _LUECKEN_CAP) -> list[str]:
    """Gaps BOTH judges name — exact strings never match across paraphrases, so a gap
    survives when the other judge names one sharing a distinctive token (≥4 chars).
    j1's phrasing wins. The measured union produced 107 'gaps' on 216 subs."""
    def toks(s):
        return {t for t in _sub_tokens(s) if len(t) >= 4}
    toks2 = [toks(l) for l in l2]
    out = [l for l in l1 if toks(l) and any(toks(l) & t2 for t2 in toks2)]
    return out[:cap]


async def _konsolidiere_subblocks(ctx: GenContext, files: dict, raw: dict, facts_map: dict,
                                  instructions: str = "", ns: str = "", lbl: str = "") -> dict:
    """In-block consolidation AFTER the facts stage: a two-judge panel sees the subs WITH
    their key points and applies the 100%-decomposition test — the embedding paths only
    catch cos ≥ 0.90, real paraphrase duplicates measure down to 0.61, and only the facts
    reveal a subset. Every action needs UNANIMITY of both judges:
      gruppen  — same-statement/subset entries fold into the judge-named `haupt` (base
                 before detail; heuristic fallback), facts union, losers → `variant`
      kataloge — pure enumeration entries of one kind bundle into a NEW named sub row
                 (members → `variant`); runs before levels/relevance, so the new row
                 gets classified normally
      fremd    — statements off-topic for the TOPIC → `discarded` (removal test)
    Questions/artefacts do not exist yet — no orphans. Gaps are returned per block so the
    caller can run the single follow-up finder round (`_luecken_runde`).
    Judge replies persist as j-files keyed by a subs-list hash (resume-safe).
    → {block title: [luecken]}"""
    topic = ctx.topic
    work_dir = files["arbeit"]
    luecken_by_title: dict[str, list[str]] = {}
    for title, subs in list(raw.items()):
        if ctx.is_cancelled():
            return luecken_by_title
        n = len(subs)
        if n < 2:
            continue
        bnorm = _norm_title(title)
        bfacts = facts_map.setdefault(title, {})

        def _kp(s):
            return (bfacts.get(_norm_title(s)) or {}).get("key_points") or []

        # prompt shows max 3 key points per sub — full lists blew past the judge timeout
        # (measured: 15 % timeouts at 585 s); the facts UNION on merge stays complete
        lines = "\n".join(f"{k}. {s}" + "".join(f"\n   - {p}" for p in _kp(s)[:3])
                          for k, s in enumerate(subs, 1))
        h = hashlib.md5("\n".join(subs).encode()).hexdigest()[:8]
        paths = [work_dir / f"sub-konsolidierung-{ns}{h}-j{j}.json" for j in (1, 2)]

        async def _judge(j, path):
            if _konsolidierung_schema(_json_file(path), n) is not None:
                return  # resume
            status, _v = await run_single_slot(
                ctx, f"{lbl}Sub-Konsolidierung j{j}",
                key=f"blocks-{topic}-{ns}sub-konsolidierung-{h}-j{j}",
                prompt=_prompt("Subblock-Konsolidierung", topic=topic, block=title, subs=lines, extra=_extra(instructions)),
                role="judge", capabilities="none",
                payload=lambda result, p=path: _sink_json(result, p, lambda d: _konsolidierung_schema(d, n)),
                timeout=_timeout("konsolidierung", n))
            if status == FAILED:
                _log(topic, f"Sub-Konsolidierung {title} j{j} ohne Ergebnis — fail-open")

        await asyncio.gather(*[_judge(j, p) for j, p in zip((1, 2), paths)])
        if ctx.is_cancelled():
            return luecken_by_title
        outs = [o for p in paths if (o := _konsolidierung_schema(_json_file(p), n)) is not None]
        if len(outs) == 1:  # Ersatz-Richter: EIN Timeout darf die gute Stimme nicht entwerten
            ersatz = work_dir / f"sub-konsolidierung-{ns}{h}-j3.json"
            await _judge(3, ersatz)
            if ctx.is_cancelled():
                return luecken_by_title
            outs = [o for p in [*paths, ersatz]
                    if (o := _konsolidierung_schema(_json_file(p), n)) is not None]
        # gaps need UNANIMITY (token-overlap match) — the union of both judges was uncalibrated
        luecken = (_luecken_schnitt(outs[0]["luecken"], outs[1]["luecken"])
                   if len(outs) == _KONSOLIDIERUNG_PANEL else [])
        journal = {"block": title, "richter": len(outs), "vorher": n,
                   "luecken_roh": [len(o["luecken"]) for o in outs],
                   "gruppen": [], "kataloge": [], "fremd": [], "luecken": luecken}
        if len(outs) == _KONSOLIDIERUNG_PANEL:
            negs = [_neg_set(s) for s in subs]
            keep = list(subs)
            gone: set[int] = set()

            async def _fold(k: int, wf: dict | None):
                lose_title = subs[k - 1]
                lf = bfacts.pop(_norm_title(lose_title), None) or {}
                if wf is not None:
                    _facts_union(wf, lf)
                await db.set_subblock_fields(topic, bnorm, _norm_title(lose_title), status="variant")
                keep.remove(lose_title)
                gone.add(k)

            # 1. Fremd (removal test): off-topic for the TOPIC → discarded, no heir.
            for k in sorted(outs[0]["fremd"] & outs[1]["fremd"]):
                ft = subs[k - 1]
                bfacts.pop(_norm_title(ft), None)
                await db.set_subblock_fields(topic, bnorm, _norm_title(ft), status="discarded")
                keep.remove(ft)
                gone.add(k)
                journal["fremd"].append(ft)

            # 2. Gruppen: winner = judge-named haupt (majority), else key_points/length heuristic.
            haupt_votes: dict[int, int] = {}
            for o in outs:
                for g in o["gruppen"]:
                    if g["haupt"]:
                        haupt_votes[g["haupt"]] = haupt_votes.get(g["haupt"], 0) + 1
            for g in _agreed_cliques([_pairs_of([x["ids"] for x in o["gruppen"]]) for o in outs], negs, n):
                g = [k for k in g if k not in gone]
                if len(g) < 2:
                    continue
                win = max(g, key=lambda k: (haupt_votes.get(k, 0),
                                            len(_kp(subs[k - 1])), len(subs[k - 1]), -k))
                wf = bfacts.setdefault(_norm_title(subs[win - 1]), {})
                for k in g:
                    if k != win:
                        await _fold(k, wf)
                journal["gruppen"].append({"behalten": subs[win - 1],
                                           "gefaltet": [subs[k - 1] for k in g if k != win]})

            # 3. Kataloge: bundle enumeration rows into ONE new named sub (facts union).
            for g in _agreed_cliques([_pairs_of([x["ids"] for x in o["kataloge"]]) for o in outs], negs, n):
                g = [k for k in g if k not in gone]
                if len(g) < 2:
                    continue
                titel = next((clean_title(x["titel"]) for x in outs[0]["kataloge"] + outs[1]["kataloge"]
                              if set(x["ids"]) & set(g) and clean_title(x["titel"])), "")
                kn = _norm_title(titel)
                if not kn or kn in {_norm_title(s) for s in keep}:
                    continue  # no usable/colliding title → members stay
                kf: dict = {}
                for k in g:
                    await _fold(k, kf)
                bfacts[kn] = kf
                keep.append(titel)
                await db.put_subblock(topic, bnorm, kn, title, titel, status="consensus")
                journal["kataloge"].append({"titel": titel, "gefaltet": [subs[k - 1] for k in g]})

            if len(keep) != n:
                raw[title] = keep
                _log(topic, f"Sub-Konsolidierung {title}: {n} → {len(keep)}")
        elif outs:
            _log(topic, f"Sub-Konsolidierung {title}: nur {len(outs)}/{_KONSOLIDIERUNG_PANEL} Richter — fail-open")
        if luecken:
            luecken_by_title[title] = luecken
            _log(topic, f"Sub-Konsolidierung {title}: mögliche Lücken: {', '.join(luecken[:5])}")
        atomic_write_json(work_dir / f"sub-konsolidierung-{ns}{h}.json", journal, indent=1)
    return luecken_by_title


async def _luecken_runde(ctx: GenContext, files: dict, title: str, luecken: list[str],
                         raw: dict, facts_map: dict, q: dict, folder, instructions: str = "",
                         ns: str = "", lbl: str = "", sources: list[str] | None = None) -> int:
    """ONE targeted finder round for the consolidation judges' reported gaps — no loop.
    Finds are deduped against the existing subs (token containment + embedding +
    negation guard, seed-guarantee pattern) and must pass the facts evidence gate
    (own work subdir `nf` — the block's facts resume files must not collide) before
    they join raw/facts_map as consensus rows. They then flow through levels/relevance/
    questions/artefacts like any other sub. → count of adopted subs."""
    topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
    work_dir = files["arbeit"]
    have = list(raw.get(title) or [])
    focus = (instructions + "\n\nFinde NUR belegbare Subbausteine zu diesen bisher fehlenden "
             "Aspekten des Blocks — nichts anderes:\n" + "\n".join(f"- {l}" for l in luecken))
    known = ("\n\nBEREITS ERFASST — liste diese NICHT erneut:\n"
             + "\n".join(f"- {s}" for s in have)) if have else ""
    paths = [work_dir / f"luecken-{ns}r1-{i}.md" for i in (1, 2, 3)]
    slots = [{
        "key": f"blocks-{topic}-{ns}luecken-r1-{i}",
        "prompt": _prompt("Subblock-Research", topic=topic, assignment=f"- {title}", known=known, out_path=p, extra=_extra(focus)),
        "role": "quick", "capabilities": "files" if folder else "full",
        "payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None),
    } for i, p in zip((1, 2, 3), paths)]
    agent_texts = await _race(topic, f"{lbl}Lücken-Nachfass", slots, 2,
                              _timeout("subblock", 1), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
    if is_cancelled() or not agent_texts:
        return 0
    cands: list[str] = []
    seen = {_norm_title(s) for s in have}
    for d in agent_texts:
        for subs in d.values():  # single-block call — every marker means this block
            for s in subs:
                sn = _norm_title(s)
                if sn and sn not in seen:
                    seen.add(sn)
                    cands.append(s)
    if not cands:
        return 0
    emb_on = EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available)
    fresh: list[str] = []
    for s in cands:
        st = _sub_tokens(s)
        base = have + fresh
        if any(st <= _sub_tokens(b) or _sub_tokens(b) <= st for b in base):
            continue
        if emb_on and base:
            sims = await asyncio.to_thread(embedding.embed_sims, [s] + base)
            if sims is not None:
                negs = [_neg_set(t) for t in [s] + base]
                if any(float(sims[0][j]) >= SEED_COVER_COS and negs[0] == negs[j]
                       for j in range(1, len(base) + 1)):
                    continue
        fresh.append(s)
    if not fresh:
        return 0
    nf_dir = work_dir / "nf"
    nf_dir.mkdir(parents=True, exist_ok=True)
    res = await _facts_block(ctx, lambda *a, **k: None, {**files, "arbeit": nf_dir},
                             {title: list(fresh)}, q, folder, instructions,
                             ns=f"{ns}nf-", lbl=lbl, sources=sources, slim=True)
    if is_cancelled() or res is None:
        return 0
    nf_facts, discarded = res
    dropped = (discarded or {}).get(title) or set()
    nf_map = nf_facts.get(title) or {}

    def _belegt(s: str) -> bool:  # HARD gate: no facts entry = no evidence = no adoption
        fk = nf_map.get(_norm_title(s))
        return bool(fk and (fk.get("key_points") or fk.get("cited_facts")))

    kept = [s for s in fresh if _norm_title(s) not in dropped and _belegt(s)]
    if not kept:
        return 0
    bnorm = _norm_title(title)
    bfacts = facts_map.setdefault(title, {})
    for s in kept:
        sn = _norm_title(s)
        await db.upsert_subblock(topic, bnorm, sn, title, s)
        await db.set_subblock_fields(topic, bnorm, sn, status="consensus")
        bfacts[sn] = nf_map[sn]
    raw.setdefault(title, []).extend(kept)
    _log(topic, f"Lücken-Nachfass {title}: {len(kept)}/{len(fresh)} Funde übernommen")
    return len(kept)


def _subs_hash(sidecar_or_raw: dict) -> str:
    """Sub-set identity for the resume files of the sub-CONSUMING stages (levels/relevance/
    questions/artefacts). Without it a re-run with a recut sub set adopted the stale stage
    results (measured: 626 orphans — artefacts of the old 425-sub set re-imported)."""
    parts: list[str] = []
    for title, subs in sidecar_or_raw.items():
        parts.append(str(title))
        for s in subs:
            parts.append(s["title"] if isinstance(s, dict) else str(s))
    return hashlib.md5("\n".join(parts).encode()).hexdigest()[:8]


def _code_vote(rater: list[dict], n: int) -> tuple[dict, dict]:
    """Majority vote over rater dicts on local ids 1..n → (outcome, disputed). A clear winner
    needs ≥2 votes and no tie; otherwise the id is disputed (kept with its vote list)."""
    outcome: dict[int, str] = {}
    disputed: dict[int, list[str]] = {}
    for k in range(1, n + 1):
        vote_list = [d[k] for d in rater if k in d]
        counter: dict[str, int] = {}
        for s in vote_list:
            counter[s] = counter.get(s, 0) + 1
        best = max(counter.values(), default=0)
        winners = [s for s, v in counter.items() if v == best]
        if len(winners) == 1 and best >= 2:
            outcome[k] = winners[0]
        else:
            disputed[k] = vote_list
    return outcome, disputed


def _disputed_lines(items, item_idxs, disputed: dict) -> str:
    """Render disputed items as `k. [block] sub — Stimmen: a, b` lines for the judge prompt."""
    return "\n".join(
        f"{k}. [{items[item_idxs[k - 1]][0]}] {items[item_idxs[k - 1]][1]} — Stimmen: {', '.join(vote_list) or 'none'}"
        for k, vote_list in disputed.items()
    )


async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None:
    """Block C: three phases with a barrier — find (classify), select (vote), clarify.
    Local IDs 1..n per package, mapped to global gid afterwards.
    → {block title: [{title, level}, …]} or None."""
    topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
    work_dir = files["arbeit"]
    # key points per sub as concise context (better-grounded classification; classification needs little).
    facts_map = _json_file(files["facts"])
    facts_map = facts_map if isinstance(facts_map, dict) else {}
    items = [(title, sub) for title, subs in raw.items() for sub in subs]  # global id = index+1
    if not items:
        return {title: [] for title in raw}
    # pack chunks from WHOLE blocks (don't split a block) → each rater sees per block
    # all subs and can classify relatively. Item indices per block in raw order.
    chunks, cur, i = [], [], 0
    for _title_b, subs in raw.items():
        g = list(range(i, i + len(subs)))
        i += len(subs)
        if cur and len(cur) + len(g) > LEVEL_CHUNK:
            chunks.append(cur)
            cur = []
        cur.extend(g)
    if cur:
        chunks.append(cur)
    n = len(chunks)
    sh = _subs_hash(raw)  # resume must invalidate when the sub set changed

    def rater_paths(c):
        return [work_dir / f"level-{sh}-c{c}-{i}.json" for i in (1, 2, 3)]

    def lset(item_idxs):
        return set(range(1, len(item_idxs) + 1))

    # Phase "Levels find": 3 raters per package (min. 2), local IDs.
    async def _rate(c, item_idxs):
        local_set = lset(item_idxs)
        paths = rater_paths(c)
        existing = sum(1 for p in paths if _levels_schema(_json_file(p), local_set))
        if existing >= 2:
            return True
        enum_lines, cur_b = [], None
        for k, j in enumerate(item_idxs, 1):
            b, sub = items[j]
            if b != cur_b:
                enum_lines.append(f"\nBAUSTEIN: {b}")
                cur_b = b
            enum_lines.append(f"{k}. {sub}")
            if (kz := _core_line(facts_map.get(b, {}).get(_norm_title(sub)))):
                enum_lines.append(f"   {kz}")
        enum = "\n".join(enum_lines).strip()
        pending = [(i, p) for i, p in enumerate(paths, 1) if not _levels_schema(_json_file(p), local_set)]
        slots = [{
            "key": f"blocks-{topic}-{ns}level-c{c}-{i}",
            "prompt": _prompt("Levels-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)),
            "role": "quick", "capabilities": "none",  # pure mapping, everything inline → text reply
            "payload": (lambda result, p=p, ids=local_set: _sink_json(result, p, lambda d: _levels_schema(d, ids))),
        } for i, p in pending]
        new = await _race(topic, f"{lbl}Levels package {c}", slots, 2 - existing, _timeout("level", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
        return not is_cancelled() and new is not None

    oks = await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Levels find"))
    if is_cancelled():
        return None
    if not all(ok is True for ok in oks):
        _blocks_errors[topic] = "Classification failed (research)"
        return None

    # Phase "Levels select": code vote per package → (outcome, disputed).
    set_p(f"Levels select ({n} packages)…", step=_step_idx(topic, "Levels select"))
    vote_by_c = {}
    for c, item_idxs in enumerate(chunks, 1):
        local_set = lset(item_idxs)
        rater = [d for p in rater_paths(c) if (d := _levels_schema(_json_file(p), local_set))]
        vote_by_c[c] = _code_vote(rater, len(item_idxs))

    # Phase "Levels clarify": one judge per package on the disputed items, all in parallel.
    async def _clarify(c, item_idxs):
        outcome, strittig = vote_by_c[c]
        if strittig:
            judge_path = work_dir / f"level-final-{sh}-c{c}.json"
            decision = _levels_schema(_json_file(judge_path), set(strittig))
            if decision is None:
                disputed_block = _disputed_lines(items, item_idxs, strittig)
                status, decision = await run_single_slot(
                    ctx, f"{lbl}Levels-Clarification {c}",
                    key=f"blocks-{topic}-{ns}level-final-c{c}",
                    prompt=_prompt("Levels-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)),
                    role="judge", capabilities="none",
                    payload=lambda result, p=judge_path, ids=set(strittig): _sink_json(result, p, lambda d: _levels_schema(d, ids)),
                    timeout=_timeout("level_check", len(strittig)),
                )
                if status == FAILED:
                    _log(topic, f"Levels clarification package {c} failed — default 'advanced'")
                decision = decision if isinstance(decision, dict) else {}
            # disputed without a decision → 'advanced'; vote winners stay; judge overrides.
            outcome = {**{k: "advanced" for k in strittig}, **outcome, **decision}
        return {item_idxs[k - 1] + 1: level for k, level in outcome.items()}

    parts = await _gather_progress([_clarify(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Levels clarify"))
    if is_cancelled():
        return None
    level_by_id: dict[int, str] = {}
    for c, part in enumerate(parts, 1):
        if not isinstance(part, dict):
            # clarification is not fatal: vote outcome + default 'advanced' for disputed.
            if isinstance(part, BaseException):
                _log(topic, f"Levels clarification package {c}: {type(part).__name__}: {part}")
            outcome, strittig = vote_by_c[c]
            item_idxs = chunks[c - 1]
            merged = {**{k: "advanced" for k in strittig}, **outcome}
            part = {item_idxs[k - 1] + 1: s for k, s in merged.items()}
        level_by_id.update(part)

    # assemble the sidecar — same order as items → gid matches
    sidecar: dict[str, list[dict]] = {}
    gid = 0
    for title, subs in raw.items():
        lst = []
        for sub in subs:
            gid += 1
            lst.append({"title": sub, "level": level_by_id.get(gid, "advanced")})
        sidecar[title] = lst
    return sidecar


_FACTS_FIELDS = ("key_points", "prerequisites", "hurdles", "cited_facts", "example_idea")


def _facts_schema(data) -> list[dict] | None:
    """{"facts": [{block, subblock, …}]} → valid list · otherwise None.
    Strictly separates belegte_facts (with source) from example_idee (generative)."""
    if not isinstance(data, dict) or not isinstance(data.get("facts"), list):
        return None
    out = []
    for e in data["facts"]:
        if not isinstance(e, dict):
            continue
        blk = str(e.get("block", "")).strip()
        sub = str(e.get("subblock", "")).strip()
        if not blk or not sub:
            continue
        bf = [{"text": t, "source": str(f.get("source", "")).strip()}
              for f in (e.get("cited_facts") or []) if isinstance(f, dict) and (t := str(f.get("text", "")).strip())]
        out.append({
            "block": blk, "subblock": sub,
            "key_points": [k for x in (e.get("key_points") or []) if (k := str(x).strip())],
            "prerequisites": str(e.get("prerequisites", "")).strip(),
            "hurdles": str(e.get("hurdles", "")).strip(),
            "cited_facts": bf,
            "example_idea": str(e.get("example_idea", "")).strip(),
        })
    return out or None


def _facts_check_schema(data) -> list[tuple[str, bool]] | None:
    """Facts check → [(sub_norm, verwerfen)] per objection · {ok:true}→[] · None if invalid.
    verwerfen=True: sub not supportable in substance (remove). verwerfen=False: only correct the fact."""
    if not isinstance(data, dict):
        return None
    if data.get("ok") is True:
        return []
    pr = data.get("problems")
    if not isinstance(pr, list):
        return None
    return [(sn, bool(p.get("discard")))
            for p in pr if isinstance(p, dict) and (sn := _norm_title(str(p.get("subblock", ""))))]


def _core_line(fk) -> str:
    """Concise key-point line for classification (level/relevance) — less context suffices there.
    Empty if no facts/key points (legacy)."""
    if not isinstance(fk, dict) or not fk.get("key_points"):
        return ""
    return "Kern: " + " · ".join(str(k) for k in fk["key_points"])


def _facts_lines(fk: dict) -> str:
    z = []
    if fk.get("key_points"):
        z.append("Kernpunkte: " + " · ".join(str(k) for k in fk["key_points"]))
    if fk.get("prerequisites"):
        z.append("Voraussetzung: " + fk["prerequisites"])
    if fk.get("hurdles"):
        z.append("Hürde: " + fk["hurdles"])
    for bf in fk.get("cited_facts", []):
        z.append(f"FAKT: {bf['text']} (Source: {bf.get('source', '?')})")
    if fk.get("example_idea"):
        z.append("Example: " + fk["example_idea"])
    return "\n".join(z)


async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str, ns: str = "", lbl: str = "", sources: list[str] | None = None, slim: bool = False) -> tuple | None:
    """Block: per sub extract source facts (find) → verify (check) → correct/discard (fix).
    Extract-once grounding: the result feeds level/relevance/questions/guide.
    slim=True (gap follow-up): no supplement pass, ONE check judge — the full program cost
    230 agent-minutes per run for a handful of finds; the hard adoption gate stays.
    → (facts_map, discarded_map) — facts_map {block: {sub_norm: facts}}, discarded_map
      {block: {sub_norm}} (unsupportable subs to remove) — or None on cancel/error."""
    topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
    work_dir = files["arbeit"]
    caps = "files" if folder else "full"
    type = q.get("type", "thema")
    source = _prompt(_SOURCE_TEMPLATE[type], project=folder) if type in _SOURCE_TEMPLATE else _prompt("Blocks-Source-Thema", topic=topic)
    blocks = [(title, [str(s).strip() for s in subs if str(s).strip()]) for title, subs in raw.items() if subs]
    if not blocks:
        return {}, {}
    chunks = _lpt_chunks([len(subs) for _, subs in blocks], FACTS_CHUNK_SUBS)
    sh = _subs_hash(raw)  # resume must invalidate when the sub set changed

    def raw_path(ci): return work_dir / f"facts-{sh}-c{ci}.json"
    def supp_path(ci): return work_dir / f"facts-erg-{sh}-c{ci}.json"
    def chk_path(ci, j): return work_dir / f"facts-check-{sh}-c{ci}-j{j}.json"
    def fix_path(ci): return work_dir / f"facts-fix-{sh}-c{ci}.json"
    def ctitle(idxs): return [blocks[i][0] for i in idxs]
    def block_text(idxs):
        return "\n\n".join(
            f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in blocks[i][1])
            for i in idxs)

    # raw facts of a chunk → {block: {sub_norm: {sub, …fields}}}, matched to chunk titles.
    def raw_map(ci, path):
        idxs = chunks[ci]
        rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
        ct = ctitle(idxs)
        out: dict[str, dict] = {}
        for e in _facts_schema(_json_file(path)) or []:
            bt = _match_sub(e["block"], ct)
            if bt not in rel_by:
                continue
            sub = _match_sub(e["subblock"], rel_by[bt])
            out.setdefault(bt, {})[_norm_title(sub)] = {"sub": sub, **{k: e[k] for k in _FACTS_FIELDS}}
        return out

    # union raw facts + completeness supplements (recall): only subs that exist in raw.
    def _chunk_facts(ci):
        raw = raw_map(ci, raw_path(ci))
        erg = raw_map(ci, supp_path(ci)) if supp_path(ci).exists() else {}
        if not erg:
            return raw
        for bt, fm in raw.items():
            ebt = erg.get(bt, {})
            for sn, fk in fm.items():
                ek = ebt.get(sn)
                if not ek:
                    continue
                seen = {str(k).strip().casefold() for k in fk.get("key_points", [])}
                for k in ek.get("key_points", []):
                    if str(k).strip().casefold() not in seen:
                        seen.add(str(k).strip().casefold())
                        fk["key_points"].append(k)
                seent = {bf["text"].strip().casefold() for bf in fk.get("cited_facts", [])}
                for bf in ek.get("cited_facts", []):
                    if bf["text"].strip().casefold() not in seent:
                        seent.add(bf["text"].strip().casefold())
                        fk["cited_facts"].append(bf)
                for f in ("prerequisites", "hurdles", "example_idea"):
                    if not fk.get(f) and ek.get(f):
                        fk[f] = ek[f]
        return raw

    # Phase "Facts find": 1 generator per chunk.
    async def _find(ci, idxs):
        fp = raw_path(ci)
        if _facts_schema(_json_file(fp)):
            return True
        subs_total = sum(len(blocks[i][1]) for i in idxs)
        status, _r = await run_single_slot(
            ctx, f"{lbl}Facts {ci}", key=f"blocks-{topic}-{ns}facts-c{ci}",
            prompt=_prompt("Facts-Research", topic=topic, source=source, blocks=block_text(idxs), out_path=fp, extra=_extra(instructions)),
            role="quick", capabilities=caps,
            payload=lambda result, p=fp: _facts_schema(_json_file(p)),
            timeout=_timeout("content", subs_total))
        return status != FAILED and _facts_schema(_json_file(fp)) is not None

    oks = await _gather_progress([_find(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts find"))
    if is_cancelled():
        return None
    if not any(ok is True for ok in oks):
        _blocks_errors[topic] = "Facts extraction failed"
        return None

    # Phase "Facts ergänzen" (recall): a targeted gap hunt per chunk looks for source-backed facts that
    # the single pass missed. Best-effort — never fails (no erg file → merge uses only raw).
    async def _supplement(ci, idxs):
        ep = supp_path(ci)
        if _facts_schema(_json_file(ep)):
            return
        per = raw_map(ci, raw_path(ci))
        if not per:
            return
        block = "\n\n".join(
            f"BLOCK: {bt}\nSUBBAUSTEINE (mit bereits erfassten Facts):\n" + "\n".join(
                f"- {fk['sub']}\n  Erfasst: " + ("; ".join(
                    list(fk.get("key_points", [])) + [bf["text"] for bf in fk.get("cited_facts", [])]) or "(nichts)")
                for fk in fm.values())
            for bt, fm in per.items())
        subs_total = sum(len(blocks[i][1]) for i in idxs)
        await run_single_slot(
            ctx, f"{lbl}Facts supplement {ci}", key=f"blocks-{topic}-{ns}facts-erg-c{ci}",
            prompt=_prompt("Facts-Supplement", topic=topic, source=source, blocks=block, out_path=ep, extra=_extra(instructions)),
            role="quick", capabilities=caps,
            payload=lambda result, p=ep: _facts_schema(_json_file(p)),
            timeout=_timeout("content", subs_total))

    if not slim:
        set_p("Facts supplement…", step=_step_idx(topic, "Facts find"))
        await _gather_progress([_supplement(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts find"))
        if is_cancelled():
            return None
    panel = (1,) if slim else (1, 2, 3)[:FACTS_CHECK_PANEL]
    min_discard = 1 if slim else 2

    # Phase "Facts check": FACTS_CHECK_PANEL judges per chunk. Two majority sets:
    # flagged (fact inaccurate → correct) and discard (sub not supportable → remove).
    async def _check(ci, idxs):
        per = _chunk_facts(ci)  # raw + supplements → panel verifies the union
        if not per:
            return ci, set(), set()
        facts_text = "\n\n".join(f"SUBBAUSTEIN: {fk['sub']}\n{_facts_lines(fk)}" for fm in per.values() for fk in fm.values())
        # Inline evidence: the EXACT cited regions (falls back to keyword excerpts) go into
        # the prompt; the judge answers as TEXT, the engine persists the check file.
        cites = [bf.get("source", "") for fm in per.values() for fk in fm.values()
                 for bf in fk.get("cited_facts", [])]
        fallback = [blocks[i][0] for i in idxs] + [fk["sub"] for fm in per.values() for fk in fm.values()]
        ev = _cited_evidence(folder, sources, cites, fallback) if folder else ""
        c_source = _prompt("Blocks-Source-Inline", excerpts=ev) if ev else source
        pending = [j for j in panel if _facts_check_schema(_json_file(chk_path(ci, j))) is None]
        rs = await asyncio.gather(*[
            run_agent(f"blocks-{topic}-{ns}facts-check-c{ci}-j{j}",
                _prompt("Facts-Check", topic=topic, source=c_source, facts=facts_text, out_path=chk_path(ci, j), extra=_extra(instructions)),
                _timeout("content_check", len(per)), provider=provider, role="judge",
                capabilities="none" if ev else caps,
                scope=topic, label=f"{lbl}Facts check {ci}/{j}")
            for j in pending], return_exceptions=True)
        for j, r in zip(pending, rs):
            if isinstance(r, tuple):
                _sink_json(r, chk_path(ci, j), _facts_check_schema)
        outs = [s for j in panel if (s := _facts_check_schema(_json_file(chk_path(ci, j)))) is not None]
        bvotes: dict[str, int] = {}
        vvotes: dict[str, int] = {}
        for s in outs:  # s = [(sub_norm, verwerfen)] of one judge
            gb, gv = set(), set()
            for sn, disc in s:
                if sn not in gb:
                    gb.add(sn); bvotes[sn] = bvotes.get(sn, 0) + 1
                if disc and sn not in gv:
                    gv.add(sn); vvotes[sn] = vvotes.get(sn, 0) + 1
        threshold = len(outs) / 2 if outs else 99
        flagged = {sn for sn, v in bvotes.items() if v > threshold}
        # Discarding is irreversible → stricter than flagging: majority AND ≥2 agreeing judges
        # (prevents deletion by a single vote when the panel is degraded). slim runs ONE judge
        # by design — there its single vote must be allowed to discard.
        to_discard = {sn for sn, v in vvotes.items() if v > threshold and v >= min_discard}
        return ci, flagged, to_discard

    check = await _gather_progress([_check(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts check"))
    if is_cancelled():
        return None
    flagged: dict[int, set] = {}
    to_discard: dict[int, set] = {}
    for r in check:
        if isinstance(r, tuple) and len(r) == 3:
            ci, b, v = r
            flagged[ci] = b
            to_discard[ci] = v

    # Phase "Facts fix": re-extract only CORRECTABLE ones (flagged without discard).
    correctable = {ci: (flagged.get(ci, set()) - to_discard.get(ci, set())) for ci in flagged}
    n_problem = sum(len(s) for s in correctable.values())
    if n_problem:
        set_p(f"Correcting facts ({n_problem})…", step=_step_idx(topic, "Facts fix"))
        async def _fix(ci):
            subs_norm = correctable.get(ci, set())
            if not subs_norm or _facts_schema(_json_file(fix_path(ci))):
                return
            idxs = chunks[ci]
            rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
            goal = []
            for bt, subs in rel_by.items():
                affected_subs = [s for s in subs if _norm_title(s) in subs_norm]
                if affected_subs:
                    goal.append(f"BLOCK: {bt}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in affected_subs))
            if not goal:
                return
            await run_single_slot(
                ctx, f"{lbl}Facts-Fix {ci}", key=f"blocks-{topic}-{ns}facts-fix-c{ci}",
                prompt=_prompt("Facts-Research", topic=topic, source=source, blocks="\n\n".join(goal), out_path=fix_path(ci), extra=_extra(instructions)),
                role="quick", capabilities=caps,
                payload=lambda result, p=fix_path(ci): _facts_schema(_json_file(p)),
                timeout=_timeout("content", len(subs_norm)))
        await _gather_progress([_fix(ci) for ci in correctable], len(correctable), _report_p(set_p, topic, "Facts fix"))
        if is_cancelled():
            return None

    # assemble: raw + fix overrides for corrected. Discarded subs out (+ report per block).
    outcome: dict[str, dict] = {}
    discarded_map: dict[str, set] = {}
    for ci in range(len(chunks)):
        per = _chunk_facts(ci)  # raw + supplements (recall); fix overrides only corrected
        fix = raw_map(ci, fix_path(ci)) if fix_path(ci).exists() else {}
        disc = to_discard.get(ci, set())
        for bt, fm in per.items():
            for sn, fk in fm.items():
                if sn in disc:
                    discarded_map.setdefault(bt, set()).add(sn)
                    continue
                winners = fix.get(bt, {}).get(sn, fk) if sn in correctable.get(ci, set()) else fk
                outcome.setdefault(bt, {})[sn] = {k: winners[k] for k in _FACTS_FIELDS}
    if discarded_map:
        _log(topic, f"Facts check discards {sum(len(s) for s in discarded_map.values())} unsupportable subblocks")
    return outcome, discarded_map


async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None:
    """Block D: three phases with a barrier — find (relevant/peripheral), select (vote), clarify.
    Items from the sidecar; local IDs 1..n per package → global gid.
    → {gid: relevance} or None on cancel/research error. Default on gap/dispute: 'relevant'."""
    topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
    work_dir = files["arbeit"]
    items = [(title, sub["title"], sub.get("facts")) for title, subs in sidecar.items() for sub in subs]  # global id = index+1
    if not items:
        return {}
    chunks = _chunk_nums(list(range(len(items))), _n_chunks(len(items), LEVEL_CHUNK))
    n = len(chunks)
    sh = _subs_hash(sidecar)  # resume must invalidate when the sub set changed

    def rater_paths(c):
        return [work_dir / f"relevance-{sh}-c{c}-{i}.json" for i in (1, 2, 3)]

    def lset(item_idxs):
        return set(range(1, len(item_idxs) + 1))

    # Phase "Relevance find": 3 raters per package (min. 2), local IDs.
    async def _rate(c, item_idxs):
        local_set = lset(item_idxs)
        paths = rater_paths(c)
        existing = sum(1 for p in paths if _relevance_schema(_json_file(p), local_set))
        if existing >= 2:
            return True
        enum_lines = []
        for k, j in enumerate(item_idxs, 1):
            enum_lines.append(f"{k}. [{items[j][0]}] {items[j][1]}")
            if (kz := _core_line(items[j][2])):
                enum_lines.append(f"   {kz}")
        enum = "\n".join(enum_lines)
        pending = [(i, p) for i, p in enumerate(paths, 1) if not _relevance_schema(_json_file(p), local_set)]
        slots = [{
            "key": f"blocks-{topic}-{ns}relevance-c{c}-{i}",
            "prompt": _prompt("Relevance-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)),
            "role": "quick", "capabilities": "none",  # pure mapping, everything inline → text reply
            "payload": (lambda result, p=p, ids=local_set: _sink_json(result, p, lambda d: _relevance_schema(d, ids))),
        } for i, p in pending]
        new = await _race(topic, f"{lbl}Relevance package {c}", slots, 2 - existing, _timeout("relevance", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
        return not is_cancelled() and new is not None

    oks = await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Relevance find"))
    if is_cancelled():
        return None
    if not all(ok is True for ok in oks):
        _blocks_errors[topic] = "Relevance failed (research)"
        return None

    # Phase "Relevance select": code vote per package → (outcome, disputed).
    set_p(f"Relevance select ({n} packages)…", step=_step_idx(topic, "Relevance select"))
    vote_by_c = {}
    for c, item_idxs in enumerate(chunks, 1):
        local_set = lset(item_idxs)
        rater = [d for p in rater_paths(c) if (d := _relevance_schema(_json_file(p), local_set))]
        vote_by_c[c] = _code_vote(rater, len(item_idxs))

    # Phase "Relevance clarify": one judge per package on the disputed items, all in parallel.
    async def _clarify(c, item_idxs):
        outcome, strittig = vote_by_c[c]
        if strittig:
            judge_path = work_dir / f"relevance-final-{sh}-c{c}.json"
            decision = _relevance_schema(_json_file(judge_path), set(strittig))
            if decision is None:
                disputed_block = _disputed_lines(items, item_idxs, strittig)
                status, decision = await run_single_slot(
                    ctx, f"{lbl}Relevance-Clarification {c}",
                    key=f"blocks-{topic}-{ns}relevance-final-c{c}",
                    prompt=_prompt("Relevance-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)),
                    role="judge", capabilities="none",
                    payload=lambda result, p=judge_path, ids=set(strittig): _sink_json(result, p, lambda d: _relevance_schema(d, ids)),
                    timeout=_timeout("relevance_check", len(strittig)),
                )
                if status == FAILED:
                    _log(topic, f"Relevance clarification package {c} failed — default 'relevant'")
                decision = decision if isinstance(decision, dict) else {}
            # disputed without a decision → 'relevant' (never accidentally exclude).
            outcome = {**{k: "relevant" for k in strittig}, **outcome, **decision}
        return {item_idxs[k - 1] + 1: rel for k, rel in outcome.items()}

    parts = await _gather_progress([_clarify(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Relevance clarify"))
    if is_cancelled():
        return None
    relevance_by_id: dict[int, str] = {}
    for c, part in enumerate(parts, 1):
        if not isinstance(part, dict):
            # clarification is not fatal: vote outcome + default 'relevant' for disputed.
            if isinstance(part, BaseException):
                _log(topic, f"Relevance clarification package {c}: {type(part).__name__}: {part}")
            outcome, strittig = vote_by_c[c]
            item_idxs = chunks[c - 1]
            merged = {**{k: "relevant" for k in strittig}, **outcome}
            part = {item_idxs[k - 1] + 1: s for k, s in merged.items()}
        relevance_by_id.update(part)
    return relevance_by_id


def _match_sub(agent_sub: str, rel: list[str]) -> str:
    """Map the agent's subblock title to the matching relevant title — exact,
    then normalized, then substring (the agent drops e.g. the prefix "Question: ").
    No match → keep the agent title. This way NO pattern is lost to a title mismatch."""
    if agent_sub in rel:
        return agent_sub
    an = _norm_title(agent_sub)
    for r in rel:
        rn = _norm_title(r)
        if an and rn and (an == rn or an in rn or rn in an):
            return r
    return agent_sub


async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None:
    """Block E (chunks of 10): find (1 generator per ~10 blocks, parallel), select (code:
    group per block + dedup), clarify (1 critic per chunk), check (catch-up round).
    Assignment per entry via the `block` field (a chunk file carries several blocks).
    → {block title: [{subblock, question}, …]} or None on cancel."""
    topic, is_cancelled = ctx.topic, ctx.is_cancelled
    work_dir = files["arbeit"]
    # ALL subblocks (including peripheral) get a pattern — peripheral is testable in the FGuide level.
    # facts_by: full facts context per sub (generation benefits — better questions).
    blocks = []
    facts_by: dict[tuple, dict] = {}
    for title, subs in sidecar.items():
        all_titles = []
        for s in subs:
            if isinstance(s, dict) and (st := str(s.get("title", "")).strip()):
                all_titles.append(st)
                if isinstance(s.get("facts"), dict):
                    facts_by[(title, _norm_title(st))] = s["facts"]
        if all_titles:
            blocks.append((title, all_titles))
    if not blocks:
        return {}
    chunks = _lpt_chunks([len(rel) for _, rel in blocks], QUESTION_CHUNK_SUBS)  # load-balanced by sub count
    sh = _subs_hash(sidecar)  # resume must invalidate when the sub set changed

    def raw_path(ci):
        return work_dir / f"question-pattern-{sh}-c{ci}.json"

    def final_path(ci):
        return work_dir / f"question-pattern-final-{sh}-c{ci}.json"

    def _chunk_title(idxs):
        return [blocks[i][0] for i in idxs]

    # Phase "Questions find": 1 generator per chunk, all in parallel.
    async def _find(ci, idxs):
        fp = raw_path(ci)
        if _question_pattern_chunk_schema(_json_file(fp)):
            return  # Resume
        def _sub_line(bi, s):
            line = f"- {s}"
            fk = facts_by.get((blocks[bi][0], _norm_title(s)))
            if fk and (ft := _facts_lines(fk)):
                line += "\n" + "\n".join("    " + l for l in ft.split("\n"))
            return line
        block = "\n\n".join(
            f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(_sub_line(i, s) for s in blocks[i][1])
            for i in idxs
        )
        subs_total = sum(len(blocks[i][1]) for i in idxs)
        status, _ = await run_single_slot(
            ctx, f"{lbl}Question-Pattern {ci}",
            key=f"blocks-{topic}-{ns}question-pattern-c{ci}",
            prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=block,
                           out_path=fp, extra=_extra(instructions)),
            role="quick", capabilities="files",
            payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
            timeout=_timeout("question_pattern", subs_total),
        )
        if status == FAILED:
            _log(topic, f"Question pattern chunk {ci} failed — blocks in fallback (catch-up round/live)")

    async def find_all(ci_list):
        ci_list = list(ci_list)
        await _gather_progress([_find(ci, chunks[ci]) for ci in ci_list], len(ci_list), _report_p(set_p, topic, "Questions find"))

    await find_all(range(len(chunks)))
    if is_cancelled():
        return None

    # Phase "Questions select": code — group chunk files per block, drop duplicates,
    # loosely map block/subblock titles to the targets (discard nothing for a mismatch).
    def _select_chunk(ci):
        idxs = chunks[ci]
        ctitle = _chunk_title(idxs)
        rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
        out, seen_set = {}, {}
        for e in _question_pattern_chunk_schema(_json_file(raw_path(ci))) or []:
            title = _match_sub(e["block"], ctitle)
            if title not in rel_by:
                continue  # not assignable → discard
            sub = _match_sub(e["subblock"], rel_by[title])
            seen = seen_set.setdefault(title, set())
            if sub in seen:
                continue  # exactly one pattern per subblock
            seen.add(sub)
            out.setdefault(title, []).append({"subblock": sub, "question": e["question"]})
        return out

    def _select_all(ci_list):
        raw = {}
        for ci in ci_list:
            for title, eintraege in _select_chunk(ci).items():
                raw.setdefault(title, []).extend(eintraege)
        return raw

    set_p("Questions select…", step=_step_idx(topic, "Questions select"))
    raw_by_title = _select_all(range(len(chunks)))

    # Phase "Questions clarify": 1 critic per chunk cleans up the tables (grouped by block).
    async def _clarify(ci, idxs):
        fp = final_path(ci)
        if _question_pattern_chunk_schema(_json_file(fp)):
            return  # resume
        block_texts = []
        for i in idxs:
            t = blocks[i][0]
            eintraege = raw_by_title.get(t) or []
            if not eintraege:
                continue
            lines = "\n".join(f"- ({e['subblock']}) {e['question']}" for e in eintraege)
            block_texts.append(f"BLOCK: {t}\n{lines}")
        if not block_texts:
            return  # nothing to clarify in this chunk
        subs_total = sum(len(blocks[i][1]) for i in idxs)
        status, _ = await run_single_slot(
            ctx, f"{lbl}Question-Pattern-Clarification {ci}",
            key=f"blocks-{topic}-{ns}question-pattern-final-c{ci}",
            prompt=_prompt("Question-Pattern-Critique", topic=topic, table="\n\n".join(block_texts), out_path=fp, extra=_extra(instructions)),
            role="judge", capabilities="none",  # pure review, everything inline → text reply
            payload=lambda result, p=fp: _sink_json(result, p, _question_pattern_chunk_schema),
            timeout=_timeout("question_pattern_check", subs_total),
        )
        if status == FAILED:
            _log(topic, f"Question pattern clarification chunk {ci} failed — raw pattern adopted")

    async def clarify_all(ci_list):
        ci_list = list(ci_list)
        await _gather_progress([_clarify(ci, chunks[ci]) for ci in ci_list], len(ci_list), _report_p(set_p, topic, "Questions clarify"))

    await clarify_all(range(len(chunks)))
    if is_cancelled():
        return None

    # Clarified chunk table per block, fallback to raw pattern. Map titles loosely.
    def _final_by_title(ci_list):
        out = {}
        for ci in ci_list:
            idxs = chunks[ci]
            ctitle = _chunk_title(idxs)
            rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
            for e in _question_pattern_chunk_schema(_json_file(final_path(ci))) or []:
                title = _match_sub(e["block"], ctitle)
                if title not in rel_by:
                    continue
                out.setdefault(title, []).append(
                    {"subblock": _match_sub(e["subblock"], rel_by[title]), "question": e["question"]})
        return out

    final_by_title = _final_by_title(range(len(chunks)))
    outcome = {t: (final_by_title.get(t) or raw_by_title.get(t) or []) for t, _ in blocks}

    # Phase "Questions check": per-sub completeness. Generators crash randomly (~15 %),
    # 1 agent per chunk without retry → subs (whole blocks) fall through silently. Hence several
    # rounds that re-request ONLY the missing subs (short packages, Question-Pattern-Research).
    set_p("Questions check…", step=_step_idx(topic, "Questions check"))

    def _missing_subs() -> list[tuple[str, list[str]]]:
        out = []
        for t, subs in blocks:
            have_set = {_norm_title(e["subblock"]) for e in outcome.get(t) or []}
            miss = [s for s in subs if _norm_title(s) not in have_set]
            if miss:
                out.append((t, miss))
        return out

    def _followup_block(items):  # items: [(block_title, [missing sub_title])]
        block_texts = []
        for title, subs in items:
            lines = []
            for s in subs:
                z = f"- {s}"
                fk = facts_by.get((title, _norm_title(s)))
                if fk and (ft := _facts_lines(fk)):
                    z += "\n" + "\n".join("    " + l for l in ft.split("\n"))
                lines.append(z)
            block_texts.append(f"BLOCK: {title}\nSUBBAUSTEINE:\n" + "\n".join(lines))
        return "\n\n".join(block_texts)

    async def _request_more(round_n, pi, items):
        fp = work_dir / f"question-pattern-nach{round_n}-{sh}-c{pi}.json"
        if _question_pattern_chunk_schema(_json_file(fp)):
            return  # resume
        subs_total = sum(len(s) for _, s in items)
        await run_single_slot(
            ctx, f"{lbl}Question pattern catch-up R{round_n}/{pi}",
            key=f"blocks-{topic}-{ns}question-pattern-nach{round_n}-c{pi}",
            prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=_followup_block(items),
                           out_path=fp, extra=_extra(instructions)),
            role="quick", capabilities="files",
            payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
            timeout=_timeout("question_pattern", subs_total),
        )

    for round_n in range(1, QUESTION_MAX_ROUNDS + 1):
        missing_subs = _missing_subs()
        if not missing_subs:
            break
        n_subs = sum(len(s) for _, s in missing_subs)
        _log(topic, f"Question pattern round {round_n}: {n_subs} sub(s) in {len(missing_subs)} block(s) without a pattern — re-request")
        packages = _lpt_chunks([len(s) for _, s in missing_subs], QUESTION_CHUNK_SUBS)
        package_items = [[missing_subs[i] for i in idxs] for idxs in packages]
        await _gather_progress(
            [_request_more(round_n, pi, items) for pi, items in enumerate(package_items)],
            len(package_items), _report_p(set_p, topic, "Questions check"))
        if is_cancelled():
            return None
        # parse output per package + merge newly gained subs (don't overwrite existing ones).
        for pi, items in enumerate(package_items):
            title_subs = {t: subs for t, subs in items}
            ctitle = list(title_subs.keys())
            for e in _question_pattern_chunk_schema(_json_file(work_dir / f"question-pattern-nach{round_n}-{sh}-c{pi}.json")) or []:
                title = _match_sub(e["block"], ctitle)
                if title not in title_subs:
                    continue
                sub = _match_sub(e["subblock"], title_subs[title])
                have_set = {_norm_title(x["subblock"]) for x in outcome.get(title) or []}
                if _norm_title(sub) in have_set:
                    continue
                outcome.setdefault(title, []).append({"subblock": sub, "question": e["question"]})

    rest = _missing_subs()
    if rest:
        n = sum(len(s) for _, s in rest)
        _log(topic, f"Question pattern: {n} sub(s) in {len(rest)} block(s) remain empty after {QUESTION_MAX_ROUNDS} rounds: {[t for t, _ in rest][:5]}")
    return outcome


# ── Inventory in the DB: research loop · consolidation · clarification ────────────



def _crawl_index(folder) -> dict[str, str]:
    """Alias (filename OR QUELLE: URL, lowercase) → canonical page key (filename)."""
    idx: dict[str, str] = {}
    if not folder or not Path(folder).is_dir():
        return idx
    for p in sorted(Path(folder).glob("*.txt")):
        key = p.name
        idx[key.lower()] = key
        try:
            first_line = p.read_text(encoding="utf-8").splitlines()[0]
        except (OSError, IndexError):
            first_line = ""
        if first_line.startswith("QUELLE:"):
            url = first_line[len("QUELLE:"):].strip()
            if url:
                idx[url.lower()] = key
                idx[url.rstrip("/").lower()] = key
    return idx






def _triage_rules(folder, pages: list[str]) -> tuple[list[str], list[str]]:
    """Deterministic content/noise filter (config.CRAWL_*). Substring match (lowercase) against
    URL + filename. Order: keep > noise > min_chars > keep. → (content, noise)."""
    folder = Path(folder)
    content, noise = [], []
    for fn in pages:
        lines = _read(folder / fn).splitlines()
        url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else ""
        body = "\n".join(lines[1:]).strip()
        hay = f"{url}\n{fn}".lower()
        if any(p in hay for p in CRAWL_KEEP_PATTERNS):
            content.append(fn)
        elif any(p in hay for p in CRAWL_NOISE_PATTERNS):
            noise.append(fn)
        elif len(body) < CRAWL_MIN_CHARS:
            noise.append(fn)
        else:
            content.append(fn)  # default: keep — everything with content stays
    return content, noise


def _page_snippet(folder, fn: str) -> tuple[str, str]:
    """(url, snippet) of a crawl page for the relevance gate. url from the QUELLE: line;
    snippet = body excerpt (navigation boilerplate is up front — the prompt ignores it).
    The URL is the primary signal (meaningful slug), the snippet only supports it."""
    lines = _read(Path(folder) / fn).splitlines()
    url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else ""
    body = "\n".join(lines[1:]).strip()
    snippet = " ".join(body.split())[:QUELLE_RELEVANZ_SNIPPET]
    return (url or fn), snippet


async def _relevance_triage(ctx: GenContext, set_p, files: dict, folder, content: list[str], spec: str, instructions: str) -> tuple[list[str], list[str]]:
    """LLM topic gate after the rule filter: each content page ja/nein against the spec.
    Off-topic (different field) → out. Pattern like `_relevance_block`: small packages, 3 raters
    (`fast`), 2-of-3 consensus. CONSERVATIVE: drop only on a clear "nein" majority; dispute/gap/
    race error → keep. SAFETY: if the gate would drop ≥80 % (or all), everything stays
    (a spec mismatch/bug must not empty the source). → (kept, out) as filenames."""
    topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
    work_dir = files["arbeit"]
    pages = sorted(content)
    if not pages:
        return content, []
    items = [_page_snippet(folder, fn) for fn in pages]  # index aligns with `pages`
    chunks = _chunk_nums(list(range(len(pages))), _n_chunks(len(pages), QUELLE_RELEVANZ_CHUNK))
    n = len(chunks)

    def rater_paths(c):
        return [work_dir / f"source-relevance-c{c}-{i}.json" for i in (1, 2, 3)]

    def lset(idxs):
        return set(range(1, len(idxs) + 1))

    async def _rate(c, idxs):
        local_set = lset(idxs)
        paths = rater_paths(c)
        existing = sum(1 for p in paths if _yesno_schema(_json_file(p), local_set))
        if existing >= 2:
            return True
        enum_lines = []
        for k, j in enumerate(idxs, 1):
            url, snip = items[j]
            enum_lines.append(f"{k}. {url}")
            if snip:
                enum_lines.append(f"   {snip}")
        enum = "\n".join(enum_lines)
        pending = [(i, p) for i, p in enumerate(paths, 1) if not _yesno_schema(_json_file(p), local_set)]
        slots = [{
            "key": f"blocks-{topic}-source-relevance-c{c}-{i}",
            "prompt": _prompt("Source-Relevance", topic=topic, spec=spec, pages=enum, out_path=p, extra=_extra(instructions)),
            "role": "fast", "capabilities": "files",
            "payload": (lambda result, p=p, ids=local_set: _yesno_schema(_json_file(p), ids)),
        } for i, p in pending]
        new = await _race(topic, f"Relevance triage package {c}", slots, 2 - existing, _timeout("relevance", len(idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
        return not is_cancelled() and new is not None

    _qidx = _step_idx(topic, "Source prep")  # gate runs in the source step (no own step)
    set_p(f"Check relevance against spec ({n} packages)…", step=_qidx)
    async def _report_triage(d, t):
        set_p(f"Check relevance against spec {d}/{t}…", step=_qidx)
    await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], n, _report_triage)
    if is_cancelled():
        return content, []  # cancel → drop nothing (caller aborts)

    # Vote per page: only a clear "nein" majority (≥2 and more than "ja") throws it out.
    dropped: list[str] = []
    for c, idxs in enumerate(chunks, 1):
        local_set = lset(idxs)
        rater = [d for p in rater_paths(c) if (d := _yesno_schema(_json_file(p), local_set))]
        for k in range(1, len(idxs) + 1):
            vote_list = [d[k] for d in rater if k in d]
            nein, ja = vote_list.count("nein"), vote_list.count("ja")
            if nein >= 2 and nein > ja:
                dropped.append(pages[idxs[k - 1]])

    if dropped and len(dropped) >= max(1, int(len(pages) * 0.8)):
        _log(topic, f"Relevance triage: would drop {len(dropped)}/{len(pages)} — discarded (spec mismatch?), keeping all")
        return content, []
    dropped_set = set(dropped)
    keepers = [fn for fn in pages if fn not in dropped_set]
    return keepers, dropped


async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool:
    """Step "Source prep": crawl (link) + PDF convert + content/noise triage.
    Persists the triage in the coverage table (content). → True (ok) / False (cancel/error).
    thema: nothing. projekt/uni: only PDFs (curated folder, no triage)."""
    topic, is_cancelled = ctx.topic, ctx.is_cancelled
    if not folder:
        return True  # thema → no source to prepare
    if q["type"] != "link":
        await asyncio.to_thread(_convert_pdfs, folder)  # projekt/uni: only PDFs, no triage
        return True
    if await db.get_step_status(topic, "Source prep") == "done":
        return True
    if not _crawl_done(topic):
        set_p("Loading source (crawl)…", step=_step_idx(topic, "Source prep"))
        n = await asyncio.to_thread(crawl, q["location"], folder, cancelled=is_cancelled)
        if is_cancelled():
            return False
        if not n:
            _blocks_errors[topic] = "Crawl yielded no content — check link/domain"
            return False
    await asyncio.to_thread(_convert_pdfs, folder)
    pages = sorted(set(_crawl_index(folder).values()))
    if pages:
        set_p("Triaging pages…", step=_step_idx(topic, "Source prep"))
        await db.delete_coverage(topic)
        content, noise = _triage_rules(folder, pages)  # deterministic rule filter
        if q.get("spec") and content:  # topic gate: separates the field (rules can't)
            content, dropped = await _relevance_triage(ctx, set_p, files, folder, content, q["spec"], instructions)
            if is_cancelled():
                return False
            if dropped:
                noise = sorted(set(noise) | set(dropped))
                _log(topic, f"LLM relevance: {len(dropped)} pages off-topic → noise")
        await db.mark_content(topic, sorted(content), sorted(noise))
        _log(topic, f"Triage: {len(content)} content / {len(noise)} noise of {len(pages)} (rules + LLM gate)")
    await db.set_step_status(topic, "Source prep", "done")
    return True






_ASPECT_MARKER = ("∈ np", "∈np", " in np", "np-schwer", "np-vollständig", "verifizierer",
                  "zertifikat", "ndtm", "nicht-determ", "lower bound", "untere schranke",
                  "bzgl", "als sprache", "beweis", "güte", "austausch",
                  # generic bound/limit/runtime property stems (a "…-Grenze"/"…-Schranke"/"…-Laufzeit"
                  # is a property OF a concept, not the concept — MDM survivorship must never pick it as
                  # the representative, so it scores >0 here like the other aspect markers):
                  "grenze", "schranke", "laufzeit")


def _aspect_marker(title: str) -> int:
    """Number of property markers in the title (∈NP, NP-hard, verifier, lower bound …).
    0 = generic main concept (the problem itself); >0 = a property of it."""
    t = title.casefold()
    return sum(1 for m in _ASPECT_MARKER if m in t)


_REFERENCE_RE = re.compile(r'^(Satz|Lemma|Korollar|Bemerkung|Definition)\s*[\d.]+\s*(\([a-z]\)|[a-z])?\s*$', re.I)


def _is_reference(title: str) -> bool:
    """True for pure reference/placeholder titles WITHOUT meaningful content: "Satz 7.18", "Lemma 6.2",
    "Korollar 6.18" (number without a name) as well as marked spots "Bedingung (**)". NOT "Satz 6.24:
    Cook/Levin" (has a name) and NOT short technical symbols like "P⊆NP"/"Σ*" (real concepts)."""
    t = title.strip()
    if _REFERENCE_RE.match(t):
        return True
    if re.search(r'\(\*+\)', t):                       # marked spot "(**)" / "(*)"
        return True
    return False


def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) -> dict:
    """Representative of a cluster = the main concept (fewest property markers — the problem
    itself, not "… ∈ NP"); tie → most frequent norm title → most readers. Title globally unique
    (suffix ' (2)') so it works as a key."""
    by_norm: dict[str, list[int]] = {}
    for k in idxs:
        by_norm.setdefault(_norm_title(candidates[k]["title"]), []).append(k)

    def weight(nb: str):
        ms = by_norm[nb]
        reader = set().union(*[set(candidates[m]["reader"]) for m in ms]) if ms else set()
        # reference/placeholder titles ("Satz 7.18") last — prefer a meaningful member.
        is_real = not _is_reference(candidates[ms[0]]["title"])
        return (is_real, -_aspect_marker(nb), len(ms), len(reader))

    best = max(by_norm, key=weight)
    k = max(by_norm[best], key=lambda m: len(candidates[m]["description"]))
    title = candidates[k]["title"]
    n = 2
    while _norm_title(title) in seen_norm:
        title = f"{candidates[k]['title']} ({n})"
        n += 1
    seen_norm.add(_norm_title(title))
    return {"title": title, "description": candidates[k]["description"]}












def _pairs_schema(data) -> dict[int, bool] | None:
    """{"pairs": {"1": "ja", "2": "nein", …}} → {pair_nr: True/False} · otherwise None."""
    if not isinstance(data, dict) or not isinstance(data.get("pairs"), dict):
        return None
    out: dict[int, bool] = {}
    for k, v in data["pairs"].items():
        try:
            nr = int(k)
        except (ValueError, TypeError):
            continue
        out[nr] = str(v).strip().casefold() in ("ja", "yes", "true", "1")
    return out or None


def _cliques(n: int, edge_list: list[tuple[int, int]]) -> list[list[int]]:
    """Complete-link: greedy maximal cliques over the confirmed duplicate edges. A group
    forms only if ALL its nodes are pairwise connected → no chaining (A=B + B=C forms
    NO group {A,B,C} as long as A=C is missing). Only cliques ≥2 are returned."""
    adj: dict[int, set[int]] = {i: set() for i in range(n)}
    for a, b in edge_list:
        adj[a].add(b)
        adj[b].add(a)
    used: set[int] = set()
    groups: list[list[int]] = []
    for v in sorted(range(n), key=lambda x: -len(adj[x])):
        if v in used or not adj[v]:
            continue
        clique = {v}
        for u in sorted(adj[v], key=lambda x: -len(adj[x])):
            if u not in used and clique <= adj[u] | {u}:  # u connected to ALL previous ones
                clique.add(u)
        if len(clique) >= 2:
            groups.append(sorted(clique))
            used |= clique
    return groups


# Canonical-name blocking (dedup recall): entity resolution's recall ceiling is set by candidate
# generation — a pair that never shares a candidate can never be merged. Normalize a title to a
# scaffolding-free, operator-class-normalized, order-independent key so surface variants of ONE entity
# collapse ("Offenes Problem P=NP?" ≡ "P vs NP" ≡ "P=NP"). Generic (no course terms): strip a small
# stoplist of catalogue/wrapper words, fold the relation operators into class tokens, sort content tokens.
_CANON_STOP = re.compile(
    r'\b(?:offenes?|open|problem|frage|question|algorithmus|algorithm|verfahren|method|methode|'
    r'satz|theorem|lemma|korollar|definition|def|das|der|die|the|ein|eine|einen|a|an|'
    r'von|of|für|for|und|and|zum|zur|im)\b', re.I)


# Catalogue references ("Definition 6.19", "Satz 7.8") are scaffolding INCLUDING their number —
# stripped as a phrase, so the digits don't pollute the key. Variant digits ("3-SAT") have no
# scaffolding word in front and survive.
_CANON_CATALOGUE = re.compile(
    r'\b(?:definition|def|satz|lemma|korollar|corollary|theorem|proposition'
    r'|kapitel|chapter|abschnitt|section)\s*\d+(?:\.\d+)*\b', re.I)


_PAREN_GROUP = re.compile(r'^\s*(.*?)\s*\(([^()]{2,60})\)\s*$')


def _title_variants(title: str) -> set[str]:
    """Acronym/expansion variants of a "X (Y)" title — normalized outer part and paren
    content. „Satisfiability Problem (SAT)" → {'satisfiability problem', 'sat'}: matched
    against another card's norm/key this makes acronym↔expansion pairs dedup CANDIDATES
    (measured: title cosine 'sat' vs the long form is 0.53, far below the floor).
    Titles without exactly one trailing paren group → empty set."""
    from textkit import _norm_title
    m = _PAREN_GROUP.match(title or "")
    if not m:
        return set()
    return {v for v in (_norm_title(m.group(1)), _norm_title(m.group(2))) if v}


def _canonical_key(title: str) -> str:
    """Order-independent canonical key of a title (scaffolding stripped, relation operators normalized).
    Two titles with the same key denote the same entity with ~100% precision (ER blocking). Empty string
    if nothing survives (never auto-merged)."""
    s = unicodedata.normalize("NFKC", title)
    s = re.sub(r'([a-zäöüß])([A-ZÄÖÜ])', r'\1 \2', s)                        # CamelCase → two tokens
    s = s.casefold()
    s = _CANON_CATALOGUE.sub(' ', s)
    s = re.sub(r'≟|\bversus\b|\bvs\.?\b|=', ' opeq ', s)                      # equality / "vs" → one token
    s = re.sub(r'≤|⪯|→|⇒|⟹|\breduces?\s+to\b|\breduziert\b', ' opred ', s)   # reduction → one token
    s = _CANON_STOP.sub(' ', s)
    s = re.sub(r'[^\w ]', ' ', s)                                            # drop punctuation/symbols
    return " ".join(sorted(t for t in s.split() if t))


# Relation-triple individuation (dedup precision): a reduction/relation "A ≤ B" is identified by BOTH
# operands AND direction (RDF-triple identity / SKOS narrowMatch — a subset/restriction is NOT the same).
# So "SAT ≤ Clique" ≠ "3-SAT ≤ Clique" (source differs) and "A → B" ≠ "B → A" (direction). Two DIFFERENT
# relations must never merge, even if a judge or a high title-cosine says so.
_REL_STRIP = re.compile(r'^\s*(?:satz|lemma|korollar|corollary|theorem|proposition)\s*[\d.]*\s*:?\s*'
                        r'|^\s*redu[ck]tion(?:en|s)?\s*:?\s*', re.I)
# trailing scaffolding ("SetCover ≤ HittingSet Reduktion") is NOT part of the target operand —
# without this strip the guard false-alarms and blocks the correct merge with the bare relation
_REL_STRIP_TAIL = re.compile(r'[\s\-–]*(?:redu[ck]tion(?:en|s)?|transformation(?:en|s)?)\s*$', re.I)
# an attached p/m marker ("≤p", "≤m") is operator notation, not part of the right operand
_REL_OPERATOR = re.compile(r'[≤⪯≥⊆⊊→⇒⟹⇔←][pm]?|=>|<=|->', re.I)


def _relation_operands(title: str) -> tuple[str, str] | None:
    """(canonical_source, canonical_target) of a relation/reduction title, else None (not a relation).
    Operands canonicalized (lowercased, non-alphanumerics stripped) so spacing/hyphenation don't matter."""
    t = _REL_STRIP.sub('', title, count=1)
    t = _REL_STRIP_TAIL.sub('', t, count=1)
    m = _REL_OPERATOR.search(t)
    if not m:
        return None
    left = re.sub(r'[\W_]', '', t[:m.start()].casefold())    # keep unicode letters/digits (umlauts), drop the rest
    right = re.sub(r'[\W_]', '', t[m.end():].casefold())
    if not left or not right:
        return None
    return (left, right)


def _relation_conflict(title_a: str, title_b: str) -> bool:
    """True if BOTH titles are relations/reductions but denote DIFFERENT ones (operands or direction
    differ) → they must NOT be merged. False if either is not a relation, or they are the same relation."""
    a, b = _relation_operands(title_a), _relation_operands(title_b)
    return a is not None and b is not None and a != b




def _filter_schema(data) -> dict[int, int] | None:
    """{"fragments": {"3": 7, "12": 8}} → {block_nr: parent_nr} · None on invalid structure.
    Empty dict = valid (nothing to degrade). Parent ≠ itself."""
    if not isinstance(data, dict) or not isinstance(data.get("fragments"), dict):
        return None
    out: dict[int, int] = {}
    for k, v in data["fragments"].items():
        try:
            nr, parent = int(k), int(v)
        except (ValueError, TypeError):
            continue
        if nr != parent:
            out[nr] = parent
    return out


# Pure notation/symbols without a standalone concept — kept narrow (FP~0, checked against aak;
# "KNF"/"MST"/"NP" do NOT match). These are discarded autonomously (need no parent).
_FILTER_NOTATION = re.compile(r'^\s*\|.{1,6}\|\s*$|^Güte\s+\d+\s*$')
# Exercise-sheet / cross-reference artefacts — the SECOND gate for a judge `drop` verdict: a hard-drop
# (parentless removal) only fires when the judge lists the number in `drop` AND `_is_artifact(title)`.
# A judge-drop without a match degrades to "keep" (logged), never deleted (FP~0 discipline like
# _FILTER_NOTATION). Shape-matching on a NORMALIZED, title-side string (see _is_artifact): four branches —
# (1) lettered/Roman/numeric sub-claims "(Aussage i)"/"Aussage (a)"/"Teil (b)"/"Fall (2)" in ANY
# parenthesization; (2) sheet refs "Blatt 10"/"Aufgabe 3"/"Übung"; (3) worked-example/table/figure refs
# "Beispiel Tab. 7.1" (a NUMBER is required — guards polysemes "Hash-Tabelle"/"Bijektive Abbildung");
# (4) parenthesized "(Variante)". Theorem words (Satz/Definition/Lemma) are deliberately NOT in the
# vocabulary, so "Satz 6.24 Cook/Levin — SAT ist NP-vollständig" is kept. Word boundaries guard prefixes
# (Aussagenlogik/Teilmenge/Blattknoten/Fallunterscheidung).
_FILTER_ARTIFACT = re.compile(r"""
      \b(?:aussage|teil|behauptung|fall)\b[\s(]*(?:[ivx]{1,4}|[a-z]|\d{1,2})\)?(?![a-zäöüß])
    | \b(?:blatt|aufgabe|serie|hausaufgabe)\s*\d+(?:[.,]\d+)*
    | \b(?:übungsblatt|übung|uebung)\b
    | \b(?:beispiel|abbildung|abb|tabelle|tab|bild|grafik|diagramm|figur|skizze)\b\.?\s*\d+(?:[.,]\d+)*
    | \(\s*(?:variante|variation|spezialfall|sonderfall)\s*\)
""", re.VERBOSE)


def _is_artifact(title: str) -> bool:
    """True if the TITLE looks like exercise-sheet / cross-reference scaffolding (P1 hard-drop gate).
    NFKC + casefold normalization (position/case/umlaut/Unicode-invariant: folds Ⅱ→ii, full-width, NBSP),
    then match only the title side of the em-dash — a real concept whose *description* merely cites a
    sheet ("… — vgl. Aufgabe 3") is never dropped. Normalize for matching only; never store the result."""
    norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip()
    head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0]  # spaced dash only → "3-SAT"/"np-schwer" unsplit
    return bool(_FILTER_ARTIFACT.search(head))
# Property/runtime suspicion — marks lines for the judge's verdict (NO auto-drop, FP too high:
# "NP-Schwere", reductions with "∈NP" are real blocks). Complements _aspekt_marker.
_FILTER_PREDICATE = re.compile(
    r'ist NP-(vollständig|schwer)|NP-(Vollständigkeit|Schwere) von|ETH (Konsequenz|Lower Bound)'
    r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP'
    # fragment families that the aspect substrings miss (all title/desc, advisory ⚠ only):
    r'|\bSatz\s+\d|\bLemma\s+\d|\bKorollar\s+\d'              # bare theorem/proof references (with number)
    r'|^\s*(?:Remark|Bemerkung|Anmerkung|Note|Notiz|Beobachtung|Observation)\b'  # EN+DE remark labels
    r'|\bSatz\s*:|\bSatz\s+[A-Z]\b'                           # "Satz:" (colon, no number) / "Satz D*" letter label
    r'|\bGegenbeispiel\b|\bWorst[- ]?Case\b|Schärfe\s+der\b'  # proof-example / sharpness facets
    r'|2\s*\^\s*[{(]?\s*[Ωωoο]\s*\('                          # ETH exponential bound 2^Ω(…)/2^o(…)
    r'|\d\s*[−–-]\s*1\s*/\s*m'                                # approximation-güte ratio "2 − 1/m"
    r'|Variablenungleichung|[α-ωΑ-Ω][a-z]?-?Variablen', re.I)  # proof variables (αu-Variablen …)


def _filter_suspect(b: dict) -> bool:
    """Heuristic flag: could be a property/detail of another block."""
    return _aspect_marker(b["title"]) > 0 or bool(_FILTER_PREDICATE.search(f"{b['title']} {b['description'] or ''}"))


def _root(nr: int, fragments: dict[int, int]) -> tuple[int, bool]:
    """Follow the fragment→parent chain to the first ancestor that is NOT itself a fragment.
    Returns (root, cyclic). This walk IS the fixpoint (no separate iteration): a mid-tier
    fragment whose own parent is also a fragment resolves to the top-level real block, so all
    chain levels collapse in one pass (unlike the old single-level parent_set shield).
    Cycle guard: on revisiting a node returns (node, True) → caller keeps both (no annihilation)."""
    seen: set[int] = set()
    cur = nr
    while cur in fragments:
        if cur in seen:
            return cur, True
        seen.add(cur)
        cur = fragments[cur]
    return cur, False


def _containment_parent(frag_norm: str, others: list[tuple[int, str]]) -> int | None:
    """Deterministic parent-by-name-containment for a ⚠-flagged survivor: a fragment whose title NAMES
    another block ("Lower Bound … für VERTEX COVER" → Vertex Cover; "List Scheduling Güte …" → List
    Scheduling). `others` = (nr, title_norm) of the OTHER blocks. A parent is a block whose normalized
    title occurs as a WHOLE-WORD span inside `frag_norm`, is SIGNIFICANT (≥2 tokens or ≥6 chars — never
    "p"/"np"/"sat") and strictly shorter than the fragment. Returns the parent nr on EXACTLY ONE match,
    else None (0 or ≥2 → leave to the judge). Whole-word + significance + exactly-one → near-FP-0."""
    hits = []
    for nr, ptitle in others:
        if not ptitle or ptitle == frag_norm or len(ptitle) >= len(frag_norm):
            continue
        if len(ptitle) < 6 and ptitle.count(" ") < 1:  # reject short single-token names (P/NP/SAT)
            continue
        if re.search(r'(?<!\w)' + re.escape(ptitle) + r'(?!\w)', frag_norm):
            hits.append(nr)
    return hits[0] if len(hits) == 1 else None


# Two provably-noise, parent-less fragment classes safe to hard-drop (precision ~≥0.95): a bare
# label reference with an empty/thin residual ("Remark 7.28", "Satz D*", "Lemma 3.2") and a single-
# variable notation assignment ("r = n + m"). Matched on the normalized title HEAD (before the em-dash).
_PARENTLESS_NOISE = re.compile(
    r'^\s*(?:remark|bemerkung|anmerkung|note|satz|lemma|korollar|proposition|folgerung)\s*[\d.]*\s*[a-z]?\*?\s*$'
    # single-var arithmetic assignment ("r = n + m") — NOT a tuple/set/language/cardinality definition
    # ("T = (Q,…)", "L = {…}", "n = |V|"): the RHS must not open with a bracket/pipe.
    r'|^\s*[a-z](?:_?[a-z0-9])?\s*:?=\s*(?![({\[|])\S', re.I)
# KEEP-guards: never drop a named theorem WITH an author, a complexity-class (in)equality, or a Definition.
_PARENTLESS_KEEP = re.compile(
    r'\b(?:cook|levin|karp|savitch|ladner|immerman|rice|håstad|hastad|christofides|dijkstra|bellman|'
    r'ford|kruskal|prim|edmonds|blum|sipser|papadimitriou)\b'
    r'|\b(?:p|np|conp|nl|pspace|exp|nexp|bpp|rp|zpp)\b|⇔|⇒|\bdefinition\b', re.I)


def _is_parentless_noise(title: str) -> bool:
    """True for the narrow, parent-less noise classes safe to hard-drop (subject to KEEP-guards)."""
    norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip()
    head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0]
    return bool(_PARENTLESS_NOISE.search(head)) and not _PARENTLESS_KEEP.search(norm)


# F1 — statement-gate keep-guard (Knowledge-Component / OMDoc theory): a self-contained ASSERTION is its
# own learning unit, not a whole-part fragment. Protect two general classes from demotion:
#   (a) a named reduction between two PROBLEMS  ("3-SAT ≤ Clique", "Clique → Vertex Cover"), and
#   (b) a LABELED or ATTRIBUTED theorem carrying its own biconditional/implication ("Satz 6.37: … ⇔ …").
# NOT protected: a bare label ("Satz 7.18", "Remark 7.28"), unary status ("X ist NP-vollständig", "X ∈ NP"),
# a proof-size step ("Strikte Reduktion |A| = O(m)"), or a güte/bound facet — those carry neither a
# two-sided reduction operator NOR a ⇔/⇒ assertion. General: attribution is structural, no author whitelist.
_STMT_LABEL = re.compile(r'^\s*(?:satz|lemma|korollar|theorem|proposition|prop|folgerung)\b', re.I)
_STMT_ATTRIB = re.compile(r'\b(?:satz|lemma|theorem|korollar)\s+von\s+[A-ZÄÖÜ]')   # "Satz von Cook/Levin"
_STMT_ASSERT = re.compile(r'⇔|⇒|⟺|⟹|\bgdw\.?\b|\bgenau dann\b', re.I)
_REDUCTION_ONLY_OP = re.compile(r'[≤⪯]|→|⇒|⟹|->')  # genuine reduction operators (NOT plain "=")


def _is_reduction_statement(title: str) -> bool:
    """True if the title is a reduction between two NAMED problems (both sides carry ≥3 letters and
    neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP" and proof-size "|A| = O(m)"."""
    t = _REL_STRIP.sub('', title, count=1)
    m = _REDUCTION_ONLY_OP.search(t)
    if not m:
        return False

    def _named(s: str) -> bool:
        return len(re.findall(r'[a-zäöüß]', s, re.I)) >= 3 and not re.match(r'\s*[Oo]\s*\(', s)

    return _named(t[:m.start()]) and _named(t[m.end():])


def _is_named_statement(title: str, desc: str = "") -> bool:
    """Statement-gate keep-guard: a named reduction (a) or a labeled/attributed theorem WITH its own
    ⇔/⇒ assertion (b). Bare labels / unary status / proof-size steps return False (stay demotable)."""
    if _is_reduction_statement(title):
        return True
    if (_STMT_LABEL.match(title) or _STMT_ATTRIB.search(title)) and _STMT_ASSERT.search(f"{title} {desc or ''}"):
        return True
    return False




def _umbrella_schema(data, ids: set[int]):
    """{"umbrellas":[{"title":str,"description":str,"members":[int,…]}, …]}
    → [(title, description, [member ids])]. [] = valid (no umbrella); None ONLY on broken JSON
    (so resume treats a valid-but-empty file as done, like _filter_schema's {} vs None). Each id
    used at most once across all umbrellas (first wins); members filtered to `ids`; an umbrella
    needs ≥2 surviving members; `description` required + non-empty — it MUST enumerate the children,
    else the source-scoped subblock step can't re-derive them (no web on uni)."""
    if not isinstance(data, dict) or not isinstance(data.get("umbrellas"), list):
        return None
    out, used = [], set()
    for u in data["umbrellas"]:
        if not isinstance(u, dict):
            continue
        title = str(u.get("title", "")).strip()
        desc = str(u.get("description", "")).strip()
        raw_members = u.get("members")
        if not title or not desc or not isinstance(raw_members, list):
            continue
        members = []
        for x in raw_members:
            try:
                m = int(x)
            except (ValueError, TypeError):
                continue
            if m in ids and m not in used:
                used.add(m)
                members.append(m)
        if len(members) >= 2:
            out.append((title, desc, members))
    return out


def _completion_schema(data, n_umbrellas: int, ids: set[int]):
    """{"additions":[{"umbrella":int,"members":[int,…]}, …]} → [(umbrella_idx, [member ids])].
    [] = valid (nothing to absorb); None only on broken JSON. umbrella idx in range; members drawn
    from `ids` (the still-standalone leftovers), de-duped within an addition."""
    if not isinstance(data, dict) or not isinstance(data.get("additions"), list):
        return None
    out = []
    for a in data["additions"]:
        if not isinstance(a, dict):
            continue
        try:
            k = int(a.get("umbrella"))
        except (ValueError, TypeError):
            continue
        if not (0 <= k < n_umbrellas):
            continue
        mem = []
        for x in (a.get("members") or []):
            try:
                m = int(x)
            except (ValueError, TypeError):
                continue
            if m in ids and m not in mem:
                mem.append(m)
        if mem:
            out.append((k, mem))
    return out


# Deterministic backstop to the grouping judge's TEST 1 (type gate): an umbrella may bundle ONLY
# constituent sub-definitions of ONE definition. If a member title carries a standalone-unit signal
# (a named algorithm / problem / reduction / theorem), the umbrella is dissolved — those stay their own
# blocks. Kept narrow so real definition-parts (Konfiguration, Übergangsfunktion δ, Literale, Makespan,
# m Maschinen) never match; checked against the aak over-merge (member „Greedy-Algorithmus GA" hits).
# Suffix-anchored head nouns (German compounds are head-final: „Approximations+algorithmus" has NO word
# boundary before „algorithmus", so \bAlgorithmus\b misses it → the MAX-SAT over-merge). \w* absorbs the
# modifier; the head noun stays the discriminator. FP-safe: no real TM/KNF definition-part ends in these
# heads (Berechnung is deliberately NOT a head → „Akzeptierende Berechnung" stays a valid member).
_GROUP_STANDALONE = re.compile(
    r'\w*algorithm(?:us|en)\b|\w*problem(?:e|s|en)?\b|\w*reduktion(?:en)?\b|\bscheduling\b|[≤⪯]'
    r'|^\s*(?:Satz|Lemma|Korollar|Theorem|Bemerkung|Beobachtung)\s*\d'
    # atomicity: a named COMPLEXITY CLASS / a "…-Vollständigkeit(completeness)" / a "…Transformation" is a
    # self-contained concept (learning-object / atomic-KC), never a sub-definition — a bundle of ≥1 such
    # member is siblings, not one model → dissolve (catches the P/NP/NP-Vollständigkeit over-merge that NO
    # cosine floor separates). Head-final compounds (\w*klasse absorbs "Komplexitäts+klasse"); FP-safe —
    # no real TM/KNF/TSP/Scheduling definition-part carries these heads.
    r'|\w*vollständigkeit\b|\w*completeness\b|\w*transformation(?:en)?\b|\w*klasse[nr]?\b', re.I)




# --- Outline (blocks artifact: chapter structure, only read by the guide) ---

def _outline_review_schema(data, valid: set[int], n_chapters: int, n_blocks: int):
    """{"moves": {"<blocknr>": <chapter-idx>}} → {nr: idx} (may be {}) · None if broken/invalid.
    A mass rewrite (more than a third of all blocks) is rejected — the reviewer's job is
    spotting misplacements, not re-designing the outline."""
    if not isinstance(data, dict) or not isinstance(data.get("moves"), dict):
        return None
    out: dict[int, int] = {}
    for k, v in data["moves"].items():
        try:
            nr, ch = int(k), int(v)
        except (ValueError, TypeError):
            return None
        if nr not in valid or not (1 <= ch <= n_chapters):
            return None
        out[nr] = ch
    if len(out) * 3 > n_blocks:
        return None
    return out


def _outline_schema(data, valid: set[int]):
    """{"chapters":[{title,numbers}]} → cleaned (valid numbers, each exactly once) ·
    None at <80 % coverage (agent/judge omitted too much)."""
    if not isinstance(data, dict) or not isinstance(data.get("chapters"), list):
        return None
    out, seen = [], set()
    for ch in data["chapters"]:
        if not isinstance(ch, dict):
            continue
        title = str(ch.get("title", "")).strip() or "Chapter"
        nums = []
        for n in (ch.get("numbers") or []):
            try:
                n = int(n)
            except (ValueError, TypeError):
                continue
            if n in valid and n not in seen:
                seen.add(n)
                nums.append(n)
        if nums:
            out.append({"title": title, "numbers": nums})
    if not out or len(seen) < 0.8 * len(valid):
        return None
    return {"chapters": out}


def _prereq_schema(data, valid: set[int]) -> dict[int, list[int]]:
    """{"prereqs": {"3": [1, 7]}} → {num: [prereq nums]} · only numbers from `valid`, no self-edge.
    Invalid/empty → {} (best-effort: then original order)."""
    if not isinstance(data, dict) or not isinstance(data.get("prereqs"), dict):
        return {}
    out: dict[int, list[int]] = {}
    for k, v in data["prereqs"].items():
        try:
            num = int(k)
        except (ValueError, TypeError):
            continue
        if num not in valid or not isinstance(v, list):
            continue
        pres = []
        for p in v:
            try:
                p = int(p)
            except (ValueError, TypeError):
                continue
            if p in valid and p != num and p not in pres:
                pres.append(p)
        if pres:
            out[num] = pres
    return out


def _topo_order(nums: list[int], edges: dict[int, list[int]]) -> list[int]:
    """Kahn topo sort: prerequisites first. `edges[num]` = numbers that must come BEFORE num.
    Stable tie-break (original order of `nums`); cycles are broken (never deadlock)."""
    pos = {n: i for i, n in enumerate(nums)}
    # remaining in-degree over valid nodes only; self/foreign edges ignored.
    pre = {n: [p for p in edges.get(n, []) if p in pos and p != n] for n in nums}
    done: list[int] = []
    finished: set[int] = set()
    rest = list(nums)
    while rest:
        ready_nodes = [n for n in rest if all(p in finished for p in pre[n])]
        if not ready_nodes:  # cycle → force the earliest remaining node in original order
            ready_nodes = [min(rest, key=lambda n: pos[n])]
        nxt = min(ready_nodes, key=lambda n: pos[n])  # stable: smallest original position first
        done.append(nxt)
        finished.add(nxt)
        rest.remove(nxt)
    return done


async def _learning_order(ctx: GenContext, set_p, files: dict, entries: dict, valid: set[int], instructions: str) -> dict:
    """Put entries (num→title) into learning order: the LLM extracts prereq edges from the
    extracted `prerequisites`, code solves via topo sort. Best-effort → otherwise entries unchanged."""
    if len(entries) < 3:
        return entries
    topic = ctx.topic
    facts_map = _json_file(files["facts"])
    facts_map = facts_map if isinstance(facts_map, dict) else {}

    def _hint(title):
        fm = facts_map.get(title) or {}
        vs = [v for fk in fm.values() if isinstance(fk, dict) and (v := str(fk.get("prerequisites", "")).strip())]
        return " · ".join(dict.fromkeys(vs))

    pp = files["arbeit"] / "outline-prereqs.json"

    def _payload(result, p=pp):
        d = _json_file(p)
        return d if isinstance(d, dict) and "prereqs" in d else None

    existing = _json_file(pp)
    if not (isinstance(existing, dict) and "prereqs" in existing):
        lines = [f"{n}. {t}" + (f"\n   braucht vorher: {h}" if (h := _hint(t)) else "") for n, t in entries.items()]
        set_p("Outline — learning order…", step=_step_idx(topic, "Outline"))
        await run_single_slot(
            ctx, "Outline-Prerequisites", key=f"blocks-{topic}-outline-prereqs",
            prompt=_prompt("Outline-Prerequisites", topic=topic, blocks="\n".join(lines), out_path=pp, extra=_extra(instructions)),
            role="guide", capabilities="files", payload=_payload, timeout=_timeout("plan", len(entries)))
    edges = _prereq_schema(_json_file(pp), valid)
    if not edges:
        return entries  # no/invalid edges → original order (no regression)
    ordered = _topo_order(list(entries), edges)
    return {n: entries[n] for n in ordered}


async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict:
    """Format-agnostic outline over ALL blocks — 3 proposals → judge merges.
    Never aborts: 0 valid → one chapter with everything; missing blocks land in "Other".
    → {"chapters":[{title,numbers}]} (also in files["outline"])."""
    topic, is_cancelled = ctx.topic, ctx.is_cancelled
    valid = set(entries)
    step = _step_idx(topic, "Outline")

    # Establish learning order (LLM-modulo): the LLM extracts prereq edges from the extracted
    # `prerequisites`, code solves via topo sort. Best-effort → otherwise original order.
    entries = await _learning_order(ctx, set_p, files, entries, valid, instructions)
    liste = "\n".join(f"{n}. {t}" for n, t in entries.items())
    set_p("Outline — proposals…", step=step)

    async def _proposal(i, path):
        if _outline_schema(_json_file(path), valid):
            return True
        await run_single_slot(
            ctx, f"Outline {i}", key=f"blocks-{topic}-outline-{i}",
            prompt=_prompt("Guide-Outline", topic=topic, blocks=liste, out_path=path, extra=_extra(instructions)),
            role="guide", capabilities="files",
            payload=lambda result, p=path: _outline_schema(_json_file(p), valid),
            timeout=_timeout("plan", len(entries)))
        return _outline_schema(_json_file(path), valid) is not None

    slots = files["outline_slots"]
    await _gather_progress([_proposal(i, p) for i, p in enumerate(slots, 1)], len(slots), _report_p(set_p, topic, "Outline"))
    if is_cancelled():
        return {}
    proposals = [v for p in slots if (v := _outline_schema(_json_file(p), valid))]

    if not proposals:
        plan = {"chapters": [{"title": "Contents", "numbers": list(entries)}]}
    elif len(proposals) == 1:
        plan = proposals[0]
    else:
        set_p("Outline merging…", step=step)
        block_texts = "\n\n".join(
            f"### Vorschlag {i}\n" + "\n".join(
                f"KAPITEL: {ch['title']}\n  Nummern: {', '.join(str(n) for n in ch['numbers'])}" for ch in v["chapters"])
            for i, v in enumerate(proposals, 1))
        await run_single_slot(
            ctx, "Outline-Judge", key=f"blocks-{topic}-outline-judge",
            prompt=_prompt("Guide-Outline-Judge", topic=topic, format_name="den Guide",
                           purpose="alle Blocks in einem roten Faden", n=len(proposals),
                           blocks=liste, outlines=block_texts, out_path=files["outline"], extra=_extra(instructions)),
            role="judge", capabilities="files",
            payload=lambda result: _outline_schema(_json_file(files["outline"]), valid),
            timeout=_timeout("plan_judge", len(entries)))
        plan = _outline_schema(_json_file(files["outline"]), valid) or proposals[0]

    # Placement review (best-effort): ONE judge checks every block→chapter assignment and
    # reports ONLY misplacements as moves. Invalid/mass output → plan unchanged.
    if proposals and len(plan["chapters"]) >= 2 and not is_cancelled():
        rp = files["arbeit"] / "outline-review.json"
        moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
        if moves is None:
            chapter_text = "\n\n".join(
                f"KAPITEL {k}: {ch['title']}\n" + "\n".join(f"  {n}. {_title(entries[n])}" for n in ch["numbers"])
                for k, ch in enumerate(plan["chapters"], 1))
            set_p("Outline review…", step=step)
            await run_single_slot(
                ctx, "Outline-Review", key=f"blocks-{topic}-outline-review",
                prompt=_prompt("Guide-Outline-Review", topic=topic, chapters=chapter_text,
                               out_path=rp, extra=_extra(instructions)),
                role="judge", capabilities="files",
                payload=lambda result: _outline_review_schema(
                    _json_file(rp), valid, len(plan["chapters"]), len(entries)),
                timeout=_timeout("plan_judge", len(entries)))
            moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
        for nr, target in (moves or {}).items():
            for ch in plan["chapters"]:
                if nr in ch["numbers"]:
                    ch["numbers"].remove(nr)
            plan["chapters"][target - 1]["numbers"].append(nr)
        if moves:
            plan["chapters"] = [ch for ch in plan["chapters"] if ch["numbers"]]
            _log(topic, f"Outline-Review: {len(moves)} Block/Blöcke umsortiert")

    # Completeness: every block appears — missing in "Other" (against omitting agents/judge).
    included = {n for ch in plan["chapters"] for n in ch["numbers"]}
    missing = [n for n in entries if n not in included]
    if missing:
        plan["chapters"].append({"title": "Other", "numbers": missing})
    atomic_write_json(files["outline"], plan, indent=1)
    return plan


# --- Learning artefacts (flashcards/examples from the facts) ---

def _cards_schema(data):
    """{"cards":[{block,subblock,question,answer}]} → list (also empty) · None if broken."""
    if not isinstance(data, dict) or not isinstance(data.get("cards"), list):
        return None
    out = []
    for e in data["cards"]:
        if isinstance(e, dict) and (f := str(e.get("question", "")).strip()) and (a := str(e.get("answer", "")).strip()):
            out.append({"block": str(e.get("block", "")).strip(), "subblock": str(e.get("subblock", "")).strip(),
                        "question": f, "answer": a})
    return out


def _example_schema(data):
    """{"examples":[{block,subblock,problem,steps,result}]} → list (also empty) · None if broken."""
    if not isinstance(data, dict) or not isinstance(data.get("examples"), list):
        return None
    out = []
    for e in data["examples"]:
        if not isinstance(e, dict):
            continue
        problem = str(e.get("problem", "")).strip()
        steps = [s for x in (e.get("steps") or []) if (s := str(x).strip())]
        if problem and steps:
            out.append({"block": str(e.get("block", "")).strip(), "subblock": str(e.get("subblock", "")).strip(),
                        "problem": problem, "steps": steps, "result": str(e.get("result", "")).strip()})
    return out


def _example_check_schema(data):
    """Worked-example check → {"ok": true} → set() (all correct); {"problems":[{"index":N}]} →
    {N, …} (1-based flagged indices); None if broken."""
    if not isinstance(data, dict):
        return None
    if data.get("ok") is True:
        return set()
    pr = data.get("problems")
    if not isinstance(pr, list):
        return None
    out: set[int] = set()
    for p in pr:
        if isinstance(p, dict):
            try:
                out.add(int(p.get("index")))
            except (ValueError, TypeError):
                continue
    return out


_ARTEFACT_SCHEMA = {"flashcard": _cards_schema, "example": _example_schema}
_ARTEFACT_PROMPT = {"flashcard": "Artifact-Flashcard", "example": "Artifact-Example"}
_ARTEFACT_STEP = {"flashcard": "Flashcards", "example": "Examples"}


async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None:
    """Generate learning artefacts per type from the stored facts — one generation pass
    per type over chunks. Worked examples are verified against the facts (wrong ones discarded);
    flashcards are low-risk and stay unchecked. → {type: [entries]} (also in files)."""
    topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
    work_dir = files["arbeit"]
    caps = "files"
    sh = _subs_hash(sidecar)  # resume must invalidate when the sub set changed
    # Blocks with subs + facts lines as input block (extract-once from the facts).
    blocks = []
    for btitle, subs in sidecar.items():
        if not isinstance(subs, list):
            continue
        lines = []
        for s in subs:
            if not isinstance(s, dict) or not (st := str(s.get("title", "")).strip()):
                continue
            fk = s.get("facts") if isinstance(s.get("facts"), dict) else {}
            line = f"- {st}"
            if fk and (fk_text := _facts_lines(fk)):
                line += "\n" + "\n".join("  " + l for l in fk_text.split("\n"))
            lines.append(line)
        if lines:
            blocks.append((btitle, lines))
    if not blocks:
        empty_map = {t: [] for t in ARTEFACT_TYPES}
        atomic_write_json(files["artefakte"], empty_map, indent=1)
        return empty_map

    chunks = _lpt_chunks([len(z) for _, z in blocks], ARTEFACT_CHUNK_SUBS)
    def block_text(idxs):
        return "\n\n".join(f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(blocks[i][1]) for i in idxs)

    # Check worked examples against the facts (panel majority) — discard wrong ones. CoT steps are
    # error-prone; a wrong example imprints a faulty schema → no example > a wrong one.
    async def _check_examples(ci, idxs, items):
        if is_cancelled() or not items:
            return items
        def cpath(j): return work_dir / f"artifact-example-check-{sh}-c{ci}-j{j}.json"
        examples_txt = "\n\n".join(
            f"{k}. PROBLEM: {e['problem']}\n   SCHRITTE: " + " | ".join(e.get("steps", []))
            + (f"\n   ERGEBNIS: {e['result']}" if e.get("result") else "")
            for k, e in enumerate(items, 1))
        pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _example_check_schema(_json_file(cpath(j))) is None]
        if pending:
            # ground truth (facts) is fully inline → no tools, text reply, engine persists
            rs = await asyncio.gather(*[
                run_agent(f"blocks-{topic}-{ns}artifact-example-check-c{ci}-j{j}",
                    _prompt("Artifact-Example-Check", topic=topic, facts=block_text(idxs), examples=examples_txt, out_path=cpath(j), extra=_extra(instructions)),
                    _timeout("content_check", len(items)), provider=provider, role="judge", capabilities="none",
                    scope=topic, label=f"{lbl}Beispiel-Check {ci}/{j}")
                for j in pending], return_exceptions=True)
            for j, r in zip(pending, rs):
                if isinstance(r, tuple):
                    _sink_json(r, cpath(j), _example_check_schema)
        outs = [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if (s := _example_check_schema(_json_file(cpath(j)))) is not None]
        if not outs:
            return items  # no exam possible → keep (best-effort)
        votes: dict[int, int] = {}
        for s in outs:
            for idx in s:
                votes[idx] = votes.get(idx, 0) + 1
        threshold = len(outs) / 2
        dropped = {idx for idx, v in votes.items() if v > threshold}  # majority (≥2 of 3) flagged → out
        if dropped:
            _log(topic, f"Worked-example check chunk {ci}: {len(dropped)}/{len(items)} discarded")
        return [e for k, e in enumerate(items, 1) if k not in dropped]

    async def _one_type(typ: str) -> list | None:
        """Full pipeline of ONE artefact type — the types are independent (own schemas,
        own files) and run in parallel."""
        schema = _ARTEFACT_SCHEMA[typ]

        def apath(ci): return work_dir / f"artifact-{typ}-{sh}-c{ci}.json"

        async def _gen(ci, idxs):
            p = apath(ci)
            if schema(_json_file(p)) is not None:
                return True
            await run_single_slot(
                ctx, f"{lbl}{_ARTEFACT_STEP[typ]} {ci}", key=f"blocks-{topic}-{ns}artifact-{typ}-c{ci}",
                prompt=_prompt(_ARTEFACT_PROMPT[typ], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)),
                role="guide", capabilities="files",
                payload=lambda result, p=p: schema(_json_file(p)),
                timeout=_timeout("content", sum(len(blocks[i][1]) for i in idxs)))
            return schema(_json_file(p)) is not None

        await _gather_progress([_gen(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, _ARTEFACT_STEP[typ]))
        if is_cancelled():
            return None
        eintraege: list = []
        for ci in range(len(chunks)):
            chunk_items = schema(_json_file(apath(ci))) or []
            if typ == "example" and chunk_items:
                chunk_items = await _check_examples(ci, chunks[ci], chunk_items)
            eintraege += chunk_items
        return eintraege

    results = await asyncio.gather(*[_one_type(t) for t in ARTEFACT_TYPES])
    if is_cancelled() or any(r is None for r in results):
        return None
    outcome: dict[str, list] = dict(zip(ARTEFACT_TYPES, results))
    atomic_write_json(files["artefakte"], outcome, indent=1)
    return outcome


async def _mirror_sidecar_db(topic: str, sidecar: dict) -> None:
    """Mirror the sidecar {block title: [{title, level, relevance}]} into the DB table subblocks."""
    for btitle, subs in sidecar.items():
        bnorm = _norm_title(btitle)
        if not bnorm or not isinstance(subs, list):
            continue
        for s in subs:
            if not isinstance(s, dict):
                continue
            st = str(s.get("title", "")).strip()
            sn = _norm_title(st)
            if not sn:
                continue
            facts = json.dumps(s["facts"], ensure_ascii=False) if isinstance(s.get("facts"), dict) else None
            await db.put_subblock(topic, bnorm, sn, btitle, st,
                                     level=s.get("level"), relevance=s.get("relevance"),
                                     facts=facts, status="consensus")




async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER,
                          research: bool = True, qa_force: bool = False) -> None:
    """Kanban entry point: source prep, then both boards (inventory + artefacts) until
    quiescence. research=False = Continue (drain the existing queue, no new search).
    A run on a finished topic ADDS research (live extension) — full rebuild = DELETE /blocks."""
    if topic in _blocks_progress:
        return
    _blocks_progress[topic] = "Warten…"
    _blocks_errors.pop(topic, None)

    files = _blocks_files(topic)
    q = load_source(topic)
    folder = source_folder(topic)  # projekt/uni/link → folder, thema → None
    instructions = q.get("spec") or instructions  # prefer the persisted specification (also on resume)

    def set_p(msg: str, step: int | None = None) -> None:
        _blocks_progress[topic] = msg
        if step is not None:
            _blocks_step[topic] = step

    def is_cancelled() -> bool:
        return topic in _blocks_cancelled

    ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled)
    try:
        async with _semaphore:
            files["arbeit"].mkdir(parents=True, exist_ok=True)
            # Step "Source prep": crawl (link) + PDFs + content/noise triage.
            if not await _prepare_source(ctx, set_p, files, q, folder, instructions):
                if is_cancelled():
                    _blocks_errors[topic] = "Cancelled — progress is preserved"
                return
            import board_inventory  # lazy: the boards import blocks
            ok = await board_inventory.run_boards(ctx, set_p, files, q, folder, instructions,
                                                  research=research, qa_force=qa_force)
            if not ok and is_cancelled():
                _blocks_errors[topic] = "Cancelled — progress is preserved"
    except Exception as e:
        log.exception("[%s] Blocks generation failed", topic)
        _blocks_errors[topic] = str(e)[:2000]
    finally:
        # No file cleanup: intermediate files stay for resume / traceability.
        _blocks_progress.pop(topic, None)
        _blocks_step.pop(topic, None)
        _blocks_cancelled.discard(topic)
        clear_scope(f"blocks-{topic}-")  # clear the scope → restart isn't blocked