"""Guide generation as a consensus pipeline.

Outline: select the blocks (deterministic per format) → 3 proposals
(Grace) that order the blocks into chapters by NUMBER → a judge merges the
proposals into one coherent order.
Writing: one writer per block. Reading exam: Check→Fix (one round),
follow-up rounds check only replaced sections; remaining complaints then stand.
Step files are kept → an abort preserves progress, ▶ resumes at the open step.
"""

import asyncio
import json
import logging
import math
from datetime import datetime, timezone
from pathlib import Path

import uuid

from agents import run_agent
from blocks import _convert_pdfs, source_folder
from config import (
    DEFAULT_PROVIDER, FORMAT_PURPOSE, CONSENSUS_GRACE,
    READABILITY_ACTIVE, TEMPLATES_DIR,
)
import readability
from database import list_guides, update_guide, list_blocks, list_subblocks, set_guide_content, get_guide_content, get_outline
from fsutil import atomic_write_json, atomic_write_text
from jsonio import read_json_file as _json_file, parse_json_text as _parse_json_text
from paths import blocks_path, guide_content_path, project_dir, subblocks_path
from pipeline import (
    CANCELLED, FAILED, GenContext, _claude_error, _extra,
    _fail, _gather_error, _gather_progress, _log, _prompt, _race,
    _semaphore, _set_progress, _set_step, _timeout, clear_guide_cancelled,
    is_guide_cancelled, run_single_slot,
)
from textkit import (
    _unique_title, _load_blocks, _norm_title, _parse_fragment, _split_chunks,
    _title, _resolve_title, _title_index,
)

log = logging.getLogger("creator.guide")

GUIDE_STEPS = ("Outline", "Content", "Content-Check", "Writing", "Reading-Exam")

# Content/Content-Check/Reading-Exam run in packets of ~GUIDE_CHUNK blocks per agent.
# Only the writer (Writing) stays at 1 agent per block (variable lengths, no trimming, no
# length alignment between blocks).
GUIDE_CHUNK = 10

# Check steps as a panel: CHECK_PANEL judges per chunk, section flagged on a majority.
# A single judge is bias/sampling prone; a small panel is more stable.
CHECK_PANEL = 3

# Reading exam: only ONE round (Check + Fix). Follow-up rounds added little value
# (1 agent per block checks finely anyway) but cost extra agents.
READING_ROUNDS = 1


# Valid level values: new (learning path) + old (difficulty) backward-compatible.
_LEVELS_OK = ("beginner", "advanced", "expert", "easy", "medium", "hard")


async def _load_subblocks(topic: str) -> dict[str, list[dict]]:
    """Subblocks per block — DB-first ({title, level, relevance}), fallback sidecar file.
    Both missing → {} (guide takes everything)."""
    out: dict[str, list[dict]] = {}
    for r in await list_subblocks(topic):
        if r["status"] == "consensus" and r["sub_title"] and r["level"] in _LEVELS_OK:
            try:
                facts = json.loads(r["facts"]) if r.get("facts") else {}
            except (ValueError, TypeError):
                facts = {}
            out.setdefault(r["block"], []).append(
                {"title": r["sub_title"], "level": r["level"], "relevance": r["relevance"], "facts": facts})
    if out:
        return out
    data = _json_file(subblocks_path(topic))
    if not isinstance(data, dict):
        return {}
    for title, subs in data.items():
        if not isinstance(subs, list):
            continue
        good = [s for s in subs if isinstance(s, dict) and str(s.get("title", "")).strip()
                and s.get("level") in _LEVELS_OK]
        if good:
            out[title] = good
    return out


def _level_label(s: dict) -> str:
    """View level of a subblock: peripheral → 'peripheral' (level 4), otherwise the level (1–3)."""
    return "peripheral" if s.get("relevance") == "peripheral" else (s.get("level") or "beginner")


def _assignment_subs(chunk: list[dict], entries: dict[int, str], subs_by_title: dict[str, list[dict]]) -> str:
    """Lists the blocks per chapter, with their subblocks and level labels beneath."""
    lines: list[str] = []
    for ch in chunk:
        lines.append(f"CHAPTER: {ch['title']}")
        for num in ch["nums"]:
            lines.append(f"- {entries[num]}")
            for s in subs_by_title.get(_title(entries[num]), []):
                lines.append(f"    [{_level_label(s)}] {s['title']}")
    return "\n".join(lines)


def _guide_files(content_path: Path) -> dict:
    d, stem = content_path.parent, content_path.stem
    return {
        "outline_slots": [d / f"{stem}.outline-{i}.json" for i in (1, 2, 3)],
        "outline": d / f"{stem}.outline.json",  # judge output
        # chunk/reading-check/fix files are dynamic:
        # {stem}.chunk-i.md, {stem}.lese-check-r{n}-{i}.json, {stem}.fix-r{n}-{i}.md
    }


def guide_slot_files(content_path: Path) -> list[Path]:
    """All step files of a guide (for a fresh start)."""
    return [p for p in content_path.parent.glob(f"{content_path.stem}.*") if p != content_path]


def _done_path(content_path: Path) -> Path:
    return content_path.parent / f"{content_path.stem}.done"


def guide_done_step(content_path: Path) -> int:
    """Highest FULLY completed step index (marker per topic+format). -1 = none.
    If the content file exists, all steps are done."""
    if content_path.exists():
        return len(GUIDE_STEPS) - 1
    try:
        return int(_done_path(content_path).read_text(encoding="utf-8").strip())
    except (OSError, ValueError):
        return -1


def _set_done(content_path: Path, step: int) -> None:
    """Set marker to `step` — monotone (only increase), except on the re-run reset (force)."""
    if step > guide_done_step(content_path):
        atomic_write_text(_done_path(content_path), str(step))


def _reset_done(content_path: Path, step: int) -> None:
    """Set marker hard to `step` (for re-run from step; step may decrease)."""
    if step < 0:
        _done_path(content_path).unlink(missing_ok=True)
    else:
        atomic_write_text(_done_path(content_path), str(step))


# Slot-file globs per step (index = GUIDE_STEPS). Stem-anchored, collision-free.
_STEP_GLOBS = (
    ("outline*",),                        # 0 Outline (incl. selection filter)
    ("content-chunk-*", "content-nach-*"),     # 1 Content (incl. follow-up round)
    ("content-check-*", "content-fix-*"),      # 2 Content-Check
    ("chunk-*",),                            # 3 Writing (chunk-* also matches chunk-nach-*)
    ("lese-check-*", "fix-r*"),           # 4 Reading-Exam
)


def _reset_guide_from_step(content_path: Path, step: int) -> None:
    """Re-run from step: delete content + all slot files of steps ≥ step.
    Earlier steps stay → the resume rebuilds from `step` (everything below is reused)."""
    content_path.unlink(missing_ok=True)  # no longer "done" → no fresh-start wipe
    d, stem = content_path.parent, content_path.stem
    for globs in _STEP_GLOBS[step:]:
        for pat in globs:
            for p in d.glob(f"{stem}.{pat}"):
                p.unlink(missing_ok=True)
    _reset_done(content_path, step - 1)  # steps < step count as done


def _read_problems_schema(data):
    """{"ok": true} → [] · {"problems": [{"section", "problem"}]} → list · else None."""
    if not isinstance(data, dict):
        return None
    if data.get("ok") is True:
        return []
    p = data.get("problems")
    if not isinstance(p, list) or not p:
        return None
    out = []
    for x in p:
        if not isinstance(x, dict) or not isinstance(x.get("section"), str) or not isinstance(x.get("problem"), str):
            return None
        out.append({"section": x["section"].strip(), "problem": x["problem"].strip()})
    return out or None


def _panel_problems(judge_paths: list[Path], valid: set[int], idx: dict[str, int]) -> dict[int, str]:
    """Panel aggregation: several judge outputs of a chunk → flagged {num: problem}.

    One vote per judge that names a section. Flagged when more than half of the
    DELIVERED (validly parsed) judges name it (3→≥2, 2→≥2, 1→≥1). Robust against a
    single failure: missing files do not count. Problem text from the first naming judge.
    """
    outputs = [p for p in (_read_problems_schema(_json_file(j)) for j in judge_paths) if p is not None]
    if not outputs:
        return {}
    votes: dict[int, int] = {}
    problem: dict[int, str] = {}
    for out in outputs:
        seen: set[int] = set()
        for item in out:
            num = _resolve_title(idx, item["section"])
            if num is None or num not in valid or num in seen:
                continue
            seen.add(num)
            votes[num] = votes.get(num, 0) + 1
            problem.setdefault(num, item["problem"])
    threshold = len(outputs) / 2
    return {num: problem[num] for num, v in votes.items() if v > threshold}


def _resolve_outline(data, entries: dict[int, str], target_min: int, target_max: int) -> list[dict] | None:
    """{"chapters": [{"title", "numbers": [1, 3, 7]}]} → [{"title", "nums"}].

    Numbers are the IDs from `entries` (1-based, as presented to the agent).
    `target_min`/`target_max` = allowed range of selected blocks (with a small tolerance).
    """
    if not isinstance(data, dict) or not isinstance(data.get("chapters"), list):
        return None
    valid = set(entries)
    chapters: list[dict] = []
    seen: set[int] = set()
    total = unknown = 0
    for ch in data["chapters"]:
        if not isinstance(ch, dict) or not isinstance(ch.get("numbers"), list):
            return None
        nums = []
        for t in ch["numbers"]:
            total += 1
            num = t if isinstance(t, int) and not isinstance(t, bool) else None
            if num is None or num not in valid:
                unknown += 1
            elif num not in seen:
                nums.append(num)
                seen.add(num)
        if nums:
            chapters.append({"title": str(ch.get("title", "")).strip() or "Chapter", "nums": nums})
    if not chapters or total == 0:
        return None
    if (total - unknown) / total < 0.85:
        return None
    if len(seen) < 0.9 * target_min or len(seen) > 1.1 * target_max:
        return None
    return chapters


def _fallback_outline(entries: dict[int, str]) -> list[dict]:
    """Deterministic outline when the agents deliver none: one chapter with
    all selected blocks in order. Guarantees full coverage."""
    return [{"title": "Contents", "nums": list(entries)}]


def _with_remainder(plan: list[dict], entries: dict[int, str]) -> list[dict]:
    """Ensures that EVERY selected block is in the plan — missing ones land in
    an "Additional" chapter (against agents/judges that drop blocks)."""
    present = {num for ch in plan for num in ch.get("nums", [])}
    missing = [num for num in entries if num not in present]
    return [*plan, {"title": "Additional", "nums": missing}] if missing else plan


def _facts_grounding(subs_raw: dict[str, list[dict]]) -> str:
    """Verified sub-facts (extract-once from the blocks phase) as a grounding block for the
    content agent. Empty if no facts are stored (legacy data → fallback to a source hint)."""
    blocks = []
    for title, subs in subs_raw.items():
        lines = []
        for s in subs:
            fk = s.get("facts") if isinstance(s.get("facts"), dict) else None
            if not fk:
                continue
            parts = []
            if fk.get("key_points"):
                parts.append("Core: " + " · ".join(fk["key_points"]))
            for bf in fk.get("cited_facts", []):
                parts.append(f"FACT[{bf.get('source', '?')}]: {bf.get('text', '')}")
            if fk.get("prerequisites"):
                parts.append("Prerequisite: " + fk["prerequisites"])
            if fk.get("hurdles"):
                parts.append("Hurdle: " + fk["hurdles"])
            if fk.get("example_idea"):
                parts.append("Example: " + fk["example_idea"])
            if parts:
                lines.append(f"- {s['title']}: " + " | ".join(parts))
        if lines:
            blocks.append(f"BLOCK: {title}\n" + "\n".join(lines))
    if not blocks:
        return ""
    return ("VERIFIED FACTS per subblock — binding basis. Quote cited facts (FACT[Source]) "
            "VERBATIM, invent nothing extra, do NOT re-research. Use examples as examples, "
            "never as fact.\n\n" + "\n\n".join(blocks))


async def _outline_from_db(topic: str, sel_entries: dict[int, str]) -> list[dict] | None:
    """Read the outline from the blocks artifact (DB) and map it onto the selected blocks.
    Title-based (robust against number drift): blocks outside the selection are ignored
    (format filter), missing ones are added later by _with_remainder. None → no artifact (legacy)."""
    raw = await get_outline(topic)
    if not raw:
        return None
    try:
        data = json.loads(raw)
    except (ValueError, TypeError):
        return None
    chapters = data.get("chapters") if isinstance(data, dict) else None
    if not isinstance(chapters, list):
        return None
    norm_to_num = {_norm_title(_title(t)): num for num, t in sel_entries.items()}
    plan, seen = [], set()
    for ch in chapters:
        if not isinstance(ch, dict):
            continue
        nums = []
        for bt in ch.get("blocks", []):
            num = norm_to_num.get(_norm_title(str(bt)))
            if num is not None and num not in seen:
                seen.add(num)
                nums.append(num)
        if nums:
            plan.append({"title": str(ch.get("title", "")).strip() or "Chapter", "nums": nums})
    return plan or None


async def _generate_sections(
    guide_id: str, topic: str, format_name: str, entries: dict[int, str],
    facts: str, instructions: str, provider: str,
    content_path: Path,
) -> list[dict] | None:
    def is_cancelled() -> bool:
        return is_guide_cancelled(guide_id)

    ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled, guide_id=guide_id)
    spec = (TEMPLATES_DIR / "Format" / "Section.md").read_text(encoding="utf-8")
    files = _guide_files(content_path)
    zweck = FORMAT_PURPOSE[format_name]

    # Subblocks per block (DB-first) — loaded early: drives selection + sub-filter per format.
    # Missing → {} (fallback: guide takes everything).
    subs_raw = await _load_subblocks(topic)
    # Extract-once grounding: stored, verified facts replace the generic source hint.
    # The content agent phrases from them instead of reading the source again.
    if (facts_block := _facts_grounding(subs_raw)):
        facts = facts_block

    def _has_relevance(num, kind):
        return any(isinstance(s, dict) and s.get("relevance") == kind for s in subs_raw.get(_title(entries[num]), []))

    # Selection: ONE full document with ALL blocks (incl. peripheral). The views E/M/S/F
    # filter later per subblock level. (FullGuide/Rest remain as legacy branches.)
    if format_name == "Rest":
        selection = [num for num in entries if not _has_relevance(num, "relevant")]
    else:  # Guide / FullGuide → all blocks
        selection = list(entries)
    if not selection:
        await _fail(guide_id, "No matching blocks for this format")
        return None

    sel_entries = {num: entries[num] for num in selection}
    target = len(sel_entries)
    # Numbered list (ID = block number from entries) — agents/judge order by number.
    sel_list = "\n".join(f"{num}. {t}" for num, t in sel_entries.items())

    # Step 0: outline. Prefers the blocks artifact (DB) — the guide only presents,
    # no longer structures itself. Missing (legacy) → previous agents/judge logic as fallback.
    # 0 valid → code fallback, 1 → direct, ≥2 → judge (with proposal as fallback).
    plan = await _outline_from_db(topic, sel_entries)
    if plan is not None:
        _log(topic, f"Outline from blocks artifact ({len(plan)} chapters)")
    if plan is None:
        plan = _resolve_outline(_json_file(files["outline"]), sel_entries, target, target)
    if plan is None:
        await _set_step(guide_id, 0, "Outline proposals (3 agents)…")
        files["outline"].unlink(missing_ok=True)
        proposals: list[list[dict]] = []
        pending = []
        for i, path in enumerate(files["outline_slots"], 1):
            res = _resolve_outline(_json_file(path), sel_entries, target, target)
            if res is not None:
                proposals.append(res)
            else:
                pending.append((i, path))
        if len(proposals) < 3 and pending:
            slots = [
                {
                    "key": f"{guide_id}-outline-{i}",
                    "prompt": _prompt(
                        "Guide-Outline",
                        topic=topic, format_name=format_name, blocks=sel_list,
                        out_path=path, extra=_extra(instructions),
                    ),
                    "role": "guide", "capabilities": "files",
                    "payload": (lambda result, p=path: _resolve_outline(_json_file(p), sel_entries, target, target)),
                }
                for i, path in pending
            ]
            # Quorum 1: take whatever comes — no minimum requirement, no abort.
            new = await _race(
                topic, "Outline", slots, 1, _timeout("plan", target),
                provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE,
            )
            if is_cancelled():
                return None
            proposals += new or []

        if not proposals:
            _log(topic, "Outline: no valid proposal — deterministic fallback")
            plan = _fallback_outline(sel_entries)
        elif len(proposals) == 1:
            plan = proposals[0]  # one proposal → no judge needed
        else:
            await _set_step(guide_id, 0, "Merging outlines…")
            proposals_text = "\n\n".join(
                f"### Proposal {i}\n"
                + "\n".join(f"CHAPTER: {ch['title']}\n  Numbers: {', '.join(str(num) for num in ch['nums'])}" for ch in v)
                for i, v in enumerate(proposals, 1)
            )
            status, plan = await run_single_slot(
                ctx, "Outline-Judge",
                key=f"{guide_id}-outline-judge",
                prompt=_prompt(
                    "Guide-Outline-Judge",
                    topic=topic, format_name=format_name, purpose=zweck, n=len(proposals),
                    blocks=sel_list, outlines=proposals_text,
                    out_path=files["outline"], extra=_extra(instructions),
                ),
                role="judge", capabilities="files",
                payload=lambda result: _resolve_outline(_json_file(files["outline"]), sel_entries, target, target),
                timeout=_timeout("plan_judge", target),
            )
            if status == CANCELLED:
                return None
            if status == FAILED or plan is None:
                _log(topic, "Outline judge produced no result — best proposal kept")
                plan = proposals[0]

    # Guarantee: every selected block is in the plan (against dropping agents/judges).
    plan = _with_remainder(plan, sel_entries)
    _set_done(content_path, 0)  # outline ready

    # Coarse chunks (~GUIDE_CHUNK blocks per agent) for content, content-check and reading-exam.
    # The writer builds per block beneath (its own fine chunks, see below) → variable lengths.
    total_sections = sum(len(c["nums"]) for c in plan)
    chunks = _split_chunks(plan, max(1, math.ceil(total_sections / GUIDE_CHUNK)))
    # Subblocks per block: Guide/FullGuide take ALL (incl. peripheral → level 4 in the view);
    # only the legacy Rest branch filters to peripheral. So the one document carries all levels.
    if format_name == "Rest":
        subs_by_title = {t: [s for s in subs if s.get("relevance") == "peripheral"] for t, subs in subs_raw.items()}
    else:  # Guide / FullGuide
        subs_by_title = {t: list(subs) for t, subs in subs_raw.items()}
    subs_by_title = {t: subs for t, subs in subs_by_title.items() if subs}
    assignments = [_assignment_subs(chunk, entries, subs_by_title) for chunk in chunks]
    chunk_sizes = [sum(len(c["nums"]) for c in chunk) for chunk in chunks]
    writer_count = len(chunks)
    idx = _title_index(entries)

    # Step 2: identify content per block — one agent per chunk (marker output, resume).
    content_paths = [content_path.parent / f"{content_path.stem}.content-chunk-{i}.md" for i in range(1, writer_count + 1)]
    pending = [i for i, p in enumerate(content_paths) if not p.exists()]
    if pending:
        async def report(d, t): await _set_step(guide_id, 1, f"Gathering content {d}/{t}…")
        results = await _gather_progress([
            run_agent(
                f"{guide_id}-content-{i + 1}",
                _prompt(
                    "Guide-Content",
                    topic=topic, assignment=assignments[i], facts=facts,
                    out_path=content_paths[i], extra=_extra(instructions),
                ),
                _timeout("content", chunk_sizes[i]), provider=provider, role="guide", capabilities="full", scope=topic,
            )
            for i in pending
        ], writer_count, report, start=writer_count - len(pending))
        if is_cancelled():
            return None
        if not any(p.exists() for p in content_paths):
            await _fail(guide_id, _gather_error("Content error", list(results)))
            return None

    content_by_num: dict[int, str] = {}
    for p in content_paths:
        if not p.exists():
            continue
        for sec in _parse_fragment(p.read_text(encoding="utf-8")):
            num = _resolve_title(idx, sec["title"])
            if num is not None and num not in content_by_num and sec["md"].strip():
                content_by_num[num] = sec["md"]
    if not content_by_num:
        await _fail(guide_id, "No content identified")
        return None

    # Follow-up round: pull missing blocks (chunk failure or lazy output) deliberately — one round.
    planned_nums = [num for ch in plan for num in ch["nums"]]
    missing = [num for num in planned_nums if num not in content_by_num]
    if missing:
        _log(topic, f"Content: {len(missing)} block(s) missing — follow-up round…")
        followup_chunks = [[{"title": "Additional", "nums": missing[k:k + GUIDE_CHUNK]}] for k in range(0, len(missing), GUIDE_CHUNK)]
        followup_paths = [content_path.parent / f"{content_path.stem}.content-nach-{k}.md" for k in range(1, len(followup_chunks) + 1)]
        followup_pending = [k for k, p in enumerate(followup_paths) if not p.exists()]
        if followup_pending:
            async def report_n(d, t): await _set_step(guide_id, 1, f"Gathering missing content {d}/{t}…")
            await _gather_progress([
                run_agent(
                    f"{guide_id}-content-nach-{k + 1}",
                    _prompt(
                        "Guide-Content",
                        topic=topic, assignment=_assignment_subs(followup_chunks[k], entries, subs_by_title),
                        facts=facts, out_path=followup_paths[k], extra=_extra(instructions),
                    ),
                    _timeout("content", len(followup_chunks[k][0]["nums"])), provider=provider, role="guide", capabilities="full", scope=topic,
                )
                for k in followup_pending
            ], len(followup_chunks), report_n, start=len(followup_chunks) - len(followup_pending))
            if is_cancelled():
                return None
        for p in followup_paths:
            if not p.exists():
                continue
            for sec in _parse_fragment(p.read_text(encoding="utf-8")):
                num = _resolve_title(idx, sec["title"])
                if num is not None and num not in content_by_num and sec["md"].strip():
                    content_by_num[num] = sec["md"]

    if all(p.exists() for p in content_paths):
        _set_done(content_path, 1)  # content complete

    content_chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in content_by_num] for chunk in chunks]

    # Step 3: check content — CHECK_PANEL judges per chunk, majority flags.
    # + revise flagged ones once. Resume: only restart missing judge files.
    check_judge_paths = [
        [content_path.parent / f"{content_path.stem}.content-check-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)]
        for i in range(1, writer_count + 1)
    ]
    pending_slots = [
        (i, j) for i in range(writer_count) if content_chunk_nums[i]
        for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None
    ]
    if pending_slots:
        await _set_step(guide_id, 2, "Checking content…")
        sections_per_chunk = {
            i: "\n\n".join(f"SECTION: {_title(entries[num])}\n{content_by_num[num]}" for num in content_chunk_nums[i])
            for i, _ in pending_slots
        }
        slots = [{
            "key": f"{guide_id}-content-check-{i + 1}-j{j + 1}",
            "prompt": _prompt(
                "Guide-Content-Check",
                topic=topic, format_name=format_name, sections=sections_per_chunk[i],
                out_path=check_judge_paths[i][j], extra=_extra(instructions),
            ),
            "role": "judge", "capabilities": "files",
            "payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))),
        } for i, j in pending_slots]
        n_checks = len(slots)
        upd = lambda n: asyncio.create_task(_set_step(guide_id, 2, f"Checking content {n}/{n_checks}…"))
        await _race(topic, "Content-Exam", slots, len(slots), _timeout("content_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
        if is_cancelled():
            return None

    problems_by_num: dict[int, str] = {}
    for i in range(writer_count):
        if content_chunk_nums[i]:
            problems_by_num.update(_panel_problems(check_judge_paths[i], set(content_chunk_nums[i]), idx))

    if problems_by_num:
        _log(topic, f"Content exam: {len(problems_by_num)} block(s) flagged")
        await _set_step(guide_id, 2, f"Revising {len(problems_by_num)} content(s)…")
        fix_chunks = [[num for num in nums if num in problems_by_num] for nums in content_chunk_nums]
        fix_paths = [content_path.parent / f"{content_path.stem}.content-fix-{i + 1}.md" for i in range(writer_count)]
        fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()]
        results = await asyncio.gather(*[
            run_agent(
                f"{guide_id}-content-fix-{i + 1}",
                _prompt(
                    "Guide-Content-Fix",
                    topic=topic, facts=facts,
                    tasks="\n\n".join(
                        f"SECTION: {_title(entries[num])}\nPROBLEM: {problems_by_num[num]}\nCURRENT:\n{content_by_num[num]}"
                        for num in fix_chunks[i]
                    ),
                    out_path=fix_paths[i], extra=_extra(instructions),
                ),
                _timeout("content", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full", scope=topic,
            )
            for i in fix_pending
        ], return_exceptions=True)
        if is_cancelled():
            return None
        for p in fix_paths:
            if not p.exists():
                continue
            for sec in _parse_fragment(p.read_text(encoding="utf-8")):
                num = _resolve_title(idx, sec["title"])
                if num in problems_by_num and sec["md"].strip():
                    content_by_num[num] = sec["md"]

    _set_done(content_path, 2)  # content check done

    # Step 4: writing — the writer phrases out the checked content (resume).
    # FINE chunks: exactly 1 block per writer → variable lengths, no budget rationing.
    def content_text(chunk) -> str:
        nums = [num for ch in chunk for num in ch["nums"] if num in content_by_num]
        return "\n\n".join(f"<!-- section: {_title(entries[num])} -->\n{content_by_num[num]}" for num in nums)

    w_chunks = [[{"title": ch["title"], "nums": [num]}] for ch in plan for num in ch["nums"]]
    w_assignments = [_assignment_subs(c, entries, subs_by_title) for c in w_chunks]
    paths = [content_path.parent / f"{content_path.stem}.chunk-{i}.md" for i in range(1, len(w_chunks) + 1)]
    pending = [i for i, p in enumerate(paths) if not p.exists()]
    if pending:
        async def report(d, t): await _set_step(guide_id, 3, f"Writing sections {d}/{t}…")
        results = await _gather_progress([
            run_agent(
                f"{guide_id}-w{i + 1}",
                _prompt(
                    "Guide-Writer",
                    topic=topic, format_name=format_name, assignment=w_assignments[i],
                    contents=content_text(w_chunks[i]),
                    spec=spec, out_path=paths[i], extra=_extra(instructions),
                ),
                _timeout("writer", 1), provider=provider, role="guide", capabilities="files", scope=topic,
            )
            for i in pending
        ], len(w_chunks), report, start=len(w_chunks) - len(pending))
        if is_cancelled():
            return None
        for i, r in zip(pending, results):
            if isinstance(r, BaseException):
                _log(topic, f"Writer {i + 1}: {type(r).__name__}: {r}")
            elif r[0] != 0:
                _log(topic, f"Writer {i + 1}: {_claude_error('Error', *r)}")
            elif not paths[i].exists():
                _log(topic, f"Writer {i + 1}: no output file created")
        if not any(p.exists() for p in paths):
            await _fail(guide_id, _gather_error("Writer error", list(results)))
            return None

    by_num: dict[int, dict] = {}
    for p in paths:
        if not p.exists():
            continue
        for sec in _parse_fragment(p.read_text(encoding="utf-8")):
            num = _resolve_title(idx, sec["title"])
            if num is None:
                _log(topic, f"Writer produced unknown section '{sec['title'][:40]}' (ignored)")
            elif num not in by_num:
                by_num[num] = sec
    if not by_num:
        await _fail(guide_id, "No sections found in writer output")
        return None

    # Follow-up round: write missing sections (writer failure) deliberately — one round.
    missing_after = [num for num in planned_nums if num not in by_num]
    if missing_after:
        _log(topic, f"Writing: {len(missing_after)} section(s) missing — follow-up round…")
        nw_chunks = [[{"title": "Additional", "nums": [num]}] for num in missing_after]
        nw_paths = [content_path.parent / f"{content_path.stem}.chunk-nach-{k}.md" for k in range(1, len(nw_chunks) + 1)]
        nw_pending = [k for k, p in enumerate(nw_paths) if not p.exists()]
        if nw_pending:
            async def report_nw(d, t): await _set_step(guide_id, 3, f"Writing missing sections {d}/{t}…")
            await _gather_progress([
                run_agent(
                    f"{guide_id}-w-nach-{k + 1}",
                    _prompt(
                        "Guide-Writer",
                        topic=topic, format_name=format_name, assignment=_assignment_subs(nw_chunks[k], entries, subs_by_title),
                        contents=content_text(nw_chunks[k]), spec=spec, out_path=nw_paths[k], extra=_extra(instructions),
                    ),
                    _timeout("writer", 1), provider=provider, role="guide", capabilities="files", scope=topic,
                )
                for k in nw_pending
            ], len(nw_chunks), report_nw, start=len(nw_chunks) - len(nw_pending))
            if is_cancelled():
                return None
        for p in nw_paths:
            if not p.exists():
                continue
            for sec in _parse_fragment(p.read_text(encoding="utf-8")):
                num = _resolve_title(idx, sec["title"])
                if num is not None and num not in by_num and sec["md"].strip():
                    by_num[num] = sec

    if all(p.exists() for p in paths):
        _set_done(content_path, 3)  # writing complete

    # Step 3: reading-exam loop — check per writer packet, fix only for
    # flagged sections; follow-up rounds check ONLY the replaced sections.
    # After the round cap, open complaints stand.
    chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in by_num] for chunk in chunks]

    def sections_text(nums: list[int]) -> str:
        return "\n\n".join(f"SECTION: {_title(entries[num])}\n{by_num[num]['md']}" for num in nums)

    def _sub_list(num: int) -> str:
        subs = subs_by_title.get(_title(entries[num]), [])
        return "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)"

    def tasks_text(nums: list[int], problems: dict[int, str]) -> str:
        return "\n\n".join(
            f"SECTION: {_title(entries[num])}\n"
            f"SUBBLOCKS (set one `<!-- sub: LABEL | title -->` marker each, label/order as here):\n{_sub_list(num)}\n"
            f"PROBLEM: {problems[num]}\nCURRENT CONTENT:\n{by_num[num]['md']}"
            for num in nums
        )

    scope = chunk_nums
    for round_no in range(1, READING_ROUNDS + 1):
        # CHECK_PANEL judges per packet; majority flags. Aggregation robust against a single failure.
        check_judge_paths = [
            [content_path.parent / f"{content_path.stem}.lese-check-r{round_no}-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)]
            for i in range(1, writer_count + 1)
        ]
        pending_slots = [
            (i, j) for i in range(writer_count) if scope[i]
            for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None
        ]
        if pending_slots:
            await _set_step(guide_id, 4, "Checking readability…")
            sections_per_chunk = {i: sections_text(scope[i]) for i, _ in pending_slots}
            slots = [{
                "key": f"{guide_id}-lese-check-r{round_no}-{i + 1}-j{j + 1}",
                "prompt": _prompt(
                    "Guide-Lese-Check",
                    topic=topic, format_name=format_name, spec=spec,
                    sections=sections_per_chunk[i],
                    out_path=check_judge_paths[i][j], extra=_extra(instructions),
                ),
                "role": "judge", "capabilities": "files",
                "payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))),
            } for i, j in pending_slots]
            n_checks = len(slots)
            upd = lambda n: asyncio.create_task(_set_step(guide_id, 4, f"Checking readability {n}/{n_checks}…"))
            res = await _race(topic, f"Reading-Exam r{round_no}", slots, len(slots), _timeout("lese_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
            if is_cancelled():
                return None
            if res is None:
                _log(topic, f"Reading exam round {round_no}: no full quorum — aggregated available judges")

        problems_by_num: dict[int, str] = {}
        for i in range(writer_count):
            if scope[i]:
                problems_by_num.update(_panel_problems(check_judge_paths[i], set(scope[i]), idx))

        # Deterministic readability gate: queue too-hard sections into the same
        # revision (LLM complaint takes precedence). Gate off → no-op.
        if READABILITY_ACTIVE:
            md_by_num = {num: by_num[num]["md"] for nums in scope for num in nums if num in by_num}
            hints = await asyncio.to_thread(readability.rate_sections, md_by_num)
            if hints:
                _log(topic, f"Readability: {len(hints)} section(s) too hard")
            for num, hint in hints.items():
                problems_by_num.setdefault(num, hint)

        if not problems_by_num:
            break

        _log(topic, f"Reading exam round {round_no}: {len(problems_by_num)} section(s) flagged")
        await _set_step(guide_id, 4, f"Revising {len(problems_by_num)} section(s) (round {round_no})…")
        fix_chunks = [[num for num in nums if num in problems_by_num] for nums in chunk_nums]
        fix_paths = [content_path.parent / f"{content_path.stem}.fix-r{round_no}-{i + 1}.md" for i in range(writer_count)]
        fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()]
        results = await asyncio.gather(*[
            run_agent(
                f"{guide_id}-fix-r{round_no}-w{i + 1}",
                _prompt(
                    "Guide-Sections-Fix",
                    topic=topic, format_name=format_name, facts=facts, spec=spec,
                    tasks=tasks_text(fix_chunks[i], problems_by_num),
                    out_path=fix_paths[i], extra=_extra(instructions),
                ),
                _timeout("writer", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full", scope=topic,
            )
            for i in fix_pending
        ], return_exceptions=True)
        if is_cancelled():
            return None
        for i, r in zip(fix_pending, results):
            if isinstance(r, BaseException) or (not isinstance(r, BaseException) and r[0] != 0):
                _log(topic, f"Sections fix {i + 1} (round {round_no}) failed — original kept")
        replaced: set[int] = set()
        for p in fix_paths:
            if not p.exists():
                continue
            for sec in _parse_fragment(p.read_text(encoding="utf-8")):
                num = _resolve_title(idx, sec["title"])
                if num not in problems_by_num or not sec["md"].strip():
                    continue
                # Marker invariant: if the fix loses the sub markers although the original had
                # some, it is discarded — otherwise the level filter (E/M/S/F) dies silently.
                if by_num[num].get("subs") and not sec.get("subs"):
                    _log(topic, f"Reading fix for '{sec['title']}' without sub markers — discarded, tagged original kept")
                    continue
                by_num[num] = sec
                replaced.add(num)
        _log(topic, f"Reading exam round {round_no}: {len(replaced)} section(s) revised")
        if not replaced:
            break
        if round_no == READING_ROUNDS:
            _log(topic, f"Reading exam: 1 round — revision stays unchecked")
            break
        scope = [[num for num in nums if num in replaced] for nums in chunk_nums]
    _set_done(content_path, 4)  # reading exam done

    # Checkable = format has an exam AND the block has ≥1 relevant subblock.
    # Guide is always checkable (even without relevance data, fallback = everything).
    def _checkable(num):
        if format_name == "Guide":
            return True
        if format_name == "FullGuide":
            return any(isinstance(s, dict) and s.get("relevance") == "relevant"
                       for s in subs_raw.get(_title(entries[num]), []))
        return False  # Rest etc. → pure reading sections

    await _set_progress(guide_id, "Assembling…")
    chapters: list[dict] = []
    for ch in plan:
        sections = [
            {"num": num, "title": _title(entries[num]), "md": by_num[num]["md"],
             "compact": by_num[num].get("compact", ""),
             "anchor": by_num[num].get("anchor", ""), "anker_compact": by_num[num].get("anker_compact", ""),
             "subs": by_num[num].get("subs", []), "checkable": _checkable(num)}
            for num in ch["nums"] if num in by_num
        ]
        if sections:
            chapters.append({"title": ch["title"], "sections": sections})
    planned = {num for ch in plan for num in ch["nums"]}
    missing = sorted(planned - set(by_num))
    if missing:
        _log(topic, f"Sections missing from writer output: {[_title(entries[n]) for n in missing]}")
    if not chapters:
        await _fail(guide_id, "No sections found in writer output")
        return None
    return chapters


_LEVEL_RANK = {"beginner": 1, "advanced": 2, "expert": 3, "peripheral": 4,
               "easy": 1, "medium": 2, "hard": 3}  # old values backward-compatible


def _section_for_level(sec: dict, level: int) -> dict:
    """Reconstruct md/compact of a section from subblocks up to the level (anchor stays)."""
    subs = sec.get("subs") or []
    if not subs:
        return sec  # no sub tags (legacy) → unchanged, visible
    visible = [s for s in subs if _LEVEL_RANK.get(s.get("level"), 1) <= level]
    md = "\n\n".join(t for t in [sec.get("anchor", ""), *(s.get("md", "") for s in visible)] if t).strip()
    compact = "\n".join(t for t in [sec.get("anker_compact", ""), *(s.get("compact", "") for s in visible)] if t).strip()
    return {**sec, "md": md, "compact": compact, "leer": not visible}


def content_fuer_level(content: dict, level: int) -> dict:
    """Filter guide content to a view level (1=A · 2=F · 3=E · 4=V). Level 4 = full version.
    Sections without visible subs are hidden, empty chapters drop out."""
    if not isinstance(content, dict) or level >= 4:
        return content
    chapters = []
    for ch in content.get("chapters", []):
        secs = [s for s in (_section_for_level(x, level) for x in ch.get("sections", [])) if not s.get("leer")]
        if secs:
            chapters.append({**ch, "sections": secs})
    return {**content, "chapters": chapters}


async def reconcile_guides() -> None:
    """Reconcile DB↔filesystem: status=done without content file → error.

    Runs at server start (after init_db) — catches crashes between
    file write and status update.
    """
    for g in await list_guides():
        if g["status"] == "done" and not guide_content_path(g["topic"], g["format"]).exists():
            log.warning("[%s] Guide %s: done without content file — set to error", g["topic"], g["id"])
            now = datetime.now(timezone.utc).isoformat()
            await update_guide(g["id"], status="error", error_msg="Content missing — regenerate", updated_at=now)


async def generate_guide(guide_id: str, topic: str, format_name: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, ab_step: int | None = None) -> None:
    async with _semaphore:
        now = datetime.now(timezone.utc).isoformat()
        await update_guide(guide_id, status="generating", progress="Starting…", updated_at=now)

        content_path = guide_content_path(topic, format_name)
        content_path.parent.mkdir(parents=True, exist_ok=True)
        project = source_folder(topic)  # folder source (project/uni/link) → path, else None

        try:
            if is_guide_cancelled(guide_id):
                return

            if project:
                await asyncio.to_thread(_convert_pdfs, project)

            # Re-run from step: delete content + slots from `ab_step`, rest stays → resume rebuilds from there.
            # Otherwise "recreate": a finished guide → complete fresh start.
            # Otherwise step files are leftovers of an abort/error → resume.
            if ab_step is not None:
                _reset_guide_from_step(content_path, ab_step)
            elif content_path.exists():
                for p_alt in guide_slot_files(content_path):
                    p_alt.unlink(missing_ok=True)

            bs = await list_blocks(topic, status="consensus")
            if bs:
                alle = {i: (f"{b['title']} — {b['description']}" if b["description"] else b["title"])
                        for i, b in enumerate(bs, 1)}
            else:  # fallback: blocks.md (legacy topics)
                bp = blocks_path(topic)
                alle = _load_blocks(bp.read_text(encoding="utf-8")) if bp.exists() else {}
            if not alle:
                await _fail(guide_id, "No blocks found")
                return
            entries = _unique_title(alle)
            facts = _prompt("Guide-Facts-Projekt", project=project) if project else _prompt("Guide-Facts-Thema")
            chapters = await _generate_sections(
                guide_id, topic, format_name, entries,
                facts, instructions, provider, content_path,
            )
            if chapters is None or is_guide_cancelled(guide_id):
                return
            content = {"topic": topic, "format": format_name, "chapters": chapters}

            atomic_write_json(content_path, content, indent=1)  # bridge (resume/fallback)
            await set_guide_content(topic, format_name, json.dumps(content, ensure_ascii=False))

            now = datetime.now(timezone.utc).isoformat()
            await update_guide(guide_id, status="done", progress=None, step=None, updated_at=now)

        except asyncio.TimeoutError:
            await _fail(guide_id, "Timeout during generation")
        except FileNotFoundError:
            await _fail(guide_id, "Blocks missing")
        except Exception as e:
            log.exception("[%s] Guide generation failed (%s)", topic, guide_id)
            await _fail(guide_id, str(e)[:2000])
        finally:
            clear_guide_cancelled(guide_id)


# --- On-demand: check / fix / rewrite one section (focus, interactive) ---

SECTION_CHECK_TIMEOUT = 300


def _section_spec() -> str:
    return (TEMPLATES_DIR / "Format" / "Section.md").read_text(encoding="utf-8")


def _section_facts(topic: str) -> str:
    project = source_folder(topic)
    return _prompt("Guide-Facts-Projekt", project=project) if project else _prompt("Guide-Facts-Thema")


def _hint_block(hint: str) -> str:
    hint = (hint or "").strip()
    return f"NOTE FROM THE USER (pay special attention):\n{hint}" if hint else ""


async def _subs_text(topic: str, block: str) -> str:
    """Relevant subblocks of a block with level — as a checklist for the agents."""
    subs_raw = await _load_subblocks(topic)
    subs = [s for s in subs_raw.get(_title(block), []) if s.get("relevance") != "peripheral"]
    if not subs:
        return "(no subblocks recorded — cover 3–7 concise points)"
    return "\n".join(f"- [{s['level']}] {s['title']}" for s in subs)


async def _load_guide_content(topic: str, format_name: str) -> dict | None:
    js = await get_guide_content(topic, format_name)
    if not js:
        return None
    try:
        return json.loads(js)
    except ValueError:
        return None


def _find_section(content: dict, block: str) -> dict | None:
    for ch in content.get("chapters", []):
        for s in ch.get("sections", []):
            if s.get("title") == block:
                return s
    return None


async def block_pruefen(topic: str, format_name: str, block: str, spot: str, snippet: str, hint: str = "", provider: str = DEFAULT_PROVIDER) -> str | None:
    """Check one section (Markdown block) against the guide rules → corrected
    block version as Markdown. None = error/section missing."""
    content = await _load_guide_content(topic, format_name)
    sec = _find_section(content, block) if content else None
    if sec is None:
        return None
    whole = sec.get("compact", "") if str(spot).startswith("compact") else sec.get("md", "")
    prompt = _prompt(
        "Block-Pruefen", topic=topic, spec=_section_spec(), facts=_section_facts(topic),
        subblocks=await _subs_text(topic, block), context=whole, snippet=snippet, hint=_hint_block(hint),
    )
    rc, stdout, _ = await run_agent(
        f"block-pruefen-{uuid.uuid4()}", prompt, SECTION_CHECK_TIMEOUT,
        provider=provider, role="judge", capabilities="none", lane="interactive",
    )
    new = stdout.strip() if rc == 0 else ""
    return new or None


async def block_adopt(topic: str, format_name: str, block: str, spot: str, old: str, new: str) -> dict | None:
    """Replace one block (old→new) in the compact/detailed field + persist.
    → {compact, md, found}; None = section missing."""
    content = await _load_guide_content(topic, format_name)
    sec = _find_section(content, block) if content else None
    if sec is None:
        return None
    is_compact = str(spot).startswith("compact")
    field = "compact" if is_compact else "md"
    current = sec.get(field, "") or ""
    found = old in current
    if found:
        sec[field] = current.replace(old, new, 1)
        # Also replace in anchor + subs (sources of the filtered E/M/S view), otherwise the
        # leveled view keeps showing the old block.
        anchor_field = "anker_compact" if is_compact else "anchor"
        if old in (sec.get(anchor_field) or ""):
            sec[anchor_field] = sec[anchor_field].replace(old, new, 1)
        for sub in sec.get("subs", []):
            if old in (sub.get(field, "") or ""):
                sub[field] = sub[field].replace(old, new, 1)
                break
        js = json.dumps(content, ensure_ascii=False)
        await set_guide_content(topic, format_name, js)
        atomic_write_json(guide_content_path(topic, format_name), content, indent=1)
    return {"compact": sec.get("compact", ""), "md": sec.get("md", ""), "found": found}