From 41c9f29a37acb2790c45e699a194a432c657d924 Mon Sep 17 00:00:00 2001 From: team3 Date: Thu, 2 Jul 2026 03:05:57 +0200 Subject: [PATCH] update --- backend/agents.py | 204 ++- backend/blocks.py | 1675 +------------------ backend/board_artefacts.py | 333 ++++ backend/board_inventory.py | 1395 +++++++++++++++ backend/config.py | 68 +- backend/database.py | 446 +++++ backend/guide.py | 670 +------- backend/guide_board.py | 521 ++++++ backend/kanban.py | 265 +++ backend/models.py | 17 +- backend/pipeline.py | 36 +- backend/pytest.ini | 3 + backend/routes.py | 114 +- backend/tests/conftest.py | 18 + backend/tests/test_board_inventory.py | 210 +++ backend/tests/test_guide_board.py | 68 + backend/tests/test_kanban.py | 128 ++ backend/tests/test_roles.py | 41 + frontend/src/App.vue | 85 +- frontend/src/api.js | 43 +- frontend/src/components/BlocksOverview.vue | 231 +-- frontend/src/components/GuideBoard.vue | 187 +++ frontend/src/components/KanbanBoard.vue | 191 +++ frontend/src/components/TopicSidebar.vue | 41 +- templates/Prompt/Blocks-Block-Grouping.md | 20 - templates/Prompt/Blocks-Naming-Check.md | 15 + templates/Prompt/Blocks-Naming.md | 15 + templates/Prompt/Blocks-Research-Mapping.md | 16 - templates/Prompt/Guide-Content-Check.md | 19 - templates/Prompt/Guide-Content-Fix.md | 18 - templates/Prompt/Guide-Content.md | 30 - templates/Prompt/Guide-Coverage.md | 19 + templates/Prompt/Guide-Fakten-Fix.md | 19 + templates/Prompt/Guide-Fakten-Gate.md | 21 + templates/Prompt/Guide-Lernziele.md | 23 + templates/Prompt/Guide-Lese-Check.md | 2 +- templates/Prompt/Guide-Writer-Board.md | 47 + templates/Prompt/Guide-Writer.md | 51 - 38 files changed, 4671 insertions(+), 2634 deletions(-) create mode 100644 backend/board_artefacts.py create mode 100644 backend/board_inventory.py create mode 100644 backend/guide_board.py create mode 100644 backend/kanban.py create mode 100644 backend/pytest.ini create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/test_board_inventory.py create mode 100644 backend/tests/test_guide_board.py create mode 100644 backend/tests/test_kanban.py create mode 100644 backend/tests/test_roles.py create mode 100644 frontend/src/components/GuideBoard.vue create mode 100644 frontend/src/components/KanbanBoard.vue delete mode 100644 templates/Prompt/Blocks-Block-Grouping.md create mode 100644 templates/Prompt/Blocks-Naming-Check.md create mode 100644 templates/Prompt/Blocks-Naming.md delete mode 100644 templates/Prompt/Blocks-Research-Mapping.md delete mode 100644 templates/Prompt/Guide-Content-Check.md delete mode 100644 templates/Prompt/Guide-Content-Fix.md delete mode 100644 templates/Prompt/Guide-Content.md create mode 100644 templates/Prompt/Guide-Coverage.md create mode 100644 templates/Prompt/Guide-Fakten-Fix.md create mode 100644 templates/Prompt/Guide-Fakten-Gate.md create mode 100644 templates/Prompt/Guide-Lernziele.md create mode 100644 templates/Prompt/Guide-Writer-Board.md delete mode 100644 templates/Prompt/Guide-Writer.md diff --git a/backend/agents.py b/backend/agents.py index ee3bec3..b30c033 100644 --- a/backend/agents.py +++ b/backend/agents.py @@ -2,9 +2,14 @@ Both runners are independent. If a binary/key is missing, only the respective provider fails — the other keeps running unchanged. + +Role routing: config.resolve_role maps (run_provider, role) → (provider, model) +ACROSS stacks, so one run can generate on MiniMax and judge on Claude. If the +routed provider is unavailable, the call falls back to the run's provider. """ import asyncio +import heapq import logging import os import re @@ -13,13 +18,27 @@ import signal import tempfile import time import urllib.request +from contextlib import asynccontextmanager from pathlib import Path -from config import PROVIDERS, DEFAULT_PROVIDER, MAX_CONCURRENT_AGENTS, MAX_CONCURRENT_INTERACTIVE +from config import (PROVIDERS, DEFAULT_PROVIDER, MAX_CONCURRENT_AGENTS, + MAX_CONCURRENT_AGENTS_PER_TOPIC, MAX_CONCURRENT_INTERACTIVE, + resolve_role) log = logging.getLogger("creator.agents") _active_processes: dict[str, asyncio.subprocess.Process] = {} +_active_started: dict[str, float] = {} # agent_key → wall-clock start (for the live runtime display) + + +def active_agents(scope_prefix: str | None = None) -> list[dict]: + """Currently running agents and how long they've been running. Filter by key prefix + (e.g. f"blocks-{topic}-") for one topic. → [{key, runtime}] sorted longest-first.""" + now = time.time() + out = [{"key": k, "runtime": round(now - t, 1)} + for k, t in list(_active_started.items()) + if k in _active_processes and (not scope_prefix or k.startswith(scope_prefix))] + return sorted(out, key=lambda a: -a["runtime"]) # Cancelled scopes (key prefixes, symmetric to kill_process). An agent whose # key starts with one of these prefixes aborts BEFORE the spawn — so agents WAITING @@ -41,14 +60,90 @@ def _scope_cancelled(agent_key: str) -> bool: # Caps the real CLI processes — independent of the pipeline semaphore in # generator.py. The acquire happens BEFORE the spawn so that queue wait time # does not count against the agent timeout. -_batch_sem = asyncio.Semaphore(MAX_CONCURRENT_AGENTS) +class _PrioritySemaphore: + """asyncio.Semaphore variant: when slots are scarce, the LOWEST priority number is served first + (FIFO within the same priority). Lets earlier pipeline columns grab agents before later ones.""" + def __init__(self, value: int): + self._value = value + self._waiters: list = [] # heap of [priority, seq, future] + self._seq = 0 + + async def acquire(self, priority: int = 100): + if self._value > 0: + self._value -= 1 + return + fut = asyncio.get_event_loop().create_future() + entry = [priority, self._seq, fut] + self._seq += 1 + heapq.heappush(self._waiters, entry) + try: + await fut # release() hands us the slot directly (no value change) + except BaseException: + entry[2] = None # tombstone so release() skips this dead waiter + if fut.done() and not fut.cancelled(): + self.release() # granted just before we were cancelled → pass it on + raise + + def release(self): + while self._waiters: + entry = heapq.heappop(self._waiters) + if entry[2] is not None and not entry[2].done(): + entry[2].set_result(None) # hand the slot straight to the highest-priority waiter + return + self._value += 1 + + +_batch_sem = _PrioritySemaphore(MAX_CONCURRENT_AGENTS) _interactive_sem = asyncio.Semaphore(MAX_CONCURRENT_INTERACTIVE) -# Serialize OpenCode starts: processes starting simultaneously collide on the -# internal session DB ("database is locked", exit after <1s). The short -# stagger spreads out the starts; afterwards the processes run in parallel normally. +# Per-topic caps (lazily created): each topic gets its own priority semaphore of size +# MAX_CONCURRENT_AGENTS_PER_TOPIC, nested INSIDE the global _batch_sem. Priority-based too, so the +# per-topic queue can't undo the global priority when one topic is the only load. +_topic_sems: dict[str, _PrioritySemaphore] = {} + +# Earlier kanban columns get the scarce global slot first (smaller = higher priority). +_STAGE_PRIORITY = ("research", "ingest", "cluster", "pair", "clarify", "naming", "filter", "grouping") + + +def _agent_priority(key: str) -> int: + for i, tag in enumerate(_STAGE_PRIORITY): + if f"-{tag}-" in key or key.endswith(f"-{tag}"): + return i + return len(_STAGE_PRIORITY) # downstream agents (subblocks/facts/…) after the inventory columns + + +@asynccontextmanager +async def _batch_gate(scope: str | None, priority: int): + """Per-topic slot FIRST (fair), then the GLOBAL slot by priority (earlier columns win when + agents are scarce). Order matters — a waiter holds only its per-topic slot while queueing globally.""" + topic_sem = _topic_sems.setdefault(scope, _PrioritySemaphore(MAX_CONCURRENT_AGENTS_PER_TOPIC)) if scope else None + if topic_sem is not None: + await topic_sem.acquire(priority) + await _batch_sem.acquire(priority) + try: + yield + finally: + _batch_sem.release() + if topic_sem is not None: + topic_sem.release() + +# Space OpenCode starts: processes starting simultaneously collide on the internal +# session DB ("database is locked", exit after <1s). Token bucket instead of a lock +# held through spawn+sleep: the lock only assigns a start slot, the sleep happens +# outside — a wave of starts is spaced by OPENCODE_START_DELAY without a global convoy. _opencode_start_lock = asyncio.Lock() -_OPENCODE_START_DELAY = 1.0 +_OPENCODE_START_DELAY = float(os.getenv("OPENCODE_START_DELAY", "0.5")) +_opencode_next_start = 0.0 + + +async def _opencode_slot() -> None: + global _opencode_next_start + loop = asyncio.get_running_loop() + async with _opencode_start_lock: + now = loop.time() + start_at = max(now, _opencode_next_start) + _opencode_next_start = start_at + _OPENCODE_START_DELAY + await asyncio.sleep(max(0.0, start_at - now)) # Capability → Claude --allowedTools _CLAUDE_TOOLS = { @@ -85,6 +180,22 @@ def provider_available(provider: str) -> bool: return True +# Availability cache for role routing: the routed target is probed at most once per +# TTL (check_url providers would otherwise block the loop on every call). +_avail_cache: dict[str, tuple[float, bool]] = {} +_AVAIL_TTL = 60.0 + + +def _available_cached(provider: str) -> bool: + now = time.monotonic() + hit = _avail_cache.get(provider) + if hit and now - hit[0] < _AVAIL_TTL: + return hit[1] + ok = provider_available(provider) + _avail_cache[provider] = (now, ok) + return ok + + def _kill(process) -> None: """Kill the agent and its child processes via the process group (otherwise the children spawned by the CLI survive, keep the pipes open and block communicate()).""" @@ -102,6 +213,7 @@ def kill_process(agent_key_prefix: str) -> None: for key, process in list(_active_processes.items()): if process.returncode is not None: # clean up dead entries while iterating _active_processes.pop(key, None) + _active_started.pop(key, None) continue if key.startswith(agent_key_prefix): log.debug("kill agent %s", key) @@ -116,23 +228,32 @@ async def run_agent( role: str = "fast", capabilities: str = "none", lane: str = "batch", + scope: str | None = None, + on_line=None, ) -> tuple[int, str, str]: if _scope_cancelled(agent_key): # before queueing: don't even enter the queue return 1, "", "cancelled" if provider not in PROVIDERS: return 1, "", f"Unknown provider: {provider}" + run_provider = provider + provider, model = resolve_role(run_provider, role) + if provider != run_provider and not _available_cached(provider): + provider, model = run_provider, PROVIDERS[run_provider].get(role, "") + if not model: + return 1, "", f"No model for role '{role}' (provider: {provider})" if shutil.which(PROVIDERS[provider]["cli"]) is None: return 1, "", f"CLI '{PROVIDERS[provider]['cli']}' not installed (provider: {provider})" - sem = _interactive_sem if lane == "interactive" else _batch_sem - async with sem: + gate = _interactive_sem if lane == "interactive" else _batch_gate(scope, _agent_priority(agent_key)) + async with gate: if _scope_cancelled(agent_key): # after the acquire: cancelled in the queue → no spawn return 1, "", "cancelled" + log.info("agent %s: %s %s (role %s)", agent_key, provider, model, role) if PROVIDERS[provider]["cli"] == "opencode": - return await _run_opencode(agent_key, prompt, timeout, provider, role, capabilities) - return await _run_claude_cli(agent_key, prompt, timeout, role, capabilities) + return await _run_opencode(agent_key, prompt, timeout, provider, model, capabilities, on_line=on_line) + return await _run_claude_cli(agent_key, prompt, timeout, model, capabilities) -async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, timeout: int, stagger: bool = False) -> tuple[int, str, str]: +async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, timeout: int, stagger: bool = False, on_line=None) -> tuple[int, str, str]: start = time.monotonic() async def spawn(): @@ -145,18 +266,40 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, ) if stagger: - async with _opencode_start_lock: - process = await spawn() - await asyncio.sleep(_OPENCODE_START_DELAY) - else: - process = await spawn() - _active_processes[agent_key] = process + await _opencode_slot() # spaced start slot; spawn itself is not serialized + process = await spawn() + # Collision-safe tracking: identical keys (e.g. same chunk label from parallel cards) + # get a ~n suffix — prefix-based kill/cancel still matches, nothing becomes an orphan. + track_key = agent_key + n = 2 + while track_key in _active_processes: + track_key = f"{agent_key}~{n}" + n += 1 + _active_processes[track_key] = process + _active_started[track_key] = time.time() try: try: - stdout, stderr = await asyncio.wait_for( - process.communicate(input=stdin_data), - timeout=timeout, - ) + if on_line is not None: + # Streaming path: read stdout line by line, hand each raw line to on_line LIVE. + out_chunks: list[str] = [] + + async def _pump(): + async for raw in process.stdout: + s = raw.decode("utf-8", errors="replace") + out_chunks.append(s) + try: + on_line(s) + except Exception: + log.debug("on_line callback failed", exc_info=True) + await asyncio.wait_for(_pump(), timeout=timeout) + await process.wait() + stderr_b = await process.stderr.read() + stdout, stderr = "".join(out_chunks).encode("utf-8"), stderr_b + else: + stdout, stderr = await asyncio.wait_for( + process.communicate(input=stdin_data), + timeout=timeout, + ) except asyncio.TimeoutError: _kill(process) try: @@ -173,13 +316,14 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, finally: # Pop only on identity: a slot restart under the same key must not evict # the NEW process from tracking. - if _active_processes.get(agent_key) is process: - del _active_processes[agent_key] + if _active_processes.get(track_key) is process: + del _active_processes[track_key] + _active_started.pop(track_key, None) -async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, role: str, capabilities: str) -> tuple[int, str, str]: +async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, model: str, capabilities: str) -> tuple[int, str, str]: cfg = PROVIDERS["claude"] - cmd = [cfg["cli"], "-p", "--model", cfg[role]] + cmd = [cfg["cli"], "-p", "--model", model] tools = _CLAUDE_TOOLS.get(capabilities) if tools: cmd += ["--allowedTools", tools] @@ -187,7 +331,7 @@ async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, role: str, return await _communicate(agent_key, cmd, prompt.encode("utf-8"), timeout) -async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, role: str, capabilities: str) -> tuple[int, str, str]: +async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, model: str, capabilities: str, on_line=None) -> tuple[int, str, str]: cfg = PROVIDERS[provider] # Prompt via temp file instead of argv (ARG_MAX protection for large project prompts) with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8", dir=tempfile.gettempdir()) as f: @@ -198,14 +342,16 @@ async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str cmd = [ cfg["cli"], "run", "Folge exakt den Anweisungen in der angehängten Datei. Sie sind der vollständige Auftrag.", - "-m", cfg[role], + "-m", model, "--agent", _OPENCODE_AGENTS.get(capabilities, "text"), "--dangerously-skip-permissions", "-f", str(prompt_path), ] + if on_line is not None: + cmd += ["--format", "json"] # raw JSON events → parsed live by on_line try: - rc, stdout, stderr = await _communicate(agent_key, cmd, None, timeout, stagger=True) - return rc, _clean_opencode_output(stdout), stderr + rc, stdout, stderr = await _communicate(agent_key, cmd, None, timeout, stagger=True, on_line=on_line) + return rc, (stdout if on_line is not None else _clean_opencode_output(stdout)), stderr finally: prompt_path.unlink(missing_ok=True) diff --git a/backend/blocks.py b/backend/blocks.py index 21bbae0..e771a09 100644 --- a/backend/blocks.py +++ b/backend/blocks.py @@ -24,7 +24,7 @@ from pathlib import Path import database as db import embedding from agents import kill_process, cancel_scope, clear_scope, run_agent -from config import CONSENSUS_GRACE, RESEARCH_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP, GROUP_RECONCILE_FLOOR, GROUP_MIN_COS_FLOOR +from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP, GROUP_RECONCILE_FLOOR, GROUP_MIN_COS_FLOOR from fsutil import atomic_write_text, atomic_write_json from jsonio import read_json_file as _json_file from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder @@ -242,27 +242,8 @@ PHASEN = ( ) -def _phases(topic: str) -> list[tuple[str, int]]: - """[(coarse_label, number of present fine steps)] for the current source.""" - fine_steps = _blocks_steps(topic) - return [(label, n) for label, members in PHASEN if (n := sum(f in members for f in fine_steps))] -def _phases_status(topic: str, current: int | None) -> list[dict]: - """Coarse phase states from the fine progress `current` (None = all pending, - len(feine) = all done). → [{label, state}] with state done/active/pending.""" - out, start = [], 0 - for label, n in _phases(topic): - end = start + n - if current is None or current < start: - state = "pending" - elif current >= end: - state = "done" - else: - state = "active" - out.append({"label": label, "state": state}) - start = end - return out def _blocks_files(topic: str) -> dict: @@ -313,74 +294,27 @@ def cancel_blocks(topic: str) -> bool: return True -async def _resume_step(topic: str) -> int: - """First step still open. While blocks.md is missing (never built OR a reset deleted it) the - inventory sub-step comes fine-grained from the DB step status; once blocks.md exists the inventory - counts as done (the artefact is the source of truth, robust for legacy topics) and later phases - come from the persisted artefacts. A reset-from-inventory deletes blocks.md, so this stays exact.""" - files = _blocks_files(topic) - steps_all = _blocks_steps(topic) - if not files["final"].exists(): - for step in ("Source prep", "Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung"): - if step in steps_all and await db.get_step_status(topic, step) != "done": - return _step_idx(topic, step) - return _step_idx(topic, "Blocks-Gruppierung") # statuses done but artefact gone → rewrite - q = load_source(topic) - if q["type"] == "projekt" and not files["ergaenzung"].exists(): - return _step_idx(topic, "Supplement") - sidecar = _json_file(files["sidecar"]) - if _sidecar_schema(sidecar) is not None: - # Levels done; only relevance still open? - if not _relevance_complete(sidecar): - return _step_idx(topic, "Relevance find") - # Relevance done; outline (blocks artifact for the guide) open? - if not _outline_complete(files): - return _step_idx(topic, "Outline") - # Outline done; question patterns open? - if not _question_pattern_complete(topic): - return _step_idx(topic, "Questions find") - # Questions done; learning artefacts (flashcards/examples) open? - if not _artefacts_complete(files): - return _step_idx(topic, "Flashcards") - return len(_blocks_steps(topic)) - if _sub_raw_schema(_json_file(files["sub_roh"])) is None: - return _step_idx(topic, "Subblocks find") - # Subblocks done; facts still open? (Facts come before the levels.) - if not _facts_complete(files): - return _step_idx(topic, "Facts find") - return _step_idx(topic, "Levels find") -def _fine_status(topic: str, current: int | None) -> list[dict]: - """Fine sub-step status: per step {label, phase, state}. state from `current` - (done = idx). Phase label from PHASEN.""" - step_phase = {s: label for label, steps in PHASEN for s in steps} - out = [] - for i, s in enumerate(_blocks_steps(topic)): - state = "pending" if current is None or current < i else "done" if current > i else "active" - out.append({"label": s, "phase": step_phase.get(s, ""), "state": state}) - return out async def blocks_status(topic: str) -> dict: - # Internally fine-grained (resume/progress); bundled into 5 coarse phases for display. - fine_steps = _blocks_steps(topic) + """Kanban-based status: `generating` from the run registry, progress from the card + counts. `partial` = cards sit in non-terminal columns while nothing runs (continue-able).""" ready = blocks_path(topic).exists() # inventory written → block overview available generating = topic in _blocks_progress - if generating: - current = _blocks_step.get(topic) - else: - # True progress (inventory from DB step status, later phases from artefacts). - current = await _resume_step(topic) - partial = not generating and 0 < current < len(fine_steps) + counts = await db.kanban_stage_counts(topic) + terminal = {"clustered", "done_cluster", "grouped", "rejected", "done_block", "done_artefact", "dead"} + open_cards = sum(n for stages in counts.values() + for stage, n in stages.items() if stage not in terminal) return { "ready": ready, "generating": generating, "progress": _blocks_progress.get(topic), "error": _blocks_errors.get(topic), - "partial": partial, - "steps": _phases_status(topic, current), - "feine_steps": _fine_status(topic, current), + "partial": not generating and open_cards > 0, + "steps": [], # legacy phase pills — replaced by the live board + "feine_steps": [], } @@ -402,164 +336,12 @@ def reset_blocks(topic: str) -> None: # source.json intentionally stays — that is the topic config. -def _phase_idx(label: str) -> int: - """Index of the coarse phase in the canonical order (Source=0 … Questions=5).""" - order = [l for l, _ in PHASEN] - return order.index(label) if label in order else 1 -def _reset_from_phase(topic: str, label: str) -> None: - """Delete file artefacts FROM the coarse phase `label` (Source/Inventory … Questions), keeping - earlier ones. Cumulative. source.json + crawl (.done) always stay (re-crawl only on full reset).""" - files = _blocks_files(topic) - work_dir = files["arbeit"] - idx = _phase_idx(label) - - def glob_del(pat: str) -> None: - if work_dir.is_dir(): - for p in work_dir.glob(pat): - p.unlink(missing_ok=True) - - # Phase index: Source=0 · Inventory=1 · Subblocks=2 · Facts=3 · Levels=4 · Relevance=5 · Outline=6 · Questions=7 · Artefacts=8 - if idx <= 8: # Artefacts (flashcards/examples) - files["artefakte"].unlink(missing_ok=True) - glob_del("artifact-*") - if idx <= 7: # Questions - files["question_pattern"].unlink(missing_ok=True) - glob_del("question-pattern-*") - if idx <= 6: # Outline - files["outline"].unlink(missing_ok=True) - glob_del("outline-*") - if idx <= 5: # Relevance - glob_del("relevance-*") - if idx <= 4: # Levels + relevance share the sidecar → from Levels rebuild entirely - files["sidecar"].unlink(missing_ok=True) - glob_del("level-*") - else: # from Relevance: keep levels, strip only the relevance fields - sc = _json_file(files["sidecar"]) - if isinstance(sc, dict): - for subs in sc.values(): - for s in (subs if isinstance(subs, list) else []): - if isinstance(s, dict): - s.pop("relevance", None) - atomic_write_json(files["sidecar"], sc, indent=1) - if idx <= 3: # Facts (before the levels) — facts map + work files gone - files["facts"].unlink(missing_ok=True) - glob_del("facts-*") - if idx <= 2: # Subblocks - files["sub_roh"].unlink(missing_ok=True) - glob_del("subblock-*") - if idx <= 1: # Inventory (and source) = inventory files + blocks.md gone - for p_old in _all_slot_files(files): - p_old.unlink(missing_ok=True) - files["final"].unlink(missing_ok=True) -async def _reset_from_step(topic: str, step_idx: int) -> None: - """Fine reset FROM a sub-step (0-based index in _blocks_steps). Resets - pipeline_state + artefacts + DB from here on; earlier steps stay. Inventory sub-steps - reconstruct the DB status from the artefacts (dedup/filter safe; clarification robustly falls back - to consolidation, because the clarification renames → a title mismatch would be fragile).""" - fine_steps = list(_blocks_steps(topic)) - if not (0 <= step_idx < len(fine_steps)): - return - affected = set(fine_steps[step_idx:]) - files = _blocks_files(topic) - work_dir = files["arbeit"] - - def gd(pat: str) -> None: - if work_dir.is_dir(): - for p in work_dir.glob(pat): - p.unlink(missing_ok=True) - - await db.delete_pipeline_state(topic, list(fine_steps[step_idx:])) - # Later artefacts/DB cumulatively from the affected step (back to front). - if {"Examples", "Flashcards"} & affected: - files["artefakte"].unlink(missing_ok=True); gd("artifact-*"); await db.delete_sub_artefakte(topic) - if any(s.startswith("Questions") for s in affected): - files["question_pattern"].unlink(missing_ok=True); gd("question-pattern-*"); await db.delete_question_pattern(topic) - if "Outline" in affected: - files["outline"].unlink(missing_ok=True); gd("outline-*"); await db.delete_outline(topic) - if any(s.startswith("Relevance") for s in affected): - gd("relevance-*") - if any(s.startswith("Levels") for s in affected): - gd("level-*") - if any(s.startswith("Facts") for s in affected): - files["facts"].unlink(missing_ok=True); gd("facts-*") - # The sidecar carries subblocks + their fields level/relevance/facts. From subblocks rebuild entirely; - # otherwise strip only the fields of the phases to rebuild — subblocks are preserved. - if any(s.startswith("Subblock") for s in affected): - files["sidecar"].unlink(missing_ok=True) - else: - strip = {f for s, f in (("Facts", "facts"), ("Levels", "level"), ("Relevance", "relevance")) - if any(x.startswith(s) for x in affected)} - if strip: - sc = _json_file(files["sidecar"]) - if isinstance(sc, dict): - for subs in sc.values(): - for s in (subs if isinstance(subs, list) else []): - if isinstance(s, dict): - for f in strip: - s.pop(f, None) - atomic_write_json(files["sidecar"], sc, indent=1) - if any(s.startswith("Subblock") for s in affected): - files["sub_roh"].unlink(missing_ok=True); gd("subblock-*"); await db.delete_subblocks(topic) - # --- Inventory (DB status cascades) --- - if "Blocks-Gruppierung" in affected and not ({"Consolidation", "Research"} & affected): - # Only status-flip resets (grouping/filter/dedup level): undo umbrella grouping — members back - # to consensus (with their original description), synthesized umbrellas discarded. On a - # Consolidation/Research reset the blocks are wiped below anyway, so skip there. Independent of - # the filter/dedup branches (a reset-from-Blocks-Filter needs BOTH grouping- and filter-undo). - d = _json_file(work_dir / "inventar-gruppierung.json") - for u in (d.get("umbrellas", []) if isinstance(d, dict) else []): - member_norms = set() - for m in u.get("mitglieder", []): - mn = m.get("title_norm") or _norm_title(m.get("title", "")) - if not mn: - continue - member_norms.add(mn) - await db.set_block_status(topic, mn, "consensus", description=m.get("description")) - un = _norm_title(u.get("umbrella", "")) - if un and un not in member_norms: # synthesized umbrella (not a reused member title) → drop it - await db.set_block_status(topic, un, "discarded") - gd("gruppierung-*"); gd("inventar-gruppierung*") - if "Blocks-Filter" in affected and not ({"Clarification", "Consolidation", "Research", "Dedup"} & affected): - # Only filter rebuilt: degraded blocks back to consensus. - d = _json_file(work_dir / "inventar-filter.json") - for f in (d.get("fragments", []) if isinstance(d, dict) else []): - await db.set_block_status(topic, _norm_title(f.get("fragment", "")), "consensus") - gd("inventar-filter*") - if "Dedup" in affected and not ({"Clarification", "Consolidation", "Research"} & affected): - # Dedup (+filter) rebuilt: all blocks discarded in dedup/filter back to consensus. - for kind in ("dedup-runde-1.json", "inventar-filter.json"): - d = _json_file(work_dir / kind) - title = ([t for g in d.get("groups", []) for t in g] if isinstance(d, dict) and "groups" in d - else [f.get("fragment", "") for f in d.get("fragments", [])] if isinstance(d, dict) else []) - for t in title: - await db.set_block_status(topic, _norm_title(t), "consensus") - gd("dedup-*"); gd("inventar-filter*") - if {"Clarification", "Consolidation"} & affected and not ({"Research"} & affected): - # Clarification/consolidation rebuilt: clear inventory DB (research readers stay). Clarification rollback - # would be fragile due to renaming → cleanly rebuild from consolidation. - await db.delete_blocks(topic) - gd("clarification*"); gd("consolidation-*"); gd("dedup-*"); gd("inventar-filter*") - if "Research" in affected: # whole inventory like a phase reset - for p_old in _all_slot_files(files): - p_old.unlink(missing_ok=True) - await db.delete_blocks(topic) - # blocks.md is the inventory aggregate — stale once any inventory sub-step is reset. Delete it so the - # status/resume see the inventory as open from the reset step (the pipeline rewrites it; the DB step - # statuses of the kept earlier steps let those skip). - if {"Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung"} & affected: - files["final"].unlink(missing_ok=True) -async def reset_blocks_ab_step(topic: str, step_idx: int) -> None: - """Public: ONLY reset from a sub-step — no re-generation. Leaves a - partial state (the steps from here count as open). If a generation is running → ignore.""" - if topic in _blocks_progress: - return - await _reset_from_step(topic, step_idx) def _supplement_schema(data): @@ -774,10 +556,12 @@ def _lpt_chunks(weights: list[int], target: int) -> list[list[int]]: -async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict | None: +async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str, + wipe: bool = True, ns: str = "") -> dict | None: """Block B (DB + loop): per package, find subblocks in rounds (3 finders, until 0 new/cap), collect in the DB (≥2 mentions = consensus, 1× discarded), a judge cleans up per package. - → {block title: [subblock, …]} (consensus) or None. Fills DB table `subblocks`.""" + → {block title: [subblock, …]} (consensus) or None. Fills DB table `subblocks`. + wipe=False (kanban board: one call per block) keeps the other blocks' rows.""" topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled work_dir = files["arbeit"] folder = source_folder(topic) @@ -790,7 +574,8 @@ async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, i n = len(chunks) title_by_num = {num: _title(entries[num]) for num in nums} norm_by_num = {num: _norm_title(title_by_num[num]) for num in nums} - await db.delete_subblocks(topic) # fresh start of the block (idempotent counter) + if wipe: + await db.delete_subblocks(topic) # fresh start of the block (idempotent counter) async def _known_block(chunk): known = [] @@ -817,7 +602,7 @@ async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, i for p in paths: p.unlink(missing_ok=True) slots = [{ - "key": f"blocks-{topic}-subblock-c{c}-r{round_n}-{i}", + "key": f"blocks-{topic}-{ns}subblock-c{c}-r{round_n}-{i}", "prompt": _prompt("Subblock-Research", topic=topic, assignment=assignment, known=bekannt, out_path=p, extra=_extra(instructions)), "role": "quick", "capabilities": caps, "payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None), @@ -894,7 +679,7 @@ async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, i p.unlink(missing_ok=True) if pending: slots = [{ - "key": f"blocks-{topic}-subblock-final-c{c}-j{j}", + "key": f"blocks-{topic}-{ns}subblock-final-c{c}-j{j}", "prompt": _prompt("Subblock-Mapping", topic=topic, source=source, blocks="\n\n".join(block_texts), out_path=p, extra=_extra(instructions)), "role": "judge", "capabilities": caps, "payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None), @@ -964,8 +749,11 @@ async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, i await db.set_subblock_fields(topic, norm_by_num[num], sn, status="consensus") await _dedup_subblocks(topic, raw) # near-dup filter per block (deterministic, no LLM) if not raw: - _blocks_errors[topic] = "No subblocks determined" - return None + # Finders ran but nothing survived the consensus/evidence gates: a legitimately + # thin block (e.g. a bare named reduction). {} = done-without-subs — the guide + # writes it from description+facts. Agent FAILURES return None elsewhere (retry). + _log(topic, "Subblocks: nichts Belegbares gefunden — Block bleibt ohne Subbausteine") + return {} return raw @@ -1024,7 +812,7 @@ def _disputed_lines(items, item_idxs, disputed: dict) -> str: ) -async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instructions: str) -> dict | None: +async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instructions: str, ns: str = "") -> dict | None: """Block C: three phases with a barrier — find (classify), select (vote), clarify. Local IDs 1..n per package, mapped to global gid afterwards. → {block title: [{title, level}, …]} or None.""" @@ -1075,9 +863,9 @@ async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instruct enum = "\n".join(enum_lines).strip() pending = [(i, p) for i, p in enumerate(paths, 1) if not _levels_schema(_json_file(p), local_set)] slots = [{ - "key": f"blocks-{topic}-level-c{c}-{i}", + "key": f"blocks-{topic}-{ns}level-c{c}-{i}", "prompt": _prompt("Levels-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)), - "role": "fast", "capabilities": "files", + "role": "quick", "capabilities": "files", "payload": (lambda result, p=p, ids=local_set: _levels_schema(_json_file(p), ids)), } for i, p in pending] new = await _race(topic, f"Levels package {c}", slots, 2 - existing, _timeout("level", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) @@ -1108,7 +896,7 @@ async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instruct disputed_block = _disputed_lines(items, item_idxs, strittig) status, decision = await run_single_slot( ctx, f"Levels-Clarification {c}", - key=f"blocks-{topic}-level-final-c{c}", + key=f"blocks-{topic}-{ns}level-final-c{c}", prompt=_prompt("Levels-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)), role="judge", capabilities="files", payload=lambda result, p=judge_path, ids=set(strittig): _levels_schema(_json_file(p), ids), @@ -1220,7 +1008,7 @@ def _facts_complete(files: dict) -> bool: return isinstance(d, dict) and bool(d) -async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str) -> tuple | None: +async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str, ns: str = "") -> tuple | None: """Block: per sub extract source facts (find) → verify (check) → correct/discard (fix). Extract-once grounding: the result feeds level/relevance/questions/guide. → (facts_map, discarded_map) — facts_map {block: {sub_norm: facts}}, discarded_map @@ -1293,9 +1081,9 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst return True subs_total = sum(len(blocks[i][1]) for i in idxs) status, _r = await run_single_slot( - ctx, f"Facts {ci}", key=f"blocks-{topic}-facts-c{ci}", + ctx, f"Facts {ci}", key=f"blocks-{topic}-{ns}facts-c{ci}", prompt=_prompt("Facts-Research", topic=topic, source=source, blocks=block_text(idxs), out_path=fp, extra=_extra(instructions)), - role="guide", capabilities=caps, + role="quick", capabilities=caps, payload=lambda result, p=fp: _facts_schema(_json_file(p)), timeout=_timeout("content", subs_total)) return status != FAILED and _facts_schema(_json_file(fp)) is not None @@ -1324,9 +1112,9 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst for bt, fm in per.items()) subs_total = sum(len(blocks[i][1]) for i in idxs) await run_single_slot( - ctx, f"Facts supplement {ci}", key=f"blocks-{topic}-facts-erg-c{ci}", + ctx, f"Facts supplement {ci}", key=f"blocks-{topic}-{ns}facts-erg-c{ci}", prompt=_prompt("Facts-Supplement", topic=topic, source=source, blocks=block, out_path=ep, extra=_extra(instructions)), - role="guide", capabilities=caps, + role="quick", capabilities=caps, payload=lambda result, p=ep: _facts_schema(_json_file(p)), timeout=_timeout("content", subs_total)) @@ -1344,7 +1132,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst facts_text = "\n\n".join(f"SUBBAUSTEIN: {fk['sub']}\n{_facts_lines(fk)}" for fm in per.values() for fk in fm.values()) pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _facts_check_schema(_json_file(chk_path(ci, j))) is None] await asyncio.gather(*[ - run_agent(f"blocks-{topic}-facts-check-c{ci}-j{j}", + run_agent(f"blocks-{topic}-{ns}facts-check-c{ci}-j{j}", _prompt("Facts-Check", topic=topic, source=source, facts=facts_text, out_path=chk_path(ci, j), extra=_extra(instructions)), _timeout("content_check", len(per)), provider=provider, role="judge", capabilities=caps) for j in pending], return_exceptions=True) @@ -1395,9 +1183,9 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst if not goal: return await run_single_slot( - ctx, f"Facts-Fix {ci}", key=f"blocks-{topic}-facts-fix-c{ci}", + ctx, f"Facts-Fix {ci}", key=f"blocks-{topic}-{ns}facts-fix-c{ci}", prompt=_prompt("Facts-Research", topic=topic, source=source, blocks="\n\n".join(goal), out_path=fix_path(ci), extra=_extra(instructions)), - role="guide", capabilities=caps, + role="quick", capabilities=caps, payload=lambda result, p=fix_path(ci): _facts_schema(_json_file(p)), timeout=_timeout("content", len(subs_norm))) await _gather_progress([_fix(ci) for ci in correctable], len(correctable), _report_p(set_p, topic, "Facts fix")) @@ -1423,7 +1211,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst return outcome, discarded_map -async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: +async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "") -> dict | None: """Block D: three phases with a barrier — find (relevant/peripheral), select (vote), clarify. Items from the sidecar; local IDs 1..n per package → global gid. → {gid: relevance} or None on cancel/research error. Default on gap/dispute: 'relevant'.""" @@ -1456,9 +1244,9 @@ async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i enum = "\n".join(enum_lines) pending = [(i, p) for i, p in enumerate(paths, 1) if not _relevance_schema(_json_file(p), local_set)] slots = [{ - "key": f"blocks-{topic}-relevance-c{c}-{i}", + "key": f"blocks-{topic}-{ns}relevance-c{c}-{i}", "prompt": _prompt("Relevance-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)), - "role": "fast", "capabilities": "files", + "role": "quick", "capabilities": "files", "payload": (lambda result, p=p, ids=local_set: _relevance_schema(_json_file(p), ids)), } for i, p in pending] new = await _race(topic, f"Relevance package {c}", slots, 2 - existing, _timeout("relevance", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) @@ -1489,7 +1277,7 @@ async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i disputed_block = _disputed_lines(items, item_idxs, strittig) status, decision = await run_single_slot( ctx, f"Relevance-Clarification {c}", - key=f"blocks-{topic}-relevance-final-c{c}", + key=f"blocks-{topic}-{ns}relevance-final-c{c}", prompt=_prompt("Relevance-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)), role="judge", capabilities="files", payload=lambda result, p=judge_path, ids=set(strittig): _relevance_schema(_json_file(p), ids), @@ -1533,7 +1321,7 @@ def _match_sub(agent_sub: str, rel: list[str]) -> str: return agent_sub -async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: +async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "") -> dict | None: """Block E (chunks of 10): find (1 generator per ~10 blocks, parallel), select (code: group per block + dedup), clarify (1 critic per chunk), check (catch-up round). Assignment per entry via the `block` field (a chunk file carries several blocks). @@ -1584,10 +1372,10 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: subs_total = sum(len(blocks[i][1]) for i in idxs) status, _ = await run_single_slot( ctx, f"Question-Pattern {ci}", - key=f"blocks-{topic}-question-pattern-c{ci}", + key=f"blocks-{topic}-{ns}question-pattern-c{ci}", prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=block, out_path=fp, extra=_extra(instructions)), - role="fast", capabilities="files", + role="quick", capabilities="files", payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)), timeout=_timeout("question_pattern", subs_total), ) @@ -1649,7 +1437,7 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: subs_total = sum(len(blocks[i][1]) for i in idxs) status, _ = await run_single_slot( ctx, f"Question-Pattern-Clarification {ci}", - key=f"blocks-{topic}-question-pattern-final-c{ci}", + key=f"blocks-{topic}-{ns}question-pattern-final-c{ci}", prompt=_prompt("Question-Pattern-Critique", topic=topic, table="\n\n".join(block_texts), out_path=fp, extra=_extra(instructions)), role="judge", capabilities="files", payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)), @@ -1718,10 +1506,10 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: subs_total = sum(len(s) for _, s in items) await run_single_slot( ctx, f"Question pattern catch-up R{round_n}/{pi}", - key=f"blocks-{topic}-question-pattern-nach{round_n}-c{pi}", + key=f"blocks-{topic}-{ns}question-pattern-nach{round_n}-c{pi}", prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=_followup_block(items), out_path=fp, extra=_extra(instructions)), - role="fast", capabilities="files", + role="quick", capabilities="files", payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)), timeout=_timeout("question_pattern", subs_total), ) @@ -1942,185 +1730,8 @@ async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, return True -async def _research_batch(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool: - """Fills DB table `blocks` with candidates (+ mention counter). FIXED file batches: - each crawl page is assigned to exactly one batch and read by RESEARCH_READERS agents - (consensus ≥2 in the batch). All assigned pages are marked as read → 100 % coverage. - Without a crawl folder (source "thema") → free web research, one round. → True/False.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - if await db.get_step_status(topic, "Research") == "done": - return True - work_dir = files["arbeit"] - await db.delete_blocks(topic) # coverage/content belongs to the triage — do NOT delete - await db.set_step_status(topic, "Research", "running") - - async def _ingest(reader_id: str, text: str, source: str | None = None) -> None: - seen_set = set() - for record in _parse_selection(text).values(): - title = _title(record) - norm = _norm_title(title) - if not norm or norm in seen_set: - continue - seen_set.add(norm) # one reader = one vote per concept - split_parts = [t.strip() for t in record.split(" — ")] - desc = split_parts[1] if len(split_parts) >= 2 else "" - # Provenance out-of-band: in section mode the source filename is known deterministically - # (`source`), so the agent no longer appends it to the line — the filename must never be - # inlined into title/desc (that leaked "— alle_klausuren.txt" into descriptions and polluted - # title_norm, breaking dedup). Fall back to a segment-3 source only where the section file - # isn't 1:1 with the reader (thema/crawl). - if source: - sources = [source] - else: - sources = [split_parts[2]] if len(split_parts) >= 3 and split_parts[2] else [] - await db.upsert_block(topic, norm, title, desc, sources, reader=reader_id) - - pages = await db.list_content(topic) # pages marked as content by the triage - if not pages and folder: - pages = sorted(set(_crawl_index(folder).values())) # fallback (projekt/uni: no triage) - - if not pages: - # source "thema" (or no crawl): free web research, one round. - set_p("Research running…", step=_step_idx(topic, "Research")) - caps = "files" if folder else "full" - paths = [work_dir / f"research-{i}.md" for i in range(1, RESEARCH_THEMA_AGENTS + 1)] - for p in paths: - p.unlink(missing_ok=True) - slots = [{ - "key": f"blocks-{topic}-research-{i}", - "prompt": _build_research_prompt(topic, p, instructions, q["type"], folder), - "role": "quick", "capabilities": caps, - "payload": (lambda result, p=p, rid=f"t{i}": ((rid, t) if (t := _file_payload(p)) else None)), - } for i, p in enumerate(paths, 1)] - agent_texts = await _race(topic, "Research", slots, 3, _timeout("research"), provider, - cancelled=is_cancelled, grace=RESEARCH_GRACE) - if is_cancelled(): - return False - if not agent_texts: - _blocks_errors[topic] = "Research failed (minimum not reached)" - return False - for rid, text in agent_texts: - await _ingest(rid, text) - await db.set_step_status(topic, "Research", "done") - return True - - # uni/projekt: curated, often LARGE files (script). Instead of reading all at once - # (lost-in-the-middle), chunk into sections and have 2 readers thoroughly read EACH — - # text directly in the prompt (small context), mentions accumulate to consensus. - if q["type"] in ("uni", "projekt"): - eintraege: list[tuple[str, str]] = [] # (filename, section text) - for fn in sorted(pages): - for section_text in _text_sections(_read(folder / fn)): - eintraege.append((fn, section_text)) - if not eintraege: - _blocks_errors[topic] = "Research: source empty" - return False - set_p(f"Research ({len(eintraege)} sections)…", step=_step_idx(topic, "Research")) - - async def _read_section(ei: int, fn: str, section_text: str) -> None: - block = (f"ARBEITE AUSSCHLIESSLICH MIT DIESEM TEXTABSCHNITT. Lies ihn VOLLSTÄNDIG, " - f"überspringe nichts. Suche NICHT im Web — nur diese Section zählt." - f"\n\n-----\n{section_text}\n-----") - paths = [work_dir / f"research-a{ei}-{i}.md" for i in range(1, RESEARCH_READERS + 1)] - # reader file reuse: if all reader outputs are present and valid (resume / - # re-run without research change), re-ingest instead of spawning agents again. - existing = [(f"a{ei}-{i}", t) for i, p in enumerate(paths, 1) if (t := _file_payload(p))] - if len(existing) == len(paths): - for rid, text in existing: - await _ingest(rid, text, fn) # provenance injected deterministically from the section file - return - for p in paths: - p.unlink(missing_ok=True) - if is_cancelled(): - return - slots = [{ - "key": f"blocks-{topic}-research-a{ei}-{i}", - "prompt": _build_research_prompt(topic, p, instructions, q["type"], folder, section=block), - "role": "quick", "capabilities": "files", - "payload": (lambda result, p=p, rid=f"a{ei}-{i}": ((rid, t) if (t := _file_payload(p)) else None)), - } for i, p in enumerate(paths, 1)] - # quorum 2: both readers per section should pass (more eyes = more concepts + - # real consensus); after timeout _race falls back to what exists. - agent_texts = await _race(topic, f"Research section {ei}", slots, 2, _timeout("research", 1), - provider, cancelled=is_cancelled, grace=RESEARCH_GRACE) - for rid, text in (agent_texts or []): - await _ingest(rid, text, fn) # provenance injected deterministically from the section file - - await _gather_progress([_read_section(ei, fn, a) for ei, (fn, a) in enumerate(eintraege, 1)], - len(eintraege), _report_p(set_p, topic, "Research")) - if is_cancelled(): - return False - await db.mark_sources_read_done(topic, sorted(pages)) - total = len(await db.list_blocks(topic)) - _log(topic, f"Research (uni/projekt): {total} candidates from {len(eintraege)} sections ({len(pages)} files)") - if not total: - _blocks_errors[topic] = "Research failed (no blocks)" - return False - await db.set_step_status(topic, "Research", "done") - return True - - # Crawl/link: many small content pages (triage in the "Source prep" step). - # Fixed batches, RESEARCH_READERS readers per batch reading EXACTLY these files. - batches = _chunk_nums(sorted(pages), max(1, math.ceil(len(pages) / RESEARCH_BATCH))) - - async def _read_batch(bi: int, batch: list[str]) -> bool: - liste = "\n".join(f"- {p}" for p in batch) - fokus = ("WICHTIG — feste Assignment: Bearbeite AUSSCHLIESSLICH diese Dateien und lies JEDE " - f"vollständig. Ignoriere alle anderen Dateien im Ordner:\n{liste}") - paths = [work_dir / f"research-b{bi}-{i}.md" for i in range(1, RESEARCH_READERS + 1)] - for p in paths: - p.unlink(missing_ok=True) - if not is_cancelled(): - slots = [{ - "key": f"blocks-{topic}-research-b{bi}-{i}", - "prompt": _build_research_prompt(topic, p, instructions, q["type"], folder, focus=fokus), - "role": "quick", "capabilities": "files", - "payload": (lambda result, p=p, rid=f"b{bi}-{i}": ((rid, t) if (t := _file_payload(p)) else None)), - } for i, p in enumerate(paths, 1)] - agent_texts = await _race(topic, f"Research batch {bi}", slots, 1, _timeout("research", len(batch)), - provider, cancelled=is_cancelled, grace=RESEARCH_GRACE) - for rid, text in (agent_texts or []): - await _ingest(rid, text) - await db.mark_sources_read_done(topic, batch) # tick off all assigned pages (even without hits) - return not is_cancelled() - - await _gather_progress([_read_batch(bi, b) for bi, b in enumerate(batches, 1)], - len(batches), _report_p(set_p, topic, "Research")) - if is_cancelled(): - return False - total = len(await db.list_blocks(topic)) - coverage = len(await db.list_coverage(topic)) - _log(topic, f"Research: {total} candidates, coverage {coverage}/{len(pages)} pages ({len(batches)} batches)") - if not total: - _blocks_errors[topic] = "Research failed (no blocks)" - return False - await db.set_step_status(topic, "Research", "done") - return True -def _grp_schema(data, ids: set[int]): - """{"groups": [[1,3],[2], …]} → partition of `ids` as a list of index groups. - Tolerant: ignores foreign/duplicate numbers; forgotten candidates are added standalone - (singleton group). None only on structurally broken JSON.""" - if not isinstance(data, dict) or not isinstance(data.get("groups"), list): - return None - groups, seen_set = [], set() - for g in data["groups"]: - if not isinstance(g, list): - return None - grp = [] - for x in g: - try: - num = int(x) - except (ValueError, TypeError): - continue - if num in ids and num not in seen_set: - seen_set.add(num) - grp.append(num) - if grp: - groups.append(grp) - groups += [[r] for r in sorted(ids - seen_set)] # forgotten candidates stay standalone - return groups or None _ASPECT_MARKER = ("∈ np", "∈np", " in np", "np-schwer", "np-vollständig", "verifizierer", @@ -2180,342 +1791,14 @@ def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) -> return {"title": title, "description": candidates[k]["description"]} -async def _group_blocks(ctx: GenContext, set_p, work_dir: Path, candidates: list[dict], - blocks: list[list[int]], prefix: str = "consolidation", - step: str = "Consolidation") -> list[list[int]]: - """Per similarity block, a judge groups the titles into the real blocks (merge - paraphrases, split over-merges). Singletons directly. Error/timeout → conservatively each - candidate alone (avoids false over-merging). → final groups (global indices). - `praefix`/`step` separate consolidation and dedup (artefacts, race key, progress).""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - multi = [(bi, b) for bi, b in enumerate(blocks) if len(b) > 1] - outcome: list[list[int]] = [list(b) for b in blocks if len(b) == 1] # singletons directly - - def _line(k: int, g: int) -> str: - b = candidates[g] - return f"{k}. {b['title']}" + (f" — {b['description']}" if b["description"] else "") - - async def _grp(bi: int, block: list[int]) -> None: - ids = set(range(1, len(block) + 1)) - p = work_dir / f"{prefix}-block-c{bi}.json" - part = _grp_schema(_json_file(p), ids) - if part is None: # resume: don't recompute a valid file - p.unlink(missing_ok=True) - if is_cancelled(): - return - lines = [_line(k, block[k - 1]) for k in range(1, len(block) + 1)] - status, part = await run_single_slot( - ctx, f"Block grouping {bi}", - key=f"blocks-{topic}-{prefix}-block-c{bi}", - prompt=_prompt("Blocks-Block-Grouping", topic=topic, entries="\n".join(lines), out_path=p), - role="judge", capabilities="files", - payload=(lambda result, p=p, ids=ids: _grp_schema(_json_file(p), ids)), - timeout=_timeout("research_mapping", len(block)), - ) - part = part if status == OK else None - if part is None: # judge failed → individually (no over-merge) - outcome.extend([idx] for idx in block) - else: # local numbers → global candidate indices - outcome.extend([block[k - 1] for k in g] for g in part) - - await _gather_progress([_grp(bi, b) for bi, b in multi], - len(multi), _report_p(set_p, topic, step)) - return outcome -async def _consolidate_embedding(ctx: GenContext, set_p, files: dict, candidates: list[dict]) -> bool: - """Two-stage: embeddings → coarse capped blocks (high recall) → one judge per multi-block, - grouping the titles into the real blocks → reader union (≥2 = consensus).""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - work_dir = files["arbeit"] - texts = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in candidates] - sims = await asyncio.to_thread(embedding.embed_sims, texts) - if sims is None: # model not available after all → fallback - return await _consolidate_llm(ctx, set_p, files, candidates) - # Level 1: coarse similarity blocks (capped, no giant component). - blocks = await asyncio.to_thread(embedding.capped_blocks, sims, None, None) - # Level 2: one judge groups EACH multi-block into the real blocks. - groups = await _group_blocks(ctx, set_p, work_dir, candidates, blocks) - if is_cancelled(): - return False - - def _min_cos(idxs): # internal coherence as a check (chains would be ~0.3) - if len(idxs) < 2: - return 1.0 - return round(min(float(sims[i][j]) for n, i in enumerate(idxs) for j in idxs[n + 1:]), 3) - - # Consensus = ≥2 distinct readers per cluster. Legacy DBs without reader tracking (research ran - # before the migration, no re-ingest) have empty reader sets → fall back to a title heuristic - # (otherwise EVERYTHING would land in the rest). - hat_reader = any(b["reader"] for b in candidates) - consensus, rest, debug, seen_norm = [], [], [], set() - for idxs in groups: - reader = set().union(*[set(candidates[k]["reader"]) for k in idxs]) if idxs else set() - if hat_reader: - score = len(reader) - else: # without reader data: max(mentions, number of distinct title variants in the cluster) - score = max(max(candidates[k]["mentions"] for k in idxs), - len({candidates[k]["title_norm"] for k in idxs})) - rep = _canonical(candidates, idxs, seen_norm) - record = f"{rep['title']} — {rep['description']}" if rep["description"] else rep["title"] - (consensus if score >= 2 else rest).append(record) - debug.append({"title": rep["title"], "reader": sorted(reader), "score": score, - "consensus": score >= 2, "min_cos": _min_cos(idxs), - "mitglieder": [candidates[k]["title"] for k in idxs]}) - atomic_write_json(work_dir / "consolidation-cluster.json", debug, indent=1) - multi_blocks = sum(1 for b in blocks if len(b) > 1) - _log(topic, f"Consolidation (embedding): {len(blocks)} blocks ({multi_blocks} grouped via LLM) " - f"→ {len(groups)} clusters from {len(candidates)} candidates " - f"→ {len(consensus)} consensus / {len(rest)} rest") - - await db.delete_blocks(topic) - for t in consensus: - await _set_inventory(topic, t, "consensus") - for t in rest: - await _set_inventory(topic, t, "rest") - await db.set_step_status(topic, "Consolidation", "done") - return True -async def _consolidate(ctx: GenContext, set_p, files: dict) -> bool: - """Merges raw candidates into consensus (≥2 readers)/rest. Deterministic via embedding clustering; - if the model is missing → fall back to the LLM panel (`_consolidate_llm`). Status in DB.""" - topic = ctx.topic - if await db.get_step_status(topic, "Consolidation") == "done": - return True - set_p("Consolidating research…", step=_step_idx(topic, "Consolidation")) - candidates = await db.list_blocks(topic) - if not candidates: - _blocks_errors[topic] = "Consolidation: no candidates" - return False - if EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available): - return await _consolidate_embedding(ctx, set_p, files, candidates) - return await _consolidate_llm(ctx, set_p, files, candidates) -async def _consolidate_llm(ctx: GenContext, set_p, files: dict, candidates: list[dict]) -> bool: - """Fallback (only without an embedding model): a panel (KONSOLIDIERUNG_PANEL judges) merges - candidates semantically; a reconcile judge combines the panel outputs into the final - consensus (≥2)/rest (1×) list. Panel instead of a single judge: a single judge is bias-prone and unstable.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - work_dir = files["arbeit"] - chunks = _chunk_nums(candidates, max(1, math.ceil(len(candidates) / CONSOLIDATION_CHUNK))) - - async def _map_panel(c: int, eintraege: str, amount: int): - """3 mapping judges over `eintraege` → reconcile judge → (consensus, rest). None on cancel/error.""" - paths = [work_dir / f"consolidation-c{c}-j{j}.json" for j in range(1, CONSOLIDATION_PANEL + 1)] - pending = [(j, p) for j, p in enumerate(paths, 1) if _mapping_schema(_json_file(p)) is None] - for _, p in pending: - p.unlink(missing_ok=True) - if pending: - slots = [{ - "key": f"blocks-{topic}-consolidation-c{c}-j{j}", - "prompt": _prompt("Blocks-Research-Mapping", topic=topic, n=RESEARCH_READERS, entries=eintraege, out_path=p), - "role": "judge", "capabilities": "files", - "payload": (lambda result, p=p: _mapping_schema(_json_file(p))), - } for j, p in pending] - existing = CONSOLIDATION_PANEL - len(pending) - await _race(topic, f"Consolidation {c}", slots, max(1, 2 - existing), - _timeout("research_mapping", amount), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) - if is_cancelled(): - return None - outs = [m for p in paths if (m := _mapping_schema(_json_file(p)))] - if not outs: - return None - # union of panel titles; per title count how many judges list it as consensus. - kvotes: dict[str, int] = {} - form: dict[str, str] = {} # norm → display title (first occurrence) - order: list[str] = [] - for kk, rr in outs: - for t in kk + rr: - nt = _norm_title(_title(t)) - if not nt: - continue - if nt not in form: - form[nt] = t - order.append(nt) - kvotes.setdefault(nt, 0) - for t in kk: - nt = _norm_title(_title(t)) - if nt: - kvotes[nt] = kvotes.get(nt, 0) + 1 - # Reconcile: one merge judge over the union, annotated with judge votes ("k× genannt"). - rp = work_dir / f"consolidation-c{c}-reconcile.json" - recon = _mapping_schema(_json_file(rp)) - if recon is None: - rp.unlink(missing_ok=True) - entries_r = "\n".join(f"{i}. {form[nt]} ({max(1, kvotes[nt])}× genannt)" for i, nt in enumerate(order, 1)) - status, recon = await run_single_slot( - ctx, f"Consolidation Reconcile {c}", - key=f"blocks-{topic}-consolidation-c{c}-reconcile", - prompt=_prompt("Blocks-Research-Mapping", topic=topic, n=CONSOLIDATION_PANEL, entries=entries_r, out_path=rp), - role="judge", capabilities="files", - payload=lambda result, p=rp: _mapping_schema(_json_file(p)), - timeout=_timeout("research_mapping", len(order)), - ) - if status == CANCELLED: - return None - recon = recon if status != FAILED else None - if recon: - return recon - # Fallback (reconcile failed): code majority — consensus if a majority of judges say consensus. - consensus = [form[nt] for nt in order if kvotes[nt] * 2 >= len(outs) and kvotes[nt] > 0] - kset = {_norm_title(_title(t)) for t in consensus} - return consensus, [form[nt] for nt in order if nt not in kset] - - consensus, rest = [], [] - for c, chunk in enumerate(chunks, 1): - eintraege = "\n".join( - f"{i}. {b['title']} — {b['description']} ({b['mentions']}× genannt)" for i, b in enumerate(chunk, 1) - ) - res = await _map_panel(c, eintraege, len(chunk)) - if res is None: - if is_cancelled(): - return False - _blocks_errors[topic] = "Research mapping failed" - return False - k, r = res - consensus += k - rest += r - # With multiple chunks: a global merge pass over the combined consensus entries, - # so duplicates across chunk boundaries (DAL×4, PHPUnit×5 …) merge. - if len(chunks) > 1 and consensus: - fp = work_dir / "consolidation-merge.json" - fp.unlink(missing_ok=True) - eintraege = "\n".join(f"{i}. {t} (2× genannt)" for i, t in enumerate(consensus, 1)) - status, mapping = await run_single_slot( - ctx, "Consolidation Merge", - key=f"blocks-{topic}-consolidation-merge", - prompt=_prompt("Blocks-Research-Mapping", topic=topic, n=RESEARCH_READERS, entries=eintraege, out_path=fp), - role="judge", capabilities="files", - payload=lambda result, p=fp: _mapping_schema(_json_file(p)), - timeout=_timeout("research_mapping", len(consensus)), - ) - if status == CANCELLED: - return False - if status != FAILED and mapping: - consensus, r2 = mapping - rest += r2 # entries downgraded by the merge into the rest - # Judge output is authoritative → re-set the inventory in the DB. - await db.delete_blocks(topic) - for t in consensus: - await _set_inventory(topic, t, "consensus") - for t in rest: - await _set_inventory(topic, t, "rest") - await db.set_step_status(topic, "Consolidation", "done") - return True -async def _clarify_inventory(ctx: GenContext, set_p, files: dict) -> bool: - """A panel (KONSOLIDIERUNG_PANEL judges) decides on the rest (1×-mentioned): majority `aufnehmen` - → consensus, otherwise discarded. Panel instead of a single judge — the rest cut is the sharpest - intervention; a single judge is too unstable here. Conservative tie → keep (never lose a concept).""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - if await db.get_step_status(topic, "Clarification") == "done": - return True - set_p("Clarification running…", step=_step_idx(topic, "Clarification")) - work_dir = files["arbeit"] - rest_rows = await db.list_blocks(topic, status="rest") - # C2 — deterministic pre-reject (precision gate, BEFORE the recall-biased panel): a single-reader - # `rest` item whose title is a mechanical exercise/notation artefact (Blatt/Aufgabe/Beispiel N, - # "(Variante)", "|·|", "Güte N") and is NOT a named statement/reduction is dropped without the panel. - # Reuses the filter's FP≈0, title-head-only helpers, gated by the statement keep-guard so real - # reductions/theorems survive. Deterministic → resume-identical; reset-safe (rest→discarded only). - pruned: list[dict] = [] - pre_rejected = [b for b in rest_rows - if (_is_artifact(b["title"]) or _FILTER_NOTATION.search(b["title"])) - and not _is_named_statement(b["title"], b["description"])] - if pre_rejected: - pr_norms = {b["title_norm"] for b in pre_rejected} - for b in pre_rejected: - await db.set_block_status(topic, b["title_norm"], "discarded") - pruned.append({"title": b["title"], "grund": "pre-reject"}) - rest_rows = [b for b in rest_rows if b["title_norm"] not in pr_norms] - _log(topic, f"Clarification: {len(pre_rejected)} single-reader artefact(s) pre-rejected (deterministic)") - # Continuous gate (EDC "Define"): also check consensus blocks with reference/placeholder titles - # ("Satz 7.18", "Korollar 6.18", "Bedingung (**)") — otherwise they bypass every exam. - suspicious = [b for b in await db.list_blocks(topic, status="consensus") if _is_reference(b["title"])] - rest_norms = {b["title_norm"] for b in rest_rows} # single-reader origin → stricter quorum (C1) - check_rows = rest_rows + suspicious - if check_rows: - paths = [work_dir / f"clarification-j{j}.json" for j in range(1, CONSOLIDATION_PANEL + 1)] - # final=False: a judge with an accidentally non-empty `rest` must not fail entirely - # (otherwise the panel collapses to 1 judge). Its `aufnehmen` counts; rest entries count as - # not-accepted. The "rest empty" requirement still stands in the prompt. - pending = [(j, p) for j, p in enumerate(paths, 1) if _runde_schema(_json_file(p)) is None] - for _, p in pending: - p.unlink(missing_ok=True) - if pending: - slots = [{ - "key": f"blocks-{topic}-clarification-j{j}", - "prompt": _prompt( - "Blocks-Klaerung", topic=topic, - rest="\n".join(f"- {b['title']} — {b['description']}" if b['description'] else f"- {b['title']}" - for b in check_rows), - final="\n- Entscheide JEDEN Eintrag. `rest` MUSS leer sein.", - out_path=p, - ), - "role": "judge", "capabilities": "files", - "payload": (lambda result, p=p: _runde_schema(_json_file(p))), - } for j, p in pending] - existing = CONSOLIDATION_PANEL - len(pending) - await _race(topic, "Clarification", slots, max(1, 2 - existing), - _timeout("selection_mapping", len(check_rows)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) - if is_cancelled(): - return False - outs = [r for p in paths if (r := _runde_schema(_json_file(p)))] - if not outs: - _blocks_errors[topic] = "Clarification failed" - return False - # Majority per rest entry (by norm title). Tie → keep (votes*2 >= n). - votes: dict[str, int] = {} - for accepted, _ in outs: - for nt in {_norm_title(_title(t)) for t in accepted}: - votes[nt] = votes.get(nt, 0) + 1 - # Rename suggestions (additive from the raw JSON — _runde_schema doesn't know the field): - # kept reference/placeholder titles → meaningful name from the content. Old title norm - # stays stable (doesn't break the votes match); per old title the most frequent suggestion. - renames: dict[str, dict[str, int]] = {} - for p in paths: - d = _json_file(p) - rename_raw = d.get("rename") if isinstance(d, dict) else None - if isinstance(rename_raw, dict): - for old, new in rename_raw.items(): - new = str(new).strip() - if new: - renames.setdefault(_norm_title(str(old)), {}).setdefault(new, 0) - renames[_norm_title(str(old))][new] += 1 - seen_norm = {b["title_norm"] for b in await db.list_blocks(topic, status="consensus")} - for b in check_rows: - v = votes.get(b["title_norm"], 0) - # C1 — origin-split quorum: a single-reader `rest` item needs UNANIMITY (a corroboration - # proxy — collapsing the ≥2-reader gate is what over-admitted ~99%); an already-consensus - # reference title keeps the majority rule (it only needs a rename/exam, not a harder bar). - accept = (v >= len(outs)) if b["title_norm"] in rest_norms else (v * 2 >= len(outs)) - if not accept: - await db.set_block_status(topic, b["title_norm"], "discarded") - pruned.append({"title": b["title"], "grund": "failed-quorum", "votes": v, "judges": len(outs)}) - continue - new_title = None - if _is_reference(b["title"]) and (suggestions := renames.get(b["title_norm"])): - cands = max(suggestions, key=lambda k: (suggestions[k], len(k))) - if not _is_reference(cands): - new_title = cands - if new_title: - nn, t, n = _norm_title(new_title), new_title, 2 - while nn in seen_norm: - t, nn, n = f"{new_title} ({n})", _norm_title(f"{new_title} ({n})"), n + 1 - seen_norm.add(nn) - await db.set_block_status(topic, b["title_norm"], "consensus", title=t, neu_norm=nn) - else: - await db.set_block_status(topic, b["title_norm"], "consensus") - # C4 — audit journal (swept by the existing gd("clarification*") on reset; not needed for - # correctness — clarification is rebuild-not-rollback — only to see what the gate pruned). - atomic_write_json(work_dir / "clarification-journal.json", - {"pre_rejected": len(pre_rejected), "quorum_pruned": len(pruned) - len(pre_rejected), - "pruned": pruned}, indent=1) - await db.set_step_status(topic, "Clarification", "done") - return True def _pairs_schema(data) -> dict[int, bool] | None: @@ -2607,142 +1890,6 @@ def _relation_conflict(title_a: str, title_b: str) -> bool: return a is not None and b is not None and a != b -async def _dedup_inventory(ctx: GenContext, set_p, files: dict) -> bool: - """Final dedup pass over the finished consensus list: pairwise verification (entity - resolution). Embedding yields candidate PAIRS (cosine ≥ DEDUP_PAAR_FLOOR), a judge - confirms EACH pair individually (ja = the same duplicate). ONLY confirmed pairs become - merge edges (union-find) — no chaining, no aspect over-merging like the block mixer. - Per group ONE representative (main concept) stays, the rest is discarded.""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - if await db.get_step_status(topic, "Dedup") == "done": - return True - if not (EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available)): - await db.set_step_status(topic, "Dedup", "done") # without a model: silently skip - return True - set_p("Dedup…", step=_step_idx(topic, "Dedup")) - work_dir = files["arbeit"] - consensus = await db.list_blocks(topic, status="consensus") - if len(consensus) >= 2: - import numpy as np - # Candidate cosine = MEAN of title-only and title+description similarity. Dedup is entity - # resolution: the TITLE identifies the concept, the description only elaborates. Title-duplicates - # with divergent descriptions ("SetCover" vs "SetCover-Problem": title-cos ~0.69 but title+desc - # ~0.54) fell below the 0.6 floor and were never checked. Averaging boosts them over the floor - # while keeping pairs that are unrelated in BOTH signals out (title-similar-only reduction - # families like "3-SAT ≤ X"/"3-SAT ≤ Y" stay below 0.6 via the divergent description). The - # per-pair judge still verifies each candidate, so recall rises without loosening precision. - texts_full = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in consensus] - texts_title = [b["title"] for b in consensus] - sims_full = await asyncio.to_thread(embedding.embed_sims, texts_full) - sims_title = await asyncio.to_thread(embedding.embed_sims, texts_title) - sims = (sims_full + sims_title) / 2 if (sims_full is not None and sims_title is not None) else sims_full - if sims is not None: - n = len(consensus) - iu = np.triu_indices(n, k=1) - cands = [(int(iu[0][m]), int(iu[1][m])) for m in np.where(sims[iu] >= DEDUP_PAIR_FLOOR)[0]] - # D3 — canonical-name blocking: raise recall by force-adding pairs whose scaffolding-stripped, - # operator-normalized key is EQUAL (embedding can miss them: "Offenes Problem P=NP?" vs "P vs - # NP" share no wording). The pairwise judge still verifies each; exact-key pairs additionally - # auto-merge below (ER-standard, ~100% precision). key_groups reused for the auto-merge. - key_groups: dict[str, list[int]] = {} - for i, b in enumerate(consensus): - k = _canonical_key(b["title"]) - if k: - key_groups.setdefault(k, []).append(i) - cand_set = set(cands) - for grp in key_groups.values(): - for x in range(len(grp)): - for y in range(x + 1, len(grp)): - cand_set.add((grp[x], grp[y])) - cands = sorted(cand_set) - _log(topic, f"Dedup: {len(cands)} candidate pairs (mean(title,title+desc) cosine ≥ {DEDUP_PAIR_FLOOR}, " - f"+ canonical-key) → pairwise filter") - packages = [cands[i:i + DEDUP_PAIRS_CHUNK] for i in range(0, len(cands), DEDUP_PAIRS_CHUNK)] - - def pair_path(pi): return work_dir / f"dedup-paar-c{pi}.json" - - async def _filt(pi, paare): - fp = pair_path(pi) - if _pairs_schema(_json_file(fp)): - return # resume - lines = "\n\n".join( - f"{j + 1}.\nA: {consensus[a]['title']} — {consensus[a]['description']}" - f"\nB: {consensus[b]['title']} — {consensus[b]['description']}" - for j, (a, b) in enumerate(paare)) - await run_single_slot( - ctx, f"Dedup pairs {pi}", - key=f"blocks-{topic}-dedup-paar-c{pi}", - prompt=_prompt("Blocks-Paar-Filter", topic=topic, pairs=lines, out_path=fp), - role="judge", capabilities="files", - payload=lambda result, p=fp: _pairs_schema(_json_file(p)), - timeout=_timeout("selection_mapping", len(paare)), - ) - - await _gather_progress([_filt(pi, p) for pi, p in enumerate(packages)], - len(packages), _report_p(set_p, topic, "Dedup")) - if is_cancelled(): - return False - # Collect confirmed "ja" edges, then COMPLETE-LINK (greedy cliques) instead of single-link - # union-find — prevents chaining (A=B + B=C does NOT merge A,C without a direct A=C). - edge_list, ja, rel_blocked = [], 0, 0 - for pi, paare in enumerate(packages): - verdict = _pairs_schema(_json_file(pair_path(pi))) or {} - for j, (a, b) in enumerate(paare): - if verdict.get(j + 1): - # D2 — relation-triple guard: never merge two DIFFERENT reductions/relations even - # if the judge said "ja" (a reduction is individuated by both operands + direction). - if _relation_conflict(consensus[a]["title"], consensus[b]["title"]): - rel_blocked += 1 - continue - edge_list.append((a, b)) - ja += 1 - # Autonomous recall net: near-identical TITLE cosine ⇒ same entity, merge without the judge - # (catches false-negatives like "Algorithmus ΔTSP1"/"ΔTSP1"). Duplicate edges are harmless - # (_cliques builds a set adjacency). These land in `groups` → dedup-runde-1.json → reversible. - auto = 0 - if sims_title is not None: - for a, b in cands: - if float(sims_title[a][b]) >= DEDUP_TITLE_AUTO and not _relation_conflict( - consensus[a]["title"], consensus[b]["title"]): - edge_list.append((a, b)) - auto += 1 - # D3 — exact-canonical-key auto-merge (equal key ⇒ same entity, ER ~100% precision). The - # relation guard still applies: a sorted key can collide on direction ("A→B"/"B→A") — the - # operand check blocks that, so only genuinely identical entities merge. - key_auto = 0 - for grp in key_groups.values(): - for x in range(len(grp)): - for y in range(x + 1, len(grp)): - a, b = grp[x], grp[y] - if not _relation_conflict(consensus[a]["title"], consensus[b]["title"]): - edge_list.append((a, b)) - key_auto += 1 - groups = _cliques(n, edge_list) - removed = 0 - for idxs in groups: - # D1 — survivorship / golden-record (MDM standard): the representative is the MOST - # INFORMATIVE record, NEVER the shortest. Cascade (higher wins): fewest property markers - # (the main concept, not a "…-Grenze"/"…-Schranke" fragment) → richest description (most - # complete) → more specific/longer title (never shortest) → stable low index. - rep = max(idxs, key=lambda k: ( - -_aspect_marker(consensus[k]["title"]), - len(consensus[k]["description"] or ""), - len(consensus[k]["title"]), - -k)) - for k in idxs: - if k != rep: - await db.set_block_status(topic, consensus[k]["title_norm"], "discarded") - removed += 1 - from collections import Counter - atomic_write_json(work_dir / "dedup-runde-1.json", - {"vorher": n, "entfernt": removed, "paare_geprueft": len(cands), "paare_ja": ja, - "auto_titel": auto, "auto_key": key_auto, "relation_blockiert": rel_blocked, - "clique_groessen": dict(sorted(Counter(len(g) for g in groups).items())), - "groups": [[consensus[k]["title"] for k in g] for g in groups]}, indent=1) - _log(topic, f"Dedup (pairwise): {n} → {n - removed} (−{removed}); {ja}/{len(cands)} pairs confirmed, " - f"{auto} auto-title, {key_auto} auto-key, {rel_blocked} relation-conflicts blocked") - await db.set_step_status(topic, "Dedup", "done") - return True def _filter_schema(data) -> dict[int, int] | None: @@ -2904,207 +2051,6 @@ def _is_named_statement(title: str, desc: str = "") -> bool: return False -async def _filter_inventory(ctx: GenContext, set_p, files: dict) -> bool: - """Degrade pass (granularity): separates real blocks from fragments (properties, - proof gadgets, notation, runtime details). Each judge sees the FULL block list - (self-containment is relational) and marks fragments WITH a parent block from the list. - Fragment + parent-in-list → discarded (content comes back as a subblock of the parent). - No parent or in doubt → keep (no concept loss).""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - if await db.get_step_status(topic, "Blocks-Filter") == "done": - return True - set_p("Blocks-Filter…", step=_step_idx(topic, "Blocks-Filter")) - work_dir = files["arbeit"] - consensus_all = await db.list_blocks(topic, status="consensus") - # Safety net: discard pure notation autonomously (FP~0, no parent needed). The judge - # reliably overlooks such symbols (recall problem), hence deterministically beforehand. - consensus, notation_dropped = [], [] - for b in consensus_all: - if _FILTER_NOTATION.search(b["title"]): - await db.set_block_status(topic, b["title_norm"], "discarded") - notation_dropped.append(b["title"]) - else: - consensus.append(b) - if notation_dropped: - _log(topic, f"Blocks-Filter: {len(notation_dropped)} pure notation discarded: {notation_dropped[:6]}") - if len(consensus) < 2: - await db.set_step_status(topic, "Blocks-Filter", "done") - return True - n = len(consensus) - # ⚠ marks suspicious lines (property/runtime) — the judge MUST check them per entry. - def _line(i, b): - mark = "⚠ " if _filter_suspect(b) else "" - return f"{i}. {mark}{b['title']} — {b['description']}" if b["description"] else f"{i}. {mark}{b['title']}" - full_list = "\n".join(_line(i, b) for i, b in enumerate(consensus, 1)) - chunks = [list(range(i, min(i + FILTER_CHUNK, n + 1))) for i in range(1, n + 1, FILTER_CHUNK)] - - def filt_path(ci): return work_dir / f"inventar-filter-c{ci}.json" - - async def _assess(ci, numbers): - fp = filt_path(ci) - if _filter_schema(_json_file(fp)) is not None: - return # resume - await run_single_slot( - ctx, f"Blocks-Filter {ci}", - key=f"blocks-{topic}-inventar-filter-c{ci}", - prompt=_prompt("Blocks-Filter", topic=topic, list=full_list, - from_n=numbers[0], to_n=numbers[-1], out_path=fp), - role="judge", capabilities="files", - payload=lambda result, p=fp: _filter_schema(_json_file(p)), - timeout=_timeout("selection_mapping", len(numbers)), - ) - - await _gather_progress([_assess(ci, nm) for ci, nm in enumerate(chunks)], - len(chunks), _report_p(set_p, topic, "Blocks-Filter")) - if is_cancelled(): - return False - fragments: dict[int, int] = {} - drops: set[int] = set() - for ci, numbers in enumerate(chunks): - raw = _json_file(filt_path(ci)) - verdict = _filter_schema(raw) or {} - nset = set(numbers) - for nr, parent in verdict.items(): - if 1 <= parent <= n and nr in nset: - fragments[nr] = parent - # `drop` is read additively from the raw JSON (like the `rename` field in _clarify_inventory) - # so _filter_schema keeps returning dict[int,int] and its three call sites stay untouched. - for x in (raw.get("drop", []) if isinstance(raw, dict) else []): - try: - dnr = int(x) - except (ValueError, TypeError): - continue - if 1 <= dnr <= n and dnr in nset: - drops.add(dnr) - # Hard-drop double-gate (Fix B): honour a judge `drop` ONLY if the title also matches the artefact - # regex — a parentless drop is the one irreversible-by-design deletion, so a single judge FP must - # not delete a concept. A judge-drop without a match degrades to "keep" (logged), never deleted. - honored_drops = {nr for nr in drops if _is_artifact(consensus[nr - 1]["title"])} - refused = drops - honored_drops - if refused: - _log(topic, f"Blocks-Filter: {len(refused)} judge-drop(s) refused (no artefact match, kept): " - f"{[consensus[nr - 1]['title'] for nr in sorted(refused)][:6]}") - for nr in honored_drops: # a hard-drop wins over a demote (it is noise, not a fragment of X) - fragments.pop(nr, None) - # Deterministic parent-by-containment demote (recall net for the single judge): a ⚠-flagged survivor - # whose title NAMES another block is demoted to it without the judge (near-FP-0: whole-word + - # significant parent + exactly-one match). Feeds the same `fragments` journal → transitive + reversible. - norms = [(i, consensus[i - 1]["title_norm"]) for i in range(1, n + 1)] - cont = 0 - for i in range(1, n + 1): - if i in fragments or i in honored_drops or not _filter_suspect(consensus[i - 1]): - continue - parent = _containment_parent(consensus[i - 1]["title_norm"], [(nr, t) for nr, t in norms if nr != i]) - if parent is not None and parent != i and parent not in honored_drops: - fragments[i] = parent - cont += 1 - if cont: - _log(topic, f"Blocks-Filter: {cont} fragment(s) auto-demoted by title-containment") - # Narrow journaled parent-less hard-drop (F3): bare label refs (Remark 7.28 / Satz D*) and single-var - # notation (r = n + m) that have NO parent — routed through honored_drops so they land in the `fragments` - # journal (2822-style) and stay reversible, unlike the un-journaled _FILTER_NOTATION prepass. KEEP-guards - # inside _is_parentless_noise protect named theorems / class (in)equalities / definitions. - pl = 0 - for i in range(1, n + 1): - if i in fragments or i in honored_drops or not _is_parentless_noise(consensus[i - 1]["title"]): - continue - if _containment_parent(consensus[i - 1]["title_norm"], [(nr, t) for nr, t in norms if nr != i]) is None: - honored_drops.add(i) - pl += 1 - if pl: - _log(topic, f"Blocks-Filter: {pl} parent-less noise block(s) hard-dropped (Remark/Satz-label/notation)") - # F2 — survivor-recheck panel: the main filter is a SINGLE judge and "fragment" is a rare positive - # class (single-pass recall ~30-60%). A focused 3-judge panel over ONLY the still-⚠-surviving, - # un-demoted blocks (majority ≥2) recovers the judge-missable ones. Same {fragments, drop} schema → - # merged into fragments/honored_drops (→ journaled by the loop below, reversible). - survivors = [i for i in range(1, n + 1) - if i not in fragments and i not in honored_drops and _filter_suspect(consensus[i - 1])] - if survivors: - rchunks = [survivors[k:k + FILTER_CHUNK] for k in range(0, len(survivors), FILTER_CHUNK)] - - def rc_path(ci, j): return work_dir / f"inventar-filter-recheck-c{ci}-j{j}.json" - - async def _recheck(ci, nums): - block = "\n".join( - (f"{i}. {consensus[i - 1]['title']} — {consensus[i - 1]['description']}" - if consensus[i - 1]['description'] else f"{i}. {consensus[i - 1]['title']}") for i in nums) - pending = [j for j in range(1, FILTER_RECHECK_PANEL + 1) if _filter_schema(_json_file(rc_path(ci, j))) is None] - if is_cancelled(): - return - await asyncio.gather(*[ - run_agent(f"blocks-{topic}-filter-recheck-c{ci}-j{j}", - _prompt("Blocks-Filter-Recheck", topic=topic, survivors=block, list=full_list, out_path=rc_path(ci, j)), - _timeout("selection_mapping", len(nums)), provider=ctx.provider, role="judge", capabilities="files") - for j in pending], return_exceptions=True) - - await _gather_progress([_recheck(ci, nm) for ci, nm in enumerate(rchunks)], - len(rchunks), _report_p(set_p, topic, "Blocks-Filter")) - if is_cancelled(): - return False - rc = 0 - for ci, nums in enumerate(rchunks): - nset, dem, drp = set(nums), {}, {} - for j in range(1, FILTER_RECHECK_PANEL + 1): - raw = _json_file(rc_path(ci, j)) - v = _filter_schema(raw) - if v is None: - continue - for nr, parent in v.items(): - if nr in nset and 1 <= parent <= n and nr != parent: - dem.setdefault(nr, []).append(parent) - for x in (raw.get("drop", []) if isinstance(raw, dict) else []): - try: - dnr = int(x) - except (ValueError, TypeError): - continue - if dnr in nset: - drp[dnr] = drp.get(dnr, 0) + 1 - for nr in nums: # majority ≥2; drop only if the title is also artefact/parent-less noise - if nr in fragments or nr in honored_drops: - continue - if drp.get(nr, 0) >= 2 and (_is_artifact(consensus[nr - 1]["title"]) or _is_parentless_noise(consensus[nr - 1]["title"])): - honored_drops.add(nr); rc += 1 - elif len(dem.get(nr, [])) >= 2: - fragments[nr] = max(set(dem[nr]), key=dem[nr].count); rc += 1 - if rc: - _log(topic, f"Blocks-Filter: recheck panel demoted/dropped {rc} more survivor(s)") - # F1 — statement-gate keep-guard (final override): a named reduction between two problems, or a - # labeled/attributed theorem WITH its own ⇔/⇒ assertion, is a knowledge unit — rescue it from ANY - # producer (judge, containment, parentless, recheck). Runs last so it wins; bare labels / unary status - # / proof-size steps are untouched (they fail _is_named_statement) and stay demoted/dropped. - def _protected(nr): - return _is_named_statement(consensus[nr - 1]["title"], consensus[nr - 1]["description"]) - saved = [nr for nr in list(fragments) if _protected(nr)] - for nr in saved: - fragments.pop(nr, None) - saved_drops = {nr for nr in honored_drops if _protected(nr)} - honored_drops -= saved_drops - if saved or saved_drops: - _log(topic, f"Blocks-Filter: {len(saved) + len(saved_drops)} named statement(s)/reduction(s) " - f"protected from demotion (statement-gate)") - # Transitive resolution (Fix C) + drops (Fix B) in one pass. Every discard is recorded under the - # SAME `fragments` key of inventar-filter.json → the existing reset rollback (see _reset_from_step) - # restores it, so both fixes stay wiring-free and reversible. - removed, debug = 0, [] - for nr in fragments: - root, cyclic = _root(nr, fragments) - if cyclic or not (1 <= root <= n): # cycle / invalid root → keep (no loss) - continue - await db.set_block_status(topic, consensus[nr - 1]["title_norm"], "discarded") - removed += 1 - if root in honored_drops: # ancestor judged noise → the child is collateral noise too - debug.append({"fragment": consensus[nr - 1]["title"], "eltern": None, "grund": "drop-collateral"}) - else: - debug.append({"fragment": consensus[nr - 1]["title"], "eltern": consensus[root - 1]["title"]}) - for nr in sorted(honored_drops): # explicit parentless hard-drops - await db.set_block_status(topic, consensus[nr - 1]["title_norm"], "discarded") - removed += 1 - debug.append({"fragment": consensus[nr - 1]["title"], "eltern": None, "grund": "drop"}) - atomic_write_json(work_dir / "inventar-filter.json", - {"vorher": n, "degradiert": removed, "fragments": debug}, indent=1) - _log(topic, f"Blocks-Filter: {n} → {n - removed} (−{removed}: fragments→subblocks + {len(honored_drops)} hard-drops)") - await db.set_step_status(topic, "Blocks-Filter", "done") - return True def _umbrella_schema(data, ids: set[int]): @@ -3188,243 +2134,6 @@ _GROUP_STANDALONE = re.compile( r'|\w*vollständigkeit\b|\w*completeness\b|\w*transformation(?:en)?\b|\w*klasse[nr]?\b', re.I) -async def _group_inventory(ctx: GenContext, set_p, files: dict, dry_run: bool = False) -> bool: - """Umbrella grouping (granularity level 2, AFTER the filter): collapse sibling DEFINITIONS that - are components of ONE umbrella concept into a single block whose description enumerates the - children (→ the subblock step re-derives them from the source). Embedding builds coarse capped - candidate clusters (low sibling-floor → high recall); one judge per multi-cluster sees the FULL - surviving list as context (hybrid recall) and synthesizes umbrella title+description+members. - Skipped (like Dedup) when the flag is off or no embedding model. Idempotent: per-cluster judge - artefacts + a single tail DB pass. dry_run writes the decision artefact WITHOUT any DB mutation - (and does NOT mark the step done) — for safe tuning on a clone.""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - if not BLOCKS_GRUPPIERUNG_AKTIV: - await db.set_step_status(topic, "Blocks-Gruppierung", "done") - return True - if not dry_run and await db.get_step_status(topic, "Blocks-Gruppierung") == "done": - return True - if not (EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available)): - await db.set_step_status(topic, "Blocks-Gruppierung", "done") - return True - set_p("Blocks-Gruppierung…", step=_step_idx(topic, "Blocks-Gruppierung")) - work_dir = files["arbeit"] - consensus = await db.list_blocks(topic, status="consensus") - n = len(consensus) - if n < 3: # nothing meaningful to group - await db.set_step_status(topic, "Blocks-Gruppierung", "done") - return True - texts = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in consensus] - sims = await asyncio.to_thread(embedding.embed_sims, texts) - if sims is None: - await db.set_step_status(topic, "Blocks-Gruppierung", "done") - return True - clusters = await asyncio.to_thread(embedding.capped_blocks, sims, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP) - multi = [c for c in clusters if len(c) > 1] - - def _min_cos(idxs): # internal coherence (chains would be ~0.3 → over-merge signal) - if len(idxs) < 2: - return 1.0 - return round(min(float(sims[i][j]) for a, i in enumerate(idxs) for j in idxs[a + 1:]), 3) - - all_ids = set(range(1, n + 1)) - full_list = "\n".join(f"{i}. {texts[i - 1]}" for i in range(1, n + 1)) - - def grp_path(ci): return work_dir / f"gruppierung-block-c{ci}.json" - - async def _assess(ci, cluster): - p = grp_path(ci) - if _umbrella_schema(_json_file(p), all_ids) is not None: - return # resume: keep a valid file - p.unlink(missing_ok=True) - if is_cancelled(): - return - cand = "\n".join(f"{g + 1}. {texts[g]}" for g in cluster) # global numbers (1-based) - await run_single_slot( - ctx, f"Blocks-Gruppierung {ci}", - key=f"blocks-{topic}-gruppierung-block-c{ci}", - prompt=_prompt("Blocks-Gruppierung", topic=topic, candidates=cand, list=full_list, out_path=p), - role="judge", capabilities="files", - payload=lambda result, p=p: _umbrella_schema(_json_file(p), all_ids), - timeout=_timeout("research_mapping", len(cluster)), - ) - - # Top-down pass (recall): one judge sees ONLY the full list (no cluster seed) and proposes umbrellas - # across it — cures the single-linkage fragmentation where heterogeneous siblings (TM-model, KNF) - # never co-cluster and so are never seeded as one candidate. Its output flows through the SAME - # collection loop → type-gate → reconcile (which merges it with any cluster-proposed duplicate). - async def _assess_top(): - p = grp_path("TOP") - if _umbrella_schema(_json_file(p), all_ids) is not None: - return # resume - p.unlink(missing_ok=True) - if is_cancelled(): - return - cand = ("Scan the ENTIRE block list below and propose EVERY genuine umbrella you find — " - "do not restrict yourself to any subset.") - await run_single_slot( - ctx, "Blocks-Gruppierung top", - key=f"blocks-{topic}-gruppierung-block-cTOP", - prompt=_prompt("Blocks-Gruppierung", topic=topic, candidates=cand, list=full_list, out_path=p), - role="judge", capabilities="files", - payload=lambda result, p=p: _umbrella_schema(_json_file(p), all_ids), - timeout=_timeout("research_mapping", n), - ) - - await _gather_progress([_assess(ci, c) for ci, c in enumerate(multi)] + [_assess_top()], - len(multi) + 1, _report_p(set_p, topic, "Blocks-Gruppierung")) - if is_cancelled(): - return False - - # Collect umbrellas across clusters; GLOBAL first-wins dedup on members (a block can be pulled - # into only one umbrella). Resolve title collisions BEFORE any DB write. - used, seen_norm = set(), {b["title_norm"] for b in consensus} - chosen, skipped, plan = [], [], [] - sources = [grp_path("TOP")] + [grp_path(ci) for ci in range(len(multi))] # TOP first → wins used-ties - for src in sources: - for title, desc, members in (_umbrella_schema(_json_file(src), all_ids) or []): - # " — " (space-em/en-dash-space) is the reserved title/description separator; a title - # containing it would be truncated by _title() downstream ("P||Cmax — X" → "P||Cmax", - # colliding with the real "P||Cmax" block). Defuse it regardless of what the judge emits. - title = re.sub(r"\s+[—–]\s+", ": ", title).strip() - members = [m for m in members if m not in used] - if len(members) < 2: - continue - rows = [consensus[m - 1] for m in members] - # Type gate (deterministic backstop): a member that is a named algorithm/problem/reduction/ - # theorem is a standalone block, never a sub-definition → dissolve the umbrella, keep members. - if any(_GROUP_STANDALONE.search(r["title"]) for r in rows): - skipped.append({"umbrella": title, "grund": "type-gate", - "mitglieder": [r["title"] for r in rows]}) - continue - # No-structure backstop ONLY (member-vs-member cosine is the WRONG instrument for over-merge — - # meronymy ≠ similarity, empirically inverted; the atomicity type-guard above is the real - # precision floor). Rejects a literally structureless chain (below the random-pair baseline); - # the floor sits BELOW the legitimate heterogeneous minimum so it never kills a real model. - mc = _min_cos([m - 1 for m in members]) - if mc < GROUP_MIN_COS_FLOOR: - skipped.append({"umbrella": title, "grund": "min-cos", "min_cos": mc, - "mitglieder": [r["title"] for r in rows]}) - continue - unorm = _norm_title(title) - member_norms = {r["title_norm"] for r in rows} - if unorm not in member_norms and unorm in seen_norm: # collision with a NON-member block - skipped.append({"umbrella": title, "grund": "title-collision", - "mitglieder": [r["title"] for r in rows]}) - continue - used.update(members) - seen_norm.add(unorm) - chosen.append({"umbrella": title, "description": desc, "min_cos": mc, - "mitglieder": [{"title": r["title"], "description": r["description"], - "title_norm": r["title_norm"]} for r in rows]}) - plan.append((unorm, title, desc, rows)) - - # Reconcile pass: two independently-judged clusters can emit the SAME parent under different titles - # + disjoint members (e.g. two „Turingmaschine"-umbrellas) — neither member-id nor title dedup catches - # that. Merge umbrella pairs whose title+description cosine ≥ GROUP_RECONCILE_FLOOR: union members, - # concatenate the enumerating descriptions (every child stays named → subblock re-derivation intact), - # keep the title of the umbrella with the most members. Deterministic (embed_sims), no extra LLM. - if len(chosen) >= 2: - u_sims = await asyncio.to_thread( - embedding.embed_sims, [f"{c['umbrella']} — {c['description']}" for c in chosen]) - if u_sims is not None: - parent = list(range(len(chosen))) - for i in range(len(chosen)): - for j in range(i + 1, len(chosen)): - if float(u_sims[i][j]) >= GROUP_RECONCILE_FLOOR: - embedding._union(parent, i, j) - comp: dict[int, list[int]] = {} - for i in range(len(chosen)): - comp.setdefault(embedding._find(parent, i), []).append(i) - m_chosen, m_plan = [], [] - for grp in comp.values(): - if len(grp) == 1: - m_chosen.append(chosen[grp[0]]) - m_plan.append(plan[grp[0]]) - continue - seen_m, rows = set(), [] # union member rows (dedup by norm) - for gi in grp: - for r in plan[gi][3]: - if r["title_norm"] not in seen_m: - seen_m.add(r["title_norm"]) - rows.append(r) - rep = max(grp, key=lambda gi: len(plan[gi][3])) # richest umbrella keeps its title - title = plan[rep][1] - desc = " · ".join(chosen[gi]["description"] for gi in grp) - m_chosen.append({**chosen[rep], "umbrella": title, "description": desc, - "mitglieder": [{"title": r["title"], "description": r["description"], - "title_norm": r["title_norm"]} for r in rows], - "reconciled_from": [chosen[gi]["umbrella"] for gi in grp]}) - m_plan.append((_norm_title(title), title, desc, rows)) - if len(m_chosen) < len(chosen): - _log(topic, f"Blocks-Gruppierung: reconciled {len(chosen)} → {len(m_chosen)} umbrellas (merged same-parent duplicates)") - chosen, plan = m_chosen, m_plan - - # Membership-completion pass (G-B): reconcile only merges umbrella↔umbrella, so a partial umbrella - # (e.g. TM with 2 of ~6 parts) + leftover standalone siblings never unites. One judge — anchored on the - # chosen umbrellas (parent + enumerated parts) — decides which of the still-standalone blocks are ALSO - # constituent parts of each parent; a per-member type-guard veto keeps precision. Updates chosen+plan+ - # used in lockstep so the artefact stays reset-faithful. Empty/absent → no-op. - leftover = sorted(all_ids - used) - if chosen and leftover: - cp = work_dir / "gruppierung-completion.json" - add = _completion_schema(_json_file(cp), len(chosen), set(leftover)) - if add is None: - cp.unlink(missing_ok=True) - if not is_cancelled(): - anchors = "\n".join( - f"UMBRELLA {k}: {c['umbrella']} — {c['description']}\n bereits: " - + ", ".join(m["title"] for m in c["mitglieder"]) for k, c in enumerate(chosen)) - rest = "\n".join(f"{i}. {texts[i - 1]}" for i in leftover) - status, add = await run_single_slot( - ctx, "Blocks-Gruppierung completion", - key=f"blocks-{topic}-gruppierung-completion", - prompt=_prompt("Blocks-Gruppierung-Completion", topic=topic, umbrellas=anchors, rest=rest, out_path=cp), - role="judge", capabilities="files", - payload=lambda result, p=cp: _completion_schema(_json_file(p), len(chosen), set(leftover)), - timeout=_timeout("research_mapping", len(leftover))) - add = add if status == OK else [] - if is_cancelled(): - return False - norm2idx = {consensus[i - 1]["title_norm"]: i for i in range(1, n + 1)} - absorbed = 0 - for k, new_members in (add or []): - hit = False - for m in new_members: - if m in used or not (1 <= m <= n) or _GROUP_STANDALONE.search(consensus[m - 1]["title"]): - continue # type-guard veto: never absorb a named algorithm/problem/theorem - r = consensus[m - 1] - used.add(m) - chosen[k]["mitglieder"].append({"title": r["title"], "description": r["description"], "title_norm": r["title_norm"]}) - plan[k][3].append(r) - absorbed += 1 - hit = True - if hit: # refresh the diagnostic min_cos over the enlarged member set - idxs = [norm2idx[mm["title_norm"]] - 1 for mm in chosen[k]["mitglieder"] if mm["title_norm"] in norm2idx] - chosen[k]["min_cos"] = _min_cos(idxs) - if absorbed: - _log(topic, f"Blocks-Gruppierung: completion absorbed {absorbed} standalone part(s) into umbrellas") - - entfernt = sum(len(c["mitglieder"]) for c in chosen) - len(chosen) - atomic_write_json(work_dir / "inventar-gruppierung.json", - {"vorher": n, "nachher": n - entfernt, "umbrellas": chosen, "skipped": skipped}, indent=1) - _log(topic, f"Blocks-Gruppierung: {len(multi)} clusters → {len(chosen)} umbrellas " - f"(−{entfernt} blocks{', DRY-RUN (no DB write)' if dry_run else ''})") - if dry_run: - return True # decision written; no DB mutation, step NOT marked done - - # Single tail pass — all DB writes here (idempotent under the step gate). The umbrella row carries - # the enumerating description; members (except a title-reused anchor) go to discarded. - for unorm, title, desc, rows in plan: - await db.upsert_block(topic, unorm, title, desc) - # Force title+desc: upsert's ON CONFLICT keeps the OLD description, so an anchor-reuse umbrella - # (unorm == a member's norm) or a reset→rerun would otherwise keep a stale/narrow description and - # the enumerating child list — the subblock step's re-derivation anchor — would be lost. - await db.set_block_status(topic, unorm, "consensus", title=title, description=desc) - for r in rows: - if r["title_norm"] != unorm: - await db.set_block_status(topic, r["title_norm"], "discarded") - await db.set_step_status(topic, "Blocks-Gruppierung", "done") - return True # --- Outline (blocks artifact: chapter structure, only read by the guide) --- @@ -3663,7 +2372,7 @@ def _artefacts_complete(files: dict) -> bool: return isinstance(d, dict) and all(t in d for t in ARTEFACT_TYPES) -async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: +async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "") -> dict | None: """Generate learning artefacts per type from the stored facts — one generation pass per type over chunks. Worked examples are verified against the facts (wrong ones discarded); flashcards are low-risk and stay unchecked. → {type: [entries]} (also in files).""" @@ -3708,7 +2417,7 @@ async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _example_check_schema(_json_file(cpath(j))) is None] if pending: await asyncio.gather(*[ - run_agent(f"blocks-{topic}-artifact-example-check-c{ci}-j{j}", + run_agent(f"blocks-{topic}-{ns}artifact-example-check-c{ci}-j{j}", _prompt("Artifact-Example-Check", topic=topic, facts=block_text(idxs), examples=examples_txt, out_path=cpath(j), extra=_extra(instructions)), _timeout("content_check", len(items)), provider=provider, role="judge", capabilities=caps) for j in pending], return_exceptions=True) @@ -3725,33 +2434,40 @@ async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i _log(topic, f"Worked-example check chunk {ci}: {len(dropped)}/{len(items)} discarded") return [e for k, e in enumerate(items, 1) if k not in dropped] - outcome: dict[str, list] = {} - for type in ARTEFACT_TYPES: - schema = _ARTEFACT_SCHEMA[type] - def apath(ci, t=type): return work_dir / f"artifact-{t}-c{ci}.json" + async def _one_type(typ: str) -> list | None: + """Full pipeline of ONE artefact type — the types are independent (own schemas, + own files) and run in parallel.""" + schema = _ARTEFACT_SCHEMA[typ] - async def _gen(ci, idxs, t=type, schema=schema): - p = apath(ci, t) + def apath(ci): return work_dir / f"artifact-{typ}-c{ci}.json" + + async def _gen(ci, idxs): + p = apath(ci) if schema(_json_file(p)) is not None: return True await run_single_slot( - ctx, f"{_ARTEFACT_STEP[t]} {ci}", key=f"blocks-{topic}-artifact-{t}-c{ci}", - prompt=_prompt(_ARTEFACT_PROMPT[t], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)), + ctx, f"{_ARTEFACT_STEP[typ]} {ci}", key=f"blocks-{topic}-{ns}artifact-{typ}-c{ci}", + prompt=_prompt(_ARTEFACT_PROMPT[typ], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)), role="guide", capabilities="files", - payload=lambda result, p=p, schema=schema: schema(_json_file(p)), + payload=lambda result, p=p: schema(_json_file(p)), timeout=_timeout("content", sum(len(blocks[i][1]) for i in idxs))) return schema(_json_file(p)) is not None - await _gather_progress([_gen(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, _ARTEFACT_STEP[type])) + await _gather_progress([_gen(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, _ARTEFACT_STEP[typ])) if is_cancelled(): return None eintraege: list = [] for ci in range(len(chunks)): chunk_items = schema(_json_file(apath(ci))) or [] - if type == "example" and chunk_items: + if typ == "example" and chunk_items: chunk_items = await _check_examples(ci, chunks[ci], chunk_items) eintraege += chunk_items - outcome[type] = eintraege + return eintraege + + results = await asyncio.gather(*[_one_type(t) for t in ARTEFACT_TYPES]) + if is_cancelled() or any(r is None for r in results): + return None + outcome: dict[str, list] = dict(zip(ARTEFACT_TYPES, results)) atomic_write_json(files["artefakte"], outcome, indent=1) return outcome @@ -3807,33 +2523,19 @@ async def _mirror_question_pattern_db(topic: str, pattern: dict) -> None: await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question) -async def _reset_db_from_phase(topic: str, label: str) -> None: - """Discard DB content of phases ≥ `label` (canonical order Source…Artefacts).""" - idx = _phase_idx(label) - if idx <= 8: # Artefacts (flashcards/examples) - await db.delete_sub_artefakte(topic) - if idx <= 7: # Questions - await db.delete_question_pattern(topic) - if idx <= 6: # Outline - await db.delete_outline(topic) - if idx <= 2: # Subblocks (facts/levels/relevance go through sidecar→mirror) - await db.delete_subblocks(topic) - if idx <= 1: # Inventory: inventory + research steps — triage stays - await db.delete_blocks(topic) - await db.delete_pipeline_state(topic, ["Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung"]) - if idx <= 0: # Source: redo triage (coverage/content + step) - await db.delete_coverage(topic) - await db.delete_pipeline_state(topic, ["Source prep"]) -async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, ab_phase: int | None = None, ab_step: int | None = None, to_step: int | None = None) -> None: +async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, + research: bool = True) -> None: + """Kanban entry point: source prep, then both boards (inventory + artefacts) until + quiescence. research=False = Continue (drain the existing queue, no new search). + A run on a finished topic ADDS research (live extension) — full rebuild = DELETE /blocks.""" if topic in _blocks_progress: return - _blocks_progress[topic] = "Waiting…" + _blocks_progress[topic] = "Warten…" _blocks_errors.pop(topic, None) files = _blocks_files(topic) - final_path = files["final"] q = load_source(topic) folder = source_folder(topic) # projekt/uni/link → folder, thema → None instructions = q.get("spec") or instructions # prefer the persisted specification (also on resume) @@ -3846,237 +2548,20 @@ async def generate_blocks(topic: str, instructions: str = "", provider: str = DE def is_cancelled() -> bool: return topic in _blocks_cancelled - def aborted() -> None: - _blocks_errors[topic] = "Cancelled — progress is preserved" - - _step_list = _blocks_steps(topic) - def _past_limit(step: str) -> bool: # optional end limit: stop before any step past to_step - return to_step is not None and step in _step_list and _step_list.index(step) > to_step - ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled) - try: async with _semaphore: files["arbeit"].mkdir(parents=True, exist_ok=True) - # Re-run from the chosen phase: delete artefacts from there; the fresh-start block - # below is skipped (with a preserved sidecar it would otherwise wipe everything). - if ab_step is not None: # fine sub-step re-run (takes precedence over ab_phase) - await _reset_from_step(topic, ab_step) - elif ab_phase is not None: - phasen = _phases(topic) - label = phasen[ab_phase - 1][0] if 1 <= ab_phase <= len(phasen) else "Inventory" - _reset_from_phase(topic, label) - await _reset_db_from_phase(topic, label) - # A stage returning False ends generation; if it was a cancel, mark aborted first. - async def _stage(coro) -> bool: - ok = await coro - if not ok and is_cancelled(): - aborted() - return ok - # Step "Source prep": crawl (link) + PDFs + content/noise triage. - if not await _stage(_prepare_source(ctx, set_p, files, q, folder, instructions)): - return - # "Create new": ONLY if truly everything is done (blocks.md AND - # sidecar) → complete fresh start. If blocks.md exists without the sidecar, - # it's a partial state (block B/C open) → resume, don't wipe. - # On an explicit re-run (ab_phase) _reset_ab_phase already handled that. - done = ab_phase is None and ab_step is None and final_path.exists() and _sidecar_schema(_json_file(files["sidecar"])) is not None - if done: - for p_old in _all_slot_files(files): - p_old.unlink(missing_ok=True) - await db.delete_pipeline_state(topic) - await db.delete_blocks(topic) - await db.delete_subblocks(topic) - await db.delete_question_pattern(topic) - await db.delete_coverage(topic) - await db.delete_outline(topic) - await db.delete_sub_artefakte(topic) - - # Inventory (DB): research loop → consolidation → clarification. - if _past_limit("Research"): return - if not await _stage(_research_batch(ctx, set_p, files, q, folder, instructions)): - return - if _past_limit("Consolidation"): return - if not await _stage(_consolidate(ctx, set_p, files)): - return - if _past_limit("Clarification"): return - if not await _stage(_clarify_inventory(ctx, set_p, files)): - return - if _past_limit("Dedup"): return - if not await _stage(_dedup_inventory(ctx, set_p, files)): - return - if _past_limit("Blocks-Filter"): return - if not await _stage(_filter_inventory(ctx, set_p, files)): - return - if _past_limit("Blocks-Gruppierung"): return - if not await _stage(_group_inventory(ctx, set_p, files)): - return - consensus_rows = await db.list_blocks(topic, status="consensus") - entries = { - i: (f"{b['title']} — {b['description']}" if b["description"] else b["title"]) - for i, b in enumerate(consensus_rows, 1) - } - - # Projects only: subject-field supplement — script/project is an excerpt, - # a web agent adds canonically missing blocks, marked with [Supplement]. - if q["type"] == "projekt" and not _past_limit("Supplement"): - set_p("Supplementing subject field…", step=_step_idx(topic, "Supplement")) - supp_path = files["ergaenzung"] - supplements = _supplement_schema(_json_file(supp_path)) - if supplements is None: - supp_path.unlink(missing_ok=True) - status, supplements = await run_single_slot( - ctx, "Supplement", - key=f"blocks-{topic}-ergaenzung-1", - prompt=_prompt( - "Blocks-Supplement", - topic=topic, blocks="\n".join(f"- {t}" for t in entries.values()), - out_path=supp_path, extra=_extra(instructions), - ), - role="quick", capabilities="full", - payload=lambda result: _supplement_schema(_json_file(supp_path)), - timeout=_timeout("ergaenzung"), - ) - if status == CANCELLED: - aborted() - return - if status == FAILED: - _blocks_errors[topic] = "Supplement failed (no valid result)" - return - idx = _title_index(entries) - new = [(t, b) for t, b in supplements if _resolve_title(idx, t) is None] - if new: - _log(topic, f"Supplement: {len(new)} block(s) added from the subject field") - start = max(entries, default=0) + 1 - for off, (t, b) in enumerate(new): - entries[start + off] = f"{t} — {b} [Supplement]" - - # Make titles unique and write the unsorted inventory - entries = _unique_title(entries) - atomic_write_text(final_path, "\n".join(f"{i}. {t}" for i, t in entries.items()) + "\n") - if _past_limit("Subblocks find"): return # end limit inside the inventory → stop with blocks.md written - - # Block B + C: subblocks per block + levels → sidecar subblocks.json. - # Non-destructive: blocks.md already exists; if the sidecar is missing, only - # this part is retried on the next run. The guide falls back without the sidecar. - if _sidecar_schema(_json_file(files["sidecar"])) is None: - raw = _sub_raw_schema(_json_file(files["sub_roh"])) - if raw is None: - raw = await _subblocks_block(ctx, set_p, files, entries, instructions) - if is_cancelled(): - aborted() - return - if raw is None: - return # error is set - atomic_write_json(files["sub_roh"], raw, indent=1) - if _past_limit("Facts find"): return # end limit after subblocks - # Facts per sub (BEFORE the level): extract + verify source facts → facts.json. - # Extract-once grounding — level/relevance/questions/guide feed on it. - if not _facts_complete(files): - res = await _facts_block(ctx, set_p, files, raw, q, folder, instructions) - if is_cancelled(): - aborted() - return - if res is None: - return # error is set - facts_map, discarded = res - # Strike discarded (unsupportable) subs from raw — FIRST (resume-robust), then - # facts.json. This way levels/relevance/outline/questions/guide no longer see them. - if discarded: - for bt, sns in discarded.items(): - if bt in raw: - raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns] - raw = {bt: subs for bt, subs in raw.items() if subs} # drop empty blocks (_sub_roh_schema requires ≥1) - atomic_write_json(files["sub_roh"], raw, indent=1) - atomic_write_json(files["facts"], facts_map, indent=1) - if _past_limit("Levels find"): return # end limit after facts (sidecar not yet valid → no DB mirror) - sidecar = await _levels_block(ctx, set_p, files, raw, instructions) + if not await _prepare_source(ctx, set_p, files, q, folder, instructions): if is_cancelled(): - aborted() - return - if sidecar is None: - return - # Merge facts into the sidecar subs (DB mirror + guide use). - facts_map = _json_file(files["facts"]) - if isinstance(facts_map, dict): - for btitle, subs in sidecar.items(): - fm = facts_map.get(btitle, {}) - for sub in subs: - if (fk := fm.get(_norm_title(sub["title"]))): - sub["facts"] = fk - atomic_write_json(files["sidecar"], sidecar, indent=1) - - # Block D: relevance per subblock (relevant/peripheral) → merge into the sidecar. - # Own phase after the levels; drives the ProGuide format (all blocks - # with ≥1 relevant subblock) and filters peripheral subs out of the guides. - sidecar = _json_file(files["sidecar"]) - if not _past_limit("Relevance find") and _sidecar_schema(sidecar) is not None and not _relevance_complete(sidecar): - relevance_by_id = await _relevance_block(ctx, set_p, files, sidecar, instructions) - if is_cancelled(): - aborted() - return - if relevance_by_id is None: - return # error is set - gid = 0 - for subs in sidecar.values(): - for sub in subs: - gid += 1 - sub["relevance"] = relevance_by_id.get(gid, "relevant") - atomic_write_json(files["sidecar"], sidecar, indent=1) - - # Block D.5: outline (blocks artifact) — chapter structure over ALL blocks, - # only read by the guide. Format-agnostic; the guide filters per format. - if not _past_limit("Outline") and not _outline_complete(files): - await _outline_block(ctx, set_p, files, entries, instructions) - if is_cancelled(): - aborted() - return - - # Block E: question pattern per relevant subblock × type → own sidecar. - # At exam time each agent draws a pattern without replacement and formulates - # a question from it — distinct seeding prevents the duplicate questions of live generation. - sidecar = _json_file(files["sidecar"]) - if not _past_limit("Questions find") and _sidecar_schema(sidecar) is not None and _relevance_complete(sidecar) and not _question_pattern_complete(topic): - pattern = await _question_pattern_block(ctx, set_p, files, sidecar, instructions) - if is_cancelled(): - aborted() - return - if pattern is None: - return # cancel - atomic_write_json(files["question_pattern"], pattern, indent=1) - - # Block F: learning artefacts (flashcards/examples) from the facts — bonus, - # presented by the frontend. Does not abort the run (artefacts are optional). - sidecar = _json_file(files["sidecar"]) - if not _past_limit("Flashcards") and _sidecar_schema(sidecar) is not None and not _artefacts_complete(files): - artefacts = await _artefacts_block(ctx, set_p, files, sidecar, instructions) - if is_cancelled(): - aborted() - return - if artefacts is None: - return # cancel (error/cancel) - - # DB mirror (bridge): write the final sidecar + question-pattern state into the DB. - sidecar = _json_file(files["sidecar"]) - if _sidecar_schema(sidecar) is not None: - await _mirror_sidecar_db(topic, sidecar) - pattern = _json_file(files["question_pattern"]) - if isinstance(pattern, dict) and pattern: - await _mirror_question_pattern_db(topic, pattern) - # Outline (title-based, robust against number drift) → DB. - plan = _json_file(files["outline"]) - if isinstance(plan, dict) and plan.get("chapters"): - kapitel = [ - {"title": ch.get("title", "Chapter"), - "blocks": [_title(entries[n]) for n in ch.get("numbers", []) if n in entries]} - for ch in plan["chapters"] - ] - await db.set_outline(topic, json.dumps({"chapters": kapitel}, ensure_ascii=False)) - # Artefacts → DB (flashcard/example per sub, diagram per block). - artefacts = _json_file(files["artefakte"]) - if isinstance(artefacts, dict) and _sidecar_schema(sidecar) is not None: - await _mirror_artefacts_db(topic, sidecar, artefacts) + _blocks_errors[topic] = "Cancelled — progress is preserved" + return + import board_inventory # lazy: the boards import blocks + ok = await board_inventory.run_boards(ctx, set_p, files, q, folder, instructions, + research=research) + if not ok and is_cancelled(): + _blocks_errors[topic] = "Cancelled — progress is preserved" except Exception as e: log.exception("[%s] Blocks generation failed", topic) _blocks_errors[topic] = str(e)[:2000] diff --git a/backend/board_artefacts.py b/backend/board_artefacts.py new file mode 100644 index 0000000..1fd8bef --- /dev/null +++ b/backend/board_artefacts.py @@ -0,0 +1,333 @@ +"""Board 2 „Artefakte": per finished block, prepare the learning artefacts the guide presents. + +A card is spawned by board 1's `done` column per mirrored block and runs through: + subblocks → facts → levels → relevance → question_pattern → artefacts → finalize +finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte +files + the DB tables. `outline` is a topic-wide BARRIER singleton at the very end +(prerequisite graph → chapter order), re-run once per generation run. + +The heavy lifting is the existing per-block functions in blocks.py — each card gets its own +work subdirectory + facts/artefakte paths, so their slot files never collide across blocks.""" + +import asyncio +import json +import logging +import re + +import database as db +import blocks +from blocks import ( + ARTEFACT_TYPES, _artefacts_block, _facts_block, _levels_block, _match_sub, + _question_pattern_block, _relevance_block, _subblocks_block, _outline_block, +) +from fsutil import atomic_write_json +from jsonio import read_json_file as _json_file +from kanban import Flow, Stage +from pipeline import GenContext, _log +from textkit import _norm_title, _title + +log = logging.getLogger("creator.board_artefacts") + +BOARD = "artefacts" +DONE = "done_artefact" + + +def _nset(msg: str, step: int | None = None) -> None: + """Progress no-op — the kanban board itself is the progress display.""" + + +def _safe(norm: str) -> str: + return re.sub(r"\W+", "-", norm).strip("-")[:24] or "block" + + +def _card_set_p(flow: Flow, norm: str): + """Per-card progress: the inner step messages land in-memory on the flow — + board_snapshot shows them as the card's info line while it is active.""" + info = flow.state.setdefault("card_info", {}) + + def set_p(msg: str, step: int | None = None) -> None: + info[f"{BOARD}:{norm}"] = msg + return set_p + + +def _pfiles(files: dict, norm: str) -> dict: + """Per-block file namespace: own work dir + facts/artefakte paths, global rest.""" + sub = files["arbeit"] / f"ab-{_norm_title(norm).replace(' ', '_')[:60]}" + sub.mkdir(parents=True, exist_ok=True) + return {**files, "arbeit": sub, "facts": sub / "facts.json", "artefakte": sub / "artefakte.json"} + + +def _entry_line(p: dict) -> str: + d = p.get("description") + return f"{p['title']} — {d}" if d else p["title"] + + +def make_spawner(topic: str, files: dict): + """Hook for board 1's `done` column: one artefact card per mirrored block.""" + + async def spawn(block_card_id: str, payload: dict): + norm = payload.get("mirrored_norm") + if not norm: + return + await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "subblocks", { + "title": payload.get("title", ""), + "description": payload.get("description", ""), + }) + return spawn + + +async def _gather_cards(ctx: GenContext, flow: Flow, cards, one): + results = await asyncio.gather(*[one(c) for c in cards], return_exceptions=True) + errs = [r for r in results if isinstance(r, Exception)] + if errs: + raise errs[0] + flow.wake.set() + + +def _fail_or_cancel(ctx: GenContext, what: str): + # A per-card failure belongs on the card (last_error/dead-letter), never in the + # topic banner — the inner block functions may have set it there. + blocks._blocks_errors.pop(ctx.topic, None) + if ctx.is_cancelled(): + return None # leave the card where it is + raise RuntimeError(f"{what} ohne Ergebnis") + + +# ── Stage processors (one call per card, all parallel) ───────────────────────────── +async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): + topic = flow.topic + # Fix 4: fragments demoted to a parent become seed candidates of the parent's subblocks. + seeds: dict[str, list[str]] = {} + for r in await db.kanban_cards(topic, board="inventory", stage="rejected"): + pn = r["payload"].get("parent_norm") + if pn: + seeds.setdefault(pn, []).append(r["payload"].get("title", "")) + + async def one(c): + p = c["payload"] + norm = c["card_id"] + instr = instructions + if (sd := [s for s in seeds.get(norm, []) if s]): + instr = (instructions + "\n\nBereits identifizierte Unterpunkt-Kandidaten dieses " + "Blocks (unbedingt prüfen und, wenn belegt, aufnehmen):\n" + + "\n".join(f"- {s}" for s in sd)) + raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), + {1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-") + if raw is None: + return _fail_or_cancel(ctx, f"Subblocks {p.get('title', norm)}") + p["raw"] = raw + await db.kanban_set_payload(topic, BOARD, norm, p) + await db.kanban_advance(topic, BOARD, norm, "facts") + + await _gather_cards(ctx, flow, cards, one) + + +async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder, + instructions: str, cards): + topic = flow.topic + + async def one(c): + p = c["payload"] + norm = c["card_id"] + raw = p.get("raw") or {} + res = await _facts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), raw, q, + folder, instructions, ns=f"{_safe(norm)}-") + if res is None: + return _fail_or_cancel(ctx, f"Facts {p.get('title', norm)}") + facts_map, discarded = res + if discarded: # unsupportable subs vanish from raw too (guide never sees them) + for bt, sns in discarded.items(): + if bt in raw: + raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns] + raw = {bt: subs for bt, subs in raw.items() if subs} + p["raw"], p["facts"] = raw, facts_map + await db.kanban_set_payload(topic, BOARD, norm, p) + await db.kanban_advance(topic, BOARD, norm, "levels") + + await _gather_cards(ctx, flow, cards, one) + + +async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): + topic = flow.topic + + async def one(c): + p = c["payload"] + norm = c["card_id"] + sidecar = await _levels_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), + p.get("raw") or {}, instructions, ns=f"{_safe(norm)}-") + if sidecar is None: + return _fail_or_cancel(ctx, f"Levels {p.get('title', norm)}") + facts_map = p.get("facts") or {} + for btitle, subs in sidecar.items(): # merge facts into the subs (mirror + guide) + fm = facts_map.get(btitle, {}) + for sub in subs: + if (fk := fm.get(_norm_title(sub["title"]))): + sub["facts"] = fk + p["sidecar"] = sidecar + await db.kanban_set_payload(topic, BOARD, norm, p) + await db.kanban_advance(topic, BOARD, norm, "relevance") + + await _gather_cards(ctx, flow, cards, one) + + +async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): + topic = flow.topic + + async def one(c): + p = c["payload"] + norm = c["card_id"] + sidecar = p.get("sidecar") or {} + rel = await _relevance_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), + sidecar, instructions, ns=f"{_safe(norm)}-") + if rel is None: + return _fail_or_cancel(ctx, f"Relevance {p.get('title', norm)}") + gid = 0 + for subs in sidecar.values(): + for sub in subs: + gid += 1 + sub["relevance"] = rel.get(gid, "relevant") + p["sidecar"] = sidecar + await db.kanban_set_payload(topic, BOARD, norm, p) + await db.kanban_advance(topic, BOARD, norm, "question_pattern") + + await _gather_cards(ctx, flow, cards, one) + + +async def _proc_question_pattern(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): + topic = flow.topic + + async def one(c): + p = c["payload"] + norm = c["card_id"] + pattern = await _question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), + p.get("sidecar") or {}, instructions, + ns=f"{_safe(norm)}-") + if pattern is None: + return _fail_or_cancel(ctx, f"Fragen {p.get('title', norm)}") + p["pattern"] = pattern + await db.kanban_set_payload(topic, BOARD, norm, p) + await db.kanban_advance(topic, BOARD, norm, "artefacts") + + await _gather_cards(ctx, flow, cards, one) + + +async def _proc_artefacts(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): + topic = flow.topic + + async def one(c): + p = c["payload"] + norm = c["card_id"] + artefacts = await _artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), + p.get("sidecar") or {}, instructions, + ns=f"{_safe(norm)}-") + if artefacts is None and ctx.is_cancelled(): + return None + p["artefacts"] = artefacts or {} # artefacts are optional — never fatal + await db.kanban_set_payload(topic, BOARD, norm, p) + await db.kanban_advance(topic, BOARD, norm, "finalize") + + await _gather_cards(ctx, flow, cards, one) + + +# ── Finalize (SERIAL): merge into the global files + DB tables ───────────────────── +def _merge_json(path, block_keys: dict) -> None: + data = _json_file(path) + if not isinstance(data, dict): + data = {} + data.update(block_keys) + atomic_write_json(path, data, indent=1) + + +async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards): + topic = flow.topic + for c in cards: + p = c["payload"] + title = p.get("title", "") + sidecar = p.get("sidecar") or {} + pattern = p.get("pattern") or {} + artefacts = p.get("artefacts") or {} + # global sidecar files (the legacy read path of guide/frontend/resume) + _merge_json(files["sub_roh"], {t: subs for t, subs in (p.get("raw") or {}).items()}) + _merge_json(files["facts"], p.get("facts") or {}) + _merge_json(files["sidecar"], sidecar) + _merge_json(files["question_pattern"], pattern) + art_global = _json_file(files["artefakte"]) + if not isinstance(art_global, dict): + art_global = {} + for typ in ARTEFACT_TYPES: + kept = [e for e in art_global.get(typ, []) + if _norm_title(_title(str(e.get("block", "")))) != _norm_title(title)] + art_global[typ] = kept + list(artefacts.get(typ, [])) + atomic_write_json(files["artefakte"], art_global, indent=1) + # DB mirrors — per block only (no global deletes) + await blocks._mirror_sidecar_db(topic, sidecar) + for btitle, entries in pattern.items(): + bnorm = _norm_title(btitle) + for e in entries if isinstance(entries, list) else []: + sub = str(e.get("subblock", "")).strip() + sn = _norm_title(sub) + question = str(e.get("question", "")).strip() + if bnorm and sn and question: + await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question) + btitles = list(sidecar.keys()) + for typ in ARTEFACT_TYPES: + for e in artefacts.get(typ, []): + bt = _match_sub(str(e.get("block", "")), btitles) + bnorm, sn = _norm_title(bt), _norm_title(str(e.get("subblock", ""))) + if not bnorm or not sn: + continue + data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")}, + ensure_ascii=False) + await db.put_sub_artifact(topic, bnorm, sn, typ, data, bt, str(e.get("subblock", ""))) + await db.kanban_advance(topic, BOARD, c["card_id"], DONE) + _log(topic, f"Artefakte fertig: {title}") + flow.wake.set() + + +# ── Outline (topic-wide barrier singleton) ───────────────────────────────────────── +OUTLINE_CARD = "outline" + + +async def ensure_outline_card(topic: str) -> None: + """(Re-)queue the outline singleton — run once per generation run, after everything.""" + await db.kanban_upsert_card(topic, BOARD, OUTLINE_CARD, "outline", "outline", + {"title": "Gliederung"}) + + +async def _proc_outline(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards): + topic = flow.topic + done = await db.kanban_cards(topic, board="inventory", stage="done_block") + done.sort(key=lambda c: c["updated_at"]) + entries = {i: _entry_line(c["payload"]) for i, c in enumerate(done, 1) + if c["payload"].get("title")} + if entries: + plan = await _outline_block(ctx, _nset, files, entries, instructions) + if ctx.is_cancelled(): + return + if isinstance(plan, dict) and plan.get("chapters"): + chapters = [ + {"title": ch.get("title", "Kapitel"), + "blocks": [_title(entries[n]) for n in ch.get("numbers", []) if n in entries]} + for ch in plan["chapters"] + ] + await db.set_outline(topic, json.dumps({"chapters": chapters}, ensure_ascii=False)) + await db.kanban_advance_many(topic, BOARD, [(c["card_id"], DONE) for c in cards]) + flow.wake.set() + + +# ── Stage list (appended after board 1 in chain order) ───────────────────────────── +def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder, + instructions: str) -> list[Stage]: + research_done = lambda: flow.research_done # noqa: E731 + return [ + Stage(BOARD, "subblocks", lambda cs: _proc_subblocks(ctx, flow, files, instructions, cs)), + Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)), + Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)), + Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)), + Stage(BOARD, "question_pattern", + lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)), + Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)), + Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True), + Stage(BOARD, "outline", lambda cs: _proc_outline(ctx, flow, files, instructions, cs), + barrier=True, drain=True, gate=research_done), + ] diff --git a/backend/board_inventory.py b/backend/board_inventory.py new file mode 100644 index 0000000..5fa93eb --- /dev/null +++ b/backend/board_inventory.py @@ -0,0 +1,1395 @@ +"""Board 1 „Inventar": streaming kanban stages for the block inventory. + +Research producers stream candidate titles into the board; the columns pull, filter, +merge and reject them until an optimized block inventory remains. The mature filter +logic (relation guard, complete-link cliques, degrade pass, umbrella grouping, gates) +is reused from blocks.py — this module only re-orchestrates it as streaming stages. + +Stages (cards): + ingest title exact dedup folded at DB-add (reader union) + cluster title SERIAL — online embedding nearest-neighbour clustering + pair_check cluster judge per candidate pair (relation guard, cliques) → splits + consensus_gate cluster code: reader union ≥2 (or supplement) → naming, else clarify + clarify cluster 3-judge panel, unanimity for single-reader finds + naming cluster judge picks the canonical member title + naming_check cluster second judge verifies → spawns the block card + fragment_filter block BARRIER/drain: global re-merge + degrade pass (full list) + grouping block BARRIER/drain: umbrella grouping (type gate, reconcile) + gap_check block BARRIER/drain: one supplement round (web) → feeds ingest + done block mirror into the legacy `blocks` table → done_block + +Terminal: clustered (titles), done_cluster, grouped, rejected (with journal), done_block. +""" + +import asyncio +import hashlib +import json +import logging +import math +import uuid + +import database as db +import embedding +import kanban +from kanban import Flow, Stage, chain_stages +import blocks +from blocks import ( + DEDUP_PAIR_FLOOR, DEDUP_PAIRS_CHUNK, DEDUP_TITLE_AUTO, FILTER_CHUNK, + FILTER_RECHECK_PANEL, CONSOLIDATION_PANEL, RESEARCH_BATCH, RESEARCH_READERS, + RESEARCH_THEMA_AGENTS, _FILTER_NOTATION, _GROUP_STANDALONE, + _build_research_prompt, _canonical, _canonical_key, _chunk_nums, _cliques, + _completion_schema, _containment_parent, _crawl_index, _file_payload, + _filter_schema, _filter_suspect, _is_artifact, _is_named_statement, + _is_parentless_noise, _is_reference, _pairs_schema, _read, + _relation_conflict, _root, _supplement_schema, _text_sections, _umbrella_schema, + _aspect_marker, +) +from config import ( + BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_AKTIV, EMBEDDING_BLOCK_CAP, + EMBEDDING_SIBLING_CAP, EMBEDDING_SIBLING_FLOOR, GROUP_MIN_COS_FLOOR, + GROUP_RECONCILE_FLOOR, +) +from fsutil import atomic_write_json, atomic_write_text +from jsonio import read_json_file as _json_file +from pipeline import (CANCELLED, FAILED, OK, GenContext, _extra, _log, _prompt, + _runde_schema, _timeout, run_single_slot) +from textkit import _norm_title, _parse_selection, _title + +log = logging.getLogger("creator.board_inventory") + +BOARD = "inventory" +RESEARCH_RUNTIME = 900 # one research agent, one round — the tail ingests live while it writes +_POLL_RESEARCH = 3 # seconds between live reads of a running research file + +_ingest_lock = asyncio.Lock() # serializes the read-modify-write title upserts + + +def _h(*parts: str) -> str: + """Short stable hash for slot filenames — a rework with different members must not + reuse a stale judge file (indices would no longer match).""" + return hashlib.sha1("|".join(parts).encode("utf-8")).hexdigest()[:8] + + +def _t_text(p: dict) -> str: + return f"{p['title']} — {p['description']}" if p.get("description") else p["title"] + + +def _line(i: int, p: dict, mark: str = "") -> str: + d = p.get("description") + return f"{i}. {mark}{p['title']} — {d}" if d else f"{i}. {mark}{p['title']}" + + +def _naming_schema(data, count: int) -> int | None: + """{"best": N} → 1-based member index in [1, count] · otherwise None.""" + if not isinstance(data, dict): + return None + try: + n = int(data.get("best")) + except (ValueError, TypeError): + return None + return n if 1 <= n <= count else None + + +# ── Embedding cache (per flow) ───────────────────────────────────────────────────── +async def _emb_ok(flow: Flow) -> bool: + if "emb_ok" not in flow.state: + flow.state["emb_ok"] = EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available) + return flow.state["emb_ok"] + + +async def _vec_rows(flow: Flow, texts: list[str]): + """L2-normalized vectors for `texts`, cached per flow (a title is embedded once, + not once per stage). → (n, d) ndarray | None when the model is off.""" + if not texts or not await _emb_ok(flow): + return None + cache = flow.state.setdefault("vecs", {}) + missing = list(dict.fromkeys(t for t in texts if t not in cache)) + if missing: + arr = await asyncio.to_thread(embedding.embed, missing) + if arr is None: + return None + for t, v in zip(missing, arr): + cache[t] = v + import numpy as np + return np.vstack([cache[t] for t in texts]) + + +# ── Research producers ───────────────────────────────────────────────────────────── +def _extract_text(raw_line: str) -> str: + """Best-effort: pull assistant/tool text out of ONE opencode `--format json` event line. + Recursively collects every `text`/`content` string — robust to the exact event schema.""" + try: + obj = json.loads(raw_line) + except Exception: + return "" + parts: list[str] = [] + + def _walk(o): + if isinstance(o, dict): + for k, v in o.items(): + if k in ("text", "content") and isinstance(v, str): + parts.append(v) + else: + _walk(v) + elif isinstance(o, list): + for v in o: + _walk(v) + _walk(obj) + return "".join(parts) + + +async def _ingest_titles(flow: Flow, text: str, reader: str, source: str = "") -> int: + """Parse a reader text into title cards (stage 'ingest'). Exact dupes fold via + reader union. Repeated drains over a growing buffer are idempotent. → new count.""" + n, seen = 0, set() + for record in _parse_selection(text).values(): + title = _title(record) + norm = _norm_title(title) + if not norm or norm in seen: + continue + seen.add(norm) # one reader = one vote per concept + parts = [t.strip() for t in record.split(" — ")] + desc = parts[1] if len(parts) >= 2 else "" + src = source or (parts[2] if len(parts) >= 3 else "") + async with _ingest_lock: + if await db.kanban_add_title(flow.topic, BOARD, norm, title, desc, src, reader): + n += 1 + if n: + flow.wake.set() + return n + + +async def _research_once(ctx: GenContext, flow: Flow, q: dict, folder, instructions: str, + tag: str, *, section: str = "", fokus: str = "", source_file: str = ""): + """ONE agent searches; its titles stream into the ingest queue LIVE. Two sources feed + the ingest: the JSON event stream (on_line → text buffer) AND the file the agent writes — + whichever the agent uses, cards stream in immediately (not only after it finishes).""" + work_dir = flow.work_dir + caps = "files" if folder else "full" + p = work_dir / f"research-{tag}.md" + stop = asyncio.Event() + buf: list[str] = [] # assistant text streamed live from the JSON events + + def _on_line(raw: str): # sync, called per stdout line by the agent runner + if (t := _extract_text(raw)): + buf.append(t) + + async def _drain() -> bool: # ingest from BOTH event buffer and file (idempotent) + text = "".join(buf) + if (ft := _file_payload(p)): + text += "\n" + ft + return bool(text) and await _ingest_titles(flow, text, tag, source_file) + + async def _tail(): # live-ingest loop while the agent runs + while not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), timeout=_POLL_RESEARCH) + except asyncio.TimeoutError: + pass + await _drain() + + if not _file_payload(p): # resume: a valid reader file is re-ingested without an agent + p.unlink(missing_ok=True) + tail = asyncio.create_task(_tail()) + try: + await run_single_slot( + ctx, f"Research {tag}", key=f"blocks-{ctx.topic}-research-{tag}", + prompt=_build_research_prompt(ctx.topic, p, instructions, q["type"], folder, + fokus=fokus, section=section), + role="quick", capabilities=caps, + payload=(lambda result, p=p: _file_payload(p)), + timeout=RESEARCH_RUNTIME, on_line=_on_line, + ) + finally: + stop.set() + await tail + await _drain() # final catch-up (also the whole resume path) + _log(ctx.topic, f"Research {tag}: Titel → ingest") + + +def _build_producers(ctx: GenContext, flow: Flow, q: dict, folder, instructions: str) -> list: + """Producer coroutines per source mode. thema = N web agents; uni/projekt = 2 readers per + text section; link = 2 readers per fixed page batch (coverage ticked per batch).""" + topic = ctx.topic + + if not folder: # thema: free web research + return [_research_once(ctx, flow, q, folder, instructions, str(i)) + for i in range(1, RESEARCH_THEMA_AGENTS + 1)] + + if q["type"] in ("uni", "projekt"): + sections: list[tuple[str, str]] = [] + for fn in sorted(set(_crawl_index(folder).values()) or + [f.name for f in sorted(folder.glob("**/*.txt"))]): + for sec in _text_sections(_read(folder / fn)): + sections.append((fn, sec)) + prods = [] + for ei, (fn, sec) in enumerate(sections, 1): + block = (f"ARBEITE AUSSCHLIESSLICH MIT DIESEM TEXTABSCHNITT. Lies ihn VOLLSTÄNDIG, " + f"überspringe nichts. Suche NICHT im Web — nur diese Section zählt." + f"\n\n-----\n{sec}\n-----") + prods += [_research_once(ctx, flow, q, folder, instructions, f"a{ei}-{r}", + section=block, source_file=fn) + for r in range(1, RESEARCH_READERS + 1)] + return prods + + # link/crawl: fixed page batches, RESEARCH_READERS readers each + async def _batch(bi: int, batch: list[str]): + liste = "\n".join(f"- {p}" for p in batch) + fokus = ("WICHTIG — feste Assignment: Bearbeite AUSSCHLIESSLICH diese Dateien und lies JEDE " + f"vollständig. Ignoriere alle anderen Dateien im Ordner:\n{liste}") + try: + await asyncio.gather(*[ + _research_once(ctx, flow, q, folder, instructions, f"b{bi}-{r}", fokus=fokus) + for r in range(1, RESEARCH_READERS + 1)], return_exceptions=True) + finally: + await db.mark_sources_read_done(topic, batch) # tick coverage even without hits + + return [_batch(bi, b) for bi, b in enumerate( + _chunk_nums(sorted(flow.state["pages"]), + max(1, math.ceil(len(flow.state["pages"]) / RESEARCH_BATCH))), 1)] + + +# ── Column processors ────────────────────────────────────────────────────────────── +async def _proc_ingest(flow: Flow, cards): + """Visibility column — exact dedup already folded at DB-add. Just advance.""" + await db.kanban_advance_many(flow.topic, BOARD, [(c["card_id"], "cluster") for c in cards]) + flow.wake.set() + + +async def _proc_cluster(ctx: GenContext, flow: Flow, cards): + """SERIAL online clustering: nearest neighbour over ALL known titles (including the + batch mates registered a moment ago — fixes main's same-batch blindness). Join the + neighbour's cluster (cos ≥ floor, cap 25) or open a new one; the touched cluster is + reworked from pair_check.""" + topic = flow.topic + emb = await _emb_ok(flow) + membership = await db.kanban_membership(topic) + sizes: dict[str, int] = {} + for g in membership.values(): + sizes[g] = sizes.get(g, 0) + 1 + nn = flow.state.setdefault("nn", {"norms": [], "texts": []}) + moves = [] + for c in cards: + nm, p = c["card_id"], c["payload"] + text = _t_text(p) + target = None + if emb and nn["norms"]: + mat = await _vec_rows(flow, nn["texts"] + [text]) + if mat is not None: + import numpy as np + cos = mat[:-1] @ mat[-1] + best = int(np.argmax(cos)) + if float(cos[best]) >= DEDUP_PAIR_FLOOR: + cand = membership.get(nn["norms"][best]) + if cand and sizes.get(cand, 0) < EMBEDDING_BLOCK_CAP: + target = cand + cid = target or f"cl-{uuid.uuid4().hex[:12]}" + membership[nm] = cid + sizes[cid] = sizes.get(cid, 0) + 1 + await db.kanban_set_member(topic, nm, cid) + # gaining a member reworks the cluster from pair_check (live re-clustering) + await db.kanban_upsert_card(topic, BOARD, cid, "cluster", "pair_check", + {"title": p["title"], "description": p.get("description", "")}) + nn["norms"].append(nm) + nn["texts"].append(text) + moves.append((nm, "clustered")) + await db.kanban_advance_many(topic, BOARD, moves) + flow.wake.set() + + +async def _member_rows(topic: str, cid: str) -> list[dict]: + rows = [] + for nm in await db.kanban_members_of(topic, cid): + tc = await db.kanban_get_card(topic, BOARD, nm) + if tc: + rows.append({"norm": nm, **tc["payload"]}) + return rows + + +async def _proc_pair_check(ctx: GenContext, flow: Flow, cards): + results = await asyncio.gather(*[_pair_one(ctx, flow, c) for c in cards], return_exceptions=True) + errs = [r for r in results if isinstance(r, Exception)] + if errs: + raise errs[0] # unadvanced cards go to backoff (engine) + flow.wake.set() + + +async def _pair_one(ctx: GenContext, flow: Flow, c): + """Pairwise entity resolution inside one cluster: candidate pairs = mean(title, title+desc) + cosine ≥ floor ∪ canonical-key blocking; a judge confirms each pair; auto-merges (title ≥0.95, + exact key); relation guard blocks different reductions; complete-link cliques merge — the + rest splits into fresh clusters. All resulting clusters → consensus_gate.""" + topic = flow.topic + cid = c["card_id"] + rows = await _member_rows(topic, cid) + if len(rows) <= 1: + await db.kanban_advance(topic, BOARD, cid, "consensus_gate") + return + n = len(rows) + texts_full = [_t_text(r) for r in rows] + texts_title = [r["title"] for r in rows] + vf = await _vec_rows(flow, texts_full) + vt = await _vec_rows(flow, texts_title) + sims_title = vt @ vt.T if vt is not None else None + pairs: set[tuple[int, int]] = set() + if vf is not None and vt is not None: + sims = (vf @ vf.T + sims_title) / 2 + pairs |= {(i, j) for i in range(n) for j in range(i + 1, n) + if float(sims[i][j]) >= DEDUP_PAIR_FLOOR} + else: # no model → judge all pairs (clusters are small) + pairs |= {(i, j) for i in range(n) for j in range(i + 1, n)} + key_groups: dict[str, list[int]] = {} + for i, r in enumerate(rows): + if (k := _canonical_key(r["title"])): + key_groups.setdefault(k, []).append(i) + for grp in key_groups.values(): + for x in range(len(grp)): + for y in range(x + 1, len(grp)): + pairs.add((grp[x], grp[y])) + ordered = sorted(pairs) + edges: list[tuple[int, int]] = [] + chunks = [ordered[k:k + DEDUP_PAIRS_CHUNK] for k in range(0, len(ordered), DEDUP_PAIRS_CHUNK)] + h = _h(*[r["norm"] for r in rows]) + + async def _judge_pairs(ci, chunk): # chunks are independent — one parallel wave + path = flow.work_dir / f"pair-{cid}-{h}-c{ci}.json" + if _pairs_schema(_json_file(path)) is not None: + return # resume + lines = "\n\n".join( + f"{j + 1}.\nA: {_t_text(rows[a])}\nB: {_t_text(rows[b])}" + for j, (a, b) in enumerate(chunk)) + status, _v = await run_single_slot( + ctx, f"Paar-Check {cid}", key=f"blocks-{topic}-pair-{cid}-{h}-c{ci}", + prompt=_prompt("Blocks-Paar-Filter", topic=topic, pairs=lines, out_path=path), + role="judge", capabilities="files", + payload=lambda result, p=path: _pairs_schema(_json_file(p)), + timeout=_timeout("selection_mapping", len(chunk))) + if status == FAILED: + raise RuntimeError(f"Paar-Filter {cid} chunk {ci} ohne Ergebnis") + + results = await asyncio.gather(*[_judge_pairs(ci, c) for ci, c in enumerate(chunks)], + return_exceptions=True) + errs = [r for r in results if isinstance(r, Exception)] + if errs: + raise errs[0] + if ctx.is_cancelled(): + return + for ci, chunk in enumerate(chunks): + verdict = _pairs_schema(_json_file(flow.work_dir / f"pair-{cid}-{h}-c{ci}.json")) or {} + for j, (a, b) in enumerate(chunk): + if verdict.get(j + 1) and not _relation_conflict(rows[a]["title"], rows[b]["title"]): + edges.append((a, b)) + if sims_title is not None: # auto recall net: near-identical titles merge without the judge + for a, b in ordered: + if float(sims_title[a][b]) >= DEDUP_TITLE_AUTO and not _relation_conflict( + rows[a]["title"], rows[b]["title"]): + edges.append((a, b)) + for grp in key_groups.values(): # exact-canonical-key auto-merge (ER ~100% precision) + for x in range(len(grp)): + for y in range(x + 1, len(grp)): + a, b = grp[x], grp[y] + if not _relation_conflict(rows[a]["title"], rows[b]["title"]): + edges.append((a, b)) + groups = _cliques(n, edges) + used = set().union(*[set(g) for g in groups]) if groups else set() + groups += [[i] for i in range(n) if i not in used] + groups.sort(key=len, reverse=True) + for gi, g in enumerate(groups): + gid = cid if gi == 0 else f"cl-{uuid.uuid4().hex[:12]}" + await db.kanban_set_members(topic, gid, [rows[i]["norm"] for i in g]) + rep = rows[max(g, key=lambda k: len(rows[k].get("description") or ""))] + await db.kanban_upsert_card(topic, BOARD, gid, "cluster", "consensus_gate", + {"title": rep["title"], "description": rep.get("description", "")}) + + +def _rep(rows: list[dict]) -> dict: + """Cluster representative via the survivorship cascade (main concept, not an aspect).""" + cands = [{"title": r["title"], "description": r.get("description") or "", + "reader": r.get("readers") or []} for r in rows] + return _canonical(cands, list(range(len(cands))), set()) + + +async def _proc_consensus_gate(ctx: GenContext, flow: Flow, cards): + """Code gate: reader union ≥2 (or supplement) passes; single finds → clarify. + Reference-titled consensus clusters also go to clarify (majority quorum + rename).""" + topic = flow.topic + moves = [] + for c in cards: + cid = c["card_id"] + rows = await _member_rows(topic, cid) + if not rows: + moves.append((cid, "done_cluster")) + continue + readers = set().union(*[set(r.get("readers") or []) for r in rows]) + supplement = any(r.get("supplement") for r in rows) + rep = _rep(rows) + p = c["payload"] + p.update(title=rep["title"], description=rep["description"], readers=sorted(readers), + supplement=supplement) + if supplement or len(readers) >= 2: + if _is_reference(rep["title"]) and not supplement: + p["quorum"] = "majority" # consensus reference title: rename/exam, not the hard bar + moves.append((cid, "clarify")) + else: + moves.append((cid, "naming")) + else: + moves.append((cid, "clarify")) + await db.kanban_set_payload(topic, BOARD, cid, p) + await db.kanban_advance_many(topic, BOARD, moves) + flow.wake.set() + + +async def _proc_clarify(ctx: GenContext, flow: Flow, cards): + """Single-reader finds: deterministic pre-reject, then a 3-judge panel (Blocks-Klaerung). + Quorum: UNANIMITY for single-reader clusters (origin-split C1), majority for reference-titled + consensus clusters. Kept reference titles get the panel's rename.""" + topic = flow.topic + moves: list[tuple[str, str]] = [] + pending = [] + for c in cards: + t = c["payload"].get("title", "") + d = c["payload"].get("description", "") + if (_is_artifact(t) or _FILTER_NOTATION.search(t)) and not _is_named_statement(t, d): + c["payload"]["reason"] = "pre-reject" + await db.kanban_set_payload(topic, BOARD, c["card_id"], c["payload"]) + moves.append((c["card_id"], "rejected")) + else: + pending.append(c) + if pending: + h = _h(*[c["card_id"] for c in pending]) + rest = "\n".join(f"- {_t_text(c['payload'])}" for c in pending) + judges = [(j, flow.work_dir / f"clarify-{h}-j{j}.json") + for j in range(1, CONSOLIDATION_PANEL + 1)] + + async def _judge(j, path): + if _runde_schema(_json_file(path)) is not None: + return # resume: keep a valid judge file + await run_single_slot( + ctx, f"Klärung j{j}", key=f"blocks-{topic}-clarify-{h}-j{j}", + prompt=_prompt("Blocks-Klaerung", topic=topic, rest=rest, + final="\n- Entscheide JEDEN Eintrag. `rest` MUSS leer sein.", + out_path=path), + role="judge", capabilities="files", + payload=lambda result, p=path: _runde_schema(_json_file(p)), + timeout=_timeout("selection_mapping", len(pending))) + + await asyncio.gather(*[_judge(j, p) for j, p in judges], return_exceptions=True) + if ctx.is_cancelled(): + return + outs, raws = [], [] + for _, path in judges: + raw = _json_file(path) + r = _runde_schema(raw) + if r is not None: + outs.append(r) + raws.append(raw) + if not outs: + raise RuntimeError("Klärung: kein Judge lieferte ein Ergebnis") + votes: dict[str, int] = {} + for accepted, _ in outs: + for nt in {_norm_title(_title(t)) for t in accepted}: + votes[nt] = votes.get(nt, 0) + 1 + renames: dict[str, dict[str, int]] = {} + for raw in raws: + if isinstance(raw, dict) and isinstance(raw.get("rename"), dict): + for old, new in raw["rename"].items(): + new = str(new).strip() + if new: + renames.setdefault(_norm_title(str(old)), {}).setdefault(new, 0) + renames[_norm_title(str(old))][new] += 1 + for c in pending: + p = c["payload"] + norm = _norm_title(p.get("title", "")) + v = votes.get(norm, 0) + majority = p.get("quorum") == "majority" + accept = (v * 2 >= len(outs)) if majority else (v >= len(outs)) + if not accept: + p.update(reason="failed-quorum", votes=v, judges=len(outs)) + await db.kanban_set_payload(topic, BOARD, c["card_id"], p) + moves.append((c["card_id"], "rejected")) + continue + if _is_reference(p.get("title", "")) and (sug := renames.get(norm)): + best = max(sug, key=lambda k: (sug[k], len(k))) + if not _is_reference(best): + p.update(title=best, renamed=True) + await db.kanban_set_payload(topic, BOARD, c["card_id"], p) + moves.append((c["card_id"], "naming")) + await db.kanban_advance_many(topic, BOARD, moves) + flow.wake.set() + + +async def _choose_title(ctx: GenContext, flow: Flow, cid: str, rows: list[dict], + template: str, current: int | None = None) -> str: + """Judge picks the best member title (index). Parse-fail → survivorship fallback.""" + topic = flow.topic + h = _h(*[r["norm"] for r in rows], template) + path = flow.work_dir / f"naming-{cid}-{h}.json" + best = _naming_schema(_json_file(path), len(rows)) + if best is None: + lines = "\n".join(f"{k + 1}. {_t_text(r)}" for k, r in enumerate(rows)) + kw = dict(topic=topic, members=lines, out_path=path) + if current is not None: + kw["current"] = current + status, best = await run_single_slot( + ctx, f"Naming {cid}", key=f"blocks-{topic}-naming-{cid}-{h}", + prompt=_prompt(template, **kw), role="judge", capabilities="files", + payload=lambda result, p=path, n=len(rows): _naming_schema(_json_file(p), n), + timeout=_timeout("selection_mapping", len(rows))) + if status == CANCELLED: + return "" + if status == FAILED: + best = None + if best is None: + rep = _rep(rows) + w = _norm_title(rep["title"]) + norms = [r["norm"] for r in rows] + return w if w in norms else norms[0] + return rows[best - 1]["norm"] + + +async def _proc_naming(ctx: GenContext, flow: Flow, cards): + results = await asyncio.gather(*[_name_one(ctx, flow, c) for c in cards], return_exceptions=True) + errs = [r for r in results if isinstance(r, Exception)] + if errs: + raise errs[0] + flow.wake.set() + + +async def _name_one(ctx: GenContext, flow: Flow, c): + topic = flow.topic + cid = c["card_id"] + p = c["payload"] + rows = await _member_rows(topic, cid) + if len(rows) > 1 and not p.get("renamed"): + winner = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming") + if not winner: # cancelled + return + p["main_norm"] = winner + w = next(r for r in rows if r["norm"] == winner) + p["title"], p["description"] = w["title"], w.get("description") or p.get("description", "") + await db.kanban_set_payload(topic, BOARD, cid, p) + await db.kanban_advance(topic, BOARD, cid, "naming_check") + + +async def _proc_naming_check(ctx: GenContext, flow: Flow, cards): + results = await asyncio.gather(*[_namecheck_one(ctx, flow, c) for c in cards], return_exceptions=True) + errs = [r for r in results if isinstance(r, Exception)] + if errs: + raise errs[0] + flow.wake.set() + + +async def _namecheck_one(ctx: GenContext, flow: Flow, c): + """Second judge verifies the title choice, then spawns the block card.""" + topic = flow.topic + cid = c["card_id"] + p = c["payload"] + rows = await _member_rows(topic, cid) + if len(rows) > 1 and not p.get("renamed"): + norms = [r["norm"] for r in rows] + cur = p.get("main_norm") + current = norms.index(cur) + 1 if cur in norms else 1 + winner = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming-Check", current=current) + if not winner: + return + w = next(r for r in rows if r["norm"] == winner) + p["title"], p["description"] = w["title"], w.get("description") or p.get("description", "") + readers = sorted(set().union(*[set(r.get("readers") or []) for r in rows])) if rows else [] + sources = sorted(set().union(*[set(r.get("sources") or []) for r in rows])) if rows else [] + await db.kanban_upsert_card(topic, BOARD, f"b-{cid}", "block", "fragment_filter", { + "title": p.get("title", ""), "description": p.get("description", ""), + "readers": readers, "sources": sources, + "supplement": bool(p.get("supplement")), "cluster": cid, + }) + await db.kanban_advance(topic, BOARD, cid, "done_cluster") + + +# ── Block barriers ───────────────────────────────────────────────────────────────── +async def _context_blocks(topic: str, exclude: set[str]) -> list[dict]: + """Blocks already past the filter (grouping/gap_check/done_block) — context for the + relational judges of a LATER pass (supplement feedback), never demotable themselves.""" + out = [] + for r in await db.kanban_cards(topic, board=BOARD, kind="block"): + if r["card_id"] in exclude or r["stage"] not in ("grouping", "gap_check", "done", "done_block"): + continue + p = r["payload"] + out.append({"card_id": r["card_id"], "payload": p, + "title": p.get("title", ""), "description": p.get("description") or "", + "title_norm": _norm_title(p.get("title", ""))}) + return out + + +async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards): + """BARRIER/drain — the global pass over ALL block cards in the stage: + (a) deterministic re-merge across cluster borders (canonical key + title-cos ≥0.95, + catches cap splits), complete-link + relation guard, survivorship keeps the champion; + (b) the degrade pass from blocks.py: full list per judge, hard-drop double gate, + containment demote, parentless-noise drop, recheck panel, statement-gate rescue, + transitive root resolution. Fragments/drops → rejected (journal), survivors → grouping.""" + topic = flow.topic + work_dir = flow.work_dir + rows = [{"card_id": c["card_id"], "payload": c["payload"], + "title": c["payload"].get("title", ""), + "description": c["payload"].get("description") or "", + "title_norm": _norm_title(c["payload"].get("title", ""))} for c in cards] + moves: list[tuple[str, str]] = [] + + # (a) global re-merge + n = len(rows) + if n >= 2: + edges = [] + vt = await _vec_rows(flow, [r["title"] for r in rows]) + keys: dict[str, list[int]] = {} + for i, r in enumerate(rows): + if (k := _canonical_key(r["title"])): + keys.setdefault(k, []).append(i) + for grp in keys.values(): + for x in range(len(grp)): + for y in range(x + 1, len(grp)): + a, b = grp[x], grp[y] + if not _relation_conflict(rows[a]["title"], rows[b]["title"]): + edges.append((a, b)) + if vt is not None: + sims_t = vt @ vt.T + for i in range(n): + for j in range(i + 1, n): + if float(sims_t[i][j]) >= DEDUP_TITLE_AUTO and not _relation_conflict( + rows[i]["title"], rows[j]["title"]): + edges.append((i, j)) + merged = 0 + for g in _cliques(n, edges): + rep = max(g, key=lambda k: (-_aspect_marker(rows[k]["title"]), + len(rows[k]["description"]), len(rows[k]["title"]), -k)) + rp = rows[rep]["payload"] + for k in g: + if k == rep: + continue + lp = rows[k]["payload"] + rp["readers"] = sorted(set(rp.get("readers") or []) | set(lp.get("readers") or [])) + rp["sources"] = sorted(set(rp.get("sources") or []) | set(lp.get("sources") or [])) + lp.update(reason="merged", merged_into=rows[rep]["title"]) + await db.kanban_set_payload(topic, BOARD, rows[k]["card_id"], lp) + moves.append((rows[k]["card_id"], "grouped")) + merged += 1 + await db.kanban_set_payload(topic, BOARD, rows[rep]["card_id"], rp) + if merged: + dropped = {cid for cid, _ in moves} + rows = [r for r in rows if r["card_id"] not in dropped] + _log(topic, f"Fragment-Filter: {merged} Cap-Split-Dublette(n) re-merged") + + # (b) degrade pass — full list = demotable stage rows + already-confirmed context + context = await _context_blocks(topic, exclude={r["card_id"] for r in rows}) + allrows = rows + context + n_dem, n_all = len(rows), len(allrows) + if n_dem: + def _fline(i): + r = allrows[i - 1] + mark = "⚠ " if i <= n_dem and _filter_suspect(r) else "" + tail = "" if i <= n_dem else " (bereits bestätigt)" + d = r.get("description") + return (f"{i}. {mark}{r['title']} — {d}{tail}" if d else f"{i}. {mark}{r['title']}{tail}") + + full_list = "\n".join(_fline(i) for i in range(1, n_all + 1)) + h = _h(*[r["card_id"] for r in rows]) + chunks = [list(range(i, min(i + FILTER_CHUNK, n_dem + 1))) for i in range(1, n_dem + 1, FILTER_CHUNK)] + + # ONE wave: all chunk judges in parallel (verdicts are independent; the voting/ + # containment evaluation below reads the files strictly afterwards). + async def _judge_chunk(ci, numbers): + path = work_dir / f"filter-{h}-c{ci}.json" + if _filter_schema(_json_file(path)) is not None: + return # resume: keep a valid judge file + status, _v = await run_single_slot( + ctx, f"Fragment-Filter {ci}", key=f"blocks-{topic}-filter-{h}-c{ci}", + prompt=_prompt("Blocks-Filter", topic=topic, list=full_list, + from_n=numbers[0], to_n=numbers[-1], out_path=path), + role="judge", capabilities="files", + payload=lambda result, p=path: _filter_schema(_json_file(p)), + timeout=_timeout("selection_mapping", len(numbers))) + if status == FAILED: + raise RuntimeError(f"Fragment-Filter chunk {ci} ohne Ergebnis") + + results = await asyncio.gather(*[_judge_chunk(ci, nm) for ci, nm in enumerate(chunks)], + return_exceptions=True) + errs = [r for r in results if isinstance(r, Exception)] + if errs: + raise errs[0] + if ctx.is_cancelled(): + return + fragments: dict[int, int] = {} + drops: set[int] = set() + for ci, numbers in enumerate(chunks): + raw = _json_file(work_dir / f"filter-{h}-c{ci}.json") + verdict = _filter_schema(raw) or {} + nset = set(numbers) + for nr, parent in verdict.items(): + if 1 <= parent <= n_all and nr in nset: + fragments[nr] = parent + for x in (raw.get("drop", []) if isinstance(raw, dict) else []): + try: + dnr = int(x) + except (ValueError, TypeError): + continue + if dnr in nset: + drops.add(dnr) + # hard-drop double gate + honored = {nr for nr in drops if _is_artifact(allrows[nr - 1]["title"])} + for nr in honored: + fragments.pop(nr, None) + # containment demote + parentless noise (deterministic) + norms = [(i, allrows[i - 1]["title_norm"]) for i in range(1, n_all + 1)] + for i in range(1, n_dem + 1): + if i in fragments or i in honored or not _filter_suspect(allrows[i - 1]): + continue + parent = _containment_parent(allrows[i - 1]["title_norm"], [(nr, t) for nr, t in norms if nr != i]) + if parent is not None and parent != i and parent not in honored: + fragments[i] = parent + for i in range(1, n_dem + 1): + if i in fragments or i in honored or not _is_parentless_noise(allrows[i - 1]["title"]): + continue + if _containment_parent(allrows[i - 1]["title_norm"], [(nr, t) for nr, t in norms if nr != i]) is None: + honored.add(i) + # recheck panel over still-⚠ survivors (rare-positive recall, majority ≥2). + # ONE wave over ALL (chunk, judge) slots; a single failed judge is tolerated + # (panel votes over whatever answered — legacy semantics). Voting afterwards. + survivors = [i for i in range(1, n_dem + 1) + if i not in fragments and i not in honored and _filter_suspect(allrows[i - 1])] + if survivors: + rchunks = [survivors[k:k + FILTER_CHUNK] for k in range(0, len(survivors), FILTER_CHUNK)] + + async def _recheck_judge(ci, nums, j): + path = work_dir / f"filter-recheck-{h}-c{ci}-j{j}.json" + if _filter_schema(_json_file(path)) is not None: + return # resume + await run_single_slot( + ctx, f"Filter-Recheck {ci}/{j}", key=f"blocks-{topic}-filter-recheck-{h}-c{ci}-j{j}", + prompt=_prompt("Blocks-Filter-Recheck", topic=topic, + survivors="\n".join(_fline(i) for i in nums), + list=full_list, out_path=path), + role="judge", capabilities="files", + payload=lambda result, p=path: _filter_schema(_json_file(p)), + timeout=_timeout("selection_mapping", len(nums))) + + await asyncio.gather(*[ + _recheck_judge(ci, nums, j) + for ci, nums in enumerate(rchunks) for j in range(1, FILTER_RECHECK_PANEL + 1)], + return_exceptions=True) + if ctx.is_cancelled(): + return + for ci, nums in enumerate(rchunks): + dem: dict[int, list[int]] = {} + drp: dict[int, int] = {} + nset = set(nums) + for j in range(1, FILTER_RECHECK_PANEL + 1): + raw = _json_file(work_dir / f"filter-recheck-{h}-c{ci}-j{j}.json") + v = _filter_schema(raw) + if v is None: + continue + for nr, parent in v.items(): + if nr in nset and 1 <= parent <= n_all and nr != parent: + dem.setdefault(nr, []).append(parent) + for x in (raw.get("drop", []) if isinstance(raw, dict) else []): + try: + dnr = int(x) + except (ValueError, TypeError): + continue + if dnr in nset: + drp[dnr] = drp.get(dnr, 0) + 1 + for nr in nums: + if nr in fragments or nr in honored: + continue + if drp.get(nr, 0) >= 2 and (_is_artifact(allrows[nr - 1]["title"]) + or _is_parentless_noise(allrows[nr - 1]["title"])): + honored.add(nr) + elif len(dem.get(nr, [])) >= 2: + fragments[nr] = max(set(dem[nr]), key=dem[nr].count) + # statement-gate rescue (final override) + def _protected(nr): + return _is_named_statement(allrows[nr - 1]["title"], allrows[nr - 1]["description"]) + for nr in [nr for nr in list(fragments) if _protected(nr)]: + fragments.pop(nr, None) + honored -= {nr for nr in honored if _protected(nr)} + # transitive resolution + moves + journal = [] + for nr in sorted(fragments): + root, cyclic = _root(nr, fragments) + if cyclic or not (1 <= root <= n_all) or nr > n_dem: + continue + r = allrows[nr - 1] + parent_norm = None if root in honored else allrows[root - 1]["title_norm"] + r["payload"].update(reason="fragment" if parent_norm else "drop-collateral", + parent_norm=parent_norm) + await db.kanban_set_payload(topic, BOARD, r["card_id"], r["payload"]) + moves.append((r["card_id"], "rejected")) + journal.append({"fragment": r["title"], + "eltern": allrows[root - 1]["title"] if parent_norm else None}) + for nr in sorted(honored): + if nr > n_dem: + continue + r = allrows[nr - 1] + r["payload"].update(reason="drop", parent_norm=None) + await db.kanban_set_payload(topic, BOARD, r["card_id"], r["payload"]) + moves.append((r["card_id"], "rejected")) + journal.append({"fragment": r["title"], "eltern": None, "grund": "drop"}) + atomic_write_json(work_dir / "inventar-filter.json", + {"vorher": n_dem, "degradiert": len(journal), "fragments": journal}, indent=1) + _log(topic, f"Fragment-Filter: {n_dem} → {n_dem - len(journal)} (−{len(journal)})") + demoted = {cid for cid, _ in moves} + moves += [(r["card_id"], "grouping") for r in rows if r["card_id"] not in demoted] + await db.kanban_advance_many(topic, BOARD, moves) + flow.wake.set() + + +async def _proc_grouping(ctx: GenContext, flow: Flow, cards): + """BARRIER/drain — umbrella grouping over the filter survivors: embedding sibling clusters + (low floor, high recall) + top-down pass, one judge per cluster, type gate + min-cos backstop, + reconcile pass, completion judge. Members → grouped; the umbrella becomes a new block card.""" + topic = flow.topic + work_dir = flow.work_dir + rows = [{"card_id": c["card_id"], "payload": c["payload"], + "title": c["payload"].get("title", ""), + "description": c["payload"].get("description") or "", + "title_norm": _norm_title(c["payload"].get("title", ""))} for c in cards] + n = len(rows) + if not (BLOCKS_GRUPPIERUNG_AKTIV and await _emb_ok(flow)) or n < 3: + await db.kanban_advance_many(topic, BOARD, [(r["card_id"], "gap_check") for r in rows]) + flow.wake.set() + return + texts = [_t_text(r) for r in rows] + vv = await _vec_rows(flow, texts) + if vv is None: + await db.kanban_advance_many(topic, BOARD, [(r["card_id"], "gap_check") for r in rows]) + flow.wake.set() + return + sims = vv @ vv.T + clusters = await asyncio.to_thread(embedding.capped_blocks, sims, + EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP) + multi = [c for c in clusters if len(c) > 1] + all_ids = set(range(1, n + 1)) + full_list = "\n".join(f"{i}. {texts[i - 1]}" for i in range(1, n + 1)) + h = _h(*[r["card_id"] for r in rows]) + + def _min_cos(idxs): + if len(idxs) < 2: + return 1.0 + return round(min(float(sims[i][j]) for a, i in enumerate(idxs) for j in idxs[a + 1:]), 3) + + async def _assess(tag, cand_text, count): + path = work_dir / f"gruppierung-{h}-c{tag}.json" + if _umbrella_schema(_json_file(path), all_ids) is None: + status, _v = await run_single_slot( + ctx, f"Gruppierung {tag}", key=f"blocks-{topic}-gruppierung-{h}-c{tag}", + prompt=_prompt("Blocks-Gruppierung", topic=topic, candidates=cand_text, + list=full_list, out_path=path), + role="judge", capabilities="files", + payload=lambda result, p=path: _umbrella_schema(_json_file(p), all_ids), + timeout=_timeout("research_mapping", count)) + if status == CANCELLED: + return None + return path + + # ONE wave: TOP + all cluster judges in parallel. The used-ties precedence lives in the + # ORDER of `sources` (TOP first), not in execution order — a failed judge simply leaves + # no valid file and is skipped in the collection below (legacy semantics). + jobs = [("TOP", "Scan the ENTIRE block list below and propose EVERY genuine umbrella " + "you find — do not restrict yourself to any subset.", n)] + jobs += [(str(ci), "\n".join(f"{g + 1}. {texts[g]}" for g in cluster), len(cluster)) + for ci, cluster in enumerate(multi)] + await asyncio.gather(*[_assess(tag, cand, count) for tag, cand, count in jobs], + return_exceptions=True) + if ctx.is_cancelled(): + return + sources = [work_dir / f"gruppierung-{h}-c{tag}.json" for tag, _, _ in jobs] # TOP first → wins used-ties + + existing_norms = {r["title_norm"] for r in rows} + for other in await db.kanban_cards(topic, board=BOARD, kind="block"): + existing_norms.add(_norm_title(other["payload"].get("title", ""))) + used: set[int] = set() + seen_norm = set(existing_norms) + chosen, skipped = [], [] + for src in sources: + for title, desc, members in (_umbrella_schema(_json_file(src), all_ids) or []): + import re as _re + title = _re.sub(r"\s+[—–]\s+", ": ", title).strip() + members = [m for m in members if m not in used] + if len(members) < 2: + continue + mrows = [rows[m - 1] for m in members] + if any(_GROUP_STANDALONE.search(r["title"]) for r in mrows): + skipped.append({"umbrella": title, "grund": "type-gate", + "mitglieder": [r["title"] for r in mrows]}) + continue + mc = _min_cos([m - 1 for m in members]) + if mc < GROUP_MIN_COS_FLOOR: + skipped.append({"umbrella": title, "grund": "min-cos", "min_cos": mc, + "mitglieder": [r["title"] for r in mrows]}) + continue + unorm = _norm_title(title) + member_norms = {r["title_norm"] for r in mrows} + if unorm not in member_norms and unorm in seen_norm: + skipped.append({"umbrella": title, "grund": "title-collision", + "mitglieder": [r["title"] for r in mrows]}) + continue + used.update(members) + seen_norm.add(unorm) + chosen.append({"umbrella": title, "description": desc, "min_cos": mc, "members": members}) + # reconcile: same parent proposed twice under different titles → union + if len(chosen) >= 2: + uv = await _vec_rows(flow, [f"{c['umbrella']} — {c['description']}" for c in chosen]) + if uv is not None: + u_sims = uv @ uv.T + parent = list(range(len(chosen))) + for i in range(len(chosen)): + for j in range(i + 1, len(chosen)): + if float(u_sims[i][j]) >= GROUP_RECONCILE_FLOOR: + embedding._union(parent, i, j) + comp: dict[int, list[int]] = {} + for i in range(len(chosen)): + comp.setdefault(embedding._find(parent, i), []).append(i) + merged = [] + for grp in comp.values(): + if len(grp) == 1: + merged.append(chosen[grp[0]]) + continue + rep = max(grp, key=lambda gi: len(chosen[gi]["members"])) + members = list(dict.fromkeys(m for gi in grp for m in chosen[gi]["members"])) + merged.append({**chosen[rep], + "description": " · ".join(chosen[gi]["description"] for gi in grp), + "members": members}) + chosen = merged + # completion: absorb leftover standalone parts (per-member type-guard veto) + leftover = sorted(all_ids - used) + if chosen and leftover: + cp = work_dir / f"gruppierung-completion-{h}.json" + add = _completion_schema(_json_file(cp), len(chosen), set(leftover)) + if add is None: + anchors = "\n".join( + f"UMBRELLA {k}: {c['umbrella']} — {c['description']}\n bereits: " + + ", ".join(rows[m - 1]["title"] for m in c["members"]) for k, c in enumerate(chosen)) + rest = "\n".join(f"{i}. {texts[i - 1]}" for i in leftover) + status, add = await run_single_slot( + ctx, "Gruppierung completion", key=f"blocks-{topic}-gruppierung-completion-{h}", + prompt=_prompt("Blocks-Gruppierung-Completion", topic=topic, umbrellas=anchors, + rest=rest, out_path=cp), + role="judge", capabilities="files", + payload=lambda result, p=cp: _completion_schema(_json_file(p), len(chosen), set(leftover)), + timeout=_timeout("research_mapping", len(leftover))) + if status == CANCELLED: + return + if status != OK: + add = [] + for k, new_members in (add or []): + for m in new_members: + if m in used or not (1 <= m <= n) or _GROUP_STANDALONE.search(rows[m - 1]["title"]): + continue + used.add(m) + chosen[k]["members"].append(m) + # apply: umbrella card + member moves + moves = [] + for c in chosen: + mrows = [rows[m - 1] for m in c["members"]] + readers = sorted(set().union(*[set(r["payload"].get("readers") or []) for r in mrows])) + srcs = sorted(set().union(*[set(r["payload"].get("sources") or []) for r in mrows])) + await db.kanban_upsert_card(topic, BOARD, f"b-u-{uuid.uuid4().hex[:8]}", "block", "gap_check", { + "title": c["umbrella"], "description": c["description"], + "readers": readers, "sources": srcs, "umbrella": True, + "children": [r["title"] for r in mrows], + }) + for r in mrows: + r["payload"].update(reason="umbrella", merged_into=c["umbrella"]) + await db.kanban_set_payload(topic, BOARD, r["card_id"], r["payload"]) + moves.append((r["card_id"], "grouped")) + grouped = {cid for cid, _ in moves} + moves += [(r["card_id"], "gap_check") for r in rows if r["card_id"] not in grouped] + atomic_write_json(work_dir / "inventar-gruppierung.json", + {"vorher": n, "umbrellas": [{"umbrella": c["umbrella"], + "mitglieder": [rows[m - 1]["title"] for m in c["members"]]} + for c in chosen], "skipped": skipped}, indent=1) + if chosen: + _log(topic, f"Gruppierung: {len(chosen)} Umbrella(s), −{len(grouped)} Blöcke") + await db.kanban_advance_many(topic, BOARD, moves) + flow.wake.set() + + +async def _proc_gap_check(ctx: GenContext, flow: Flow, cards): + """BARRIER/drain — the finished blocks advance to `done` IMMEDIATELY (board 2 starts + while the supplement searches). ONE supplement round runs as an async PRODUCER: the + producer count keeps the flow alive and the research_done gates closed, so the new + titles feed back through every gate in a clean second pass. Failure never fatal.""" + topic = flow.topic + if not flow.state.get("supplement_done"): + flow.state["supplement_done"] = True # one round (loop stop), set before the task + titles = [c["payload"].get("title", "") for c in cards] + titles += [b["title"] for b in await _context_blocks(topic, exclude=set())] + flow.add_producer() # SYNC before create_task — gates/exit must see the producer + + async def _run(): + try: + await _supplement_producer(ctx, flow, sorted({t for t in titles if t})) + except Exception: + log.exception("[%s] supplement failed", topic) + finally: + flow.done_producer() + + asyncio.create_task(_run()) + await db.kanban_advance_many(topic, BOARD, [(c["card_id"], "done") for c in cards]) + flow.wake.set() + + +async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]): + """One web agent proposes canonically missing blocks → new title cards (reader + 'supplement' skips only the ≥2 consensus bar, every other gate applies).""" + topic = flow.topic + path = flow.work_dir / "supplement.json" + supplements = _supplement_schema(_json_file(path)) + if supplements is None: + status, supplements = await run_single_slot( + ctx, "Supplement", key=f"blocks-{topic}-supplement", + prompt=_prompt("Blocks-Supplement", topic=topic, + blocks="\n".join(f"- {t}" for t in titles), + out_path=path, extra=_extra(flow.state.get("instructions", ""))), + role="quick", capabilities="full", + payload=lambda result, p=path: _supplement_schema(_json_file(p)), + timeout=_timeout("ergaenzung")) + if status == CANCELLED: + flow.state["supplement_done"] = False # in-memory only; resume re-derives from the file + return + if status != OK: + _log(topic, "Supplement fehlgeschlagen — übersprungen (optional)") + supplements = [] + known_norms = set() + known_keys = set() + for t in await db.kanban_cards(topic, board=BOARD): + tt = t["payload"].get("title", "") + if tt: + known_norms.add(_norm_title(tt)) + if (k := _canonical_key(tt)): + known_keys.add(k) + new = 0 + for t, d in (supplements or []): + norm = _norm_title(t) + key = _canonical_key(t) + if not norm or norm in known_norms or (key and key in known_keys): + continue + known_norms.add(norm) + desc = f"{d} [Supplement]".strip() + async with _ingest_lock: + await db.kanban_add_title(topic, BOARD, norm, t, desc, "supplement", "supplement") + card = await db.kanban_get_card(topic, BOARD, norm) + if card: + card["payload"]["supplement"] = True + await db.kanban_set_payload(topic, BOARD, norm, card["payload"]) + new += 1 + if new: + _log(topic, f"Supplement: {new} Block-Kandidat(en) → ingest") + flow.wake.set() + + +async def _proc_done(ctx: GenContext, flow: Flow, cards): + """Mirror finished blocks into the legacy `blocks` table (status consensus) — the + interface everything downstream (guide, exam, frontend) already reads. Unique-title + collisions get a numeric suffix; a re-run under a new name discards the old mirror.""" + topic = flow.topic + mirrored: dict[str, str] = flow.state.setdefault("mirrored", {}) + moves = [] + for c in cards: + p = c["payload"] + base = p.get("title", "") + if not base: + moves.append((c["card_id"], "done_block")) + continue + title, norm, i = base, _norm_title(base), 2 + while mirrored.get(norm) not in (None, c["card_id"]): + title = f"{base} ({i})" + norm = _norm_title(title) + i += 1 + old = p.get("mirrored_norm") + if old and old != norm: + await db.set_block_status(topic, old, "discarded") + await db.upsert_block(topic, norm, title, p.get("description", ""), p.get("sources") or []) + await db.set_block_status(topic, norm, "consensus", + title=title, description=p.get("description", "")) + mirrored[norm] = c["card_id"] + p.update(mirrored_norm=norm, title=title) + await db.kanban_set_payload(topic, BOARD, c["card_id"], p) + if (spawn := flow.state.get("spawn_artefact")): + await spawn(c["card_id"], p) # board 2 card per finished block + moves.append((c["card_id"], "done_block")) + await db.kanban_advance_many(topic, BOARD, moves) + flow.wake.set() + + +# ── Orchestration ────────────────────────────────────────────────────────────────── +def inventory_stages(ctx: GenContext, flow: Flow) -> list[Stage]: + research_done = lambda: flow.research_done # noqa: E731 + return [ + Stage(BOARD, "ingest", lambda cs: _proc_ingest(flow, cs)), + Stage(BOARD, "cluster", lambda cs: _proc_cluster(ctx, flow, cs), serial=True), + Stage(BOARD, "pair_check", lambda cs: _proc_pair_check(ctx, flow, cs)), + # gate (no barrier): hold clusters until ALL research producers are done — a title's + # second reader may arrive minutes later, and the ≥2 vote must count it. + Stage(BOARD, "consensus_gate", lambda cs: _proc_consensus_gate(ctx, flow, cs), + gate=research_done), + Stage(BOARD, "clarify", lambda cs: _proc_clarify(ctx, flow, cs)), + Stage(BOARD, "naming", lambda cs: _proc_naming(ctx, flow, cs)), + Stage(BOARD, "naming_check", lambda cs: _proc_naming_check(ctx, flow, cs)), + Stage(BOARD, "fragment_filter", lambda cs: _proc_fragment_filter(ctx, flow, cs), + barrier=True, drain=True, gate=research_done), + Stage(BOARD, "grouping", lambda cs: _proc_grouping(ctx, flow, cs), + barrier=True, drain=True, gate=research_done), + Stage(BOARD, "gap_check", lambda cs: _proc_gap_check(ctx, flow, cs), + barrier=True, drain=True, gate=research_done), + Stage(BOARD, "done", lambda cs: _proc_done(ctx, flow, cs)), + ] + + +async def _preload_state(flow: Flow): + """Continue/resume: rebuild the in-memory caches from the persisted cards.""" + titles = await db.kanban_cards(flow.topic, board=BOARD, kind="title") + known = [t for t in titles if t["stage"] != "ingest"] + flow.state["nn"] = {"norms": [t["card_id"] for t in known], + "texts": [_t_text(t["payload"]) for t in known]} + flow.state["mirrored"] = { + c["payload"]["mirrored_norm"]: c["card_id"] + for c in await db.kanban_cards(flow.topic, board=BOARD, stage="done_block") + if c["payload"].get("mirrored_norm")} + flow.state["supplement_done"] = _supplement_schema( + _json_file(flow.work_dir / "supplement.json")) is not None + + +async def run_boards(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str, + research: bool = True, artefacts: bool = True) -> bool: + """Run the inventory board (plus board 2 „Artefakte") until quiescence. + research=False = Continue: drain the existing queue, search nothing new.""" + topic = ctx.topic + flow = Flow(topic, files["arbeit"]) + flow.state["instructions"] = instructions + if q["type"] == "link" and folder: + pages = await db.list_content(topic) + flow.state["pages"] = pages or sorted(set(_crawl_index(folder).values())) + await _preload_state(flow) + stages = inventory_stages(ctx, flow) + if artefacts: + import board_artefacts # lazy — board_artefacts imports blocks too + flow.state["spawn_artefact"] = board_artefacts.make_spawner(topic, files) + await board_artefacts.ensure_outline_card(topic) + stages += board_artefacts.artefact_stages(ctx, flow, files, q, folder, instructions) + stages = chain_stages(stages) + producers = _build_producers(ctx, flow, q, folder, instructions) if research else [] + + async def _as_producer(coro): + try: + await coro + except Exception: + log.exception("[%s] research producer failed", topic) + finally: + flow.done_producer() + + for _ in producers: + flow.add_producer() + flow.spawn_research = lambda: _research_once( + ctx, flow, q, folder, instructions, f"x{flow.next_tag()}") + # cancel hook: blocks.cancel_blocks flips is_cancelled; stop the flow with it + stopper = asyncio.create_task(_stop_on_cancel(ctx, flow)) + try: + await kanban.run_flow(flow, stages, [_as_producer(p) for p in producers], set_p) + finally: + stopper.cancel() + if ctx.is_cancelled(): + return False + await _write_final(topic, files) + return True + + +async def _stop_on_cancel(ctx: GenContext, flow: Flow): + while not flow.stop: + if ctx.is_cancelled(): + flow.stop = True + flow.wake.set() + return + await asyncio.sleep(0.3) + + +async def _write_final(topic: str, files: dict): + """blocks.md from the finished cards (flow order — titles are already unique).""" + done = await db.kanban_cards(topic, board=BOARD, stage="done_block") + done.sort(key=lambda c: c["updated_at"]) + lines = [_line(i, c["payload"]) for i, c in enumerate(done, 1) if c["payload"].get("title")] + if lines: + atomic_write_text(files["final"], "\n".join(lines) + "\n") + _log(topic, f"Kanban: fertig — {len(lines)} Blöcke") + + +# ── Board API (snapshot, reset, dead-letter) ─────────────────────────────────────── +# (board, stage, label, kind) — display order of the live board. Terminal columns last. +COLUMNS = [ + ("inventory", "ingest", "Eingang", "title"), + ("inventory", "cluster", "Cluster", "title"), + ("inventory", "pair_check", "Paar-Check", "cluster"), + ("inventory", "consensus_gate", "Konsens", "cluster"), + ("inventory", "clarify", "Klärung", "cluster"), + ("inventory", "naming", "Naming", "cluster"), + ("inventory", "naming_check", "Naming-Check", "cluster"), + ("inventory", "fragment_filter", "Fragment-Filter", "block"), + ("inventory", "grouping", "Gruppierung", "block"), + ("inventory", "gap_check", "Lücken-Check", "block"), + ("inventory", "done", "Spiegeln", "block"), + ("inventory", "done_block", "Fertig", "block"), + ("inventory", "rejected", "Verworfen", None), + ("inventory", "grouped", "Zusammengelegt", "block"), + ("artefacts", "subblocks", "Subbausteine", "ablock"), + ("artefacts", "facts", "Fakten", "ablock"), + ("artefacts", "levels", "Stufen", "ablock"), + ("artefacts", "relevance", "Relevanz", "ablock"), + ("artefacts", "question_pattern", "Fragen", "ablock"), + ("artefacts", "artefacts", "Lernkarten", "ablock"), + ("artefacts", "finalize", "Zusammenführen", "ablock"), + ("artefacts", "outline", "Gliederung", "outline"), + ("artefacts", "done_artefact", "Fertig", "ablock"), +] + +_TITLE_STAGES = ["ingest", "cluster"] +_CLUSTER_STAGES = ["pair_check", "consensus_gate", "clarify", "naming", "naming_check"] +_BLOCK_STAGES = ["fragment_filter", "grouping", "gap_check", "done"] +_ART_STAGES = ["subblocks", "facts", "levels", "relevance", "question_pattern", + "artefacts", "finalize", "outline"] +# where a requeued dead card restarts, by kind +_DEAD_RESTART = {"title": "cluster", "cluster": "pair_check", "block": "fragment_filter", + "ablock": "subblocks", "outline": "outline"} +_VERDICT_KEYS = ("reason", "votes", "judges", "merged_into", "parent_norm", "mirrored_norm") +DONE_ART = "done_artefact" + + +def _card_view(r: dict, active: set[str], live_info: dict) -> dict: + p = r["payload"] + key = f"{r['board']}:{r['card_id']}" + is_active = key in active + info = r.get("last_error") or p.get("reason") or "" + if p.get("merged_into"): + info = f"→ {p['merged_into']}" + elif p.get("parent_norm"): + info = f"Fragment von: {p['parent_norm']}" + if is_active and live_info.get(key): # live step message wins while the card is worked + info = live_info[key] + status = "error" if r["retries"] else ("active" if is_active else "open") + return {"title": p.get("title") or r["card_id"], "status": status, + "info": info, "retries": r["retries"]} + + +async def board_snapshot(topic: str, limit: int = 20) -> dict: + """Live board: per column count + the newest cards (title, status, info).""" + counts = await db.kanban_stage_counts(topic) + flow = kanban.active_flows.get(topic) + active = flow.active_cards if flow else set() + live_info = flow.state.get("card_info", {}) if flow else {} + columns = [] + for board, stage, label, _kind in COLUMNS: + total = counts.get(board, {}).get(stage, 0) + cards = ([_card_view(r, active, live_info) + for r in await db.kanban_stage_cards(topic, board, stage, limit)] + if total else []) + columns.append({"board": board, "key": stage, "label": label, "total": total, "cards": cards}) + dead = [{"card_id": r["card_id"], "board": r["board"], "kind": r["kind"], + "title": r["payload"].get("title") or r["card_id"], "error": r.get("last_error") or ""} + for r in await db.kanban_dead(topic)] + return {"columns": columns, "dead": dead, + "done": counts.get("inventory", {}).get("done_block", 0)} + + +async def _clean_artefact_state(topic: str, files: dict) -> None: + """Artefact data is fully derived — wipe cards, DB tables and sidecar files.""" + await db.kanban_delete_cards(topic, "artefacts") + await db.delete_subblocks(topic) + await db.delete_question_pattern(topic) + await db.delete_sub_artefakte(topic) + await db.delete_outline(topic) + for key in ("sub_roh", "sidecar", "facts", "question_pattern", "artefakte", "outline"): + files[key].unlink(missing_ok=True) + + +async def reset_board_from_stage(topic: str, board: str, stage: str, files: dict) -> int: + """Reset ab Spalte: cards from `stage` onward (incl. terminal rejected/grouped/dead) + return to `stage`; downstream derived state is wiped. → moved card count. + Only call while nothing is generating (the route guards).""" + moved = 0 + + async def _requeue(card: dict, to_stage: str): + nonlocal moved + p = {k: v for k, v in card["payload"].items() if k not in _VERDICT_KEYS} + await db.kanban_set_payload(topic, card["board"], card["card_id"], p) + await db.kanban_advance(topic, card["board"], card["card_id"], to_stage) + moved += 1 + + if board == "inventory" and stage in _TITLE_STAGES: + # everything below titles is derived → full re-derive + for r in await db.kanban_cards(topic, board="inventory"): + if r["kind"] == "title": + await _requeue(r, stage) + await db.kanban_delete_cards(topic, "inventory", "cluster") + await db.kanban_delete_cards(topic, "inventory", "block") + dbc = await db.get_db() + await dbc.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,)) + await dbc.commit() + await db.delete_blocks(topic) + await _clean_artefact_state(topic, files) + elif board == "inventory" and stage in _CLUSTER_STAGES: + idx = _CLUSTER_STAGES.index(stage) + later = set(_CLUSTER_STAGES[idx:]) | {"done_cluster", "rejected", "dead"} + for r in await db.kanban_cards(topic, board="inventory", kind="cluster"): + if r["stage"] in later: + await _requeue(r, stage) + await db.kanban_delete_cards(topic, "inventory", "block") + await db.delete_blocks(topic) + await _clean_artefact_state(topic, files) + elif board == "inventory": + idx = _BLOCK_STAGES.index(stage) if stage in _BLOCK_STAGES else 0 + later = set(_BLOCK_STAGES[idx:]) | {"done_block", "rejected", "grouped", "dead"} + for r in await db.kanban_cards(topic, board="inventory", kind="block"): + if r["stage"] in later: + await _requeue(r, stage) + await db.delete_blocks(topic) + await _clean_artefact_state(topic, files) + else: # artefacts board + idx = _ART_STAGES.index(stage) if stage in _ART_STAGES else 0 + later = set(_ART_STAGES[idx:]) | {DONE_ART, "dead"} + for r in await db.kanban_cards(topic, board="artefacts"): + if r["kind"] == "outline": + await _requeue(r, "outline") + elif r["stage"] in later: + await _requeue(r, stage) + if stage == "subblocks": # full artefact re-derive → also the DB mirrors + await db.delete_subblocks(topic) + await db.delete_question_pattern(topic) + await db.delete_sub_artefakte(topic) + return moved + + +async def requeue_dead(topic: str) -> int: + """Dead-letter → restart stage by card kind (fresh retries). → requeued count.""" + n = 0 + for r in await db.kanban_dead(topic): + stage = _DEAD_RESTART.get(r["kind"]) + if stage: + await db.kanban_advance(topic, r["board"], r["card_id"], stage) + n += 1 + return n + + +def add_research_agent(topic: str) -> bool: + """Attach one more research agent to a live flow. Counted SYNCHRONOUSLY before the task + (quiescence race). → True if a run was live to attach to.""" + flow = kanban.active_flows.get(topic) + if flow is None or flow.stop or flow.spawn_research is None: + return False + flow.add_producer() + + async def _run(): + try: + await flow.spawn_research() + except Exception: + log.exception("[%s] extra research failed", topic) + finally: + flow.done_producer() + + asyncio.create_task(_run()) + return True diff --git a/backend/config.py b/backend/config.py index adaa0f3..0e01ef7 100644 --- a/backend/config.py +++ b/backend/config.py @@ -1,3 +1,4 @@ +import os from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parent.parent @@ -8,6 +9,26 @@ DB_PATH = STORAGE_DIR / "creator.db" PROJECTS_DIR = PROJECT_ROOT / "projects" UNI_DIR = PROJECT_ROOT / "uni" + +def _load_env(path: Path) -> None: + """Mini .env loader (no dependency): KEY=VALUE lines; existing env always wins + (`make dev` already exports .env — this covers bare `uvicorn`/pytest starts).""" + try: + text = path.read_text(encoding="utf-8") + except OSError: + return + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + key, value = key.strip(), value.strip().strip('"').strip("'") + if key and key not in os.environ: + os.environ[key] = value + + +_load_env(PROJECT_ROOT / ".env") + MAX_CONCURRENT_GENERATIONS = 10 # Readability gate: deterministic checker (small German complexity model, @@ -21,12 +42,9 @@ READABILITY_MAX = 3.5 # section too hard when the sentence average is a READABILITY_HARD = 4.0 # an individual sentence is "hard" from here on READABILITY_HARD_SHARE = 0.30 # … OR when this share of sentences is hard -# Block consolidation: semantic embedding clustering instead of an LLM list merge. -# A small multilingual sentence embedding (mean-pool) builds the candidate clusters -# GLOBALLY (no chunk loss) via cosine + union-find. Title variants of the same concept -# ("Vertex Cover" / "Vertex Cover Definition") merge; the consensus then counts the -# real readers per cluster (≥2 = consensus). If transformers/torch are missing or the model -# won't load → embedding silently off, `_consolidate` falls back to the old panel-judge path. +# Kanban clustering: semantic embeddings drive the online title clustering and the +# candidate pairs of the pair check. If transformers/torch are missing or the model +# won't load → embedding silently off (all pairs go to the judge, clusters stay singletons). EMBEDDING_AKTIV = True EMBEDDING_MODELL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" # CPU, multilingual, ~470 MB # Stronger (larger) CPU alternative if needed: "BAAI/bge-m3". @@ -66,10 +84,12 @@ GROUP_RECONCILE_FLOOR = 0.75 # chain (random-pair baseline), set BELOW the legitimate heterogeneous minimum so it never kills a real model. GROUP_MIN_COS_FLOOR = 0.15 -# Cap for concurrent CLI agent processes (across all generations). -# Own lane for interactive calls (chat, elements) so they don't hang behind -# running writers in the queue. -MAX_CONCURRENT_AGENTS = 10 +# Caps for concurrent CLI agent processes (env-overridable). Two nested limits, both always active: +# a per-topic cap and a global cap across all topics. Defaults 10/10 = previous behavior (global +# dominates). Locally raise the global cap to actually parallelize across topics (per-topic stays 10). +# Own lane for interactive calls (chat, elements) so they don't hang behind running writers. +MAX_CONCURRENT_AGENTS = int(os.getenv("MAX_CONCURRENT_AGENTS", "16")) # global, all topics +MAX_CONCURRENT_AGENTS_PER_TOPIC = int(os.getenv("MAX_CONCURRENT_AGENTS_PER_TOPIC", "12")) # per topic MAX_CONCURRENT_INTERACTIVE = 8 # Grace window of the consensus races (blocks, guide, OnePager): after the first @@ -123,6 +143,10 @@ TIMEOUTS = { "question_pattern_check": (300, 10), # critic cleans up the pattern table per block "writer": (600, 120), # per section in the chunk "lese_check": (300, 10), # per section in the package + # guide board (per card = one block) + "lernziele": (300, 5), # backward-design objectives per block + "fakten_gate": (600, 5), # CoVe claim check per block + "coverage": (300, 5), # objective↔section mapping per block } # Purpose per format — flows into the outline judge (what the guide should achieve). @@ -169,3 +193,27 @@ PROVIDERS = { "check_url": "http://localhost:11434/api/tags", # Ollama reachable? }, } + +# Role routing ACROSS provider stacks: generation (quick/guide) and judging (judge) +# may run on different providers within ONE run — judge model ≠ generator model +# (research-backed: cross-model judging avoids self-preference bias). +# Value: "" = provider of the run; "minimax" = that stack's role model; +# "provider:model" = explicit model override. +ROLE_ROUTING = { + "quick": os.getenv("ROLE_QUICK", "minimax"), + "judge": os.getenv("ROLE_JUDGE", "claude"), + "guide": os.getenv("ROLE_GUIDE", "minimax"), + "fast": os.getenv("ROLE_FAST", ""), +} + + +def resolve_role(run_provider: str, role: str) -> tuple[str, str]: + """→ (provider, model) for one agent call. Pure routing, no availability check — + the caller (agents.run_agent) falls back to run_provider if the target is unavailable.""" + target = ROLE_ROUTING.get(role, "") or run_provider + provider, _, model = target.partition(":") + if provider not in PROVIDERS: + provider, model = run_provider, "" + if not model: + model = PROVIDERS.get(provider, {}).get(role, "") + return provider, model diff --git a/backend/database.py b/backend/database.py index 2ec89c5..32e4cdf 100644 --- a/backend/database.py +++ b/backend/database.py @@ -199,6 +199,73 @@ CREATE TABLE IF NOT EXISTS sub_artefakte ( ) """ +# Kanban dataflow (boards 'inventory' + 'artefacts'): ONE generic card table for all card kinds +# (title/cluster/block). `stage` is the queue key — a worker pulls WHERE stage = . +# `payload` is a JSON blob (title, description, sources, readers, mentions, parent_norm, journal …); +# retries/not_before/last_error implement backoff + dead-letter (stage 'dead' after MAX_CARD_RETRIES). +CREATE_KANBAN_CARDS = """ +CREATE TABLE IF NOT EXISTS kanban_cards ( + topic TEXT NOT NULL, + board TEXT NOT NULL, + card_id TEXT NOT NULL, + kind TEXT NOT NULL, + stage TEXT NOT NULL, + payload TEXT NOT NULL DEFAULT '{}', + retries INTEGER NOT NULL DEFAULT 0, + not_before TEXT NOT NULL DEFAULT '', + last_error TEXT, + updated_at TEXT NOT NULL, + PRIMARY KEY (topic, board, card_id) +) +""" + +CREATE_KANBAN_PULL_INDEX = """ +CREATE INDEX IF NOT EXISTS idx_kanban_pull ON kanban_cards(topic, board, stage, not_before, updated_at) +""" + +# title_norm → cluster membership (one title belongs to exactly one cluster). +CREATE_KANBAN_MEMBERS = """ +CREATE TABLE IF NOT EXISTS kanban_members ( + topic TEXT NOT NULL, + member_id TEXT NOT NULL, + group_id TEXT NOT NULL, + PRIMARY KEY (topic, member_id) +) +""" + +# Guide board: one card per block, linear stages (lernziele … lesbarkeit → done). +# `md` carries the writer fragment (with kapitel/section/sub markers) between the gates. +CREATE_GUIDE_CARDS = """ +CREATE TABLE IF NOT EXISTS guide_cards ( + topic TEXT NOT NULL, + format TEXT NOT NULL DEFAULT 'Guide', + block_norm TEXT NOT NULL, + block TEXT NOT NULL, + chapter TEXT NOT NULL DEFAULT '', + ord INTEGER NOT NULL DEFAULT 0, + stage TEXT NOT NULL DEFAULT 'lernziele', + status TEXT NOT NULL DEFAULT 'open', + writer_rounds INTEGER NOT NULL DEFAULT 0, + gate_info TEXT NOT NULL DEFAULT '', + md TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL, + PRIMARY KEY (topic, format, block_norm) +) +""" + +CREATE_GUIDE_LERNZIELE = """ +CREATE TABLE IF NOT EXISTS guide_lernziele ( + topic TEXT NOT NULL, + block_norm TEXT NOT NULL, + ziel_id TEXT NOT NULL, + text TEXT NOT NULL, + sub_norm TEXT NOT NULL DEFAULT '', + covered INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL, + PRIMARY KEY (topic, block_norm, ziel_id) +) +""" + _db: aiosqlite.Connection | None = None @@ -213,7 +280,9 @@ async def get_db() -> aiosqlite.Connection: async def init_db(): db = await get_db() # WAL survives crashes much better; busy_timeout absorbs short locks. + # synchronous=NORMAL: safe under WAL, much less fsync — the kanban flow commits often. await db.execute("PRAGMA journal_mode=WAL") + await db.execute("PRAGMA synchronous=NORMAL") await db.execute("PRAGMA busy_timeout=5000") await db.execute(CREATE_GUIDES) await db.execute(CREATE_PROGRESS) @@ -230,6 +299,11 @@ async def init_db(): await db.execute(CREATE_SOURCE) await db.execute(CREATE_GUIDE_OUTLINE) await db.execute(CREATE_SUB_ARTEFAKTE) + await db.execute(CREATE_KANBAN_CARDS) + await db.execute(CREATE_KANBAN_PULL_INDEX) + await db.execute(CREATE_KANBAN_MEMBERS) + await db.execute(CREATE_GUIDE_CARDS) + await db.execute(CREATE_GUIDE_LERNZIELE) try: # migration for existing DBs without the step column await db.execute("ALTER TABLE guides ADD COLUMN step INTEGER") except aiosqlite.OperationalError: @@ -706,6 +780,378 @@ async def delete_blocks(topic: str) -> None: await db.commit() +# ── Kanban dataflow (generic card layer, boards 'inventory' + 'artefacts') ──────── +def _now_plus(seconds: float) -> str: + from datetime import datetime, timedelta, timezone + return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat() + + +def _card(row, cursor) -> dict: + """Row → dict with the JSON payload decoded (payload keys stay under 'payload').""" + d = _row_to_dict(row, cursor) + try: + d["payload"] = json.loads(d.get("payload") or "{}") + except (TypeError, ValueError): + d["payload"] = {} + return d + + +async def kanban_pull(topic: str, board: str, stage: str, limit: int) -> list[dict]: + """Oldest `limit` cards sitting in `stage` whose backoff has expired (FIFO via updated_at).""" + db = await get_db() + cursor = await db.execute( + """SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? AND not_before <= ? + ORDER BY updated_at LIMIT ?""", + (topic, board, stage, _now(), limit)) + return [_card(row, cursor) for row in await cursor.fetchall()] + + +async def kanban_count(topic: str, stages, board: str | None = None) -> int: + """Cards sitting in any of `stages` (str or list) — queue length / quiescence. + Cards in backoff still count: their work is not done. `board=None` = across boards.""" + if isinstance(stages, str): + stages = [stages] + if not stages: + return 0 + db = await get_db() + ph = ",".join("?" * len(stages)) + sql = f"SELECT count(*) FROM kanban_cards WHERE topic = ? AND stage IN ({ph})" + args: tuple = (topic, *stages) + if board: + sql += " AND board = ?" + args += (board,) + cursor = await db.execute(sql, args) + return (await cursor.fetchone())[0] + + +async def kanban_advance(topic: str, board: str, card_id: str, stage: str) -> None: + """Move a card to `stage` (next column, or back for rework). Clears backoff/error.""" + await kanban_advance_many(topic, board, [(card_id, stage)]) + + +async def kanban_advance_many(topic: str, board: str, moves: list[tuple[str, str]]) -> None: + """Batch stage moves in ONE commit (the flow advances whole packages).""" + if not moves: + return + db = await get_db() + now = _now() + await db.executemany( + """UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ? + WHERE topic = ? AND board = ? AND card_id = ?""", + [(stage, now, topic, board, cid) for cid, stage in moves]) + await db.commit() + + +async def kanban_upsert_card(topic: str, board: str, card_id: str, kind: str, stage: str, + payload: dict | None = None) -> None: + """Insert or overwrite a card (stable ids → growing clusters upsert, never duplicate). + payload=None keeps the existing payload on conflict.""" + db = await get_db() + await db.execute( + """INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at) + VALUES (?, ?, ?, ?, ?, COALESCE(?, '{}'), ?) + ON CONFLICT(topic, board, card_id) DO UPDATE SET + kind = excluded.kind, stage = excluded.stage, + payload = COALESCE(?, kanban_cards.payload), + retries = 0, not_before = '', last_error = NULL, updated_at = excluded.updated_at""", + (topic, board, card_id, kind, stage, + json.dumps(payload, ensure_ascii=False) if payload is not None else None, _now(), + json.dumps(payload, ensure_ascii=False) if payload is not None else None)) + await db.commit() + + +async def kanban_set_payload(topic: str, board: str, card_id: str, payload: dict) -> None: + db = await get_db() + await db.execute( + "UPDATE kanban_cards SET payload = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?", + (json.dumps(payload, ensure_ascii=False), _now(), topic, board, card_id)) + await db.commit() + + +async def kanban_get_card(topic: str, board: str, card_id: str) -> dict | None: + db = await get_db() + cursor = await db.execute( + "SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?", (topic, board, card_id)) + row = await cursor.fetchone() + return _card(row, cursor) if row else None + + +async def kanban_cards(topic: str, board: str | None = None, stage: str | None = None, + kind: str | None = None) -> list[dict]: + db = await get_db() + sql, args = "SELECT * FROM kanban_cards WHERE topic = ?", [topic] + for col, val in (("board", board), ("stage", stage), ("kind", kind)): + if val is not None: + sql += f" AND {col} = ?" + args.append(val) + cursor = await db.execute(sql, args) + return [_card(row, cursor) for row in await cursor.fetchall()] + + +async def kanban_fail_card(topic: str, board: str, card_id: str, error: str, + max_retries: int, backoff_base: float = 30.0) -> bool: + """Register a processing failure: retries++, exponential backoff (not_before), and after + `max_retries` → stage 'dead' (dead-letter, requeue-able). → True if the card went dead.""" + db = await get_db() + cursor = await db.execute( + "SELECT retries FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?", + (topic, board, card_id)) + row = await cursor.fetchone() + if row is None: + return False + retries = (row[0] or 0) + 1 + dead = retries >= max_retries + if dead: + await db.execute( + """UPDATE kanban_cards SET stage = 'dead', retries = ?, not_before = '', + last_error = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?""", + (retries, error[:500], _now(), topic, board, card_id)) + else: + await db.execute( + """UPDATE kanban_cards SET retries = ?, not_before = ?, last_error = ?, updated_at = ? + WHERE topic = ? AND board = ? AND card_id = ?""", + (retries, _now_plus(backoff_base * (2 ** (retries - 1))), error[:500], _now(), + topic, board, card_id)) + await db.commit() + return dead + + +async def kanban_dead(topic: str) -> list[dict]: + """Dead-letter cards across boards (for the board UI + requeue).""" + return await kanban_cards(topic, stage="dead") + + +async def kanban_requeue_dead(topic: str, board: str, stage: str) -> int: + """dead → `stage` (fresh retries). → number of requeued cards.""" + db = await get_db() + cursor = await db.execute( + """UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ? + WHERE topic = ? AND board = ? AND stage = 'dead'""", + (stage, _now(), topic, board)) + await db.commit() + return cursor.rowcount + + +async def kanban_stage_counts(topic: str) -> dict[str, dict[str, int]]: + """{board: {stage: count}} — the live board.""" + db = await get_db() + cursor = await db.execute( + "SELECT board, stage, count(*) FROM kanban_cards WHERE topic = ? GROUP BY board, stage", (topic,)) + out: dict[str, dict[str, int]] = {} + for board, stage, n in await cursor.fetchall(): + out.setdefault(board, {})[stage] = n + return out + + +async def kanban_stage_cards(topic: str, board: str, stage: str, limit: int = 20) -> list[dict]: + """Newest `limit` cards of one column (for the live card display).""" + db = await get_db() + cursor = await db.execute( + """SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? + ORDER BY updated_at DESC LIMIT ?""", + (topic, board, stage, limit)) + return [_card(row, cursor) for row in await cursor.fetchall()] + + +async def kanban_delete_cards(topic: str, board: str, kind: str | None = None) -> None: + """Delete derived cards (board reset) — kind=None wipes the whole board.""" + db = await get_db() + if kind: + await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ? AND kind = ?", + (topic, board, kind)) + else: + await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ?", (topic, board)) + await db.commit() + + +async def kanban_reset(topic: str, board: str | None = None) -> None: + db = await get_db() + if board: + await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ?", (topic, board)) + if board == "inventory": + await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,)) + else: + await db.execute("DELETE FROM kanban_cards WHERE topic = ?", (topic,)) + await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,)) + await db.commit() + + +async def kanban_set_members(topic: str, group_id: str, members: list[str]) -> None: + """Replace the member set of a cluster (one member belongs to exactly one cluster).""" + db = await get_db() + await db.execute("DELETE FROM kanban_members WHERE topic = ? AND group_id = ?", (topic, group_id)) + await db.executemany( + """INSERT INTO kanban_members (topic, member_id, group_id) VALUES (?, ?, ?) + ON CONFLICT(topic, member_id) DO UPDATE SET group_id = excluded.group_id""", + [(topic, m, group_id) for m in members]) + await db.commit() + + +async def kanban_members_of(topic: str, group_id: str) -> list[str]: + db = await get_db() + cursor = await db.execute( + "SELECT member_id FROM kanban_members WHERE topic = ? AND group_id = ?", (topic, group_id)) + return [r[0] for r in await cursor.fetchall()] + + +async def kanban_member_group(topic: str, member_id: str) -> str | None: + db = await get_db() + cursor = await db.execute( + "SELECT group_id FROM kanban_members WHERE topic = ? AND member_id = ?", (topic, member_id)) + row = await cursor.fetchone() + return row[0] if row else None + + +# ── Guide board (one card per block, linear stages) ────────────────────────────── +async def upsert_guide_card(topic: str, format: str, block_norm: str, block: str, + stage: str = "lernziele") -> None: + """Insert a card; an existing one keeps its stage/progress (resume).""" + db = await get_db() + await db.execute( + """INSERT INTO guide_cards (topic, format, block_norm, block, stage, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(topic, format, block_norm) DO UPDATE SET + block = excluded.block, updated_at = excluded.updated_at""", + (topic, format, block_norm, block, stage, _now())) + await db.commit() + + +async def list_guide_cards(topic: str, format: str) -> list[dict]: + db = await get_db() + cursor = await db.execute( + "SELECT * FROM guide_cards WHERE topic = ? AND format = ? ORDER BY ord, block_norm", + (topic, format)) + return [_row_to_dict(row, cursor) for row in await cursor.fetchall()] + + +async def set_guide_card(topic: str, format: str, block_norm: str, **fields) -> None: + if not fields: + return + db = await get_db() + cols = ", ".join(f"{k} = ?" for k in fields) + await db.execute( + f"UPDATE guide_cards SET {cols}, updated_at = ? WHERE topic = ? AND format = ? AND block_norm = ?", + (*fields.values(), _now(), topic, format, block_norm)) + await db.commit() + + +async def guide_stage_counts(topic: str, format: str) -> dict[str, int]: + db = await get_db() + cursor = await db.execute( + "SELECT stage, count(*) FROM guide_cards WHERE topic = ? AND format = ? GROUP BY stage", + (topic, format)) + return {stage: n for stage, n in await cursor.fetchall()} + + +async def reset_guide_cards_from_stage(topic: str, format: str, stages: list[str], + to_stage: str, clear_md: bool = False) -> int: + """Cards sitting in any of `stages` → back to `to_stage` (fresh rounds/gate info).""" + if not stages: + return 0 + db = await get_db() + ph = ",".join("?" * len(stages)) + md = ", md = ''" if clear_md else "" + cursor = await db.execute( + f"""UPDATE guide_cards SET stage = ?, status = 'open', writer_rounds = 0, + gate_info = ''{md}, updated_at = ? + WHERE topic = ? AND format = ? AND stage IN ({ph})""", + (to_stage, _now(), topic, format, *stages)) + await db.commit() + return cursor.rowcount + + +async def delete_guide_board(topic: str, format: str | None = None) -> None: + db = await get_db() + if format: + await db.execute("DELETE FROM guide_cards WHERE topic = ? AND format = ?", (topic, format)) + cursor = await db.execute("SELECT count(*) FROM guide_cards WHERE topic = ?", (topic,)) + if (await cursor.fetchone())[0] == 0: + await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,)) + else: + await db.execute("DELETE FROM guide_cards WHERE topic = ?", (topic,)) + await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,)) + await db.commit() + + +async def put_lernziel(topic: str, block_norm: str, ziel_id: str, text: str, sub_norm: str = "") -> None: + db = await get_db() + await db.execute( + """INSERT INTO guide_lernziele (topic, block_norm, ziel_id, text, sub_norm, covered, updated_at) + VALUES (?, ?, ?, ?, ?, 0, ?) + ON CONFLICT(topic, block_norm, ziel_id) DO UPDATE SET + text = excluded.text, sub_norm = excluded.sub_norm, updated_at = excluded.updated_at""", + (topic, block_norm, ziel_id, text, sub_norm, _now())) + await db.commit() + + +async def list_lernziele(topic: str, block_norm: str | None = None) -> list[dict]: + db = await get_db() + if block_norm is None: + cursor = await db.execute( + "SELECT * FROM guide_lernziele WHERE topic = ? ORDER BY block_norm, ziel_id", (topic,)) + else: + cursor = await db.execute( + "SELECT * FROM guide_lernziele WHERE topic = ? AND block_norm = ? ORDER BY ziel_id", + (topic, block_norm)) + return [_row_to_dict(row, cursor) for row in await cursor.fetchall()] + + +async def set_ziel_covered(topic: str, block_norm: str, ziel_id: str, covered: bool) -> None: + db = await get_db() + await db.execute( + "UPDATE guide_lernziele SET covered = ?, updated_at = ? WHERE topic = ? AND block_norm = ? AND ziel_id = ?", + (1 if covered else 0, _now(), topic, block_norm, ziel_id)) + await db.commit() + + +async def delete_lernziele(topic: str, block_norm: str) -> None: + db = await get_db() + await db.execute("DELETE FROM guide_lernziele WHERE topic = ? AND block_norm = ?", (topic, block_norm)) + await db.commit() + + +async def kanban_membership(topic: str) -> dict[str, str]: + """{member_id: group_id} for the whole topic (the cluster worker's working map).""" + db = await get_db() + cursor = await db.execute("SELECT member_id, group_id FROM kanban_members WHERE topic = ?", (topic,)) + return {m: g for m, g in await cursor.fetchall()} + + +async def kanban_set_member(topic: str, member_id: str, group_id: str) -> None: + db = await get_db() + await db.execute( + """INSERT INTO kanban_members (topic, member_id, group_id) VALUES (?, ?, ?) + ON CONFLICT(topic, member_id) DO UPDATE SET group_id = excluded.group_id""", + (topic, member_id, group_id)) + await db.commit() + + +async def kanban_add_title(topic: str, board: str, card_id: str, title: str, + description: str, source: str, reader: str) -> bool: + """Ingest one research title (stage 'ingest'). An exact dupe folds instead of duplicating: + reader-set union, source union, longer description wins; stage stays untouched (a title + already consumed into a cluster is not re-queued). NOT concurrency-safe across awaits — + callers serialize through one ingest lock. → True if the card is new.""" + row = await kanban_get_card(topic, board, card_id) + if row is None: + payload = {"title": title, "description": description, + "sources": [source] if source else [], "readers": [reader]} + db = await get_db() + await db.execute( + """INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at) + VALUES (?, ?, ?, 'title', 'ingest', ?, ?)""", + (topic, board, card_id, json.dumps(payload, ensure_ascii=False), _now())) + await db.commit() + return True + p = row["payload"] + p["readers"] = list(dict.fromkeys((p.get("readers") or []) + [reader])) + p["sources"] = list(dict.fromkeys((p.get("sources") or []) + ([source] if source else []))) + if len(description or "") > len(p.get("description") or ""): + p["description"] = description + await kanban_set_payload(topic, board, card_id, p) + return False + + async def upsert_subblock(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str) -> None: db = await get_db() await db.execute( diff --git a/backend/guide.py b/backend/guide.py index f653512..802f9be 100644 --- a/backend/guide.py +++ b/backend/guide.py @@ -24,7 +24,8 @@ from config import ( READABILITY_ACTIVE, TEMPLATES_DIR, ) import readability -from database import list_guides, update_guide, list_blocks, list_subblocks, set_guide_content, get_guide_content, get_outline +from database import (list_guides, update_guide, list_blocks, list_subblocks, set_guide_content, + get_guide_content, get_outline, guide_stage_counts, delete_guide_board) from fsutil import atomic_write_json, atomic_write_text from jsonio import read_json_file as _json_file, parse_json_text as _parse_json_text from paths import blocks_path, guide_content_path, project_dir, subblocks_path @@ -41,20 +42,16 @@ from textkit import ( log = logging.getLogger("creator.guide") -GUIDE_STEPS = ("Outline", "Content", "Content-Check", "Writing", "Reading-Exam") # Content/Content-Check/Reading-Exam run in packets of ~GUIDE_CHUNK blocks per agent. # Only the writer (Writing) stays at 1 agent per block (variable lengths, no trimming, no # length alignment between blocks). -GUIDE_CHUNK = 10 # Check steps as a panel: CHECK_PANEL judges per chunk, section flagged on a majority. # A single judge is bias/sampling prone; a small panel is more stable. -CHECK_PANEL = 3 # Reading exam: only ONE round (Check + Fix). Follow-up rounds added little value # (1 agent per block checks finely anyway) but cost extra agents. -READING_ROUNDS = 1 # Valid level values: new (learning path) + old (difficulty) backward-compatible. @@ -93,26 +90,8 @@ def _level_label(s: dict) -> str: return "peripheral" if s.get("relevance") == "peripheral" else (s.get("level") or "beginner") -def _assignment_subs(chunk: list[dict], entries: dict[int, str], subs_by_title: dict[str, list[dict]]) -> str: - """Lists the blocks per chapter, with their subblocks and level labels beneath.""" - lines: list[str] = [] - for ch in chunk: - lines.append(f"CHAPTER: {ch['title']}") - for num in ch["nums"]: - lines.append(f"- {entries[num]}") - for s in subs_by_title.get(_title(entries[num]), []): - lines.append(f" [{_level_label(s)}] {s['title']}") - return "\n".join(lines) -def _guide_files(content_path: Path) -> dict: - d, stem = content_path.parent, content_path.stem - return { - "outline_slots": [d / f"{stem}.outline-{i}.json" for i in (1, 2, 3)], - "outline": d / f"{stem}.outline.json", # judge output - # chunk/reading-check/fix files are dynamic: - # {stem}.chunk-i.md, {stem}.lese-check-r{n}-{i}.json, {stem}.fix-r{n}-{i}.md - } def guide_slot_files(content_path: Path) -> list[Path]: @@ -120,132 +99,23 @@ def guide_slot_files(content_path: Path) -> list[Path]: return [p for p in content_path.parent.glob(f"{content_path.stem}.*") if p != content_path] -def _done_path(content_path: Path) -> Path: - return content_path.parent / f"{content_path.stem}.done" -def guide_done_step(content_path: Path) -> int: - """Highest FULLY completed step index (marker per topic+format). -1 = none. - If the content file exists, all steps are done.""" - if content_path.exists(): - return len(GUIDE_STEPS) - 1 - try: - return int(_done_path(content_path).read_text(encoding="utf-8").strip()) - except (OSError, ValueError): - return -1 -def _set_done(content_path: Path, step: int) -> None: - """Set marker to `step` — monotone (only increase), except on the re-run reset (force).""" - if step > guide_done_step(content_path): - atomic_write_text(_done_path(content_path), str(step)) -def _reset_done(content_path: Path, step: int) -> None: - """Set marker hard to `step` (for re-run from step; step may decrease).""" - if step < 0: - _done_path(content_path).unlink(missing_ok=True) - else: - atomic_write_text(_done_path(content_path), str(step)) # Slot-file globs per step (index = GUIDE_STEPS). Stem-anchored, collision-free. -_STEP_GLOBS = ( - ("outline*",), # 0 Outline (incl. selection filter) - ("content-chunk-*", "content-nach-*"), # 1 Content (incl. follow-up round) - ("content-check-*", "content-fix-*"), # 2 Content-Check - ("chunk-*",), # 3 Writing (chunk-* also matches chunk-nach-*) - ("lese-check-*", "fix-r*"), # 4 Reading-Exam -) -def _reset_guide_from_step(content_path: Path, step: int) -> None: - """Re-run from step: delete content + all slot files of steps ≥ step. - Earlier steps stay → the resume rebuilds from `step` (everything below is reused).""" - content_path.unlink(missing_ok=True) # no longer "done" → no fresh-start wipe - d, stem = content_path.parent, content_path.stem - for globs in _STEP_GLOBS[step:]: - for pat in globs: - for p in d.glob(f"{stem}.{pat}"): - p.unlink(missing_ok=True) - _reset_done(content_path, step - 1) # steps < step count as done -def _read_problems_schema(data): - """{"ok": true} → [] · {"problems": [{"section", "problem"}]} → list · else None.""" - if not isinstance(data, dict): - return None - if data.get("ok") is True: - return [] - p = data.get("problems") - if not isinstance(p, list) or not p: - return None - out = [] - for x in p: - if not isinstance(x, dict) or not isinstance(x.get("section"), str) or not isinstance(x.get("problem"), str): - return None - out.append({"section": x["section"].strip(), "problem": x["problem"].strip()}) - return out or None -def _panel_problems(judge_paths: list[Path], valid: set[int], idx: dict[str, int]) -> dict[int, str]: - """Panel aggregation: several judge outputs of a chunk → flagged {num: problem}. - - One vote per judge that names a section. Flagged when more than half of the - DELIVERED (validly parsed) judges name it (3→≥2, 2→≥2, 1→≥1). Robust against a - single failure: missing files do not count. Problem text from the first naming judge. - """ - outputs = [p for p in (_read_problems_schema(_json_file(j)) for j in judge_paths) if p is not None] - if not outputs: - return {} - votes: dict[int, int] = {} - problem: dict[int, str] = {} - for out in outputs: - seen: set[int] = set() - for item in out: - num = _resolve_title(idx, item["section"]) - if num is None or num not in valid or num in seen: - continue - seen.add(num) - votes[num] = votes.get(num, 0) + 1 - problem.setdefault(num, item["problem"]) - threshold = len(outputs) / 2 - return {num: problem[num] for num, v in votes.items() if v > threshold} -def _resolve_outline(data, entries: dict[int, str], target_min: int, target_max: int) -> list[dict] | None: - """{"chapters": [{"title", "numbers": [1, 3, 7]}]} → [{"title", "nums"}]. - - Numbers are the IDs from `entries` (1-based, as presented to the agent). - `target_min`/`target_max` = allowed range of selected blocks (with a small tolerance). - """ - if not isinstance(data, dict) or not isinstance(data.get("chapters"), list): - return None - valid = set(entries) - chapters: list[dict] = [] - seen: set[int] = set() - total = unknown = 0 - for ch in data["chapters"]: - if not isinstance(ch, dict) or not isinstance(ch.get("numbers"), list): - return None - nums = [] - for t in ch["numbers"]: - total += 1 - num = t if isinstance(t, int) and not isinstance(t, bool) else None - if num is None or num not in valid: - unknown += 1 - elif num not in seen: - nums.append(num) - seen.add(num) - if nums: - chapters.append({"title": str(ch.get("title", "")).strip() or "Chapter", "nums": nums}) - if not chapters or total == 0: - return None - if (total - unknown) / total < 0.85: - return None - if len(seen) < 0.9 * target_min or len(seen) > 1.1 * target_max: - return None - return chapters def _fallback_outline(entries: dict[int, str]) -> list[dict]: @@ -324,515 +194,6 @@ async def _outline_from_db(topic: str, sel_entries: dict[int, str]) -> list[dict return plan or None -async def _generate_sections( - guide_id: str, topic: str, format_name: str, entries: dict[int, str], - facts: str, instructions: str, provider: str, - content_path: Path, -) -> list[dict] | None: - def is_cancelled() -> bool: - return is_guide_cancelled(guide_id) - - ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled, guide_id=guide_id) - spec = (TEMPLATES_DIR / "Format" / "Section.md").read_text(encoding="utf-8") - files = _guide_files(content_path) - zweck = FORMAT_PURPOSE[format_name] - - # Subblocks per block (DB-first) — loaded early: drives selection + sub-filter per format. - # Missing → {} (fallback: guide takes everything). - subs_raw = await _load_subblocks(topic) - # Extract-once grounding: stored, verified facts replace the generic source hint. - # The content agent phrases from them instead of reading the source again. - if (facts_block := _facts_grounding(subs_raw)): - facts = facts_block - - def _has_relevance(num, kind): - return any(isinstance(s, dict) and s.get("relevance") == kind for s in subs_raw.get(_title(entries[num]), [])) - - # Selection: ONE full document with ALL blocks (incl. peripheral). The views E/M/S/F - # filter later per subblock level. (FullGuide/Rest remain as legacy branches.) - if format_name == "Rest": - selection = [num for num in entries if not _has_relevance(num, "relevant")] - else: # Guide / FullGuide → all blocks - selection = list(entries) - if not selection: - await _fail(guide_id, "No matching blocks for this format") - return None - - sel_entries = {num: entries[num] for num in selection} - target = len(sel_entries) - # Numbered list (ID = block number from entries) — agents/judge order by number. - sel_list = "\n".join(f"{num}. {t}" for num, t in sel_entries.items()) - - # Step 0: outline. Prefers the blocks artifact (DB) — the guide only presents, - # no longer structures itself. Missing (legacy) → previous agents/judge logic as fallback. - # 0 valid → code fallback, 1 → direct, ≥2 → judge (with proposal as fallback). - plan = await _outline_from_db(topic, sel_entries) - if plan is not None: - _log(topic, f"Outline from blocks artifact ({len(plan)} chapters)") - if plan is None: - plan = _resolve_outline(_json_file(files["outline"]), sel_entries, target, target) - if plan is None: - await _set_step(guide_id, 0, "Outline proposals (3 agents)…") - files["outline"].unlink(missing_ok=True) - proposals: list[list[dict]] = [] - pending = [] - for i, path in enumerate(files["outline_slots"], 1): - res = _resolve_outline(_json_file(path), sel_entries, target, target) - if res is not None: - proposals.append(res) - else: - pending.append((i, path)) - if len(proposals) < 3 and pending: - slots = [ - { - "key": f"{guide_id}-outline-{i}", - "prompt": _prompt( - "Guide-Outline", - topic=topic, format_name=format_name, blocks=sel_list, - out_path=path, extra=_extra(instructions), - ), - "role": "guide", "capabilities": "files", - "payload": (lambda result, p=path: _resolve_outline(_json_file(p), sel_entries, target, target)), - } - for i, path in pending - ] - # Quorum 1: take whatever comes — no minimum requirement, no abort. - new = await _race( - topic, "Outline", slots, 1, _timeout("plan", target), - provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE, - ) - if is_cancelled(): - return None - proposals += new or [] - - if not proposals: - _log(topic, "Outline: no valid proposal — deterministic fallback") - plan = _fallback_outline(sel_entries) - elif len(proposals) == 1: - plan = proposals[0] # one proposal → no judge needed - else: - await _set_step(guide_id, 0, "Merging outlines…") - proposals_text = "\n\n".join( - f"### Proposal {i}\n" - + "\n".join(f"CHAPTER: {ch['title']}\n Numbers: {', '.join(str(num) for num in ch['nums'])}" for ch in v) - for i, v in enumerate(proposals, 1) - ) - status, plan = await run_single_slot( - ctx, "Outline-Judge", - key=f"{guide_id}-outline-judge", - prompt=_prompt( - "Guide-Outline-Judge", - topic=topic, format_name=format_name, purpose=zweck, n=len(proposals), - blocks=sel_list, outlines=proposals_text, - out_path=files["outline"], extra=_extra(instructions), - ), - role="judge", capabilities="files", - payload=lambda result: _resolve_outline(_json_file(files["outline"]), sel_entries, target, target), - timeout=_timeout("plan_judge", target), - ) - if status == CANCELLED: - return None - if status == FAILED or plan is None: - _log(topic, "Outline judge produced no result — best proposal kept") - plan = proposals[0] - - # Guarantee: every selected block is in the plan (against dropping agents/judges). - plan = _with_remainder(plan, sel_entries) - _set_done(content_path, 0) # outline ready - - # Coarse chunks (~GUIDE_CHUNK blocks per agent) for content, content-check and reading-exam. - # The writer builds per block beneath (its own fine chunks, see below) → variable lengths. - total_sections = sum(len(c["nums"]) for c in plan) - chunks = _split_chunks(plan, max(1, math.ceil(total_sections / GUIDE_CHUNK))) - # Subblocks per block: Guide/FullGuide take ALL (incl. peripheral → level 4 in the view); - # only the legacy Rest branch filters to peripheral. So the one document carries all levels. - if format_name == "Rest": - subs_by_title = {t: [s for s in subs if s.get("relevance") == "peripheral"] for t, subs in subs_raw.items()} - else: # Guide / FullGuide - subs_by_title = {t: list(subs) for t, subs in subs_raw.items()} - subs_by_title = {t: subs for t, subs in subs_by_title.items() if subs} - assignments = [_assignment_subs(chunk, entries, subs_by_title) for chunk in chunks] - chunk_sizes = [sum(len(c["nums"]) for c in chunk) for chunk in chunks] - writer_count = len(chunks) - idx = _title_index(entries) - - # Step 2: identify content per block — one agent per chunk (marker output, resume). - content_paths = [content_path.parent / f"{content_path.stem}.content-chunk-{i}.md" for i in range(1, writer_count + 1)] - pending = [i for i, p in enumerate(content_paths) if not p.exists()] - if pending: - async def report(d, t): await _set_step(guide_id, 1, f"Gathering content {d}/{t}…") - results = await _gather_progress([ - run_agent( - f"{guide_id}-content-{i + 1}", - _prompt( - "Guide-Content", - topic=topic, assignment=assignments[i], facts=facts, - out_path=content_paths[i], extra=_extra(instructions), - ), - _timeout("content", chunk_sizes[i]), provider=provider, role="guide", capabilities="full", - ) - for i in pending - ], writer_count, report, start=writer_count - len(pending)) - if is_cancelled(): - return None - if not any(p.exists() for p in content_paths): - await _fail(guide_id, _gather_error("Content error", list(results))) - return None - - content_by_num: dict[int, str] = {} - for p in content_paths: - if not p.exists(): - continue - for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _resolve_title(idx, sec["title"]) - if num is not None and num not in content_by_num and sec["md"].strip(): - content_by_num[num] = sec["md"] - if not content_by_num: - await _fail(guide_id, "No content identified") - return None - - # Follow-up round: pull missing blocks (chunk failure or lazy output) deliberately — one round. - planned_nums = [num for ch in plan for num in ch["nums"]] - missing = [num for num in planned_nums if num not in content_by_num] - if missing: - _log(topic, f"Content: {len(missing)} block(s) missing — follow-up round…") - followup_chunks = [[{"title": "Additional", "nums": missing[k:k + GUIDE_CHUNK]}] for k in range(0, len(missing), GUIDE_CHUNK)] - followup_paths = [content_path.parent / f"{content_path.stem}.content-nach-{k}.md" for k in range(1, len(followup_chunks) + 1)] - followup_pending = [k for k, p in enumerate(followup_paths) if not p.exists()] - if followup_pending: - async def report_n(d, t): await _set_step(guide_id, 1, f"Gathering missing content {d}/{t}…") - await _gather_progress([ - run_agent( - f"{guide_id}-content-nach-{k + 1}", - _prompt( - "Guide-Content", - topic=topic, assignment=_assignment_subs(followup_chunks[k], entries, subs_by_title), - facts=facts, out_path=followup_paths[k], extra=_extra(instructions), - ), - _timeout("content", len(followup_chunks[k][0]["nums"])), provider=provider, role="guide", capabilities="full", - ) - for k in followup_pending - ], len(followup_chunks), report_n, start=len(followup_chunks) - len(followup_pending)) - if is_cancelled(): - return None - for p in followup_paths: - if not p.exists(): - continue - for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _resolve_title(idx, sec["title"]) - if num is not None and num not in content_by_num and sec["md"].strip(): - content_by_num[num] = sec["md"] - - if all(p.exists() for p in content_paths): - _set_done(content_path, 1) # content complete - - content_chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in content_by_num] for chunk in chunks] - - # Step 3: check content — CHECK_PANEL judges per chunk, majority flags. - # + revise flagged ones once. Resume: only restart missing judge files. - check_judge_paths = [ - [content_path.parent / f"{content_path.stem}.content-check-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)] - for i in range(1, writer_count + 1) - ] - pending_slots = [ - (i, j) for i in range(writer_count) if content_chunk_nums[i] - for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None - ] - if pending_slots: - await _set_step(guide_id, 2, "Checking content…") - sections_per_chunk = { - i: "\n\n".join(f"SECTION: {_title(entries[num])}\n{content_by_num[num]}" for num in content_chunk_nums[i]) - for i, _ in pending_slots - } - slots = [{ - "key": f"{guide_id}-content-check-{i + 1}-j{j + 1}", - "prompt": _prompt( - "Guide-Content-Check", - topic=topic, format_name=format_name, sections=sections_per_chunk[i], - out_path=check_judge_paths[i][j], extra=_extra(instructions), - ), - "role": "judge", "capabilities": "files", - "payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))), - } for i, j in pending_slots] - n_checks = len(slots) - upd = lambda n: asyncio.create_task(_set_step(guide_id, 2, f"Checking content {n}/{n_checks}…")) - await _race(topic, "Content-Exam", slots, len(slots), _timeout("content_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE) - if is_cancelled(): - return None - - problems_by_num: dict[int, str] = {} - for i in range(writer_count): - if content_chunk_nums[i]: - problems_by_num.update(_panel_problems(check_judge_paths[i], set(content_chunk_nums[i]), idx)) - - if problems_by_num: - _log(topic, f"Content exam: {len(problems_by_num)} block(s) flagged") - await _set_step(guide_id, 2, f"Revising {len(problems_by_num)} content(s)…") - fix_chunks = [[num for num in nums if num in problems_by_num] for nums in content_chunk_nums] - fix_paths = [content_path.parent / f"{content_path.stem}.content-fix-{i + 1}.md" for i in range(writer_count)] - fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()] - results = await asyncio.gather(*[ - run_agent( - f"{guide_id}-content-fix-{i + 1}", - _prompt( - "Guide-Content-Fix", - topic=topic, facts=facts, - tasks="\n\n".join( - f"SECTION: {_title(entries[num])}\nPROBLEM: {problems_by_num[num]}\nCURRENT:\n{content_by_num[num]}" - for num in fix_chunks[i] - ), - out_path=fix_paths[i], extra=_extra(instructions), - ), - _timeout("content", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full", - ) - for i in fix_pending - ], return_exceptions=True) - if is_cancelled(): - return None - for p in fix_paths: - if not p.exists(): - continue - for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _resolve_title(idx, sec["title"]) - if num in problems_by_num and sec["md"].strip(): - content_by_num[num] = sec["md"] - - _set_done(content_path, 2) # content check done - - # Step 4: writing — the writer phrases out the checked content (resume). - # FINE chunks: exactly 1 block per writer → variable lengths, no budget rationing. - def content_text(chunk) -> str: - nums = [num for ch in chunk for num in ch["nums"] if num in content_by_num] - return "\n\n".join(f"\n{content_by_num[num]}" for num in nums) - - w_chunks = [[{"title": ch["title"], "nums": [num]}] for ch in plan for num in ch["nums"]] - w_assignments = [_assignment_subs(c, entries, subs_by_title) for c in w_chunks] - paths = [content_path.parent / f"{content_path.stem}.chunk-{i}.md" for i in range(1, len(w_chunks) + 1)] - pending = [i for i, p in enumerate(paths) if not p.exists()] - if pending: - async def report(d, t): await _set_step(guide_id, 3, f"Writing sections {d}/{t}…") - results = await _gather_progress([ - run_agent( - f"{guide_id}-w{i + 1}", - _prompt( - "Guide-Writer", - topic=topic, format_name=format_name, assignment=w_assignments[i], - contents=content_text(w_chunks[i]), - spec=spec, out_path=paths[i], extra=_extra(instructions), - ), - _timeout("writer", 1), provider=provider, role="guide", capabilities="files", - ) - for i in pending - ], len(w_chunks), report, start=len(w_chunks) - len(pending)) - if is_cancelled(): - return None - for i, r in zip(pending, results): - if isinstance(r, BaseException): - _log(topic, f"Writer {i + 1}: {type(r).__name__}: {r}") - elif r[0] != 0: - _log(topic, f"Writer {i + 1}: {_claude_error('Error', *r)}") - elif not paths[i].exists(): - _log(topic, f"Writer {i + 1}: no output file created") - if not any(p.exists() for p in paths): - await _fail(guide_id, _gather_error("Writer error", list(results))) - return None - - by_num: dict[int, dict] = {} - for p in paths: - if not p.exists(): - continue - for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _resolve_title(idx, sec["title"]) - if num is None: - _log(topic, f"Writer produced unknown section '{sec['title'][:40]}' (ignored)") - elif num not in by_num: - by_num[num] = sec - if not by_num: - await _fail(guide_id, "No sections found in writer output") - return None - - # Follow-up round: write missing sections (writer failure) deliberately — one round. - missing_after = [num for num in planned_nums if num not in by_num] - if missing_after: - _log(topic, f"Writing: {len(missing_after)} section(s) missing — follow-up round…") - nw_chunks = [[{"title": "Additional", "nums": [num]}] for num in missing_after] - nw_paths = [content_path.parent / f"{content_path.stem}.chunk-nach-{k}.md" for k in range(1, len(nw_chunks) + 1)] - nw_pending = [k for k, p in enumerate(nw_paths) if not p.exists()] - if nw_pending: - async def report_nw(d, t): await _set_step(guide_id, 3, f"Writing missing sections {d}/{t}…") - await _gather_progress([ - run_agent( - f"{guide_id}-w-nach-{k + 1}", - _prompt( - "Guide-Writer", - topic=topic, format_name=format_name, assignment=_assignment_subs(nw_chunks[k], entries, subs_by_title), - contents=content_text(nw_chunks[k]), spec=spec, out_path=nw_paths[k], extra=_extra(instructions), - ), - _timeout("writer", 1), provider=provider, role="guide", capabilities="files", - ) - for k in nw_pending - ], len(nw_chunks), report_nw, start=len(nw_chunks) - len(nw_pending)) - if is_cancelled(): - return None - for p in nw_paths: - if not p.exists(): - continue - for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _resolve_title(idx, sec["title"]) - if num is not None and num not in by_num and sec["md"].strip(): - by_num[num] = sec - - if all(p.exists() for p in paths): - _set_done(content_path, 3) # writing complete - - # Step 3: reading-exam loop — check per writer packet, fix only for - # flagged sections; follow-up rounds check ONLY the replaced sections. - # After the round cap, open complaints stand. - chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in by_num] for chunk in chunks] - - def sections_text(nums: list[int]) -> str: - return "\n\n".join(f"SECTION: {_title(entries[num])}\n{by_num[num]['md']}" for num in nums) - - def _sub_list(num: int) -> str: - subs = subs_by_title.get(_title(entries[num]), []) - return "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)" - - def tasks_text(nums: list[int], problems: dict[int, str]) -> str: - return "\n\n".join( - f"SECTION: {_title(entries[num])}\n" - f"SUBBLOCKS (set one `` marker each, label/order as here):\n{_sub_list(num)}\n" - f"PROBLEM: {problems[num]}\nCURRENT CONTENT:\n{by_num[num]['md']}" - for num in nums - ) - - scope = chunk_nums - for round_no in range(1, READING_ROUNDS + 1): - # CHECK_PANEL judges per packet; majority flags. Aggregation robust against a single failure. - check_judge_paths = [ - [content_path.parent / f"{content_path.stem}.lese-check-r{round_no}-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)] - for i in range(1, writer_count + 1) - ] - pending_slots = [ - (i, j) for i in range(writer_count) if scope[i] - for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None - ] - if pending_slots: - await _set_step(guide_id, 4, "Checking readability…") - sections_per_chunk = {i: sections_text(scope[i]) for i, _ in pending_slots} - slots = [{ - "key": f"{guide_id}-lese-check-r{round_no}-{i + 1}-j{j + 1}", - "prompt": _prompt( - "Guide-Lese-Check", - topic=topic, format_name=format_name, spec=spec, - sections=sections_per_chunk[i], - out_path=check_judge_paths[i][j], extra=_extra(instructions), - ), - "role": "judge", "capabilities": "files", - "payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))), - } for i, j in pending_slots] - n_checks = len(slots) - upd = lambda n: asyncio.create_task(_set_step(guide_id, 4, f"Checking readability {n}/{n_checks}…")) - res = await _race(topic, f"Reading-Exam r{round_no}", slots, len(slots), _timeout("lese_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE) - if is_cancelled(): - return None - if res is None: - _log(topic, f"Reading exam round {round_no}: no full quorum — aggregated available judges") - - problems_by_num: dict[int, str] = {} - for i in range(writer_count): - if scope[i]: - problems_by_num.update(_panel_problems(check_judge_paths[i], set(scope[i]), idx)) - - # Deterministic readability gate: queue too-hard sections into the same - # revision (LLM complaint takes precedence). Gate off → no-op. - if READABILITY_ACTIVE: - md_by_num = {num: by_num[num]["md"] for nums in scope for num in nums if num in by_num} - hints = await asyncio.to_thread(readability.rate_sections, md_by_num) - if hints: - _log(topic, f"Readability: {len(hints)} section(s) too hard") - for num, hint in hints.items(): - problems_by_num.setdefault(num, hint) - - if not problems_by_num: - break - - _log(topic, f"Reading exam round {round_no}: {len(problems_by_num)} section(s) flagged") - await _set_step(guide_id, 4, f"Revising {len(problems_by_num)} section(s) (round {round_no})…") - fix_chunks = [[num for num in nums if num in problems_by_num] for nums in chunk_nums] - fix_paths = [content_path.parent / f"{content_path.stem}.fix-r{round_no}-{i + 1}.md" for i in range(writer_count)] - fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()] - results = await asyncio.gather(*[ - run_agent( - f"{guide_id}-fix-r{round_no}-w{i + 1}", - _prompt( - "Guide-Sections-Fix", - topic=topic, format_name=format_name, facts=facts, spec=spec, - tasks=tasks_text(fix_chunks[i], problems_by_num), - out_path=fix_paths[i], extra=_extra(instructions), - ), - _timeout("writer", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full", - ) - for i in fix_pending - ], return_exceptions=True) - if is_cancelled(): - return None - for i, r in zip(fix_pending, results): - if isinstance(r, BaseException) or (not isinstance(r, BaseException) and r[0] != 0): - _log(topic, f"Sections fix {i + 1} (round {round_no}) failed — original kept") - replaced: set[int] = set() - for p in fix_paths: - if not p.exists(): - continue - for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _resolve_title(idx, sec["title"]) - if num not in problems_by_num or not sec["md"].strip(): - continue - # Marker invariant: if the fix loses the sub markers although the original had - # some, it is discarded — otherwise the level filter (E/M/S/F) dies silently. - if by_num[num].get("subs") and not sec.get("subs"): - _log(topic, f"Reading fix for '{sec['title']}' without sub markers — discarded, tagged original kept") - continue - by_num[num] = sec - replaced.add(num) - _log(topic, f"Reading exam round {round_no}: {len(replaced)} section(s) revised") - if not replaced: - break - if round_no == READING_ROUNDS: - _log(topic, f"Reading exam: 1 round — revision stays unchecked") - break - scope = [[num for num in nums if num in replaced] for nums in chunk_nums] - _set_done(content_path, 4) # reading exam done - - # Checkable = format has an exam AND the block has ≥1 relevant subblock. - # Guide is always checkable (even without relevance data, fallback = everything). - def _checkable(num): - if format_name == "Guide": - return True - if format_name == "FullGuide": - return any(isinstance(s, dict) and s.get("relevance") == "relevant" - for s in subs_raw.get(_title(entries[num]), [])) - return False # Rest etc. → pure reading sections - - await _set_progress(guide_id, "Assembling…") - chapters: list[dict] = [] - for ch in plan: - sections = [ - {"num": num, "title": _title(entries[num]), "md": by_num[num]["md"], - "compact": by_num[num].get("compact", ""), - "anchor": by_num[num].get("anchor", ""), "anker_compact": by_num[num].get("anker_compact", ""), - "subs": by_num[num].get("subs", []), "checkable": _checkable(num)} - for num in ch["nums"] if num in by_num - ] - if sections: - chapters.append({"title": ch["title"], "sections": sections}) - planned = {num for ch in plan for num in ch["nums"]} - missing = sorted(planned - set(by_num)) - if missing: - _log(topic, f"Sections missing from writer output: {[_title(entries[n]) for n in missing]}") - if not chapters: - await _fail(guide_id, "No sections found in writer output") - return None - return chapters _LEVEL_RANK = {"beginner": 1, "advanced": 2, "expert": 3, "peripheral": 4, @@ -892,14 +253,18 @@ async def generate_guide(guide_id: str, topic: str, format_name: str, instructio if project: await asyncio.to_thread(_convert_pdfs, project) - # Re-run from step: delete content + slots from `ab_step`, rest stays → resume rebuilds from there. - # Otherwise "recreate": a finished guide → complete fresh start. - # Otherwise step files are leftovers of an abort/error → resume. + import guide_board # lazy — guide_board imports helpers from this module + # Re-run from stage: cards from `ab_step` onward back to that column. + # A FINISHED guide without ab_step → complete fresh start (board + slots wiped). + # Otherwise cards are leftovers of an abort/error → resume at their stored stage. if ab_step is not None: - _reset_guide_from_step(content_path, ab_step) + await guide_board.reset_from_stage(topic, format_name, ab_step) elif content_path.exists(): - for p_alt in guide_slot_files(content_path): - p_alt.unlink(missing_ok=True) + counts = await guide_stage_counts(topic, format_name) + if not counts or set(counts) == {"done"}: + await delete_guide_board(topic, format_name) + for p_alt in guide_slot_files(content_path): + p_alt.unlink(missing_ok=True) bs = await list_blocks(topic, status="consensus") if bs: @@ -912,12 +277,13 @@ async def generate_guide(guide_id: str, topic: str, format_name: str, instructio await _fail(guide_id, "No blocks found") return entries = _unique_title(alle) - facts = _prompt("Guide-Facts-Projekt", project=project) if project else _prompt("Guide-Facts-Thema") - chapters = await _generate_sections( - guide_id, topic, format_name, entries, - facts, instructions, provider, content_path, + chapters = await guide_board.run_guide_board( + guide_id, topic, format_name, entries, instructions, provider, content_path, ) - if chapters is None or is_guide_cancelled(guide_id): + if is_guide_cancelled(guide_id): + return + if chapters is None: + await _fail(guide_id, "No finished sections (see board — cards with errors)") return content = {"topic": topic, "format": format_name, "chapters": chapters} diff --git a/backend/guide_board.py b/backend/guide_board.py new file mode 100644 index 0000000..72e996c --- /dev/null +++ b/backend/guide_board.py @@ -0,0 +1,521 @@ +"""Board 3 „Guide": one card per block, linear stages with gates between them. + + lernziele judge Backward Design — objectives BEFORE writing + zuweisung code chapter/order from the outline artefact + facts grounding + writer guide ONE coherent per-block text, only from VERIFIED FACTS + fakten_gate judge CoVe: atomic claims, each binary against the facts → minimal fix + coverage judge objective↔section mapping; gap → back to writer (max 2 rounds) + lesbarkeit judge Lese-Check + deterministic readability gate → fix → done + +Runner: one asyncio task per card (cards are fixed from the start — no queue engine +needed); stage transitions are persisted in guide_cards, so the board is live and +cancel/resume just picks cards up at their stored stage. Assembly keeps the exact +legacy content format → content_fuer_level / TopicDetail stay untouched. +""" + +import asyncio +import json +import logging +import re + +import database as db +import readability +from config import FORMAT_PURPOSE, READABILITY_ACTIVE, TEMPLATES_DIR +from fsutil import atomic_write_json +from jsonio import read_json_file as _json_file +from pipeline import (CANCELLED, FAILED, OK, GenContext, _extra, _log, _prompt, + _timeout, is_guide_cancelled, run_single_slot) +from textkit import _norm_title, _parse_fragment, _title + +log = logging.getLogger("creator.guide_board") + +GUIDE_STAGES = ("lernziele", "zuweisung", "writer", "fakten_gate", "coverage", "lesbarkeit") +STAGE_LABELS = {"lernziele": "Lernziele", "zuweisung": "Zuweisung", "writer": "Writer", + "fakten_gate": "Fakten-Gate", "coverage": "Coverage", + "lesbarkeit": "Lesbarkeit", "done": "Fertig"} +MAX_WRITER_ROUNDS = 2 # coverage → writer feedback loop cap (gains die after round 1–2) +CARD_CONCURRENCY = 10 # simultaneous cards (the per-topic agent semaphore is the hard cap) + + +def _safe(norm: str) -> str: + return re.sub(r"\W+", "_", norm)[:50] or "block" + + +def _ziele_schema(data): + """{"ziele":[{id,text,sub}]} → list of dicts · None on invalid structure.""" + if not isinstance(data, dict) or not isinstance(data.get("ziele"), list) or not data["ziele"]: + return None + out, seen = [], set() + for z in data["ziele"]: + if not isinstance(z, dict): + return None + zid = str(z.get("id", "")).strip() + text = str(z.get("text", "")).strip() + if not zid or not text or zid in seen or len(out) >= 12: + continue + seen.add(zid) + out.append({"id": zid, "text": text, "sub": str(z.get("sub", "")).strip()}) + return out or None + + +def _gate_schema(data): + """{"ok":true} → [] · {"claims":[{text,grund}]} → list · None invalid.""" + if not isinstance(data, dict): + return None + if data.get("ok") is True: + return [] + claims = data.get("claims") + if not isinstance(claims, list) or not claims: + return None + out = [] + for c in claims: + if isinstance(c, dict) and str(c.get("text", "")).strip(): + out.append({"text": str(c["text"]).strip(), "grund": str(c.get("grund", "")).strip()}) + return out + + +def _coverage_schema(data, ziel_ids: set[str]): + """{"ziele":{id:bool}, "luecken":[{ziel,fehlt}], "ballast":[str]} — ziele must cover all ids.""" + if not isinstance(data, dict) or not isinstance(data.get("ziele"), dict): + return None + ziele = {} + for k, v in data["ziele"].items(): + ziele[str(k)] = str(v).strip().casefold() in ("true", "ja", "yes", "1") + if not ziel_ids <= set(ziele): + return None + luecken = [{"ziel": str(l.get("ziel", "")), "fehlt": str(l.get("fehlt", ""))} + for l in data.get("luecken", []) if isinstance(l, dict) and str(l.get("fehlt", "")).strip()] + ballast = [str(b).strip() for b in data.get("ballast", []) if str(b).strip()] + return {"ziele": ziele, "luecken": luecken, "ballast": ballast} + + +def _problems_schema(data): + """Lese-Check: {"ok":true} → [] · {"problems":[{section,problem}]} → [problem…].""" + if not isinstance(data, dict): + return None + if data.get("ok") is True: + return [] + probs = data.get("problems") + if not isinstance(probs, list) or not probs: + return None + out = [str(p.get("problem", "")).strip() for p in probs + if isinstance(p, dict) and str(p.get("problem", "")).strip()] + return out or None + + +def _first_section(md: str) -> dict | None: + secs = _parse_fragment(md) if md else [] + return secs[0] if secs else None + + +class _Env: + """Shared per-run context for the card tasks.""" + + def __init__(self, ctx, guide_id, topic, format_name, instructions, content_path, + subs_by_title, chapter_map, fallback_facts, spec): + self.ctx = ctx + self.guide_id = guide_id + self.topic = topic + self.format = format_name + self.instructions = instructions + self.content_path = content_path + self.subs_by_title = subs_by_title # block title → [sub dicts] + self.chapter_map = chapter_map # block_norm → (chapter title, ord) + self.fallback_facts = fallback_facts # generic source hint (legacy topics without facts) + self.spec = spec + + def slot(self, name: str): + return self.content_path.parent / f"{self.content_path.stem}.{name}" + + +def _card_facts(env: _Env, block_title: str) -> str: + from guide import _facts_grounding # lazy: guide imports this module + grounding = _facts_grounding({block_title: env.subs_by_title.get(block_title, [])}) + return grounding or env.fallback_facts + + +def _card_assignment(env: _Env, card: dict) -> str: + from guide import _level_label + lines = [f"- {card['block']}"] + for s in env.subs_by_title.get(card["block"], []): + lines.append(f" [{_level_label(s)}] {s['title']}") + return "\n".join(lines) + + +async def _set(env: _Env, card: dict, **fields): + card.update(fields) + await db.set_guide_card(env.topic, env.format, card["block_norm"], **fields) + + +# ── Stages ───────────────────────────────────────────────────────────────────────── +async def _stage_lernziele(env: _Env, card: dict) -> bool: + norm = card["block_norm"] + if not await db.list_lernziele(env.topic, norm): + path = env.slot(f"ziele-{_safe(norm)}.json") + subs = "\n".join(f"- [{s.get('level', 'beginner')}] {s['title']}" + for s in env.subs_by_title.get(card["block"], [])) or "(keine)" + status, ziele = await run_single_slot( + env.ctx, f"Lernziele {card['block']}", key=f"{env.guide_id}-ziele-{_safe(norm)}", + prompt=_prompt("Guide-Lernziele", topic=env.topic, block=card["block"], + subs=subs, facts=_card_facts(env, card["block"]), + out_path=path, extra=_extra(env.instructions)), + role="judge", capabilities="files", + payload=lambda result: _ziele_schema(_json_file(path)), + timeout=_timeout("lernziele", len(env.subs_by_title.get(card["block"], [])))) + if status == CANCELLED: + return False + if status == FAILED: + await _set(env, card, status="error", gate_info="Lernziele ohne Ergebnis") + return False + for z in ziele: + await db.put_lernziel(env.topic, norm, z["id"], z["text"], _norm_title(z["sub"])) + await _set(env, card, stage="zuweisung", status="open") + return True + + +async def _stage_zuweisung(env: _Env, card: dict) -> bool: + chapter, ord_ = env.chapter_map.get(card["block_norm"], ("Weitere Inhalte", 10_000)) + await _set(env, card, chapter=chapter, ord=ord_, stage="writer") + return True + + +async def _stage_writer(env: _Env, card: dict) -> bool: + norm = card["block_norm"] + ziele = await db.list_lernziele(env.topic, norm) + ziele_text = "\n".join(f"- ({z['ziel_id']}) {z['text']}" for z in ziele) or "(keine definiert)" + gaps = "" + if card["writer_rounds"] > 0 and card.get("gate_info"): + gaps = ("\nREVISION ROUND — a previous version exists. Revise it: close exactly the gaps " + "below, cut the listed ballast, keep everything else as-is.\n" + f"PREVIOUS VERSION:\n{card.get('md', '')}\n\nGAPS/BALLAST:\n{card['gate_info']}\n") + path = env.slot(f"card-{_safe(norm)}-r{card['writer_rounds']}.md") + path.unlink(missing_ok=True) + + def _payload(result): + text = path.read_text(encoding="utf-8") if path.exists() else "" + sec = _first_section(text) + return text if sec and sec.get("md", "").strip() else None + + status, text = await run_single_slot( + env.ctx, f"Writer {card['block']}", key=f"{env.guide_id}-w-{_safe(norm)}-r{card['writer_rounds']}", + prompt=_prompt("Guide-Writer-Board", topic=env.topic, format_name=env.format, + chapter=card.get("chapter") or "Inhalte", + assignment=_card_assignment(env, card), ziele=ziele_text, + facts=_card_facts(env, card["block"]), gaps=gaps, spec=env.spec, + out_path=path, extra=_extra(env.instructions)), + role="guide", capabilities="files", payload=_payload, + timeout=_timeout("writer", 1)) + if status == CANCELLED: + return False + if status == FAILED: + await _set(env, card, status="error", gate_info="Writer ohne Ergebnis") + return False + await _set(env, card, md=text, stage="fakten_gate", status="open") + return True + + +async def _stage_fakten_gate(env: _Env, card: dict) -> bool: + norm = card["block_norm"] + sec = _first_section(card["md"]) + if sec is None: + await _set(env, card, status="error", gate_info="Writer-Fragment unlesbar") + return False + facts = _card_facts(env, card["block"]) + path = env.slot(f"gate-{_safe(norm)}-r{card['writer_rounds']}.json") + status, claims = await run_single_slot( + env.ctx, f"Fakten-Gate {card['block']}", key=f"{env.guide_id}-gate-{_safe(norm)}-r{card['writer_rounds']}", + prompt=_prompt("Guide-Fakten-Gate", topic=env.topic, block=card["block"], + section=sec["md"], facts=facts, out_path=path, + extra=_extra(env.instructions)), + role="judge", capabilities="files", + payload=lambda result: _gate_schema(_json_file(path)), + timeout=_timeout("fakten_gate", 1)) + if status == CANCELLED: + return False + if status == FAILED: + claims = [] # gate failure must not block the card — logged, text stands + _log(env.topic, f"Fakten-Gate {card['block']}: kein Ergebnis — Text bleibt ungeprüft") + if claims: + _log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} unbelegte Claims → Fix") + fixp = env.slot(f"gatefix-{_safe(norm)}-r{card['writer_rounds']}.md") + fixp.unlink(missing_ok=True) + claims_text = "\n".join(f"- {c['text']}" + (f" ({c['grund']})" if c["grund"] else "") + for c in claims) + + def _fixload(result): + text = fixp.read_text(encoding="utf-8") if fixp.exists() else "" + return text if _first_section(text) else None + + fstatus, fixed = await run_single_slot( + env.ctx, f"Fakten-Fix {card['block']}", key=f"{env.guide_id}-gatefix-{_safe(norm)}-r{card['writer_rounds']}", + prompt=_prompt("Guide-Fakten-Fix", topic=env.topic, block=card["block"], + section=card["md"], claims=claims_text, facts=facts, + out_path=fixp, extra=_extra(env.instructions)), + role="guide", capabilities="files", payload=_fixload, # Fix ≠ Gate-Modell (kein Selbst-Check) + timeout=_timeout("fakten_gate", 1)) + if fstatus == CANCELLED: + return False + if fstatus == OK and fixed: + new_sec = _first_section(fixed) + # marker invariant: a fix that loses the sub markers kills the level filter → discard + if sec.get("subs") and not (new_sec and new_sec.get("subs")): + _log(env.topic, f"Fakten-Fix {card['block']} ohne Sub-Marker — verworfen") + else: + card["md"] = fixed + await _set(env, card, md=card["md"], stage="coverage", status="open") + return True + + +async def _stage_coverage(env: _Env, card: dict) -> bool: + norm = card["block_norm"] + ziele = await db.list_lernziele(env.topic, norm) + if not ziele: + await _set(env, card, stage="lesbarkeit") + return True + sec = _first_section(card["md"]) + ziele_text = "\n".join(f"- ({z['ziel_id']}) {z['text']}" for z in ziele) + path = env.slot(f"coverage-{_safe(norm)}-r{card['writer_rounds']}.json") + ids = {z["ziel_id"] for z in ziele} + status, res = await run_single_slot( + env.ctx, f"Coverage {card['block']}", key=f"{env.guide_id}-cov-{_safe(norm)}-r{card['writer_rounds']}", + prompt=_prompt("Guide-Coverage", topic=env.topic, block=card["block"], + ziele=ziele_text, section=sec["md"] if sec else card["md"], + out_path=path, extra=_extra(env.instructions)), + role="judge", capabilities="files", + payload=lambda result: _coverage_schema(_json_file(path), ids), + timeout=_timeout("coverage", len(ziele))) + if status == CANCELLED: + return False + if status == FAILED: + _log(env.topic, f"Coverage {card['block']}: kein Ergebnis — weiter ohne Gate") + await _set(env, card, stage="lesbarkeit") + return True + for zid, ok in res["ziele"].items(): + if zid in ids: + await db.set_ziel_covered(env.topic, norm, zid, ok) + if res["luecken"] and card["writer_rounds"] < MAX_WRITER_ROUNDS: + info = "\n".join(f"- Lücke ({l['ziel']}): {l['fehlt']}" for l in res["luecken"]) + if res["ballast"]: + info += "\n" + "\n".join(f"- Ballast (kürzen): {b}" for b in res["ballast"]) + _log(env.topic, f"Coverage {card['block']}: {len(res['luecken'])} Lücke(n) → Writer-Runde " + f"{card['writer_rounds'] + 1}") + await _set(env, card, writer_rounds=card["writer_rounds"] + 1, gate_info=info, + stage="writer", status="open") + return True + if res["luecken"]: + _log(env.topic, f"Coverage {card['block']}: Lücken bleiben nach {MAX_WRITER_ROUNDS} Runden") + await _set(env, card, gate_info="", stage="lesbarkeit") + return True + + +async def _stage_lesbarkeit(env: _Env, card: dict) -> bool: + norm = card["block_norm"] + sec = _first_section(card["md"]) + if sec is None: + await _set(env, card, status="error", gate_info="Fragment unlesbar") + return False + problems: list[str] = [] + path = env.slot(f"lese-{_safe(norm)}-r{card['writer_rounds']}.json") + status, res = await run_single_slot( + env.ctx, f"Lese-Check {card['block']}", key=f"{env.guide_id}-lese-{_safe(norm)}", + prompt=_prompt("Guide-Lese-Check", topic=env.topic, format_name=env.format, + spec=env.spec, sections=f"SECTION: {card['block']}\n{sec['md']}", + out_path=path, extra=_extra(env.instructions)), + role="judge", capabilities="files", + payload=lambda result: _problems_schema(_json_file(path)), + timeout=_timeout("lese_check", 1)) + if status == CANCELLED: + return False + if status == OK and res: + problems += res + if READABILITY_ACTIVE: # deterministic gate, external grounding + hints = await asyncio.to_thread(readability.rate_sections, {1: sec["md"]}) + if hints.get(1): + problems.append(hints[1]) + if problems: + from guide import _level_label + subs = env.subs_by_title.get(card["block"], []) + sub_list = "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)" + tasks = (f"SECTION: {card['block']}\n" + f"SUBBLOCKS (set one `` marker each, label/order as here):\n{sub_list}\n" + f"PROBLEM: {' · '.join(problems)}\nCURRENT CONTENT:\n{sec['md']}") + fixp = env.slot(f"lesefix-{_safe(norm)}.md") + fixp.unlink(missing_ok=True) + + def _fixload(result): + text = fixp.read_text(encoding="utf-8") if fixp.exists() else "" + return text if _first_section(text) else None + + fstatus, fixed = await run_single_slot( + env.ctx, f"Lese-Fix {card['block']}", key=f"{env.guide_id}-lesefix-{_safe(norm)}", + prompt=_prompt("Guide-Sections-Fix", topic=env.topic, format_name=env.format, + facts=_card_facts(env, card["block"]), spec=env.spec, tasks=tasks, + out_path=fixp, extra=_extra(env.instructions)), + role="guide", capabilities="files", payload=_fixload, + timeout=_timeout("writer", 1)) + if fstatus == CANCELLED: + return False + if fstatus == OK and fixed: + new_sec = _first_section(fixed) + if sec.get("subs") and not (new_sec and new_sec.get("subs")): + _log(env.topic, f"Lese-Fix {card['block']} ohne Sub-Marker — verworfen") + else: + card["md"] = fixed + await _set(env, card, md=card["md"], stage="done", status="ok", gate_info="") + return True + + +_STAGE_FN = {"lernziele": _stage_lernziele, "zuweisung": _stage_zuweisung, + "writer": _stage_writer, "fakten_gate": _stage_fakten_gate, + "coverage": _stage_coverage, "lesbarkeit": _stage_lesbarkeit} + + +async def _run_card(env: _Env, card: dict, sem: asyncio.Semaphore) -> None: + async with sem: + while card["stage"] != "done": + if is_guide_cancelled(env.guide_id): + await _set(env, card, status="open") # no longer being worked + return + fn = _STAGE_FN.get(card["stage"]) + if fn is None: # unknown stage → park as error + await _set(env, card, status="error", gate_info=f"Unbekannte Stage {card['stage']}") + return + if card["status"] != "active": + await _set(env, card, status="active") # live board: this card is being worked + try: + if not await fn(env, card): + return + except Exception as e: + log.exception("[%s] guide card %s failed", env.topic, card["block"]) + await _set(env, card, status="error", gate_info=f"{type(e).__name__}: {e}"[:300]) + return + + +# ── Orchestration ────────────────────────────────────────────────────────────────── +async def _chapter_map(topic: str, entries: dict[int, str]) -> dict[str, tuple[str, int]]: + """block_norm → (chapter title, global order) from the outline artefact.""" + from guide import _outline_from_db, _fallback_outline, _with_remainder + plan = await _outline_from_db(topic, entries) or _fallback_outline(entries) + plan = _with_remainder(plan, entries) + out: dict[str, tuple[str, int]] = {} + i = 0 + for ch in plan: + for num in ch.get("nums", []): + if num in entries: + out[_norm_title(_title(entries[num]))] = (ch.get("title") or "Kapitel", i) + i += 1 + return out + + +async def run_guide_board(guide_id: str, topic: str, format_name: str, entries: dict[int, str], + instructions: str, provider: str, content_path) -> list[dict] | None: + """Seed one card per block (existing cards keep their stage — resume), run all cards, + assemble the chapters in the legacy content format. → chapters | None (cancel/empty).""" + from blocks import source_folder + from guide import _load_subblocks + ctx = GenContext(topic=topic, provider=provider, + is_cancelled=lambda: is_guide_cancelled(guide_id), guide_id=guide_id) + spec = (TEMPLATES_DIR / "Format" / "Section.md").read_text(encoding="utf-8") + subs_raw = await _load_subblocks(topic) + project = source_folder(topic) + fallback = (_prompt("Guide-Facts-Projekt", project=project) if project + else _prompt("Guide-Facts-Thema")) + env = _Env(ctx, guide_id, topic, format_name, instructions, content_path, + subs_raw, await _chapter_map(topic, entries), fallback, spec) + for num, line in entries.items(): + title = _title(line) + await db.upsert_guide_card(topic, format_name, _norm_title(title), title) + cards = await db.list_guide_cards(topic, format_name) + open_cards = [c for c in cards if c["stage"] != "done"] + if open_cards: + sem = asyncio.Semaphore(CARD_CONCURRENCY) + + async def _progress(): + while True: + counts = await db.guide_stage_counts(topic, format_name) + done = counts.get("done", 0) + total = sum(counts.values()) + await db.update_guide(guide_id, progress=f"Board: {done}/{total} Karten fertig") + await asyncio.sleep(2.0) + + reporter = asyncio.create_task(_progress()) + try: + await asyncio.gather(*[_run_card(env, c, sem) for c in open_cards]) + finally: + reporter.cancel() + if is_guide_cancelled(guide_id): + return None + # assembly — identical shape to the legacy pipeline + cards = await db.list_guide_cards(topic, format_name) + chapters: list[dict] = [] + by_chapter: dict[str, list[dict]] = {} + order: list[str] = [] + for c in sorted(cards, key=lambda c: (c["ord"], c["block_norm"])): + if c["stage"] != "done": + _log(topic, f"Guide: Karte '{c['block']}' nicht fertig ({c['stage']}) — Abschnitt fehlt") + continue + sec = _first_section(c["md"]) + if sec is None: + continue + ch = c["chapter"] or "Inhalte" + if ch not in by_chapter: + by_chapter[ch] = [] + order.append(ch) + by_chapter[ch].append({ + "num": c["ord"], "title": c["block"], "md": sec["md"], + "compact": sec.get("compact", ""), "anchor": sec.get("anchor", ""), + "anker_compact": sec.get("anker_compact", ""), "subs": sec.get("subs", []), + "checkable": format_name == "Guide" or bool( + any(s.get("relevance") == "relevant" for s in subs_raw.get(c["block"], []))), + }) + for ch in order: + chapters.append({"title": ch, "sections": by_chapter[ch]}) + return chapters or None + + +async def done_step(topic: str, format_name: str) -> int: + """Sidebar dots: highest fully completed stage index. -1 = nothing, len(stages) at done.""" + counts = await db.guide_stage_counts(topic, format_name) + if not counts: + return -1 + if set(counts) == {"done"}: + return len(GUIDE_STAGES) + lowest = min(GUIDE_STAGES.index(s) for s in counts if s in GUIDE_STAGES) + return lowest - 1 if lowest > 0 else -1 + + +async def board_snapshot(topic: str, format_name: str, limit: int = 20) -> dict: + """Live guide board: columns with counts + cards (title, rounds, covered objectives).""" + cards = await db.list_guide_cards(topic, format_name) + ziele = {} + for z in await db.list_lernziele(topic): + d = ziele.setdefault(z["block_norm"], [0, 0]) + d[1] += 1 + d[0] += 1 if z["covered"] else 0 + columns = [] + for stage in (*GUIDE_STAGES, "done"): + in_stage = [c for c in cards if c["stage"] == stage] + views = [] + for c in in_stage[:limit]: + zc = ziele.get(c["block_norm"]) + views.append({"title": c["block"], + "status": c["status"] if c["status"] in ("error", "active") else "open", + "rounds": c["writer_rounds"], + "info": c["gate_info"][:200] if c["status"] == "error" else "", + "ziele": f"{zc[0]}/{zc[1]}" if zc else ""}) + columns.append({"key": stage, "label": STAGE_LABELS[stage], + "total": len(in_stage), "cards": views}) + return {"columns": columns} + + +async def reset_from_stage(topic: str, format_name: str, ab_stage: int) -> int: + """Cards in stages ≥ ab_stage (incl. done) back to GUIDE_STAGES[ab_stage].""" + ab_stage = max(0, min(ab_stage, len(GUIDE_STAGES) - 1)) + target = GUIDE_STAGES[ab_stage] + stages = list(GUIDE_STAGES[ab_stage:]) + ["done"] + if ab_stage == 0: + for c in await db.list_guide_cards(topic, format_name): + await db.delete_lernziele(topic, c["block_norm"]) + moved = await db.reset_guide_cards_from_stage(topic, format_name, stages, target, + clear_md=ab_stage <= 2) + return moved diff --git a/backend/kanban.py b/backend/kanban.py new file mode 100644 index 0000000..603bcf1 --- /dev/null +++ b/backend/kanban.py @@ -0,0 +1,265 @@ +"""Generic streaming kanban engine (no concrete stages — boards define those). + +Each column is a worker that pulls cards from its input `stage` (the queue = kanban_cards rows +WHERE stage = ), processes up to KANBAN_BATCH at a time, and advances them. Streaming +columns run continuously; BARRIER columns start only at QUIESCENCE of every stage before them +(no active worker + no queued card). SERIAL columns process one package at a time (their +processor mutates shared cross-card state). + +Failure handling: a processor exception (including parse-fails it raises) sends the package's +unadvanced cards into exponential backoff (retries++, not_before); after MAX_CARD_RETRIES the +card goes to stage 'dead' (dead-letter — visible on the board, requeue-able via API). No card +is ever deleted by the engine. + +Board definitions live in board_inventory.py / board_artefacts.py; run via run_flow(). +""" + +import asyncio +import logging + +import database as db +from config import MAX_CONCURRENT_AGENTS_PER_TOPIC + +log = logging.getLogger("creator.kanban") + +KANBAN_BATCH = 5 # cards a worker pulls per package (micro-batching) +# How many packages ONE worker keeps in flight at once. A worker no longer blocks on a single +# package — it keeps pulling and dispatching until this many run concurrently, so a busy column +# fills the agent slots (the per-topic semaphore is the real cap; over-dispatch just queues cheaply). +WORKER_INFLIGHT = MAX_CONCURRENT_AGENTS_PER_TOPIC +_POLL = 0.3 # seconds between empty-queue polls +MAX_CARD_RETRIES = 3 # failures per card until dead-letter +RETRY_BACKOFF = 30.0 # base seconds; backoff = base · 2^(retries-1) + +# Live registry of running flows (topic → Flow), so routes can attach research agents, +# report `generating`, and cancel. +active_flows: dict[str, "Flow"] = {} + + +class Flow: + """Shared runtime state of one topic run: active-task counters per stage + a wakeup event. + `producers` counts running research agents (initial + any added live); research counts as done + only when ALL producers have finished, so the flow stays awake while extras still search.""" + + def __init__(self, topic: str, work_dir=None): + self.topic = topic + self.work_dir = work_dir + self.active: dict[str, int] = {} + self.producers = 0 + self.producer_tag = 0 + self.stop = False + self.wake = asyncio.Event() + self.spawn_research = None # set by the board: () → coroutine adding one more research agent + self.state: dict = {} # board-private shared state (embedding caches, one-shot flags …) + self.active_cards: set[str] = set() # "board:card_id" currently inside a processor (live display) + + @property + def research_done(self) -> bool: + return self.producers <= 0 + + def add_producer(self): + """MUST be called synchronously BEFORE create_task of the producer — otherwise workers + can pass their exit check in the gap and never see the new producer (quiescence race).""" + self.producers += 1 + self.wake.set() + + def done_producer(self): + self.producers -= 1 + self.wake.set() + + def next_tag(self) -> int: + self.producer_tag += 1 + return self.producer_tag + + def enter(self, stage: str): + self.active[stage] = self.active.get(stage, 0) + 1 + + def leave(self, stage: str): + self.active[stage] = max(0, self.active.get(stage, 0) - 1) + self.wake.set() + + def active_in(self, stages) -> bool: + return any(self.active.get(s, 0) > 0 for s in stages) + + +class Stage: + """One column: board + stage name + processor. `upstream` (all stages before it, across + boards) is filled by chain_stages(). process(cards) gets the pulled package (list of card + dicts with decoded payload). + + barrier: pull only when every upstream stage is quiescent (relational judgements need the + full set). gate: extra callable that must be truthy before the stage pulls (works without + barrier too — e.g. the consensus gate holds cards until research is done so late reader + votes still count). drain: pull the WHOLE queue as one package (global passes like the + fragment filter); implies serial.""" + + def __init__(self, board: str, stage: str, process, *, barrier: bool = False, + serial: bool = False, gate=None, drain: bool = False): + self.board = board + self.stage = stage + self.process = process + self.barrier = barrier + self.serial = serial or drain + self.gate = gate + self.drain = drain + self.upstream: list[str] = [] + + +def chain_stages(stages: list[Stage]) -> list[Stage]: + """Fill each stage's upstream = every stage listed before it (list order = flow order). + Producers are upstream of everything implicitly via flow.research_done.""" + seen: list[str] = [] + for s in stages: + s.upstream = list(seen) + seen.append(s.stage) + return stages + + +async def quiescent(flow: Flow, stages) -> bool: + """True iff no worker is active in `stages` AND no card is queued in any of them. + The barrier/exit condition — must include QUEUED cards, not just active workers, or a worker + could exit in a momentary lull while an upstream worker still has work to push down.""" + if not stages: + return True + if flow.active_in(stages): + return False + return await db.kanban_count(flow.topic, list(stages)) == 0 + + +async def _sleep_wake(flow: Flow): + try: + await asyncio.wait_for(flow.wake.wait(), timeout=_POLL) + except asyncio.TimeoutError: + pass + flow.wake.clear() + + +async def _fail_package(flow: Flow, spec: Stage, cards: list[dict], error: str): + """Backoff/dead-letter for the cards the processor did NOT advance (their stage is unchanged — + advanced cards must not be punished for a failure after their move).""" + for c in cards: + cur = await db.kanban_get_card(flow.topic, spec.board, c["card_id"]) + if cur is None or cur["stage"] != spec.stage: + continue + dead = await db.kanban_fail_card(flow.topic, spec.board, c["card_id"], error, + MAX_CARD_RETRIES, RETRY_BACKOFF) + if dead: + log.warning("kanban %s/%s: card %s → dead (%s)", flow.topic, spec.stage, c["card_id"], error) + + +async def _worker(flow: Flow, spec: Stage, inflight: int, all_stages: list[str]): + """Pull cards from spec.stage, run spec.process — keeping up to `inflight` packages running + CONCURRENTLY so a busy column fills the agent slots. A barrier worker only pulls when upstream + is fully quiescent (and its gate, if any, is open). ANY worker exits only when research is + done and the WHOLE flow is quiescent — global instead of per-stage, so a downstream stage + that feeds cards back upstream (gap-check → ingest) never strands work. Double-checked over + one grace sleep (a producer attached in the lull keeps the flow alive). + + Double-pull safety: each stage has exactly ONE worker, so an in-memory `claimed` set of + card-ids (held while a package runs) keeps concurrent pulls from grabbing the same cards.""" + topic = flow.topic + claimed: set[str] = set() + tasks: set[asyncio.Task] = set() + batch = 100_000 if spec.drain else KANBAN_BATCH + + async def _run(cards): + ids = [c["card_id"] for c in cards] + flow.enter(spec.stage) + flow.active_cards.update(f"{spec.board}:{i}" for i in ids) + try: + await spec.process(cards) + except Exception as e: # one bad package must not kill the worker → backoff/dead-letter + log.info("kanban %s/%s: %s: %s", topic, spec.stage, type(e).__name__, e) + try: + await _fail_package(flow, spec, cards, f"{type(e).__name__}: {e}") + except Exception: + log.exception("kanban %s/%s: fail-handling broke", topic, spec.stage) + finally: + flow.leave(spec.stage) + for i in ids: + claimed.discard(i) + flow.active_cards.discard(f"{spec.board}:{i}") + flow.wake.set() + + async def _idle_exit() -> bool: + return (flow.research_done and not flow.active_in(all_stages) + and await db.kanban_count(topic, all_stages) == 0) + + async def _may_pull() -> bool: + if spec.gate is not None and not spec.gate(): + return False + if not spec.barrier: + return True + return await quiescent(flow, spec.upstream) + + try: + while not flow.stop: + tasks = {t for t in tasks if not t.done()} + # Fill the pipeline: pull fresh cards and dispatch until `inflight` packages run. + if await _may_pull(): + while len(tasks) < inflight: + rows = await db.kanban_pull(topic, spec.board, spec.stage, batch + len(claimed)) + fresh = [r for r in rows if r["card_id"] not in claimed][:batch] + if not fresh: + break + for r in fresh: + claimed.add(r["card_id"]) + tasks.add(asyncio.create_task(_run(list(fresh)))) + if tasks: # busy → wait for a package to finish, then refill + await asyncio.wait(tasks, timeout=_POLL, return_when=asyncio.FIRST_COMPLETED) + continue + # idle: nothing in flight and nothing pulled + if await _idle_exit(): + # Real grace sleep (NOT _sleep_wake — the wake event is usually already set + # by the last package and would collapse the window to 0ms). A producer + # attached during the lull flips research_done and keeps us alive. + await asyncio.sleep(_POLL) + if await _idle_exit(): + return # nothing left and nothing upstream can produce + continue + await _sleep_wake(flow) + finally: + for t in tasks: + t.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + +async def run_flow(flow: Flow, stages: list[Stage], producers=(), set_p=None) -> None: + """Run producers + one worker per stage until global quiescence. `producers` are coroutines + already counted via flow.add_producer() BEFORE this call (quiescence race). Registers the + flow in active_flows for live attach/cancel.""" + active_flows[flow.topic] = flow + names = [s.stage for s in stages] + + def _spawn_workers(): + return [asyncio.ensure_future(_worker(flow, s, 1 if s.serial else WORKER_INFLIGHT, names)) + for s in stages] + + workers = [asyncio.ensure_future(p) for p in producers] + _spawn_workers() + progress = asyncio.create_task(_progress(flow, set_p)) if set_p else None + try: + while True: + await asyncio.gather(*workers, return_exceptions=True) + # Restart round: a producer attached exactly as the workers exited (missed even the + # grace sleep) leaves live producers or queued cards behind → run the workers again. + if flow.stop or (flow.research_done and await quiescent(flow, names)): + break + workers = _spawn_workers() + finally: + flow.stop = True + if progress: + progress.cancel() + if active_flows.get(flow.topic) is flow: + active_flows.pop(flow.topic, None) + + +async def _progress(flow: Flow, set_p): + while not flow.stop: + try: + counts = await db.kanban_stage_counts(flow.topic) + total = sum(n for stages in counts.values() for n in stages.values()) + set_p(f"Kanban: {total} Karten im Fluss") + except Exception: + pass + await asyncio.sleep(1.0) diff --git a/backend/models.py b/backend/models.py index 182638e..52a49f6 100644 --- a/backend/models.py +++ b/backend/models.py @@ -17,7 +17,13 @@ class GuideCreateRequest(BaseModel): format: FormatType instructions: str = Field(default="", max_length=2000) provider: ProviderType = "claude" - ab_step: int | None = Field(default=None, ge=0, le=4) # re-run from guide step (0 outline … 4 read-exam); None = full/resume + ab_step: int | None = Field(default=None, ge=0, le=5) # re-run from board stage (0 lernziele … 5 lesbarkeit); None = full/resume + + +class GuideBoardResetRequest(BaseModel): + topic: str = Field(min_length=1, max_length=100) + format: FormatType = "Guide" + ab_stage: int = Field(ge=0, le=5) # reset cards back to this board stage (no generation) class TopicCreateRequest(BaseModel): @@ -30,14 +36,13 @@ class BlocksCreateRequest(BaseModel): provider: ProviderType = "claude" source_type: SourceType = "thema" source_location: str = Field(default="", max_length=2000) - ab_phase: int | None = Field(default=None, ge=1, le=9) # re-run from a coarse phase (position in _phasen(topic), 1-based; up to 9: …outline/questions/artifacts); None = resume/continue without deleting - ab_step: int | None = Field(default=None, ge=0) # re-run from a fine sub-step (0-based index into _blocks_steps); takes precedence over ab_phase - to_step: int | None = Field(default=None, ge=0) # stop AFTER this fine sub-step (0-based index into _blocks_steps); None = run to the end + research: bool = True # False = Continue: drain the existing kanban queue, no new search -class BlocksResetStepRequest(BaseModel): +class BlocksResetStageRequest(BaseModel): topic: str = Field(min_length=1, max_length=100) - ab_step: int = Field(ge=0) # ONLY reset from this sub-step (no regeneration) + board: Literal["inventory", "artefacts"] + stage: str = Field(min_length=1, max_length=40) # kanban column to reset back to class BlocksStep(BaseModel): diff --git a/backend/pipeline.py b/backend/pipeline.py index ecfdea6..02c4f8f 100644 --- a/backend/pipeline.py +++ b/backend/pipeline.py @@ -167,7 +167,7 @@ _yesno_schema = _enum_map_schema("relevant", _YESNO) # triage gate _MAX_RESTARTS = 2 -async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: int, provider: str, on_update=None, cancelled=None, *, grace: int | None = None) -> list | None: +async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: int, provider: str, on_update=None, cancelled=None, *, grace: int | None = None, min_runtime: int | None = None, max_runtime: int | None = None) -> list | None: """Starts all slots in parallel and collects `quorum` valid results. Slot spec: {key, prompt, role, capabilities, payload}. `payload(result)` @@ -180,10 +180,18 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: a timer of `grace` seconds. After it expires, running agents are only killed if the minimum stands — otherwise the race, including restarts, keeps running until it stands. Returns: `quorum` to `len(slots)` results. + + `min_runtime` (wall-clock from start): the race does not return before it + elapses while agents are still running — gives them time to search thoroughly. + `max_runtime` (wall-clock from start): hard cap — returns whatever is collected + (or None if nothing), killing the rest. Both default off; only Research sets them. """ attempts = {i: 0 for i in range(len(slots))} tasks: dict[asyncio.Task, int] = {} loop = asyncio.get_running_loop() + start = loop.time() + min_deadline = start + min_runtime if min_runtime else None + max_deadline = start + max_runtime if max_runtime else None deadline: float | None = None def spawn(i: int) -> None: @@ -191,6 +199,7 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: task = asyncio.create_task(run_agent( slot["key"], slot["prompt"], timeout, provider=provider, role=slot["role"], capabilities=slot["capabilities"], + scope=topic, on_line=slot.get("on_line"), )) tasks[task] = i @@ -202,12 +211,22 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: while tasks: if cancelled and cancelled(): return None - if deadline is not None and len(results) >= quorum and loop.time() >= deadline: + # Hard wall-clock cap: return whatever we have (None if empty), kill the rest. + if max_deadline is not None and loop.time() >= max_deadline: + _log(topic, f"{label}: max runtime {max_runtime}s reached ({len(results)} valid)") + return results or None + min_ok = min_deadline is None or loop.time() >= min_deadline + if deadline is not None and len(results) >= quorum and loop.time() >= deadline and min_ok: return results - # Grace set and minimum reached → only wait for the remaining deadline - wait_timeout = None + # Wake up for the earliest relevant deadline (grace, min, or max). + waits = [] if deadline is not None and len(results) >= quorum: - wait_timeout = max(0.0, deadline - loop.time()) + waits.append(deadline - loop.time()) + if min_deadline is not None: + waits.append(min_deadline - loop.time()) + if max_deadline is not None: + waits.append(max_deadline - loop.time()) + wait_timeout = max(0.0, min(waits)) if waits else None done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED, timeout=wait_timeout) if not done: continue @@ -234,7 +253,8 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: _log(topic, f"{label}: first result — grace {grace}s running") if on_update: on_update(len(results)) - if len(results) >= quorum and (grace is None or loop.time() >= deadline): + if (len(results) >= quorum and (grace is None or loop.time() >= deadline) + and (min_deadline is None or loop.time() >= min_deadline)): return results continue @@ -272,13 +292,13 @@ OK, CANCELLED, FAILED = "ok", "cancelled", "failed" async def run_single_slot( ctx: GenContext, label: str, *, - key: str, prompt: str, role: str, capabilities: str, payload, timeout: int, + key: str, prompt: str, role: str, capabilities: str, payload, timeout: int, on_line=None, ) -> tuple[str, object]: """One agent, one valid result (race with quorum 1). → (OK, value) | (CANCELLED, None) | (FAILED, None) """ - slots = [{"key": key, "prompt": prompt, "role": role, "capabilities": capabilities, "payload": payload}] + slots = [{"key": key, "prompt": prompt, "role": role, "capabilities": capabilities, "payload": payload, "on_line": on_line}] res = await _race(ctx.topic, label, slots, 1, timeout, ctx.provider, cancelled=ctx.is_cancelled) if ctx.is_cancelled(): return CANCELLED, None diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 0000000..78c5011 --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +asyncio_mode = auto +testpaths = tests diff --git a/backend/routes.py b/backend/routes.py index 7404be1..54738c2 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -7,7 +7,7 @@ from datetime import datetime, timezone from fastapi import APIRouter, HTTPException from fastapi.responses import Response -from agents import provider_available +from agents import active_agents, provider_available from config import PROJECTS_DIR, UNI_DIR, PROVIDERS from database import ( create_guide, delete_guide, get_guide, list_guides, @@ -18,19 +18,20 @@ from database import ( set_block_score_and_streak, set_block_completed, delete_block_data, delete_block_progress, subs_per_level, subs_per_level_raw, delete_topic_pipeline, delete_source, get_guide_content, delete_guide_content, - get_sub_artefakte, + get_sub_artefakte, kanban_reset, delete_guide_board, ) -from blocks import generate_blocks, cancel_blocks, blocks_status, active_blocks, reset_blocks, reset_blocks_ab_step, load_source, load_overview, subblocks_title, subblocks_frei, load_question_pattern, load_question_pattern_free +from blocks import generate_blocks, cancel_blocks, blocks_status, active_blocks, reset_blocks, load_source, load_overview, subblocks_title, subblocks_frei, load_question_pattern, load_question_pattern_free, _blocks_files +from board_inventory import add_research_agent, board_snapshot, requeue_dead, reset_board_from_stage from elements import generate_element, chat_with_guide, chat_with_element, check_element, style_element, refine_suggestion from learning import block_chat, block_discussion, create_block_element, exam_rating, exam_rating_fast, exam_question, exam_question_variant, generate_quiz, generate_gapchoice, generate_gaptext, check_gaptext, hurdles_distractor_block, compute_score, floor_from_score, level_from_score, cap_final, cap_aktuell, freie_level, thresholds, points_delta, cap_followup -from guide import generate_guide, guide_slot_files, guide_done_step, block_pruefen, block_adopt, content_fuer_level +from guide import generate_guide, guide_slot_files, block_pruefen, block_adopt, content_fuer_level from pipeline import cancel_guide from rules import FORMATE, formats_stats, guide_lock, ist_completed, load_learnstate, topic_completed from models import ( GuideCreateRequest, GuideResponse, TopicCreateRequest, - BlocksCreateRequest, BlocksResetStepRequest, BlocksStatusResponse, - GuideChatRequest, GuideChatResponse, + BlocksCreateRequest, BlocksResetStageRequest, BlocksStatusResponse, + GuideBoardResetRequest, GuideChatRequest, GuideChatResponse, ElementCreateRequest, ElementChatRequest, ElementChatResponse, ElementResponse, ElementUpdateRequest, ElementCheckRequest, ElementCheckResponse, ElementStyleResponse, ElementRefineRequest, ElementRefineResponse, @@ -164,10 +165,55 @@ async def create_blocks(req: BlocksCreateRequest): raise HTTPException(400, "Link must start with http:// or https://.") qp.parent.mkdir(parents=True, exist_ok=True) atomic_write_json(qp, {"type": type, "location": location, "spec": req.instructions.strip()}) - asyncio.create_task(generate_blocks(topic, req.instructions.strip(), req.provider, ab_phase=req.ab_phase, ab_step=req.ab_step, to_step=req.to_step)) + asyncio.create_task(generate_blocks(topic, req.instructions.strip(), req.provider, research=req.research)) return {"ok": True} +@router.get("/blocks/board") +async def get_blocks_board(topic: str): + """Live kanban board: columns with counts + newest cards, dead-letter, agents.""" + snap = await board_snapshot(topic) + status = await blocks_status(topic) + snap["generating"] = status["generating"] + snap["progress"] = status["progress"] + snap["error"] = status["error"] + snap["agents"] = [{"label": a["key"].removeprefix(f"blocks-{topic}-"), "runtime": a["runtime"]} + for a in active_agents(f"blocks-{topic}-")] + return snap + + +@router.get("/blocks/agents") +async def get_blocks_agents(topic: str): + return [{"label": a["key"].removeprefix(f"blocks-{topic}-"), "runtime": a["runtime"]} + for a in active_agents(f"blocks-{topic}-")] + + +@router.post("/blocks/research") +async def add_blocks_research(topic: str, provider: str = "claude"): + """Attach one more research agent — to the live flow, or attach-or-start.""" + if add_research_agent(topic): + return {"ok": True, "attached": True} + if (await blocks_status(topic))["generating"]: + return {"ok": False, "status": "starting"} # flow is booting, try again shortly + asyncio.create_task(generate_blocks(topic, "", provider, research=True)) + return {"ok": True, "attached": False} + + +@router.post("/blocks/reset-stage") +async def reset_blocks_stage(req: BlocksResetStageRequest): + """Reset cards from a column onward back to that column (no regeneration).""" + topic = req.topic.strip() + if (await blocks_status(topic))["generating"]: + return {"ok": True, "status": "generating"} # don't interfere with a running generation + moved = await reset_board_from_stage(topic, req.board, req.stage, _blocks_files(topic)) + return {"ok": True, "moved": moved} + + +@router.post("/blocks/requeue-dead") +async def requeue_blocks_dead(topic: str): + return {"ok": True, "requeued": await requeue_dead(topic)} + + @router.post("/blocks/cancel") async def cancel_blocks_route(topic: str): if not cancel_blocks(topic): @@ -179,15 +225,7 @@ async def cancel_blocks_route(topic: str): async def remove_blocks(topic: str): reset_blocks(topic) # Files: crawl + triage + inventory…questions gone; source.json stays await delete_topic_pipeline(topic) # DB: blocks area gone; topic config (source) stays - return {"ok": True} - - -@router.post("/blocks/reset-step") -async def reset_blocks_step(req: BlocksResetStepRequest): - topic = req.topic.strip() - if (await blocks_status(topic))["generating"]: - return {"ok": True, "status": "generating"} # don't interfere with a running generation - await reset_blocks_ab_step(topic, req.ab_step) + await kanban_reset(topic) # kanban cards + cluster membership gone return {"ok": True} @@ -509,9 +547,46 @@ async def guide_locks(topic: str): @router.get("/guides/steps") async def guide_steps(topic: str): - """Highest fully completed step index per format (artifact-based, -1 = none). - Drives the clickable step bubbles (like the blocks phases).""" - return {fmt: guide_done_step(guide_content_path(topic, fmt)) for fmt in ("Guide", "FullGuide", "Rest")} + """Highest fully completed stage index per format (card-based, -1 = none). + Content file present (legacy without cards) → everything done.""" + import guide_board + out = {} + for fmt in ("Guide", "FullGuide", "Rest"): + step = await guide_board.done_step(topic, fmt) + if step < 0 and guide_content_path(topic, fmt).exists(): + step = len(guide_board.GUIDE_STAGES) + out[fmt] = step + return out + + +@router.get("/guides/board") +async def get_guide_board(topic: str, format: str = "Guide"): + """Live guide board: columns with counts + cards (rounds, covered objectives), agents.""" + import guide_board + snap = await guide_board.board_snapshot(topic, format) + guide = next((g for g in await list_guides() + if g["topic"] == topic and g["format"] == format), None) + snap["generating"] = bool(guide and guide["status"] in ("queued", "generating")) + snap["guide_id"] = guide["id"] if guide else None + snap["progress"] = guide.get("progress") if guide else None + snap["error"] = guide.get("error_msg") if guide else None + prefix = f"{guide['id']}-" if guide else "-" + snap["agents"] = [{"label": a["key"].removeprefix(prefix), "runtime": a["runtime"]} + for a in active_agents(prefix)] + return snap + + +@router.post("/guides/board/reset") +async def reset_guide_board(req: GuideBoardResetRequest): + """Reset cards from a stage onward — without generation (pendant to blocks reset-stage).""" + import guide_board + topic = req.topic.strip() + guide = next((g for g in await list_guides() + if g["topic"] == topic and g["format"] == req.format), None) + if guide and guide["status"] in ("queued", "generating"): + return {"ok": True, "status": "generating"} + moved = await guide_board.reset_from_stage(topic, req.format, req.ab_stage) + return {"ok": True, "moved": moved} @router.get("/guides/{guide_id}", response_model=GuideResponse) @@ -680,6 +755,7 @@ async def remove(guide_id: str, slots: bool = False): rest = [g for g in await list_guides() if g["topic"] == guide["topic"] and g["format"] == guide["format"]] if not rest: await delete_guide_content(guide["topic"], guide["format"]) + await delete_guide_board(guide["topic"], guide["format"]) # board cards + lernziele content = guide_content_path(guide["topic"], guide["format"]) if slots or content.exists(): for p in guide_slot_files(content): diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..5ecd4f7 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,18 @@ +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import database # noqa: E402 + + +@pytest.fixture +async def testdb(tmp_path, monkeypatch): + """Fresh sqlite file per test; resets the module-global connection.""" + monkeypatch.setattr(database, "DB_PATH", tmp_path / "test.db") + database._db = None + await database.init_db() + yield database + await database.close_db() diff --git a/backend/tests/test_board_inventory.py b/backend/tests/test_board_inventory.py new file mode 100644 index 0000000..d76678c --- /dev/null +++ b/backend/tests/test_board_inventory.py @@ -0,0 +1,210 @@ +"""Board 1 end-to-end through the real engine — agents faked, no LLM, no embedding model.""" + +import json +import re + +import pytest + +import board_inventory as bi +import kanban +from pipeline import GenContext + +TOPIC = "t" +B = bi.BOARD + +_PATH_RE = re.compile(r"(/\S+\.json)") + + +def _fake_single_slot(tmp_path): + """Deterministic judge stand-in: writes the expected JSON to the out_path found in the + prompt and returns it — keyed off the agent-key naming convention.""" + + async def fake(ctx, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None): + m = _PATH_RE.search(prompt) + out = None + if "-pair-" in key: + pairs = prompt.count("\nA: ") + out = {"pairs": {str(i + 1): "ja" for i in range(pairs)}} + elif "-clarify-" in key: + keep = [line[2:].split(" — ")[0] for line in prompt.splitlines() + if line.startswith("- ")] + out = {"keep": keep, "rest": []} + elif "-naming-" in key: + out = {"best": 1} + elif "-filter-recheck-" in key or "-filter-" in key: + out = {"fragments": {}, "drop": []} + elif "-gruppierung-completion-" in key: + out = {"additions": []} + elif "-gruppierung-" in key: + out = {"umbrellas": []} + elif "-supplement" in key: + out = {"blocks": [{"title": "Zeta-Konzept", "description": "kanonisch fehlend"}]} + if m and out is not None: + with open(m.group(1), "w", encoding="utf-8") as f: + json.dump(out, f) + return "ok", payload(None) + return fake + + +@pytest.fixture +async def board_env(testdb, tmp_path, monkeypatch): + import board_artefacts as ba + from textkit import _norm_title + + monkeypatch.setattr(bi, "run_single_slot", _fake_single_slot(tmp_path)) + + async def no_emb(flow): + return False + monkeypatch.setattr(bi, "_emb_ok", no_emb) + + async def fake_subblocks(ctx, set_p, files, entries, instructions, wipe=True, ns=""): + title = list(entries.values())[0].split(" — ")[0] + return {title: ["Sub Eins", "Sub Zwei"]} + + async def fake_facts(ctx, set_p, files, raw, q, folder, instructions, ns=""): + facts = {t: {_norm_title(s): {"key_points": [f"Fakt zu {s}"], "cited_facts": []} + for s in subs} for t, subs in raw.items()} + return facts, {} + + async def fake_levels(ctx, set_p, files, raw, instructions, ns=""): + return {t: [{"title": s, "level": "beginner"} for s in subs] for t, subs in raw.items()} + + async def fake_relevance(ctx, set_p, files, sidecar, instructions, ns=""): + return {1: "relevant", 2: "peripheral"} + + async def fake_pattern(ctx, set_p, files, sidecar, instructions, ns=""): + return {t: [{"subblock": subs[0]["title"], "question": f"Was ist {t}?"}] + for t, subs in sidecar.items()} + + async def fake_artefacts(ctx, set_p, files, sidecar, instructions, ns=""): + return {"flashcard": [{"block": t, "subblock": subs[0]["title"], "front": "F", "back": "B"} + for t, subs in sidecar.items()], "example": []} + + async def fake_outline(ctx, set_p, files, entries, instructions): + return {"chapters": [{"title": "Kapitel 1", "numbers": sorted(entries)}]} + + for name, fn in [("_subblocks_block", fake_subblocks), ("_facts_block", fake_facts), + ("_levels_block", fake_levels), ("_relevance_block", fake_relevance), + ("_question_pattern_block", fake_pattern), ("_artefacts_block", fake_artefacts), + ("_outline_block", fake_outline)]: + monkeypatch.setattr(ba, name, fn) + + work = tmp_path / "arbeit" + work.mkdir() + files = {"arbeit": work, "final": tmp_path / "blocks.md", + "sub_roh": tmp_path / "sub_roh.json", "sidecar": tmp_path / "subblocks.json", + "facts": tmp_path / "facts.json", "question_pattern": tmp_path / "question_pattern.json", + "artefakte": tmp_path / "artefakte.json", "outline": tmp_path / "outline.json", + "outline_slots": tmp_path / "outline_slots"} + ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False) + return testdb, ctx, files + + +async def _seed(db): + # 2 consensus titles (2 readers), 1 single find (kept by panel), 1 artifact (pre-reject) + await db.kanban_add_title(TOPIC, B, "alpha", "Alpha", "Grundkonzept", "s1", "r1") + await db.kanban_add_title(TOPIC, B, "alpha", "Alpha", "Grundkonzept", "s2", "r2") + await db.kanban_add_title(TOPIC, B, "beta", "Beta", "Zweites Konzept", "s1", "r1") + await db.kanban_add_title(TOPIC, B, "beta", "Beta", "Zweites Konzept", "s2", "r2") + await db.kanban_add_title(TOPIC, B, "gamma", "Gamma", "Einzelfund", "s1", "r1") + await db.kanban_add_title(TOPIC, B, "aufgabe 3", "Aufgabe 3", "Übungszettel", "s1", "r1") + + +async def test_board1_full_flow(board_env): + db, ctx, files = board_env + await _seed(db) + import asyncio + ok = await asyncio.wait_for( + bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "", research=False), + timeout=30) + assert ok + done = await db.kanban_cards(TOPIC, board=B, stage="done_block") + titles = sorted(c["payload"]["title"] for c in done) + # Alpha/Beta/Gamma survive; the supplement's Zeta flows through the whole board too + assert titles == ["Alpha", "Beta", "Gamma", "Zeta-Konzept"] + rejected = await db.kanban_cards(TOPIC, board=B, stage="rejected") + assert [c["payload"]["title"] for c in rejected] == ["Aufgabe 3"] + assert rejected[0]["payload"]["reason"] == "pre-reject" + # legacy mirror carries the survivors as consensus + legacy = {b["title"] for b in await db.list_blocks(TOPIC, status="consensus")} + assert legacy == set(titles) + # blocks.md written in flow order + lines = files["final"].read_text(encoding="utf-8").strip().splitlines() + assert len(lines) == 4 + # reader union survived the pipeline (consensus evidence on the block card) + alpha = next(c for c in done if c["payload"]["title"] == "Alpha") + assert set(alpha["payload"]["readers"]) == {"r1", "r2"} + # board 2: one artefact card per block ran through to done_artefact (+ outline singleton) + art_done = await db.kanban_cards(TOPIC, board="artefacts", stage="done_artefact") + assert len(art_done) == 5 # 4 blocks + outline card + sidecar = json.loads(files["sidecar"].read_text(encoding="utf-8")) + assert set(sidecar) == set(titles) + assert sidecar["Alpha"][0]["facts"]["key_points"] == ["Fakt zu Sub Eins"] + assert sidecar["Alpha"][0]["relevance"] == "relevant" + assert sidecar["Alpha"][1]["relevance"] == "peripheral" + # DB mirrors: subblocks, question pattern, artefacts, outline + subs = await db.list_subblocks(TOPIC, "alpha") + assert {s["sub_title"] for s in subs} == {"Sub Eins", "Sub Zwei"} + outline = await db.get_outline(TOPIC) + assert outline and "Kapitel 1" in outline + + +async def test_filter_judges_run_parallel(board_env, monkeypatch): + """40 Blöcke → 2 Filter-Chunks: die Judge-Welle muss parallel laufen (Perf-Fix).""" + import asyncio + db, ctx, files = board_env + state = {"cur": 0, "max": 0} + base = bi.run_single_slot # instant fake from the fixture + + async def slow(*a, **k): + state["cur"] += 1 + state["max"] = max(state["max"], state["cur"]) + try: + await asyncio.sleep(0.05) + return await base(*a, **k) + finally: + state["cur"] -= 1 + + monkeypatch.setattr(bi, "run_single_slot", slow) + for i in range(40): + await db.kanban_upsert_card(TOPIC, B, f"b-x{i}", "block", "fragment_filter", + {"title": f"Block {i}", "description": "d"}) + ok = await asyncio.wait_for( + bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "", research=False), + timeout=30) + assert ok + # 40 seeded + 1 supplement candidate (Zeta) flow through to done_block + assert await db.kanban_count(TOPIC, "done_block", board=B) == 41 + assert state["max"] >= 2 # chunk judges ran as one wave, not sequentially + + +async def test_empty_subblocks_completes_without_deadletter(board_env, monkeypatch): + """Legitim leere Subbausteine ({} statt None) → Karte läuft bis done_artefact durch.""" + import asyncio + import board_artefacts as ba + import blocks as blx + db, ctx, files = board_env + + async def empty_subs(ctx, set_p, files, entries, instructions, wipe=True, ns=""): + return {} + monkeypatch.setattr(ba, "_subblocks_block", empty_subs) + await db.kanban_upsert_card(TOPIC, "artefacts", "leer", "ablock", "subblocks", + {"title": "Leerer Block", "description": "d"}) + ok = await asyncio.wait_for( + bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "", research=False), + timeout=30) + assert ok + card = await db.kanban_get_card(TOPIC, "artefacts", "leer") + assert card["stage"] == "done_artefact" + assert card["retries"] == 0 and not card.get("last_error") + assert TOPIC not in blx._blocks_errors # kein globales Banner + + +async def test_reader_union_folds_exact_dupes(testdb): + db = testdb + assert await db.kanban_add_title(TOPIC, B, "x", "X", "d", "s1", "r1") is True + assert await db.kanban_add_title(TOPIC, B, "x", "X", "d länger", "s2", "r2") is False + card = await db.kanban_get_card(TOPIC, B, "x") + assert set(card["payload"]["readers"]) == {"r1", "r2"} + assert set(card["payload"]["sources"]) == {"s1", "s2"} + assert card["payload"]["description"] == "d länger" diff --git a/backend/tests/test_guide_board.py b/backend/tests/test_guide_board.py new file mode 100644 index 0000000..2a37334 --- /dev/null +++ b/backend/tests/test_guide_board.py @@ -0,0 +1,68 @@ +"""Guide board: schema parsers + card reset semantics (no LLM).""" + +import guide_board as gb + +TOPIC, FMT = "t", "Guide" + + +def test_ziele_schema(): + ok = gb._ziele_schema({"ziele": [{"id": "z1", "text": "Erklären, warum X", "sub": "S"}, + {"id": "z2", "text": "Nennen von Y"}]}) + assert [z["id"] for z in ok] == ["z1", "z2"] + assert gb._ziele_schema({"ziele": []}) is None + assert gb._ziele_schema({"ziele": [{"id": "z1", "text": "a"}, {"id": "z1", "text": "b"}]}) \ + == [{"id": "z1", "text": "a", "sub": ""}] # duplicate ids fold + assert gb._ziele_schema("quatsch") is None + + +def test_gate_schema(): + assert gb._gate_schema({"ok": True}) == [] + claims = gb._gate_schema({"claims": [{"text": "Falsch", "grund": "fehlt"}]}) + assert claims == [{"text": "Falsch", "grund": "fehlt"}] + assert gb._gate_schema({}) is None + + +def test_coverage_schema(): + res = gb._coverage_schema({"ziele": {"z1": True, "z2": "false"}, + "luecken": [{"ziel": "z2", "fehlt": "Beweis"}], + "ballast": ["Abschweifung"]}, {"z1", "z2"}) + assert res["ziele"] == {"z1": True, "z2": False} + assert res["luecken"][0]["fehlt"] == "Beweis" + assert gb._coverage_schema({"ziele": {"z1": True}}, {"z1", "z2"}) is None # z2 missing + + +def test_problems_schema(): + assert gb._problems_schema({"ok": True}) == [] + assert gb._problems_schema({"problems": [{"section": "S", "problem": "zu lang"}]}) == ["zu lang"] + assert gb._problems_schema({"problems": []}) is None + + +async def test_reset_from_stage(testdb): + db = testdb + await db.upsert_guide_card(TOPIC, FMT, "a", "A") + await db.upsert_guide_card(TOPIC, FMT, "b", "B") + await db.set_guide_card(TOPIC, FMT, "a", stage="done", md="text", writer_rounds=2) + await db.set_guide_card(TOPIC, FMT, "b", stage="coverage", md="text") + await db.put_lernziel(TOPIC, "a", "z1", "Ziel") + # reset ab writer (idx 2): beide Karten zurück, md geleert, Ziele bleiben + moved = await gb.reset_from_stage(TOPIC, FMT, 2) + assert moved == 2 + cards = {c["block_norm"]: c for c in await db.list_guide_cards(TOPIC, FMT)} + assert cards["a"]["stage"] == "writer" and cards["a"]["md"] == "" and cards["a"]["writer_rounds"] == 0 + assert cards["b"]["stage"] == "writer" + assert await db.list_lernziele(TOPIC, "a") + # reset ab lernziele (idx 0): Ziele weg + await gb.reset_from_stage(TOPIC, FMT, 0) + assert not await db.list_lernziele(TOPIC, "a") + assert (await db.list_guide_cards(TOPIC, FMT))[0]["stage"] == "lernziele" + + +async def test_done_step(testdb): + db = testdb + assert await gb.done_step(TOPIC, FMT) == -1 + await db.upsert_guide_card(TOPIC, FMT, "a", "A") + assert await gb.done_step(TOPIC, FMT) == -1 # alles in lernziele + await db.set_guide_card(TOPIC, FMT, "a", stage="coverage") + assert await gb.done_step(TOPIC, FMT) == 3 # bis fakten_gate fertig + await db.set_guide_card(TOPIC, FMT, "a", stage="done") + assert await gb.done_step(TOPIC, FMT) == len(gb.GUIDE_STAGES) diff --git a/backend/tests/test_kanban.py b/backend/tests/test_kanban.py new file mode 100644 index 0000000..6e12636 --- /dev/null +++ b/backend/tests/test_kanban.py @@ -0,0 +1,128 @@ +"""Engine tests with fake processors (no LLM): flow, barrier, retry/dead-letter, producer race.""" + +import asyncio + +import pytest + +import kanban +from kanban import Flow, Stage, chain_stages, run_flow + +TOPIC = "t" +BOARD = "inventory" + + +def _advance_proc(db, to_stage): + async def proc(cards): + await db.kanban_advance_many(TOPIC, BOARD, [(c["card_id"], to_stage) for c in cards]) + return proc + + +async def _seed(db, n, stage="s1"): + for i in range(n): + await db.kanban_upsert_card(TOPIC, BOARD, f"card-{i}", "title", stage, {"title": f"T{i}"}) + + +async def test_cards_flow_through_stages(testdb): + db = testdb + await _seed(db, 7) + flow = Flow(TOPIC) + stages = chain_stages([ + Stage(BOARD, "s1", _advance_proc(db, "s2")), + Stage(BOARD, "s2", _advance_proc(db, "done")), + ]) + await asyncio.wait_for(run_flow(flow, stages), timeout=10) + assert await db.kanban_count(TOPIC, "done", board=BOARD) == 7 + assert await db.kanban_count(TOPIC, ["s1", "s2"], board=BOARD) == 0 + + +async def test_barrier_waits_for_upstream(testdb): + db = testdb + await _seed(db, 6) + upstream_left: list[int] = [] + + async def slow_s1(cards): + await asyncio.sleep(0.05) # keep upstream busy so an eager barrier would see queued cards + await db.kanban_advance_many(TOPIC, BOARD, [(c["card_id"], "gate") for c in cards]) + + async def barrier_proc(cards): + upstream_left.append(await db.kanban_count(TOPIC, ["s1"], board=BOARD)) + await db.kanban_advance_many(TOPIC, BOARD, [(c["card_id"], "done") for c in cards]) + + flow = Flow(TOPIC) + stages = chain_stages([ + Stage(BOARD, "s1", slow_s1), + Stage(BOARD, "gate", barrier_proc, barrier=True), + ]) + await asyncio.wait_for(run_flow(flow, stages), timeout=10) + assert await db.kanban_count(TOPIC, "done", board=BOARD) == 6 + assert upstream_left and all(n == 0 for n in upstream_left) # barrier never ran with s1 queued + + +async def test_retry_backoff_then_dead(testdb, monkeypatch): + db = testdb + monkeypatch.setattr(kanban, "RETRY_BACKOFF", 0.02) + await _seed(db, 1) + attempts = [] + + async def failing(cards): + attempts.append(cards[0]["retries"]) + raise RuntimeError("kaputt") + + flow = Flow(TOPIC) + stages = chain_stages([Stage(BOARD, "s1", failing)]) + await asyncio.wait_for(run_flow(flow, stages), timeout=10) + card = await db.kanban_get_card(TOPIC, BOARD, "card-0") + assert card["stage"] == "dead" + assert card["retries"] == kanban.MAX_CARD_RETRIES + assert "kaputt" in card["last_error"] + assert attempts == [0, 1, 2] # backoff between attempts, then dead-letter + + +async def test_requeue_dead(testdb): + db = testdb + await db.kanban_upsert_card(TOPIC, BOARD, "card-0", "title", "s1") + for _ in range(kanban.MAX_CARD_RETRIES): + await db.kanban_fail_card(TOPIC, BOARD, "card-0", "x", kanban.MAX_CARD_RETRIES, 0.0) + assert (await db.kanban_get_card(TOPIC, BOARD, "card-0"))["stage"] == "dead" + assert await db.kanban_requeue_dead(TOPIC, BOARD, "s1") == 1 + card = await db.kanban_get_card(TOPIC, BOARD, "card-0") + assert card["stage"] == "s1" and card["retries"] == 0 + + +async def test_producer_attach_in_idle_lull(testdb): + """Fix-6 regression: a producer attached while workers sit in the exit grace poll + must keep the flow alive and its cards must still be processed.""" + db = testdb + flow = Flow(TOPIC) + stages = chain_stages([Stage(BOARD, "s1", _advance_proc(db, "done"))]) + + async def producer_a(): + await db.kanban_upsert_card(TOPIC, BOARD, "card-a", "title", "s1") + flow.wake.set() + flow.done_producer() + + async def attacher(): + while await db.kanban_count(TOPIC, "done", board=BOARD) == 0: # wait for card-a done + await asyncio.sleep(0.01) + flow.add_producer() # synchronous BEFORE the work — the grace poll must see it + + async def producer_b(): + await db.kanban_upsert_card(TOPIC, BOARD, "card-b", "title", "s1") + flow.wake.set() + flow.done_producer() + await producer_b() + + flow.add_producer() # producer_a, counted before run_flow (sync add) + asyncio.get_event_loop().create_task(attacher()) + await asyncio.wait_for(run_flow(flow, stages, producers=[producer_a()]), timeout=10) + assert await db.kanban_count(TOPIC, "done", board=BOARD) == 2 + + +async def test_backoff_delays_pull(testdb, monkeypatch): + db = testdb + await db.kanban_upsert_card(TOPIC, BOARD, "card-0", "title", "s1") + await db.kanban_fail_card(TOPIC, BOARD, "card-0", "x", 5, 0.2) + assert await db.kanban_pull(TOPIC, BOARD, "s1", 10) == [] # in backoff → not pullable + assert await db.kanban_count(TOPIC, "s1", board=BOARD) == 1 # but still counts as queued + await asyncio.sleep(0.25) + assert len(await db.kanban_pull(TOPIC, BOARD, "s1", 10)) == 1 diff --git a/backend/tests/test_roles.py b/backend/tests/test_roles.py new file mode 100644 index 0000000..e953cb4 --- /dev/null +++ b/backend/tests/test_roles.py @@ -0,0 +1,41 @@ +"""Role routing: resolve_role maps (run_provider, role) → (provider, model) across stacks.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import config +from config import PROVIDERS, resolve_role + + +def test_default_quick_routes_to_minimax(monkeypatch): + monkeypatch.setitem(config.ROLE_ROUTING, "quick", "minimax") + assert resolve_role("claude", "quick") == ("minimax", PROVIDERS["minimax"]["quick"]) + + +def test_default_judge_routes_to_claude(monkeypatch): + monkeypatch.setitem(config.ROLE_ROUTING, "judge", "claude") + assert resolve_role("minimax", "judge") == ("claude", PROVIDERS["claude"]["judge"]) + + +def test_empty_routing_keeps_run_provider(monkeypatch): + monkeypatch.setitem(config.ROLE_ROUTING, "fast", "") + assert resolve_role("claude", "fast") == ("claude", PROVIDERS["claude"]["fast"]) + assert resolve_role("minimax", "fast") == ("minimax", PROVIDERS["minimax"]["fast"]) + + +def test_explicit_model_syntax(monkeypatch): + monkeypatch.setitem(config.ROLE_ROUTING, "guide", "claude:claude-opus-4-8") + assert resolve_role("minimax", "guide") == ("claude", "claude-opus-4-8") + + +def test_unknown_target_falls_back_to_run_provider(monkeypatch): + monkeypatch.setitem(config.ROLE_ROUTING, "quick", "gibtsnicht") + assert resolve_role("claude", "quick") == ("claude", PROVIDERS["claude"]["quick"]) + + +def test_unknown_role_yields_empty_model(): + provider, model = resolve_role("claude", "nope") + assert provider == "claude" + assert model == "" diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 6811578..d87ccef 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -1,10 +1,11 @@ + + + + diff --git a/frontend/src/components/KanbanBoard.vue b/frontend/src/components/KanbanBoard.vue new file mode 100644 index 0000000..b7c2708 --- /dev/null +++ b/frontend/src/components/KanbanBoard.vue @@ -0,0 +1,191 @@ + + + + + diff --git a/frontend/src/components/TopicSidebar.vue b/frontend/src/components/TopicSidebar.vue index 9aefcb0..face691 100644 --- a/frontend/src/components/TopicSidebar.vue +++ b/frontend/src/components/TopicSidebar.vue @@ -26,7 +26,7 @@ const props = defineProps({ stufeAnsicht: { type: Number, default: 4 }, // 1=A · 2=F · 3=E · 4=V }) -const emit = defineEmits(['select', 'createThema', 'updateSource', 'formatClick', 'bausteineClick', 'cancelBlocks', 'resetBausteine', 'deleteTopic', 'cancelGuide', 'deleteGuide', 'dismissError', 'dismissUiError', 'preview', 'openBausteineView', 'openElements', 'togglePin', 'sidebarLeave', 'toggleDark', 'setAnsicht', 'setStufe', 'generalExam', 'setProvider']) +const emit = defineEmits(['select', 'createThema', 'updateSource', 'formatClick', 'bausteineClick', 'cancelBlocks', 'resetBausteine', 'deleteTopic', 'cancelGuide', 'deleteGuide', 'dismissError', 'dismissUiError', 'preview', 'openBausteineView', 'openGuideBoard', 'openElements', 'togglePin', 'sidebarLeave', 'toggleDark', 'setAnsicht', 'setStufe', 'generalExam', 'setProvider']) // Accordion: at most one panel open. IDs: 'blocks', 'fmt-', 'topic-'. const openPanel = ref(null) @@ -108,11 +108,10 @@ function guideStatus(format) { return latest.status } -// Step dots of the guide pipeline -const GUIDE_STEPS = ['Outline', 'Content', 'Content check', 'Writing', 'Reading exam'] +// Stage dots of the guide board (display only — restart/reset lives on the board) +const GUIDE_STEPS = ['Lernziele', 'Zuweisung', 'Writer', 'Fakten', 'Coverage', 'Lesbarkeit'] -// Dots from the artifact-based "done" marker (like blocks, not the DB counter): -// ≤ done = done. Running → the next step (done+1) is active. +// Dots from the card-based "done" marker: ≤ done = done. Running → done+1 active. function guideSteps(format) { const labels = GUIDE_STEPS const done = props.guideStepsDone[format] ?? -1 @@ -124,20 +123,9 @@ function guideSteps(format) { })) } -// Re-run from a guide step (1-based dot per format). null = full/resume. -const selectedStep = reactive({}) -// Dots clickable once artifacts exist (marker ≥ 0 or done) and not generating. -function guideSelectable(format) { - const st = guideStatus(format) - if (st === 'generating' || st === 'queued') return false - return (props.guideStepsDone[format] ?? -1) >= 0 || st === 'done' -} -function guideStepClick(format, n) { - if (!guideSelectable(format)) return - selectedStep[format] = selectedStep[format] === n ? null : n -} -function selectedStepLabel(format) { - return GUIDE_STEPS[(selectedStep[format] || 0) - 1] || '' +// Dot click → open the live guide board (the board hosts restart/reset per column). +function guideStepClick(format) { + emit('openGuideBoard', format) } function errorMsg(format) { @@ -169,10 +157,7 @@ function playLock(format) { function handlePlay(format) { if (playLock(format)) return - // Selected dot (1-based) → ab_step (0-based). Only for a (partially) built guide. - const abStep = guideSelectable(format) && selectedStep[format] ? selectedStep[format] - 1 : null - emit('formatClick', { format, instructions: '', abStep }) - selectedStep[format] = null + emit('formatClick', { format, instructions: '', abStep: null }) // Restart-ab-Stage lebt auf dem Board } // Flash-message behavior: × only hides, nothing is deleted @@ -408,10 +393,10 @@ function saveSource() { {{ i + 1 }} @@ -427,7 +412,7 @@ function saveSource() { :title="playLock(f.key) || (aborted(f.key) ? 'Resume' : 'Generate')" :disabled="!!playLock(f.key)" @click="handlePlay(f.key)" - >{{ selectedStep[f.key] ? `Restart from «${selectedStepLabel(f.key)}»` : aborted(f.key) ? 'Resume' : guideStatus(f.key) === 'done' ? 'Regenerate' : 'Generate' }} + >{{ aborted(f.key) ? 'Resume' : guideStatus(f.key) === 'done' ? 'Regenerate' : 'Generate' }}