marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-02 03:05:57 +02:00
parent afa8b36105
commit 41c9f29a37
38 changed files with 4671 additions and 2634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
backend/agents.py
View File

@@ -2,9 +2,14 @@
Both runners are independent. If a binary/key is missing, only the
respective provider fails — the other keeps running unchanged.
Role routing: config.resolve_role maps (run_provider, role) → (provider, model)
ACROSS stacks, so one run can generate on MiniMax and judge on Claude. If the
routed provider is unavailable, the call falls back to the run's provider.
"""
import asyncio
import heapq
import logging
import os
import re
@@ -13,13 +18,27 @@ import signal
import tempfile
import time
import urllib.request
from contextlib import asynccontextmanager
from pathlib import Path
from config import PROVIDERS, DEFAULT_PROVIDER, MAX_CONCURRENT_AGENTS, MAX_CONCURRENT_INTERACTIVE
from config import (PROVIDERS, DEFAULT_PROVIDER, MAX_CONCURRENT_AGENTS,
MAX_CONCURRENT_AGENTS_PER_TOPIC, MAX_CONCURRENT_INTERACTIVE,
resolve_role)
log = logging.getLogger("creator.agents")
_active_processes: dict[str, asyncio.subprocess.Process] = {}
_active_started: dict[str, float] = {} # agent_key → wall-clock start (for the live runtime display)
def active_agents(scope_prefix: str | None = None) -> list[dict]:
"""Currently running agents and how long they've been running. Filter by key prefix
(e.g. f"blocks-{topic}-") for one topic. → [{key, runtime}] sorted longest-first."""
now = time.time()
out = [{"key": k, "runtime": round(now - t, 1)}
for k, t in list(_active_started.items())
if k in _active_processes and (not scope_prefix or k.startswith(scope_prefix))]
return sorted(out, key=lambda a: -a["runtime"])
# Cancelled scopes (key prefixes, symmetric to kill_process). An agent whose
# key starts with one of these prefixes aborts BEFORE the spawn — so agents WAITING
@@ -41,14 +60,90 @@ def _scope_cancelled(agent_key: str) -> bool:
# Caps the real CLI processes — independent of the pipeline semaphore in
# generator.py. The acquire happens BEFORE the spawn so that queue wait time
# does not count against the agent timeout.
_batch_sem = asyncio.Semaphore(MAX_CONCURRENT_AGENTS)
class _PrioritySemaphore:
"""asyncio.Semaphore variant: when slots are scarce, the LOWEST priority number is served first
(FIFO within the same priority). Lets earlier pipeline columns grab agents before later ones."""
def __init__(self, value: int):
self._value = value
self._waiters: list = [] # heap of [priority, seq, future]
self._seq = 0
async def acquire(self, priority: int = 100):
if self._value > 0:
self._value -= 1
return
fut = asyncio.get_event_loop().create_future()
entry = [priority, self._seq, fut]
self._seq += 1
heapq.heappush(self._waiters, entry)
try:
await fut # release() hands us the slot directly (no value change)
except BaseException:
entry[2] = None # tombstone so release() skips this dead waiter
if fut.done() and not fut.cancelled():
self.release() # granted just before we were cancelled → pass it on
raise
def release(self):
while self._waiters:
entry = heapq.heappop(self._waiters)
if entry[2] is not None and not entry[2].done():
entry[2].set_result(None) # hand the slot straight to the highest-priority waiter
return
self._value += 1
_batch_sem = _PrioritySemaphore(MAX_CONCURRENT_AGENTS)
_interactive_sem = asyncio.Semaphore(MAX_CONCURRENT_INTERACTIVE)
# Serialize OpenCode starts: processes starting simultaneously collide on the
# internal session DB ("database is locked", exit after <1s). The short
# stagger spreads out the starts; afterwards the processes run in parallel normally.
# Per-topic caps (lazily created): each topic gets its own priority semaphore of size
# MAX_CONCURRENT_AGENTS_PER_TOPIC, nested INSIDE the global _batch_sem. Priority-based too, so the
# per-topic queue can't undo the global priority when one topic is the only load.
_topic_sems: dict[str, _PrioritySemaphore] = {}
# Earlier kanban columns get the scarce global slot first (smaller = higher priority).
_STAGE_PRIORITY = ("research", "ingest", "cluster", "pair", "clarify", "naming", "filter", "grouping")
def _agent_priority(key: str) -> int:
for i, tag in enumerate(_STAGE_PRIORITY):
if f"-{tag}-" in key or key.endswith(f"-{tag}"):
return i
return len(_STAGE_PRIORITY) # downstream agents (subblocks/facts/…) after the inventory columns
@asynccontextmanager
async def _batch_gate(scope: str | None, priority: int):
"""Per-topic slot FIRST (fair), then the GLOBAL slot by priority (earlier columns win when
agents are scarce). Order matters — a waiter holds only its per-topic slot while queueing globally."""
topic_sem = _topic_sems.setdefault(scope, _PrioritySemaphore(MAX_CONCURRENT_AGENTS_PER_TOPIC)) if scope else None
if topic_sem is not None:
await topic_sem.acquire(priority)
await _batch_sem.acquire(priority)
try:
yield
finally:
_batch_sem.release()
if topic_sem is not None:
topic_sem.release()
# Space OpenCode starts: processes starting simultaneously collide on the internal
# session DB ("database is locked", exit after <1s). Token bucket instead of a lock
# held through spawn+sleep: the lock only assigns a start slot, the sleep happens
# outside — a wave of starts is spaced by OPENCODE_START_DELAY without a global convoy.
_opencode_start_lock = asyncio.Lock()
_OPENCODE_START_DELAY = 1.0
_OPENCODE_START_DELAY = float(os.getenv("OPENCODE_START_DELAY", "0.5"))
_opencode_next_start = 0.0
async def _opencode_slot() -> None:
global _opencode_next_start
loop = asyncio.get_running_loop()
async with _opencode_start_lock:
now = loop.time()
start_at = max(now, _opencode_next_start)
_opencode_next_start = start_at + _OPENCODE_START_DELAY
await asyncio.sleep(max(0.0, start_at - now))
# Capability → Claude --allowedTools
_CLAUDE_TOOLS = {
@@ -85,6 +180,22 @@ def provider_available(provider: str) -> bool:
return True
# Availability cache for role routing: the routed target is probed at most once per
# TTL (check_url providers would otherwise block the loop on every call).
_avail_cache: dict[str, tuple[float, bool]] = {}
_AVAIL_TTL = 60.0
def _available_cached(provider: str) -> bool:
now = time.monotonic()
hit = _avail_cache.get(provider)
if hit and now - hit[0] < _AVAIL_TTL:
return hit[1]
ok = provider_available(provider)
_avail_cache[provider] = (now, ok)
return ok
def _kill(process) -> None:
"""Kill the agent and its child processes via the process group (otherwise the
children spawned by the CLI survive, keep the pipes open and block communicate())."""
@@ -102,6 +213,7 @@ def kill_process(agent_key_prefix: str) -> None:
for key, process in list(_active_processes.items()):
if process.returncode is not None: # clean up dead entries while iterating
_active_processes.pop(key, None)
_active_started.pop(key, None)
continue
if key.startswith(agent_key_prefix):
log.debug("kill agent %s", key)
@@ -116,23 +228,32 @@ async def run_agent(
role: str = "fast",
capabilities: str = "none",
lane: str = "batch",
scope: str | None = None,
on_line=None,
) -> tuple[int, str, str]:
if _scope_cancelled(agent_key): # before queueing: don't even enter the queue
return 1, "", "cancelled"
if provider not in PROVIDERS:
return 1, "", f"Unknown provider: {provider}"
run_provider = provider
provider, model = resolve_role(run_provider, role)
if provider != run_provider and not _available_cached(provider):
provider, model = run_provider, PROVIDERS[run_provider].get(role, "")
if not model:
return 1, "", f"No model for role '{role}' (provider: {provider})"
if shutil.which(PROVIDERS[provider]["cli"]) is None:
return 1, "", f"CLI '{PROVIDERS[provider]['cli']}' not installed (provider: {provider})"
sem = _interactive_sem if lane == "interactive" else _batch_sem
async with sem:
gate = _interactive_sem if lane == "interactive" else _batch_gate(scope, _agent_priority(agent_key))
async with gate:
if _scope_cancelled(agent_key): # after the acquire: cancelled in the queue → no spawn
return 1, "", "cancelled"
log.info("agent %s: %s %s (role %s)", agent_key, provider, model, role)
if PROVIDERS[provider]["cli"] == "opencode":
return await _run_opencode(agent_key, prompt, timeout, provider, role, capabilities)
return await _run_claude_cli(agent_key, prompt, timeout, role, capabilities)
return await _run_opencode(agent_key, prompt, timeout, provider, model, capabilities, on_line=on_line)
return await _run_claude_cli(agent_key, prompt, timeout, model, capabilities)
async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, timeout: int, stagger: bool = False) -> tuple[int, str, str]:
async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, timeout: int, stagger: bool = False, on_line=None) -> tuple[int, str, str]:
start = time.monotonic()
async def spawn():
@@ -145,18 +266,40 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None,
)
if stagger:
async with _opencode_start_lock:
process = await spawn()
await asyncio.sleep(_OPENCODE_START_DELAY)
else:
process = await spawn()
_active_processes[agent_key] = process
await _opencode_slot() # spaced start slot; spawn itself is not serialized
process = await spawn()
# Collision-safe tracking: identical keys (e.g. same chunk label from parallel cards)
# get a ~n suffix — prefix-based kill/cancel still matches, nothing becomes an orphan.
track_key = agent_key
n = 2
while track_key in _active_processes:
track_key = f"{agent_key}~{n}"
n += 1
_active_processes[track_key] = process
_active_started[track_key] = time.time()
try:
try:
stdout, stderr = await asyncio.wait_for(
process.communicate(input=stdin_data),
timeout=timeout,
)
if on_line is not None:
# Streaming path: read stdout line by line, hand each raw line to on_line LIVE.
out_chunks: list[str] = []
async def _pump():
async for raw in process.stdout:
s = raw.decode("utf-8", errors="replace")
out_chunks.append(s)
try:
on_line(s)
except Exception:
log.debug("on_line callback failed", exc_info=True)
await asyncio.wait_for(_pump(), timeout=timeout)
await process.wait()
stderr_b = await process.stderr.read()
stdout, stderr = "".join(out_chunks).encode("utf-8"), stderr_b
else:
stdout, stderr = await asyncio.wait_for(
process.communicate(input=stdin_data),
timeout=timeout,
)
except asyncio.TimeoutError:
_kill(process)
try:
@@ -173,13 +316,14 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None,
finally:
# Pop only on identity: a slot restart under the same key must not evict
# the NEW process from tracking.
if _active_processes.get(agent_key) is process:
del _active_processes[agent_key]
if _active_processes.get(track_key) is process:
del _active_processes[track_key]
_active_started.pop(track_key, None)
async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, role: str, capabilities: str) -> tuple[int, str, str]:
async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, model: str, capabilities: str) -> tuple[int, str, str]:
cfg = PROVIDERS["claude"]
cmd = [cfg["cli"], "-p", "--model", cfg[role]]
cmd = [cfg["cli"], "-p", "--model", model]
tools = _CLAUDE_TOOLS.get(capabilities)
if tools:
cmd += ["--allowedTools", tools]
@@ -187,7 +331,7 @@ async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, role: str,
return await _communicate(agent_key, cmd, prompt.encode("utf-8"), timeout)
async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, role: str, capabilities: str) -> tuple[int, str, str]:
async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, model: str, capabilities: str, on_line=None) -> tuple[int, str, str]:
cfg = PROVIDERS[provider]
# Prompt via temp file instead of argv (ARG_MAX protection for large project prompts)
with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8", dir=tempfile.gettempdir()) as f:
@@ -198,14 +342,16 @@ async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str
cmd = [
cfg["cli"], "run",
"Folge exakt den Anweisungen in der angehängten Datei. Sie sind der vollständige Auftrag.",
"-m", cfg[role],
"-m", model,
"--agent", _OPENCODE_AGENTS.get(capabilities, "text"),
"--dangerously-skip-permissions",
"-f", str(prompt_path),
]
if on_line is not None:
cmd += ["--format", "json"] # raw JSON events → parsed live by on_line
try:
rc, stdout, stderr = await _communicate(agent_key, cmd, None, timeout, stagger=True)
return rc, _clean_opencode_output(stdout), stderr
rc, stdout, stderr = await _communicate(agent_key, cmd, None, timeout, stagger=True, on_line=on_line)
return rc, (stdout if on_line is not None else _clean_opencode_output(stdout)), stderr
finally:
prompt_path.unlink(missing_ok=True)

1675
backend/blocks.py
View File

File diff suppressed because it is too large Load Diff

333
backend/board_artefacts.py Normal file
View File

@@ -0,0 +1,333 @@
"""Board 2 „Artefakte": per finished block, prepare the learning artefacts the guide presents.
A card is spawned by board 1's `done` column per mirrored block and runs through:
subblocks → facts → levels → relevance → question_pattern → artefacts → finalize
finalize (SERIAL) merges the block's results into the global sidecar/facts/pattern/artefakte
files + the DB tables. `outline` is a topic-wide BARRIER singleton at the very end
(prerequisite graph → chapter order), re-run once per generation run.
The heavy lifting is the existing per-block functions in blocks.py — each card gets its own
work subdirectory + facts/artefakte paths, so their slot files never collide across blocks."""
import asyncio
import json
import logging
import re
import database as db
import blocks
from blocks import (
ARTEFACT_TYPES, _artefacts_block, _facts_block, _levels_block, _match_sub,
_question_pattern_block, _relevance_block, _subblocks_block, _outline_block,
)
from fsutil import atomic_write_json
from jsonio import read_json_file as _json_file
from kanban import Flow, Stage
from pipeline import GenContext, _log
from textkit import _norm_title, _title
log = logging.getLogger("creator.board_artefacts")
BOARD = "artefacts"
DONE = "done_artefact"
def _nset(msg: str, step: int | None = None) -> None:
"""Progress no-op — the kanban board itself is the progress display."""
def _safe(norm: str) -> str:
return re.sub(r"\W+", "-", norm).strip("-")[:24] or "block"
def _card_set_p(flow: Flow, norm: str):
"""Per-card progress: the inner step messages land in-memory on the flow —
board_snapshot shows them as the card's info line while it is active."""
info = flow.state.setdefault("card_info", {})
def set_p(msg: str, step: int | None = None) -> None:
info[f"{BOARD}:{norm}"] = msg
return set_p
def _pfiles(files: dict, norm: str) -> dict:
"""Per-block file namespace: own work dir + facts/artefakte paths, global rest."""
sub = files["arbeit"] / f"ab-{_norm_title(norm).replace(' ', '_')[:60]}"
sub.mkdir(parents=True, exist_ok=True)
return {**files, "arbeit": sub, "facts": sub / "facts.json", "artefakte": sub / "artefakte.json"}
def _entry_line(p: dict) -> str:
d = p.get("description")
return f"{p['title']} — {d}" if d else p["title"]
def make_spawner(topic: str, files: dict):
"""Hook for board 1's `done` column: one artefact card per mirrored block."""
async def spawn(block_card_id: str, payload: dict):
norm = payload.get("mirrored_norm")
if not norm:
return
await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "subblocks", {
"title": payload.get("title", ""),
"description": payload.get("description", ""),
})
return spawn
async def _gather_cards(ctx: GenContext, flow: Flow, cards, one):
results = await asyncio.gather(*[one(c) for c in cards], return_exceptions=True)
errs = [r for r in results if isinstance(r, Exception)]
if errs:
raise errs[0]
flow.wake.set()
def _fail_or_cancel(ctx: GenContext, what: str):
# A per-card failure belongs on the card (last_error/dead-letter), never in the
# topic banner — the inner block functions may have set it there.
blocks._blocks_errors.pop(ctx.topic, None)
if ctx.is_cancelled():
return None # leave the card where it is
raise RuntimeError(f"{what} ohne Ergebnis")
# ── Stage processors (one call per card, all parallel) ─────────────────────────────
async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
topic = flow.topic
# Fix 4: fragments demoted to a parent become seed candidates of the parent's subblocks.
seeds: dict[str, list[str]] = {}
for r in await db.kanban_cards(topic, board="inventory", stage="rejected"):
pn = r["payload"].get("parent_norm")
if pn:
seeds.setdefault(pn, []).append(r["payload"].get("title", ""))
async def one(c):
p = c["payload"]
norm = c["card_id"]
instr = instructions
if (sd := [s for s in seeds.get(norm, []) if s]):
instr = (instructions + "\n\nBereits identifizierte Unterpunkt-Kandidaten dieses "
"Blocks (unbedingt prüfen und, wenn belegt, aufnehmen):\n"
+ "\n".join(f"- {s}" for s in sd))
raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
{1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-")
if raw is None:
return _fail_or_cancel(ctx, f"Subblocks {p.get('title', norm)}")
p["raw"] = raw
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "facts")
await _gather_cards(ctx, flow, cards, one)
async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
instructions: str, cards):
topic = flow.topic
async def one(c):
p = c["payload"]
norm = c["card_id"]
raw = p.get("raw") or {}
res = await _facts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), raw, q,
folder, instructions, ns=f"{_safe(norm)}-")
if res is None:
return _fail_or_cancel(ctx, f"Facts {p.get('title', norm)}")
facts_map, discarded = res
if discarded: # unsupportable subs vanish from raw too (guide never sees them)
for bt, sns in discarded.items():
if bt in raw:
raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns]
raw = {bt: subs for bt, subs in raw.items() if subs}
p["raw"], p["facts"] = raw, facts_map
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "levels")
await _gather_cards(ctx, flow, cards, one)
async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
topic = flow.topic
async def one(c):
p = c["payload"]
norm = c["card_id"]
sidecar = await _levels_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
p.get("raw") or {}, instructions, ns=f"{_safe(norm)}-")
if sidecar is None:
return _fail_or_cancel(ctx, f"Levels {p.get('title', norm)}")
facts_map = p.get("facts") or {}
for btitle, subs in sidecar.items(): # merge facts into the subs (mirror + guide)
fm = facts_map.get(btitle, {})
for sub in subs:
if (fk := fm.get(_norm_title(sub["title"]))):
sub["facts"] = fk
p["sidecar"] = sidecar
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "relevance")
await _gather_cards(ctx, flow, cards, one)
async def _proc_relevance(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
topic = flow.topic
async def one(c):
p = c["payload"]
norm = c["card_id"]
sidecar = p.get("sidecar") or {}
rel = await _relevance_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
sidecar, instructions, ns=f"{_safe(norm)}-")
if rel is None:
return _fail_or_cancel(ctx, f"Relevance {p.get('title', norm)}")
gid = 0
for subs in sidecar.values():
for sub in subs:
gid += 1
sub["relevance"] = rel.get(gid, "relevant")
p["sidecar"] = sidecar
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "question_pattern")
await _gather_cards(ctx, flow, cards, one)
async def _proc_question_pattern(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
topic = flow.topic
async def one(c):
p = c["payload"]
norm = c["card_id"]
pattern = await _question_pattern_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
p.get("sidecar") or {}, instructions,
ns=f"{_safe(norm)}-")
if pattern is None:
return _fail_or_cancel(ctx, f"Fragen {p.get('title', norm)}")
p["pattern"] = pattern
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "artefacts")
await _gather_cards(ctx, flow, cards, one)
async def _proc_artefacts(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
topic = flow.topic
async def one(c):
p = c["payload"]
norm = c["card_id"]
artefacts = await _artefacts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
p.get("sidecar") or {}, instructions,
ns=f"{_safe(norm)}-")
if artefacts is None and ctx.is_cancelled():
return None
p["artefacts"] = artefacts or {} # artefacts are optional — never fatal
await db.kanban_set_payload(topic, BOARD, norm, p)
await db.kanban_advance(topic, BOARD, norm, "finalize")
await _gather_cards(ctx, flow, cards, one)
# ── Finalize (SERIAL): merge into the global files + DB tables ─────────────────────
def _merge_json(path, block_keys: dict) -> None:
data = _json_file(path)
if not isinstance(data, dict):
data = {}
data.update(block_keys)
atomic_write_json(path, data, indent=1)
async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
topic = flow.topic
for c in cards:
p = c["payload"]
title = p.get("title", "")
sidecar = p.get("sidecar") or {}
pattern = p.get("pattern") or {}
artefacts = p.get("artefacts") or {}
# global sidecar files (the legacy read path of guide/frontend/resume)
_merge_json(files["sub_roh"], {t: subs for t, subs in (p.get("raw") or {}).items()})
_merge_json(files["facts"], p.get("facts") or {})
_merge_json(files["sidecar"], sidecar)
_merge_json(files["question_pattern"], pattern)
art_global = _json_file(files["artefakte"])
if not isinstance(art_global, dict):
art_global = {}
for typ in ARTEFACT_TYPES:
kept = [e for e in art_global.get(typ, [])
if _norm_title(_title(str(e.get("block", "")))) != _norm_title(title)]
art_global[typ] = kept + list(artefacts.get(typ, []))
atomic_write_json(files["artefakte"], art_global, indent=1)
# DB mirrors — per block only (no global deletes)
await blocks._mirror_sidecar_db(topic, sidecar)
for btitle, entries in pattern.items():
bnorm = _norm_title(btitle)
for e in entries if isinstance(entries, list) else []:
sub = str(e.get("subblock", "")).strip()
sn = _norm_title(sub)
question = str(e.get("question", "")).strip()
if bnorm and sn and question:
await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question)
btitles = list(sidecar.keys())
for typ in ARTEFACT_TYPES:
for e in artefacts.get(typ, []):
bt = _match_sub(str(e.get("block", "")), btitles)
bnorm, sn = _norm_title(bt), _norm_title(str(e.get("subblock", "")))
if not bnorm or not sn:
continue
data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")},
ensure_ascii=False)
await db.put_sub_artifact(topic, bnorm, sn, typ, data, bt, str(e.get("subblock", "")))
await db.kanban_advance(topic, BOARD, c["card_id"], DONE)
_log(topic, f"Artefakte fertig: {title}")
flow.wake.set()
# ── Outline (topic-wide barrier singleton) ─────────────────────────────────────────
OUTLINE_CARD = "outline"
async def ensure_outline_card(topic: str) -> None:
"""(Re-)queue the outline singleton — run once per generation run, after everything."""
await db.kanban_upsert_card(topic, BOARD, OUTLINE_CARD, "outline", "outline",
{"title": "Gliederung"})
async def _proc_outline(ctx: GenContext, flow: Flow, files: dict, instructions: str, cards):
topic = flow.topic
done = await db.kanban_cards(topic, board="inventory", stage="done_block")
done.sort(key=lambda c: c["updated_at"])
entries = {i: _entry_line(c["payload"]) for i, c in enumerate(done, 1)
if c["payload"].get("title")}
if entries:
plan = await _outline_block(ctx, _nset, files, entries, instructions)
if ctx.is_cancelled():
return
if isinstance(plan, dict) and plan.get("chapters"):
chapters = [
{"title": ch.get("title", "Kapitel"),
"blocks": [_title(entries[n]) for n in ch.get("numbers", []) if n in entries]}
for ch in plan["chapters"]
]
await db.set_outline(topic, json.dumps({"chapters": chapters}, ensure_ascii=False))
await db.kanban_advance_many(topic, BOARD, [(c["card_id"], DONE) for c in cards])
flow.wake.set()
# ── Stage list (appended after board 1 in chain order) ─────────────────────────────
def artefact_stages(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
instructions: str) -> list[Stage]:
research_done = lambda: flow.research_done # noqa: E731
return [
Stage(BOARD, "subblocks", lambda cs: _proc_subblocks(ctx, flow, files, instructions, cs)),
Stage(BOARD, "facts", lambda cs: _proc_facts(ctx, flow, files, q, folder, instructions, cs)),
Stage(BOARD, "levels", lambda cs: _proc_levels(ctx, flow, files, instructions, cs)),
Stage(BOARD, "relevance", lambda cs: _proc_relevance(ctx, flow, files, instructions, cs)),
Stage(BOARD, "question_pattern",
lambda cs: _proc_question_pattern(ctx, flow, files, instructions, cs)),
Stage(BOARD, "artefacts", lambda cs: _proc_artefacts(ctx, flow, files, instructions, cs)),
Stage(BOARD, "finalize", lambda cs: _proc_finalize(ctx, flow, files, cs), serial=True),
Stage(BOARD, "outline", lambda cs: _proc_outline(ctx, flow, files, instructions, cs),
barrier=True, drain=True, gate=research_done),
]

1395
backend/board_inventory.py Normal file
View File

File diff suppressed because it is too large Load Diff

68
backend/config.py
View File

@@ -1,3 +1,4 @@
import os
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent
@@ -8,6 +9,26 @@ DB_PATH = STORAGE_DIR / "creator.db"
PROJECTS_DIR = PROJECT_ROOT / "projects"
UNI_DIR = PROJECT_ROOT / "uni"
def _load_env(path: Path) -> None:
"""Mini .env loader (no dependency): KEY=VALUE lines; existing env always wins
(`make dev` already exports .env — this covers bare `uvicorn`/pytest starts)."""
try:
text = path.read_text(encoding="utf-8")
except OSError:
return
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
key, value = key.strip(), value.strip().strip('"').strip("'")
if key and key not in os.environ:
os.environ[key] = value
_load_env(PROJECT_ROOT / ".env")
MAX_CONCURRENT_GENERATIONS = 10
# Readability gate: deterministic checker (small German complexity model,
@@ -21,12 +42,9 @@ READABILITY_MAX = 3.5 # section too hard when the sentence average is a
READABILITY_HARD = 4.0 # an individual sentence is "hard" from here on
READABILITY_HARD_SHARE = 0.30 # … OR when this share of sentences is hard
# Block consolidation: semantic embedding clustering instead of an LLM list merge.
# A small multilingual sentence embedding (mean-pool) builds the candidate clusters
# GLOBALLY (no chunk loss) via cosine + union-find. Title variants of the same concept
# ("Vertex Cover" / "Vertex Cover Definition") merge; the consensus then counts the
# real readers per cluster (≥2 = consensus). If transformers/torch are missing or the model
# won't load → embedding silently off, `_consolidate` falls back to the old panel-judge path.
# Kanban clustering: semantic embeddings drive the online title clustering and the
# candidate pairs of the pair check. If transformers/torch are missing or the model
# won't load → embedding silently off (all pairs go to the judge, clusters stay singletons).
EMBEDDING_AKTIV = True
EMBEDDING_MODELL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" # CPU, multilingual, ~470 MB
# Stronger (larger) CPU alternative if needed: "BAAI/bge-m3".
@@ -66,10 +84,12 @@ GROUP_RECONCILE_FLOOR = 0.75
# chain (random-pair baseline), set BELOW the legitimate heterogeneous minimum so it never kills a real model.
GROUP_MIN_COS_FLOOR = 0.15
# Cap for concurrent CLI agent processes (across all generations).
# Own lane for interactive calls (chat, elements) so they don't hang behind
# running writers in the queue.
MAX_CONCURRENT_AGENTS = 10
# Caps for concurrent CLI agent processes (env-overridable). Two nested limits, both always active:
# a per-topic cap and a global cap across all topics. Defaults 10/10 = previous behavior (global
# dominates). Locally raise the global cap to actually parallelize across topics (per-topic stays 10).
# Own lane for interactive calls (chat, elements) so they don't hang behind running writers.
MAX_CONCURRENT_AGENTS = int(os.getenv("MAX_CONCURRENT_AGENTS", "16")) # global, all topics
MAX_CONCURRENT_AGENTS_PER_TOPIC = int(os.getenv("MAX_CONCURRENT_AGENTS_PER_TOPIC", "12")) # per topic
MAX_CONCURRENT_INTERACTIVE = 8
# Grace window of the consensus races (blocks, guide, OnePager): after the first
@@ -123,6 +143,10 @@ TIMEOUTS = {
"question_pattern_check": (300, 10), # critic cleans up the pattern table per block
"writer": (600, 120), # per section in the chunk
"lese_check": (300, 10), # per section in the package
# guide board (per card = one block)
"lernziele": (300, 5), # backward-design objectives per block
"fakten_gate": (600, 5), # CoVe claim check per block
"coverage": (300, 5), # objective↔section mapping per block
}
# Purpose per format — flows into the outline judge (what the guide should achieve).
@@ -169,3 +193,27 @@ PROVIDERS = {
"check_url": "http://localhost:11434/api/tags", # Ollama reachable?
},
}
# Role routing ACROSS provider stacks: generation (quick/guide) and judging (judge)
# may run on different providers within ONE run — judge model ≠ generator model
# (research-backed: cross-model judging avoids self-preference bias).
# Value: "" = provider of the run; "minimax" = that stack's role model;
# "provider:model" = explicit model override.
ROLE_ROUTING = {
"quick": os.getenv("ROLE_QUICK", "minimax"),
"judge": os.getenv("ROLE_JUDGE", "claude"),
"guide": os.getenv("ROLE_GUIDE", "minimax"),
"fast": os.getenv("ROLE_FAST", ""),
}
def resolve_role(run_provider: str, role: str) -> tuple[str, str]:
"""→ (provider, model) for one agent call. Pure routing, no availability check —
the caller (agents.run_agent) falls back to run_provider if the target is unavailable."""
target = ROLE_ROUTING.get(role, "") or run_provider
provider, _, model = target.partition(":")
if provider not in PROVIDERS:
provider, model = run_provider, ""
if not model:
model = PROVIDERS.get(provider, {}).get(role, "")
return provider, model

446
backend/database.py
View File

@@ -199,6 +199,73 @@ CREATE TABLE IF NOT EXISTS sub_artefakte (
)
"""
# Kanban dataflow (boards 'inventory' + 'artefacts'): ONE generic card table for all card kinds
# (title/cluster/block). `stage` is the queue key — a worker pulls WHERE stage = <its input stage>.
# `payload` is a JSON blob (title, description, sources, readers, mentions, parent_norm, journal …);
# retries/not_before/last_error implement backoff + dead-letter (stage 'dead' after MAX_CARD_RETRIES).
CREATE_KANBAN_CARDS = """
CREATE TABLE IF NOT EXISTS kanban_cards (
topic TEXT NOT NULL,
board TEXT NOT NULL,
card_id TEXT NOT NULL,
kind TEXT NOT NULL,
stage TEXT NOT NULL,
payload TEXT NOT NULL DEFAULT '{}',
retries INTEGER NOT NULL DEFAULT 0,
not_before TEXT NOT NULL DEFAULT '',
last_error TEXT,
updated_at TEXT NOT NULL,
PRIMARY KEY (topic, board, card_id)
)
"""
CREATE_KANBAN_PULL_INDEX = """
CREATE INDEX IF NOT EXISTS idx_kanban_pull ON kanban_cards(topic, board, stage, not_before, updated_at)
"""
# title_norm → cluster membership (one title belongs to exactly one cluster).
CREATE_KANBAN_MEMBERS = """
CREATE TABLE IF NOT EXISTS kanban_members (
topic TEXT NOT NULL,
member_id TEXT NOT NULL,
group_id TEXT NOT NULL,
PRIMARY KEY (topic, member_id)
)
"""
# Guide board: one card per block, linear stages (lernziele … lesbarkeit → done).
# `md` carries the writer fragment (with kapitel/section/sub markers) between the gates.
CREATE_GUIDE_CARDS = """
CREATE TABLE IF NOT EXISTS guide_cards (
topic TEXT NOT NULL,
format TEXT NOT NULL DEFAULT 'Guide',
block_norm TEXT NOT NULL,
block TEXT NOT NULL,
chapter TEXT NOT NULL DEFAULT '',
ord INTEGER NOT NULL DEFAULT 0,
stage TEXT NOT NULL DEFAULT 'lernziele',
status TEXT NOT NULL DEFAULT 'open',
writer_rounds INTEGER NOT NULL DEFAULT 0,
gate_info TEXT NOT NULL DEFAULT '',
md TEXT NOT NULL DEFAULT '',
updated_at TEXT NOT NULL,
PRIMARY KEY (topic, format, block_norm)
)
"""
CREATE_GUIDE_LERNZIELE = """
CREATE TABLE IF NOT EXISTS guide_lernziele (
topic TEXT NOT NULL,
block_norm TEXT NOT NULL,
ziel_id TEXT NOT NULL,
text TEXT NOT NULL,
sub_norm TEXT NOT NULL DEFAULT '',
covered INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL,
PRIMARY KEY (topic, block_norm, ziel_id)
)
"""
_db: aiosqlite.Connection | None = None
@@ -213,7 +280,9 @@ async def get_db() -> aiosqlite.Connection:
async def init_db():
db = await get_db()
# WAL survives crashes much better; busy_timeout absorbs short locks.
# synchronous=NORMAL: safe under WAL, much less fsync — the kanban flow commits often.
await db.execute("PRAGMA journal_mode=WAL")
await db.execute("PRAGMA synchronous=NORMAL")
await db.execute("PRAGMA busy_timeout=5000")
await db.execute(CREATE_GUIDES)
await db.execute(CREATE_PROGRESS)
@@ -230,6 +299,11 @@ async def init_db():
await db.execute(CREATE_SOURCE)
await db.execute(CREATE_GUIDE_OUTLINE)
await db.execute(CREATE_SUB_ARTEFAKTE)
await db.execute(CREATE_KANBAN_CARDS)
await db.execute(CREATE_KANBAN_PULL_INDEX)
await db.execute(CREATE_KANBAN_MEMBERS)
await db.execute(CREATE_GUIDE_CARDS)
await db.execute(CREATE_GUIDE_LERNZIELE)
try: # migration for existing DBs without the step column
await db.execute("ALTER TABLE guides ADD COLUMN step INTEGER")
except aiosqlite.OperationalError:
@@ -706,6 +780,378 @@ async def delete_blocks(topic: str) -> None:
await db.commit()
# ── Kanban dataflow (generic card layer, boards 'inventory' + 'artefacts') ────────
def _now_plus(seconds: float) -> str:
from datetime import datetime, timedelta, timezone
return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat()
def _card(row, cursor) -> dict:
"""Row → dict with the JSON payload decoded (payload keys stay under 'payload')."""
d = _row_to_dict(row, cursor)
try:
d["payload"] = json.loads(d.get("payload") or "{}")
except (TypeError, ValueError):
d["payload"] = {}
return d
async def kanban_pull(topic: str, board: str, stage: str, limit: int) -> list[dict]:
"""Oldest `limit` cards sitting in `stage` whose backoff has expired (FIFO via updated_at)."""
db = await get_db()
cursor = await db.execute(
"""SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? AND not_before <= ?
ORDER BY updated_at LIMIT ?""",
(topic, board, stage, _now(), limit))
return [_card(row, cursor) for row in await cursor.fetchall()]
async def kanban_count(topic: str, stages, board: str | None = None) -> int:
"""Cards sitting in any of `stages` (str or list) — queue length / quiescence.
Cards in backoff still count: their work is not done. `board=None` = across boards."""
if isinstance(stages, str):
stages = [stages]
if not stages:
return 0
db = await get_db()
ph = ",".join("?" * len(stages))
sql = f"SELECT count(*) FROM kanban_cards WHERE topic = ? AND stage IN ({ph})"
args: tuple = (topic, *stages)
if board:
sql += " AND board = ?"
args += (board,)
cursor = await db.execute(sql, args)
return (await cursor.fetchone())[0]
async def kanban_advance(topic: str, board: str, card_id: str, stage: str) -> None:
"""Move a card to `stage` (next column, or back for rework). Clears backoff/error."""
await kanban_advance_many(topic, board, [(card_id, stage)])
async def kanban_advance_many(topic: str, board: str, moves: list[tuple[str, str]]) -> None:
"""Batch stage moves in ONE commit (the flow advances whole packages)."""
if not moves:
return
db = await get_db()
now = _now()
await db.executemany(
"""UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ?
WHERE topic = ? AND board = ? AND card_id = ?""",
[(stage, now, topic, board, cid) for cid, stage in moves])
await db.commit()
async def kanban_upsert_card(topic: str, board: str, card_id: str, kind: str, stage: str,
payload: dict | None = None) -> None:
"""Insert or overwrite a card (stable ids → growing clusters upsert, never duplicate).
payload=None keeps the existing payload on conflict."""
db = await get_db()
await db.execute(
"""INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at)
VALUES (?, ?, ?, ?, ?, COALESCE(?, '{}'), ?)
ON CONFLICT(topic, board, card_id) DO UPDATE SET
kind = excluded.kind, stage = excluded.stage,
payload = COALESCE(?, kanban_cards.payload),
retries = 0, not_before = '', last_error = NULL, updated_at = excluded.updated_at""",
(topic, board, card_id, kind, stage,
json.dumps(payload, ensure_ascii=False) if payload is not None else None, _now(),
json.dumps(payload, ensure_ascii=False) if payload is not None else None))
await db.commit()
async def kanban_set_payload(topic: str, board: str, card_id: str, payload: dict) -> None:
db = await get_db()
await db.execute(
"UPDATE kanban_cards SET payload = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?",
(json.dumps(payload, ensure_ascii=False), _now(), topic, board, card_id))
await db.commit()
async def kanban_get_card(topic: str, board: str, card_id: str) -> dict | None:
db = await get_db()
cursor = await db.execute(
"SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?", (topic, board, card_id))
row = await cursor.fetchone()
return _card(row, cursor) if row else None
async def kanban_cards(topic: str, board: str | None = None, stage: str | None = None,
kind: str | None = None) -> list[dict]:
db = await get_db()
sql, args = "SELECT * FROM kanban_cards WHERE topic = ?", [topic]
for col, val in (("board", board), ("stage", stage), ("kind", kind)):
if val is not None:
sql += f" AND {col} = ?"
args.append(val)
cursor = await db.execute(sql, args)
return [_card(row, cursor) for row in await cursor.fetchall()]
async def kanban_fail_card(topic: str, board: str, card_id: str, error: str,
max_retries: int, backoff_base: float = 30.0) -> bool:
"""Register a processing failure: retries++, exponential backoff (not_before), and after
`max_retries` → stage 'dead' (dead-letter, requeue-able). → True if the card went dead."""
db = await get_db()
cursor = await db.execute(
"SELECT retries FROM kanban_cards WHERE topic = ? AND board = ? AND card_id = ?",
(topic, board, card_id))
row = await cursor.fetchone()
if row is None:
return False
retries = (row[0] or 0) + 1
dead = retries >= max_retries
if dead:
await db.execute(
"""UPDATE kanban_cards SET stage = 'dead', retries = ?, not_before = '',
last_error = ?, updated_at = ? WHERE topic = ? AND board = ? AND card_id = ?""",
(retries, error[:500], _now(), topic, board, card_id))
else:
await db.execute(
"""UPDATE kanban_cards SET retries = ?, not_before = ?, last_error = ?, updated_at = ?
WHERE topic = ? AND board = ? AND card_id = ?""",
(retries, _now_plus(backoff_base * (2 ** (retries - 1))), error[:500], _now(),
topic, board, card_id))
await db.commit()
return dead
async def kanban_dead(topic: str) -> list[dict]:
"""Dead-letter cards across boards (for the board UI + requeue)."""
return await kanban_cards(topic, stage="dead")
async def kanban_requeue_dead(topic: str, board: str, stage: str) -> int:
"""dead → `stage` (fresh retries). → number of requeued cards."""
db = await get_db()
cursor = await db.execute(
"""UPDATE kanban_cards SET stage = ?, retries = 0, not_before = '', last_error = NULL, updated_at = ?
WHERE topic = ? AND board = ? AND stage = 'dead'""",
(stage, _now(), topic, board))
await db.commit()
return cursor.rowcount
async def kanban_stage_counts(topic: str) -> dict[str, dict[str, int]]:
"""{board: {stage: count}} — the live board."""
db = await get_db()
cursor = await db.execute(
"SELECT board, stage, count(*) FROM kanban_cards WHERE topic = ? GROUP BY board, stage", (topic,))
out: dict[str, dict[str, int]] = {}
for board, stage, n in await cursor.fetchall():
out.setdefault(board, {})[stage] = n
return out
async def kanban_stage_cards(topic: str, board: str, stage: str, limit: int = 20) -> list[dict]:
"""Newest `limit` cards of one column (for the live card display)."""
db = await get_db()
cursor = await db.execute(
"""SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ?
ORDER BY updated_at DESC LIMIT ?""",
(topic, board, stage, limit))
return [_card(row, cursor) for row in await cursor.fetchall()]
async def kanban_delete_cards(topic: str, board: str, kind: str | None = None) -> None:
"""Delete derived cards (board reset) — kind=None wipes the whole board."""
db = await get_db()
if kind:
await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ? AND kind = ?",
(topic, board, kind))
else:
await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ?", (topic, board))
await db.commit()
async def kanban_reset(topic: str, board: str | None = None) -> None:
db = await get_db()
if board:
await db.execute("DELETE FROM kanban_cards WHERE topic = ? AND board = ?", (topic, board))
if board == "inventory":
await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,))
else:
await db.execute("DELETE FROM kanban_cards WHERE topic = ?", (topic,))
await db.execute("DELETE FROM kanban_members WHERE topic = ?", (topic,))
await db.commit()
async def kanban_set_members(topic: str, group_id: str, members: list[str]) -> None:
"""Replace the member set of a cluster (one member belongs to exactly one cluster)."""
db = await get_db()
await db.execute("DELETE FROM kanban_members WHERE topic = ? AND group_id = ?", (topic, group_id))
await db.executemany(
"""INSERT INTO kanban_members (topic, member_id, group_id) VALUES (?, ?, ?)
ON CONFLICT(topic, member_id) DO UPDATE SET group_id = excluded.group_id""",
[(topic, m, group_id) for m in members])
await db.commit()
async def kanban_members_of(topic: str, group_id: str) -> list[str]:
db = await get_db()
cursor = await db.execute(
"SELECT member_id FROM kanban_members WHERE topic = ? AND group_id = ?", (topic, group_id))
return [r[0] for r in await cursor.fetchall()]
async def kanban_member_group(topic: str, member_id: str) -> str | None:
db = await get_db()
cursor = await db.execute(
"SELECT group_id FROM kanban_members WHERE topic = ? AND member_id = ?", (topic, member_id))
row = await cursor.fetchone()
return row[0] if row else None
# ── Guide board (one card per block, linear stages) ──────────────────────────────
async def upsert_guide_card(topic: str, format: str, block_norm: str, block: str,
stage: str = "lernziele") -> None:
"""Insert a card; an existing one keeps its stage/progress (resume)."""
db = await get_db()
await db.execute(
"""INSERT INTO guide_cards (topic, format, block_norm, block, stage, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(topic, format, block_norm) DO UPDATE SET
block = excluded.block, updated_at = excluded.updated_at""",
(topic, format, block_norm, block, stage, _now()))
await db.commit()
async def list_guide_cards(topic: str, format: str) -> list[dict]:
db = await get_db()
cursor = await db.execute(
"SELECT * FROM guide_cards WHERE topic = ? AND format = ? ORDER BY ord, block_norm",
(topic, format))
return [_row_to_dict(row, cursor) for row in await cursor.fetchall()]
async def set_guide_card(topic: str, format: str, block_norm: str, **fields) -> None:
if not fields:
return
db = await get_db()
cols = ", ".join(f"{k} = ?" for k in fields)
await db.execute(
f"UPDATE guide_cards SET {cols}, updated_at = ? WHERE topic = ? AND format = ? AND block_norm = ?",
(*fields.values(), _now(), topic, format, block_norm))
await db.commit()
async def guide_stage_counts(topic: str, format: str) -> dict[str, int]:
db = await get_db()
cursor = await db.execute(
"SELECT stage, count(*) FROM guide_cards WHERE topic = ? AND format = ? GROUP BY stage",
(topic, format))
return {stage: n for stage, n in await cursor.fetchall()}
async def reset_guide_cards_from_stage(topic: str, format: str, stages: list[str],
to_stage: str, clear_md: bool = False) -> int:
"""Cards sitting in any of `stages` → back to `to_stage` (fresh rounds/gate info)."""
if not stages:
return 0
db = await get_db()
ph = ",".join("?" * len(stages))
md = ", md = ''" if clear_md else ""
cursor = await db.execute(
f"""UPDATE guide_cards SET stage = ?, status = 'open', writer_rounds = 0,
gate_info = ''{md}, updated_at = ?
WHERE topic = ? AND format = ? AND stage IN ({ph})""",
(to_stage, _now(), topic, format, *stages))
await db.commit()
return cursor.rowcount
async def delete_guide_board(topic: str, format: str | None = None) -> None:
db = await get_db()
if format:
await db.execute("DELETE FROM guide_cards WHERE topic = ? AND format = ?", (topic, format))
cursor = await db.execute("SELECT count(*) FROM guide_cards WHERE topic = ?", (topic,))
if (await cursor.fetchone())[0] == 0:
await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,))
else:
await db.execute("DELETE FROM guide_cards WHERE topic = ?", (topic,))
await db.execute("DELETE FROM guide_lernziele WHERE topic = ?", (topic,))
await db.commit()
async def put_lernziel(topic: str, block_norm: str, ziel_id: str, text: str, sub_norm: str = "") -> None:
db = await get_db()
await db.execute(
"""INSERT INTO guide_lernziele (topic, block_norm, ziel_id, text, sub_norm, covered, updated_at)
VALUES (?, ?, ?, ?, ?, 0, ?)
ON CONFLICT(topic, block_norm, ziel_id) DO UPDATE SET
text = excluded.text, sub_norm = excluded.sub_norm, updated_at = excluded.updated_at""",
(topic, block_norm, ziel_id, text, sub_norm, _now()))
await db.commit()
async def list_lernziele(topic: str, block_norm: str | None = None) -> list[dict]:
db = await get_db()
if block_norm is None:
cursor = await db.execute(
"SELECT * FROM guide_lernziele WHERE topic = ? ORDER BY block_norm, ziel_id", (topic,))
else:
cursor = await db.execute(
"SELECT * FROM guide_lernziele WHERE topic = ? AND block_norm = ? ORDER BY ziel_id",
(topic, block_norm))
return [_row_to_dict(row, cursor) for row in await cursor.fetchall()]
async def set_ziel_covered(topic: str, block_norm: str, ziel_id: str, covered: bool) -> None:
db = await get_db()
await db.execute(
"UPDATE guide_lernziele SET covered = ?, updated_at = ? WHERE topic = ? AND block_norm = ? AND ziel_id = ?",
(1 if covered else 0, _now(), topic, block_norm, ziel_id))
await db.commit()
async def delete_lernziele(topic: str, block_norm: str) -> None:
db = await get_db()
await db.execute("DELETE FROM guide_lernziele WHERE topic = ? AND block_norm = ?", (topic, block_norm))
await db.commit()
async def kanban_membership(topic: str) -> dict[str, str]:
"""{member_id: group_id} for the whole topic (the cluster worker's working map)."""
db = await get_db()
cursor = await db.execute("SELECT member_id, group_id FROM kanban_members WHERE topic = ?", (topic,))
return {m: g for m, g in await cursor.fetchall()}
async def kanban_set_member(topic: str, member_id: str, group_id: str) -> None:
db = await get_db()
await db.execute(
"""INSERT INTO kanban_members (topic, member_id, group_id) VALUES (?, ?, ?)
ON CONFLICT(topic, member_id) DO UPDATE SET group_id = excluded.group_id""",
(topic, member_id, group_id))
await db.commit()
async def kanban_add_title(topic: str, board: str, card_id: str, title: str,
description: str, source: str, reader: str) -> bool:
"""Ingest one research title (stage 'ingest'). An exact dupe folds instead of duplicating:
reader-set union, source union, longer description wins; stage stays untouched (a title
already consumed into a cluster is not re-queued). NOT concurrency-safe across awaits —
callers serialize through one ingest lock. → True if the card is new."""
row = await kanban_get_card(topic, board, card_id)
if row is None:
payload = {"title": title, "description": description,
"sources": [source] if source else [], "readers": [reader]}
db = await get_db()
await db.execute(
"""INSERT INTO kanban_cards (topic, board, card_id, kind, stage, payload, updated_at)
VALUES (?, ?, ?, 'title', 'ingest', ?, ?)""",
(topic, board, card_id, json.dumps(payload, ensure_ascii=False), _now()))
await db.commit()
return True
p = row["payload"]
p["readers"] = list(dict.fromkeys((p.get("readers") or []) + [reader]))
p["sources"] = list(dict.fromkeys((p.get("sources") or []) + ([source] if source else [])))
if len(description or "") > len(p.get("description") or ""):
p["description"] = description
await kanban_set_payload(topic, board, card_id, p)
return False
async def upsert_subblock(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str) -> None:
db = await get_db()
await db.execute(

670
backend/guide.py
View File

@@ -24,7 +24,8 @@ from config import (
READABILITY_ACTIVE, TEMPLATES_DIR,
)
import readability
from database import list_guides, update_guide, list_blocks, list_subblocks, set_guide_content, get_guide_content, get_outline
from database import (list_guides, update_guide, list_blocks, list_subblocks, set_guide_content,
get_guide_content, get_outline, guide_stage_counts, delete_guide_board)
from fsutil import atomic_write_json, atomic_write_text
from jsonio import read_json_file as _json_file, parse_json_text as _parse_json_text
from paths import blocks_path, guide_content_path, project_dir, subblocks_path
@@ -41,20 +42,16 @@ from textkit import (
log = logging.getLogger("creator.guide")
GUIDE_STEPS = ("Outline", "Content", "Content-Check", "Writing", "Reading-Exam")
# Content/Content-Check/Reading-Exam run in packets of ~GUIDE_CHUNK blocks per agent.
# Only the writer (Writing) stays at 1 agent per block (variable lengths, no trimming, no
# length alignment between blocks).
GUIDE_CHUNK = 10
# Check steps as a panel: CHECK_PANEL judges per chunk, section flagged on a majority.
# A single judge is bias/sampling prone; a small panel is more stable.
CHECK_PANEL = 3
# Reading exam: only ONE round (Check + Fix). Follow-up rounds added little value
# (1 agent per block checks finely anyway) but cost extra agents.
READING_ROUNDS = 1
# Valid level values: new (learning path) + old (difficulty) backward-compatible.
@@ -93,26 +90,8 @@ def _level_label(s: dict) -> str:
return "peripheral" if s.get("relevance") == "peripheral" else (s.get("level") or "beginner")
def _assignment_subs(chunk: list[dict], entries: dict[int, str], subs_by_title: dict[str, list[dict]]) -> str:
"""Lists the blocks per chapter, with their subblocks and level labels beneath."""
lines: list[str] = []
for ch in chunk:
lines.append(f"CHAPTER: {ch['title']}")
for num in ch["nums"]:
lines.append(f"- {entries[num]}")
for s in subs_by_title.get(_title(entries[num]), []):
lines.append(f" [{_level_label(s)}] {s['title']}")
return "\n".join(lines)
def _guide_files(content_path: Path) -> dict:
d, stem = content_path.parent, content_path.stem
return {
"outline_slots": [d / f"{stem}.outline-{i}.json" for i in (1, 2, 3)],
"outline": d / f"{stem}.outline.json", # judge output
# chunk/reading-check/fix files are dynamic:
# {stem}.chunk-i.md, {stem}.lese-check-r{n}-{i}.json, {stem}.fix-r{n}-{i}.md
}
def guide_slot_files(content_path: Path) -> list[Path]:
@@ -120,132 +99,23 @@ def guide_slot_files(content_path: Path) -> list[Path]:
return [p for p in content_path.parent.glob(f"{content_path.stem}.*") if p != content_path]
def _done_path(content_path: Path) -> Path:
return content_path.parent / f"{content_path.stem}.done"
def guide_done_step(content_path: Path) -> int:
"""Highest FULLY completed step index (marker per topic+format). -1 = none.
If the content file exists, all steps are done."""
if content_path.exists():
return len(GUIDE_STEPS) - 1
try:
return int(_done_path(content_path).read_text(encoding="utf-8").strip())
except (OSError, ValueError):
return -1
def _set_done(content_path: Path, step: int) -> None:
"""Set marker to `step` — monotone (only increase), except on the re-run reset (force)."""
if step > guide_done_step(content_path):
atomic_write_text(_done_path(content_path), str(step))
def _reset_done(content_path: Path, step: int) -> None:
"""Set marker hard to `step` (for re-run from step; step may decrease)."""
if step < 0:
_done_path(content_path).unlink(missing_ok=True)
else:
atomic_write_text(_done_path(content_path), str(step))
# Slot-file globs per step (index = GUIDE_STEPS). Stem-anchored, collision-free.
_STEP_GLOBS = (
("outline*",), # 0 Outline (incl. selection filter)
("content-chunk-*", "content-nach-*"), # 1 Content (incl. follow-up round)
("content-check-*", "content-fix-*"), # 2 Content-Check
("chunk-*",), # 3 Writing (chunk-* also matches chunk-nach-*)
("lese-check-*", "fix-r*"), # 4 Reading-Exam
)
def _reset_guide_from_step(content_path: Path, step: int) -> None:
"""Re-run from step: delete content + all slot files of steps ≥ step.
Earlier steps stay → the resume rebuilds from `step` (everything below is reused)."""
content_path.unlink(missing_ok=True) # no longer "done" → no fresh-start wipe
d, stem = content_path.parent, content_path.stem
for globs in _STEP_GLOBS[step:]:
for pat in globs:
for p in d.glob(f"{stem}.{pat}"):
p.unlink(missing_ok=True)
_reset_done(content_path, step - 1) # steps < step count as done
def _read_problems_schema(data):
"""{"ok": true} → [] · {"problems": [{"section", "problem"}]} → list · else None."""
if not isinstance(data, dict):
return None
if data.get("ok") is True:
return []
p = data.get("problems")
if not isinstance(p, list) or not p:
return None
out = []
for x in p:
if not isinstance(x, dict) or not isinstance(x.get("section"), str) or not isinstance(x.get("problem"), str):
return None
out.append({"section": x["section"].strip(), "problem": x["problem"].strip()})
return out or None
def _panel_problems(judge_paths: list[Path], valid: set[int], idx: dict[str, int]) -> dict[int, str]:
"""Panel aggregation: several judge outputs of a chunk → flagged {num: problem}.
One vote per judge that names a section. Flagged when more than half of the
DELIVERED (validly parsed) judges name it (3→≥2, 2→≥2, 1→≥1). Robust against a
single failure: missing files do not count. Problem text from the first naming judge.
"""
outputs = [p for p in (_read_problems_schema(_json_file(j)) for j in judge_paths) if p is not None]
if not outputs:
return {}
votes: dict[int, int] = {}
problem: dict[int, str] = {}
for out in outputs:
seen: set[int] = set()
for item in out:
num = _resolve_title(idx, item["section"])
if num is None or num not in valid or num in seen:
continue
seen.add(num)
votes[num] = votes.get(num, 0) + 1
problem.setdefault(num, item["problem"])
threshold = len(outputs) / 2
return {num: problem[num] for num, v in votes.items() if v > threshold}
def _resolve_outline(data, entries: dict[int, str], target_min: int, target_max: int) -> list[dict] | None:
"""{"chapters": [{"title", "numbers": [1, 3, 7]}]} → [{"title", "nums"}].
Numbers are the IDs from `entries` (1-based, as presented to the agent).
`target_min`/`target_max` = allowed range of selected blocks (with a small tolerance).
"""
if not isinstance(data, dict) or not isinstance(data.get("chapters"), list):
return None
valid = set(entries)
chapters: list[dict] = []
seen: set[int] = set()
total = unknown = 0
for ch in data["chapters"]:
if not isinstance(ch, dict) or not isinstance(ch.get("numbers"), list):
return None
nums = []
for t in ch["numbers"]:
total += 1
num = t if isinstance(t, int) and not isinstance(t, bool) else None
if num is None or num not in valid:
unknown += 1
elif num not in seen:
nums.append(num)
seen.add(num)
if nums:
chapters.append({"title": str(ch.get("title", "")).strip() or "Chapter", "nums": nums})
if not chapters or total == 0:
return None
if (total - unknown) / total < 0.85:
return None
if len(seen) < 0.9 * target_min or len(seen) > 1.1 * target_max:
return None
return chapters
def _fallback_outline(entries: dict[int, str]) -> list[dict]:
@@ -324,515 +194,6 @@ async def _outline_from_db(topic: str, sel_entries: dict[int, str]) -> list[dict
return plan or None
async def _generate_sections(
guide_id: str, topic: str, format_name: str, entries: dict[int, str],
facts: str, instructions: str, provider: str,
content_path: Path,
) -> list[dict] | None:
def is_cancelled() -> bool:
return is_guide_cancelled(guide_id)
ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled, guide_id=guide_id)
spec = (TEMPLATES_DIR / "Format" / "Section.md").read_text(encoding="utf-8")
files = _guide_files(content_path)
zweck = FORMAT_PURPOSE[format_name]
# Subblocks per block (DB-first) — loaded early: drives selection + sub-filter per format.
# Missing → {} (fallback: guide takes everything).
subs_raw = await _load_subblocks(topic)
# Extract-once grounding: stored, verified facts replace the generic source hint.
# The content agent phrases from them instead of reading the source again.
if (facts_block := _facts_grounding(subs_raw)):
facts = facts_block
def _has_relevance(num, kind):
return any(isinstance(s, dict) and s.get("relevance") == kind for s in subs_raw.get(_title(entries[num]), []))
# Selection: ONE full document with ALL blocks (incl. peripheral). The views E/M/S/F
# filter later per subblock level. (FullGuide/Rest remain as legacy branches.)
if format_name == "Rest":
selection = [num for num in entries if not _has_relevance(num, "relevant")]
else: # Guide / FullGuide → all blocks
selection = list(entries)
if not selection:
await _fail(guide_id, "No matching blocks for this format")
return None
sel_entries = {num: entries[num] for num in selection}
target = len(sel_entries)
# Numbered list (ID = block number from entries) — agents/judge order by number.
sel_list = "\n".join(f"{num}. {t}" for num, t in sel_entries.items())
# Step 0: outline. Prefers the blocks artifact (DB) — the guide only presents,
# no longer structures itself. Missing (legacy) → previous agents/judge logic as fallback.
# 0 valid → code fallback, 1 → direct, ≥2 → judge (with proposal as fallback).
plan = await _outline_from_db(topic, sel_entries)
if plan is not None:
_log(topic, f"Outline from blocks artifact ({len(plan)} chapters)")
if plan is None:
plan = _resolve_outline(_json_file(files["outline"]), sel_entries, target, target)
if plan is None:
await _set_step(guide_id, 0, "Outline proposals (3 agents)…")
files["outline"].unlink(missing_ok=True)
proposals: list[list[dict]] = []
pending = []
for i, path in enumerate(files["outline_slots"], 1):
res = _resolve_outline(_json_file(path), sel_entries, target, target)
if res is not None:
proposals.append(res)
else:
pending.append((i, path))
if len(proposals) < 3 and pending:
slots = [
{
"key": f"{guide_id}-outline-{i}",
"prompt": _prompt(
"Guide-Outline",
topic=topic, format_name=format_name, blocks=sel_list,
out_path=path, extra=_extra(instructions),
),
"role": "guide", "capabilities": "files",
"payload": (lambda result, p=path: _resolve_outline(_json_file(p), sel_entries, target, target)),
}
for i, path in pending
]
# Quorum 1: take whatever comes — no minimum requirement, no abort.
new = await _race(
topic, "Outline", slots, 1, _timeout("plan", target),
provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE,
)
if is_cancelled():
return None
proposals += new or []
if not proposals:
_log(topic, "Outline: no valid proposal — deterministic fallback")
plan = _fallback_outline(sel_entries)
elif len(proposals) == 1:
plan = proposals[0] # one proposal → no judge needed
else:
await _set_step(guide_id, 0, "Merging outlines…")
proposals_text = "\n\n".join(
f"### Proposal {i}\n"
+ "\n".join(f"CHAPTER: {ch['title']}\n Numbers: {', '.join(str(num) for num in ch['nums'])}" for ch in v)
for i, v in enumerate(proposals, 1)
)
status, plan = await run_single_slot(
ctx, "Outline-Judge",
key=f"{guide_id}-outline-judge",
prompt=_prompt(
"Guide-Outline-Judge",
topic=topic, format_name=format_name, purpose=zweck, n=len(proposals),
blocks=sel_list, outlines=proposals_text,
out_path=files["outline"], extra=_extra(instructions),
),
role="judge", capabilities="files",
payload=lambda result: _resolve_outline(_json_file(files["outline"]), sel_entries, target, target),
timeout=_timeout("plan_judge", target),
)
if status == CANCELLED:
return None
if status == FAILED or plan is None:
_log(topic, "Outline judge produced no result — best proposal kept")
plan = proposals[0]
# Guarantee: every selected block is in the plan (against dropping agents/judges).
plan = _with_remainder(plan, sel_entries)
_set_done(content_path, 0) # outline ready
# Coarse chunks (~GUIDE_CHUNK blocks per agent) for content, content-check and reading-exam.
# The writer builds per block beneath (its own fine chunks, see below) → variable lengths.
total_sections = sum(len(c["nums"]) for c in plan)
chunks = _split_chunks(plan, max(1, math.ceil(total_sections / GUIDE_CHUNK)))
# Subblocks per block: Guide/FullGuide take ALL (incl. peripheral → level 4 in the view);
# only the legacy Rest branch filters to peripheral. So the one document carries all levels.
if format_name == "Rest":
subs_by_title = {t: [s for s in subs if s.get("relevance") == "peripheral"] for t, subs in subs_raw.items()}
else: # Guide / FullGuide
subs_by_title = {t: list(subs) for t, subs in subs_raw.items()}
subs_by_title = {t: subs for t, subs in subs_by_title.items() if subs}
assignments = [_assignment_subs(chunk, entries, subs_by_title) for chunk in chunks]
chunk_sizes = [sum(len(c["nums"]) for c in chunk) for chunk in chunks]
writer_count = len(chunks)
idx = _title_index(entries)
# Step 2: identify content per block — one agent per chunk (marker output, resume).
content_paths = [content_path.parent / f"{content_path.stem}.content-chunk-{i}.md" for i in range(1, writer_count + 1)]
pending = [i for i, p in enumerate(content_paths) if not p.exists()]
if pending:
async def report(d, t): await _set_step(guide_id, 1, f"Gathering content {d}/{t}…")
results = await _gather_progress([
run_agent(
f"{guide_id}-content-{i + 1}",
_prompt(
"Guide-Content",
topic=topic, assignment=assignments[i], facts=facts,
out_path=content_paths[i], extra=_extra(instructions),
),
_timeout("content", chunk_sizes[i]), provider=provider, role="guide", capabilities="full",
)
for i in pending
], writer_count, report, start=writer_count - len(pending))
if is_cancelled():
return None
if not any(p.exists() for p in content_paths):
await _fail(guide_id, _gather_error("Content error", list(results)))
return None
content_by_num: dict[int, str] = {}
for p in content_paths:
if not p.exists():
continue
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
num = _resolve_title(idx, sec["title"])
if num is not None and num not in content_by_num and sec["md"].strip():
content_by_num[num] = sec["md"]
if not content_by_num:
await _fail(guide_id, "No content identified")
return None
# Follow-up round: pull missing blocks (chunk failure or lazy output) deliberately — one round.
planned_nums = [num for ch in plan for num in ch["nums"]]
missing = [num for num in planned_nums if num not in content_by_num]
if missing:
_log(topic, f"Content: {len(missing)} block(s) missing — follow-up round…")
followup_chunks = [[{"title": "Additional", "nums": missing[k:k + GUIDE_CHUNK]}] for k in range(0, len(missing), GUIDE_CHUNK)]
followup_paths = [content_path.parent / f"{content_path.stem}.content-nach-{k}.md" for k in range(1, len(followup_chunks) + 1)]
followup_pending = [k for k, p in enumerate(followup_paths) if not p.exists()]
if followup_pending:
async def report_n(d, t): await _set_step(guide_id, 1, f"Gathering missing content {d}/{t}…")
await _gather_progress([
run_agent(
f"{guide_id}-content-nach-{k + 1}",
_prompt(
"Guide-Content",
topic=topic, assignment=_assignment_subs(followup_chunks[k], entries, subs_by_title),
facts=facts, out_path=followup_paths[k], extra=_extra(instructions),
),
_timeout("content", len(followup_chunks[k][0]["nums"])), provider=provider, role="guide", capabilities="full",
)
for k in followup_pending
], len(followup_chunks), report_n, start=len(followup_chunks) - len(followup_pending))
if is_cancelled():
return None
for p in followup_paths:
if not p.exists():
continue
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
num = _resolve_title(idx, sec["title"])
if num is not None and num not in content_by_num and sec["md"].strip():
content_by_num[num] = sec["md"]
if all(p.exists() for p in content_paths):
_set_done(content_path, 1) # content complete
content_chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in content_by_num] for chunk in chunks]
# Step 3: check content — CHECK_PANEL judges per chunk, majority flags.
# + revise flagged ones once. Resume: only restart missing judge files.
check_judge_paths = [
[content_path.parent / f"{content_path.stem}.content-check-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)]
for i in range(1, writer_count + 1)
]
pending_slots = [
(i, j) for i in range(writer_count) if content_chunk_nums[i]
for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None
]
if pending_slots:
await _set_step(guide_id, 2, "Checking content…")
sections_per_chunk = {
i: "\n\n".join(f"SECTION: {_title(entries[num])}\n{content_by_num[num]}" for num in content_chunk_nums[i])
for i, _ in pending_slots
}
slots = [{
"key": f"{guide_id}-content-check-{i + 1}-j{j + 1}",
"prompt": _prompt(
"Guide-Content-Check",
topic=topic, format_name=format_name, sections=sections_per_chunk[i],
out_path=check_judge_paths[i][j], extra=_extra(instructions),
),
"role": "judge", "capabilities": "files",
"payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))),
} for i, j in pending_slots]
n_checks = len(slots)
upd = lambda n: asyncio.create_task(_set_step(guide_id, 2, f"Checking content {n}/{n_checks}…"))
await _race(topic, "Content-Exam", slots, len(slots), _timeout("content_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
if is_cancelled():
return None
problems_by_num: dict[int, str] = {}
for i in range(writer_count):
if content_chunk_nums[i]:
problems_by_num.update(_panel_problems(check_judge_paths[i], set(content_chunk_nums[i]), idx))
if problems_by_num:
_log(topic, f"Content exam: {len(problems_by_num)} block(s) flagged")
await _set_step(guide_id, 2, f"Revising {len(problems_by_num)} content(s)…")
fix_chunks = [[num for num in nums if num in problems_by_num] for nums in content_chunk_nums]
fix_paths = [content_path.parent / f"{content_path.stem}.content-fix-{i + 1}.md" for i in range(writer_count)]
fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()]
results = await asyncio.gather(*[
run_agent(
f"{guide_id}-content-fix-{i + 1}",
_prompt(
"Guide-Content-Fix",
topic=topic, facts=facts,
tasks="\n\n".join(
f"SECTION: {_title(entries[num])}\nPROBLEM: {problems_by_num[num]}\nCURRENT:\n{content_by_num[num]}"
for num in fix_chunks[i]
),
out_path=fix_paths[i], extra=_extra(instructions),
),
_timeout("content", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full",
)
for i in fix_pending
], return_exceptions=True)
if is_cancelled():
return None
for p in fix_paths:
if not p.exists():
continue
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
num = _resolve_title(idx, sec["title"])
if num in problems_by_num and sec["md"].strip():
content_by_num[num] = sec["md"]
_set_done(content_path, 2) # content check done
# Step 4: writing — the writer phrases out the checked content (resume).
# FINE chunks: exactly 1 block per writer → variable lengths, no budget rationing.
def content_text(chunk) -> str:
nums = [num for ch in chunk for num in ch["nums"] if num in content_by_num]
return "\n\n".join(f"<!-- section: {_title(entries[num])} -->\n{content_by_num[num]}" for num in nums)
w_chunks = [[{"title": ch["title"], "nums": [num]}] for ch in plan for num in ch["nums"]]
w_assignments = [_assignment_subs(c, entries, subs_by_title) for c in w_chunks]
paths = [content_path.parent / f"{content_path.stem}.chunk-{i}.md" for i in range(1, len(w_chunks) + 1)]
pending = [i for i, p in enumerate(paths) if not p.exists()]
if pending:
async def report(d, t): await _set_step(guide_id, 3, f"Writing sections {d}/{t}…")
results = await _gather_progress([
run_agent(
f"{guide_id}-w{i + 1}",
_prompt(
"Guide-Writer",
topic=topic, format_name=format_name, assignment=w_assignments[i],
contents=content_text(w_chunks[i]),
spec=spec, out_path=paths[i], extra=_extra(instructions),
),
_timeout("writer", 1), provider=provider, role="guide", capabilities="files",
)
for i in pending
], len(w_chunks), report, start=len(w_chunks) - len(pending))
if is_cancelled():
return None
for i, r in zip(pending, results):
if isinstance(r, BaseException):
_log(topic, f"Writer {i + 1}: {type(r).__name__}: {r}")
elif r[0] != 0:
_log(topic, f"Writer {i + 1}: {_claude_error('Error', *r)}")
elif not paths[i].exists():
_log(topic, f"Writer {i + 1}: no output file created")
if not any(p.exists() for p in paths):
await _fail(guide_id, _gather_error("Writer error", list(results)))
return None
by_num: dict[int, dict] = {}
for p in paths:
if not p.exists():
continue
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
num = _resolve_title(idx, sec["title"])
if num is None:
_log(topic, f"Writer produced unknown section '{sec['title'][:40]}' (ignored)")
elif num not in by_num:
by_num[num] = sec
if not by_num:
await _fail(guide_id, "No sections found in writer output")
return None
# Follow-up round: write missing sections (writer failure) deliberately — one round.
missing_after = [num for num in planned_nums if num not in by_num]
if missing_after:
_log(topic, f"Writing: {len(missing_after)} section(s) missing — follow-up round…")
nw_chunks = [[{"title": "Additional", "nums": [num]}] for num in missing_after]
nw_paths = [content_path.parent / f"{content_path.stem}.chunk-nach-{k}.md" for k in range(1, len(nw_chunks) + 1)]
nw_pending = [k for k, p in enumerate(nw_paths) if not p.exists()]
if nw_pending:
async def report_nw(d, t): await _set_step(guide_id, 3, f"Writing missing sections {d}/{t}…")
await _gather_progress([
run_agent(
f"{guide_id}-w-nach-{k + 1}",
_prompt(
"Guide-Writer",
topic=topic, format_name=format_name, assignment=_assignment_subs(nw_chunks[k], entries, subs_by_title),
contents=content_text(nw_chunks[k]), spec=spec, out_path=nw_paths[k], extra=_extra(instructions),
),
_timeout("writer", 1), provider=provider, role="guide", capabilities="files",
)
for k in nw_pending
], len(nw_chunks), report_nw, start=len(nw_chunks) - len(nw_pending))
if is_cancelled():
return None
for p in nw_paths:
if not p.exists():
continue
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
num = _resolve_title(idx, sec["title"])
if num is not None and num not in by_num and sec["md"].strip():
by_num[num] = sec
if all(p.exists() for p in paths):
_set_done(content_path, 3) # writing complete
# Step 3: reading-exam loop — check per writer packet, fix only for
# flagged sections; follow-up rounds check ONLY the replaced sections.
# After the round cap, open complaints stand.
chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in by_num] for chunk in chunks]
def sections_text(nums: list[int]) -> str:
return "\n\n".join(f"SECTION: {_title(entries[num])}\n{by_num[num]['md']}" for num in nums)
def _sub_list(num: int) -> str:
subs = subs_by_title.get(_title(entries[num]), [])
return "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)"
def tasks_text(nums: list[int], problems: dict[int, str]) -> str:
return "\n\n".join(
f"SECTION: {_title(entries[num])}\n"
f"SUBBLOCKS (set one `<!-- sub: LABEL | title -->` marker each, label/order as here):\n{_sub_list(num)}\n"
f"PROBLEM: {problems[num]}\nCURRENT CONTENT:\n{by_num[num]['md']}"
for num in nums
)
scope = chunk_nums
for round_no in range(1, READING_ROUNDS + 1):
# CHECK_PANEL judges per packet; majority flags. Aggregation robust against a single failure.
check_judge_paths = [
[content_path.parent / f"{content_path.stem}.lese-check-r{round_no}-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)]
for i in range(1, writer_count + 1)
]
pending_slots = [
(i, j) for i in range(writer_count) if scope[i]
for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None
]
if pending_slots:
await _set_step(guide_id, 4, "Checking readability…")
sections_per_chunk = {i: sections_text(scope[i]) for i, _ in pending_slots}
slots = [{
"key": f"{guide_id}-lese-check-r{round_no}-{i + 1}-j{j + 1}",
"prompt": _prompt(
"Guide-Lese-Check",
topic=topic, format_name=format_name, spec=spec,
sections=sections_per_chunk[i],
out_path=check_judge_paths[i][j], extra=_extra(instructions),
),
"role": "judge", "capabilities": "files",
"payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))),
} for i, j in pending_slots]
n_checks = len(slots)
upd = lambda n: asyncio.create_task(_set_step(guide_id, 4, f"Checking readability {n}/{n_checks}…"))
res = await _race(topic, f"Reading-Exam r{round_no}", slots, len(slots), _timeout("lese_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
if is_cancelled():
return None
if res is None:
_log(topic, f"Reading exam round {round_no}: no full quorum — aggregated available judges")
problems_by_num: dict[int, str] = {}
for i in range(writer_count):
if scope[i]:
problems_by_num.update(_panel_problems(check_judge_paths[i], set(scope[i]), idx))
# Deterministic readability gate: queue too-hard sections into the same
# revision (LLM complaint takes precedence). Gate off → no-op.
if READABILITY_ACTIVE:
md_by_num = {num: by_num[num]["md"] for nums in scope for num in nums if num in by_num}
hints = await asyncio.to_thread(readability.rate_sections, md_by_num)
if hints:
_log(topic, f"Readability: {len(hints)} section(s) too hard")
for num, hint in hints.items():
problems_by_num.setdefault(num, hint)
if not problems_by_num:
break
_log(topic, f"Reading exam round {round_no}: {len(problems_by_num)} section(s) flagged")
await _set_step(guide_id, 4, f"Revising {len(problems_by_num)} section(s) (round {round_no})…")
fix_chunks = [[num for num in nums if num in problems_by_num] for nums in chunk_nums]
fix_paths = [content_path.parent / f"{content_path.stem}.fix-r{round_no}-{i + 1}.md" for i in range(writer_count)]
fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()]
results = await asyncio.gather(*[
run_agent(
f"{guide_id}-fix-r{round_no}-w{i + 1}",
_prompt(
"Guide-Sections-Fix",
topic=topic, format_name=format_name, facts=facts, spec=spec,
tasks=tasks_text(fix_chunks[i], problems_by_num),
out_path=fix_paths[i], extra=_extra(instructions),
),
_timeout("writer", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full",
)
for i in fix_pending
], return_exceptions=True)
if is_cancelled():
return None
for i, r in zip(fix_pending, results):
if isinstance(r, BaseException) or (not isinstance(r, BaseException) and r[0] != 0):
_log(topic, f"Sections fix {i + 1} (round {round_no}) failed — original kept")
replaced: set[int] = set()
for p in fix_paths:
if not p.exists():
continue
for sec in _parse_fragment(p.read_text(encoding="utf-8")):
num = _resolve_title(idx, sec["title"])
if num not in problems_by_num or not sec["md"].strip():
continue
# Marker invariant: if the fix loses the sub markers although the original had
# some, it is discarded — otherwise the level filter (E/M/S/F) dies silently.
if by_num[num].get("subs") and not sec.get("subs"):
_log(topic, f"Reading fix for '{sec['title']}' without sub markers — discarded, tagged original kept")
continue
by_num[num] = sec
replaced.add(num)
_log(topic, f"Reading exam round {round_no}: {len(replaced)} section(s) revised")
if not replaced:
break
if round_no == READING_ROUNDS:
_log(topic, f"Reading exam: 1 round — revision stays unchecked")
break
scope = [[num for num in nums if num in replaced] for nums in chunk_nums]
_set_done(content_path, 4) # reading exam done
# Checkable = format has an exam AND the block has ≥1 relevant subblock.
# Guide is always checkable (even without relevance data, fallback = everything).
def _checkable(num):
if format_name == "Guide":
return True
if format_name == "FullGuide":
return any(isinstance(s, dict) and s.get("relevance") == "relevant"
for s in subs_raw.get(_title(entries[num]), []))
return False # Rest etc. → pure reading sections
await _set_progress(guide_id, "Assembling…")
chapters: list[dict] = []
for ch in plan:
sections = [
{"num": num, "title": _title(entries[num]), "md": by_num[num]["md"],
"compact": by_num[num].get("compact", ""),
"anchor": by_num[num].get("anchor", ""), "anker_compact": by_num[num].get("anker_compact", ""),
"subs": by_num[num].get("subs", []), "checkable": _checkable(num)}
for num in ch["nums"] if num in by_num
]
if sections:
chapters.append({"title": ch["title"], "sections": sections})
planned = {num for ch in plan for num in ch["nums"]}
missing = sorted(planned - set(by_num))
if missing:
_log(topic, f"Sections missing from writer output: {[_title(entries[n]) for n in missing]}")
if not chapters:
await _fail(guide_id, "No sections found in writer output")
return None
return chapters
_LEVEL_RANK = {"beginner": 1, "advanced": 2, "expert": 3, "peripheral": 4,
@@ -892,14 +253,18 @@ async def generate_guide(guide_id: str, topic: str, format_name: str, instructio
if project:
await asyncio.to_thread(_convert_pdfs, project)
# Re-run from step: delete content + slots from `ab_step`, rest stays → resume rebuilds from there.
# Otherwise "recreate": a finished guide → complete fresh start.
# Otherwise step files are leftovers of an abort/error → resume.
import guide_board # lazy — guide_board imports helpers from this module
# Re-run from stage: cards from `ab_step` onward back to that column.
# A FINISHED guide without ab_step → complete fresh start (board + slots wiped).
# Otherwise cards are leftovers of an abort/error → resume at their stored stage.
if ab_step is not None:
_reset_guide_from_step(content_path, ab_step)
await guide_board.reset_from_stage(topic, format_name, ab_step)
elif content_path.exists():
for p_alt in guide_slot_files(content_path):
p_alt.unlink(missing_ok=True)
counts = await guide_stage_counts(topic, format_name)
if not counts or set(counts) == {"done"}:
await delete_guide_board(topic, format_name)
for p_alt in guide_slot_files(content_path):
p_alt.unlink(missing_ok=True)
bs = await list_blocks(topic, status="consensus")
if bs:
@@ -912,12 +277,13 @@ async def generate_guide(guide_id: str, topic: str, format_name: str, instructio
await _fail(guide_id, "No blocks found")
return
entries = _unique_title(alle)
facts = _prompt("Guide-Facts-Projekt", project=project) if project else _prompt("Guide-Facts-Thema")
chapters = await _generate_sections(
guide_id, topic, format_name, entries,
facts, instructions, provider, content_path,
chapters = await guide_board.run_guide_board(
guide_id, topic, format_name, entries, instructions, provider, content_path,
)
if chapters is None or is_guide_cancelled(guide_id):
if is_guide_cancelled(guide_id):
return
if chapters is None:
await _fail(guide_id, "No finished sections (see board — cards with errors)")
return
content = {"topic": topic, "format": format_name, "chapters": chapters}

521
backend/guide_board.py Normal file
View File

@@ -0,0 +1,521 @@
"""Board 3 „Guide": one card per block, linear stages with gates between them.
lernziele judge Backward Design — objectives BEFORE writing
zuweisung code chapter/order from the outline artefact + facts grounding
writer guide ONE coherent per-block text, only from VERIFIED FACTS
fakten_gate judge CoVe: atomic claims, each binary against the facts → minimal fix
coverage judge objective↔section mapping; gap → back to writer (max 2 rounds)
lesbarkeit judge Lese-Check + deterministic readability gate → fix → done
Runner: one asyncio task per card (cards are fixed from the start — no queue engine
needed); stage transitions are persisted in guide_cards, so the board is live and
cancel/resume just picks cards up at their stored stage. Assembly keeps the exact
legacy content format → content_fuer_level / TopicDetail stay untouched.
"""
import asyncio
import json
import logging
import re
import database as db
import readability
from config import FORMAT_PURPOSE, READABILITY_ACTIVE, TEMPLATES_DIR
from fsutil import atomic_write_json
from jsonio import read_json_file as _json_file
from pipeline import (CANCELLED, FAILED, OK, GenContext, _extra, _log, _prompt,
_timeout, is_guide_cancelled, run_single_slot)
from textkit import _norm_title, _parse_fragment, _title
log = logging.getLogger("creator.guide_board")
GUIDE_STAGES = ("lernziele", "zuweisung", "writer", "fakten_gate", "coverage", "lesbarkeit")
STAGE_LABELS = {"lernziele": "Lernziele", "zuweisung": "Zuweisung", "writer": "Writer",
"fakten_gate": "Fakten-Gate", "coverage": "Coverage",
"lesbarkeit": "Lesbarkeit", "done": "Fertig"}
MAX_WRITER_ROUNDS = 2 # coverage → writer feedback loop cap (gains die after round 1–2)
CARD_CONCURRENCY = 10 # simultaneous cards (the per-topic agent semaphore is the hard cap)
def _safe(norm: str) -> str:
return re.sub(r"\W+", "_", norm)[:50] or "block"
def _ziele_schema(data):
"""{"ziele":[{id,text,sub}]} → list of dicts · None on invalid structure."""
if not isinstance(data, dict) or not isinstance(data.get("ziele"), list) or not data["ziele"]:
return None
out, seen = [], set()
for z in data["ziele"]:
if not isinstance(z, dict):
return None
zid = str(z.get("id", "")).strip()
text = str(z.get("text", "")).strip()
if not zid or not text or zid in seen or len(out) >= 12:
continue
seen.add(zid)
out.append({"id": zid, "text": text, "sub": str(z.get("sub", "")).strip()})
return out or None
def _gate_schema(data):
"""{"ok":true} → [] · {"claims":[{text,grund}]} → list · None invalid."""
if not isinstance(data, dict):
return None
if data.get("ok") is True:
return []
claims = data.get("claims")
if not isinstance(claims, list) or not claims:
return None
out = []
for c in claims:
if isinstance(c, dict) and str(c.get("text", "")).strip():
out.append({"text": str(c["text"]).strip(), "grund": str(c.get("grund", "")).strip()})
return out
def _coverage_schema(data, ziel_ids: set[str]):
"""{"ziele":{id:bool}, "luecken":[{ziel,fehlt}], "ballast":[str]} — ziele must cover all ids."""
if not isinstance(data, dict) or not isinstance(data.get("ziele"), dict):
return None
ziele = {}
for k, v in data["ziele"].items():
ziele[str(k)] = str(v).strip().casefold() in ("true", "ja", "yes", "1")
if not ziel_ids <= set(ziele):
return None
luecken = [{"ziel": str(l.get("ziel", "")), "fehlt": str(l.get("fehlt", ""))}
for l in data.get("luecken", []) if isinstance(l, dict) and str(l.get("fehlt", "")).strip()]
ballast = [str(b).strip() for b in data.get("ballast", []) if str(b).strip()]
return {"ziele": ziele, "luecken": luecken, "ballast": ballast}
def _problems_schema(data):
"""Lese-Check: {"ok":true} → [] · {"problems":[{section,problem}]} → [problem…]."""
if not isinstance(data, dict):
return None
if data.get("ok") is True:
return []
probs = data.get("problems")
if not isinstance(probs, list) or not probs:
return None
out = [str(p.get("problem", "")).strip() for p in probs
if isinstance(p, dict) and str(p.get("problem", "")).strip()]
return out or None
def _first_section(md: str) -> dict | None:
secs = _parse_fragment(md) if md else []
return secs[0] if secs else None
class _Env:
"""Shared per-run context for the card tasks."""
def __init__(self, ctx, guide_id, topic, format_name, instructions, content_path,
subs_by_title, chapter_map, fallback_facts, spec):
self.ctx = ctx
self.guide_id = guide_id
self.topic = topic
self.format = format_name
self.instructions = instructions
self.content_path = content_path
self.subs_by_title = subs_by_title # block title → [sub dicts]
self.chapter_map = chapter_map # block_norm → (chapter title, ord)
self.fallback_facts = fallback_facts # generic source hint (legacy topics without facts)
self.spec = spec
def slot(self, name: str):
return self.content_path.parent / f"{self.content_path.stem}.{name}"
def _card_facts(env: _Env, block_title: str) -> str:
from guide import _facts_grounding # lazy: guide imports this module
grounding = _facts_grounding({block_title: env.subs_by_title.get(block_title, [])})
return grounding or env.fallback_facts
def _card_assignment(env: _Env, card: dict) -> str:
from guide import _level_label
lines = [f"- {card['block']}"]
for s in env.subs_by_title.get(card["block"], []):
lines.append(f" [{_level_label(s)}] {s['title']}")
return "\n".join(lines)
async def _set(env: _Env, card: dict, **fields):
card.update(fields)
await db.set_guide_card(env.topic, env.format, card["block_norm"], **fields)
# ── Stages ─────────────────────────────────────────────────────────────────────────
async def _stage_lernziele(env: _Env, card: dict) -> bool:
norm = card["block_norm"]
if not await db.list_lernziele(env.topic, norm):
path = env.slot(f"ziele-{_safe(norm)}.json")
subs = "\n".join(f"- [{s.get('level', 'beginner')}] {s['title']}"
for s in env.subs_by_title.get(card["block"], [])) or "(keine)"
status, ziele = await run_single_slot(
env.ctx, f"Lernziele {card['block']}", key=f"{env.guide_id}-ziele-{_safe(norm)}",
prompt=_prompt("Guide-Lernziele", topic=env.topic, block=card["block"],
subs=subs, facts=_card_facts(env, card["block"]),
out_path=path, extra=_extra(env.instructions)),
role="judge", capabilities="files",
payload=lambda result: _ziele_schema(_json_file(path)),
timeout=_timeout("lernziele", len(env.subs_by_title.get(card["block"], []))))
if status == CANCELLED:
return False
if status == FAILED:
await _set(env, card, status="error", gate_info="Lernziele ohne Ergebnis")
return False
for z in ziele:
await db.put_lernziel(env.topic, norm, z["id"], z["text"], _norm_title(z["sub"]))
await _set(env, card, stage="zuweisung", status="open")
return True
async def _stage_zuweisung(env: _Env, card: dict) -> bool:
chapter, ord_ = env.chapter_map.get(card["block_norm"], ("Weitere Inhalte", 10_000))
await _set(env, card, chapter=chapter, ord=ord_, stage="writer")
return True
async def _stage_writer(env: _Env, card: dict) -> bool:
norm = card["block_norm"]
ziele = await db.list_lernziele(env.topic, norm)
ziele_text = "\n".join(f"- ({z['ziel_id']}) {z['text']}" for z in ziele) or "(keine definiert)"
gaps = ""
if card["writer_rounds"] > 0 and card.get("gate_info"):
gaps = ("\nREVISION ROUND — a previous version exists. Revise it: close exactly the gaps "
"below, cut the listed ballast, keep everything else as-is.\n"
f"PREVIOUS VERSION:\n{card.get('md', '')}\n\nGAPS/BALLAST:\n{card['gate_info']}\n")
path = env.slot(f"card-{_safe(norm)}-r{card['writer_rounds']}.md")
path.unlink(missing_ok=True)
def _payload(result):
text = path.read_text(encoding="utf-8") if path.exists() else ""
sec = _first_section(text)
return text if sec and sec.get("md", "").strip() else None
status, text = await run_single_slot(
env.ctx, f"Writer {card['block']}", key=f"{env.guide_id}-w-{_safe(norm)}-r{card['writer_rounds']}",
prompt=_prompt("Guide-Writer-Board", topic=env.topic, format_name=env.format,
chapter=card.get("chapter") or "Inhalte",
assignment=_card_assignment(env, card), ziele=ziele_text,
facts=_card_facts(env, card["block"]), gaps=gaps, spec=env.spec,
out_path=path, extra=_extra(env.instructions)),
role="guide", capabilities="files", payload=_payload,
timeout=_timeout("writer", 1))
if status == CANCELLED:
return False
if status == FAILED:
await _set(env, card, status="error", gate_info="Writer ohne Ergebnis")
return False
await _set(env, card, md=text, stage="fakten_gate", status="open")
return True
async def _stage_fakten_gate(env: _Env, card: dict) -> bool:
norm = card["block_norm"]
sec = _first_section(card["md"])
if sec is None:
await _set(env, card, status="error", gate_info="Writer-Fragment unlesbar")
return False
facts = _card_facts(env, card["block"])
path = env.slot(f"gate-{_safe(norm)}-r{card['writer_rounds']}.json")
status, claims = await run_single_slot(
env.ctx, f"Fakten-Gate {card['block']}", key=f"{env.guide_id}-gate-{_safe(norm)}-r{card['writer_rounds']}",
prompt=_prompt("Guide-Fakten-Gate", topic=env.topic, block=card["block"],
section=sec["md"], facts=facts, out_path=path,
extra=_extra(env.instructions)),
role="judge", capabilities="files",
payload=lambda result: _gate_schema(_json_file(path)),
timeout=_timeout("fakten_gate", 1))
if status == CANCELLED:
return False
if status == FAILED:
claims = [] # gate failure must not block the card — logged, text stands
_log(env.topic, f"Fakten-Gate {card['block']}: kein Ergebnis — Text bleibt ungeprüft")
if claims:
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} unbelegte Claims → Fix")
fixp = env.slot(f"gatefix-{_safe(norm)}-r{card['writer_rounds']}.md")
fixp.unlink(missing_ok=True)
claims_text = "\n".join(f"- {c['text']}" + (f" ({c['grund']})" if c["grund"] else "")
for c in claims)
def _fixload(result):
text = fixp.read_text(encoding="utf-8") if fixp.exists() else ""
return text if _first_section(text) else None
fstatus, fixed = await run_single_slot(
env.ctx, f"Fakten-Fix {card['block']}", key=f"{env.guide_id}-gatefix-{_safe(norm)}-r{card['writer_rounds']}",
prompt=_prompt("Guide-Fakten-Fix", topic=env.topic, block=card["block"],
section=card["md"], claims=claims_text, facts=facts,
out_path=fixp, extra=_extra(env.instructions)),
role="guide", capabilities="files", payload=_fixload, # Fix ≠ Gate-Modell (kein Selbst-Check)
timeout=_timeout("fakten_gate", 1))
if fstatus == CANCELLED:
return False
if fstatus == OK and fixed:
new_sec = _first_section(fixed)
# marker invariant: a fix that loses the sub markers kills the level filter → discard
if sec.get("subs") and not (new_sec and new_sec.get("subs")):
_log(env.topic, f"Fakten-Fix {card['block']} ohne Sub-Marker — verworfen")
else:
card["md"] = fixed
await _set(env, card, md=card["md"], stage="coverage", status="open")
return True
async def _stage_coverage(env: _Env, card: dict) -> bool:
norm = card["block_norm"]
ziele = await db.list_lernziele(env.topic, norm)
if not ziele:
await _set(env, card, stage="lesbarkeit")
return True
sec = _first_section(card["md"])
ziele_text = "\n".join(f"- ({z['ziel_id']}) {z['text']}" for z in ziele)
path = env.slot(f"coverage-{_safe(norm)}-r{card['writer_rounds']}.json")
ids = {z["ziel_id"] for z in ziele}
status, res = await run_single_slot(
env.ctx, f"Coverage {card['block']}", key=f"{env.guide_id}-cov-{_safe(norm)}-r{card['writer_rounds']}",
prompt=_prompt("Guide-Coverage", topic=env.topic, block=card["block"],
ziele=ziele_text, section=sec["md"] if sec else card["md"],
out_path=path, extra=_extra(env.instructions)),
role="judge", capabilities="files",
payload=lambda result: _coverage_schema(_json_file(path), ids),
timeout=_timeout("coverage", len(ziele)))
if status == CANCELLED:
return False
if status == FAILED:
_log(env.topic, f"Coverage {card['block']}: kein Ergebnis — weiter ohne Gate")
await _set(env, card, stage="lesbarkeit")
return True
for zid, ok in res["ziele"].items():
if zid in ids:
await db.set_ziel_covered(env.topic, norm, zid, ok)
if res["luecken"] and card["writer_rounds"] < MAX_WRITER_ROUNDS:
info = "\n".join(f"- Lücke ({l['ziel']}): {l['fehlt']}" for l in res["luecken"])
if res["ballast"]:
info += "\n" + "\n".join(f"- Ballast (kürzen): {b}" for b in res["ballast"])
_log(env.topic, f"Coverage {card['block']}: {len(res['luecken'])} Lücke(n) → Writer-Runde "
f"{card['writer_rounds'] + 1}")
await _set(env, card, writer_rounds=card["writer_rounds"] + 1, gate_info=info,
stage="writer", status="open")
return True
if res["luecken"]:
_log(env.topic, f"Coverage {card['block']}: Lücken bleiben nach {MAX_WRITER_ROUNDS} Runden")
await _set(env, card, gate_info="", stage="lesbarkeit")
return True
async def _stage_lesbarkeit(env: _Env, card: dict) -> bool:
norm = card["block_norm"]
sec = _first_section(card["md"])
if sec is None:
await _set(env, card, status="error", gate_info="Fragment unlesbar")
return False
problems: list[str] = []
path = env.slot(f"lese-{_safe(norm)}-r{card['writer_rounds']}.json")
status, res = await run_single_slot(
env.ctx, f"Lese-Check {card['block']}", key=f"{env.guide_id}-lese-{_safe(norm)}",
prompt=_prompt("Guide-Lese-Check", topic=env.topic, format_name=env.format,
spec=env.spec, sections=f"SECTION: {card['block']}\n{sec['md']}",
out_path=path, extra=_extra(env.instructions)),
role="judge", capabilities="files",
payload=lambda result: _problems_schema(_json_file(path)),
timeout=_timeout("lese_check", 1))
if status == CANCELLED:
return False
if status == OK and res:
problems += res
if READABILITY_ACTIVE: # deterministic gate, external grounding
hints = await asyncio.to_thread(readability.rate_sections, {1: sec["md"]})
if hints.get(1):
problems.append(hints[1])
if problems:
from guide import _level_label
subs = env.subs_by_title.get(card["block"], [])
sub_list = "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)"
tasks = (f"SECTION: {card['block']}\n"
f"SUBBLOCKS (set one `<!-- sub: LABEL | title -->` marker each, label/order as here):\n{sub_list}\n"
f"PROBLEM: {' · '.join(problems)}\nCURRENT CONTENT:\n{sec['md']}")
fixp = env.slot(f"lesefix-{_safe(norm)}.md")
fixp.unlink(missing_ok=True)
def _fixload(result):
text = fixp.read_text(encoding="utf-8") if fixp.exists() else ""
return text if _first_section(text) else None
fstatus, fixed = await run_single_slot(
env.ctx, f"Lese-Fix {card['block']}", key=f"{env.guide_id}-lesefix-{_safe(norm)}",
prompt=_prompt("Guide-Sections-Fix", topic=env.topic, format_name=env.format,
facts=_card_facts(env, card["block"]), spec=env.spec, tasks=tasks,
out_path=fixp, extra=_extra(env.instructions)),
role="guide", capabilities="files", payload=_fixload,
timeout=_timeout("writer", 1))
if fstatus == CANCELLED:
return False
if fstatus == OK and fixed:
new_sec = _first_section(fixed)
if sec.get("subs") and not (new_sec and new_sec.get("subs")):
_log(env.topic, f"Lese-Fix {card['block']} ohne Sub-Marker — verworfen")
else:
card["md"] = fixed
await _set(env, card, md=card["md"], stage="done", status="ok", gate_info="")
return True
_STAGE_FN = {"lernziele": _stage_lernziele, "zuweisung": _stage_zuweisung,
"writer": _stage_writer, "fakten_gate": _stage_fakten_gate,
"coverage": _stage_coverage, "lesbarkeit": _stage_lesbarkeit}
async def _run_card(env: _Env, card: dict, sem: asyncio.Semaphore) -> None:
async with sem:
while card["stage"] != "done":
if is_guide_cancelled(env.guide_id):
await _set(env, card, status="open") # no longer being worked
return
fn = _STAGE_FN.get(card["stage"])
if fn is None: # unknown stage → park as error
await _set(env, card, status="error", gate_info=f"Unbekannte Stage {card['stage']}")
return
if card["status"] != "active":
await _set(env, card, status="active") # live board: this card is being worked
try:
if not await fn(env, card):
return
except Exception as e:
log.exception("[%s] guide card %s failed", env.topic, card["block"])
await _set(env, card, status="error", gate_info=f"{type(e).__name__}: {e}"[:300])
return
# ── Orchestration ──────────────────────────────────────────────────────────────────
async def _chapter_map(topic: str, entries: dict[int, str]) -> dict[str, tuple[str, int]]:
"""block_norm → (chapter title, global order) from the outline artefact."""
from guide import _outline_from_db, _fallback_outline, _with_remainder
plan = await _outline_from_db(topic, entries) or _fallback_outline(entries)
plan = _with_remainder(plan, entries)
out: dict[str, tuple[str, int]] = {}
i = 0
for ch in plan:
for num in ch.get("nums", []):
if num in entries:
out[_norm_title(_title(entries[num]))] = (ch.get("title") or "Kapitel", i)
i += 1
return out
async def run_guide_board(guide_id: str, topic: str, format_name: str, entries: dict[int, str],
instructions: str, provider: str, content_path) -> list[dict] | None:
"""Seed one card per block (existing cards keep their stage — resume), run all cards,
assemble the chapters in the legacy content format. → chapters | None (cancel/empty)."""
from blocks import source_folder
from guide import _load_subblocks
ctx = GenContext(topic=topic, provider=provider,
is_cancelled=lambda: is_guide_cancelled(guide_id), guide_id=guide_id)
spec = (TEMPLATES_DIR / "Format" / "Section.md").read_text(encoding="utf-8")
subs_raw = await _load_subblocks(topic)
project = source_folder(topic)
fallback = (_prompt("Guide-Facts-Projekt", project=project) if project
else _prompt("Guide-Facts-Thema"))
env = _Env(ctx, guide_id, topic, format_name, instructions, content_path,
subs_raw, await _chapter_map(topic, entries), fallback, spec)
for num, line in entries.items():
title = _title(line)
await db.upsert_guide_card(topic, format_name, _norm_title(title), title)
cards = await db.list_guide_cards(topic, format_name)
open_cards = [c for c in cards if c["stage"] != "done"]
if open_cards:
sem = asyncio.Semaphore(CARD_CONCURRENCY)
async def _progress():
while True:
counts = await db.guide_stage_counts(topic, format_name)
done = counts.get("done", 0)
total = sum(counts.values())
await db.update_guide(guide_id, progress=f"Board: {done}/{total} Karten fertig")
await asyncio.sleep(2.0)
reporter = asyncio.create_task(_progress())
try:
await asyncio.gather(*[_run_card(env, c, sem) for c in open_cards])
finally:
reporter.cancel()
if is_guide_cancelled(guide_id):
return None
# assembly — identical shape to the legacy pipeline
cards = await db.list_guide_cards(topic, format_name)
chapters: list[dict] = []
by_chapter: dict[str, list[dict]] = {}
order: list[str] = []
for c in sorted(cards, key=lambda c: (c["ord"], c["block_norm"])):
if c["stage"] != "done":
_log(topic, f"Guide: Karte '{c['block']}' nicht fertig ({c['stage']}) — Abschnitt fehlt")
continue
sec = _first_section(c["md"])
if sec is None:
continue
ch = c["chapter"] or "Inhalte"
if ch not in by_chapter:
by_chapter[ch] = []
order.append(ch)
by_chapter[ch].append({
"num": c["ord"], "title": c["block"], "md": sec["md"],
"compact": sec.get("compact", ""), "anchor": sec.get("anchor", ""),
"anker_compact": sec.get("anker_compact", ""), "subs": sec.get("subs", []),
"checkable": format_name == "Guide" or bool(
any(s.get("relevance") == "relevant" for s in subs_raw.get(c["block"], []))),
})
for ch in order:
chapters.append({"title": ch, "sections": by_chapter[ch]})
return chapters or None
async def done_step(topic: str, format_name: str) -> int:
"""Sidebar dots: highest fully completed stage index. -1 = nothing, len(stages) at done."""
counts = await db.guide_stage_counts(topic, format_name)
if not counts:
return -1
if set(counts) == {"done"}:
return len(GUIDE_STAGES)
lowest = min(GUIDE_STAGES.index(s) for s in counts if s in GUIDE_STAGES)
return lowest - 1 if lowest > 0 else -1
async def board_snapshot(topic: str, format_name: str, limit: int = 20) -> dict:
"""Live guide board: columns with counts + cards (title, rounds, covered objectives)."""
cards = await db.list_guide_cards(topic, format_name)
ziele = {}
for z in await db.list_lernziele(topic):
d = ziele.setdefault(z["block_norm"], [0, 0])
d[1] += 1
d[0] += 1 if z["covered"] else 0
columns = []
for stage in (*GUIDE_STAGES, "done"):
in_stage = [c for c in cards if c["stage"] == stage]
views = []
for c in in_stage[:limit]:
zc = ziele.get(c["block_norm"])
views.append({"title": c["block"],
"status": c["status"] if c["status"] in ("error", "active") else "open",
"rounds": c["writer_rounds"],
"info": c["gate_info"][:200] if c["status"] == "error" else "",
"ziele": f"{zc[0]}/{zc[1]}" if zc else ""})
columns.append({"key": stage, "label": STAGE_LABELS[stage],
"total": len(in_stage), "cards": views})
return {"columns": columns}
async def reset_from_stage(topic: str, format_name: str, ab_stage: int) -> int:
"""Cards in stages ≥ ab_stage (incl. done) back to GUIDE_STAGES[ab_stage]."""
ab_stage = max(0, min(ab_stage, len(GUIDE_STAGES) - 1))
target = GUIDE_STAGES[ab_stage]
stages = list(GUIDE_STAGES[ab_stage:]) + ["done"]
if ab_stage == 0:
for c in await db.list_guide_cards(topic, format_name):
await db.delete_lernziele(topic, c["block_norm"])
moved = await db.reset_guide_cards_from_stage(topic, format_name, stages, target,
clear_md=ab_stage <= 2)
return moved

265
backend/kanban.py Normal file
View File

@@ -0,0 +1,265 @@
"""Generic streaming kanban engine (no concrete stages — boards define those).
Each column is a worker that pulls cards from its input `stage` (the queue = kanban_cards rows
WHERE stage = <input>), processes up to KANBAN_BATCH at a time, and advances them. Streaming
columns run continuously; BARRIER columns start only at QUIESCENCE of every stage before them
(no active worker + no queued card). SERIAL columns process one package at a time (their
processor mutates shared cross-card state).
Failure handling: a processor exception (including parse-fails it raises) sends the package's
unadvanced cards into exponential backoff (retries++, not_before); after MAX_CARD_RETRIES the
card goes to stage 'dead' (dead-letter — visible on the board, requeue-able via API). No card
is ever deleted by the engine.
Board definitions live in board_inventory.py / board_artefacts.py; run via run_flow().
"""
import asyncio
import logging
import database as db
from config import MAX_CONCURRENT_AGENTS_PER_TOPIC
log = logging.getLogger("creator.kanban")
KANBAN_BATCH = 5 # cards a worker pulls per package (micro-batching)
# How many packages ONE worker keeps in flight at once. A worker no longer blocks on a single
# package — it keeps pulling and dispatching until this many run concurrently, so a busy column
# fills the agent slots (the per-topic semaphore is the real cap; over-dispatch just queues cheaply).
WORKER_INFLIGHT = MAX_CONCURRENT_AGENTS_PER_TOPIC
_POLL = 0.3 # seconds between empty-queue polls
MAX_CARD_RETRIES = 3 # failures per card until dead-letter
RETRY_BACKOFF = 30.0 # base seconds; backoff = base · 2^(retries-1)
# Live registry of running flows (topic → Flow), so routes can attach research agents,
# report `generating`, and cancel.
active_flows: dict[str, "Flow"] = {}
class Flow:
"""Shared runtime state of one topic run: active-task counters per stage + a wakeup event.
`producers` counts running research agents (initial + any added live); research counts as done
only when ALL producers have finished, so the flow stays awake while extras still search."""
def __init__(self, topic: str, work_dir=None):
self.topic = topic
self.work_dir = work_dir
self.active: dict[str, int] = {}
self.producers = 0
self.producer_tag = 0
self.stop = False
self.wake = asyncio.Event()
self.spawn_research = None # set by the board: () → coroutine adding one more research agent
self.state: dict = {} # board-private shared state (embedding caches, one-shot flags …)
self.active_cards: set[str] = set() # "board:card_id" currently inside a processor (live display)
@property
def research_done(self) -> bool:
return self.producers <= 0
def add_producer(self):
"""MUST be called synchronously BEFORE create_task of the producer — otherwise workers
can pass their exit check in the gap and never see the new producer (quiescence race)."""
self.producers += 1
self.wake.set()
def done_producer(self):
self.producers -= 1
self.wake.set()
def next_tag(self) -> int:
self.producer_tag += 1
return self.producer_tag
def enter(self, stage: str):
self.active[stage] = self.active.get(stage, 0) + 1
def leave(self, stage: str):
self.active[stage] = max(0, self.active.get(stage, 0) - 1)
self.wake.set()
def active_in(self, stages) -> bool:
return any(self.active.get(s, 0) > 0 for s in stages)
class Stage:
"""One column: board + stage name + processor. `upstream` (all stages before it, across
boards) is filled by chain_stages(). process(cards) gets the pulled package (list of card
dicts with decoded payload).
barrier: pull only when every upstream stage is quiescent (relational judgements need the
full set). gate: extra callable that must be truthy before the stage pulls (works without
barrier too — e.g. the consensus gate holds cards until research is done so late reader
votes still count). drain: pull the WHOLE queue as one package (global passes like the
fragment filter); implies serial."""
def __init__(self, board: str, stage: str, process, *, barrier: bool = False,
serial: bool = False, gate=None, drain: bool = False):
self.board = board
self.stage = stage
self.process = process
self.barrier = barrier
self.serial = serial or drain
self.gate = gate
self.drain = drain
self.upstream: list[str] = []
def chain_stages(stages: list[Stage]) -> list[Stage]:
"""Fill each stage's upstream = every stage listed before it (list order = flow order).
Producers are upstream of everything implicitly via flow.research_done."""
seen: list[str] = []
for s in stages:
s.upstream = list(seen)
seen.append(s.stage)
return stages
async def quiescent(flow: Flow, stages) -> bool:
"""True iff no worker is active in `stages` AND no card is queued in any of them.
The barrier/exit condition — must include QUEUED cards, not just active workers, or a worker
could exit in a momentary lull while an upstream worker still has work to push down."""
if not stages:
return True
if flow.active_in(stages):
return False
return await db.kanban_count(flow.topic, list(stages)) == 0
async def _sleep_wake(flow: Flow):
try:
await asyncio.wait_for(flow.wake.wait(), timeout=_POLL)
except asyncio.TimeoutError:
pass
flow.wake.clear()
async def _fail_package(flow: Flow, spec: Stage, cards: list[dict], error: str):
"""Backoff/dead-letter for the cards the processor did NOT advance (their stage is unchanged —
advanced cards must not be punished for a failure after their move)."""
for c in cards:
cur = await db.kanban_get_card(flow.topic, spec.board, c["card_id"])
if cur is None or cur["stage"] != spec.stage:
continue
dead = await db.kanban_fail_card(flow.topic, spec.board, c["card_id"], error,
MAX_CARD_RETRIES, RETRY_BACKOFF)
if dead:
log.warning("kanban %s/%s: card %s → dead (%s)", flow.topic, spec.stage, c["card_id"], error)
async def _worker(flow: Flow, spec: Stage, inflight: int, all_stages: list[str]):
"""Pull cards from spec.stage, run spec.process — keeping up to `inflight` packages running
CONCURRENTLY so a busy column fills the agent slots. A barrier worker only pulls when upstream
is fully quiescent (and its gate, if any, is open). ANY worker exits only when research is
done and the WHOLE flow is quiescent — global instead of per-stage, so a downstream stage
that feeds cards back upstream (gap-check → ingest) never strands work. Double-checked over
one grace sleep (a producer attached in the lull keeps the flow alive).
Double-pull safety: each stage has exactly ONE worker, so an in-memory `claimed` set of
card-ids (held while a package runs) keeps concurrent pulls from grabbing the same cards."""
topic = flow.topic
claimed: set[str] = set()
tasks: set[asyncio.Task] = set()
batch = 100_000 if spec.drain else KANBAN_BATCH
async def _run(cards):
ids = [c["card_id"] for c in cards]
flow.enter(spec.stage)
flow.active_cards.update(f"{spec.board}:{i}" for i in ids)
try:
await spec.process(cards)
except Exception as e: # one bad package must not kill the worker → backoff/dead-letter
log.info("kanban %s/%s: %s: %s", topic, spec.stage, type(e).__name__, e)
try:
await _fail_package(flow, spec, cards, f"{type(e).__name__}: {e}")
except Exception:
log.exception("kanban %s/%s: fail-handling broke", topic, spec.stage)
finally:
flow.leave(spec.stage)
for i in ids:
claimed.discard(i)
flow.active_cards.discard(f"{spec.board}:{i}")
flow.wake.set()
async def _idle_exit() -> bool:
return (flow.research_done and not flow.active_in(all_stages)
and await db.kanban_count(topic, all_stages) == 0)
async def _may_pull() -> bool:
if spec.gate is not None and not spec.gate():
return False
if not spec.barrier:
return True
return await quiescent(flow, spec.upstream)
try:
while not flow.stop:
tasks = {t for t in tasks if not t.done()}
# Fill the pipeline: pull fresh cards and dispatch until `inflight` packages run.
if await _may_pull():
while len(tasks) < inflight:
rows = await db.kanban_pull(topic, spec.board, spec.stage, batch + len(claimed))
fresh = [r for r in rows if r["card_id"] not in claimed][:batch]
if not fresh:
break
for r in fresh:
claimed.add(r["card_id"])
tasks.add(asyncio.create_task(_run(list(fresh))))
if tasks: # busy → wait for a package to finish, then refill
await asyncio.wait(tasks, timeout=_POLL, return_when=asyncio.FIRST_COMPLETED)
continue
# idle: nothing in flight and nothing pulled
if await _idle_exit():
# Real grace sleep (NOT _sleep_wake — the wake event is usually already set
# by the last package and would collapse the window to 0ms). A producer
# attached during the lull flips research_done and keeps us alive.
await asyncio.sleep(_POLL)
if await _idle_exit():
return # nothing left and nothing upstream can produce
continue
await _sleep_wake(flow)
finally:
for t in tasks:
t.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
async def run_flow(flow: Flow, stages: list[Stage], producers=(), set_p=None) -> None:
"""Run producers + one worker per stage until global quiescence. `producers` are coroutines
already counted via flow.add_producer() BEFORE this call (quiescence race). Registers the
flow in active_flows for live attach/cancel."""
active_flows[flow.topic] = flow
names = [s.stage for s in stages]
def _spawn_workers():
return [asyncio.ensure_future(_worker(flow, s, 1 if s.serial else WORKER_INFLIGHT, names))
for s in stages]
workers = [asyncio.ensure_future(p) for p in producers] + _spawn_workers()
progress = asyncio.create_task(_progress(flow, set_p)) if set_p else None
try:
while True:
await asyncio.gather(*workers, return_exceptions=True)
# Restart round: a producer attached exactly as the workers exited (missed even the
# grace sleep) leaves live producers or queued cards behind → run the workers again.
if flow.stop or (flow.research_done and await quiescent(flow, names)):
break
workers = _spawn_workers()
finally:
flow.stop = True
if progress:
progress.cancel()
if active_flows.get(flow.topic) is flow:
active_flows.pop(flow.topic, None)
async def _progress(flow: Flow, set_p):
while not flow.stop:
try:
counts = await db.kanban_stage_counts(flow.topic)
total = sum(n for stages in counts.values() for n in stages.values())
set_p(f"Kanban: {total} Karten im Fluss")
except Exception:
pass
await asyncio.sleep(1.0)

17
backend/models.py
View File

@@ -17,7 +17,13 @@ class GuideCreateRequest(BaseModel):
format: FormatType
instructions: str = Field(default="", max_length=2000)
provider: ProviderType = "claude"
ab_step: int | None = Field(default=None, ge=0, le=4) # re-run from guide step (0 outline … 4 read-exam); None = full/resume
ab_step: int | None = Field(default=None, ge=0, le=5) # re-run from board stage (0 lernziele … 5 lesbarkeit); None = full/resume
class GuideBoardResetRequest(BaseModel):
topic: str = Field(min_length=1, max_length=100)
format: FormatType = "Guide"
ab_stage: int = Field(ge=0, le=5) # reset cards back to this board stage (no generation)
class TopicCreateRequest(BaseModel):
@@ -30,14 +36,13 @@ class BlocksCreateRequest(BaseModel):
provider: ProviderType = "claude"
source_type: SourceType = "thema"
source_location: str = Field(default="", max_length=2000)
ab_phase: int | None = Field(default=None, ge=1, le=9) # re-run from a coarse phase (position in _phasen(topic), 1-based; up to 9: …outline/questions/artifacts); None = resume/continue without deleting
ab_step: int | None = Field(default=None, ge=0) # re-run from a fine sub-step (0-based index into _blocks_steps); takes precedence over ab_phase
to_step: int | None = Field(default=None, ge=0) # stop AFTER this fine sub-step (0-based index into _blocks_steps); None = run to the end
research: bool = True # False = Continue: drain the existing kanban queue, no new search
class BlocksResetStepRequest(BaseModel):
class BlocksResetStageRequest(BaseModel):
topic: str = Field(min_length=1, max_length=100)
ab_step: int = Field(ge=0) # ONLY reset from this sub-step (no regeneration)
board: Literal["inventory", "artefacts"]
stage: str = Field(min_length=1, max_length=40) # kanban column to reset back to
class BlocksStep(BaseModel):

36
backend/pipeline.py
View File

@@ -167,7 +167,7 @@ _yesno_schema = _enum_map_schema("relevant", _YESNO) # triage gate
_MAX_RESTARTS = 2
async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: int, provider: str, on_update=None, cancelled=None, *, grace: int | None = None) -> list | None:
async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: int, provider: str, on_update=None, cancelled=None, *, grace: int | None = None, min_runtime: int | None = None, max_runtime: int | None = None) -> list | None:
"""Starts all slots in parallel and collects `quorum` valid results.
Slot spec: {key, prompt, role, capabilities, payload}. `payload(result)`
@@ -180,10 +180,18 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout:
a timer of `grace` seconds. After it expires, running agents are only
killed if the minimum stands — otherwise the race, including restarts,
keeps running until it stands. Returns: `quorum` to `len(slots)` results.
`min_runtime` (wall-clock from start): the race does not return before it
elapses while agents are still running — gives them time to search thoroughly.
`max_runtime` (wall-clock from start): hard cap — returns whatever is collected
(or None if nothing), killing the rest. Both default off; only Research sets them.
"""
attempts = {i: 0 for i in range(len(slots))}
tasks: dict[asyncio.Task, int] = {}
loop = asyncio.get_running_loop()
start = loop.time()
min_deadline = start + min_runtime if min_runtime else None
max_deadline = start + max_runtime if max_runtime else None
deadline: float | None = None
def spawn(i: int) -> None:
@@ -191,6 +199,7 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout:
task = asyncio.create_task(run_agent(
slot["key"], slot["prompt"], timeout,
provider=provider, role=slot["role"], capabilities=slot["capabilities"],
scope=topic, on_line=slot.get("on_line"),
))
tasks[task] = i
@@ -202,12 +211,22 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout:
while tasks:
if cancelled and cancelled():
return None
if deadline is not None and len(results) >= quorum and loop.time() >= deadline:
# Hard wall-clock cap: return whatever we have (None if empty), kill the rest.
if max_deadline is not None and loop.time() >= max_deadline:
_log(topic, f"{label}: max runtime {max_runtime}s reached ({len(results)} valid)")
return results or None
min_ok = min_deadline is None or loop.time() >= min_deadline
if deadline is not None and len(results) >= quorum and loop.time() >= deadline and min_ok:
return results
# Grace set and minimum reached → only wait for the remaining deadline
wait_timeout = None
# Wake up for the earliest relevant deadline (grace, min, or max).
waits = []
if deadline is not None and len(results) >= quorum:
wait_timeout = max(0.0, deadline - loop.time())
waits.append(deadline - loop.time())
if min_deadline is not None:
waits.append(min_deadline - loop.time())
if max_deadline is not None:
waits.append(max_deadline - loop.time())
wait_timeout = max(0.0, min(waits)) if waits else None
done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED, timeout=wait_timeout)
if not done:
continue
@@ -234,7 +253,8 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout:
_log(topic, f"{label}: first result — grace {grace}s running")
if on_update:
on_update(len(results))
if len(results) >= quorum and (grace is None or loop.time() >= deadline):
if (len(results) >= quorum and (grace is None or loop.time() >= deadline)
and (min_deadline is None or loop.time() >= min_deadline)):
return results
continue
@@ -272,13 +292,13 @@ OK, CANCELLED, FAILED = "ok", "cancelled", "failed"
async def run_single_slot(
ctx: GenContext, label: str, *,
key: str, prompt: str, role: str, capabilities: str, payload, timeout: int,
key: str, prompt: str, role: str, capabilities: str, payload, timeout: int, on_line=None,
) -> tuple[str, object]:
"""One agent, one valid result (race with quorum 1).
→ (OK, value) | (CANCELLED, None) | (FAILED, None)
"""
slots = [{"key": key, "prompt": prompt, "role": role, "capabilities": capabilities, "payload": payload}]
slots = [{"key": key, "prompt": prompt, "role": role, "capabilities": capabilities, "payload": payload, "on_line": on_line}]
res = await _race(ctx.topic, label, slots, 1, timeout, ctx.provider, cancelled=ctx.is_cancelled)
if ctx.is_cancelled():
return CANCELLED, None

3
backend/pytest.ini Normal file
View File

@@ -0,0 +1,3 @@
[pytest]
asyncio_mode = auto
testpaths = tests

114
backend/routes.py
View File

@@ -7,7 +7,7 @@ from datetime import datetime, timezone
from fastapi import APIRouter, HTTPException
from fastapi.responses import Response
from agents import provider_available
from agents import active_agents, provider_available
from config import PROJECTS_DIR, UNI_DIR, PROVIDERS
from database import (
create_guide, delete_guide, get_guide, list_guides,
@@ -18,19 +18,20 @@ from database import (
set_block_score_and_streak, set_block_completed,
delete_block_data, delete_block_progress, subs_per_level, subs_per_level_raw,
delete_topic_pipeline, delete_source, get_guide_content, delete_guide_content,
get_sub_artefakte,
get_sub_artefakte, kanban_reset, delete_guide_board,
)
from blocks import generate_blocks, cancel_blocks, blocks_status, active_blocks, reset_blocks, reset_blocks_ab_step, load_source, load_overview, subblocks_title, subblocks_frei, load_question_pattern, load_question_pattern_free
from blocks import generate_blocks, cancel_blocks, blocks_status, active_blocks, reset_blocks, load_source, load_overview, subblocks_title, subblocks_frei, load_question_pattern, load_question_pattern_free, _blocks_files
from board_inventory import add_research_agent, board_snapshot, requeue_dead, reset_board_from_stage
from elements import generate_element, chat_with_guide, chat_with_element, check_element, style_element, refine_suggestion
from learning import block_chat, block_discussion, create_block_element, exam_rating, exam_rating_fast, exam_question, exam_question_variant, generate_quiz, generate_gapchoice, generate_gaptext, check_gaptext, hurdles_distractor_block, compute_score, floor_from_score, level_from_score, cap_final, cap_aktuell, freie_level, thresholds, points_delta, cap_followup
from guide import generate_guide, guide_slot_files, guide_done_step, block_pruefen, block_adopt, content_fuer_level
from guide import generate_guide, guide_slot_files, block_pruefen, block_adopt, content_fuer_level
from pipeline import cancel_guide
from rules import FORMATE, formats_stats, guide_lock, ist_completed, load_learnstate, topic_completed
from models import (
GuideCreateRequest, GuideResponse,
TopicCreateRequest,
BlocksCreateRequest, BlocksResetStepRequest, BlocksStatusResponse,
GuideChatRequest, GuideChatResponse,
BlocksCreateRequest, BlocksResetStageRequest, BlocksStatusResponse,
GuideBoardResetRequest, GuideChatRequest, GuideChatResponse,
ElementCreateRequest, ElementChatRequest, ElementChatResponse, ElementResponse,
ElementUpdateRequest, ElementCheckRequest, ElementCheckResponse, ElementStyleResponse,
ElementRefineRequest, ElementRefineResponse,
@@ -164,10 +165,55 @@ async def create_blocks(req: BlocksCreateRequest):
raise HTTPException(400, "Link must start with http:// or https://.")
qp.parent.mkdir(parents=True, exist_ok=True)
atomic_write_json(qp, {"type": type, "location": location, "spec": req.instructions.strip()})
asyncio.create_task(generate_blocks(topic, req.instructions.strip(), req.provider, ab_phase=req.ab_phase, ab_step=req.ab_step, to_step=req.to_step))
asyncio.create_task(generate_blocks(topic, req.instructions.strip(), req.provider, research=req.research))
return {"ok": True}
@router.get("/blocks/board")
async def get_blocks_board(topic: str):
"""Live kanban board: columns with counts + newest cards, dead-letter, agents."""
snap = await board_snapshot(topic)
status = await blocks_status(topic)
snap["generating"] = status["generating"]
snap["progress"] = status["progress"]
snap["error"] = status["error"]
snap["agents"] = [{"label": a["key"].removeprefix(f"blocks-{topic}-"), "runtime": a["runtime"]}
for a in active_agents(f"blocks-{topic}-")]
return snap
@router.get("/blocks/agents")
async def get_blocks_agents(topic: str):
return [{"label": a["key"].removeprefix(f"blocks-{topic}-"), "runtime": a["runtime"]}
for a in active_agents(f"blocks-{topic}-")]
@router.post("/blocks/research")
async def add_blocks_research(topic: str, provider: str = "claude"):
"""Attach one more research agent — to the live flow, or attach-or-start."""
if add_research_agent(topic):
return {"ok": True, "attached": True}
if (await blocks_status(topic))["generating"]:
return {"ok": False, "status": "starting"} # flow is booting, try again shortly
asyncio.create_task(generate_blocks(topic, "", provider, research=True))
return {"ok": True, "attached": False}
@router.post("/blocks/reset-stage")
async def reset_blocks_stage(req: BlocksResetStageRequest):
"""Reset cards from a column onward back to that column (no regeneration)."""
topic = req.topic.strip()
if (await blocks_status(topic))["generating"]:
return {"ok": True, "status": "generating"} # don't interfere with a running generation
moved = await reset_board_from_stage(topic, req.board, req.stage, _blocks_files(topic))
return {"ok": True, "moved": moved}
@router.post("/blocks/requeue-dead")
async def requeue_blocks_dead(topic: str):
return {"ok": True, "requeued": await requeue_dead(topic)}
@router.post("/blocks/cancel")
async def cancel_blocks_route(topic: str):
if not cancel_blocks(topic):
@@ -179,15 +225,7 @@ async def cancel_blocks_route(topic: str):
async def remove_blocks(topic: str):
reset_blocks(topic) # Files: crawl + triage + inventory…questions gone; source.json stays
await delete_topic_pipeline(topic) # DB: blocks area gone; topic config (source) stays
return {"ok": True}
@router.post("/blocks/reset-step")
async def reset_blocks_step(req: BlocksResetStepRequest):
topic = req.topic.strip()
if (await blocks_status(topic))["generating"]:
return {"ok": True, "status": "generating"} # don't interfere with a running generation
await reset_blocks_ab_step(topic, req.ab_step)
await kanban_reset(topic) # kanban cards + cluster membership gone
return {"ok": True}
@@ -509,9 +547,46 @@ async def guide_locks(topic: str):
@router.get("/guides/steps")
async def guide_steps(topic: str):
"""Highest fully completed step index per format (artifact-based, -1 = none).
Drives the clickable step bubbles (like the blocks phases)."""
return {fmt: guide_done_step(guide_content_path(topic, fmt)) for fmt in ("Guide", "FullGuide", "Rest")}
"""Highest fully completed stage index per format (card-based, -1 = none).
Content file present (legacy without cards) → everything done."""
import guide_board
out = {}
for fmt in ("Guide", "FullGuide", "Rest"):
step = await guide_board.done_step(topic, fmt)
if step < 0 and guide_content_path(topic, fmt).exists():
step = len(guide_board.GUIDE_STAGES)
out[fmt] = step
return out
@router.get("/guides/board")
async def get_guide_board(topic: str, format: str = "Guide"):
"""Live guide board: columns with counts + cards (rounds, covered objectives), agents."""
import guide_board
snap = await guide_board.board_snapshot(topic, format)
guide = next((g for g in await list_guides()
if g["topic"] == topic and g["format"] == format), None)
snap["generating"] = bool(guide and guide["status"] in ("queued", "generating"))
snap["guide_id"] = guide["id"] if guide else None
snap["progress"] = guide.get("progress") if guide else None
snap["error"] = guide.get("error_msg") if guide else None
prefix = f"{guide['id']}-" if guide else "-"
snap["agents"] = [{"label": a["key"].removeprefix(prefix), "runtime": a["runtime"]}
for a in active_agents(prefix)]
return snap
@router.post("/guides/board/reset")
async def reset_guide_board(req: GuideBoardResetRequest):
"""Reset cards from a stage onward — without generation (pendant to blocks reset-stage)."""
import guide_board
topic = req.topic.strip()
guide = next((g for g in await list_guides()
if g["topic"] == topic and g["format"] == req.format), None)
if guide and guide["status"] in ("queued", "generating"):
return {"ok": True, "status": "generating"}
moved = await guide_board.reset_from_stage(topic, req.format, req.ab_stage)
return {"ok": True, "moved": moved}
@router.get("/guides/{guide_id}", response_model=GuideResponse)
@@ -680,6 +755,7 @@ async def remove(guide_id: str, slots: bool = False):
rest = [g for g in await list_guides() if g["topic"] == guide["topic"] and g["format"] == guide["format"]]
if not rest:
await delete_guide_content(guide["topic"], guide["format"])
await delete_guide_board(guide["topic"], guide["format"]) # board cards + lernziele
content = guide_content_path(guide["topic"], guide["format"])
if slots or content.exists():
for p in guide_slot_files(content):

18
backend/tests/conftest.py Normal file
View File

@@ -0,0 +1,18 @@
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import database # noqa: E402
@pytest.fixture
async def testdb(tmp_path, monkeypatch):
"""Fresh sqlite file per test; resets the module-global connection."""
monkeypatch.setattr(database, "DB_PATH", tmp_path / "test.db")
database._db = None
await database.init_db()
yield database
await database.close_db()

210
backend/tests/test_board_inventory.py Normal file
View File

@@ -0,0 +1,210 @@
"""Board 1 end-to-end through the real engine — agents faked, no LLM, no embedding model."""
import json
import re
import pytest
import board_inventory as bi
import kanban
from pipeline import GenContext
TOPIC = "t"
B = bi.BOARD
_PATH_RE = re.compile(r"(/\S+\.json)")
def _fake_single_slot(tmp_path):
"""Deterministic judge stand-in: writes the expected JSON to the out_path found in the
prompt and returns it — keyed off the agent-key naming convention."""
async def fake(ctx, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
m = _PATH_RE.search(prompt)
out = None
if "-pair-" in key:
pairs = prompt.count("\nA: ")
out = {"pairs": {str(i + 1): "ja" for i in range(pairs)}}
elif "-clarify-" in key:
keep = [line[2:].split(" — ")[0] for line in prompt.splitlines()
if line.startswith("- ")]
out = {"keep": keep, "rest": []}
elif "-naming-" in key:
out = {"best": 1}
elif "-filter-recheck-" in key or "-filter-" in key:
out = {"fragments": {}, "drop": []}
elif "-gruppierung-completion-" in key:
out = {"additions": []}
elif "-gruppierung-" in key:
out = {"umbrellas": []}
elif "-supplement" in key:
out = {"blocks": [{"title": "Zeta-Konzept", "description": "kanonisch fehlend"}]}
if m and out is not None:
with open(m.group(1), "w", encoding="utf-8") as f:
json.dump(out, f)
return "ok", payload(None)
return fake
@pytest.fixture
async def board_env(testdb, tmp_path, monkeypatch):
import board_artefacts as ba
from textkit import _norm_title
monkeypatch.setattr(bi, "run_single_slot", _fake_single_slot(tmp_path))
async def no_emb(flow):
return False
monkeypatch.setattr(bi, "_emb_ok", no_emb)
async def fake_subblocks(ctx, set_p, files, entries, instructions, wipe=True, ns=""):
title = list(entries.values())[0].split(" — ")[0]
return {title: ["Sub Eins", "Sub Zwei"]}
async def fake_facts(ctx, set_p, files, raw, q, folder, instructions, ns=""):
facts = {t: {_norm_title(s): {"key_points": [f"Fakt zu {s}"], "cited_facts": []}
for s in subs} for t, subs in raw.items()}
return facts, {}
async def fake_levels(ctx, set_p, files, raw, instructions, ns=""):
return {t: [{"title": s, "level": "beginner"} for s in subs] for t, subs in raw.items()}
async def fake_relevance(ctx, set_p, files, sidecar, instructions, ns=""):
return {1: "relevant", 2: "peripheral"}
async def fake_pattern(ctx, set_p, files, sidecar, instructions, ns=""):
return {t: [{"subblock": subs[0]["title"], "question": f"Was ist {t}?"}]
for t, subs in sidecar.items()}
async def fake_artefacts(ctx, set_p, files, sidecar, instructions, ns=""):
return {"flashcard": [{"block": t, "subblock": subs[0]["title"], "front": "F", "back": "B"}
for t, subs in sidecar.items()], "example": []}
async def fake_outline(ctx, set_p, files, entries, instructions):
return {"chapters": [{"title": "Kapitel 1", "numbers": sorted(entries)}]}
for name, fn in [("_subblocks_block", fake_subblocks), ("_facts_block", fake_facts),
("_levels_block", fake_levels), ("_relevance_block", fake_relevance),
("_question_pattern_block", fake_pattern), ("_artefacts_block", fake_artefacts),
("_outline_block", fake_outline)]:
monkeypatch.setattr(ba, name, fn)
work = tmp_path / "arbeit"
work.mkdir()
files = {"arbeit": work, "final": tmp_path / "blocks.md",
"sub_roh": tmp_path / "sub_roh.json", "sidecar": tmp_path / "subblocks.json",
"facts": tmp_path / "facts.json", "question_pattern": tmp_path / "question_pattern.json",
"artefakte": tmp_path / "artefakte.json", "outline": tmp_path / "outline.json",
"outline_slots": tmp_path / "outline_slots"}
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
return testdb, ctx, files
async def _seed(db):
# 2 consensus titles (2 readers), 1 single find (kept by panel), 1 artifact (pre-reject)
await db.kanban_add_title(TOPIC, B, "alpha", "Alpha", "Grundkonzept", "s1", "r1")
await db.kanban_add_title(TOPIC, B, "alpha", "Alpha", "Grundkonzept", "s2", "r2")
await db.kanban_add_title(TOPIC, B, "beta", "Beta", "Zweites Konzept", "s1", "r1")
await db.kanban_add_title(TOPIC, B, "beta", "Beta", "Zweites Konzept", "s2", "r2")
await db.kanban_add_title(TOPIC, B, "gamma", "Gamma", "Einzelfund", "s1", "r1")
await db.kanban_add_title(TOPIC, B, "aufgabe 3", "Aufgabe 3", "Übungszettel", "s1", "r1")
async def test_board1_full_flow(board_env):
db, ctx, files = board_env
await _seed(db)
import asyncio
ok = await asyncio.wait_for(
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "", research=False),
timeout=30)
assert ok
done = await db.kanban_cards(TOPIC, board=B, stage="done_block")
titles = sorted(c["payload"]["title"] for c in done)
# Alpha/Beta/Gamma survive; the supplement's Zeta flows through the whole board too
assert titles == ["Alpha", "Beta", "Gamma", "Zeta-Konzept"]
rejected = await db.kanban_cards(TOPIC, board=B, stage="rejected")
assert [c["payload"]["title"] for c in rejected] == ["Aufgabe 3"]
assert rejected[0]["payload"]["reason"] == "pre-reject"
# legacy mirror carries the survivors as consensus
legacy = {b["title"] for b in await db.list_blocks(TOPIC, status="consensus")}
assert legacy == set(titles)
# blocks.md written in flow order
lines = files["final"].read_text(encoding="utf-8").strip().splitlines()
assert len(lines) == 4
# reader union survived the pipeline (consensus evidence on the block card)
alpha = next(c for c in done if c["payload"]["title"] == "Alpha")
assert set(alpha["payload"]["readers"]) == {"r1", "r2"}
# board 2: one artefact card per block ran through to done_artefact (+ outline singleton)
art_done = await db.kanban_cards(TOPIC, board="artefacts", stage="done_artefact")
assert len(art_done) == 5 # 4 blocks + outline card
sidecar = json.loads(files["sidecar"].read_text(encoding="utf-8"))
assert set(sidecar) == set(titles)
assert sidecar["Alpha"][0]["facts"]["key_points"] == ["Fakt zu Sub Eins"]
assert sidecar["Alpha"][0]["relevance"] == "relevant"
assert sidecar["Alpha"][1]["relevance"] == "peripheral"
# DB mirrors: subblocks, question pattern, artefacts, outline
subs = await db.list_subblocks(TOPIC, "alpha")
assert {s["sub_title"] for s in subs} == {"Sub Eins", "Sub Zwei"}
outline = await db.get_outline(TOPIC)
assert outline and "Kapitel 1" in outline
async def test_filter_judges_run_parallel(board_env, monkeypatch):
"""40 Blöcke → 2 Filter-Chunks: die Judge-Welle muss parallel laufen (Perf-Fix)."""
import asyncio
db, ctx, files = board_env
state = {"cur": 0, "max": 0}
base = bi.run_single_slot # instant fake from the fixture
async def slow(*a, **k):
state["cur"] += 1
state["max"] = max(state["max"], state["cur"])
try:
await asyncio.sleep(0.05)
return await base(*a, **k)
finally:
state["cur"] -= 1
monkeypatch.setattr(bi, "run_single_slot", slow)
for i in range(40):
await db.kanban_upsert_card(TOPIC, B, f"b-x{i}", "block", "fragment_filter",
{"title": f"Block {i}", "description": "d"})
ok = await asyncio.wait_for(
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "", research=False),
timeout=30)
assert ok
# 40 seeded + 1 supplement candidate (Zeta) flow through to done_block
assert await db.kanban_count(TOPIC, "done_block", board=B) == 41
assert state["max"] >= 2 # chunk judges ran as one wave, not sequentially
async def test_empty_subblocks_completes_without_deadletter(board_env, monkeypatch):
"""Legitim leere Subbausteine ({} statt None) → Karte läuft bis done_artefact durch."""
import asyncio
import board_artefacts as ba
import blocks as blx
db, ctx, files = board_env
async def empty_subs(ctx, set_p, files, entries, instructions, wipe=True, ns=""):
return {}
monkeypatch.setattr(ba, "_subblocks_block", empty_subs)
await db.kanban_upsert_card(TOPIC, "artefacts", "leer", "ablock", "subblocks",
{"title": "Leerer Block", "description": "d"})
ok = await asyncio.wait_for(
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "", research=False),
timeout=30)
assert ok
card = await db.kanban_get_card(TOPIC, "artefacts", "leer")
assert card["stage"] == "done_artefact"
assert card["retries"] == 0 and not card.get("last_error")
assert TOPIC not in blx._blocks_errors # kein globales Banner
async def test_reader_union_folds_exact_dupes(testdb):
db = testdb
assert await db.kanban_add_title(TOPIC, B, "x", "X", "d", "s1", "r1") is True
assert await db.kanban_add_title(TOPIC, B, "x", "X", "d länger", "s2", "r2") is False
card = await db.kanban_get_card(TOPIC, B, "x")
assert set(card["payload"]["readers"]) == {"r1", "r2"}
assert set(card["payload"]["sources"]) == {"s1", "s2"}
assert card["payload"]["description"] == "d länger"

68
backend/tests/test_guide_board.py Normal file
View File

@@ -0,0 +1,68 @@
"""Guide board: schema parsers + card reset semantics (no LLM)."""
import guide_board as gb
TOPIC, FMT = "t", "Guide"
def test_ziele_schema():
ok = gb._ziele_schema({"ziele": [{"id": "z1", "text": "Erklären, warum X", "sub": "S"},
{"id": "z2", "text": "Nennen von Y"}]})
assert [z["id"] for z in ok] == ["z1", "z2"]
assert gb._ziele_schema({"ziele": []}) is None
assert gb._ziele_schema({"ziele": [{"id": "z1", "text": "a"}, {"id": "z1", "text": "b"}]}) \
== [{"id": "z1", "text": "a", "sub": ""}] # duplicate ids fold
assert gb._ziele_schema("quatsch") is None
def test_gate_schema():
assert gb._gate_schema({"ok": True}) == []
claims = gb._gate_schema({"claims": [{"text": "Falsch", "grund": "fehlt"}]})
assert claims == [{"text": "Falsch", "grund": "fehlt"}]
assert gb._gate_schema({}) is None
def test_coverage_schema():
res = gb._coverage_schema({"ziele": {"z1": True, "z2": "false"},
"luecken": [{"ziel": "z2", "fehlt": "Beweis"}],
"ballast": ["Abschweifung"]}, {"z1", "z2"})
assert res["ziele"] == {"z1": True, "z2": False}
assert res["luecken"][0]["fehlt"] == "Beweis"
assert gb._coverage_schema({"ziele": {"z1": True}}, {"z1", "z2"}) is None # z2 missing
def test_problems_schema():
assert gb._problems_schema({"ok": True}) == []
assert gb._problems_schema({"problems": [{"section": "S", "problem": "zu lang"}]}) == ["zu lang"]
assert gb._problems_schema({"problems": []}) is None
async def test_reset_from_stage(testdb):
db = testdb
await db.upsert_guide_card(TOPIC, FMT, "a", "A")
await db.upsert_guide_card(TOPIC, FMT, "b", "B")
await db.set_guide_card(TOPIC, FMT, "a", stage="done", md="text", writer_rounds=2)
await db.set_guide_card(TOPIC, FMT, "b", stage="coverage", md="text")
await db.put_lernziel(TOPIC, "a", "z1", "Ziel")
# reset ab writer (idx 2): beide Karten zurück, md geleert, Ziele bleiben
moved = await gb.reset_from_stage(TOPIC, FMT, 2)
assert moved == 2
cards = {c["block_norm"]: c for c in await db.list_guide_cards(TOPIC, FMT)}
assert cards["a"]["stage"] == "writer" and cards["a"]["md"] == "" and cards["a"]["writer_rounds"] == 0
assert cards["b"]["stage"] == "writer"
assert await db.list_lernziele(TOPIC, "a")
# reset ab lernziele (idx 0): Ziele weg
await gb.reset_from_stage(TOPIC, FMT, 0)
assert not await db.list_lernziele(TOPIC, "a")
assert (await db.list_guide_cards(TOPIC, FMT))[0]["stage"] == "lernziele"
async def test_done_step(testdb):
db = testdb
assert await gb.done_step(TOPIC, FMT) == -1
await db.upsert_guide_card(TOPIC, FMT, "a", "A")
assert await gb.done_step(TOPIC, FMT) == -1 # alles in lernziele
await db.set_guide_card(TOPIC, FMT, "a", stage="coverage")
assert await gb.done_step(TOPIC, FMT) == 3 # bis fakten_gate fertig
await db.set_guide_card(TOPIC, FMT, "a", stage="done")
assert await gb.done_step(TOPIC, FMT) == len(gb.GUIDE_STAGES)

128
backend/tests/test_kanban.py Normal file
View File

@@ -0,0 +1,128 @@
"""Engine tests with fake processors (no LLM): flow, barrier, retry/dead-letter, producer race."""
import asyncio
import pytest
import kanban
from kanban import Flow, Stage, chain_stages, run_flow
TOPIC = "t"
BOARD = "inventory"
def _advance_proc(db, to_stage):
async def proc(cards):
await db.kanban_advance_many(TOPIC, BOARD, [(c["card_id"], to_stage) for c in cards])
return proc
async def _seed(db, n, stage="s1"):
for i in range(n):
await db.kanban_upsert_card(TOPIC, BOARD, f"card-{i}", "title", stage, {"title": f"T{i}"})
async def test_cards_flow_through_stages(testdb):
db = testdb
await _seed(db, 7)
flow = Flow(TOPIC)
stages = chain_stages([
Stage(BOARD, "s1", _advance_proc(db, "s2")),
Stage(BOARD, "s2", _advance_proc(db, "done")),
])
await asyncio.wait_for(run_flow(flow, stages), timeout=10)
assert await db.kanban_count(TOPIC, "done", board=BOARD) == 7
assert await db.kanban_count(TOPIC, ["s1", "s2"], board=BOARD) == 0
async def test_barrier_waits_for_upstream(testdb):
db = testdb
await _seed(db, 6)
upstream_left: list[int] = []
async def slow_s1(cards):
await asyncio.sleep(0.05) # keep upstream busy so an eager barrier would see queued cards
await db.kanban_advance_many(TOPIC, BOARD, [(c["card_id"], "gate") for c in cards])
async def barrier_proc(cards):
upstream_left.append(await db.kanban_count(TOPIC, ["s1"], board=BOARD))
await db.kanban_advance_many(TOPIC, BOARD, [(c["card_id"], "done") for c in cards])
flow = Flow(TOPIC)
stages = chain_stages([
Stage(BOARD, "s1", slow_s1),
Stage(BOARD, "gate", barrier_proc, barrier=True),
])
await asyncio.wait_for(run_flow(flow, stages), timeout=10)
assert await db.kanban_count(TOPIC, "done", board=BOARD) == 6
assert upstream_left and all(n == 0 for n in upstream_left) # barrier never ran with s1 queued
async def test_retry_backoff_then_dead(testdb, monkeypatch):
db = testdb
monkeypatch.setattr(kanban, "RETRY_BACKOFF", 0.02)
await _seed(db, 1)
attempts = []
async def failing(cards):
attempts.append(cards[0]["retries"])
raise RuntimeError("kaputt")
flow = Flow(TOPIC)
stages = chain_stages([Stage(BOARD, "s1", failing)])
await asyncio.wait_for(run_flow(flow, stages), timeout=10)
card = await db.kanban_get_card(TOPIC, BOARD, "card-0")
assert card["stage"] == "dead"
assert card["retries"] == kanban.MAX_CARD_RETRIES
assert "kaputt" in card["last_error"]
assert attempts == [0, 1, 2] # backoff between attempts, then dead-letter
async def test_requeue_dead(testdb):
db = testdb
await db.kanban_upsert_card(TOPIC, BOARD, "card-0", "title", "s1")
for _ in range(kanban.MAX_CARD_RETRIES):
await db.kanban_fail_card(TOPIC, BOARD, "card-0", "x", kanban.MAX_CARD_RETRIES, 0.0)
assert (await db.kanban_get_card(TOPIC, BOARD, "card-0"))["stage"] == "dead"
assert await db.kanban_requeue_dead(TOPIC, BOARD, "s1") == 1
card = await db.kanban_get_card(TOPIC, BOARD, "card-0")
assert card["stage"] == "s1" and card["retries"] == 0
async def test_producer_attach_in_idle_lull(testdb):
"""Fix-6 regression: a producer attached while workers sit in the exit grace poll
must keep the flow alive and its cards must still be processed."""
db = testdb
flow = Flow(TOPIC)
stages = chain_stages([Stage(BOARD, "s1", _advance_proc(db, "done"))])
async def producer_a():
await db.kanban_upsert_card(TOPIC, BOARD, "card-a", "title", "s1")
flow.wake.set()
flow.done_producer()
async def attacher():
while await db.kanban_count(TOPIC, "done", board=BOARD) == 0: # wait for card-a done
await asyncio.sleep(0.01)
flow.add_producer() # synchronous BEFORE the work — the grace poll must see it
async def producer_b():
await db.kanban_upsert_card(TOPIC, BOARD, "card-b", "title", "s1")
flow.wake.set()
flow.done_producer()
await producer_b()
flow.add_producer() # producer_a, counted before run_flow (sync add)
asyncio.get_event_loop().create_task(attacher())
await asyncio.wait_for(run_flow(flow, stages, producers=[producer_a()]), timeout=10)
assert await db.kanban_count(TOPIC, "done", board=BOARD) == 2
async def test_backoff_delays_pull(testdb, monkeypatch):
db = testdb
await db.kanban_upsert_card(TOPIC, BOARD, "card-0", "title", "s1")
await db.kanban_fail_card(TOPIC, BOARD, "card-0", "x", 5, 0.2)
assert await db.kanban_pull(TOPIC, BOARD, "s1", 10) == [] # in backoff → not pullable
assert await db.kanban_count(TOPIC, "s1", board=BOARD) == 1 # but still counts as queued
await asyncio.sleep(0.25)
assert len(await db.kanban_pull(TOPIC, BOARD, "s1", 10)) == 1

41
backend/tests/test_roles.py Normal file
View File

@@ -0,0 +1,41 @@
"""Role routing: resolve_role maps (run_provider, role) → (provider, model) across stacks."""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import config
from config import PROVIDERS, resolve_role
def test_default_quick_routes_to_minimax(monkeypatch):
monkeypatch.setitem(config.ROLE_ROUTING, "quick", "minimax")
assert resolve_role("claude", "quick") == ("minimax", PROVIDERS["minimax"]["quick"])
def test_default_judge_routes_to_claude(monkeypatch):
monkeypatch.setitem(config.ROLE_ROUTING, "judge", "claude")
assert resolve_role("minimax", "judge") == ("claude", PROVIDERS["claude"]["judge"])
def test_empty_routing_keeps_run_provider(monkeypatch):
monkeypatch.setitem(config.ROLE_ROUTING, "fast", "")
assert resolve_role("claude", "fast") == ("claude", PROVIDERS["claude"]["fast"])
assert resolve_role("minimax", "fast") == ("minimax", PROVIDERS["minimax"]["fast"])
def test_explicit_model_syntax(monkeypatch):
monkeypatch.setitem(config.ROLE_ROUTING, "guide", "claude:claude-opus-4-8")
assert resolve_role("minimax", "guide") == ("claude", "claude-opus-4-8")
def test_unknown_target_falls_back_to_run_provider(monkeypatch):
monkeypatch.setitem(config.ROLE_ROUTING, "quick", "gibtsnicht")
assert resolve_role("claude", "quick") == ("claude", PROVIDERS["claude"]["quick"])
def test_unknown_role_yields_empty_model():
provider, model = resolve_role("claude", "nope")
assert provider == "claude"
assert model == ""