From c794fcaccf1844047049f5d0f2099a8f8d39b331 Mon Sep 17 00:00:00 2001 From: team3 Date: Tue, 30 Jun 2026 00:14:18 +0200 Subject: [PATCH] update --- backend/agents.py | 64 +- backend/bausteine.py | 3330 ----------------- backend/blocks.py | 3326 ++++++++++++++++ backend/config.py | 171 +- backend/crawl.py | 100 +- backend/database.py | 711 ++-- backend/elements.py | 102 +- backend/embedding.py | 82 +- backend/fsutil.py | 6 +- backend/guide.py | 1032 ++--- backend/jsonio.py | 16 +- backend/learning.py | 657 ++++ backend/lernen.py | 657 ---- backend/lesbarkeit.py | 115 - backend/logsetup.py | 2 +- backend/main.py | 8 +- backend/models.py | 172 +- backend/paths.py | 48 +- backend/pipeline.py | 205 +- backend/readability.py | 115 + backend/regeln.py | 141 - backend/routes.py | 632 ++-- backend/rules.py | 141 + backend/textkit.py | 126 +- frontend/src/App.vue | 268 +- frontend/src/api.js | 106 +- frontend/src/components/BausteinFokus.vue | 418 --- frontend/src/components/BausteinPanel.vue | 1002 ----- frontend/src/components/BlockFocus.vue | 418 +++ frontend/src/components/BlockPanel.vue | 966 +++++ ...teineUebersicht.vue => BlocksOverview.vue} | 156 +- frontend/src/components/ElementsOverview.vue | 19 +- frontend/src/components/FlashcardWidget.vue | 54 +- frontend/src/components/GeneralExamPanel.vue | 102 +- frontend/src/components/TopicDetail.vue | 310 +- frontend/src/components/TopicSidebar.vue | 344 +- .../src/components/WorkedExampleBlock.vue | 36 +- .../components/elements/ElementChatTab.vue | 10 +- .../src/components/elements/ElementDetail.vue | 99 +- .../components/elements/ElementEditTab.vue | 26 +- .../src/components/elements/ElementList.vue | 24 +- .../components/elements/ElementSuggestion.vue | 12 +- .../components/elements/ElementsSidebar.vue | 22 +- frontend/src/composables/useChat.js | 36 +- frontend/src/composables/useConfirm.js | 6 +- frontend/src/composables/usePolling.js | 6 +- frontend/src/levels.js | 35 + frontend/src/markdown.js | 29 +- frontend/src/pruefungCache.js | 28 +- frontend/src/stufen.js | 35 - templates/Prompt/Artefakt-Beispiel-Check.md | 25 - templates/Prompt/Artefakt-Beispiel.md | 22 - templates/Prompt/Artefakt-Karteikarte.md | 21 - templates/Prompt/Artifact-Example-Check.md | 25 + templates/Prompt/Artifact-Example.md | 24 + templates/Prompt/Artifact-Flashcard.md | 23 + templates/Prompt/Baustein-Bewertung-Kritik.md | 35 - templates/Prompt/Baustein-Bewertung.md | 55 - templates/Prompt/Baustein-Chat.md | 19 - templates/Prompt/Baustein-Frage-Kritik.md | 32 - templates/Prompt/Baustein-Frage-Variante.md | 31 - templates/Prompt/Baustein-Frage.md | 45 - .../Prompt/Baustein-Lueckentext-Pruefung.md | 18 - templates/Prompt/Baustein-Lueckentext.md | 29 - templates/Prompt/Baustein-Lueckwahl.md | 30 - .../Prompt/Baustein-Pruefung-Diskussion.md | 45 - templates/Prompt/Baustein-Quiz.md | 32 - .../Prompt/Bausteine-Block-Gruppieren.md | 20 - templates/Prompt/Bausteine-Ergaenzung.md | 21 - templates/Prompt/Bausteine-Filter.md | 39 - templates/Prompt/Bausteine-Klaerung.md | 32 - templates/Prompt/Bausteine-Paar-Filter.md | 20 - templates/Prompt/Bausteine-Quelle-Link.md | 1 - templates/Prompt/Bausteine-Quelle-Projekt.md | 1 - templates/Prompt/Bausteine-Quelle-Thema.md | 1 - templates/Prompt/Bausteine-Quelle-Uni.md | 1 - .../Prompt/Bausteine-Recherche-Mapping.md | 16 - templates/Prompt/Bausteine-Recherche.md | 25 - templates/Prompt/Block-Chat.md | 19 + templates/Prompt/Block-Exam-Discussion.md | 45 + templates/Prompt/Block-Gapchoice.md | 31 + templates/Prompt/Block-Gaptext-Exam.md | 18 + templates/Prompt/Block-Gaptext.md | 29 + templates/Prompt/Block-Pruefen.md | 30 +- templates/Prompt/Block-Question-Critique.md | 32 + templates/Prompt/Block-Question-Variante.md | 31 + templates/Prompt/Block-Question.md | 45 + templates/Prompt/Block-Quiz.md | 32 + templates/Prompt/Block-Rating-Critique.md | 35 + templates/Prompt/Block-Rating.md | 55 + templates/Prompt/Blocks-Block-Grouping.md | 20 + templates/Prompt/Blocks-Filter.md | 39 + templates/Prompt/Blocks-Klaerung.md | 32 + templates/Prompt/Blocks-Paar-Filter.md | 20 + templates/Prompt/Blocks-Research-Mapping.md | 16 + templates/Prompt/Blocks-Research.md | 25 + templates/Prompt/Blocks-Source-Link.md | 1 + templates/Prompt/Blocks-Source-Projekt.md | 1 + templates/Prompt/Blocks-Source-Thema.md | 1 + templates/Prompt/Blocks-Source-Uni.md | 1 + templates/Prompt/Blocks-Supplement.md | 21 + templates/Prompt/Chat.md | 20 +- templates/Prompt/Element-Chat.md | 36 +- templates/Prompt/Element-Check.md | 18 +- templates/Prompt/Element-Create.md | 42 +- templates/Prompt/Element-Refine.md | 26 +- templates/Prompt/Element-Stil.md | 37 - templates/Prompt/Element-Style.md | 37 + templates/Prompt/Element-Verify.md | 24 +- templates/Prompt/Facts-Check.md | 26 + templates/Prompt/Facts-Research.md | 30 + templates/Prompt/Facts-Supplement.md | 25 + templates/Prompt/Fakten-Check.md | 26 - templates/Prompt/Fakten-Ergaenzung.md | 24 - templates/Prompt/Fakten-Recherche.md | 30 - templates/Prompt/Frage-Muster-Kritik.md | 21 - templates/Prompt/Frage-Muster-Recherche.md | 26 - .../Prompt/Gliederung-Voraussetzungen.md | 17 - templates/Prompt/Guide-Content-Check.md | 19 + templates/Prompt/Guide-Content-Fix.md | 18 + templates/Prompt/Guide-Content.md | 30 + templates/Prompt/Guide-Facts-Projekt.md | 1 + templates/Prompt/Guide-Facts-Thema.md | 1 + templates/Prompt/Guide-Fakten-Projekt.md | 1 - templates/Prompt/Guide-Fakten-Thema.md | 1 - templates/Prompt/Guide-Gliederung-Judge.md | 24 - templates/Prompt/Guide-Gliederung.md | 23 - templates/Prompt/Guide-Inhalt-Check.md | 19 - templates/Prompt/Guide-Inhalt-Fix.md | 16 - templates/Prompt/Guide-Inhalt.md | 28 - templates/Prompt/Guide-Lese-Check.md | 44 +- templates/Prompt/Guide-Outline-Judge.md | 26 + templates/Prompt/Guide-Outline.md | 25 + templates/Prompt/Guide-Sections-Fix.md | 48 +- templates/Prompt/Guide-Writer.md | 72 +- templates/Prompt/Levels-Mapping.md | 21 + templates/Prompt/Levels-Research.md | 33 + templates/Prompt/Outline-Prerequisites.md | 17 + templates/Prompt/Quelle-Relevanz.md | 23 - templates/Prompt/Question-Pattern-Critique.md | 23 + templates/Prompt/Question-Pattern-Research.md | 28 + templates/Prompt/Relevance-Mapping.md | 18 + templates/Prompt/Relevance-Research.md | 22 + templates/Prompt/Relevanz-Mapping.md | 18 - templates/Prompt/Relevanz-Recherche.md | 22 - templates/Prompt/Source-Relevance.md | 23 + templates/Prompt/Stufen-Mapping.md | 21 - templates/Prompt/Stufen-Recherche.md | 33 - templates/Prompt/Subbaustein-Mapping.md | 27 - templates/Prompt/Subbaustein-Recherche.md | 29 - templates/Prompt/Subblock-Mapping.md | 27 + templates/Prompt/Subblock-Research.md | 29 + 152 files changed, 9485 insertions(+), 9583 deletions(-) delete mode 100644 backend/bausteine.py create mode 100644 backend/blocks.py create mode 100644 backend/learning.py delete mode 100644 backend/lernen.py delete mode 100644 backend/lesbarkeit.py create mode 100644 backend/readability.py delete mode 100644 backend/regeln.py create mode 100644 backend/rules.py delete mode 100644 frontend/src/components/BausteinFokus.vue delete mode 100644 frontend/src/components/BausteinPanel.vue create mode 100644 frontend/src/components/BlockFocus.vue create mode 100644 frontend/src/components/BlockPanel.vue rename frontend/src/components/{BausteineUebersicht.vue => BlocksOverview.vue} (61%) create mode 100644 frontend/src/levels.js delete mode 100644 frontend/src/stufen.js delete mode 100644 templates/Prompt/Artefakt-Beispiel-Check.md delete mode 100644 templates/Prompt/Artefakt-Beispiel.md delete mode 100644 templates/Prompt/Artefakt-Karteikarte.md create mode 100644 templates/Prompt/Artifact-Example-Check.md create mode 100644 templates/Prompt/Artifact-Example.md create mode 100644 templates/Prompt/Artifact-Flashcard.md delete mode 100644 templates/Prompt/Baustein-Bewertung-Kritik.md delete mode 100644 templates/Prompt/Baustein-Bewertung.md delete mode 100644 templates/Prompt/Baustein-Chat.md delete mode 100644 templates/Prompt/Baustein-Frage-Kritik.md delete mode 100644 templates/Prompt/Baustein-Frage-Variante.md delete mode 100644 templates/Prompt/Baustein-Frage.md delete mode 100644 templates/Prompt/Baustein-Lueckentext-Pruefung.md delete mode 100644 templates/Prompt/Baustein-Lueckentext.md delete mode 100644 templates/Prompt/Baustein-Lueckwahl.md delete mode 100644 templates/Prompt/Baustein-Pruefung-Diskussion.md delete mode 100644 templates/Prompt/Baustein-Quiz.md delete mode 100644 templates/Prompt/Bausteine-Block-Gruppieren.md delete mode 100644 templates/Prompt/Bausteine-Ergaenzung.md delete mode 100644 templates/Prompt/Bausteine-Filter.md delete mode 100644 templates/Prompt/Bausteine-Klaerung.md delete mode 100644 templates/Prompt/Bausteine-Paar-Filter.md delete mode 100644 templates/Prompt/Bausteine-Quelle-Link.md delete mode 100644 templates/Prompt/Bausteine-Quelle-Projekt.md delete mode 100644 templates/Prompt/Bausteine-Quelle-Thema.md delete mode 100644 templates/Prompt/Bausteine-Quelle-Uni.md delete mode 100644 templates/Prompt/Bausteine-Recherche-Mapping.md delete mode 100644 templates/Prompt/Bausteine-Recherche.md create mode 100644 templates/Prompt/Block-Chat.md create mode 100644 templates/Prompt/Block-Exam-Discussion.md create mode 100644 templates/Prompt/Block-Gapchoice.md create mode 100644 templates/Prompt/Block-Gaptext-Exam.md create mode 100644 templates/Prompt/Block-Gaptext.md create mode 100644 templates/Prompt/Block-Question-Critique.md create mode 100644 templates/Prompt/Block-Question-Variante.md create mode 100644 templates/Prompt/Block-Question.md create mode 100644 templates/Prompt/Block-Quiz.md create mode 100644 templates/Prompt/Block-Rating-Critique.md create mode 100644 templates/Prompt/Block-Rating.md create mode 100644 templates/Prompt/Blocks-Block-Grouping.md create mode 100644 templates/Prompt/Blocks-Filter.md create mode 100644 templates/Prompt/Blocks-Klaerung.md create mode 100644 templates/Prompt/Blocks-Paar-Filter.md create mode 100644 templates/Prompt/Blocks-Research-Mapping.md create mode 100644 templates/Prompt/Blocks-Research.md create mode 100644 templates/Prompt/Blocks-Source-Link.md create mode 100644 templates/Prompt/Blocks-Source-Projekt.md create mode 100644 templates/Prompt/Blocks-Source-Thema.md create mode 100644 templates/Prompt/Blocks-Source-Uni.md create mode 100644 templates/Prompt/Blocks-Supplement.md delete mode 100644 templates/Prompt/Element-Stil.md create mode 100644 templates/Prompt/Element-Style.md create mode 100644 templates/Prompt/Facts-Check.md create mode 100644 templates/Prompt/Facts-Research.md create mode 100644 templates/Prompt/Facts-Supplement.md delete mode 100644 templates/Prompt/Fakten-Check.md delete mode 100644 templates/Prompt/Fakten-Ergaenzung.md delete mode 100644 templates/Prompt/Fakten-Recherche.md delete mode 100644 templates/Prompt/Frage-Muster-Kritik.md delete mode 100644 templates/Prompt/Frage-Muster-Recherche.md delete mode 100644 templates/Prompt/Gliederung-Voraussetzungen.md create mode 100644 templates/Prompt/Guide-Content-Check.md create mode 100644 templates/Prompt/Guide-Content-Fix.md create mode 100644 templates/Prompt/Guide-Content.md create mode 100644 templates/Prompt/Guide-Facts-Projekt.md create mode 100644 templates/Prompt/Guide-Facts-Thema.md delete mode 100644 templates/Prompt/Guide-Fakten-Projekt.md delete mode 100644 templates/Prompt/Guide-Fakten-Thema.md delete mode 100644 templates/Prompt/Guide-Gliederung-Judge.md delete mode 100644 templates/Prompt/Guide-Gliederung.md delete mode 100644 templates/Prompt/Guide-Inhalt-Check.md delete mode 100644 templates/Prompt/Guide-Inhalt-Fix.md delete mode 100644 templates/Prompt/Guide-Inhalt.md create mode 100644 templates/Prompt/Guide-Outline-Judge.md create mode 100644 templates/Prompt/Guide-Outline.md create mode 100644 templates/Prompt/Levels-Mapping.md create mode 100644 templates/Prompt/Levels-Research.md create mode 100644 templates/Prompt/Outline-Prerequisites.md delete mode 100644 templates/Prompt/Quelle-Relevanz.md create mode 100644 templates/Prompt/Question-Pattern-Critique.md create mode 100644 templates/Prompt/Question-Pattern-Research.md create mode 100644 templates/Prompt/Relevance-Mapping.md create mode 100644 templates/Prompt/Relevance-Research.md delete mode 100644 templates/Prompt/Relevanz-Mapping.md delete mode 100644 templates/Prompt/Relevanz-Recherche.md create mode 100644 templates/Prompt/Source-Relevance.md delete mode 100644 templates/Prompt/Stufen-Mapping.md delete mode 100644 templates/Prompt/Stufen-Recherche.md delete mode 100644 templates/Prompt/Subbaustein-Mapping.md delete mode 100644 templates/Prompt/Subbaustein-Recherche.md create mode 100644 templates/Prompt/Subblock-Mapping.md create mode 100644 templates/Prompt/Subblock-Research.md diff --git a/backend/agents.py b/backend/agents.py index 9db9f97..ee3bec3 100644 --- a/backend/agents.py +++ b/backend/agents.py @@ -1,7 +1,7 @@ -"""Provider-Schicht: führt Agent-Aufrufe über die Claude-CLI oder OpenCode (MiniMax) aus. +"""Provider layer: runs agent calls via the Claude CLI or OpenCode (MiniMax). -Beide Runner sind unabhängig. Fehlt ein Binary/Key, schlägt nur der -jeweilige Provider fehl — der andere läuft unverändert weiter. +Both runners are independent. If a binary/key is missing, only the +respective provider fails — the other keeps running unchanged. """ import asyncio @@ -21,9 +21,9 @@ log = logging.getLogger("creator.agents") _active_processes: dict[str, asyncio.subprocess.Process] = {} -# Abgebrochene Scopes (Schlüssel-Präfixe, symmetrisch zu kill_process). Ein Agent, dessen -# Key mit einem dieser Präfixe beginnt, bricht VOR dem Spawn ab — so werden auch in der -# Semaphore-Schlange WARTENDE Agenten beim Abbruch sofort gestoppt, statt noch zu starten. +# Cancelled scopes (key prefixes, symmetric to kill_process). An agent whose +# key starts with one of these prefixes aborts BEFORE the spawn — so agents WAITING +# in the semaphore queue are also stopped immediately on abort instead of still starting. _cancelled_prefixes: set[str] = set() @@ -38,15 +38,15 @@ def clear_scope(prefix: str) -> None: def _scope_cancelled(agent_key: str) -> bool: return any(agent_key.startswith(p) for p in _cancelled_prefixes) -# Deckelt die realen CLI-Prozesse — unabhängig von der Pipeline-Semaphore in -# generator.py. Acquire passiert VOR dem Spawn, damit Wartezeit in der Queue -# nicht gegen den Agent-Timeout zählt. +# Caps the real CLI processes — independent of the pipeline semaphore in +# generator.py. The acquire happens BEFORE the spawn so that queue wait time +# does not count against the agent timeout. _batch_sem = asyncio.Semaphore(MAX_CONCURRENT_AGENTS) _interactive_sem = asyncio.Semaphore(MAX_CONCURRENT_INTERACTIVE) -# OpenCode-Starts serialisieren: gleichzeitig startende Prozesse kollidieren an -# der internen Session-DB ("database is locked", Exit nach <1s). Der kurze -# Versatz entzerrt die Starts; danach laufen die Prozesse normal parallel. +# Serialize OpenCode starts: processes starting simultaneously collide on the +# internal session DB ("database is locked", exit after <1s). The short +# stagger spreads out the starts; afterwards the processes run in parallel normally. _opencode_start_lock = asyncio.Lock() _OPENCODE_START_DELAY = 1.0 @@ -58,7 +58,7 @@ _CLAUDE_TOOLS = { "none": None, } -# Capability → OpenCode-Agent (Tool-Rechte in dev-ops/opencode.json definiert) +# Capability → OpenCode agent (tool permissions defined in dev-ops/opencode.json) _OPENCODE_AGENTS = { "full": "full", "files": "files", @@ -86,8 +86,8 @@ def provider_available(provider: str) -> bool: def _kill(process) -> None: - """Killt den Agenten samt Kindprozessen über die Prozess-Gruppe (sonst überleben die - von der CLI gestarteten Kinder, halten die Pipes offen und blockieren communicate()).""" + """Kill the agent and its child processes via the process group (otherwise the + children spawned by the CLI survive, keep the pipes open and block communicate()).""" try: os.killpg(os.getpgid(process.pid), signal.SIGKILL) except (ProcessLookupError, PermissionError): @@ -98,9 +98,9 @@ def _kill(process) -> None: def kill_process(agent_key_prefix: str) -> None: - """Killt alle aktiven Prozesse, deren Key mit dem Prefix beginnt (deckt -plan/-w1… ab).""" + """Kill all active processes whose key starts with the prefix (covers -plan/-w1…).""" for key, process in list(_active_processes.items()): - if process.returncode is not None: # tote Einträge beim Iterieren aufräumen + if process.returncode is not None: # clean up dead entries while iterating _active_processes.pop(key, None) continue if key.startswith(agent_key_prefix): @@ -117,16 +117,16 @@ async def run_agent( capabilities: str = "none", lane: str = "batch", ) -> tuple[int, str, str]: - if _scope_cancelled(agent_key): # vor dem Anstehen: gar nicht erst in die Schlange - return 1, "", "abgebrochen" + if _scope_cancelled(agent_key): # before queueing: don't even enter the queue + return 1, "", "cancelled" if provider not in PROVIDERS: - return 1, "", f"Unbekannter Provider: {provider}" + return 1, "", f"Unknown provider: {provider}" if shutil.which(PROVIDERS[provider]["cli"]) is None: - return 1, "", f"CLI '{PROVIDERS[provider]['cli']}' nicht installiert (Provider: {provider})" + return 1, "", f"CLI '{PROVIDERS[provider]['cli']}' not installed (provider: {provider})" sem = _interactive_sem if lane == "interactive" else _batch_sem async with sem: - if _scope_cancelled(agent_key): # nach dem Acquire: in der Schlange abgebrochen → kein Spawn - return 1, "", "abgebrochen" + if _scope_cancelled(agent_key): # after the acquire: cancelled in the queue → no spawn + return 1, "", "cancelled" if PROVIDERS[provider]["cli"] == "opencode": return await _run_opencode(agent_key, prompt, timeout, provider, role, capabilities) return await _run_claude_cli(agent_key, prompt, timeout, role, capabilities) @@ -141,7 +141,7 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, stdin=asyncio.subprocess.PIPE if stdin_data is not None else asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - start_new_session=True, # eigene Prozess-Gruppe → killpg killt auch Kindprozesse + start_new_session=True, # own process group → killpg also kills child processes ) if stagger: @@ -163,16 +163,16 @@ async def _communicate(agent_key: str, cmd: list[str], stdin_data: bytes | None, await asyncio.wait_for(process.wait(), timeout=5) except asyncio.TimeoutError: pass - log.info("agent %s: Timeout nach %ds", agent_key, timeout) + log.info("agent %s: timeout after %ds", agent_key, timeout) raise log.info( - "agent %s: exit %s nach %.1fs (%d Bytes stdout)", + "agent %s: exit %s after %.1fs (%d bytes stdout)", agent_key, process.returncode, time.monotonic() - start, len(stdout), ) return process.returncode, stdout.decode("utf-8", errors="replace"), stderr.decode("utf-8", errors="replace") finally: - # Pop nur bei Identität: ein Slot-Restart unter demselben Key darf den - # NEUEN Prozess nicht aus dem Tracking werfen. + # Pop only on identity: a slot restart under the same key must not evict + # the NEW process from tracking. if _active_processes.get(agent_key) is process: del _active_processes[agent_key] @@ -189,12 +189,12 @@ async def _run_claude_cli(agent_key: str, prompt: str, timeout: int, role: str, async def _run_opencode(agent_key: str, prompt: str, timeout: int, provider: str, role: str, capabilities: str) -> tuple[int, str, str]: cfg = PROVIDERS[provider] - # Prompt über Tempdatei statt argv (ARG_MAX-Schutz bei großen Projekt-Prompts) + # Prompt via temp file instead of argv (ARG_MAX protection for large project prompts) with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8", dir=tempfile.gettempdir()) as f: f.write(prompt) prompt_path = Path(f.name) - # Positional-Message MUSS vor -f stehen: -f ist ein Array-Flag und - # frisst sonst den Text als zweiten Dateinamen ("File not found"). + # The positional message MUST come before -f: -f is an array flag and + # would otherwise eat the text as a second file name ("File not found"). cmd = [ cfg["cli"], "run", "Folge exakt den Anweisungen in der angehängten Datei. Sie sind der vollständige Auftrag.", @@ -214,7 +214,7 @@ _ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") def _clean_opencode_output(text: str) -> str: - """Entfernt ANSI-Codes und den führenden Banner ("> agent · modell").""" + """Strip ANSI codes and the leading banner ("> agent · model").""" text = _ANSI_RE.sub("", text) lines = text.splitlines() while lines and (not lines[0].strip() or lines[0].lstrip().startswith(">")): diff --git a/backend/bausteine.py b/backend/bausteine.py deleted file mode 100644 index 618600e..0000000 --- a/backend/bausteine.py +++ /dev/null @@ -1,3330 +0,0 @@ -"""Bausteine-Pipeline: Recherche-Konsens + Klärungs-Loop — reines Inventar, unsortiert. - -5x Recherche (min. 3, Grace) → Mapping (Konsens/Rest) → Klärungs-Loop (max. -KONSENS_MAX_RUNDEN Runden): 3 Auswahl-Agenten (min. 2, Grace) entscheiden -über den strittigen Rest, ein Mapping-Agent sortiert in aufnehmen/verwerfen/ -weiter strittig. Leerer Rest beendet den Loop; die letzte Runde muss alles -entscheiden. Races nutzen ein Grace-Fenster statt „erste N gewinnen": Nach dem -ersten gültigen Ergebnis dürfen die übrigen Agenten KONSENS_GRACE Sekunden -fertig werden. Der Konsens wird im Code akkumuliert — kein Agent re-emittiert -die Gesamtliste. -""" - -import asyncio -import json -import logging -import math -import re -import shutil -import subprocess -import time -from pathlib import Path - -import database as db -import embedding -from agents import kill_process, cancel_scope, clear_scope, run_agent -from config import KONSENS_GRACE, RECHERCHE_GRACE, KONSENS_MAX_RUNDEN, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP -from fsutil import atomic_write_text, atomic_write_json -from jsonio import read_json_file as _json_datei -from paths import arbeit_dir, bausteine_path, frage_muster_path, project_dir, subbausteine_path, quelle_path, quelle_crawl_dir, safe_ordner -from crawl import crawl -from pipeline import ( - CANCELLED, FAILED, OK, GenContext, _extra, _gather_fortschritt, _janein_schema, _log, _prompt, _race, - _relevanz_schema, _runde_schema, _semaphore, _str_liste, _stufen_schema, _timeout, run_single_slot, -) -from textkit import ( - _eindeutige_titel, _lade_bausteine, _norm_titel, _parse_auswahl, _parse_subbausteine, _titel, - _titel_aufloesen, _titel_index, -) - -# Subbausteine (Websuche je Baustein) chunken: 1 Agent je ~10 Bausteine, gedeckelt. -SUBBAUSTEIN_CHUNK = 10 -SUBBAUSTEIN_MAX = 40 -# Einstufen ist billig (kurzes Urteil, keine Websuche) → größere Pakete, weniger Dateien/Agenten. -STUFE_CHUNK = 100 - -# Recherche: feste Datei-Batches statt Such-Loop → jede Crawl-Seite genau einmal zugeteilt. -RECHERCHE_BATCH = 20 # Crawl-Seiten je Batch -RECHERCHE_READERS = 2 # Reader-Agenten je Batch (Konsens ≥2 innerhalb des Batches) -RECHERCHE_THEMA_AGENTEN = 5 # Web-Modus (Quelle „thema", kein Crawl-Ordner) -# uni/projekt: Skript-Text in Abschnitte ~dieser Größe chunken (gegen Lost-in-the-Middle bei -# großen Dokumenten). ~12k Zeichen ≈ 3k Token → sicher unter der Recall-Abfall-Schwelle. -RECHERCHE_ABSCHNITT_ZEICHEN = 12000 -# Sichtung (Content/Noise) ist jetzt ein deterministischer Regel-Filter (config.CRAWL_*). -SUBBAUSTEIN_KAPPE = 900 # Subbaustein-Finde-Loop je Chunk (15 min) -KONSOLIDIERUNG_CHUNK = 600 # bis hierher EIN globaler Judge (dedupt alles); darüber chunked + Merge-Pass — nur Fallback-Pfad -DEDUP_PAAR_FLOOR = 0.6 # Mindest-Cosine fürs Kandidaten-Paar (Complete-Link aggregiert → kein Chaining) -DEDUP_PAARE_CHUNK = 40 # Paare je Judge-Paket (Pairwise-Verifikation statt Block-Mischer) -FILTER_CHUNK = 35 # zu beurteilende Bausteine je Judge im Degradier-Pass (volle Liste als Kontext) -# Frage-Muster-Chunks per LPT nach Sub-Last balancieren (Makespan), statt nach Baustein-Anzahl. -FRAGE_CHUNK_SUBS = 50 # Ziel-Summe relevanter Subs je Chunk -FRAGE_MAX_RUNDEN = 3 # Nachhol-Runden für Subs ohne Muster (LLM lässt pro Chunk ~18 % aus) -FAKTEN_CHUNK_SUBS = 25 # Fakten-Extraktion: kleinere Chunks (Fakten sind umfangreicher als Muster) -FAKTEN_CHECK_PANEL = 3 # Judges je Chunk im Fakten-Check (Mehrheit beanstandet) -KONSOLIDIERUNG_PANEL = 3 # Mapping-Judges je Chunk (Panel → Reconcile statt Einzel-Judge) -SUBBAUSTEIN_PANEL = 3 # Source-Judges in der Subbaustein-Klärung (Mehrheit statt Einzel-Judge) - -log = logging.getLogger("creator.bausteine") - -_bausteine_progress: dict[str, str] = {} -_bausteine_errors: dict[str, str] = {} -_bausteine_cancelled: set[str] = set() -_bausteine_step: dict[str, int] = {} - -ARTEFAKT_TYPEN = ("karteikarte", "beispiel") - - -def lade_quelle(topic: str) -> dict: - """Persistierte Quellen-Wahl lesen. Fallback (Alt-Themen ohne quelle.json): - existiert projects/ → projekt, sonst thema.""" - q = _json_datei(quelle_path(topic)) - if isinstance(q, dict) and q.get("type") in ("thema", "projekt", "uni", "link"): - return q - if project_dir(topic).is_dir(): - return {"type": "projekt", "ort": f"projects/{topic}", "spec": ""} - return {"type": "thema", "ort": "", "spec": ""} - - -def quelle_ordner(topic: str) -> Path | None: - """Ordner-Quelle (projekt/uni → Pfad, link → Crawl-Ordner) — sonst None (thema).""" - q = lade_quelle(topic) - if q["type"] == "link": - return quelle_crawl_dir(topic) - if q["type"] in ("projekt", "uni"): - return safe_ordner(q.get("ort", "")) - return None - - -def _crawl_fertig(topic: str) -> bool: - return (quelle_crawl_dir(topic) / ".done").exists() # Marker erst bei sauberem Abschluss - - -# Lernpfad-Stufen (anfaenger/fortgeschritten/experte); alte Schwierigkeits-Werte abwärtskompatibel. -_STUFEN = ("anfaenger", "fortgeschritten", "experte", "einfach", "mittel", "schwer") - - -async def subbausteine_titel(topic: str, baustein: str) -> list[str]: - """Subbaustein-Titel eines Bausteins — DB-first (Konsens), Fallback Sidecar-Datei.""" - rows = [s["sub_titel"] for s in await db.list_subbausteine(topic, _norm_titel(baustein)) - if s["status"] == "konsens" and s["sub_titel"]] - if rows: - return rows - sc = _json_datei(subbausteine_path(topic)) - if not isinstance(sc, dict): - return [] - return [ - t for s in (sc.get(baustein) or []) - if isinstance(s, dict) and (t := str(s.get("titel", "")).strip()) - ] - - -async def lade_frage_muster(topic: str, baustein: str) -> list[dict]: - """Vordefinierte Frage-Muster eines Bausteins — DB-first, Fallback Sidecar (leer = Live).""" - rows = await db.list_frage_muster(topic, _norm_titel(baustein)) - if rows: - return [{"subbaustein": r["sub_titel"], "frage": r["frage"]} for r in rows if r["frage"]] - fm = _json_datei(frage_muster_path(topic)) - if not isinstance(fm, dict): - return [] - return [ - {"subbaustein": str(e.get("subbaustein", "")).strip(), "frage": frage} - for e in (fm.get(baustein) or []) - if isinstance(e, dict) and (frage := str(e.get("frage", "")).strip()) - ] - - -async def subbausteine_frei(topic: str, baustein: str, max_ebene: int) -> list[str]: - """Subbaustein-Titel bis zur freigeschalteten Ebene (≤ max_ebene). Fallback ohne - Ebenen-Wissen (Altbestand/Sidecar): alle Subbaustein-Titel.""" - rows = await db.subs_mit_ebene(topic, baustein) - if not rows: - return await subbausteine_titel(topic, baustein) - return [s["titel"] for s in rows if s["ebene"] <= max_ebene and s["titel"]] - - -async def lade_frage_muster_frei(topic: str, baustein: str, max_ebene: int) -> list[dict]: - """Frage-Muster, gefiltert auf Subbausteine bis zur freigeschalteten Ebene. Ohne - Ebenen-Wissen (Altbestand/Sidecar) ungefiltert.""" - rows = await db.subs_mit_ebene(topic, baustein) - if not rows: - return await lade_frage_muster(topic, baustein) - frei = {s["norm"] for s in rows if s["ebene"] <= max_ebene} - return [m for m in await lade_frage_muster(topic, baustein) if _norm_titel(m["subbaustein"]) in frei] - - -async def lade_uebersicht(topic: str) -> list[dict]: - """Strukturierte Baustein-Liste für die Übersicht — DB-first (Konsens + Subs/Stufen/Relevanz), - Fallback bausteine.md + Sidecar (Alt-Themen).""" - bs = await db.list_bausteine(topic, status="konsens") - if bs: - out = [] - for num, b in enumerate(bs, 1): - subs = [s for s in await db.list_subbausteine(topic, b["titel_norm"]) if s["status"] == "konsens"] - out.append({ - "num": num, "titel": b["titel"], "beschreibung": b["beschreibung"], - "subbausteine": [ - {"titel": s["sub_titel"], - "stufe": s["stufe"] if s["stufe"] in _STUFEN else "fortgeschritten", - "relevanz": s["relevanz"] if s["relevanz"] in ("relevant", "rand") else None} - for s in subs if s["sub_titel"] - ], - }) - return out - entries = _lade_bausteine(_read(bausteine_path(topic))) - sidecar = _json_datei(subbausteine_path(topic)) - sidecar = sidecar if isinstance(sidecar, dict) else {} - out = [] - for num, entry in entries.items(): - titel = _titel(entry) - teile = entry.split(" — ", 1) - beschreibung = teile[1].strip() if len(teile) == 2 else "" - subbausteine = [ - { - "titel": t, - "stufe": s.get("stufe") if s.get("stufe") in _STUFEN else "fortgeschritten", - "relevanz": s.get("relevanz") if s.get("relevanz") in ("relevant", "rand") else None, - } - for s in (sidecar.get(titel) or []) - if isinstance(s, dict) and (t := str(s.get("titel", "")).strip()) - ] - out.append({"num": num, "titel": titel, "beschreibung": beschreibung, "subbausteine": subbausteine}) - return out - - -def _bausteine_steps(topic: str) -> tuple: - """Schritte je Quelle: link bekommt vorne „Quelle laden", projekt zusätzlich „Ergänzung". - - Subbausteine + Stufen sind je drei Phasen (Finden, Wählen, Klären). Pro Phase - laufen alle Pakete parallel; der Schritt bleibt, bis das letzte Paket fertig ist. - """ - q = lade_quelle(topic) - base = ("Recherche", "Konsolidierung", "Klärung", "Dedup", "Bausteine-Filter") - rest = ( - "Subbausteine finden", "Subbausteine wählen", "Subbausteine klären", - "Fakten finden", "Fakten prüfen", "Fakten fix", - "Stufen finden", "Stufen wählen", "Stufen klären", - "Relevanz finden", "Relevanz wählen", "Relevanz klären", - "Gliederung", - "Fragen finden", "Fragen wählen", "Fragen klären", "Fragen prüfen", - "Karteikarten", "Beispiele", - ) - mitte = base + (("Ergänzung",) if q["type"] == "projekt" else ()) + rest - return (("Quelle aufbereiten",) if q["type"] == "link" else ()) + mitte - - -def _step_idx(topic: str, name: str) -> int: - return _bausteine_steps(topic).index(name) - - -def _melde_p(set_p, topic: str, schritt: str): - """Async-Melde-Callback für _gather_fortschritt: setzt „ d/t…" + Schritt-Index.""" - idx = _step_idx(topic, schritt) - async def melde(d, t): - set_p(f"{schritt} {d}/{t}…", step=idx) - return melde - - -# Grobe Anzeige-Phasen: bündeln die Feinschritte (intern bleibt alles feingranular). -# Sonderschritte (Quelle laden, Ergänzung) gehören zur Phase „Inventar". -PHASEN = ( - ("Quelle", ("Quelle aufbereiten",)), - ("Inventar", ("Recherche", "Konsolidierung", "Klärung", "Dedup", "Bausteine-Filter", "Ergänzung")), - ("Subbausteine", ("Subbausteine finden", "Subbausteine wählen", "Subbausteine klären")), - ("Fakten", ("Fakten finden", "Fakten prüfen", "Fakten fix")), - ("Stufen", ("Stufen finden", "Stufen wählen", "Stufen klären")), - ("Relevanz", ("Relevanz finden", "Relevanz wählen", "Relevanz klären")), - ("Gliederung", ("Gliederung",)), - ("Fragen", ("Fragen finden", "Fragen wählen", "Fragen klären", "Fragen prüfen")), - ("Artefakte", ("Karteikarten", "Beispiele")), -) - - -def _phasen(topic: str) -> list[tuple[str, int]]: - """[(grob_label, Anzahl vorhandener Feinschritte)] für die aktuelle Quelle.""" - feine = _bausteine_steps(topic) - return [(label, n) for label, members in PHASEN if (n := sum(f in members for f in feine))] - - -def _phasen_status(topic: str, current: int | None) -> list[dict]: - """Grobe Phasen-Zustände aus dem feinen Fortschritt `current` (None = alles pending, - len(feine) = alles done). → [{label, state}] mit state done/active/pending.""" - out, start = [], 0 - for label, n in _phasen(topic): - end = start + n - if current is None or current < start: - state = "pending" - elif current >= end: - state = "done" - else: - state = "active" - out.append({"label": label, "state": state}) - start = end - return out - - -def _bausteine_files(topic: str) -> dict: - arbeit = arbeit_dir(topic) - runden = range(1, KONSENS_MAX_RUNDEN + 1) - return { - "final": bausteine_path(topic), - "arbeit": arbeit, - "recherche": [arbeit / f"recherche-{i}.md" for i in (1, 2, 3, 4, 5)], - "recherche_mapping": arbeit / "recherche-mapping.json", - "auswahl": {n: [arbeit / f"auswahl-r{n}-{i}.json" for i in (1, 2, 3)] for n in runden}, - "mapping": {n: arbeit / f"auswahl-mapping-r{n}.json" for n in runden}, - "ergaenzung": arbeit / "ergaenzung.json", - "sub_roh": arbeit / "subbausteine-roh.json", - "fakten": arbeit / "subbausteine-fakten.json", - "sidecar": subbausteine_path(topic), - "frage_muster": frage_muster_path(topic), - "gliederung": arbeit / "gliederung.json", - "gliederung_slots": [arbeit / f"gliederung-{i}.json" for i in (1, 2, 3)], - "artefakte": arbeit / "artefakte.json", - } - - -def _alle_slot_dateien(files: dict) -> list[Path]: - arbeit = files["arbeit"] - # Subbaustein-/Stufen-Slots sind pro Chunk dynamisch — per Glob einsammeln. - dyn = (list(arbeit.glob("subbaustein-*")) + list(arbeit.glob("fakten-*")) + list(arbeit.glob("stufe-*")) + list(arbeit.glob("relevanz-*")) - + list(arbeit.glob("frage-muster-*")) + list(arbeit.glob("gliederung-*")) + list(arbeit.glob("artefakt-*")) - + list(arbeit.glob("recherche-*")) + list(arbeit.glob("konsolidierung-*")) - + list(arbeit.glob("klaerung*")) + list(arbeit.glob("dedup-*")) - + list(arbeit.glob("inventar-filter*"))) if arbeit.is_dir() else [] - return [ - *files["recherche"], files["recherche_mapping"], - *(p for slots in files["auswahl"].values() for p in slots), - *files["mapping"].values(), files["ergaenzung"], - files["sub_roh"], files["sidecar"], files["frage_muster"], - files["fakten"], files["gliederung"], files["artefakte"], *dyn, - ] - - -def cancel_bausteine(topic: str) -> bool: - if topic not in _bausteine_progress: - return False - _bausteine_cancelled.add(topic) - cancel_scope(f"bausteine-{topic}-") # wartende Agenten bailen vorm Spawn - kill_process(f"bausteine-{topic}-") # laufende Subprozesse killen - return True - - -def _resume_step(topic: str) -> int: - """Erster noch offener Schritt anhand der persistierten Artefakte. - Inventar (Recherche→Konsolidierung→Klärung) gilt als fertig, sobald bausteine.md vorliegt.""" - files = _bausteine_files(topic) - q = lade_quelle(topic) - if q["type"] == "link" and not _crawl_fertig(topic): - return _step_idx(topic, "Quelle aufbereiten") - if not files["final"].exists(): # Inventar (DB-Loop) noch offen - return _step_idx(topic, "Recherche") - if q["type"] == "projekt" and not files["ergaenzung"].exists(): - return _step_idx(topic, "Ergänzung") - sidecar = _json_datei(files["sidecar"]) - if _sidecar_schema(sidecar) is not None: - # Stufen fertig; nur noch Relevanz offen? - if not _relevanz_komplett(sidecar): - return _step_idx(topic, "Relevanz finden") - # Relevanz fertig; Gliederung (Bausteine-Artefakt für den Guide) offen? - if not _gliederung_komplett(files): - return _step_idx(topic, "Gliederung") - # Gliederung fertig; Frage-Muster offen? - if not _frage_muster_komplett(topic): - return _step_idx(topic, "Fragen finden") - # Fragen fertig; Lern-Artefakte (Karteikarten/Beispiele) offen? - if not _artefakte_komplett(files): - return _step_idx(topic, "Karteikarten") - return len(_bausteine_steps(topic)) - if _sub_roh_schema(_json_datei(files["sub_roh"])) is None: - return _step_idx(topic, "Subbausteine finden") - # Subbausteine fertig; Fakten noch offen? (Fakten kommen vor den Stufen.) - if not _fakten_komplett(files): - return _step_idx(topic, "Fakten finden") - return _step_idx(topic, "Stufen finden") - - -def _feine_status(topic: str, current: int | None) -> list[dict]: - """Feiner Teilschritt-Status: je Schritt {label, phase, state}. state aus `current` - (done = idx). Phasen-Label aus PHASEN.""" - schritt_phase = {s: label for label, schritte in PHASEN for s in schritte} - out = [] - for i, s in enumerate(_bausteine_steps(topic)): - state = "pending" if current is None or current < i else "done" if current > i else "active" - out.append({"label": s, "phase": schritt_phase.get(s, ""), "state": state}) - return out - - -def bausteine_status(topic: str) -> dict: - # Intern feingranular (Resume/Progress); für die Anzeige zu 5 groben Phasen gebündelt. - feine = _bausteine_steps(topic) - ready = bausteine_path(topic).exists() - generating = topic in _bausteine_progress - partial = False - if generating: - current = _bausteine_step.get(topic) - elif ready: - current = len(feine) # alles fertig → alle Phasen done - else: - current = _resume_step(topic) - partial = current > 0 - return { - "ready": ready, - "generating": generating, - "progress": _bausteine_progress.get(topic), - "error": _bausteine_errors.get(topic), - "partial": partial, - "steps": _phasen_status(topic, current), - "feine_steps": _feine_status(topic, current), - } - - -def active_bausteine() -> list[dict]: - return [{"topic": t, "progress": p} for t, p in _bausteine_progress.items()] - - -def reset_bausteine(topic: str) -> None: - """„Entfernen": löscht den GESAMTEN Bausteine-Bereich — Crawl, Sichtung, Inventar … Fragen. - BEHÄLT nur die Themen-Config `quelle.json` (Typ/Link/Spec). Re-Generieren crawlt neu. - (Crawl/Sichtung gehören zu den Bausteinen; nur die Config ist „Thema".)""" - files = _bausteine_files(topic) - files["final"].unlink(missing_ok=True) - files["sidecar"].unlink(missing_ok=True) - files["frage_muster"].unlink(missing_ok=True) - shutil.rmtree(quelle_crawl_dir(topic), ignore_errors=True) # Crawl gehört zu Bausteinen - shutil.rmtree(files["arbeit"], ignore_errors=True) - _bausteine_errors.pop(topic, None) - # quelle.json bleibt bewusst stehen — das ist die Themen-Config. - - -def _phase_idx(label: str) -> int: - """Index der groben Phase in der kanonischen Reihenfolge (Quelle=0 … Fragen=5).""" - order = [l for l, _ in PHASEN] - return order.index(label) if label in order else 1 - - -def _reset_ab_phase(topic: str, label: str) -> None: - """Datei-Artefakte AB der groben Phase `label` löschen (Quelle/Inventar … Fragen), frühere - behalten. Kumulativ. quelle.json + Crawl (.done) bleiben immer (re-crawl nur bei Voll-Reset).""" - files = _bausteine_files(topic) - arbeit = files["arbeit"] - idx = _phase_idx(label) - - def glob_del(pat: str) -> None: - if arbeit.is_dir(): - for p in arbeit.glob(pat): - p.unlink(missing_ok=True) - - # Phasen-Index: Quelle=0 · Inventar=1 · Subbausteine=2 · Fakten=3 · Stufen=4 · Relevanz=5 · Gliederung=6 · Fragen=7 · Artefakte=8 - if idx <= 8: # Artefakte (Karteikarten/Beispiele) - files["artefakte"].unlink(missing_ok=True) - glob_del("artefakt-*") - if idx <= 7: # Fragen - files["frage_muster"].unlink(missing_ok=True) - glob_del("frage-muster-*") - if idx <= 6: # Gliederung - files["gliederung"].unlink(missing_ok=True) - glob_del("gliederung-*") - if idx <= 5: # Relevanz - glob_del("relevanz-*") - if idx <= 4: # Stufen + Relevanz teilen die Sidecar → ab Stufen ganz neu - files["sidecar"].unlink(missing_ok=True) - glob_del("stufe-*") - else: # ab Relevanz: Stufen behalten, nur Relevanz-Felder strippen - sc = _json_datei(files["sidecar"]) - if isinstance(sc, dict): - for subs in sc.values(): - for s in (subs if isinstance(subs, list) else []): - if isinstance(s, dict): - s.pop("relevanz", None) - atomic_write_json(files["sidecar"], sc, indent=1) - if idx <= 3: # Fakten (vor den Stufen) — Fakten-Map + Arbeitsdateien weg - files["fakten"].unlink(missing_ok=True) - glob_del("fakten-*") - if idx <= 2: # Subbausteine - files["sub_roh"].unlink(missing_ok=True) - glob_del("subbaustein-*") - if idx <= 1: # Inventar (und Quelle) = Inventar-Dateien + bausteine.md weg - for p_alt in _alle_slot_dateien(files): - p_alt.unlink(missing_ok=True) - files["final"].unlink(missing_ok=True) - - -async def _reset_ab_step(topic: str, step_idx: int) -> None: - """Feiner Reset AB einem Teilschritt (0-basierter Index in _bausteine_steps). Setzt - pipeline_state + Artefakte + DB ab hier zurück; frühere Schritte bleiben. Inventar-Teilschritte - rekonstruieren den DB-Status aus den Artefakten (Dedup/Filter sicher; Klärung fällt robust auf - Konsolidierung zurück, weil die Klärung umbenennt → Titel-Mismatch wäre fragil).""" - feine = list(_bausteine_steps(topic)) - if not (0 <= step_idx < len(feine)): - return - ab = set(feine[step_idx:]) - files = _bausteine_files(topic) - arbeit = files["arbeit"] - - def gd(pat: str) -> None: - if arbeit.is_dir(): - for p in arbeit.glob(pat): - p.unlink(missing_ok=True) - - await db.delete_pipeline_state(topic, list(feine[step_idx:])) - # Spätere Artefakte/DB kumulativ ab dem betroffenen Schritt (von hinten nach vorn). - if {"Beispiele", "Karteikarten"} & ab: - files["artefakte"].unlink(missing_ok=True); gd("artefakt-*"); await db.delete_sub_artefakte(topic) - if any(s.startswith("Fragen") for s in ab): - files["frage_muster"].unlink(missing_ok=True); gd("frage-muster-*"); await db.delete_frage_muster(topic) - if "Gliederung" in ab: - files["gliederung"].unlink(missing_ok=True); gd("gliederung-*"); await db.delete_gliederung(topic) - if any(s.startswith("Relevanz") for s in ab): - gd("relevanz-*") - if any(s.startswith("Stufen") for s in ab): - gd("stufe-*") - if any(s.startswith("Fakten") for s in ab): - files["fakten"].unlink(missing_ok=True); gd("fakten-*") - # Sidecar trägt Subbausteine + ihre Felder stufe/relevanz/fakten. Ab Subbausteinen ganz neu; - # sonst nur die Felder der neu-zu-bauenden Phasen strippen — Subbausteine bleiben erhalten. - if any(s.startswith("Subbaustein") for s in ab): - files["sidecar"].unlink(missing_ok=True) - else: - strip = {f for s, f in (("Fakten", "fakten"), ("Stufen", "stufe"), ("Relevanz", "relevanz")) - if any(x.startswith(s) for x in ab)} - if strip: - sc = _json_datei(files["sidecar"]) - if isinstance(sc, dict): - for subs in sc.values(): - for s in (subs if isinstance(subs, list) else []): - if isinstance(s, dict): - for f in strip: - s.pop(f, None) - atomic_write_json(files["sidecar"], sc, indent=1) - if any(s.startswith("Subbaustein") for s in ab): - files["sub_roh"].unlink(missing_ok=True); gd("subbaustein-*"); await db.delete_subbausteine(topic) - # --- Inventar (DB-Status kaskadiert) --- - if "Bausteine-Filter" in ab and not ({"Klärung", "Konsolidierung", "Recherche", "Dedup"} & ab): - # Nur Filter neu: degradierte Bausteine zurück auf konsens. - d = _json_datei(arbeit / "inventar-filter.json") - for f in (d.get("fragmente", []) if isinstance(d, dict) else []): - await db.set_baustein_status(topic, _norm_titel(f.get("fragment", "")), "konsens") - gd("inventar-filter*") - if "Dedup" in ab and not ({"Klärung", "Konsolidierung", "Recherche"} & ab): - # Dedup (+Filter) neu: alle im Dedup/Filter verworfenen Bausteine zurück auf konsens. - for art in ("dedup-runde-1.json", "inventar-filter.json"): - d = _json_datei(arbeit / art) - titel = ([t for g in d.get("gruppen", []) for t in g] if isinstance(d, dict) and "gruppen" in d - else [f.get("fragment", "") for f in d.get("fragmente", [])] if isinstance(d, dict) else []) - for t in titel: - await db.set_baustein_status(topic, _norm_titel(t), "konsens") - gd("dedup-*"); gd("inventar-filter*") - if {"Klärung", "Konsolidierung"} & ab and not ({"Recherche"} & ab): - # Klärung/Konsolidierung neu: Inventar-DB leeren (Recherche-Reader bleiben). Klärungs-Rückbau - # wäre wegen Umbenennung fragil → ab Konsolidierung sauber neu aufbauen. - await db.delete_bausteine(topic) - gd("klaerung*"); gd("konsolidierung-*"); gd("dedup-*"); gd("inventar-filter*") - if "Recherche" in ab: # ganzes Inventar wie Phase-Reset - for p_alt in _alle_slot_dateien(files): - p_alt.unlink(missing_ok=True) - files["final"].unlink(missing_ok=True) - await db.delete_bausteine(topic) - - -async def reset_bausteine_ab_step(topic: str, step_idx: int) -> None: - """Öffentlich: NUR ab einem Teilschritt zurücksetzen — kein Neu-Generieren. Lässt einen - Teil-Stand zurück (die Schritte ab hier gelten als offen). Läuft eine Generierung → ignorieren.""" - if topic in _bausteine_progress: - return - await _reset_ab_step(topic, step_idx) - - -def _ergaenzung_schema(data): - """{"bausteine": [{"titel", "beschreibung"}]} → Liste (leer erlaubt) · sonst None.""" - if not isinstance(data, dict) or not isinstance(data.get("bausteine"), list): - return None - out = [] - for b in data["bausteine"]: - if not isinstance(b, dict) or not isinstance(b.get("titel"), str) or not isinstance(b.get("beschreibung"), str): - return None - titel, beschreibung = b["titel"].strip(), b["beschreibung"].strip() - if not titel: - return None - out.append((titel, beschreibung)) - return out - - -def _pdfs_konvertieren(project: Path) -> None: - """PDFs im Projekt in .txt wandeln (pdftotext) — Agenten lesen Text statt Seiten-Bildern. - - Wird vor jeder Projekt-Generierung aufgerufen; konvertiert nur, wenn die - .txt fehlt oder älter als das PDF ist. Das Original bleibt unangetastet. - Fehlt pdftotext und das Projekt enthält PDFs → harter Fehler statt - unzuverlässigem Direkt-Lese-Modus (MiniMax-Bilderlimit, Vision-Kosten). - """ - pdfs = list(project.rglob("*.pdf")) - if not pdfs: - return - if shutil.which("pdftotext") is None: - raise RuntimeError("pdftotext fehlt (poppler-utils installieren) — PDFs im Projekt können nicht gelesen werden") - for pdf in pdfs: - txt = pdf.with_suffix(".txt") - if txt.exists() and txt.stat().st_mtime >= pdf.stat().st_mtime: - continue - try: - subprocess.run(["pdftotext", "-layout", str(pdf), str(txt)], check=True, timeout=120) - _log(project.name, f"PDF konvertiert: {pdf.name} → {txt.name}") - except Exception as e: - raise RuntimeError(f"PDF-Konvertierung fehlgeschlagen ({pdf.name}): {e}") from e - - -_QUELLE_TEMPLATE = {"projekt": "Bausteine-Quelle-Projekt", "uni": "Bausteine-Quelle-Uni", "link": "Bausteine-Quelle-Link"} - - -def _text_abschnitte(text: str, ziel: int = RECHERCHE_ABSCHNITT_ZEICHEN) -> list[str]: - """Text an Absatz-/Zeilengrenzen in Abschnitte ~`ziel` Zeichen splitten (gegen Lost-in-the-Middle - bei großen Dokumenten). Kleiner Text bleibt EIN Abschnitt. Inhalt bleibt vollständig — nur - Trenn-Whitespace fällt weg.""" - text = text.strip() - if len(text) <= ziel: - return [text] if text else [] - abschnitte: list[str] = [] - buf = "" - - def flush(): - nonlocal buf - if buf.strip(): - abschnitte.append(buf.strip()) - buf = "" - - for block in re.split(r"\n\s*\n", text): # an Absatz-Grenzen - block = block.strip() - if not block: - continue - if len(block) > ziel: # einzelner Riesen-Absatz → hart an Zeilen schneiden - flush() - for zeile in block.split("\n"): - if buf and len(buf) + len(zeile) + 1 > ziel: - flush() - buf += zeile + "\n" - flush() - elif buf and len(buf) + len(block) + 2 > ziel: - flush() - buf = block - else: - buf = (buf + "\n\n" + block) if buf else block - flush() - return abschnitte - - -def _build_recherche_prompt(topic: str, out_path: Path, instructions: str, typ: str, ordner: Path | None, fokus: str = "", abschnitt: str = "") -> str: - if abschnitt: - # Abschnitt-Modus (uni/projekt): Text direkt im Prompt → kleiner Kontext, kein Datei-Lesen. - source = abschnitt - elif typ in _QUELLE_TEMPLATE: - source = _prompt(_QUELLE_TEMPLATE[typ], project=ordner) - else: - source = _prompt("Bausteine-Quelle-Thema", topic=topic) - return _prompt( - "Bausteine-Recherche", - topic=topic, source=source, bausteine_path=out_path, fokus=fokus, extra=_extra(instructions), - ) - - -def _file_payload(path: Path): - """Gültig, wenn die Slot-Datei existiert und nummerierte Einträge enthält.""" - if not path.exists(): - return None - text = path.read_text(encoding="utf-8") - return text if _parse_auswahl(text) else None - - -def _mapping_schema(data): - """{"bausteine": [str, ≥1], "rest": [str]} → (bausteine, rest) · sonst None.""" - if not isinstance(data, dict): - return None - bausteine = _str_liste(data.get("bausteine")) - rest = _str_liste(data.get("rest")) - if not bausteine or rest is None: - return None - return bausteine, rest - - -def _sub_roh_schema(data): - """{Baustein-Titel: [Subbaustein, …]} → dict · sonst None (Zwischenstand Block B).""" - if not isinstance(data, dict) or not data: - return None - out: dict[str, list[str]] = {} - for k, v in data.items(): - subs = _str_liste(v) if isinstance(v, list) else None - if not isinstance(k, str) or not k.strip() or not subs: - return None - out[k] = subs - return out - - -def _sidecar_schema(data): - """{Baustein-Titel: [{titel, stufe}, …]} → dict · sonst None (Sidecar mit Stufen).""" - if not isinstance(data, dict) or not data: - return None - for v in data.values(): - if not isinstance(v, list) or not v: - return None - for s in v: - if not isinstance(s, dict) or not str(s.get("titel", "")).strip() or s.get("stufe") not in _STUFEN: - return None - return data - - -def _relevanz_komplett(data) -> bool: - """Jeder Subbaustein der Sidecar trägt eine gültige Relevanz (relevant/rand)?""" - if not isinstance(data, dict) or not data: - return False - return all( - isinstance(s, dict) and s.get("relevanz") in ("relevant", "rand") - for v in data.values() if isinstance(v, list) - for s in v - ) - - - - -def _frage_muster_chunk_schema(data) -> list[dict] | None: - """{"muster": [{baustein, subbaustein, frage}, …]} → Liste valider Einträge · sonst None. - - Ein Muster je Subbaustein (kein Typ-Kreuzprodukt — die Schwierigkeit kommt erst bei der - Prüfung aus dem Lerner-Niveau). Ungültige Einzel-Einträge werden übersprungen.""" - if not isinstance(data, dict) or not isinstance(data.get("muster"), list): - return None - out = [] - for e in data["muster"]: - if not isinstance(e, dict): - continue - bau = str(e.get("baustein", "")).strip() - sub = str(e.get("subbaustein", "")).strip() - frage = str(e.get("frage", "")).strip() - if not bau or not sub or not frage: - continue - out.append({"baustein": bau, "subbaustein": sub, "frage": frage}) - return out or None - - -def _frage_muster_komplett(topic: str) -> bool: - """Frage-Muster-Sidecar existiert (Build gelaufen)? Einzelne leere Bausteine - fallen zur Prüfungszeit auf Live-Generierung zurück — daher genügt die Datei.""" - return isinstance(_json_datei(frage_muster_path(topic)), dict) - - -def _read(p: Path) -> str: - return p.read_text(encoding="utf-8") if p.exists() else "" - - -def _chunk_nums(items: list, n: int) -> list[list]: - """Teilt eine flache Liste in n möglichst gleich große Chunks.""" - n = max(1, n) - size = max(1, math.ceil(len(items) / n)) - return [items[i:i + size] for i in range(0, len(items), size)] - - -def _n_chunks(count: int, size: int = SUBBAUSTEIN_CHUNK) -> int: - return min(SUBBAUSTEIN_MAX, max(1, math.ceil(count / size))) - - -def _lpt_chunks(gewichte: list[int], target: int) -> list[list[int]]: - """Indizes lastbalanciert auf Chunks verteilen (LPT, Makespan-minimal). Gewicht = Kosten je Index. - K = ceil(Gesamtgewicht/target); schwerste zuerst in den jeweils leichtesten Bin. → Index-Listen.""" - if not gewichte: - return [] - K = max(1, math.ceil(sum(gewichte) / max(1, target))) - bins: list[list[int]] = [[] for _ in range(K)] - last = [0] * K - for i in sorted(range(len(gewichte)), key=lambda x: gewichte[x], reverse=True): - j = min(range(K), key=lambda b: last[b]) - bins[j].append(i) - last[j] += gewichte[i] - return [b for b in bins if b] - - - - - - - - -async def _subbausteine_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict | None: - """Block B (DB + Loop): je Paket Subbausteine in Runden finden (3 Finder, bis 0 neue/Kappe), - in der DB sammeln (≥2 Nennungen = Konsens, 1× verworfen), Judge bereinigt je Paket. - → {Baustein-Titel: [Subbaustein, …]} (Konsens) oder None. Befüllt DB-Tabelle `subbausteine`.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - arbeit = files["arbeit"] - ordner = quelle_ordner(topic) - caps = "files" if ordner else "full" - # Quelle für die Beleg-Prüfung im Klär-Schritt (verwirft erfundene/unbelegbare Subs). - _typ = lade_quelle(topic).get("type", "thema") - source = _prompt(_QUELLE_TEMPLATE[_typ], project=ordner) if _typ in _QUELLE_TEMPLATE else _prompt("Bausteine-Quelle-Thema", topic=topic) - nums = list(entries) - chunks = _chunk_nums(nums, _n_chunks(len(nums))) - n = len(chunks) - titel_by_num = {num: _titel(entries[num]) for num in nums} - norm_by_num = {num: _norm_titel(titel_by_num[num]) for num in nums} - await db.delete_subbausteine(topic) # Frischstart des Blocks (idempotenter Zähler) - - async def _bekannt_block(chunk): - bl = [] - for num in chunk: - subs = [s["sub_titel"] for s in await db.list_subbausteine(topic, norm_by_num[num])] - if subs: - bl.append(f"\n" + "\n".join(f"- {s}" for s in subs)) - if not bl: - return "" - # Bekanntes NICHT erneut auflisten lassen (sonst bläht Re-Bestätigung den Mention-Count auf, - # Self-Bias/Echo) — nur Fehlendes ergänzen. Der Zähler bleibt so ein ehrliches Konsens-Signal. - return ("\n\nBEREITS ERFASST — liste diese NICHT erneut. Finde nur, was FEHLT:\n" + "\n".join(bl)) - - # Phase „Subbausteine finden": je Paket Loop bis 0 neue Subs / Zeit-Kappe. - async def _finde(c, chunk): - zuteilung = "\n".join(f"- {entries[num]}" for num in chunk) - chunk_idx = _titel_index({num: titel_by_num[num] for num in chunk}) - start = time.monotonic() - runde = 0 - while not is_cancelled(): - runde += 1 - bekannt = await _bekannt_block(chunk) if runde > 1 else "" - paths = [arbeit / f"subbaustein-c{c}-r{runde}-{i}.md" for i in (1, 2, 3)] - for p in paths: - p.unlink(missing_ok=True) - slots = [{ - "key": f"bausteine-{topic}-subbaustein-c{c}-r{runde}-{i}", - "prompt": _prompt("Subbaustein-Recherche", topic=topic, zuteilung=zuteilung, bekannt=bekannt, out_path=p, extra=_extra(instructions)), - "role": "quick", "capabilities": caps, - "payload": (lambda result, p=p: _parse_subbausteine(_read(p)) or None), - } for i, p in enumerate(paths, 1)] - texte = await _race(topic, f"Subbausteine Paket {c} R{runde}", slots, 2, _timeout("subbaustein", len(chunk)), provider, cancelled=is_cancelled, grace=KONSENS_GRACE) - if is_cancelled(): - return False - if not texte: - return runde > 1 # Runde 1 ohne Ergebnis = Fehler; spätere = einfach Ende - vorhanden = {num: {s["sub_norm"] for s in await db.list_subbausteine(topic, norm_by_num[num])} for num in chunk} - neu = 0 - for d in texte: - for marker, subs in d.items(): - num = _titel_aufloesen(chunk_idx, marker) - if num is None: - continue - gesehen = set() - for sub in subs: - sn = _norm_titel(sub) - if not sn or sn in gesehen: - continue - gesehen.add(sn) - if sn not in vorhanden[num]: - neu += 1 - vorhanden[num].add(sn) - await db.upsert_subbaustein(topic, norm_by_num[num], sn, titel_by_num[num], sub) - if neu == 0: - break - if time.monotonic() - start > SUBBAUSTEIN_KAPPE: - _log(topic, f"Subbausteine Paket {c}: Zeit-Kappe erreicht (Runde {runde})") - break - return True - - oks = await _gather_fortschritt([_finde(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _melde_p(set_p, topic, "Subbausteine finden")) - if is_cancelled(): - return None - if not all(ok is True for ok in oks): - _bausteine_errors[topic] = "Subbausteine fehlgeschlagen (Recherche)" - return None - - # Phase „Subbausteine wählen": ≥2 Nennungen = Konsens, 1× verworfen (Code). - set_p(f"Subbausteine wählen ({n} Pakete)…", step=_step_idx(topic, "Subbausteine wählen")) - for num in nums: - for s in await db.list_subbausteine(topic, norm_by_num[num]): - await db.set_subbaustein_felder(topic, norm_by_num[num], s["sub_norm"], - status=("konsens" if s["nennungen"] >= 2 else "verworfen")) - - # Phase „Subbausteine klären": Source-Panel (SUBBAUSTEIN_PANEL Judges) prüft Konsens + Unsicher (1×) - # gegen die Quelle; Code-Mehrheit je Sub. Externes, mehrstimmiges Gate gegen Einzel-Judge-Bias + Echo. - async def _klaere(c, chunk): - fp = arbeit / f"subbaustein-final-c{c}.md" - if _parse_subbausteine(_read(fp)): - return - bloecke, hat = [], False - konsens_by_num: dict[int, list[str]] = {} - for num in chunk: - rows = await db.list_subbausteine(topic, norm_by_num[num]) - kon = [s["sub_titel"] for s in rows if s["status"] == "konsens"] - uns = [s["sub_titel"] for s in rows if s["status"] != "konsens" and s["nennungen"] == 1] - konsens_by_num[num] = kon - if not kon and not uns: - continue - hat = True - k_zeilen = "\n".join(f"- {s}" for s in kon) if kon else "- (keiner)" - u_zeilen = "\n".join(f"- {s}" for s in uns) if uns else "- (keiner)" - bloecke.append(f"BAUSTEIN: {titel_by_num[num]}\nKonsens (≥2 Finder):\n{k_zeilen}\nUnsicher (1× — streng gegen Quelle prüfen):\n{u_zeilen}") - if not hat: - return - - chunk_idx = _titel_index({num: titel_by_num[num] for num in chunk}) - paths = [arbeit / f"subbaustein-final-c{c}-j{j}.md" for j in range(1, SUBBAUSTEIN_PANEL + 1)] - offen = [(j, p) for j, p in enumerate(paths, 1) if _parse_subbausteine(_read(p)) is None] - for _, p in offen: - p.unlink(missing_ok=True) - if offen: - slots = [{ - "key": f"bausteine-{topic}-subbaustein-final-c{c}-j{j}", - "prompt": _prompt("Subbaustein-Mapping", topic=topic, source=source, bausteine="\n\n".join(bloecke), out_path=p, extra=_extra(instructions)), - "role": "judge", "capabilities": caps, - "payload": (lambda result, p=p: _parse_subbausteine(_read(p)) or None), - } for j, p in offen] - vorhanden = SUBBAUSTEIN_PANEL - len(offen) - await _race(topic, f"Subbaustein-Klärung {c}", slots, max(1, 2 - vorhanden), - _timeout("subbaustein_check", len(chunk)), provider, cancelled=is_cancelled, grace=KONSENS_GRACE) - if is_cancelled(): - return - outs = [d for p in paths if (d := _parse_subbausteine(_read(p)))] - if not outs: # Panel komplett gescheitert → Konsens übernehmen (wie bisher der Fallback) - _log(topic, f"Subbaustein-Klärung Paket {c} fehlgeschlagen — Konsens übernommen") - text = "\n\n".join(f"\n" + "\n".join(f"- {s}" for s in konsens_by_num[num]) - for num in chunk if konsens_by_num[num]) - atomic_write_text(fp, text) - return - - # Code-Mehrheit je Baustein/Sub-Norm: behalten wenn Mehrheit der Judges ihn führt (Tie → behalten). - bloecke_out = [] - for num in chunk: - votes: dict[str, int] = {} - form: dict[str, str] = {} - for d in outs: - seen = set() - for marker, subs in d.items(): - if _titel_aufloesen(chunk_idx, marker) != num: - continue - for sub in subs: - sn = _norm_titel(sub) - if not sn or sn in seen: - continue - seen.add(sn) - form.setdefault(sn, sub) - votes[sn] = votes.get(sn, 0) + 1 - kept = [form[sn] for sn in form if votes[sn] * 2 >= len(outs)] - if kept: - bloecke_out.append(f"\n" + "\n".join(f"- {s}" for s in kept)) - atomic_write_text(fp, "\n\n".join(bloecke_out)) - - await _gather_fortschritt([_klaere(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _melde_p(set_p, topic, "Subbausteine klären")) - if is_cancelled(): - return None - - # Finale Liste je Baustein: Judge-Ausgabe, sonst Konsens-Fallback. DB reconcilen + roh bauen. - roh: dict[str, list[str]] = {} - for c, chunk in enumerate(chunks, 1): - final = _parse_subbausteine(_read(arbeit / f"subbaustein-final-c{c}.md")) or {} - chunk_idx = _titel_index({num: titel_by_num[num] for num in chunk}) - final_by_num = {_titel_aufloesen(chunk_idx, m): subs for m, subs in final.items() if _titel_aufloesen(chunk_idx, m) is not None} - for num in chunk: - titel = titel_by_num[num] - konsens = [s["sub_titel"] for s in await db.list_subbausteine(topic, norm_by_num[num]) if s["status"] == "konsens"] - subs = final_by_num.get(num) or konsens - if not subs: - continue - roh[titel] = subs - # DB an die finale Liste angleichen: finale = konsens, Rest verworfen, Neues ergänzen. - final_norms = {_norm_titel(s) for s in subs} - have = {s["sub_norm"] for s in await db.list_subbausteine(topic, norm_by_num[num])} - for s in await db.list_subbausteine(topic, norm_by_num[num]): - await db.set_subbaustein_felder(topic, norm_by_num[num], s["sub_norm"], - status=("konsens" if s["sub_norm"] in final_norms else "verworfen")) - for s in subs: - sn = _norm_titel(s) - if sn and sn not in have: - await db.upsert_subbaustein(topic, norm_by_num[num], sn, titel, s) - await db.set_subbaustein_felder(topic, norm_by_num[num], sn, status="konsens") - await _dedup_subbausteine(topic, roh) # Near-Dup-Filter je Baustein (deterministisch, kein LLM) - if not roh: - _bausteine_errors[topic] = "Keine Subbausteine ermittelt" - return None - return roh - - -async def _dedup_subbausteine(topic: str, roh: dict[str, list[str]]) -> None: - """Deterministischer Near-Duplicate-Filter pro Baustein: Subbausteine mit Cosine ≥ - EMBEDDING_SUB_DUP sind dieselbe Aussage (im engen Baustein-Kontext zuverlässig — kein LLM - nötig). Behält je Dublett-Gruppe den informativsten (längsten); Rest → DB verworfen + aus `roh`. - Modell fehlt → still überspringen (wie der übrige Embedding-Fallback).""" - if not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.verfuegbar): - return - for titel, subs in list(roh.items()): - if len(subs) < 2: - continue - sims = await asyncio.to_thread(embedding.embed_sims, subs) - if sims is None: - return - behalten: list[int] = [] - verworfen: list[int] = [] - for i in sorted(range(len(subs)), key=lambda x: (-len(subs[x]), x)): # informativster zuerst - if any(float(sims[i][j]) >= EMBEDDING_SUB_DUP for j in behalten): - verworfen.append(i) - else: - behalten.append(i) - if not verworfen: - continue - bnorm = _norm_titel(titel) - for i in verworfen: - await db.set_subbaustein_felder(topic, bnorm, _norm_titel(subs[i]), status="verworfen") - roh[titel] = [subs[i] for i in sorted(behalten)] # Original-Reihenfolge der Behaltenen - - -async def _stufen_block(ctx: GenContext, set_p, files: dict, roh: dict, instructions: str) -> dict | None: - """Block C: drei Phasen mit Barriere — Finden (einstufen), Wählen (Vote), Klären. - Lokale IDs 1..n pro Paket, hinterher auf globale gid gemappt. - → {Baustein-Titel: [{titel, stufe}, …]} oder None.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - arbeit = files["arbeit"] - # Kernpunkte je Sub als knapper Kontext (fundiertere Einstufung; Klassifikation braucht wenig). - fakten_map = _json_datei(files["fakten"]) - fakten_map = fakten_map if isinstance(fakten_map, dict) else {} - items = [(titel, sub) for titel, subs in roh.items() for sub in subs] # globale id = index+1 - if not items: - return {titel: [] for titel in roh} - # Chunks aus GANZEN Bausteinen packen (keinen Baustein splitten) → Rater sieht je Baustein - # alle Subs und kann relativ einstufen. Item-Indizes je Baustein in roh-Reihenfolge. - chunks, cur, i = [], [], 0 - for _titel_b, subs in roh.items(): - g = list(range(i, i + len(subs))) - i += len(subs) - if cur and len(cur) + len(g) > STUFE_CHUNK: - chunks.append(cur) - cur = [] - cur.extend(g) - if cur: - chunks.append(cur) - n = len(chunks) - - def rater_paths(c): - return [arbeit / f"stufe-c{c}-{i}.json" for i in (1, 2, 3)] - - def lset(item_idxs): - return set(range(1, len(item_idxs) + 1)) - - # Phase „Stufen finden": pro Paket 3 Rater (min. 2), lokale IDs. - async def _rate(c, item_idxs): - local_set = lset(item_idxs) - paths = rater_paths(c) - vorhanden = sum(1 for p in paths if _stufen_schema(_json_datei(p), local_set)) - if vorhanden >= 2: - return True - enum_zeilen, cur_b = [], None - for k, j in enumerate(item_idxs, 1): - b, sub = items[j] - if b != cur_b: - enum_zeilen.append(f"\nBAUSTEIN: {b}") - cur_b = b - enum_zeilen.append(f"{k}. {sub}") - if (kz := _kern_zeile(fakten_map.get(b, {}).get(_norm_titel(sub)))): - enum_zeilen.append(f" {kz}") - enum = "\n".join(enum_zeilen).strip() - offen = [(i, p) for i, p in enumerate(paths, 1) if not _stufen_schema(_json_datei(p), local_set)] - slots = [{ - "key": f"bausteine-{topic}-stufe-c{c}-{i}", - "prompt": _prompt("Stufen-Recherche", topic=topic, subbausteine=enum, out_path=p, extra=_extra(instructions)), - "role": "fast", "capabilities": "files", - "payload": (lambda result, p=p, ids=local_set: _stufen_schema(_json_datei(p), ids)), - } for i, p in offen] - neu = await _race(topic, f"Stufen Paket {c}", slots, 2 - vorhanden, _timeout("stufe", len(item_idxs)), provider, cancelled=is_cancelled, grace=KONSENS_GRACE) - return not is_cancelled() and neu is not None - - oks = await _gather_fortschritt([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _melde_p(set_p, topic, "Stufen finden")) - if is_cancelled(): - return None - if not all(ok is True for ok in oks): - _bausteine_errors[topic] = "Einstufung fehlgeschlagen (Recherche)" - return None - - # Phase „Stufen wählen": Code-Vote je Paket → (ergebnis, strittig). - set_p(f"Stufen wählen ({n} Pakete)…", step=_step_idx(topic, "Stufen wählen")) - vote_by_c = {} - for c, item_idxs in enumerate(chunks, 1): - local_set = lset(item_idxs) - rater = [d for p in rater_paths(c) if (d := _stufen_schema(_json_datei(p), local_set))] - ergebnis: dict[int, str] = {} - strittig: dict[int, list[str]] = {} - for k in range(1, len(item_idxs) + 1): - stimmen = [d[k] for d in rater if k in d] - zaehler: dict[str, int] = {} - for s in stimmen: - zaehler[s] = zaehler.get(s, 0) + 1 - best = max(zaehler.values(), default=0) - gewinner = [s for s, v in zaehler.items() if v == best] - if len(gewinner) == 1 and best >= 2: - ergebnis[k] = gewinner[0] - else: - strittig[k] = stimmen - vote_by_c[c] = (ergebnis, strittig) - - # Phase „Stufen klären": Judge je Paket mit Strittigem, alle parallel. - async def _klaere(c, item_idxs): - ergebnis, strittig = vote_by_c[c] - if strittig: - judge_path = arbeit / f"stufe-final-c{c}.json" - entsch = _stufen_schema(_json_datei(judge_path), set(strittig)) - if entsch is None: - strittig_block = "\n".join( - f"{k}. [{items[item_idxs[k - 1]][0]}] {items[item_idxs[k - 1]][1]} — Stimmen: {', '.join(stimmen) or 'keine'}" - for k, stimmen in strittig.items() - ) - status, entsch = await run_single_slot( - ctx, f"Stufen-Klärung {c}", - key=f"bausteine-{topic}-stufe-final-c{c}", - prompt=_prompt("Stufen-Mapping", topic=topic, strittig=strittig_block, out_path=judge_path, extra=_extra(instructions)), - role="judge", capabilities="files", - payload=lambda result, p=judge_path, ids=set(strittig): _stufen_schema(_json_datei(p), ids), - timeout=_timeout("stufe_check", len(strittig)), - ) - if status == FAILED: - _log(topic, f"Stufen-Klärung Paket {c} fehlgeschlagen — Default 'fortgeschritten'") - entsch = entsch if isinstance(entsch, dict) else {} - # Strittige ohne Entscheid → 'fortgeschritten'; Vote-Gewinner bleiben; Judge überschreibt. - ergebnis = {**{k: "fortgeschritten" for k in strittig}, **ergebnis, **entsch} - return {item_idxs[k - 1] + 1: stufe for k, stufe in ergebnis.items()} - - parts = await _gather_fortschritt([_klaere(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _melde_p(set_p, topic, "Stufen klären")) - if is_cancelled(): - return None - stufe_by_id: dict[int, str] = {} - for c, part in enumerate(parts, 1): - if not isinstance(part, dict): - # Klärung ist nicht fatal: Vote-Ergebnis + Default 'fortgeschritten' für Strittige. - if isinstance(part, BaseException): - _log(topic, f"Stufen-Klärung Paket {c}: {type(part).__name__}: {part}") - ergebnis, strittig = vote_by_c[c] - item_idxs = chunks[c - 1] - merged = {**{k: "fortgeschritten" for k in strittig}, **ergebnis} - part = {item_idxs[k - 1] + 1: s for k, s in merged.items()} - stufe_by_id.update(part) - - # Sidecar zusammensetzen — gleiche Reihenfolge wie items → gid stimmt - sidecar: dict[str, list[dict]] = {} - gid = 0 - for titel, subs in roh.items(): - lst = [] - for sub in subs: - gid += 1 - lst.append({"titel": sub, "stufe": stufe_by_id.get(gid, "fortgeschritten")}) - sidecar[titel] = lst - return sidecar - - -_FAKTEN_FELDER = ("kernpunkte", "voraussetzungen", "huerden", "belegte_fakten", "beispiel_idee") - - -def _fakten_schema(data) -> list[dict] | None: - """{"fakten": [{baustein, subbaustein, …}]} → valide Liste · sonst None. - Trennt belegte_fakten (mit Quelle) hart von beispiel_idee (generativ).""" - if not isinstance(data, dict) or not isinstance(data.get("fakten"), list): - return None - out = [] - for e in data["fakten"]: - if not isinstance(e, dict): - continue - bau = str(e.get("baustein", "")).strip() - sub = str(e.get("subbaustein", "")).strip() - if not bau or not sub: - continue - bf = [{"text": t, "quelle": str(f.get("quelle", "")).strip()} - for f in (e.get("belegte_fakten") or []) if isinstance(f, dict) and (t := str(f.get("text", "")).strip())] - out.append({ - "baustein": bau, "subbaustein": sub, - "kernpunkte": [k for x in (e.get("kernpunkte") or []) if (k := str(x).strip())], - "voraussetzungen": str(e.get("voraussetzungen", "")).strip(), - "huerden": str(e.get("huerden", "")).strip(), - "belegte_fakten": bf, - "beispiel_idee": str(e.get("beispiel_idee", "")).strip(), - }) - return out or None - - -def _fakten_check_schema(data) -> list[tuple[str, bool]] | None: - """Fakten-Check → [(sub_norm, verwerfen)] je Beanstandung · {ok:true}→[] · None bei ungültig. - verwerfen=True: Sub inhaltlich nicht belegbar (entfernen). verwerfen=False: nur Fakt korrigieren.""" - if not isinstance(data, dict): - return None - if data.get("ok") is True: - return [] - pr = data.get("probleme") - if not isinstance(pr, list): - return None - return [(sn, bool(p.get("verwerfen"))) - for p in pr if isinstance(p, dict) and (sn := _norm_titel(str(p.get("subbaustein", ""))))] - - -def _kern_zeile(fk) -> str: - """Knappe Kernpunkt-Zeile für Klassifikation (Stufe/Relevanz) — weniger Kontext genügt dort. - Leer, wenn keine Fakten/Kernpunkte (Altbestand).""" - if not isinstance(fk, dict) or not fk.get("kernpunkte"): - return "" - return "Kern: " + " · ".join(str(k) for k in fk["kernpunkte"]) - - -def _fakten_zeilen(fk: dict) -> str: - z = [] - if fk.get("kernpunkte"): - z.append("Kernpunkte: " + " · ".join(str(k) for k in fk["kernpunkte"])) - if fk.get("voraussetzungen"): - z.append("Voraussetzung: " + fk["voraussetzungen"]) - if fk.get("huerden"): - z.append("Hürde: " + fk["huerden"]) - for bf in fk.get("belegte_fakten", []): - z.append(f"FAKT: {bf['text']} (Quelle: {bf.get('quelle', '?')})") - if fk.get("beispiel_idee"): - z.append("Beispiel: " + fk["beispiel_idee"]) - return "\n".join(z) - - -def _fakten_komplett(files: dict) -> bool: - """Fakten-Map existiert (Block durch)? {Baustein: {sub_norm: {...}}}.""" - d = _json_datei(files["fakten"]) - return isinstance(d, dict) and bool(d) - - -async def _fakten_block(ctx, set_p, files: dict, roh: dict, q: dict, ordner, instructions: str) -> tuple | None: - """Block: je Sub Quell-Fakten extrahieren (finden) → verifizieren (prüfen) → korrigieren/verwerfen (fix). - Extract-once-Grounding: das Ergebnis nährt Stufe/Relevanz/Fragen/Guide. - → (fakten_map, verworfen_map) — fakten_map {Baustein: {sub_norm: fakten}}, verworfen_map - {Baustein: {sub_norm}} (unbelegbare Subs zum Entfernen) — oder None bei Abbruch/Fehler.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - arbeit = files["arbeit"] - caps = "files" if ordner else "full" - typ = q.get("type", "thema") - source = _prompt(_QUELLE_TEMPLATE[typ], project=ordner) if typ in _QUELLE_TEMPLATE else _prompt("Bausteine-Quelle-Thema", topic=topic) - bausteine = [(titel, [str(s).strip() for s in subs if str(s).strip()]) for titel, subs in roh.items() if subs] - if not bausteine: - return {}, {} - chunks = _lpt_chunks([len(subs) for _, subs in bausteine], FAKTEN_CHUNK_SUBS) - - def roh_path(ci): return arbeit / f"fakten-c{ci}.json" - def erg_path(ci): return arbeit / f"fakten-erg-c{ci}.json" - def chk_path(ci, j): return arbeit / f"fakten-check-c{ci}-j{j}.json" - def fix_path(ci): return arbeit / f"fakten-fix-c{ci}.json" - def ctitel(idxs): return [bausteine[i][0] for i in idxs] - def block_text(idxs): - return "\n\n".join( - f"BAUSTEIN: {bausteine[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in bausteine[i][1]) - for i in idxs) - - # Roh-Fakten eines Chunks → {Baustein: {sub_norm: {sub, …felder}}}, auf Chunk-Titel gematcht. - def roh_map(ci, path): - idxs = chunks[ci] - rel_by = {bausteine[i][0]: bausteine[i][1] for i in idxs} - ct = ctitel(idxs) - out: dict[str, dict] = {} - for e in _fakten_schema(_json_datei(path)) or []: - bt = _match_sub(e["baustein"], ct) - if bt not in rel_by: - continue - sub = _match_sub(e["subbaustein"], rel_by[bt]) - out.setdefault(bt, {})[_norm_titel(sub)] = {"sub": sub, **{k: e[k] for k in _FAKTEN_FELDER}} - return out - - # Roh-Fakten + Completeness-Ergänzungen vereinigen (Recall): nur Subs, die in roh existieren. - def _chunk_fakten(ci): - roh = roh_map(ci, roh_path(ci)) - erg = roh_map(ci, erg_path(ci)) if erg_path(ci).exists() else {} - if not erg: - return roh - for bt, fm in roh.items(): - ebt = erg.get(bt, {}) - for sn, fk in fm.items(): - ek = ebt.get(sn) - if not ek: - continue - seen = {str(k).strip().casefold() for k in fk.get("kernpunkte", [])} - for k in ek.get("kernpunkte", []): - if str(k).strip().casefold() not in seen: - seen.add(str(k).strip().casefold()) - fk["kernpunkte"].append(k) - seent = {bf["text"].strip().casefold() for bf in fk.get("belegte_fakten", [])} - for bf in ek.get("belegte_fakten", []): - if bf["text"].strip().casefold() not in seent: - seent.add(bf["text"].strip().casefold()) - fk["belegte_fakten"].append(bf) - for f in ("voraussetzungen", "huerden", "beispiel_idee"): - if not fk.get(f) and ek.get(f): - fk[f] = ek[f] - return roh - - # Phase „Fakten finden": 1 Generator je Chunk. - async def _finde(ci, idxs): - fp = roh_path(ci) - if _fakten_schema(_json_datei(fp)): - return True - subs_total = sum(len(bausteine[i][1]) for i in idxs) - status, _r = await run_single_slot( - ctx, f"Fakten {ci}", key=f"bausteine-{topic}-fakten-c{ci}", - prompt=_prompt("Fakten-Recherche", topic=topic, source=source, bausteine=block_text(idxs), out_path=fp, extra=_extra(instructions)), - role="guide", capabilities=caps, - payload=lambda result, p=fp: _fakten_schema(_json_datei(p)), - timeout=_timeout("inhalt", subs_total)) - return status != FAILED and _fakten_schema(_json_datei(fp)) is not None - - oks = await _gather_fortschritt([_finde(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _melde_p(set_p, topic, "Fakten finden")) - if is_cancelled(): - return None - if not any(ok is True for ok in oks): - _bausteine_errors[topic] = "Fakten-Extraktion fehlgeschlagen" - return None - - # Phase „Fakten ergänzen" (Recall): ein gezielter Gap-Hunt je Chunk sucht source-belegte Fakten, die - # der Single-Pass übersah. Best-effort — schlägt nie fehl (keine erg-Datei → Merge nutzt nur roh). - async def _ergaenze(ci, idxs): - ep = erg_path(ci) - if _fakten_schema(_json_datei(ep)): - return - per = roh_map(ci, roh_path(ci)) - if not per: - return - block = "\n\n".join( - f"BAUSTEIN: {bt}\nSUBBAUSTEINE (mit bereits erfassten Fakten):\n" + "\n".join( - f"- {fk['sub']}\n Erfasst: " + ("; ".join( - list(fk.get("kernpunkte", [])) + [bf["text"] for bf in fk.get("belegte_fakten", [])]) or "(nichts)") - for fk in fm.values()) - for bt, fm in per.items()) - subs_total = sum(len(bausteine[i][1]) for i in idxs) - await run_single_slot( - ctx, f"Fakten ergänzen {ci}", key=f"bausteine-{topic}-fakten-erg-c{ci}", - prompt=_prompt("Fakten-Ergaenzung", topic=topic, source=source, bausteine=block, out_path=ep, extra=_extra(instructions)), - role="guide", capabilities=caps, - payload=lambda result, p=ep: _fakten_schema(_json_datei(p)), - timeout=_timeout("inhalt", subs_total)) - - set_p("Fakten ergänzen…", step=_step_idx(topic, "Fakten finden")) - await _gather_fortschritt([_ergaenze(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _melde_p(set_p, topic, "Fakten finden")) - if is_cancelled(): - return None - - # Phase „Fakten prüfen": FAKTEN_CHECK_PANEL Judges je Chunk. Zwei Mehrheits-Mengen: - # beanstandet (Fakt ungenau → korrigieren) und verwerfen (Sub nicht belegbar → entfernen). - async def _pruefe(ci, idxs): - per = _chunk_fakten(ci) # roh + Ergänzungen → Panel verifiziert die Vereinigung - if not per: - return ci, set(), set() - fakten_text = "\n\n".join(f"SUBBAUSTEIN: {fk['sub']}\n{_fakten_zeilen(fk)}" for fm in per.values() for fk in fm.values()) - offen = [j for j in (1, 2, 3)[:FAKTEN_CHECK_PANEL] if _fakten_check_schema(_json_datei(chk_path(ci, j))) is None] - await asyncio.gather(*[ - run_agent(f"bausteine-{topic}-fakten-check-c{ci}-j{j}", - _prompt("Fakten-Check", topic=topic, source=source, fakten=fakten_text, out_path=chk_path(ci, j), extra=_extra(instructions)), - _timeout("inhalt_check", len(per)), provider=provider, role="judge", capabilities=caps) - for j in offen], return_exceptions=True) - outs = [s for j in (1, 2, 3)[:FAKTEN_CHECK_PANEL] if (s := _fakten_check_schema(_json_datei(chk_path(ci, j)))) is not None] - bvotes: dict[str, int] = {} - vvotes: dict[str, int] = {} - for s in outs: # s = [(sub_norm, verwerfen)] eines Judges - gb, gv = set(), set() - for sn, verw in s: - if sn not in gb: - gb.add(sn); bvotes[sn] = bvotes.get(sn, 0) + 1 - if verw and sn not in gv: - gv.add(sn); vvotes[sn] = vvotes.get(sn, 0) + 1 - schwelle = len(outs) / 2 if outs else 99 - beanstandet = {sn for sn, v in bvotes.items() if v > schwelle} - # Verwerfen ist irreversibel → strenger als Beanstanden: Mehrheit UND ≥2 zustimmende Judges - # (verhindert Löschung durch eine Einzelstimme, wenn das Panel degradiert ist). - verwerfen = {sn for sn, v in vvotes.items() if v > schwelle and v >= 2} - return ci, beanstandet, verwerfen - - pruef = await _gather_fortschritt([_pruefe(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _melde_p(set_p, topic, "Fakten prüfen")) - if is_cancelled(): - return None - beanstandet: dict[int, set] = {} - verwerfen: dict[int, set] = {} - for r in pruef: - if isinstance(r, tuple) and len(r) == 3: - ci, b, v = r - beanstandet[ci] = b - verwerfen[ci] = v - - # Phase „Fakten fix": nur KORRIGIERBARE (beanstandet ohne verwerfen) neu extrahieren. - korrigieren = {ci: (beanstandet.get(ci, set()) - verwerfen.get(ci, set())) for ci in beanstandet} - n_problem = sum(len(s) for s in korrigieren.values()) - if n_problem: - set_p(f"Fakten korrigieren ({n_problem})…", step=_step_idx(topic, "Fakten fix")) - async def _fix(ci): - subs_norm = korrigieren.get(ci, set()) - if not subs_norm or _fakten_schema(_json_datei(fix_path(ci))): - return - idxs = chunks[ci] - rel_by = {bausteine[i][0]: bausteine[i][1] for i in idxs} - ziel = [] - for bt, subs in rel_by.items(): - betroffen = [s for s in subs if _norm_titel(s) in subs_norm] - if betroffen: - ziel.append(f"BAUSTEIN: {bt}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in betroffen)) - if not ziel: - return - await run_single_slot( - ctx, f"Fakten-Fix {ci}", key=f"bausteine-{topic}-fakten-fix-c{ci}", - prompt=_prompt("Fakten-Recherche", topic=topic, source=source, bausteine="\n\n".join(ziel), out_path=fix_path(ci), extra=_extra(instructions)), - role="guide", capabilities=caps, - payload=lambda result, p=fix_path(ci): _fakten_schema(_json_datei(p)), - timeout=_timeout("inhalt", len(subs_norm))) - await _gather_fortschritt([_fix(ci) for ci in korrigieren], len(korrigieren), _melde_p(set_p, topic, "Fakten fix")) - if is_cancelled(): - return None - - # Zusammensetzen: roh + Fix-Overrides für korrigierte. Verworfene Subs raus (+ je Baustein melden). - ergebnis: dict[str, dict] = {} - verworfen_map: dict[str, set] = {} - for ci in range(len(chunks)): - per = _chunk_fakten(ci) # roh + Ergänzungen (Recall); Fix überschreibt nur Korrigierte - fix = roh_map(ci, fix_path(ci)) if fix_path(ci).exists() else {} - verw = verwerfen.get(ci, set()) - for bt, fm in per.items(): - for sn, fk in fm.items(): - if sn in verw: - verworfen_map.setdefault(bt, set()).add(sn) - continue - gewinner = fix.get(bt, {}).get(sn, fk) if sn in korrigieren.get(ci, set()) else fk - ergebnis.setdefault(bt, {})[sn] = {k: gewinner[k] for k in _FAKTEN_FELDER} - if verworfen_map: - _log(topic, f"Fakten-Check verwirft {sum(len(s) for s in verworfen_map.values())} unbelegbare Subbausteine") - return ergebnis, verworfen_map - - -async def _relevanz_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: - """Block D: drei Phasen mit Barriere — Finden (relevant/rand), Wählen (Vote), Klären. - Items aus der Sidecar; lokale IDs 1..n pro Paket → globale gid. - → {gid: relevanz} oder None bei Abbruch/Recherche-Fehler. Default bei Lücke/Streit: 'relevant'.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - arbeit = files["arbeit"] - items = [(titel, sub["titel"], sub.get("fakten")) for titel, subs in sidecar.items() for sub in subs] # globale id = index+1 - if not items: - return {} - chunks = _chunk_nums(list(range(len(items))), _n_chunks(len(items), STUFE_CHUNK)) - n = len(chunks) - - def rater_paths(c): - return [arbeit / f"relevanz-c{c}-{i}.json" for i in (1, 2, 3)] - - def lset(item_idxs): - return set(range(1, len(item_idxs) + 1)) - - # Phase „Relevanz finden": pro Paket 3 Rater (min. 2), lokale IDs. - async def _rate(c, item_idxs): - local_set = lset(item_idxs) - paths = rater_paths(c) - vorhanden = sum(1 for p in paths if _relevanz_schema(_json_datei(p), local_set)) - if vorhanden >= 2: - return True - enum_zeilen = [] - for k, j in enumerate(item_idxs, 1): - enum_zeilen.append(f"{k}. [{items[j][0]}] {items[j][1]}") - if (kz := _kern_zeile(items[j][2])): - enum_zeilen.append(f" {kz}") - enum = "\n".join(enum_zeilen) - offen = [(i, p) for i, p in enumerate(paths, 1) if not _relevanz_schema(_json_datei(p), local_set)] - slots = [{ - "key": f"bausteine-{topic}-relevanz-c{c}-{i}", - "prompt": _prompt("Relevanz-Recherche", topic=topic, subbausteine=enum, out_path=p, extra=_extra(instructions)), - "role": "fast", "capabilities": "files", - "payload": (lambda result, p=p, ids=local_set: _relevanz_schema(_json_datei(p), ids)), - } for i, p in offen] - neu = await _race(topic, f"Relevanz Paket {c}", slots, 2 - vorhanden, _timeout("relevanz", len(item_idxs)), provider, cancelled=is_cancelled, grace=KONSENS_GRACE) - return not is_cancelled() and neu is not None - - oks = await _gather_fortschritt([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _melde_p(set_p, topic, "Relevanz finden")) - if is_cancelled(): - return None - if not all(ok is True for ok in oks): - _bausteine_errors[topic] = "Relevanz fehlgeschlagen (Recherche)" - return None - - # Phase „Relevanz wählen": Code-Vote je Paket → (ergebnis, strittig). - set_p(f"Relevanz wählen ({n} Pakete)…", step=_step_idx(topic, "Relevanz wählen")) - vote_by_c = {} - for c, item_idxs in enumerate(chunks, 1): - local_set = lset(item_idxs) - rater = [d for p in rater_paths(c) if (d := _relevanz_schema(_json_datei(p), local_set))] - ergebnis: dict[int, str] = {} - strittig: dict[int, list[str]] = {} - for k in range(1, len(item_idxs) + 1): - stimmen = [d[k] for d in rater if k in d] - zaehler: dict[str, int] = {} - for s in stimmen: - zaehler[s] = zaehler.get(s, 0) + 1 - best = max(zaehler.values(), default=0) - gewinner = [s for s, v in zaehler.items() if v == best] - if len(gewinner) == 1 and best >= 2: - ergebnis[k] = gewinner[0] - else: - strittig[k] = stimmen - vote_by_c[c] = (ergebnis, strittig) - - # Phase „Relevanz klären": Judge je Paket mit Strittigem, alle parallel. - async def _klaere(c, item_idxs): - ergebnis, strittig = vote_by_c[c] - if strittig: - judge_path = arbeit / f"relevanz-final-c{c}.json" - entsch = _relevanz_schema(_json_datei(judge_path), set(strittig)) - if entsch is None: - strittig_block = "\n".join( - f"{k}. [{items[item_idxs[k - 1]][0]}] {items[item_idxs[k - 1]][1]} — Stimmen: {', '.join(stimmen) or 'keine'}" - for k, stimmen in strittig.items() - ) - status, entsch = await run_single_slot( - ctx, f"Relevanz-Klärung {c}", - key=f"bausteine-{topic}-relevanz-final-c{c}", - prompt=_prompt("Relevanz-Mapping", topic=topic, strittig=strittig_block, out_path=judge_path, extra=_extra(instructions)), - role="judge", capabilities="files", - payload=lambda result, p=judge_path, ids=set(strittig): _relevanz_schema(_json_datei(p), ids), - timeout=_timeout("relevanz_check", len(strittig)), - ) - if status == FAILED: - _log(topic, f"Relevanz-Klärung Paket {c} fehlgeschlagen — Default 'relevant'") - entsch = entsch if isinstance(entsch, dict) else {} - # Strittige ohne Entscheid → 'relevant' (nie versehentlich ausschließen). - ergebnis = {**{k: "relevant" for k in strittig}, **ergebnis, **entsch} - return {item_idxs[k - 1] + 1: rel for k, rel in ergebnis.items()} - - parts = await _gather_fortschritt([_klaere(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _melde_p(set_p, topic, "Relevanz klären")) - if is_cancelled(): - return None - relevanz_by_id: dict[int, str] = {} - for c, part in enumerate(parts, 1): - if not isinstance(part, dict): - # Klärung ist nicht fatal: Vote-Ergebnis + Default 'relevant' für Strittige. - if isinstance(part, BaseException): - _log(topic, f"Relevanz-Klärung Paket {c}: {type(part).__name__}: {part}") - ergebnis, strittig = vote_by_c[c] - item_idxs = chunks[c - 1] - merged = {**{k: "relevant" for k in strittig}, **ergebnis} - part = {item_idxs[k - 1] + 1: s for k, s in merged.items()} - relevanz_by_id.update(part) - return relevanz_by_id - - -def _match_sub(agent_sub: str, rel: list[str]) -> str: - """Subbaustein-Titel des Agenten auf den passenden relevanten Titel mappen — exakt, - dann normalisiert, dann Teilstring (der Agent lässt z. B. das Präfix „Frage: " weg). - Kein Treffer → Agent-Titel behalten. So geht KEIN Muster durch Titel-Abweichung verloren.""" - if agent_sub in rel: - return agent_sub - an = _norm_titel(agent_sub) - for r in rel: - rn = _norm_titel(r) - if an and rn and (an == rn or an in rn or rn in an): - return r - return agent_sub - - -async def _frage_muster_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: - """Block E (10er-Chunks): Finden (1 Generator je ~10 Bausteine, parallel), Wählen (Code: - je Baustein gruppieren + Dedup), Klären (1 Kritiker je Chunk), Prüfen (Nachrunde). - Zuordnung je Eintrag über das `baustein`-Feld (Chunk-Datei trägt mehrere Bausteine). - → {Baustein-Titel: [{subbaustein, frage}, …]} oder None bei Abbruch.""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - arbeit = files["arbeit"] - # ALLE Subbausteine (auch rand) bekommen ein Muster — Rand ist in der FGuide-Ebene prüfbar. - # fakten_by: voller Fakten-Kontext je Sub (Generierung profitiert davon — bessere Fragen). - bausteine = [] - fakten_by: dict[tuple, dict] = {} - for titel, subs in sidecar.items(): - alle = [] - for s in subs: - if isinstance(s, dict) and (st := str(s.get("titel", "")).strip()): - alle.append(st) - if isinstance(s.get("fakten"), dict): - fakten_by[(titel, _norm_titel(st))] = s["fakten"] - if alle: - bausteine.append((titel, alle)) - if not bausteine: - return {} - chunks = _lpt_chunks([len(rel) for _, rel in bausteine], FRAGE_CHUNK_SUBS) # lastbalanciert nach Sub-Zahl - - def roh_path(ci): - return arbeit / f"frage-muster-c{ci}.json" - - def final_path(ci): - return arbeit / f"frage-muster-final-c{ci}.json" - - def _chunk_titel(idxs): - return [bausteine[i][0] for i in idxs] - - # Phase „Fragen finden": je Chunk 1 Generator, alle parallel. - async def _finde(ci, idxs): - fp = roh_path(ci) - if _frage_muster_chunk_schema(_json_datei(fp)): - return # Resume - def _sub_zeile(bi, s): - zeile = f"- {s}" - fk = fakten_by.get((bausteine[bi][0], _norm_titel(s))) - if fk and (ft := _fakten_zeilen(fk)): - zeile += "\n" + "\n".join(" " + l for l in ft.split("\n")) - return zeile - block = "\n\n".join( - f"BAUSTEIN: {bausteine[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(_sub_zeile(i, s) for s in bausteine[i][1]) - for i in idxs - ) - subs_total = sum(len(bausteine[i][1]) for i in idxs) - status, _ = await run_single_slot( - ctx, f"Frage-Muster {ci}", - key=f"bausteine-{topic}-frage-muster-c{ci}", - prompt=_prompt("Frage-Muster-Recherche", topic=topic, bausteine=block, - out_path=fp, extra=_extra(instructions)), - role="fast", capabilities="files", - payload=lambda result, p=fp: _frage_muster_chunk_schema(_json_datei(p)), - timeout=_timeout("frage_muster", subs_total), - ) - if status == FAILED: - _log(topic, f"Frage-Muster Chunk {ci} fehlgeschlagen — Bausteine im Fallback (Nachrunde/Live)") - - async def finde_alle(ci_list): - ci_list = list(ci_list) - await _gather_fortschritt([_finde(ci, chunks[ci]) for ci in ci_list], len(ci_list), _melde_p(set_p, topic, "Fragen finden")) - - await finde_alle(range(len(chunks))) - if is_cancelled(): - return None - - # Phase „Fragen wählen": Code — Chunk-Dateien je Baustein gruppieren, Dubletten raus, - # Baustein-/Subbaustein-Titel locker auf die Vorgaben mappen (nichts wegen Abweichung verwerfen). - def _waehle_chunk(ci): - idxs = chunks[ci] - ctitel = _chunk_titel(idxs) - rel_by = {bausteine[i][0]: bausteine[i][1] for i in idxs} - out, gesehen = {}, {} - for e in _frage_muster_chunk_schema(_json_datei(roh_path(ci))) or []: - titel = _match_sub(e["baustein"], ctitel) - if titel not in rel_by: - continue # nicht zuordenbar → verwerfen - sub = _match_sub(e["subbaustein"], rel_by[titel]) - seen = gesehen.setdefault(titel, set()) - if sub in seen: - continue # genau ein Muster je Subbaustein - seen.add(sub) - out.setdefault(titel, []).append({"subbaustein": sub, "frage": e["frage"]}) - return out - - def _waehle_all(ci_list): - roh = {} - for ci in ci_list: - for titel, eintraege in _waehle_chunk(ci).items(): - roh.setdefault(titel, []).extend(eintraege) - return roh - - set_p("Fragen wählen…", step=_step_idx(topic, "Fragen wählen")) - roh_by_titel = _waehle_all(range(len(chunks))) - - # Phase „Fragen klären": je Chunk 1 Kritiker bereinigt die Tabellen (nach Baustein gruppiert). - async def _klaere(ci, idxs): - fp = final_path(ci) - if _frage_muster_chunk_schema(_json_datei(fp)): - return # Resume - bloecke = [] - for i in idxs: - t = bausteine[i][0] - eintraege = roh_by_titel.get(t) or [] - if not eintraege: - continue - zeilen = "\n".join(f"- ({e['subbaustein']}) {e['frage']}" for e in eintraege) - bloecke.append(f"BAUSTEIN: {t}\n{zeilen}") - if not bloecke: - return # nichts zu klären in diesem Chunk - subs_total = sum(len(bausteine[i][1]) for i in idxs) - status, _ = await run_single_slot( - ctx, f"Frage-Muster-Klärung {ci}", - key=f"bausteine-{topic}-frage-muster-final-c{ci}", - prompt=_prompt("Frage-Muster-Kritik", topic=topic, tabelle="\n\n".join(bloecke), out_path=fp, extra=_extra(instructions)), - role="judge", capabilities="files", - payload=lambda result, p=fp: _frage_muster_chunk_schema(_json_datei(p)), - timeout=_timeout("frage_muster_check", subs_total), - ) - if status == FAILED: - _log(topic, f"Frage-Muster-Klärung Chunk {ci} fehlgeschlagen — Roh-Muster übernommen") - - async def klaere_alle(ci_list): - ci_list = list(ci_list) - await _gather_fortschritt([_klaere(ci, chunks[ci]) for ci in ci_list], len(ci_list), _melde_p(set_p, topic, "Fragen klären")) - - await klaere_alle(range(len(chunks))) - if is_cancelled(): - return None - - # Geklärte Chunk-Tabelle je Baustein, Fallback auf Roh-Muster. Titel locker mappen. - def _final_by_titel(ci_list): - out = {} - for ci in ci_list: - idxs = chunks[ci] - ctitel = _chunk_titel(idxs) - rel_by = {bausteine[i][0]: bausteine[i][1] for i in idxs} - for e in _frage_muster_chunk_schema(_json_datei(final_path(ci))) or []: - titel = _match_sub(e["baustein"], ctitel) - if titel not in rel_by: - continue - out.setdefault(titel, []).append( - {"subbaustein": _match_sub(e["subbaustein"], rel_by[titel]), "frage": e["frage"]}) - return out - - final_by_titel = _final_by_titel(range(len(chunks))) - ergebnis = {t: (final_by_titel.get(t) or roh_by_titel.get(t) or []) for t, _ in bausteine} - - # Phase „Fragen prüfen": pro-Sub-Vollständigkeit. Generatoren stürzen zufällig ab (~15 %), - # je Chunk 1 Agent ohne Retry → Subs (ganze Bausteine) fallen still durch. Darum mehrere - # Runden, die NUR die fehlenden Subs gezielt nachfordern (kurze Pakete, Frage-Muster-Recherche). - set_p("Fragen prüfen…", step=_step_idx(topic, "Fragen prüfen")) - - def _fehlende_subs() -> list[tuple[str, list[str]]]: - out = [] - for t, subs in bausteine: - hab = {_norm_titel(e["subbaustein"]) for e in ergebnis.get(t) or []} - miss = [s for s in subs if _norm_titel(s) not in hab] - if miss: - out.append((t, miss)) - return out - - def _nach_block(items): # items: [(baustein_titel, [fehlende sub_titel])] - bloecke = [] - for titel, subs in items: - zeilen = [] - for s in subs: - z = f"- {s}" - fk = fakten_by.get((titel, _norm_titel(s))) - if fk and (ft := _fakten_zeilen(fk)): - z += "\n" + "\n".join(" " + l for l in ft.split("\n")) - zeilen.append(z) - bloecke.append(f"BAUSTEIN: {titel}\nSUBBAUSTEINE:\n" + "\n".join(zeilen)) - return "\n\n".join(bloecke) - - async def _nachfordere(runde, pi, items): - fp = arbeit / f"frage-muster-nach{runde}-c{pi}.json" - if _frage_muster_chunk_schema(_json_datei(fp)): - return # Resume - subs_total = sum(len(s) for _, s in items) - await run_single_slot( - ctx, f"Frage-Muster Nachholung R{runde}/{pi}", - key=f"bausteine-{topic}-frage-muster-nach{runde}-c{pi}", - prompt=_prompt("Frage-Muster-Recherche", topic=topic, bausteine=_nach_block(items), - out_path=fp, extra=_extra(instructions)), - role="fast", capabilities="files", - payload=lambda result, p=fp: _frage_muster_chunk_schema(_json_datei(p)), - timeout=_timeout("frage_muster", subs_total), - ) - - for runde in range(1, FRAGE_MAX_RUNDEN + 1): - fehlend = _fehlende_subs() - if not fehlend: - break - n_subs = sum(len(s) for _, s in fehlend) - _log(topic, f"Frage-Muster Runde {runde}: {n_subs} Sub(s) in {len(fehlend)} Baustein(en) ohne Muster — Nachforderung") - pakete = _lpt_chunks([len(s) for _, s in fehlend], FRAGE_CHUNK_SUBS) - paket_items = [[fehlend[i] for i in idxs] for idxs in pakete] - await _gather_fortschritt( - [_nachfordere(runde, pi, items) for pi, items in enumerate(paket_items)], - len(paket_items), _melde_p(set_p, topic, "Fragen prüfen")) - if is_cancelled(): - return None - # Output je Paket parsen + neu gewonnene Subs mergen (Vorhandene nicht überschreiben). - for pi, items in enumerate(paket_items): - titel_subs = {t: subs for t, subs in items} - ctitel = list(titel_subs.keys()) - for e in _frage_muster_chunk_schema(_json_datei(arbeit / f"frage-muster-nach{runde}-c{pi}.json")) or []: - titel = _match_sub(e["baustein"], ctitel) - if titel not in titel_subs: - continue - sub = _match_sub(e["subbaustein"], titel_subs[titel]) - hab = {_norm_titel(x["subbaustein"]) for x in ergebnis.get(titel) or []} - if _norm_titel(sub) in hab: - continue - ergebnis.setdefault(titel, []).append({"subbaustein": sub, "frage": e["frage"]}) - - rest = _fehlende_subs() - if rest: - n = sum(len(s) for _, s in rest) - _log(topic, f"Frage-Muster: {n} Sub(s) in {len(rest)} Baustein(en) bleiben nach {FRAGE_MAX_RUNDEN} Runden leer: {[t for t, _ in rest][:5]}") - return ergebnis - - -# ── Inventar in der DB: Recherche-Loop · Konsolidierung · Klärung ──────────── - - - -def _crawl_index(ordner) -> dict[str, str]: - """Alias (Dateiname ODER QUELLE:-URL, klein) → kanonischer Seiten-Key (Dateiname).""" - idx: dict[str, str] = {} - if not ordner or not Path(ordner).is_dir(): - return idx - for p in sorted(Path(ordner).glob("*.txt")): - key = p.name - idx[key.lower()] = key - try: - erste = p.read_text(encoding="utf-8").splitlines()[0] - except (OSError, IndexError): - erste = "" - if erste.startswith("QUELLE:"): - url = erste[len("QUELLE:"):].strip() - if url: - idx[url.lower()] = key - idx[url.rstrip("/").lower()] = key - return idx - - - - - - -async def _set_inventar(topic: str, eintrag: str, status: str) -> None: - """Einen Inventar-Eintrag ('Titel — Beschreibung') mit Status in die DB schreiben.""" - titel = _titel(eintrag) - norm = _norm_titel(titel) - if not norm: - return - teile = [t.strip() for t in eintrag.split(" — ")] - besch = teile[1] if len(teile) >= 2 else "" - await db.upsert_baustein(topic, norm, titel, besch) - await db.set_baustein_status(topic, norm, status) - - -def _sichte_regeln(ordner, pages: list[str]) -> tuple[list[str], list[str]]: - """Deterministischer Content/Noise-Filter (config.CRAWL_*). Substring-Match (klein) gegen - URL + Dateiname. Reihenfolge: keep > noise > min_chars > behalten. → (content, noise).""" - ordner = Path(ordner) - content, noise = [], [] - for fn in pages: - zeilen = _read(ordner / fn).splitlines() - url = zeilen[0][len("QUELLE:"):].strip() if zeilen and zeilen[0].startswith("QUELLE:") else "" - body = "\n".join(zeilen[1:]).strip() - hay = f"{url}\n{fn}".lower() - if any(p in hay for p in CRAWL_KEEP_PATTERNS): - content.append(fn) - elif any(p in hay for p in CRAWL_NOISE_PATTERNS): - noise.append(fn) - elif len(body) < CRAWL_MIN_CHARS: - noise.append(fn) - else: - content.append(fn) # Default: behalten — alles mit Inhalt bleibt - return content, noise - - -def _seite_snippet(ordner, fn: str) -> tuple[str, str]: - """(url, snippet) einer Crawl-Seite für das Relevanz-Gate. url aus der QUELLE:-Zeile; - snippet = Body-Auszug (Navigations-Boilerplate steht vorn — der Prompt ignoriert es). - URL ist das Primärsignal (sprechender Slug), der Snippet stützt nur.""" - zeilen = _read(Path(ordner) / fn).splitlines() - url = zeilen[0][len("QUELLE:"):].strip() if zeilen and zeilen[0].startswith("QUELLE:") else "" - body = "\n".join(zeilen[1:]).strip() - snippet = " ".join(body.split())[:QUELLE_RELEVANZ_SNIPPET] - return (url or fn), snippet - - -async def _relevanz_sichtung(ctx: GenContext, set_p, files: dict, ordner, content: list[str], spec: str, instructions: str) -> tuple[list[str], list[str]]: - """LLM-Themen-Gate nach dem Regel-Filter: jede Content-Seite ja/nein gegen die Spec. - Off-topic (anderes Fachgebiet) → raus. Muster wie `_relevanz_block`: kleine Pakete, 3 Rater - (`fast`), 2-von-3-Konsens. KONSERVATIV: nur bei klarer „nein"-Mehrheit droppen; Streit/Lücke/ - Race-Fehler → behalten. SAFETY: würde das Gate ≥80 % (oder alles) droppen, bleibt alles - (Spec-Mismatch/Bug soll die Quelle nicht leeren). → (behalten, raus) als Dateinamen.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - arbeit = files["arbeit"] - pages = sorted(content) - if not pages: - return content, [] - items = [_seite_snippet(ordner, fn) for fn in pages] # Index deckt sich mit `pages` - chunks = _chunk_nums(list(range(len(pages))), _n_chunks(len(pages), QUELLE_RELEVANZ_CHUNK)) - n = len(chunks) - - def rater_paths(c): - return [arbeit / f"quelle-relevanz-c{c}-{i}.json" for i in (1, 2, 3)] - - def lset(idxs): - return set(range(1, len(idxs) + 1)) - - async def _rate(c, idxs): - local_set = lset(idxs) - paths = rater_paths(c) - vorhanden = sum(1 for p in paths if _janein_schema(_json_datei(p), local_set)) - if vorhanden >= 2: - return True - enum_zeilen = [] - for k, j in enumerate(idxs, 1): - url, snip = items[j] - enum_zeilen.append(f"{k}. {url}") - if snip: - enum_zeilen.append(f" {snip}") - enum = "\n".join(enum_zeilen) - offen = [(i, p) for i, p in enumerate(paths, 1) if not _janein_schema(_json_datei(p), local_set)] - slots = [{ - "key": f"bausteine-{topic}-quelle-relevanz-c{c}-{i}", - "prompt": _prompt("Quelle-Relevanz", topic=topic, spec=spec, seiten=enum, out_path=p, extra=_extra(instructions)), - "role": "fast", "capabilities": "files", - "payload": (lambda result, p=p, ids=local_set: _janein_schema(_json_datei(p), ids)), - } for i, p in offen] - neu = await _race(topic, f"Relevanz-Sichtung Paket {c}", slots, 2 - vorhanden, _timeout("relevanz", len(idxs)), provider, cancelled=is_cancelled, grace=KONSENS_GRACE) - return not is_cancelled() and neu is not None - - _qidx = _step_idx(topic, "Quelle aufbereiten") # Gate läuft im Quelle-Schritt (kein eigener Step) - set_p(f"Prüfe Relevanz zur Spec ({n} Pakete)…", step=_qidx) - async def _melde_sichtung(d, t): - set_p(f"Prüfe Relevanz zur Spec {d}/{t}…", step=_qidx) - await _gather_fortschritt([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], n, _melde_sichtung) - if is_cancelled(): - return content, [] # Abbruch → nichts droppen (Caller bricht ab) - - # Vote je Seite: nur eine klare „nein"-Mehrheit (≥2 und mehr als „ja") wirft raus. - raus: list[str] = [] - for c, idxs in enumerate(chunks, 1): - local_set = lset(idxs) - rater = [d for p in rater_paths(c) if (d := _janein_schema(_json_datei(p), local_set))] - for k in range(1, len(idxs) + 1): - stimmen = [d[k] for d in rater if k in d] - nein, ja = stimmen.count("nein"), stimmen.count("ja") - if nein >= 2 and nein > ja: - raus.append(pages[idxs[k - 1]]) - - if raus and len(raus) >= max(1, int(len(pages) * 0.8)): - _log(topic, f"Relevanz-Sichtung: würde {len(raus)}/{len(pages)} droppen — verworfen (Spec-Mismatch?), alles behalten") - return content, [] - raus_set = set(raus) - behalten = [fn for fn in pages if fn not in raus_set] - return behalten, raus - - -async def _quelle_aufbereiten(ctx: GenContext, set_p, files: dict, q: dict, ordner, instructions: str) -> bool: - """Schritt „Quelle aufbereiten": Crawl (link) + PDF-Konvert + Content/Noise-Sichtung. - Persistiert die Sichtung in der Coverage-Tabelle (inhalt). → True (ok) / False (Abbruch/Fehler). - thema: nichts. projekt/uni: nur PDFs (kuratierter Ordner, keine Sichtung).""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - if not ordner: - return True # thema → keine Quelle aufzubereiten - if q["type"] != "link": - await asyncio.to_thread(_pdfs_konvertieren, ordner) # projekt/uni: nur PDFs, keine Sichtung - return True - if await db.get_step_status(topic, "Quelle aufbereiten") == "fertig": - return True - if not _crawl_fertig(topic): - set_p("Quelle laden (Crawl)…", step=_step_idx(topic, "Quelle aufbereiten")) - n = await asyncio.to_thread(crawl, q["ort"], ordner, cancelled=is_cancelled) - if is_cancelled(): - return False - if not n: - _bausteine_errors[topic] = "Crawl ergab keine Inhalte — Link/Domain prüfen" - return False - await asyncio.to_thread(_pdfs_konvertieren, ordner) - pages = sorted(set(_crawl_index(ordner).values())) - if pages: - set_p("Sichte Seiten…", step=_step_idx(topic, "Quelle aufbereiten")) - await db.delete_coverage(topic) - content, noise = _sichte_regeln(ordner, pages) # deterministischer Regel-Filter - if q.get("spec") and content: # Themen-Gate: trennt das Fachgebiet (Regeln können das nicht) - content, raus = await _relevanz_sichtung(ctx, set_p, files, ordner, content, q["spec"], instructions) - if is_cancelled(): - return False - if raus: - noise = sorted(set(noise) | set(raus)) - _log(topic, f"LLM-Relevanz: {len(raus)} Seiten off-topic → Noise") - await db.mark_inhalt(topic, sorted(content), sorted(noise)) - _log(topic, f"Sichtung: {len(content)} Content / {len(noise)} Noise von {len(pages)} (Regeln + LLM-Gate)") - await db.set_step_status(topic, "Quelle aufbereiten", "fertig") - return True - - -async def _recherche_batch(ctx: GenContext, set_p, files: dict, q: dict, ordner, instructions: str) -> bool: - """Befüllt DB-Tabelle `bausteine` mit Kandidaten (+ Nennungszähler). FESTE Datei-Batches: - jede Crawl-Seite wird genau einem Batch zugeteilt und von RECHERCHE_READERS Agenten gelesen - (Konsens ≥2 im Batch). Alle zugeteilten Seiten werden als gelesen markiert → 100 % Abdeckung. - Ohne Crawl-Ordner (Quelle „thema") → freie Web-Recherche, eine Runde. → True/False.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - if await db.get_step_status(topic, "Recherche") == "fertig": - return True - arbeit = files["arbeit"] - await db.delete_bausteine(topic) # Coverage/inhalt gehört der Sichtung — NICHT löschen - await db.set_step_status(topic, "Recherche", "laufend") - - async def _ingest(reader_id: str, text: str) -> None: - gesehen = set() - for eintrag in _parse_auswahl(text).values(): - titel = _titel(eintrag) - norm = _norm_titel(titel) - if not norm or norm in gesehen: - continue - gesehen.add(norm) # ein Reader = eine Stimme je Konzept - teile = [t.strip() for t in eintrag.split(" — ")] - besch = teile[1] if len(teile) >= 2 else "" - quelle = [teile[2]] if len(teile) >= 3 and teile[2] else [] - await db.upsert_baustein(topic, norm, titel, besch, quelle, reader=reader_id) - - pages = await db.list_content(topic) # von der Sichtung als Content markierte Seiten - if not pages and ordner: - pages = sorted(set(_crawl_index(ordner).values())) # Fallback (projekt/uni: keine Sichtung) - - if not pages: - # Quelle „thema" (oder kein Crawl): freie Web-Recherche, eine Runde. - set_p("Recherche läuft…", step=_step_idx(topic, "Recherche")) - caps = "files" if ordner else "full" - paths = [arbeit / f"recherche-{i}.md" for i in range(1, RECHERCHE_THEMA_AGENTEN + 1)] - for p in paths: - p.unlink(missing_ok=True) - slots = [{ - "key": f"bausteine-{topic}-recherche-{i}", - "prompt": _build_recherche_prompt(topic, p, instructions, q["type"], ordner), - "role": "quick", "capabilities": caps, - "payload": (lambda result, p=p, rid=f"t{i}": ((rid, t) if (t := _file_payload(p)) else None)), - } for i, p in enumerate(paths, 1)] - texte = await _race(topic, "Recherche", slots, 3, _timeout("recherche"), provider, - cancelled=is_cancelled, grace=RECHERCHE_GRACE) - if is_cancelled(): - return False - if not texte: - _bausteine_errors[topic] = "Recherche fehlgeschlagen (Minimum nicht erreicht)" - return False - for rid, text in texte: - await _ingest(rid, text) - await db.set_step_status(topic, "Recherche", "fertig") - return True - - # uni/projekt: kuratierte, oft GROSSE Dateien (Skript). Statt alle am Stück zu lesen - # (Lost-in-the-Middle), in Abschnitte chunken und JEDEN gründlich von 2 Readern lesen — - # Text direkt im Prompt (kleiner Kontext), Nennungen akkumulieren zu Konsens. - if q["type"] in ("uni", "projekt"): - eintraege: list[tuple[str, str]] = [] # (dateiname, abschnitt-text) - for fn in sorted(pages): - for absch in _text_abschnitte(_read(ordner / fn)): - eintraege.append((fn, absch)) - if not eintraege: - _bausteine_errors[topic] = "Recherche: Quelle leer" - return False - set_p(f"Recherche ({len(eintraege)} Abschnitte)…", step=_step_idx(topic, "Recherche")) - - async def _lese_abschnitt(ei: int, fn: str, absch: str) -> None: - block = (f"ARBEITE AUSSCHLIESSLICH MIT DIESEM TEXTABSCHNITT (Quelle: {fn}). Lies ihn " - f"VOLLSTÄNDIG, überspringe nichts. Notiere `{fn}` als Quelle jedes Bausteins. " - f"Suche NICHT im Web — nur dieser Abschnitt zählt.\n\n-----\n{absch}\n-----") - paths = [arbeit / f"recherche-a{ei}-{i}.md" for i in range(1, RECHERCHE_READERS + 1)] - # Reader-Datei-Wiederverwendung: liegen alle Reader-Outputs valide vor (Resume / - # Re-Run ohne Recherche-Änderung), re-ingestieren statt erneut Agenten zu spawnen. - vorhanden = [(f"a{ei}-{i}", t) for i, p in enumerate(paths, 1) if (t := _file_payload(p))] - if len(vorhanden) == len(paths): - for rid, text in vorhanden: - await _ingest(rid, text) - return - for p in paths: - p.unlink(missing_ok=True) - if is_cancelled(): - return - slots = [{ - "key": f"bausteine-{topic}-recherche-a{ei}-{i}", - "prompt": _build_recherche_prompt(topic, p, instructions, q["type"], ordner, abschnitt=block), - "role": "quick", "capabilities": "files", - "payload": (lambda result, p=p, rid=f"a{ei}-{i}": ((rid, t) if (t := _file_payload(p)) else None)), - } for i, p in enumerate(paths, 1)] - # Quorum 2: beide Reader pro Abschnitt sollen durch (mehr Augen = mehr Konzepte + - # echter Konsens); nach Timeout fällt _race auf das Vorhandene zurück. - texte = await _race(topic, f"Recherche Abschnitt {ei}", slots, 2, _timeout("recherche", 1), - provider, cancelled=is_cancelled, grace=RECHERCHE_GRACE) - for rid, text in (texte or []): - await _ingest(rid, text) - - await _gather_fortschritt([_lese_abschnitt(ei, fn, a) for ei, (fn, a) in enumerate(eintraege, 1)], - len(eintraege), _melde_p(set_p, topic, "Recherche")) - if is_cancelled(): - return False - await db.mark_quellen_gelesen(topic, sorted(pages)) - gesamt = len(await db.list_bausteine(topic)) - _log(topic, f"Recherche (uni/projekt): {gesamt} Kandidaten aus {len(eintraege)} Abschnitten ({len(pages)} Dateien)") - if not gesamt: - _bausteine_errors[topic] = "Recherche fehlgeschlagen (keine Bausteine)" - return False - await db.set_step_status(topic, "Recherche", "fertig") - return True - - # Crawl/Link: viele kleine Content-Seiten (Sichtung im Schritt „Quelle aufbereiten"). - # Feste Batches, je Batch RECHERCHE_READERS Reader, die GENAU diese Dateien lesen. - batches = _chunk_nums(sorted(pages), max(1, math.ceil(len(pages) / RECHERCHE_BATCH))) - - async def _lese_batch(bi: int, batch: list[str]) -> bool: - liste = "\n".join(f"- {p}" for p in batch) - fokus = ("WICHTIG — feste Zuteilung: Bearbeite AUSSCHLIESSLICH diese Dateien und lies JEDE " - f"vollständig. Ignoriere alle anderen Dateien im Ordner:\n{liste}") - paths = [arbeit / f"recherche-b{bi}-{i}.md" for i in range(1, RECHERCHE_READERS + 1)] - for p in paths: - p.unlink(missing_ok=True) - if not is_cancelled(): - slots = [{ - "key": f"bausteine-{topic}-recherche-b{bi}-{i}", - "prompt": _build_recherche_prompt(topic, p, instructions, q["type"], ordner, fokus=fokus), - "role": "quick", "capabilities": "files", - "payload": (lambda result, p=p, rid=f"b{bi}-{i}": ((rid, t) if (t := _file_payload(p)) else None)), - } for i, p in enumerate(paths, 1)] - texte = await _race(topic, f"Recherche Batch {bi}", slots, 1, _timeout("recherche", len(batch)), - provider, cancelled=is_cancelled, grace=RECHERCHE_GRACE) - for rid, text in (texte or []): - await _ingest(rid, text) - await db.mark_quellen_gelesen(topic, batch) # alle zugeteilten Seiten abhaken (auch ohne Treffer) - return not is_cancelled() - - await _gather_fortschritt([_lese_batch(bi, b) for bi, b in enumerate(batches, 1)], - len(batches), _melde_p(set_p, topic, "Recherche")) - if is_cancelled(): - return False - gesamt = len(await db.list_bausteine(topic)) - deckung = len(await db.list_coverage(topic)) - _log(topic, f"Recherche: {gesamt} Kandidaten, Abdeckung {deckung}/{len(pages)} Seiten ({len(batches)} Batches)") - if not gesamt: - _bausteine_errors[topic] = "Recherche fehlgeschlagen (keine Bausteine)" - return False - await db.set_step_status(topic, "Recherche", "fertig") - return True - - -def _grp_schema(data, ids: set[int]): - """{"gruppen": [[1,3],[2], …]} → Partition von `ids` als Liste von Index-Gruppen. - Tolerant: ignoriert Fremd-/Doppel-Nummern; vergessene Kandidaten werden eigenständig - (Singleton-Gruppe) ergänzt. None nur bei strukturell kaputtem JSON.""" - if not isinstance(data, dict) or not isinstance(data.get("gruppen"), list): - return None - gruppen, gesehen = [], set() - for g in data["gruppen"]: - if not isinstance(g, list): - return None - grp = [] - for x in g: - try: - num = int(x) - except (ValueError, TypeError): - continue - if num in ids and num not in gesehen: - gesehen.add(num) - grp.append(num) - if grp: - gruppen.append(grp) - gruppen += [[r] for r in sorted(ids - gesehen)] # vergessene Kandidaten bleiben eigenständig - return gruppen or None - - -_ASPEKT_MARKER = ("∈ np", "∈np", " in np", "np-schwer", "np-vollständig", "verifizierer", - "zertifikat", "ndtm", "nicht-determ", "lower bound", "untere schranke", - "bzgl", "als sprache") - - -def _aspekt_marker(titel: str) -> int: - """Anzahl Eigenschafts-Marker im Titel (∈NP, NP-schwer, Verifizierer, Lower Bound …). - 0 = generisches Hauptkonzept (das Problem selbst); >0 = eine Eigenschaft davon.""" - t = titel.casefold() - return sum(1 for m in _ASPEKT_MARKER if m in t) - - -_VERWEIS_RE = re.compile(r'^(Satz|Lemma|Korollar|Bemerkung|Definition)\s*[\d.]+\s*(\([a-z]\)|[a-z])?\s*$', re.I) - - -def _ist_verweis(titel: str) -> bool: - """True für reine Verweis-/Platzhalter-Titel OHNE sprechenden Inhalt: „Satz 7.18", „Lemma 6.2", - „Korollar 6.18" (Nummer ohne Namen) sowie markierte Stellen „Bedingung (**)". NICHT „Satz 6.24: - Cook/Levin" (hat Namen) und NICHT kurze Fachsymbole wie „P⊆NP"/„Σ*" (echte Konzepte).""" - t = titel.strip() - if _VERWEIS_RE.match(t): - return True - if re.search(r'\(\*+\)', t): # markierte Stelle „(**)" / „(*)" - return True - return False - - -def _canonical(kandidaten: list[dict], idxs: list[int], gesehen_norm: set[str]) -> dict: - """Repräsentant eines Clusters = das Hauptkonzept (wenigste Eigenschafts-Marker — das Problem - selbst, nicht „… ∈ NP"); Tie → häufigster norm-Titel → meiste Reader. Titel global eindeutig - (Suffix ' (2)'), damit er als Schlüssel taugt.""" - by_norm: dict[str, list[int]] = {} - for k in idxs: - by_norm.setdefault(_norm_titel(kandidaten[k]["titel"]), []).append(k) - - def gewicht(nb: str): - ms = by_norm[nb] - reader = set().union(*[set(kandidaten[m]["reader"]) for m in ms]) if ms else set() - # Verweis-/Platzhalter-Titel ("Satz 7.18") zuletzt — sprechendes Mitglied bevorzugen. - echt = not _ist_verweis(kandidaten[ms[0]]["titel"]) - return (echt, -_aspekt_marker(nb), len(ms), len(reader)) - - best = max(by_norm, key=gewicht) - k = max(by_norm[best], key=lambda m: len(kandidaten[m]["beschreibung"])) - titel = kandidaten[k]["titel"] - n = 2 - while _norm_titel(titel) in gesehen_norm: - titel = f"{kandidaten[k]['titel']} ({n})" - n += 1 - gesehen_norm.add(_norm_titel(titel)) - return {"titel": titel, "beschreibung": kandidaten[k]["beschreibung"]} - - -async def _block_gruppieren(ctx: GenContext, set_p, arbeit: Path, kandidaten: list[dict], - blocks: list[list[int]], praefix: str = "konsolidierung", - schritt: str = "Konsolidierung") -> list[list[int]]: - """Je Ähnlichkeits-Block gruppiert ein Judge die Titel in die echten Bausteine (merge - Paraphrasen, split Über-Merges). Singletons direkt. Fehler/Timeout → konservativ jeder - Kandidat einzeln (vermeidet fälschliches Über-Mergen). → finale Gruppen (globale Indizes). - `praefix`/`schritt` trennen Konsolidierung und Dedup (Artefakte, Race-Key, Fortschritt).""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - multi = [(bi, b) for bi, b in enumerate(blocks) if len(b) > 1] - ergebnis: list[list[int]] = [list(b) for b in blocks if len(b) == 1] # Singletons direkt - - def _zeile(k: int, g: int) -> str: - b = kandidaten[g] - return f"{k}. {b['titel']}" + (f" — {b['beschreibung']}" if b["beschreibung"] else "") - - async def _grp(bi: int, block: list[int]) -> None: - ids = set(range(1, len(block) + 1)) - p = arbeit / f"{praefix}-block-c{bi}.json" - part = _grp_schema(_json_datei(p), ids) - if part is None: # Resume: gültige Datei nicht neu rechnen - p.unlink(missing_ok=True) - if is_cancelled(): - return - zeilen = [_zeile(k, block[k - 1]) for k in range(1, len(block) + 1)] - status, part = await run_single_slot( - ctx, f"Block-Gruppieren {bi}", - key=f"bausteine-{topic}-{praefix}-block-c{bi}", - prompt=_prompt("Bausteine-Block-Gruppieren", topic=topic, eintraege="\n".join(zeilen), out_path=p), - role="judge", capabilities="files", - payload=(lambda result, p=p, ids=ids: _grp_schema(_json_datei(p), ids)), - timeout=_timeout("recherche_mapping", len(block)), - ) - part = part if status == OK else None - if part is None: # Judge gescheitert → einzeln (kein Über-Merge) - ergebnis.extend([idx] for idx in block) - else: # lokale Nummern → globale Kandidaten-Indizes - ergebnis.extend([block[k - 1] for k in g] for g in part) - - await _gather_fortschritt([_grp(bi, b) for bi, b in multi], - len(multi), _melde_p(set_p, topic, schritt)) - return ergebnis - - -async def _konsolidiere_embedding(ctx: GenContext, set_p, files: dict, kandidaten: list[dict]) -> bool: - """Zweistufig: Embeddings → grobe Capped-Blocks (High-Recall) → je Multi-Block ein Judge, - der die Titel in die echten Bausteine gruppiert → Reader-Union (≥2 = Konsens).""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - arbeit = files["arbeit"] - texts = [f"{b['titel']} — {b['beschreibung']}" if b["beschreibung"] else b["titel"] for b in kandidaten] - sims = await asyncio.to_thread(embedding.embed_sims, texts) - if sims is None: # Modell doch nicht verfügbar → Fallback - return await _konsolidiere_llm(ctx, set_p, files, kandidaten) - # Stufe 1: grobe Ähnlichkeits-Blocks (gedeckelt, kein Giant-Component). - blocks = await asyncio.to_thread(embedding.capped_blocks, sims, None, None) - # Stufe 2: ein Judge gruppiert JEDEN Multi-Block in die echten Bausteine. - gruppen = await _block_gruppieren(ctx, set_p, arbeit, kandidaten, blocks) - if is_cancelled(): - return False - - def _min_cos(idxs): # interne Kohärenz zur Kontrolle (Ketten hätten ~0,3) - if len(idxs) < 2: - return 1.0 - return round(min(float(sims[i][j]) for n, i in enumerate(idxs) for j in idxs[n + 1:]), 3) - - # Konsens = ≥2 distinkte Reader pro Cluster. Legacy-DBs ohne Reader-Tracking (Recherche lief - # vor der Migration, kein Re-Ingest) haben leere Reader-Sets → Rückfall auf Titel-Heuristik - # (sonst landete ALLES im Rest). - hat_reader = any(b["reader"] for b in kandidaten) - konsens, rest, debug, gesehen_norm = [], [], [], set() - for idxs in gruppen: - reader = set().union(*[set(kandidaten[k]["reader"]) for k in idxs]) if idxs else set() - if hat_reader: - score = len(reader) - else: # ohne Reader-Daten: max(Nennungen, Anzahl distinkter Titel-Varianten im Cluster) - score = max(max(kandidaten[k]["nennungen"] for k in idxs), - len({kandidaten[k]["titel_norm"] for k in idxs})) - rep = _canonical(kandidaten, idxs, gesehen_norm) - eintrag = f"{rep['titel']} — {rep['beschreibung']}" if rep["beschreibung"] else rep["titel"] - (konsens if score >= 2 else rest).append(eintrag) - debug.append({"titel": rep["titel"], "reader": sorted(reader), "score": score, - "konsens": score >= 2, "min_cos": _min_cos(idxs), - "mitglieder": [kandidaten[k]["titel"] for k in idxs]}) - atomic_write_json(arbeit / "konsolidierung-cluster.json", debug, indent=1) - multi_blocks = sum(1 for b in blocks if len(b) > 1) - _log(topic, f"Konsolidierung (Embedding): {len(blocks)} Blocks ({multi_blocks} per LLM gruppiert) " - f"→ {len(gruppen)} Cluster aus {len(kandidaten)} Kandidaten " - f"→ {len(konsens)} Konsens / {len(rest)} Rest") - - await db.delete_bausteine(topic) - for t in konsens: - await _set_inventar(topic, t, "konsens") - for t in rest: - await _set_inventar(topic, t, "rest") - await db.set_step_status(topic, "Konsolidierung", "fertig") - return True - - -async def _konsolidiere(ctx: GenContext, set_p, files: dict) -> bool: - """Mergt Roh-Kandidaten zu Konsens (≥2 Reader)/Rest. Deterministisch per Embedding-Clustering; - fehlt das Modell → Rückfall auf das LLM-Panel (`_konsolidiere_llm`). Status in DB.""" - topic = ctx.topic - if await db.get_step_status(topic, "Konsolidierung") == "fertig": - return True - set_p("Konsolidiere Recherche…", step=_step_idx(topic, "Konsolidierung")) - kandidaten = await db.list_bausteine(topic) - if not kandidaten: - _bausteine_errors[topic] = "Konsolidierung: keine Kandidaten" - return False - if EMBEDDING_AKTIV and await asyncio.to_thread(embedding.verfuegbar): - return await _konsolidiere_embedding(ctx, set_p, files, kandidaten) - return await _konsolidiere_llm(ctx, set_p, files, kandidaten) - - -async def _konsolidiere_llm(ctx: GenContext, set_p, files: dict, kandidaten: list[dict]) -> bool: - """Fallback (nur ohne Embedding-Modell): Panel (KONSOLIDIERUNG_PANEL Judges) mergt Kandidaten - semantisch; ein Reconcile-Judge führt die Panel-Ausgaben zur finalen Konsens (≥2)/Rest (1×)-Liste - zusammen. Panel statt Einzel-Judge: ein einzelner Judge ist bias-anfällig und instabil.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - arbeit = files["arbeit"] - chunks = _chunk_nums(kandidaten, max(1, math.ceil(len(kandidaten) / KONSOLIDIERUNG_CHUNK))) - - async def _map_panel(c: int, eintraege: str, anzahl: int): - """3 Mapping-Judges über `eintraege` → Reconcile-Judge → (konsens, rest). None bei Abbruch/Fehler.""" - paths = [arbeit / f"konsolidierung-c{c}-j{j}.json" for j in range(1, KONSOLIDIERUNG_PANEL + 1)] - offen = [(j, p) for j, p in enumerate(paths, 1) if _mapping_schema(_json_datei(p)) is None] - for _, p in offen: - p.unlink(missing_ok=True) - if offen: - slots = [{ - "key": f"bausteine-{topic}-konsolidierung-c{c}-j{j}", - "prompt": _prompt("Bausteine-Recherche-Mapping", topic=topic, n=RECHERCHE_READERS, eintraege=eintraege, out_path=p), - "role": "judge", "capabilities": "files", - "payload": (lambda result, p=p: _mapping_schema(_json_datei(p))), - } for j, p in offen] - vorhanden = KONSOLIDIERUNG_PANEL - len(offen) - await _race(topic, f"Konsolidierung {c}", slots, max(1, 2 - vorhanden), - _timeout("recherche_mapping", anzahl), provider, cancelled=is_cancelled, grace=KONSENS_GRACE) - if is_cancelled(): - return None - outs = [m for p in paths if (m := _mapping_schema(_json_datei(p)))] - if not outs: - return None - # Union der Panel-Titel; je Titel zählen, wie viele Judges ihn als Konsens führen. - kvotes: dict[str, int] = {} - form: dict[str, str] = {} # norm → Anzeigetitel (erstes Vorkommen) - order: list[str] = [] - for kk, rr in outs: - for t in kk + rr: - nt = _norm_titel(_titel(t)) - if not nt: - continue - if nt not in form: - form[nt] = t - order.append(nt) - kvotes.setdefault(nt, 0) - for t in kk: - nt = _norm_titel(_titel(t)) - if nt: - kvotes[nt] = kvotes.get(nt, 0) + 1 - # Reconcile: ein Merge-Judge über die Union, annotiert mit Judge-Stimmen ("k× genannt"). - rp = arbeit / f"konsolidierung-c{c}-reconcile.json" - recon = _mapping_schema(_json_datei(rp)) - if recon is None: - rp.unlink(missing_ok=True) - eintraege_r = "\n".join(f"{i}. {form[nt]} ({max(1, kvotes[nt])}× genannt)" for i, nt in enumerate(order, 1)) - status, recon = await run_single_slot( - ctx, f"Konsolidierung Reconcile {c}", - key=f"bausteine-{topic}-konsolidierung-c{c}-reconcile", - prompt=_prompt("Bausteine-Recherche-Mapping", topic=topic, n=KONSOLIDIERUNG_PANEL, eintraege=eintraege_r, out_path=rp), - role="judge", capabilities="files", - payload=lambda result, p=rp: _mapping_schema(_json_datei(p)), - timeout=_timeout("recherche_mapping", len(order)), - ) - if status == CANCELLED: - return None - recon = recon if status != FAILED else None - if recon: - return recon - # Fallback (Reconcile gescheitert): Code-Mehrheit — Konsens, wenn Mehrheit der Judges Konsens sagt. - konsens = [form[nt] for nt in order if kvotes[nt] * 2 >= len(outs) and kvotes[nt] > 0] - kset = {_norm_titel(_titel(t)) for t in konsens} - return konsens, [form[nt] for nt in order if nt not in kset] - - konsens, rest = [], [] - for c, chunk in enumerate(chunks, 1): - eintraege = "\n".join( - f"{i}. {b['titel']} — {b['beschreibung']} ({b['nennungen']}× genannt)" for i, b in enumerate(chunk, 1) - ) - res = await _map_panel(c, eintraege, len(chunk)) - if res is None: - if is_cancelled(): - return False - _bausteine_errors[topic] = "Recherche-Mapping fehlgeschlagen" - return False - k, r = res - konsens += k - rest += r - # Bei mehreren Chunks: ein globaler Merge-Pass über die vereinten Konsens-Einträge, - # damit Dubletten über Chunk-Grenzen (DAL×4, PHPUnit×5 …) verschmelzen. - if len(chunks) > 1 and konsens: - fp = arbeit / "konsolidierung-merge.json" - fp.unlink(missing_ok=True) - eintraege = "\n".join(f"{i}. {t} (2× genannt)" for i, t in enumerate(konsens, 1)) - status, mapping = await run_single_slot( - ctx, "Konsolidierung Merge", - key=f"bausteine-{topic}-konsolidierung-merge", - prompt=_prompt("Bausteine-Recherche-Mapping", topic=topic, n=RECHERCHE_READERS, eintraege=eintraege, out_path=fp), - role="judge", capabilities="files", - payload=lambda result, p=fp: _mapping_schema(_json_datei(p)), - timeout=_timeout("recherche_mapping", len(konsens)), - ) - if status == CANCELLED: - return False - if status != FAILED and mapping: - konsens, r2 = mapping - rest += r2 # vom Merge zurückgestufte Einträge in den Rest - # Judge-Ausgabe ist maßgeblich → Inventar in der DB neu setzen. - await db.delete_bausteine(topic) - for t in konsens: - await _set_inventar(topic, t, "konsens") - for t in rest: - await _set_inventar(topic, t, "rest") - await db.set_step_status(topic, "Konsolidierung", "fertig") - return True - - -async def _klaere_inventar(ctx: GenContext, set_p, files: dict) -> bool: - """Panel (KONSOLIDIERUNG_PANEL Judges) entscheidet über den Rest (1×-Genannte): Mehrheit `aufnehmen` - → Konsens, sonst verworfen. Panel statt Einzel-Judge — der Rest-Schnitt ist der schärfste Eingriff; - ein einzelner Judge ist hier zu instabil. Konservativer Tie → behalten (nie ein Konzept verlieren).""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - if await db.get_step_status(topic, "Klärung") == "fertig": - return True - set_p("Klärung läuft…", step=_step_idx(topic, "Klärung")) - rest_rows = await db.list_bausteine(topic, status="rest") - # Durchgängiges Gate (EDC „Define"): auch Konsens-Bausteine mit Verweis-/Platzhalter-Titel - # prüfen („Satz 7.18", „Korollar 6.18", „Bedingung (**)") — sie umgehen sonst jede Prüfung. - verdaechtig = [b for b in await db.list_bausteine(topic, status="konsens") if _ist_verweis(b["titel"])] - pruef_rows = rest_rows + verdaechtig - if pruef_rows: - arbeit = files["arbeit"] - paths = [arbeit / f"klaerung-j{j}.json" for j in range(1, KONSOLIDIERUNG_PANEL + 1)] - # final=False: ein Judge mit versehentlich nicht-leerem `rest` darf nicht komplett ausfallen - # (sonst kollabiert das Panel auf 1 Judge). Sein `aufnehmen` zählt; rest-Einträge gelten als - # nicht-aufgenommen. Die „rest leer"-Vorgabe steht weiter im Prompt. - offen = [(j, p) for j, p in enumerate(paths, 1) if _runde_schema(_json_datei(p)) is None] - for _, p in offen: - p.unlink(missing_ok=True) - if offen: - slots = [{ - "key": f"bausteine-{topic}-klaerung-j{j}", - "prompt": _prompt( - "Bausteine-Klaerung", topic=topic, - rest="\n".join(f"- {b['titel']} — {b['beschreibung']}" if b['beschreibung'] else f"- {b['titel']}" - for b in pruef_rows), - final="\n- Entscheide JEDEN Eintrag. `rest` MUSS leer sein.", - out_path=p, - ), - "role": "judge", "capabilities": "files", - "payload": (lambda result, p=p: _runde_schema(_json_datei(p))), - } for j, p in offen] - vorhanden = KONSOLIDIERUNG_PANEL - len(offen) - await _race(topic, "Klärung", slots, max(1, 2 - vorhanden), - _timeout("auswahl_mapping", len(pruef_rows)), provider, cancelled=is_cancelled, grace=KONSENS_GRACE) - if is_cancelled(): - return False - outs = [r for p in paths if (r := _runde_schema(_json_datei(p)))] - if not outs: - _bausteine_errors[topic] = "Klärung fehlgeschlagen" - return False - # Mehrheit je Rest-Eintrag (per Norm-Titel). Tie → behalten (votes*2 >= n). - votes: dict[str, int] = {} - for aufnehmen, _ in outs: - for nt in {_norm_titel(_titel(t)) for t in aufnehmen}: - votes[nt] = votes.get(nt, 0) + 1 - # Umbenenn-Vorschläge (additiv aus dem Roh-JSON — _runde_schema kennt das Feld nicht): - # behaltene Verweis-/Platzhalter-Titel → sprechender Name aus dem Inhalt. Alt-Titel-Norm - # bleibt stabil (bricht den votes-Match nicht); je Alt-Titel der häufigste Vorschlag. - umbenenn: dict[str, dict[str, int]] = {} - for p in paths: - d = _json_datei(p) - umb = d.get("umbenennen") if isinstance(d, dict) else None - if isinstance(umb, dict): - for alt, neu in umb.items(): - neu = str(neu).strip() - if neu: - umbenenn.setdefault(_norm_titel(str(alt)), {}).setdefault(neu, 0) - umbenenn[_norm_titel(str(alt))][neu] += 1 - gesehen_norm = {b["titel_norm"] for b in await db.list_bausteine(topic, status="konsens")} - for b in pruef_rows: - auf = votes.get(b["titel_norm"], 0) * 2 >= len(outs) - if not auf: - await db.set_baustein_status(topic, b["titel_norm"], "verworfen") - continue - neu_titel = None - if _ist_verweis(b["titel"]) and (vors := umbenenn.get(b["titel_norm"])): - kand = max(vors, key=lambda k: (vors[k], len(k))) - if not _ist_verweis(kand): - neu_titel = kand - if neu_titel: - nn, t, n = _norm_titel(neu_titel), neu_titel, 2 - while nn in gesehen_norm: - t, nn, n = f"{neu_titel} ({n})", _norm_titel(f"{neu_titel} ({n})"), n + 1 - gesehen_norm.add(nn) - await db.set_baustein_status(topic, b["titel_norm"], "konsens", titel=t, neu_norm=nn) - else: - await db.set_baustein_status(topic, b["titel_norm"], "konsens") - await db.set_step_status(topic, "Klärung", "fertig") - return True - - -def _paare_schema(data) -> dict[int, bool] | None: - """{"paare": {"1": "ja", "2": "nein", …}} → {paar_nr: True/False} · sonst None.""" - if not isinstance(data, dict) or not isinstance(data.get("paare"), dict): - return None - out: dict[int, bool] = {} - for k, v in data["paare"].items(): - try: - nr = int(k) - except (ValueError, TypeError): - continue - out[nr] = str(v).strip().casefold() in ("ja", "yes", "true", "1") - return out or None - - -def _cliquen(n: int, kanten: list[tuple[int, int]]) -> list[list[int]]: - """Complete-Link: greedy maximale Cliquen über die bestätigten Dubletten-Kanten. Eine Gruppe - entsteht nur, wenn ALLE ihre Knoten paarweise verbunden sind → kein Chaining (A=B + B=C bildet - KEINE Gruppe {A,B,C}, solange A=C fehlt). Nur Cliquen ≥2 werden zurückgegeben.""" - adj: dict[int, set[int]] = {i: set() for i in range(n)} - for a, b in kanten: - adj[a].add(b) - adj[b].add(a) - benutzt: set[int] = set() - gruppen: list[list[int]] = [] - for v in sorted(range(n), key=lambda x: -len(adj[x])): - if v in benutzt or not adj[v]: - continue - clique = {v} - for u in sorted(adj[v], key=lambda x: -len(adj[x])): - if u not in benutzt and clique <= adj[u] | {u}: # u mit ALLEN bisherigen verbunden - clique.add(u) - if len(clique) >= 2: - gruppen.append(sorted(clique)) - benutzt |= clique - return gruppen - - -async def _dedup_inventar(ctx: GenContext, set_p, files: dict) -> bool: - """Finaler Dedup-Pass über die fertige Konsens-Liste: Pairwise-Verifikation (Entity - Resolution). Embedding liefert Kandidaten-PAARE (Cosine ≥ DEDUP_PAAR_FLOOR), ein Judge - bestätigt JEDES Paar einzeln (ja = dieselbe Dublette). NUR bestätigte Paare werden zu - Merge-Kanten (Union-Find) — kein Chaining, kein Aspekt-Über-Mergen wie beim Block-Mischer. - Pro Gruppe bleibt EIN Repräsentant (Hauptkonzept), der Rest wird verworfen.""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - if await db.get_step_status(topic, "Dedup") == "fertig": - return True - if not (EMBEDDING_AKTIV and await asyncio.to_thread(embedding.verfuegbar)): - await db.set_step_status(topic, "Dedup", "fertig") # ohne Modell: still überspringen - return True - set_p("Dedup…", step=_step_idx(topic, "Dedup")) - arbeit = files["arbeit"] - konsens = await db.list_bausteine(topic, status="konsens") - if len(konsens) >= 2: - import numpy as np - texts = [f"{b['titel']} — {b['beschreibung']}" if b["beschreibung"] else b["titel"] for b in konsens] - sims = await asyncio.to_thread(embedding.embed_sims, texts) - if sims is not None: - n = len(konsens) - iu = np.triu_indices(n, k=1) - kand = [(int(iu[0][m]), int(iu[1][m])) for m in np.where(sims[iu] >= DEDUP_PAAR_FLOOR)[0]] - _log(topic, f"Dedup: {len(kand)} Kandidaten-Paare (Cosine ≥ {DEDUP_PAAR_FLOOR}) → Pairwise-Filter") - pakete = [kand[i:i + DEDUP_PAARE_CHUNK] for i in range(0, len(kand), DEDUP_PAARE_CHUNK)] - - def paar_path(pi): return arbeit / f"dedup-paar-c{pi}.json" - - async def _filt(pi, paare): - fp = paar_path(pi) - if _paare_schema(_json_datei(fp)): - return # Resume - zeilen = "\n\n".join( - f"{j + 1}.\nA: {konsens[a]['titel']} — {konsens[a]['beschreibung']}" - f"\nB: {konsens[b]['titel']} — {konsens[b]['beschreibung']}" - for j, (a, b) in enumerate(paare)) - await run_single_slot( - ctx, f"Dedup-Paare {pi}", - key=f"bausteine-{topic}-dedup-paar-c{pi}", - prompt=_prompt("Bausteine-Paar-Filter", topic=topic, paare=zeilen, out_path=fp), - role="judge", capabilities="files", - payload=lambda result, p=fp: _paare_schema(_json_datei(p)), - timeout=_timeout("auswahl_mapping", len(paare)), - ) - - await _gather_fortschritt([_filt(pi, p) for pi, p in enumerate(pakete)], - len(pakete), _melde_p(set_p, topic, "Dedup")) - if is_cancelled(): - return False - # Bestätigte "ja"-Kanten sammeln, dann COMPLETE-LINK (greedy Cliquen) statt Single-Link - # Union-Find — verhindert Chaining (A=B + B=C mergt NICHT A,C ohne direktes A=C). - kanten, ja = [], 0 - for pi, paare in enumerate(pakete): - urteil = _paare_schema(_json_datei(paar_path(pi))) or {} - for j, (a, b) in enumerate(paare): - if urteil.get(j + 1): - kanten.append((a, b)) - ja += 1 - gruppen = _cliquen(n, kanten) - weg = 0 - for idxs in gruppen: - # Repräsentant = Hauptkonzept (wenigste Eigenschafts-Marker), dann kürzester Titel. - rep = min(idxs, key=lambda k: (_aspekt_marker(konsens[k]["titel"]), len(konsens[k]["titel"]), k)) - for k in idxs: - if k != rep: - await db.set_baustein_status(topic, konsens[k]["titel_norm"], "verworfen") - weg += 1 - from collections import Counter - atomic_write_json(arbeit / "dedup-runde-1.json", - {"vorher": n, "entfernt": weg, "paare_geprueft": len(kand), "paare_ja": ja, - "clique_groessen": dict(sorted(Counter(len(g) for g in gruppen).items())), - "gruppen": [[konsens[k]["titel"] for k in g] for g in gruppen]}, indent=1) - _log(topic, f"Dedup (pairwise): {n} → {n - weg} (−{weg}); {ja}/{len(kand)} Paare bestätigt") - await db.set_step_status(topic, "Dedup", "fertig") - return True - - -def _filter_schema(data) -> dict[int, int] | None: - """{"fragmente": {"3": 7, "12": 8}} → {baustein_nr: eltern_nr} · None bei ungültiger Struktur. - Leeres dict = gültig (nichts zu degradieren). Eltern ≠ sich selbst.""" - if not isinstance(data, dict) or not isinstance(data.get("fragmente"), dict): - return None - out: dict[int, int] = {} - for k, v in data["fragmente"].items(): - try: - nr, eltern = int(k), int(v) - except (ValueError, TypeError): - continue - if nr != eltern: - out[nr] = eltern - return out - - -# Reine Notation/Symbole ohne eigenständiges Konzept — eng gehalten (FP~0 an aak geprüft; -# „KNF"/„MST"/„NP" treffen NICHT). Diese werden autonom verworfen (brauchen keinen Eltern). -_FILTER_NOTATION = re.compile(r'^\s*\|.{1,6}\|\s*$|^Güte\s+\d+\s*$') -# Eigenschaft-/Laufzeit-Verdacht — markiert Zeilen fürs Judge-Urteil (KEIN Auto-Drop, FP zu hoch: -# „NP-Schwere", Reduktionen mit „∈NP" sind echte Bausteine). Ergänzt _aspekt_marker. -_FILTER_PRAEDIKAT = re.compile( - r'ist NP-(vollständig|schwer)|NP-(Vollständigkeit|Schwere) von|ETH (Konsequenz|Lower Bound)' - r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP', re.I) - - -def _filter_verdacht(b: dict) -> bool: - """Heuristik-Flag: könnte eine Eigenschaft/ein Detail eines anderen Bausteins sein.""" - return _aspekt_marker(b["titel"]) > 0 or bool(_FILTER_PRAEDIKAT.search(f"{b['titel']} {b['beschreibung'] or ''}")) - - -async def _filter_inventar(ctx: GenContext, set_p, files: dict) -> bool: - """Degradier-Pass (Granularität): trennt echte Bausteine von Fragmenten (Eigenschaften, - Beweis-Gadgets, Notation, Laufzeit-Details). Jeder Judge sieht die VOLLE Baustein-Liste - (Self-Containment ist relational) und markiert Fragmente MIT Eltern-Baustein aus der Liste. - Fragment + Eltern-in-Liste → verworfen (Inhalt kommt als Subbaustein des Eltern zurück). - Ohne Eltern oder im Zweifel → behalten (kein Konzept-Verlust).""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - if await db.get_step_status(topic, "Bausteine-Filter") == "fertig": - return True - set_p("Bausteine-Filter…", step=_step_idx(topic, "Bausteine-Filter")) - arbeit = files["arbeit"] - konsens_all = await db.list_bausteine(topic, status="konsens") - # Sicherheitsnetz: reine Notation autonom verwerfen (FP~0, kein Eltern nötig). Der Judge - # übersieht solche Symbole zuverlässig (Recall-Problem), darum deterministisch vorab. - konsens, notation_weg = [], [] - for b in konsens_all: - if _FILTER_NOTATION.search(b["titel"]): - await db.set_baustein_status(topic, b["titel_norm"], "verworfen") - notation_weg.append(b["titel"]) - else: - konsens.append(b) - if notation_weg: - _log(topic, f"Bausteine-Filter: {len(notation_weg)} reine Notation verworfen: {notation_weg[:6]}") - if len(konsens) < 2: - await db.set_step_status(topic, "Bausteine-Filter", "fertig") - return True - n = len(konsens) - # ⚠ markiert Verdachts-Zeilen (Eigenschaft/Laufzeit) — der Judge MUSS sie pro Eintrag prüfen. - def _zeile(i, b): - mark = "⚠ " if _filter_verdacht(b) else "" - return f"{i}. {mark}{b['titel']} — {b['beschreibung']}" if b["beschreibung"] else f"{i}. {mark}{b['titel']}" - voll = "\n".join(_zeile(i, b) for i, b in enumerate(konsens, 1)) - chunks = [list(range(i, min(i + FILTER_CHUNK, n + 1))) for i in range(1, n + 1, FILTER_CHUNK)] - - def filt_path(ci): return arbeit / f"inventar-filter-c{ci}.json" - - async def _beurteile(ci, nummern): - fp = filt_path(ci) - if _filter_schema(_json_datei(fp)) is not None: - return # Resume - await run_single_slot( - ctx, f"Bausteine-Filter {ci}", - key=f"bausteine-{topic}-inventar-filter-c{ci}", - prompt=_prompt("Bausteine-Filter", topic=topic, liste=voll, - von=nummern[0], bis=nummern[-1], out_path=fp), - role="judge", capabilities="files", - payload=lambda result, p=fp: _filter_schema(_json_datei(p)), - timeout=_timeout("auswahl_mapping", len(nummern)), - ) - - await _gather_fortschritt([_beurteile(ci, nm) for ci, nm in enumerate(chunks)], - len(chunks), _melde_p(set_p, topic, "Bausteine-Filter")) - if is_cancelled(): - return False - frag: dict[int, int] = {} - for ci, nummern in enumerate(chunks): - urteil = _filter_schema(_json_datei(filt_path(ci))) or {} - nset = set(nummern) - for nr, eltern in urteil.items(): - if 1 <= eltern <= n and nr in nset: - frag[nr] = eltern - # Ketten-Schutz: ein Baustein, der selbst Eltern eines Fragments ist, bleibt (sein Kind braucht den Anker). - eltern_set = set(frag.values()) - weg, debug = 0, [] - for nr, eltern in frag.items(): - if nr in eltern_set: - continue - b = konsens[nr - 1] - await db.set_baustein_status(topic, b["titel_norm"], "verworfen") - weg += 1 - debug.append({"fragment": b["titel"], "eltern": konsens[eltern - 1]["titel"]}) - atomic_write_json(arbeit / "inventar-filter.json", - {"vorher": n, "degradiert": weg, "fragmente": debug}, indent=1) - _log(topic, f"Bausteine-Filter: {n} → {n - weg} (−{weg} Fragmente → Subbausteine)") - await db.set_step_status(topic, "Bausteine-Filter", "fertig") - return True - - -# --- Gliederung (Bausteine-Artefakt: Kapitel-Struktur, vom Guide nur gelesen) --- - -def _gliederung_komplett(files: dict) -> bool: - """Gliederung steht (Kapitel-Liste vorhanden)?""" - d = _json_datei(files["gliederung"]) - return isinstance(d, dict) and isinstance(d.get("kapitel"), list) and bool(d.get("kapitel")) - - -def _gliederung_schema(data, valid: set[int]): - """{"kapitel":[{titel,nummern}]} → bereinigt (valide Nummern, je genau einmal) · - None bei <80 % Abdeckung (Agent/Judge hat zu viel weggelassen).""" - if not isinstance(data, dict) or not isinstance(data.get("kapitel"), list): - return None - out, seen = [], set() - for ch in data["kapitel"]: - if not isinstance(ch, dict): - continue - titel = str(ch.get("titel", "")).strip() or "Kapitel" - nums = [] - for n in (ch.get("nummern") or []): - try: - n = int(n) - except (ValueError, TypeError): - continue - if n in valid and n not in seen: - seen.add(n) - nums.append(n) - if nums: - out.append({"titel": titel, "nummern": nums}) - if not out or len(seen) < 0.8 * len(valid): - return None - return {"kapitel": out} - - -def _prereq_schema(data, valid: set[int]) -> dict[int, list[int]]: - """{"prereqs": {"3": [1, 7]}} → {num: [prereq-nums]} · nur Nummern aus `valid`, ohne Selbstkante. - Ungültig/leer → {} (Best-effort: dann Original-Reihenfolge).""" - if not isinstance(data, dict) or not isinstance(data.get("prereqs"), dict): - return {} - out: dict[int, list[int]] = {} - for k, v in data["prereqs"].items(): - try: - num = int(k) - except (ValueError, TypeError): - continue - if num not in valid or not isinstance(v, list): - continue - pres = [] - for p in v: - try: - p = int(p) - except (ValueError, TypeError): - continue - if p in valid and p != num and p not in pres: - pres.append(p) - if pres: - out[num] = pres - return out - - -def _topo_order(nums: list[int], edges: dict[int, list[int]]) -> list[int]: - """Kahn-Topo-Sort: Voraussetzungen zuerst. `edges[num]` = Nummern, die VOR num kommen müssen. - Stabiler Tie-Break (Original-Reihenfolge von `nums`); Zyklen werden gebrochen (nie Deadlock).""" - pos = {n: i for i, n in enumerate(nums)} - # Resteingangsgrad nur über gültige Knoten; Selbst-/Fremdkanten ignoriert. - pre = {n: [p for p in edges.get(n, []) if p in pos and p != n] for n in nums} - fertig: list[int] = [] - erledigt: set[int] = set() - rest = list(nums) - while rest: - bereit = [n for n in rest if all(p in erledigt for p in pre[n])] - if not bereit: # Zyklus → den in Original-Reihenfolge frühesten Rest-Knoten erzwingen - bereit = [min(rest, key=lambda n: pos[n])] - nxt = min(bereit, key=lambda n: pos[n]) # stabil: kleinste Original-Position zuerst - fertig.append(nxt) - erledigt.add(nxt) - rest.remove(nxt) - return fertig - - -async def _lernreihenfolge(ctx: GenContext, set_p, files: dict, entries: dict, valid: set[int], instructions: str) -> dict: - """entries (num→titel) in Lernreihenfolge bringen: LLM extrahiert Prereq-Kanten aus den - extrahierten `voraussetzungen`, Code löst per Topo-Sort. Best-effort → sonst entries unverändert.""" - if len(entries) < 3: - return entries - topic = ctx.topic - fakten_map = _json_datei(files["fakten"]) - fakten_map = fakten_map if isinstance(fakten_map, dict) else {} - - def _hint(titel): - fm = fakten_map.get(titel) or {} - vs = [v for fk in fm.values() if isinstance(fk, dict) and (v := str(fk.get("voraussetzungen", "")).strip())] - return " · ".join(dict.fromkeys(vs)) - - pp = files["arbeit"] / "gliederung-prereqs.json" - - def _payload(result, p=pp): - d = _json_datei(p) - return d if isinstance(d, dict) and "prereqs" in d else None - - vorhanden = _json_datei(pp) - if not (isinstance(vorhanden, dict) and "prereqs" in vorhanden): - zeilen = [f"{n}. {t}" + (f"\n braucht vorher: {h}" if (h := _hint(t)) else "") for n, t in entries.items()] - set_p("Gliederung — Lernreihenfolge…", step=_step_idx(topic, "Gliederung")) - await run_single_slot( - ctx, "Gliederung-Voraussetzungen", key=f"bausteine-{topic}-gliederung-prereqs", - prompt=_prompt("Gliederung-Voraussetzungen", topic=topic, bausteine="\n".join(zeilen), out_path=pp, extra=_extra(instructions)), - role="guide", capabilities="files", payload=_payload, timeout=_timeout("plan", len(entries))) - edges = _prereq_schema(_json_datei(pp), valid) - if not edges: - return entries # keine/ungültige Kanten → Original-Reihenfolge (kein Regress) - ordered = _topo_order(list(entries), edges) - return {n: entries[n] for n in ordered} - - -async def _gliederung_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict: - """Format-agnostische Gliederung über ALLE Bausteine — 3 Vorschläge → Judge merged. - Bricht nie ab: 0 gültige → ein Kapitel mit allem; fehlende Bausteine landen in „Weitere". - → {"kapitel":[{titel,nummern}]} (auch in files["gliederung"]).""" - topic, is_cancelled = ctx.topic, ctx.is_cancelled - valid = set(entries) - step = _step_idx(topic, "Gliederung") - - # Lernreihenfolge gründen (LLM-Modulo): LLM extrahiert Prereq-Kanten aus den extrahierten - # `voraussetzungen`, Code löst per Topo-Sort. Best-effort → sonst Original-Reihenfolge. - entries = await _lernreihenfolge(ctx, set_p, files, entries, valid, instructions) - liste = "\n".join(f"{n}. {t}" for n, t in entries.items()) - set_p("Gliederung — Vorschläge…", step=step) - - async def _vorschlag(i, path): - if _gliederung_schema(_json_datei(path), valid): - return True - await run_single_slot( - ctx, f"Gliederung {i}", key=f"bausteine-{topic}-gliederung-{i}", - prompt=_prompt("Guide-Gliederung", topic=topic, bausteine=liste, out_path=path, extra=_extra(instructions)), - role="guide", capabilities="files", - payload=lambda result, p=path: _gliederung_schema(_json_datei(p), valid), - timeout=_timeout("plan", len(entries))) - return _gliederung_schema(_json_datei(path), valid) is not None - - slots = files["gliederung_slots"] - await _gather_fortschritt([_vorschlag(i, p) for i, p in enumerate(slots, 1)], len(slots), _melde_p(set_p, topic, "Gliederung")) - if is_cancelled(): - return {} - vorschlaege = [v for p in slots if (v := _gliederung_schema(_json_datei(p), valid))] - - if not vorschlaege: - plan = {"kapitel": [{"titel": "Inhalte", "nummern": list(entries)}]} - elif len(vorschlaege) == 1: - plan = vorschlaege[0] - else: - set_p("Gliederung zusammenführen…", step=step) - bloecke = "\n\n".join( - f"### Vorschlag {i}\n" + "\n".join( - f"KAPITEL: {ch['titel']}\n Nummern: {', '.join(str(n) for n in ch['nummern'])}" for ch in v["kapitel"]) - for i, v in enumerate(vorschlaege, 1)) - await run_single_slot( - ctx, "Gliederung-Judge", key=f"bausteine-{topic}-gliederung-judge", - prompt=_prompt("Guide-Gliederung-Judge", topic=topic, format_name="den Guide", - zweck="alle Bausteine in einem roten Faden", n=len(vorschlaege), - bausteine=liste, gliederungen=bloecke, out_path=files["gliederung"], extra=_extra(instructions)), - role="judge", capabilities="files", - payload=lambda result: _gliederung_schema(_json_datei(files["gliederung"]), valid), - timeout=_timeout("plan_judge", len(entries))) - plan = _gliederung_schema(_json_datei(files["gliederung"]), valid) or vorschlaege[0] - - # Vollständigkeit: jeder Baustein kommt vor — fehlende in „Weitere" (gegen weglassende Agenten/Judge). - drin = {n for ch in plan["kapitel"] for n in ch["nummern"]} - fehlen = [n for n in entries if n not in drin] - if fehlen: - plan["kapitel"].append({"titel": "Weitere", "nummern": fehlen}) - atomic_write_json(files["gliederung"], plan, indent=1) - return plan - - -# --- Lern-Artefakte (Karteikarten/Beispiele aus den Fakten) --- - -def _karten_schema(data): - """{"karten":[{baustein,subbaustein,frage,antwort}]} → Liste (auch leer) · None bei kaputt.""" - if not isinstance(data, dict) or not isinstance(data.get("karten"), list): - return None - out = [] - for e in data["karten"]: - if isinstance(e, dict) and (f := str(e.get("frage", "")).strip()) and (a := str(e.get("antwort", "")).strip()): - out.append({"baustein": str(e.get("baustein", "")).strip(), "subbaustein": str(e.get("subbaustein", "")).strip(), - "frage": f, "antwort": a}) - return out - - -def _beispiel_schema(data): - """{"beispiele":[{baustein,subbaustein,problem,schritte,ergebnis}]} → Liste (auch leer) · None bei kaputt.""" - if not isinstance(data, dict) or not isinstance(data.get("beispiele"), list): - return None - out = [] - for e in data["beispiele"]: - if not isinstance(e, dict): - continue - problem = str(e.get("problem", "")).strip() - schritte = [s for x in (e.get("schritte") or []) if (s := str(x).strip())] - if problem and schritte: - out.append({"baustein": str(e.get("baustein", "")).strip(), "subbaustein": str(e.get("subbaustein", "")).strip(), - "problem": problem, "schritte": schritte, "ergebnis": str(e.get("ergebnis", "")).strip()}) - return out - - -def _beispiel_check_schema(data): - """Worked-Example-Check → {"ok": true} → set() (alles korrekt); {"probleme":[{"index":N}]} → - {N, …} (1-basierte beanstandete Indizes); None bei kaputt.""" - if not isinstance(data, dict): - return None - if data.get("ok") is True: - return set() - pr = data.get("probleme") - if not isinstance(pr, list): - return None - out: set[int] = set() - for p in pr: - if isinstance(p, dict): - try: - out.add(int(p.get("index"))) - except (ValueError, TypeError): - continue - return out - - -_ARTEFAKT_SCHEMA = {"karteikarte": _karten_schema, "beispiel": _beispiel_schema} -_ARTEFAKT_PROMPT = {"karteikarte": "Artefakt-Karteikarte", "beispiel": "Artefakt-Beispiel"} -_ARTEFAKT_SCHRITT = {"karteikarte": "Karteikarten", "beispiel": "Beispiele"} - - -def _artefakte_komplett(files: dict) -> bool: - """Artefakt-Map steht (alle Typen erzeugt)? Werte dürfen leer sein (content-aware).""" - d = _json_datei(files["artefakte"]) - return isinstance(d, dict) and all(t in d for t in ARTEFAKT_TYPEN) - - -async def _artefakte_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: - """Lern-Artefakte je Typ aus den gespeicherten Fakten erzeugen — ein Generierungs-Durchlauf - je Typ über Chunks. Worked Examples werden gegen die Fakten verifiziert (falsche verworfen); - Karteikarten sind risikoarm und bleiben ungeprüft. → {typ: [eintraege]} (auch in files).""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - arbeit = files["arbeit"] - caps = "files" - # Bausteine mit Subs + Fakten-Zeilen als Input-Block (extract-once aus den Fakten). - bausteine = [] - for btitel, subs in sidecar.items(): - if not isinstance(subs, list): - continue - zeilen = [] - for s in subs: - if not isinstance(s, dict) or not (st := str(s.get("titel", "")).strip()): - continue - fk = s.get("fakten") if isinstance(s.get("fakten"), dict) else {} - zeile = f"- {st}" - if fk and (fk_text := _fakten_zeilen(fk)): - zeile += "\n" + "\n".join(" " + l for l in fk_text.split("\n")) - zeilen.append(zeile) - if zeilen: - bausteine.append((btitel, zeilen)) - if not bausteine: - leer = {t: [] for t in ARTEFAKT_TYPEN} - atomic_write_json(files["artefakte"], leer, indent=1) - return leer - - chunks = _lpt_chunks([len(z) for _, z in bausteine], FAKTEN_CHUNK_SUBS) - def block_text(idxs): - return "\n\n".join(f"BAUSTEIN: {bausteine[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(bausteine[i][1]) for i in idxs) - - # Worked Examples gegen die Fakten prüfen (Panel-Mehrheit) — falsche verwerfen. CoT-Schritte sind - # fehleranfällig; ein falsches Beispiel prägt ein fehlerhaftes Schema ein → kein Beispiel > falsches. - async def _pruefe_beispiele(ci, idxs, items): - if is_cancelled() or not items: - return items - def cpath(j): return arbeit / f"artefakt-beispiel-check-c{ci}-j{j}.json" - beispiele_txt = "\n\n".join( - f"{k}. PROBLEM: {e['problem']}\n SCHRITTE: " + " | ".join(e.get("schritte", [])) - + (f"\n ERGEBNIS: {e['ergebnis']}" if e.get("ergebnis") else "") - for k, e in enumerate(items, 1)) - offen = [j for j in (1, 2, 3)[:FAKTEN_CHECK_PANEL] if _beispiel_check_schema(_json_datei(cpath(j))) is None] - if offen: - await asyncio.gather(*[ - run_agent(f"bausteine-{topic}-artefakt-beispiel-check-c{ci}-j{j}", - _prompt("Artefakt-Beispiel-Check", topic=topic, fakten=block_text(idxs), beispiele=beispiele_txt, out_path=cpath(j), extra=_extra(instructions)), - _timeout("inhalt_check", len(items)), provider=provider, role="judge", capabilities=caps) - for j in offen], return_exceptions=True) - outs = [s for j in (1, 2, 3)[:FAKTEN_CHECK_PANEL] if (s := _beispiel_check_schema(_json_datei(cpath(j)))) is not None] - if not outs: - return items # keine Prüfung möglich → behalten (Best-effort) - votes: dict[int, int] = {} - for s in outs: - for idx in s: - votes[idx] = votes.get(idx, 0) + 1 - schwelle = len(outs) / 2 - raus = {idx for idx, v in votes.items() if v > schwelle} # Mehrheit (≥2 von 3) beanstandet → raus - if raus: - _log(topic, f"Worked-Example-Check Chunk {ci}: {len(raus)}/{len(items)} verworfen") - return [e for k, e in enumerate(items, 1) if k not in raus] - - ergebnis: dict[str, list] = {} - for typ in ARTEFAKT_TYPEN: - schema = _ARTEFAKT_SCHEMA[typ] - def apath(ci, t=typ): return arbeit / f"artefakt-{t}-c{ci}.json" - - async def _gen(ci, idxs, t=typ, schema=schema): - p = apath(ci, t) - if schema(_json_datei(p)) is not None: - return True - await run_single_slot( - ctx, f"{_ARTEFAKT_SCHRITT[t]} {ci}", key=f"bausteine-{topic}-artefakt-{t}-c{ci}", - prompt=_prompt(_ARTEFAKT_PROMPT[t], topic=topic, bausteine=block_text(idxs), out_path=p, extra=_extra(instructions)), - role="guide", capabilities="files", - payload=lambda result, p=p, schema=schema: schema(_json_datei(p)), - timeout=_timeout("inhalt", sum(len(bausteine[i][1]) for i in idxs))) - return schema(_json_datei(p)) is not None - - await _gather_fortschritt([_gen(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _melde_p(set_p, topic, _ARTEFAKT_SCHRITT[typ])) - if is_cancelled(): - return None - eintraege: list = [] - for ci in range(len(chunks)): - chunk_items = schema(_json_datei(apath(ci))) or [] - if typ == "beispiel" and chunk_items: - chunk_items = await _pruefe_beispiele(ci, chunks[ci], chunk_items) - eintraege += chunk_items - ergebnis[typ] = eintraege - atomic_write_json(files["artefakte"], ergebnis, indent=1) - return ergebnis - - -async def _mirror_artefakte_db(topic: str, sidecar: dict, artefakte: dict) -> None: - """Artefakte in die DB spiegeln. Karteikarte/Beispiel je Sub (sub_norm).""" - await db.delete_sub_artefakte(topic) - btitel_list = list(sidecar.keys()) - for typ in ARTEFAKT_TYPEN: - for e in artefakte.get(typ, []): - bt = _match_sub(e.get("baustein", ""), btitel_list) - bnorm, sn = _norm_titel(bt), _norm_titel(e.get("subbaustein", "")) - if not bnorm or not sn: - continue - daten = json.dumps({k: v for k, v in e.items() if k not in ("baustein", "subbaustein")}, ensure_ascii=False) - await db.put_sub_artefakt(topic, bnorm, sn, typ, daten, bt, e.get("subbaustein", "")) - - -async def _mirror_sidecar_db(topic: str, sidecar: dict) -> None: - """Sidecar {Baustein-Titel: [{titel, stufe, relevanz}]} in die DB-Tabelle subbausteine spiegeln.""" - for btitel, subs in sidecar.items(): - bnorm = _norm_titel(btitel) - if not bnorm or not isinstance(subs, list): - continue - for s in subs: - if not isinstance(s, dict): - continue - st = str(s.get("titel", "")).strip() - sn = _norm_titel(st) - if not sn: - continue - fakten = json.dumps(s["fakten"], ensure_ascii=False) if isinstance(s.get("fakten"), dict) else None - await db.put_subbaustein(topic, bnorm, sn, btitel, st, - stufe=s.get("stufe"), relevanz=s.get("relevanz"), - fakten=fakten, status="konsens") - - -async def _mirror_frage_muster_db(topic: str, muster: dict) -> None: - """Frage-Muster {Baustein-Titel: [{subbaustein, frage}]} in die DB-Tabelle frage_muster spiegeln.""" - await db.delete_frage_muster(topic) - for btitel, eintraege in muster.items(): - bnorm = _norm_titel(btitel) - if not bnorm or not isinstance(eintraege, list): - continue - for e in eintraege: - if not isinstance(e, dict): - continue - sub = str(e.get("subbaustein", "")).strip() - sn = _norm_titel(sub) - frage = str(e.get("frage", "")).strip() - if not (sn and frage): - continue - await db.upsert_frage_muster(topic, bnorm, sn, btitel, sub, frage) - - -async def _reset_db_ab_phase(topic: str, label: str) -> None: - """DB-Inhalt der Phasen ≥ `label` verwerfen (kanonische Reihenfolge Quelle…Artefakte).""" - idx = _phase_idx(label) - if idx <= 8: # Artefakte (Karteikarten/Beispiele) - await db.delete_sub_artefakte(topic) - if idx <= 7: # Fragen - await db.delete_frage_muster(topic) - if idx <= 6: # Gliederung - await db.delete_gliederung(topic) - if idx <= 2: # Subbausteine (Fakten/Stufen/Relevanz greifen über Sidecar→Mirror) - await db.delete_subbausteine(topic) - if idx <= 1: # Inventar: Inventar + Recherche-Schritte — Sichtung bleibt - await db.delete_bausteine(topic) - await db.delete_pipeline_state(topic, ["Recherche", "Konsolidierung", "Klärung", "Dedup", "Bausteine-Filter"]) - if idx <= 0: # Quelle: Sichtung neu (Coverage/inhalt + Schritt) - await db.delete_coverage(topic) - await db.delete_pipeline_state(topic, ["Quelle aufbereiten"]) - - -async def generate_bausteine(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, ab_phase: int | None = None, ab_step: int | None = None) -> None: - if topic in _bausteine_progress: - return - _bausteine_progress[topic] = "Wartend…" - _bausteine_errors.pop(topic, None) - - files = _bausteine_files(topic) - final_path = files["final"] - q = lade_quelle(topic) - ordner = quelle_ordner(topic) # projekt/uni/link → Ordner, thema → None - instructions = q.get("spec") or instructions # persistierte Spezifikation bevorzugen (auch bei Resume) - - def set_p(msg: str, step: int | None = None) -> None: - _bausteine_progress[topic] = msg - if step is not None: - _bausteine_step[topic] = step - - def is_cancelled() -> bool: - return topic in _bausteine_cancelled - - def abgebrochen() -> None: - _bausteine_errors[topic] = "Abgebrochen — Fortschritt bleibt erhalten" - - ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled) - - try: - async with _semaphore: - files["arbeit"].mkdir(parents=True, exist_ok=True) - # Re-Run ab gewählter Phase: Artefakte ab dort löschen; der Frischstart-Block - # unten wird übersprungen (er würde bei erhaltener Sidecar sonst alles wischen). - if ab_step is not None: # feiner Teilschritt-Re-Run (Vorrang vor ab_phase) - await _reset_ab_step(topic, ab_step) - elif ab_phase is not None: - phasen = _phasen(topic) - label = phasen[ab_phase - 1][0] if 1 <= ab_phase <= len(phasen) else "Inventar" - _reset_ab_phase(topic, label) - await _reset_db_ab_phase(topic, label) - # Schritt „Quelle aufbereiten": Crawl (link) + PDFs + Content/Noise-Sichtung. - if not await _quelle_aufbereiten(ctx, set_p, files, q, ordner, instructions): - if is_cancelled(): - abgebrochen() - return - # „Neu erstellen": NUR wenn wirklich alles fertig ist (bausteine.md UND - # Sidecar) → kompletter Frischstart. Liegt bausteine.md ohne Sidecar vor, - # ist das ein Teil-Stand (Block B/C offen) → Resume, nicht wischen. - # Bei explizitem Re-Run (ab_phase) hat _reset_ab_phase das schon erledigt. - fertig = ab_phase is None and ab_step is None and final_path.exists() and _sidecar_schema(_json_datei(files["sidecar"])) is not None - if fertig: - for p_alt in _alle_slot_dateien(files): - p_alt.unlink(missing_ok=True) - await db.delete_pipeline_state(topic) - await db.delete_bausteine(topic) - await db.delete_subbausteine(topic) - await db.delete_frage_muster(topic) - await db.delete_coverage(topic) - await db.delete_gliederung(topic) - await db.delete_sub_artefakte(topic) - - # Inventar (DB): Recherche-Loop → Konsolidierung → Klärung. - if not await _recherche_batch(ctx, set_p, files, q, ordner, instructions): - if is_cancelled(): - abgebrochen() - return - if not await _konsolidiere(ctx, set_p, files): - if is_cancelled(): - abgebrochen() - return - if not await _klaere_inventar(ctx, set_p, files): - if is_cancelled(): - abgebrochen() - return - if not await _dedup_inventar(ctx, set_p, files): - if is_cancelled(): - abgebrochen() - return - if not await _filter_inventar(ctx, set_p, files): - if is_cancelled(): - abgebrochen() - return - konsens_rows = await db.list_bausteine(topic, status="konsens") - entries = { - i: (f"{b['titel']} — {b['beschreibung']}" if b["beschreibung"] else b["titel"]) - for i, b in enumerate(konsens_rows, 1) - } - - # Nur Projekte: Themenfeld-Ergänzung — Skript/Projekt ist ein Ausschnitt, - # ein Web-Agent ergänzt kanonisch fehlende Bausteine, markiert mit [Ergänzung]. - if q["type"] == "projekt": - set_p("Ergänze Themenfeld…", step=_step_idx(topic, "Ergänzung")) - erg_path = files["ergaenzung"] - ergaenzungen = _ergaenzung_schema(_json_datei(erg_path)) - if ergaenzungen is None: - erg_path.unlink(missing_ok=True) - status, ergaenzungen = await run_single_slot( - ctx, "Ergänzung", - key=f"bausteine-{topic}-ergaenzung-1", - prompt=_prompt( - "Bausteine-Ergaenzung", - topic=topic, bausteine="\n".join(f"- {t}" for t in entries.values()), - out_path=erg_path, extra=_extra(instructions), - ), - role="quick", capabilities="full", - payload=lambda result: _ergaenzung_schema(_json_datei(erg_path)), - timeout=_timeout("ergaenzung"), - ) - if status == CANCELLED: - abgebrochen() - return - if status == FAILED: - _bausteine_errors[topic] = "Ergänzung fehlgeschlagen (kein gültiges Ergebnis)" - return - idx = _titel_index(entries) - neu = [(t, b) for t, b in ergaenzungen if _titel_aufloesen(idx, t) is None] - if neu: - _log(topic, f"Ergänzung: {len(neu)} Baustein(e) aus dem Themenfeld ergänzt") - start = max(entries, default=0) + 1 - for off, (t, b) in enumerate(neu): - entries[start + off] = f"{t} — {b} [Ergänzung]" - - # Titel eindeutig machen und unsortiertes Inventar schreiben - entries = _eindeutige_titel(entries) - atomic_write_text(final_path, "\n".join(f"{i}. {t}" for i, t in entries.items()) + "\n") - - # Block B + C: Subbausteine je Baustein + Stufen → Sidecar subbausteine.json. - # Nicht-destruktiv: bausteine.md steht schon; fehlt die Sidecar, wird beim - # nächsten Lauf nur dieser Teil neu versucht. Guide fällt ohne Sidecar zurück. - if _sidecar_schema(_json_datei(files["sidecar"])) is None: - roh = _sub_roh_schema(_json_datei(files["sub_roh"])) - if roh is None: - roh = await _subbausteine_block(ctx, set_p, files, entries, instructions) - if is_cancelled(): - abgebrochen() - return - if roh is None: - return # Fehler ist gesetzt - atomic_write_json(files["sub_roh"], roh, indent=1) - # Fakten je Sub (VOR der Stufe): Quell-Fakten extrahieren + verifizieren → fakten.json. - # Extract-once-Grounding — Stufe/Relevanz/Fragen/Guide nähren sich daraus. - if not _fakten_komplett(files): - res = await _fakten_block(ctx, set_p, files, roh, q, ordner, instructions) - if is_cancelled(): - abgebrochen() - return - if res is None: - return # Fehler ist gesetzt - fakten_map, verworfen = res - # Verworfene (unbelegbare) Subs aus roh streichen — ZUERST (Resume-robust), dann - # fakten.json. So sehen Stufen/Relevanz/Gliederung/Fragen/Guide sie nicht mehr. - if verworfen: - for bt, sns in verworfen.items(): - if bt in roh: - roh[bt] = [s for s in roh[bt] if _norm_titel(s) not in sns] - roh = {bt: subs for bt, subs in roh.items() if subs} # leere Bausteine raus (_sub_roh_schema verlangt ≥1) - atomic_write_json(files["sub_roh"], roh, indent=1) - atomic_write_json(files["fakten"], fakten_map, indent=1) - sidecar = await _stufen_block(ctx, set_p, files, roh, instructions) - if is_cancelled(): - abgebrochen() - return - if sidecar is None: - return - # Fakten in die Sidecar-Subs mergen (DB-Spiegel + Guide-Nutzung). - fakten_map = _json_datei(files["fakten"]) - if isinstance(fakten_map, dict): - for btitel, subs in sidecar.items(): - fm = fakten_map.get(btitel, {}) - for sub in subs: - if (fk := fm.get(_norm_titel(sub["titel"]))): - sub["fakten"] = fk - atomic_write_json(files["sidecar"], sidecar, indent=1) - - # Block D: Relevanz je Subbaustein (relevant/rand) → in die Sidecar mergen. - # Eigene Phase nach den Stufen; treibt das ProGuide-Format (alle Bausteine - # mit ≥1 relevantem Subbaustein) und filtert Rand-Subs aus den Guides. - sidecar = _json_datei(files["sidecar"]) - if _sidecar_schema(sidecar) is not None and not _relevanz_komplett(sidecar): - relevanz_by_id = await _relevanz_block(ctx, set_p, files, sidecar, instructions) - if is_cancelled(): - abgebrochen() - return - if relevanz_by_id is None: - return # Fehler ist gesetzt - gid = 0 - for subs in sidecar.values(): - for sub in subs: - gid += 1 - sub["relevanz"] = relevanz_by_id.get(gid, "relevant") - atomic_write_json(files["sidecar"], sidecar, indent=1) - - # Block D.5: Gliederung (Bausteine-Artefakt) — Kapitel-Struktur über ALLE Bausteine, - # vom Guide nur noch gelesen. Format-agnostisch; der Guide filtert je Format. - if not _gliederung_komplett(files): - await _gliederung_block(ctx, set_p, files, entries, instructions) - if is_cancelled(): - abgebrochen() - return - - # Block E: Frage-Muster je relevantem Subbaustein × Typ → eigenes Sidecar. - # Zur Prüfungszeit zieht jeder Agent ein Muster ohne Zurücklegen und formuliert - # daraus eine Frage — distinkte Saat verhindert die Doppelfragen der Live-Generierung. - sidecar = _json_datei(files["sidecar"]) - if _sidecar_schema(sidecar) is not None and _relevanz_komplett(sidecar) and not _frage_muster_komplett(topic): - muster = await _frage_muster_block(ctx, set_p, files, sidecar, instructions) - if is_cancelled(): - abgebrochen() - return - if muster is None: - return # Abbruch - atomic_write_json(files["frage_muster"], muster, indent=1) - - # Block F: Lern-Artefakte (Karteikarten/Beispiele) aus den Fakten — Bonus, - # vom Frontend präsentiert. Bricht den Lauf nicht ab (Artefakte sind optional). - sidecar = _json_datei(files["sidecar"]) - if _sidecar_schema(sidecar) is not None and not _artefakte_komplett(files): - artefakte = await _artefakte_block(ctx, set_p, files, sidecar, instructions) - if is_cancelled(): - abgebrochen() - return - if artefakte is None: - return # Abbruch (Fehler/Cancel) - - # DB-Spiegel (Brücke): finalen Sidecar- + Frage-Muster-Stand in die DB schreiben. - sidecar = _json_datei(files["sidecar"]) - if _sidecar_schema(sidecar) is not None: - await _mirror_sidecar_db(topic, sidecar) - muster = _json_datei(files["frage_muster"]) - if isinstance(muster, dict) and muster: - await _mirror_frage_muster_db(topic, muster) - # Gliederung (Titel-basiert, robust gegen Nummern-Drift) → DB. - plan = _json_datei(files["gliederung"]) - if isinstance(plan, dict) and plan.get("kapitel"): - kapitel = [ - {"titel": ch.get("titel", "Kapitel"), - "bausteine": [_titel(entries[n]) for n in ch.get("nummern", []) if n in entries]} - for ch in plan["kapitel"] - ] - await db.set_gliederung(topic, json.dumps({"kapitel": kapitel}, ensure_ascii=False)) - # Artefakte → DB (Karteikarte/Beispiel je Sub, Diagramm je Baustein). - artefakte = _json_datei(files["artefakte"]) - if isinstance(artefakte, dict) and _sidecar_schema(sidecar) is not None: - await _mirror_artefakte_db(topic, sidecar, artefakte) - except Exception as e: - log.exception("[%s] Bausteine-Generierung fehlgeschlagen", topic) - _bausteine_errors[topic] = str(e)[:2000] - finally: - # Kein Datei-Cleanup: Zwischendateien bleiben für Resume bzw. Nachvollziehbarkeit. - _bausteine_progress.pop(topic, None) - _bausteine_step.pop(topic, None) - _bausteine_cancelled.discard(topic) - clear_scope(f"bausteine-{topic}-") # Scope leeren → Neustart blockiert nicht diff --git a/backend/blocks.py b/backend/blocks.py new file mode 100644 index 0000000..9af8b25 --- /dev/null +++ b/backend/blocks.py @@ -0,0 +1,3326 @@ +"""Blocks pipeline: research consensus + clarification loop — pure inventory, unsorted. + +5x research (min. 3, grace) → mapping (consensus/rest) → clarification loop (max. +CONSENSUS_MAX_ROUNDS rounds): 3 selection agents (min. 2, grace) decide +on the disputed rest, a mapping agent sorts into accept/discard/ +still disputed. An empty rest ends the loop; the last round must decide +everything. Races use a grace window instead of "first N win": after the +first valid result, the remaining agents get CONSENSUS_GRACE seconds to +finish. The consensus is accumulated in code — no agent re-emits +the full list. +""" + +import asyncio +import json +import logging +import math +import re +import shutil +import subprocess +import time +from pathlib import Path + +import database as db +import embedding +from agents import kill_process, cancel_scope, clear_scope, run_agent +from config import CONSENSUS_GRACE, RESEARCH_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP +from fsutil import atomic_write_text, atomic_write_json +from jsonio import read_json_file as _json_file +from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder +from crawl import crawl +from pipeline import ( + CANCELLED, FAILED, OK, GenContext, _extra, _gather_progress, _yesno_schema, _log, _prompt, _race, + _relevance_schema, _runde_schema, _semaphore, _str_list, _levels_schema, _timeout, run_single_slot, +) +from textkit import ( + _unique_title, _load_blocks, _norm_title, _parse_selection, _parse_subblocks, _title, + _resolve_title, _title_index, +) + +# Chunk the subblocks (web search per block): 1 agent per ~10 blocks, capped. +SUBBLOCK_CHUNK = 10 +SUBBLOCK_MAX = 40 +# Classifying is cheap (short verdict, no web search) → larger packages, fewer files/agents. +LEVEL_CHUNK = 100 + +# Research: fixed file batches instead of a search loop → each crawl page is assigned exactly once. +RESEARCH_BATCH = 20 # crawl pages per batch +RESEARCH_READERS = 2 # reader agents per batch (consensus ≥2 within the batch) +RESEARCH_THEMA_AGENTS = 5 # web mode (source "thema", no crawl folder) +# uni/projekt: chunk the script text into sections of ~this size (against lost-in-the-middle on +# large documents). ~12k chars ≈ 3k tokens → safely below the recall-drop threshold. +RESEARCH_SECTION_CHARS = 12000 +# Triage (content/noise) is now a deterministic rule filter (config.CRAWL_*). +SUBBLOCK_CAP = 900 # subblock find loop per chunk (15 min) +CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (dedups everything); above that chunked + merge pass — fallback path only +DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair (complete-link aggregates → no chaining) +DEDUP_PAIRS_CHUNK = 40 # pairs per judge package (pairwise verification instead of a block mixer) +FILTER_CHUNK = 35 # blocks to assess per judge in the degrade pass (full list as context) +# Balance question-pattern chunks by sub load via LPT (makespan), not by block count. +QUESTION_CHUNK_SUBS = 50 # target sum of relevant subs per chunk +QUESTION_MAX_ROUNDS = 3 # catch-up rounds for subs without a pattern (the LLM omits ~18 % per chunk) +FACTS_CHUNK_SUBS = 25 # facts extraction: smaller chunks (facts are bulkier than patterns) +FACTS_CHECK_PANEL = 3 # judges per chunk in the facts check (majority objects) +CONSOLIDATION_PANEL = 3 # mapping judges per chunk (panel → reconcile instead of a single judge) +SUBBLOCK_PANEL = 3 # source judges in the subblock clarification (majority instead of a single judge) + +log = logging.getLogger("creator.blocks") + +_blocks_progress: dict[str, str] = {} +_blocks_errors: dict[str, str] = {} +_blocks_cancelled: set[str] = set() +_blocks_step: dict[str, int] = {} + +ARTEFACT_TYPES = ("flashcard", "example") + + +def load_source(topic: str) -> dict: + """Read the persisted source choice. Fallback (legacy topics without source.json): + if projects/ exists → projekt, otherwise thema.""" + q = _json_file(source_path(topic)) + if isinstance(q, dict) and q.get("type") in ("thema", "projekt", "uni", "link"): + return q + if project_dir(topic).is_dir(): + return {"type": "projekt", "location": f"projects/{topic}", "spec": ""} + return {"type": "thema", "location": "", "spec": ""} + + +def source_folder(topic: str) -> Path | None: + """Folder source (projekt/uni → path, link → crawl folder) — otherwise None (thema).""" + q = load_source(topic) + if q["type"] == "link": + return source_crawl_dir(topic) + if q["type"] in ("projekt", "uni"): + return safe_folder(q.get("location", "")) + return None + + +def _crawl_done(topic: str) -> bool: + return (source_crawl_dir(topic) / ".done").exists() # marker only on clean completion + + +# Learning-path levels (beginner/advanced/expert); old difficulty values are backward-compatible. +_LEVELS = ("beginner", "advanced", "expert", "easy", "medium", "hard") + + +async def subblocks_title(topic: str, block: str) -> list[str]: + """Subblock titles of a block — DB-first (consensus), fallback to the sidecar file.""" + rows = [s["sub_title"] for s in await db.list_subblocks(topic, _norm_title(block)) + if s["status"] == "consensus" and s["sub_title"]] + if rows: + return rows + sc = _json_file(subblocks_path(topic)) + if not isinstance(sc, dict): + return [] + return [ + t for s in (sc.get(block) or []) + if isinstance(s, dict) and (t := str(s.get("title", "")).strip()) + ] + + +async def load_question_pattern(topic: str, block: str) -> list[dict]: + """Predefined question patterns of a block — DB-first, fallback to sidecar (empty = live).""" + rows = await db.list_question_pattern(topic, _norm_title(block)) + if rows: + return [{"subblock": r["sub_title"], "question": r["question"]} for r in rows if r["question"]] + fm = _json_file(question_pattern_path(topic)) + if not isinstance(fm, dict): + return [] + return [ + {"subblock": str(e.get("subblock", "")).strip(), "question": question} + for e in (fm.get(block) or []) + if isinstance(e, dict) and (question := str(e.get("question", "")).strip()) + ] + + +async def subblocks_frei(topic: str, block: str, max_level: int) -> list[str]: + """Subblock titles up to the unlocked level (≤ max_level). Fallback without + level knowledge (legacy/sidecar): all subblock titles.""" + rows = await db.subs_with_level(topic, block) + if not rows: + return await subblocks_title(topic, block) + return [s["title"] for s in rows if s["level"] <= max_level and s["title"]] + + +async def load_question_pattern_free(topic: str, block: str, max_level: int) -> list[dict]: + """Question patterns, filtered to subblocks up to the unlocked level. Without + level knowledge (legacy/sidecar), unfiltered.""" + rows = await db.subs_with_level(topic, block) + if not rows: + return await load_question_pattern(topic, block) + unlocked = {s["norm"] for s in rows if s["level"] <= max_level} + return [m for m in await load_question_pattern(topic, block) if _norm_title(m["subblock"]) in unlocked] + + +async def load_overview(topic: str) -> list[dict]: + """Structured block list for the overview — DB-first (consensus + subs/levels/relevance), + fallback to blocks.md + sidecar (legacy topics).""" + bs = await db.list_blocks(topic, status="consensus") + if bs: + out = [] + for num, b in enumerate(bs, 1): + subs = [s for s in await db.list_subblocks(topic, b["title_norm"]) if s["status"] == "consensus"] + out.append({ + "num": num, "title": b["title"], "description": b["description"], + "subblocks": [ + {"title": s["sub_title"], + "level": s["level"] if s["level"] in _LEVELS else "advanced", + "relevance": s["relevance"] if s["relevance"] in ("relevant", "peripheral") else None} + for s in subs if s["sub_title"] + ], + }) + return out + entries = _load_blocks(_read(blocks_path(topic))) + sidecar = _json_file(subblocks_path(topic)) + sidecar = sidecar if isinstance(sidecar, dict) else {} + out = [] + for num, entry in entries.items(): + title = _title(entry) + split_parts = entry.split(" — ", 1) + description = split_parts[1].strip() if len(split_parts) == 2 else "" + subblocks = [ + { + "title": t, + "level": s.get("level") if s.get("level") in _LEVELS else "advanced", + "relevance": s.get("relevance") if s.get("relevance") in ("relevant", "peripheral") else None, + } + for s in (sidecar.get(title) or []) + if isinstance(s, dict) and (t := str(s.get("title", "")).strip()) + ] + out.append({"num": num, "title": title, "description": description, "subblocks": subblocks}) + return out + + +def _blocks_steps(topic: str) -> tuple: + """Steps per source: link gets "Source laden" up front, projekt additionally "Supplement". + + Subblocks + levels are three phases each (find, select, clarify). Per phase + all packages run in parallel; the step remains until the last package is done. + """ + q = load_source(topic) + base = ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter") + rest = ( + "Subblocks find", "Subblocks select", "Subblocks clarify", + "Facts find", "Facts check", "Facts fix", + "Levels find", "Levels select", "Levels clarify", + "Relevance find", "Relevance select", "Relevance clarify", + "Outline", + "Questions find", "Questions select", "Questions clarify", "Questions check", + "Flashcards", "Examples", + ) + middle = base + (("Supplement",) if q["type"] == "projekt" else ()) + rest + return (("Source prep",) if q["type"] == "link" else ()) + middle + + +def _step_idx(topic: str, name: str) -> int: + return _blocks_steps(topic).index(name) + + +def _report_p(set_p, topic: str, step: str): + """Async report callback for _gather_progress: sets " d/t…" + step index.""" + idx = _step_idx(topic, step) + async def report(d, t): + set_p(f"{step} {d}/{t}…", step=idx) + return report + + +# Coarse display phases: bundle the fine steps (internally everything stays fine-grained). +# Special steps (Source laden, Supplement) belong to the "Inventory" phase. +PHASEN = ( + ("Source", ("Source prep",)), + ("Inventory", ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Supplement")), + ("Subblocks", ("Subblocks find", "Subblocks select", "Subblocks clarify")), + ("Facts", ("Facts find", "Facts check", "Facts fix")), + ("Levels", ("Levels find", "Levels select", "Levels clarify")), + ("Relevance", ("Relevance find", "Relevance select", "Relevance clarify")), + ("Outline", ("Outline",)), + ("Questions", ("Questions find", "Questions select", "Questions clarify", "Questions check")), + ("Artefacts", ("Flashcards", "Examples")), +) + + +def _phases(topic: str) -> list[tuple[str, int]]: + """[(coarse_label, number of present fine steps)] for the current source.""" + fine_steps = _blocks_steps(topic) + return [(label, n) for label, members in PHASEN if (n := sum(f in members for f in fine_steps))] + + +def _phases_status(topic: str, current: int | None) -> list[dict]: + """Coarse phase states from the fine progress `current` (None = all pending, + len(feine) = all done). → [{label, state}] with state done/active/pending.""" + out, start = [], 0 + for label, n in _phases(topic): + end = start + n + if current is None or current < start: + state = "pending" + elif current >= end: + state = "done" + else: + state = "active" + out.append({"label": label, "state": state}) + start = end + return out + + +def _blocks_files(topic: str) -> dict: + work_dir = arbeit_dir(topic) + rounds = range(1, CONSENSUS_MAX_ROUNDS + 1) + return { + "final": blocks_path(topic), + "arbeit": work_dir, + "research": [work_dir / f"research-{i}.md" for i in (1, 2, 3, 4, 5)], + "research_mapping": work_dir / "research-mapping.json", + "selection": {n: [work_dir / f"selection-r{n}-{i}.json" for i in (1, 2, 3)] for n in rounds}, + "mapping": {n: work_dir / f"selection-mapping-r{n}.json" for n in rounds}, + "ergaenzung": work_dir / "ergaenzung.json", + "sub_roh": work_dir / "subblocks-roh.json", + "facts": work_dir / "subblocks-facts.json", + "sidecar": subblocks_path(topic), + "question_pattern": question_pattern_path(topic), + "outline": work_dir / "outline.json", + "outline_slots": [work_dir / f"outline-{i}.json" for i in (1, 2, 3)], + "artefakte": work_dir / "artefakte.json", + } + + +def _all_slot_files(files: dict) -> list[Path]: + work_dir = files["arbeit"] + # Subblock/levels slots are dynamic per chunk — collect via glob. + dyn = (list(work_dir.glob("subblock-*")) + list(work_dir.glob("facts-*")) + list(work_dir.glob("level-*")) + list(work_dir.glob("relevance-*")) + + list(work_dir.glob("question-pattern-*")) + list(work_dir.glob("outline-*")) + list(work_dir.glob("artifact-*")) + + list(work_dir.glob("research-*")) + list(work_dir.glob("consolidation-*")) + + list(work_dir.glob("clarification*")) + list(work_dir.glob("dedup-*")) + + list(work_dir.glob("inventar-filter*"))) if work_dir.is_dir() else [] + return [ + *files["research"], files["research_mapping"], + *(p for slots in files["selection"].values() for p in slots), + *files["mapping"].values(), files["ergaenzung"], + files["sub_roh"], files["sidecar"], files["question_pattern"], + files["facts"], files["outline"], files["artefakte"], *dyn, + ] + + +def cancel_blocks(topic: str) -> bool: + if topic not in _blocks_progress: + return False + _blocks_cancelled.add(topic) + cancel_scope(f"blocks-{topic}-") # waiting agents bail before spawning + kill_process(f"blocks-{topic}-") # kill running subprocesses + return True + + +async def _resume_step(topic: str) -> int: + """First step still open. While blocks.md is missing (never built OR a reset deleted it) the + inventory sub-step comes fine-grained from the DB step status; once blocks.md exists the inventory + counts as done (the artefact is the source of truth, robust for legacy topics) and later phases + come from the persisted artefacts. A reset-from-inventory deletes blocks.md, so this stays exact.""" + files = _blocks_files(topic) + steps_all = _blocks_steps(topic) + if not files["final"].exists(): + for step in ("Source prep", "Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter"): + if step in steps_all and await db.get_step_status(topic, step) != "done": + return _step_idx(topic, step) + return _step_idx(topic, "Blocks-Filter") # statuses done but artefact gone → rewrite + q = load_source(topic) + if q["type"] == "projekt" and not files["ergaenzung"].exists(): + return _step_idx(topic, "Supplement") + sidecar = _json_file(files["sidecar"]) + if _sidecar_schema(sidecar) is not None: + # Levels done; only relevance still open? + if not _relevance_complete(sidecar): + return _step_idx(topic, "Relevance find") + # Relevance done; outline (blocks artifact for the guide) open? + if not _outline_complete(files): + return _step_idx(topic, "Outline") + # Outline done; question patterns open? + if not _question_pattern_complete(topic): + return _step_idx(topic, "Questions find") + # Questions done; learning artefacts (flashcards/examples) open? + if not _artefacts_complete(files): + return _step_idx(topic, "Flashcards") + return len(_blocks_steps(topic)) + if _sub_raw_schema(_json_file(files["sub_roh"])) is None: + return _step_idx(topic, "Subblocks find") + # Subblocks done; facts still open? (Facts come before the levels.) + if not _facts_complete(files): + return _step_idx(topic, "Facts find") + return _step_idx(topic, "Levels find") + + +def _fine_status(topic: str, current: int | None) -> list[dict]: + """Fine sub-step status: per step {label, phase, state}. state from `current` + (done = idx). Phase label from PHASEN.""" + step_phase = {s: label for label, steps in PHASEN for s in steps} + out = [] + for i, s in enumerate(_blocks_steps(topic)): + state = "pending" if current is None or current < i else "done" if current > i else "active" + out.append({"label": s, "phase": step_phase.get(s, ""), "state": state}) + return out + + +async def blocks_status(topic: str) -> dict: + # Internally fine-grained (resume/progress); bundled into 5 coarse phases for display. + fine_steps = _blocks_steps(topic) + ready = blocks_path(topic).exists() # inventory written → block overview available + generating = topic in _blocks_progress + if generating: + current = _blocks_step.get(topic) + else: + # True progress (inventory from DB step status, later phases from artefacts). + current = await _resume_step(topic) + partial = not generating and 0 < current < len(fine_steps) + return { + "ready": ready, + "generating": generating, + "progress": _blocks_progress.get(topic), + "error": _blocks_errors.get(topic), + "partial": partial, + "steps": _phases_status(topic, current), + "feine_steps": _fine_status(topic, current), + } + + +def active_blocks() -> list[dict]: + return [{"topic": t, "progress": p} for t, p in _blocks_progress.items()] + + +def reset_blocks(topic: str) -> None: + """"Remove": deletes the ENTIRE blocks area — crawl, triage, inventory … questions. + KEEPS only the topic config `source.json` (type/link/spec). Re-generating crawls anew. + (Crawl/triage belong to the blocks; only the config is the "topic".)""" + files = _blocks_files(topic) + files["final"].unlink(missing_ok=True) + files["sidecar"].unlink(missing_ok=True) + files["question_pattern"].unlink(missing_ok=True) + shutil.rmtree(source_crawl_dir(topic), ignore_errors=True) # crawl belongs to the blocks + shutil.rmtree(files["arbeit"], ignore_errors=True) + _blocks_errors.pop(topic, None) + # source.json intentionally stays — that is the topic config. + + +def _phase_idx(label: str) -> int: + """Index of the coarse phase in the canonical order (Source=0 … Questions=5).""" + order = [l for l, _ in PHASEN] + return order.index(label) if label in order else 1 + + +def _reset_from_phase(topic: str, label: str) -> None: + """Delete file artefacts FROM the coarse phase `label` (Source/Inventory … Questions), keeping + earlier ones. Cumulative. source.json + crawl (.done) always stay (re-crawl only on full reset).""" + files = _blocks_files(topic) + work_dir = files["arbeit"] + idx = _phase_idx(label) + + def glob_del(pat: str) -> None: + if work_dir.is_dir(): + for p in work_dir.glob(pat): + p.unlink(missing_ok=True) + + # Phase index: Source=0 · Inventory=1 · Subblocks=2 · Facts=3 · Levels=4 · Relevance=5 · Outline=6 · Questions=7 · Artefacts=8 + if idx <= 8: # Artefacts (flashcards/examples) + files["artefakte"].unlink(missing_ok=True) + glob_del("artifact-*") + if idx <= 7: # Questions + files["question_pattern"].unlink(missing_ok=True) + glob_del("question-pattern-*") + if idx <= 6: # Outline + files["outline"].unlink(missing_ok=True) + glob_del("outline-*") + if idx <= 5: # Relevance + glob_del("relevance-*") + if idx <= 4: # Levels + relevance share the sidecar → from Levels rebuild entirely + files["sidecar"].unlink(missing_ok=True) + glob_del("level-*") + else: # from Relevance: keep levels, strip only the relevance fields + sc = _json_file(files["sidecar"]) + if isinstance(sc, dict): + for subs in sc.values(): + for s in (subs if isinstance(subs, list) else []): + if isinstance(s, dict): + s.pop("relevance", None) + atomic_write_json(files["sidecar"], sc, indent=1) + if idx <= 3: # Facts (before the levels) — facts map + work files gone + files["facts"].unlink(missing_ok=True) + glob_del("facts-*") + if idx <= 2: # Subblocks + files["sub_roh"].unlink(missing_ok=True) + glob_del("subblock-*") + if idx <= 1: # Inventory (and source) = inventory files + blocks.md gone + for p_old in _all_slot_files(files): + p_old.unlink(missing_ok=True) + files["final"].unlink(missing_ok=True) + + +async def _reset_from_step(topic: str, step_idx: int) -> None: + """Fine reset FROM a sub-step (0-based index in _blocks_steps). Resets + pipeline_state + artefacts + DB from here on; earlier steps stay. Inventory sub-steps + reconstruct the DB status from the artefacts (dedup/filter safe; clarification robustly falls back + to consolidation, because the clarification renames → a title mismatch would be fragile).""" + fine_steps = list(_blocks_steps(topic)) + if not (0 <= step_idx < len(fine_steps)): + return + affected = set(fine_steps[step_idx:]) + files = _blocks_files(topic) + work_dir = files["arbeit"] + + def gd(pat: str) -> None: + if work_dir.is_dir(): + for p in work_dir.glob(pat): + p.unlink(missing_ok=True) + + await db.delete_pipeline_state(topic, list(fine_steps[step_idx:])) + # Later artefacts/DB cumulatively from the affected step (back to front). + if {"Examples", "Flashcards"} & affected: + files["artefakte"].unlink(missing_ok=True); gd("artifact-*"); await db.delete_sub_artefakte(topic) + if any(s.startswith("Questions") for s in affected): + files["question_pattern"].unlink(missing_ok=True); gd("question-pattern-*"); await db.delete_question_pattern(topic) + if "Outline" in affected: + files["outline"].unlink(missing_ok=True); gd("outline-*"); await db.delete_outline(topic) + if any(s.startswith("Relevance") for s in affected): + gd("relevance-*") + if any(s.startswith("Levels") for s in affected): + gd("level-*") + if any(s.startswith("Facts") for s in affected): + files["facts"].unlink(missing_ok=True); gd("facts-*") + # The sidecar carries subblocks + their fields level/relevance/facts. From subblocks rebuild entirely; + # otherwise strip only the fields of the phases to rebuild — subblocks are preserved. + if any(s.startswith("Subblock") for s in affected): + files["sidecar"].unlink(missing_ok=True) + else: + strip = {f for s, f in (("Facts", "facts"), ("Levels", "level"), ("Relevance", "relevance")) + if any(x.startswith(s) for x in affected)} + if strip: + sc = _json_file(files["sidecar"]) + if isinstance(sc, dict): + for subs in sc.values(): + for s in (subs if isinstance(subs, list) else []): + if isinstance(s, dict): + for f in strip: + s.pop(f, None) + atomic_write_json(files["sidecar"], sc, indent=1) + if any(s.startswith("Subblock") for s in affected): + files["sub_roh"].unlink(missing_ok=True); gd("subblock-*"); await db.delete_subblocks(topic) + # --- Inventory (DB status cascades) --- + if "Blocks-Filter" in affected and not ({"Clarification", "Consolidation", "Research", "Dedup"} & affected): + # Only filter rebuilt: degraded blocks back to consensus. + d = _json_file(work_dir / "inventar-filter.json") + for f in (d.get("fragments", []) if isinstance(d, dict) else []): + await db.set_block_status(topic, _norm_title(f.get("fragment", "")), "consensus") + gd("inventar-filter*") + if "Dedup" in affected and not ({"Clarification", "Consolidation", "Research"} & affected): + # Dedup (+filter) rebuilt: all blocks discarded in dedup/filter back to consensus. + for kind in ("dedup-runde-1.json", "inventar-filter.json"): + d = _json_file(work_dir / kind) + title = ([t for g in d.get("groups", []) for t in g] if isinstance(d, dict) and "groups" in d + else [f.get("fragment", "") for f in d.get("fragments", [])] if isinstance(d, dict) else []) + for t in title: + await db.set_block_status(topic, _norm_title(t), "consensus") + gd("dedup-*"); gd("inventar-filter*") + if {"Clarification", "Consolidation"} & affected and not ({"Research"} & affected): + # Clarification/consolidation rebuilt: clear inventory DB (research readers stay). Clarification rollback + # would be fragile due to renaming → cleanly rebuild from consolidation. + await db.delete_blocks(topic) + gd("clarification*"); gd("consolidation-*"); gd("dedup-*"); gd("inventar-filter*") + if "Research" in affected: # whole inventory like a phase reset + for p_old in _all_slot_files(files): + p_old.unlink(missing_ok=True) + await db.delete_blocks(topic) + # blocks.md is the inventory aggregate — stale once any inventory sub-step is reset. Delete it so the + # status/resume see the inventory as open from the reset step (the pipeline rewrites it; the DB step + # statuses of the kept earlier steps let those skip). + if {"Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter"} & affected: + files["final"].unlink(missing_ok=True) + + +async def reset_blocks_ab_step(topic: str, step_idx: int) -> None: + """Public: ONLY reset from a sub-step — no re-generation. Leaves a + partial state (the steps from here count as open). If a generation is running → ignore.""" + if topic in _blocks_progress: + return + await _reset_from_step(topic, step_idx) + + +def _supplement_schema(data): + """{"blocks": [{"title", "description"}]} → list (empty allowed) · otherwise None.""" + if not isinstance(data, dict) or not isinstance(data.get("blocks"), list): + return None + out = [] + for b in data["blocks"]: + if not isinstance(b, dict) or not isinstance(b.get("title"), str) or not isinstance(b.get("description"), str): + return None + title, description = b["title"].strip(), b["description"].strip() + if not title: + return None + out.append((title, description)) + return out + + +def _convert_pdfs(project: Path) -> None: + """Convert PDFs in the project to .txt (pdftotext) — agents read text instead of page images. + + Called before every project generation; converts only if the + .txt is missing or older than the PDF. The original is left untouched. + If pdftotext is missing and the project contains PDFs → hard error instead of + an unreliable direct-read mode (MiniMax image limit, vision cost). + """ + pdfs = list(project.rglob("*.pdf")) + if not pdfs: + return + if shutil.which("pdftotext") is None: + raise RuntimeError("pdftotext missing (install poppler-utils) — PDFs in the project cannot be read") + for pdf in pdfs: + txt = pdf.with_suffix(".txt") + if txt.exists() and txt.stat().st_mtime >= pdf.stat().st_mtime: + continue + try: + subprocess.run(["pdftotext", "-layout", str(pdf), str(txt)], check=True, timeout=120) + _log(project.name, f"PDF converted: {pdf.name} → {txt.name}") + except Exception as e: + raise RuntimeError(f"PDF conversion failed ({pdf.name}): {e}") from e + + +_SOURCE_TEMPLATE = {"projekt": "Blocks-Source-Projekt", "uni": "Blocks-Source-Uni", "link": "Blocks-Source-Link"} + + +def _text_sections(text: str, goal: int = RESEARCH_SECTION_CHARS) -> list[str]: + """Split text at paragraph/line boundaries into sections of ~`ziel` chars (against lost-in-the-middle + on large documents). Small text stays ONE section. Content stays complete — only + separating whitespace is dropped.""" + text = text.strip() + if len(text) <= goal: + return [text] if text else [] + sections: list[str] = [] + buf = "" + + def flush(): + nonlocal buf + if buf.strip(): + sections.append(buf.strip()) + buf = "" + + for block in re.split(r"\n\s*\n", text): # at paragraph boundaries + block = block.strip() + if not block: + continue + if len(block) > goal: # single huge paragraph → hard-cut at lines + flush() + for line in block.split("\n"): + if buf and len(buf) + len(line) + 1 > goal: + flush() + buf += line + "\n" + flush() + elif buf and len(buf) + len(block) + 2 > goal: + flush() + buf = block + else: + buf = (buf + "\n\n" + block) if buf else block + flush() + return sections + + +def _build_research_prompt(topic: str, out_path: Path, instructions: str, type: str, folder: Path | None, fokus: str = "", section: str = "") -> str: + if section: + # Section mode (uni/projekt): text directly in the prompt → small context, no file reading. + source = section + elif type in _SOURCE_TEMPLATE: + source = _prompt(_SOURCE_TEMPLATE[type], project=folder) + else: + source = _prompt("Blocks-Source-Thema", topic=topic) + return _prompt( + "Blocks-Research", + topic=topic, source=source, blocks_path=out_path, focus=fokus, extra=_extra(instructions), + ) + + +def _file_payload(path: Path): + """Valid if the slot file exists and contains numbered entries.""" + if not path.exists(): + return None + text = path.read_text(encoding="utf-8") + return text if _parse_selection(text) else None + + +def _mapping_schema(data): + """{"blocks": [str, ≥1], "rest": [str]} → (blocks, rest) · otherwise None.""" + if not isinstance(data, dict): + return None + blocks = _str_list(data.get("blocks")) + rest = _str_list(data.get("rest")) + if not blocks or rest is None: + return None + return blocks, rest + + +def _sub_raw_schema(data): + """{block title: [subblock, …]} → dict · otherwise None (intermediate state of block B).""" + if not isinstance(data, dict) or not data: + return None + out: dict[str, list[str]] = {} + for k, v in data.items(): + subs = _str_list(v) if isinstance(v, list) else None + if not isinstance(k, str) or not k.strip() or not subs: + return None + out[k] = subs + return out + + +def _sidecar_schema(data): + """{block title: [{title, level}, …]} → dict · otherwise None (sidecar with levels).""" + if not isinstance(data, dict) or not data: + return None + for v in data.values(): + if not isinstance(v, list) or not v: + return None + for s in v: + if not isinstance(s, dict) or not str(s.get("title", "")).strip() or s.get("level") not in _LEVELS: + return None + return data + + +def _relevance_complete(data) -> bool: + """Does every subblock in the sidecar carry a valid relevance (relevant/peripheral)?""" + if not isinstance(data, dict) or not data: + return False + return all( + isinstance(s, dict) and s.get("relevance") in ("relevant", "peripheral") + for v in data.values() if isinstance(v, list) + for s in v + ) + + + + +def _question_pattern_chunk_schema(data) -> list[dict] | None: + """{"pattern": [{block, subblock, question}, …]} → list of valid entries · otherwise None. + + One pattern per subblock (no type cross-product — the difficulty only comes at + exam time from the learner's tier). Invalid individual entries are skipped.""" + if not isinstance(data, dict) or not isinstance(data.get("pattern"), list): + return None + out = [] + for e in data["pattern"]: + if not isinstance(e, dict): + continue + blk = str(e.get("block", "")).strip() + sub = str(e.get("subblock", "")).strip() + question = str(e.get("question", "")).strip() + if not blk or not sub or not question: + continue + out.append({"block": blk, "subblock": sub, "question": question}) + return out or None + + +def _question_pattern_complete(topic: str) -> bool: + """Does the question-pattern sidecar exist (build ran)? Individual empty blocks + fall back to live generation at exam time — so the file is enough.""" + return isinstance(_json_file(question_pattern_path(topic)), dict) + + +def _read(p: Path) -> str: + return p.read_text(encoding="utf-8") if p.exists() else "" + + +def _chunk_nums(items: list, n: int) -> list[list]: + """Splits a flat list into n chunks as equal in size as possible.""" + n = max(1, n) + size = max(1, math.ceil(len(items) / n)) + return [items[i:i + size] for i in range(0, len(items), size)] + + +def _n_chunks(count: int, size: int = SUBBLOCK_CHUNK) -> int: + return min(SUBBLOCK_MAX, max(1, math.ceil(count / size))) + + +def _lpt_chunks(weights: list[int], target: int) -> list[list[int]]: + """Distribute indices across chunks load-balanced (LPT, makespan-minimal). Weight = cost per index. + K = ceil(total weight/target); heaviest first into the currently lightest bin. → index lists.""" + if not weights: + return [] + K = max(1, math.ceil(sum(weights) / max(1, target))) + bins: list[list[int]] = [[] for _ in range(K)] + last = [0] * K + for i in sorted(range(len(weights)), key=lambda x: weights[x], reverse=True): + j = min(range(K), key=lambda b: last[b]) + bins[j].append(i) + last[j] += weights[i] + return [b for b in bins if b] + + + + + + + + +async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict | None: + """Block B (DB + loop): per package, find subblocks in rounds (3 finders, until 0 new/cap), + collect in the DB (≥2 mentions = consensus, 1× discarded), a judge cleans up per package. + → {block title: [subblock, …]} (consensus) or None. Fills DB table `subblocks`.""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + work_dir = files["arbeit"] + folder = source_folder(topic) + caps = "files" if folder else "full" + # Source for the evidence exam in the clarify step (discards invented/unsupportable subs). + _type = load_source(topic).get("type", "thema") + source = _prompt(_SOURCE_TEMPLATE[_type], project=folder) if _type in _SOURCE_TEMPLATE else _prompt("Blocks-Source-Thema", topic=topic) + nums = list(entries) + chunks = _chunk_nums(nums, _n_chunks(len(nums))) + n = len(chunks) + title_by_num = {num: _title(entries[num]) for num in nums} + norm_by_num = {num: _norm_title(title_by_num[num]) for num in nums} + await db.delete_subblocks(topic) # fresh start of the block (idempotent counter) + + async def _known_block(chunk): + known = [] + for num in chunk: + subs = [s["sub_title"] for s in await db.list_subblocks(topic, norm_by_num[num])] + if subs: + known.append(f"\n" + "\n".join(f"- {s}" for s in subs)) + if not known: + return "" + # Do NOT list known items again (otherwise re-confirmation inflates the mention count, + # self-bias/echo) — only add what's missing. This keeps the counter an honest consensus signal. + return ("\n\nBEREITS ERFASST — liste diese NICHT erneut. Finde nur, was FEHLT:\n" + "\n".join(known)) + + # Phase "Subblocks find": per package loop until 0 new subs / time cap. + async def _find(c, chunk): + assignment = "\n".join(f"- {entries[num]}" for num in chunk) + chunk_idx = _title_index({num: title_by_num[num] for num in chunk}) + start = time.monotonic() + round_n = 0 + while not is_cancelled(): + round_n += 1 + bekannt = await _known_block(chunk) if round_n > 1 else "" + paths = [work_dir / f"subblock-c{c}-r{round_n}-{i}.md" for i in (1, 2, 3)] + for p in paths: + p.unlink(missing_ok=True) + slots = [{ + "key": f"blocks-{topic}-subblock-c{c}-r{round_n}-{i}", + "prompt": _prompt("Subblock-Research", topic=topic, assignment=assignment, known=bekannt, out_path=p, extra=_extra(instructions)), + "role": "quick", "capabilities": caps, + "payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None), + } for i, p in enumerate(paths, 1)] + agent_texts = await _race(topic, f"Subblocks package {c} R{round_n}", slots, 2, _timeout("subblock", len(chunk)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) + if is_cancelled(): + return False + if not agent_texts: + return round_n > 1 # round 1 without result = error; later = simply the end + existing = {num: {s["sub_norm"] for s in await db.list_subblocks(topic, norm_by_num[num])} for num in chunk} + new = 0 + for d in agent_texts: + for marker, subs in d.items(): + num = _resolve_title(chunk_idx, marker) + if num is None: + continue + seen_set = set() + for sub in subs: + sn = _norm_title(sub) + if not sn or sn in seen_set: + continue + seen_set.add(sn) + if sn not in existing[num]: + new += 1 + existing[num].add(sn) + await db.upsert_subblock(topic, norm_by_num[num], sn, title_by_num[num], sub) + if new == 0: + break + if time.monotonic() - start > SUBBLOCK_CAP: + _log(topic, f"Subblocks package {c}: time cap reached (round {round_n})") + break + return True + + oks = await _gather_progress([_find(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _report_p(set_p, topic, "Subblocks find")) + if is_cancelled(): + return None + if not all(ok is True for ok in oks): + _blocks_errors[topic] = "Subblocks failed (research)" + return None + + # Phase "Subblocks select": ≥2 mentions = consensus, 1× discarded (code). + set_p(f"Subblocks select ({n} packages)…", step=_step_idx(topic, "Subblocks select")) + for num in nums: + for s in await db.list_subblocks(topic, norm_by_num[num]): + await db.set_subblock_fields(topic, norm_by_num[num], s["sub_norm"], + status=("consensus" if s["mentions"] >= 2 else "discarded")) + + # Phase "Subblocks clarify": source panel (SUBBAUSTEIN_PANEL judges) checks consensus + uncertain (1×) + # against the source; code majority per sub. External, multi-voice gate against single-judge bias + echo. + async def _clarify(c, chunk): + fp = work_dir / f"subblock-final-c{c}.md" + if _parse_subblocks(_read(fp)): + return + block_texts, has_any = [], False + consensus_by_num: dict[int, list[str]] = {} + for num in chunk: + rows = await db.list_subblocks(topic, norm_by_num[num]) + consensus_subs = [s["sub_title"] for s in rows if s["status"] == "consensus"] + uncertain = [s["sub_title"] for s in rows if s["status"] != "consensus" and s["mentions"] == 1] + consensus_by_num[num] = consensus_subs + if not consensus_subs and not uncertain: + continue + has_any = True + k_lines = "\n".join(f"- {s}" for s in consensus_subs) if consensus_subs else "- (keiner)" + u_lines = "\n".join(f"- {s}" for s in uncertain) if uncertain else "- (keiner)" + block_texts.append(f"BLOCK: {title_by_num[num]}\nKonsens (≥2 finders):\n{k_lines}\nUnsicher (1× — streng gegen Source check):\n{u_lines}") + if not has_any: + return + + chunk_idx = _title_index({num: title_by_num[num] for num in chunk}) + paths = [work_dir / f"subblock-final-c{c}-j{j}.md" for j in range(1, SUBBLOCK_PANEL + 1)] + pending = [(j, p) for j, p in enumerate(paths, 1) if _parse_subblocks(_read(p)) is None] + for _, p in pending: + p.unlink(missing_ok=True) + if pending: + slots = [{ + "key": f"blocks-{topic}-subblock-final-c{c}-j{j}", + "prompt": _prompt("Subblock-Mapping", topic=topic, source=source, blocks="\n\n".join(block_texts), out_path=p, extra=_extra(instructions)), + "role": "judge", "capabilities": caps, + "payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None), + } for j, p in pending] + existing = SUBBLOCK_PANEL - len(pending) + await _race(topic, f"Subblock-Clarification {c}", slots, max(1, 2 - existing), + _timeout("subblock_check", len(chunk)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) + if is_cancelled(): + return + outs = [d for p in paths if (d := _parse_subblocks(_read(p)))] + if not outs: # panel fully failed → adopt consensus (the fallback as before) + _log(topic, f"Subblock clarification package {c} failed — consensus adopted") + text = "\n\n".join(f"\n" + "\n".join(f"- {s}" for s in consensus_by_num[num]) + for num in chunk if consensus_by_num[num]) + atomic_write_text(fp, text) + return + + # code majority per block/sub-norm: keep if a majority of judges list it (tie → keep). + block_texts_out = [] + for num in chunk: + votes: dict[str, int] = {} + form: dict[str, str] = {} + for d in outs: + seen = set() + for marker, subs in d.items(): + if _resolve_title(chunk_idx, marker) != num: + continue + for sub in subs: + sn = _norm_title(sub) + if not sn or sn in seen: + continue + seen.add(sn) + form.setdefault(sn, sub) + votes[sn] = votes.get(sn, 0) + 1 + kept = [form[sn] for sn in form if votes[sn] * 2 >= len(outs)] + if kept: + block_texts_out.append(f"\n" + "\n".join(f"- {s}" for s in kept)) + atomic_write_text(fp, "\n\n".join(block_texts_out)) + + await _gather_progress([_clarify(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _report_p(set_p, topic, "Subblocks clarify")) + if is_cancelled(): + return None + + # Final list per block: judge output, otherwise consensus fallback. Reconcile DB + build raw. + raw: dict[str, list[str]] = {} + for c, chunk in enumerate(chunks, 1): + final = _parse_subblocks(_read(work_dir / f"subblock-final-c{c}.md")) or {} + chunk_idx = _title_index({num: title_by_num[num] for num in chunk}) + final_by_num = {_resolve_title(chunk_idx, m): subs for m, subs in final.items() if _resolve_title(chunk_idx, m) is not None} + for num in chunk: + title = title_by_num[num] + consensus = [s["sub_title"] for s in await db.list_subblocks(topic, norm_by_num[num]) if s["status"] == "consensus"] + subs = final_by_num.get(num) or consensus + if not subs: + continue + raw[title] = subs + # align DB to the final list: final = consensus, rest discarded, add new ones. + final_norms = {_norm_title(s) for s in subs} + have = {s["sub_norm"] for s in await db.list_subblocks(topic, norm_by_num[num])} + for s in await db.list_subblocks(topic, norm_by_num[num]): + await db.set_subblock_fields(topic, norm_by_num[num], s["sub_norm"], + status=("consensus" if s["sub_norm"] in final_norms else "discarded")) + for s in subs: + sn = _norm_title(s) + if sn and sn not in have: + await db.upsert_subblock(topic, norm_by_num[num], sn, title, s) + await db.set_subblock_fields(topic, norm_by_num[num], sn, status="consensus") + await _dedup_subblocks(topic, raw) # near-dup filter per block (deterministic, no LLM) + if not raw: + _blocks_errors[topic] = "No subblocks determined" + return None + return raw + + +async def _dedup_subblocks(topic: str, raw: dict[str, list[str]]) -> None: + """Deterministic near-duplicate filter per block: subblocks with cosine ≥ + EMBEDDING_SUB_DUP are the same statement (reliable in the narrow block context — no LLM + needed). Per duplicate group keeps the most informative (longest); rest → DB discarded + out of `roh`. + Model missing → silently skip (like the rest of the embedding fallback).""" + if not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available): + return + for title, subs in list(raw.items()): + if len(subs) < 2: + continue + sims = await asyncio.to_thread(embedding.embed_sims, subs) + if sims is None: + return + keepers: list[int] = [] + discarded: list[int] = [] + for i in sorted(range(len(subs)), key=lambda x: (-len(subs[x]), x)): # most informative first + if any(float(sims[i][j]) >= EMBEDDING_SUB_DUP for j in keepers): + discarded.append(i) + else: + keepers.append(i) + if not discarded: + continue + bnorm = _norm_title(title) + for i in discarded: + await db.set_subblock_fields(topic, bnorm, _norm_title(subs[i]), status="discarded") + raw[title] = [subs[i] for i in sorted(keepers)] # original order of the kept ones + + +def _code_vote(rater: list[dict], n: int) -> tuple[dict, dict]: + """Majority vote over rater dicts on local ids 1..n → (outcome, disputed). A clear winner + needs ≥2 votes and no tie; otherwise the id is disputed (kept with its vote list).""" + outcome: dict[int, str] = {} + disputed: dict[int, list[str]] = {} + for k in range(1, n + 1): + vote_list = [d[k] for d in rater if k in d] + counter: dict[str, int] = {} + for s in vote_list: + counter[s] = counter.get(s, 0) + 1 + best = max(counter.values(), default=0) + winners = [s for s, v in counter.items() if v == best] + if len(winners) == 1 and best >= 2: + outcome[k] = winners[0] + else: + disputed[k] = vote_list + return outcome, disputed + + +def _disputed_lines(items, item_idxs, disputed: dict) -> str: + """Render disputed items as `k. [block] sub — Stimmen: a, b` lines for the judge prompt.""" + return "\n".join( + f"{k}. [{items[item_idxs[k - 1]][0]}] {items[item_idxs[k - 1]][1]} — Stimmen: {', '.join(vote_list) or 'none'}" + for k, vote_list in disputed.items() + ) + + +async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instructions: str) -> dict | None: + """Block C: three phases with a barrier — find (classify), select (vote), clarify. + Local IDs 1..n per package, mapped to global gid afterwards. + → {block title: [{title, level}, …]} or None.""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + work_dir = files["arbeit"] + # key points per sub as concise context (better-grounded classification; classification needs little). + facts_map = _json_file(files["facts"]) + facts_map = facts_map if isinstance(facts_map, dict) else {} + items = [(title, sub) for title, subs in raw.items() for sub in subs] # global id = index+1 + if not items: + return {title: [] for title in raw} + # pack chunks from WHOLE blocks (don't split a block) → each rater sees per block + # all subs and can classify relatively. Item indices per block in raw order. + chunks, cur, i = [], [], 0 + for _title_b, subs in raw.items(): + g = list(range(i, i + len(subs))) + i += len(subs) + if cur and len(cur) + len(g) > LEVEL_CHUNK: + chunks.append(cur) + cur = [] + cur.extend(g) + if cur: + chunks.append(cur) + n = len(chunks) + + def rater_paths(c): + return [work_dir / f"level-c{c}-{i}.json" for i in (1, 2, 3)] + + def lset(item_idxs): + return set(range(1, len(item_idxs) + 1)) + + # Phase "Levels find": 3 raters per package (min. 2), local IDs. + async def _rate(c, item_idxs): + local_set = lset(item_idxs) + paths = rater_paths(c) + existing = sum(1 for p in paths if _levels_schema(_json_file(p), local_set)) + if existing >= 2: + return True + enum_lines, cur_b = [], None + for k, j in enumerate(item_idxs, 1): + b, sub = items[j] + if b != cur_b: + enum_lines.append(f"\nBAUSTEIN: {b}") + cur_b = b + enum_lines.append(f"{k}. {sub}") + if (kz := _core_line(facts_map.get(b, {}).get(_norm_title(sub)))): + enum_lines.append(f" {kz}") + enum = "\n".join(enum_lines).strip() + pending = [(i, p) for i, p in enumerate(paths, 1) if not _levels_schema(_json_file(p), local_set)] + slots = [{ + "key": f"blocks-{topic}-level-c{c}-{i}", + "prompt": _prompt("Levels-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)), + "role": "fast", "capabilities": "files", + "payload": (lambda result, p=p, ids=local_set: _levels_schema(_json_file(p), ids)), + } for i, p in pending] + new = await _race(topic, f"Levels package {c}", slots, 2 - existing, _timeout("level", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) + return not is_cancelled() and new is not None + + oks = await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Levels find")) + if is_cancelled(): + return None + if not all(ok is True for ok in oks): + _blocks_errors[topic] = "Classification failed (research)" + return None + + # Phase "Levels select": code vote per package → (outcome, disputed). + set_p(f"Levels select ({n} packages)…", step=_step_idx(topic, "Levels select")) + vote_by_c = {} + for c, item_idxs in enumerate(chunks, 1): + local_set = lset(item_idxs) + rater = [d for p in rater_paths(c) if (d := _levels_schema(_json_file(p), local_set))] + vote_by_c[c] = _code_vote(rater, len(item_idxs)) + + # Phase "Levels clarify": one judge per package on the disputed items, all in parallel. + async def _clarify(c, item_idxs): + outcome, strittig = vote_by_c[c] + if strittig: + judge_path = work_dir / f"level-final-c{c}.json" + decision = _levels_schema(_json_file(judge_path), set(strittig)) + if decision is None: + disputed_block = _disputed_lines(items, item_idxs, strittig) + status, decision = await run_single_slot( + ctx, f"Levels-Clarification {c}", + key=f"blocks-{topic}-level-final-c{c}", + prompt=_prompt("Levels-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)), + role="judge", capabilities="files", + payload=lambda result, p=judge_path, ids=set(strittig): _levels_schema(_json_file(p), ids), + timeout=_timeout("level_check", len(strittig)), + ) + if status == FAILED: + _log(topic, f"Levels clarification package {c} failed — default 'advanced'") + decision = decision if isinstance(decision, dict) else {} + # disputed without a decision → 'advanced'; vote winners stay; judge overrides. + outcome = {**{k: "advanced" for k in strittig}, **outcome, **decision} + return {item_idxs[k - 1] + 1: level for k, level in outcome.items()} + + parts = await _gather_progress([_clarify(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Levels clarify")) + if is_cancelled(): + return None + level_by_id: dict[int, str] = {} + for c, part in enumerate(parts, 1): + if not isinstance(part, dict): + # clarification is not fatal: vote outcome + default 'advanced' for disputed. + if isinstance(part, BaseException): + _log(topic, f"Levels clarification package {c}: {type(part).__name__}: {part}") + outcome, strittig = vote_by_c[c] + item_idxs = chunks[c - 1] + merged = {**{k: "advanced" for k in strittig}, **outcome} + part = {item_idxs[k - 1] + 1: s for k, s in merged.items()} + level_by_id.update(part) + + # assemble the sidecar — same order as items → gid matches + sidecar: dict[str, list[dict]] = {} + gid = 0 + for title, subs in raw.items(): + lst = [] + for sub in subs: + gid += 1 + lst.append({"title": sub, "level": level_by_id.get(gid, "advanced")}) + sidecar[title] = lst + return sidecar + + +_FACTS_FIELDS = ("key_points", "prerequisites", "hurdles", "cited_facts", "example_idea") + + +def _facts_schema(data) -> list[dict] | None: + """{"facts": [{block, subblock, …}]} → valid list · otherwise None. + Strictly separates belegte_facts (with source) from example_idee (generative).""" + if not isinstance(data, dict) or not isinstance(data.get("facts"), list): + return None + out = [] + for e in data["facts"]: + if not isinstance(e, dict): + continue + blk = str(e.get("block", "")).strip() + sub = str(e.get("subblock", "")).strip() + if not blk or not sub: + continue + bf = [{"text": t, "source": str(f.get("source", "")).strip()} + for f in (e.get("cited_facts") or []) if isinstance(f, dict) and (t := str(f.get("text", "")).strip())] + out.append({ + "block": blk, "subblock": sub, + "key_points": [k for x in (e.get("key_points") or []) if (k := str(x).strip())], + "prerequisites": str(e.get("prerequisites", "")).strip(), + "hurdles": str(e.get("hurdles", "")).strip(), + "cited_facts": bf, + "example_idea": str(e.get("example_idea", "")).strip(), + }) + return out or None + + +def _facts_check_schema(data) -> list[tuple[str, bool]] | None: + """Facts check → [(sub_norm, verwerfen)] per objection · {ok:true}→[] · None if invalid. + verwerfen=True: sub not supportable in substance (remove). verwerfen=False: only correct the fact.""" + if not isinstance(data, dict): + return None + if data.get("ok") is True: + return [] + pr = data.get("problems") + if not isinstance(pr, list): + return None + return [(sn, bool(p.get("discard"))) + for p in pr if isinstance(p, dict) and (sn := _norm_title(str(p.get("subblock", ""))))] + + +def _core_line(fk) -> str: + """Concise key-point line for classification (level/relevance) — less context suffices there. + Empty if no facts/key points (legacy).""" + if not isinstance(fk, dict) or not fk.get("key_points"): + return "" + return "Kern: " + " · ".join(str(k) for k in fk["key_points"]) + + +def _facts_lines(fk: dict) -> str: + z = [] + if fk.get("key_points"): + z.append("Kernpunkte: " + " · ".join(str(k) for k in fk["key_points"])) + if fk.get("prerequisites"): + z.append("Voraussetzung: " + fk["prerequisites"]) + if fk.get("hurdles"): + z.append("Hürde: " + fk["hurdles"]) + for bf in fk.get("cited_facts", []): + z.append(f"FAKT: {bf['text']} (Source: {bf.get('source', '?')})") + if fk.get("example_idea"): + z.append("Example: " + fk["example_idea"]) + return "\n".join(z) + + +def _facts_complete(files: dict) -> bool: + """Does the facts map exist (block done)? {block: {sub_norm: {...}}}.""" + d = _json_file(files["facts"]) + return isinstance(d, dict) and bool(d) + + +async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str) -> tuple | None: + """Block: per sub extract source facts (find) → verify (check) → correct/discard (fix). + Extract-once grounding: the result feeds level/relevance/questions/guide. + → (facts_map, discarded_map) — facts_map {block: {sub_norm: facts}}, discarded_map + {block: {sub_norm}} (unsupportable subs to remove) — or None on cancel/error.""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + work_dir = files["arbeit"] + caps = "files" if folder else "full" + type = q.get("type", "thema") + source = _prompt(_SOURCE_TEMPLATE[type], project=folder) if type in _SOURCE_TEMPLATE else _prompt("Blocks-Source-Thema", topic=topic) + blocks = [(title, [str(s).strip() for s in subs if str(s).strip()]) for title, subs in raw.items() if subs] + if not blocks: + return {}, {} + chunks = _lpt_chunks([len(subs) for _, subs in blocks], FACTS_CHUNK_SUBS) + + def raw_path(ci): return work_dir / f"facts-c{ci}.json" + def supp_path(ci): return work_dir / f"facts-erg-c{ci}.json" + def chk_path(ci, j): return work_dir / f"facts-check-c{ci}-j{j}.json" + def fix_path(ci): return work_dir / f"facts-fix-c{ci}.json" + def ctitle(idxs): return [blocks[i][0] for i in idxs] + def block_text(idxs): + return "\n\n".join( + f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in blocks[i][1]) + for i in idxs) + + # raw facts of a chunk → {block: {sub_norm: {sub, …fields}}}, matched to chunk titles. + def raw_map(ci, path): + idxs = chunks[ci] + rel_by = {blocks[i][0]: blocks[i][1] for i in idxs} + ct = ctitle(idxs) + out: dict[str, dict] = {} + for e in _facts_schema(_json_file(path)) or []: + bt = _match_sub(e["block"], ct) + if bt not in rel_by: + continue + sub = _match_sub(e["subblock"], rel_by[bt]) + out.setdefault(bt, {})[_norm_title(sub)] = {"sub": sub, **{k: e[k] for k in _FACTS_FIELDS}} + return out + + # union raw facts + completeness supplements (recall): only subs that exist in raw. + def _chunk_facts(ci): + raw = raw_map(ci, raw_path(ci)) + erg = raw_map(ci, supp_path(ci)) if supp_path(ci).exists() else {} + if not erg: + return raw + for bt, fm in raw.items(): + ebt = erg.get(bt, {}) + for sn, fk in fm.items(): + ek = ebt.get(sn) + if not ek: + continue + seen = {str(k).strip().casefold() for k in fk.get("key_points", [])} + for k in ek.get("key_points", []): + if str(k).strip().casefold() not in seen: + seen.add(str(k).strip().casefold()) + fk["key_points"].append(k) + seent = {bf["text"].strip().casefold() for bf in fk.get("cited_facts", [])} + for bf in ek.get("cited_facts", []): + if bf["text"].strip().casefold() not in seent: + seent.add(bf["text"].strip().casefold()) + fk["cited_facts"].append(bf) + for f in ("prerequisites", "hurdles", "example_idea"): + if not fk.get(f) and ek.get(f): + fk[f] = ek[f] + return raw + + # Phase "Facts find": 1 generator per chunk. + async def _find(ci, idxs): + fp = raw_path(ci) + if _facts_schema(_json_file(fp)): + return True + subs_total = sum(len(blocks[i][1]) for i in idxs) + status, _r = await run_single_slot( + ctx, f"Facts {ci}", key=f"blocks-{topic}-facts-c{ci}", + prompt=_prompt("Facts-Research", topic=topic, source=source, blocks=block_text(idxs), out_path=fp, extra=_extra(instructions)), + role="guide", capabilities=caps, + payload=lambda result, p=fp: _facts_schema(_json_file(p)), + timeout=_timeout("content", subs_total)) + return status != FAILED and _facts_schema(_json_file(fp)) is not None + + oks = await _gather_progress([_find(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts find")) + if is_cancelled(): + return None + if not any(ok is True for ok in oks): + _blocks_errors[topic] = "Facts extraction failed" + return None + + # Phase "Facts ergänzen" (recall): a targeted gap hunt per chunk looks for source-backed facts that + # the single pass missed. Best-effort — never fails (no erg file → merge uses only raw). + async def _supplement(ci, idxs): + ep = supp_path(ci) + if _facts_schema(_json_file(ep)): + return + per = raw_map(ci, raw_path(ci)) + if not per: + return + block = "\n\n".join( + f"BLOCK: {bt}\nSUBBAUSTEINE (mit bereits erfassten Facts):\n" + "\n".join( + f"- {fk['sub']}\n Erfasst: " + ("; ".join( + list(fk.get("key_points", [])) + [bf["text"] for bf in fk.get("cited_facts", [])]) or "(nichts)") + for fk in fm.values()) + for bt, fm in per.items()) + subs_total = sum(len(blocks[i][1]) for i in idxs) + await run_single_slot( + ctx, f"Facts supplement {ci}", key=f"blocks-{topic}-facts-erg-c{ci}", + prompt=_prompt("Facts-Supplement", topic=topic, source=source, blocks=block, out_path=ep, extra=_extra(instructions)), + role="guide", capabilities=caps, + payload=lambda result, p=ep: _facts_schema(_json_file(p)), + timeout=_timeout("content", subs_total)) + + set_p("Facts supplement…", step=_step_idx(topic, "Facts find")) + await _gather_progress([_supplement(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts find")) + if is_cancelled(): + return None + + # Phase "Facts check": FACTS_CHECK_PANEL judges per chunk. Two majority sets: + # flagged (fact inaccurate → correct) and discard (sub not supportable → remove). + async def _check(ci, idxs): + per = _chunk_facts(ci) # raw + supplements → panel verifies the union + if not per: + return ci, set(), set() + facts_text = "\n\n".join(f"SUBBAUSTEIN: {fk['sub']}\n{_facts_lines(fk)}" for fm in per.values() for fk in fm.values()) + pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _facts_check_schema(_json_file(chk_path(ci, j))) is None] + await asyncio.gather(*[ + run_agent(f"blocks-{topic}-facts-check-c{ci}-j{j}", + _prompt("Facts-Check", topic=topic, source=source, facts=facts_text, out_path=chk_path(ci, j), extra=_extra(instructions)), + _timeout("content_check", len(per)), provider=provider, role="judge", capabilities=caps) + for j in pending], return_exceptions=True) + outs = [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if (s := _facts_check_schema(_json_file(chk_path(ci, j)))) is not None] + bvotes: dict[str, int] = {} + vvotes: dict[str, int] = {} + for s in outs: # s = [(sub_norm, verwerfen)] of one judge + gb, gv = set(), set() + for sn, disc in s: + if sn not in gb: + gb.add(sn); bvotes[sn] = bvotes.get(sn, 0) + 1 + if disc and sn not in gv: + gv.add(sn); vvotes[sn] = vvotes.get(sn, 0) + 1 + threshold = len(outs) / 2 if outs else 99 + flagged = {sn for sn, v in bvotes.items() if v > threshold} + # Discarding is irreversible → stricter than flagging: majority AND ≥2 agreeing judges + # (prevents deletion by a single vote when the panel is degraded). + to_discard = {sn for sn, v in vvotes.items() if v > threshold and v >= 2} + return ci, flagged, to_discard + + check = await _gather_progress([_check(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts check")) + if is_cancelled(): + return None + flagged: dict[int, set] = {} + to_discard: dict[int, set] = {} + for r in check: + if isinstance(r, tuple) and len(r) == 3: + ci, b, v = r + flagged[ci] = b + to_discard[ci] = v + + # Phase "Facts fix": re-extract only CORRECTABLE ones (flagged without discard). + correctable = {ci: (flagged.get(ci, set()) - to_discard.get(ci, set())) for ci in flagged} + n_problem = sum(len(s) for s in correctable.values()) + if n_problem: + set_p(f"Correcting facts ({n_problem})…", step=_step_idx(topic, "Facts fix")) + async def _fix(ci): + subs_norm = correctable.get(ci, set()) + if not subs_norm or _facts_schema(_json_file(fix_path(ci))): + return + idxs = chunks[ci] + rel_by = {blocks[i][0]: blocks[i][1] for i in idxs} + goal = [] + for bt, subs in rel_by.items(): + affected_subs = [s for s in subs if _norm_title(s) in subs_norm] + if affected_subs: + goal.append(f"BLOCK: {bt}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in affected_subs)) + if not goal: + return + await run_single_slot( + ctx, f"Facts-Fix {ci}", key=f"blocks-{topic}-facts-fix-c{ci}", + prompt=_prompt("Facts-Research", topic=topic, source=source, blocks="\n\n".join(goal), out_path=fix_path(ci), extra=_extra(instructions)), + role="guide", capabilities=caps, + payload=lambda result, p=fix_path(ci): _facts_schema(_json_file(p)), + timeout=_timeout("content", len(subs_norm))) + await _gather_progress([_fix(ci) for ci in correctable], len(correctable), _report_p(set_p, topic, "Facts fix")) + if is_cancelled(): + return None + + # assemble: raw + fix overrides for corrected. Discarded subs out (+ report per block). + outcome: dict[str, dict] = {} + discarded_map: dict[str, set] = {} + for ci in range(len(chunks)): + per = _chunk_facts(ci) # raw + supplements (recall); fix overrides only corrected + fix = raw_map(ci, fix_path(ci)) if fix_path(ci).exists() else {} + disc = to_discard.get(ci, set()) + for bt, fm in per.items(): + for sn, fk in fm.items(): + if sn in disc: + discarded_map.setdefault(bt, set()).add(sn) + continue + winners = fix.get(bt, {}).get(sn, fk) if sn in correctable.get(ci, set()) else fk + outcome.setdefault(bt, {})[sn] = {k: winners[k] for k in _FACTS_FIELDS} + if discarded_map: + _log(topic, f"Facts check discards {sum(len(s) for s in discarded_map.values())} unsupportable subblocks") + return outcome, discarded_map + + +async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: + """Block D: three phases with a barrier — find (relevant/peripheral), select (vote), clarify. + Items from the sidecar; local IDs 1..n per package → global gid. + → {gid: relevance} or None on cancel/research error. Default on gap/dispute: 'relevant'.""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + work_dir = files["arbeit"] + items = [(title, sub["title"], sub.get("facts")) for title, subs in sidecar.items() for sub in subs] # global id = index+1 + if not items: + return {} + chunks = _chunk_nums(list(range(len(items))), _n_chunks(len(items), LEVEL_CHUNK)) + n = len(chunks) + + def rater_paths(c): + return [work_dir / f"relevance-c{c}-{i}.json" for i in (1, 2, 3)] + + def lset(item_idxs): + return set(range(1, len(item_idxs) + 1)) + + # Phase "Relevance find": 3 raters per package (min. 2), local IDs. + async def _rate(c, item_idxs): + local_set = lset(item_idxs) + paths = rater_paths(c) + existing = sum(1 for p in paths if _relevance_schema(_json_file(p), local_set)) + if existing >= 2: + return True + enum_lines = [] + for k, j in enumerate(item_idxs, 1): + enum_lines.append(f"{k}. [{items[j][0]}] {items[j][1]}") + if (kz := _core_line(items[j][2])): + enum_lines.append(f" {kz}") + enum = "\n".join(enum_lines) + pending = [(i, p) for i, p in enumerate(paths, 1) if not _relevance_schema(_json_file(p), local_set)] + slots = [{ + "key": f"blocks-{topic}-relevance-c{c}-{i}", + "prompt": _prompt("Relevance-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)), + "role": "fast", "capabilities": "files", + "payload": (lambda result, p=p, ids=local_set: _relevance_schema(_json_file(p), ids)), + } for i, p in pending] + new = await _race(topic, f"Relevance package {c}", slots, 2 - existing, _timeout("relevance", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) + return not is_cancelled() and new is not None + + oks = await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Relevance find")) + if is_cancelled(): + return None + if not all(ok is True for ok in oks): + _blocks_errors[topic] = "Relevance failed (research)" + return None + + # Phase "Relevance select": code vote per package → (outcome, disputed). + set_p(f"Relevance select ({n} packages)…", step=_step_idx(topic, "Relevance select")) + vote_by_c = {} + for c, item_idxs in enumerate(chunks, 1): + local_set = lset(item_idxs) + rater = [d for p in rater_paths(c) if (d := _relevance_schema(_json_file(p), local_set))] + vote_by_c[c] = _code_vote(rater, len(item_idxs)) + + # Phase "Relevance clarify": one judge per package on the disputed items, all in parallel. + async def _clarify(c, item_idxs): + outcome, strittig = vote_by_c[c] + if strittig: + judge_path = work_dir / f"relevance-final-c{c}.json" + decision = _relevance_schema(_json_file(judge_path), set(strittig)) + if decision is None: + disputed_block = _disputed_lines(items, item_idxs, strittig) + status, decision = await run_single_slot( + ctx, f"Relevance-Clarification {c}", + key=f"blocks-{topic}-relevance-final-c{c}", + prompt=_prompt("Relevance-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)), + role="judge", capabilities="files", + payload=lambda result, p=judge_path, ids=set(strittig): _relevance_schema(_json_file(p), ids), + timeout=_timeout("relevance_check", len(strittig)), + ) + if status == FAILED: + _log(topic, f"Relevance clarification package {c} failed — default 'relevant'") + decision = decision if isinstance(decision, dict) else {} + # disputed without a decision → 'relevant' (never accidentally exclude). + outcome = {**{k: "relevant" for k in strittig}, **outcome, **decision} + return {item_idxs[k - 1] + 1: rel for k, rel in outcome.items()} + + parts = await _gather_progress([_clarify(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Relevance clarify")) + if is_cancelled(): + return None + relevance_by_id: dict[int, str] = {} + for c, part in enumerate(parts, 1): + if not isinstance(part, dict): + # clarification is not fatal: vote outcome + default 'relevant' for disputed. + if isinstance(part, BaseException): + _log(topic, f"Relevance clarification package {c}: {type(part).__name__}: {part}") + outcome, strittig = vote_by_c[c] + item_idxs = chunks[c - 1] + merged = {**{k: "relevant" for k in strittig}, **outcome} + part = {item_idxs[k - 1] + 1: s for k, s in merged.items()} + relevance_by_id.update(part) + return relevance_by_id + + +def _match_sub(agent_sub: str, rel: list[str]) -> str: + """Map the agent's subblock title to the matching relevant title — exact, + then normalized, then substring (the agent drops e.g. the prefix "Question: "). + No match → keep the agent title. This way NO pattern is lost to a title mismatch.""" + if agent_sub in rel: + return agent_sub + an = _norm_title(agent_sub) + for r in rel: + rn = _norm_title(r) + if an and rn and (an == rn or an in rn or rn in an): + return r + return agent_sub + + +async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: + """Block E (chunks of 10): find (1 generator per ~10 blocks, parallel), select (code: + group per block + dedup), clarify (1 critic per chunk), check (catch-up round). + Assignment per entry via the `block` field (a chunk file carries several blocks). + → {block title: [{subblock, question}, …]} or None on cancel.""" + topic, is_cancelled = ctx.topic, ctx.is_cancelled + work_dir = files["arbeit"] + # ALL subblocks (including peripheral) get a pattern — peripheral is testable in the FGuide level. + # facts_by: full facts context per sub (generation benefits — better questions). + blocks = [] + facts_by: dict[tuple, dict] = {} + for title, subs in sidecar.items(): + all_titles = [] + for s in subs: + if isinstance(s, dict) and (st := str(s.get("title", "")).strip()): + all_titles.append(st) + if isinstance(s.get("facts"), dict): + facts_by[(title, _norm_title(st))] = s["facts"] + if all_titles: + blocks.append((title, all_titles)) + if not blocks: + return {} + chunks = _lpt_chunks([len(rel) for _, rel in blocks], QUESTION_CHUNK_SUBS) # load-balanced by sub count + + def raw_path(ci): + return work_dir / f"question-pattern-c{ci}.json" + + def final_path(ci): + return work_dir / f"question-pattern-final-c{ci}.json" + + def _chunk_title(idxs): + return [blocks[i][0] for i in idxs] + + # Phase "Questions find": 1 generator per chunk, all in parallel. + async def _find(ci, idxs): + fp = raw_path(ci) + if _question_pattern_chunk_schema(_json_file(fp)): + return # Resume + def _sub_line(bi, s): + line = f"- {s}" + fk = facts_by.get((blocks[bi][0], _norm_title(s))) + if fk and (ft := _facts_lines(fk)): + line += "\n" + "\n".join(" " + l for l in ft.split("\n")) + return line + block = "\n\n".join( + f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(_sub_line(i, s) for s in blocks[i][1]) + for i in idxs + ) + subs_total = sum(len(blocks[i][1]) for i in idxs) + status, _ = await run_single_slot( + ctx, f"Question-Pattern {ci}", + key=f"blocks-{topic}-question-pattern-c{ci}", + prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=block, + out_path=fp, extra=_extra(instructions)), + role="fast", capabilities="files", + payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)), + timeout=_timeout("question_pattern", subs_total), + ) + if status == FAILED: + _log(topic, f"Question pattern chunk {ci} failed — blocks in fallback (catch-up round/live)") + + async def find_all(ci_list): + ci_list = list(ci_list) + await _gather_progress([_find(ci, chunks[ci]) for ci in ci_list], len(ci_list), _report_p(set_p, topic, "Questions find")) + + await find_all(range(len(chunks))) + if is_cancelled(): + return None + + # Phase "Questions select": code — group chunk files per block, drop duplicates, + # loosely map block/subblock titles to the targets (discard nothing for a mismatch). + def _select_chunk(ci): + idxs = chunks[ci] + ctitle = _chunk_title(idxs) + rel_by = {blocks[i][0]: blocks[i][1] for i in idxs} + out, seen_set = {}, {} + for e in _question_pattern_chunk_schema(_json_file(raw_path(ci))) or []: + title = _match_sub(e["block"], ctitle) + if title not in rel_by: + continue # not assignable → discard + sub = _match_sub(e["subblock"], rel_by[title]) + seen = seen_set.setdefault(title, set()) + if sub in seen: + continue # exactly one pattern per subblock + seen.add(sub) + out.setdefault(title, []).append({"subblock": sub, "question": e["question"]}) + return out + + def _select_all(ci_list): + raw = {} + for ci in ci_list: + for title, eintraege in _select_chunk(ci).items(): + raw.setdefault(title, []).extend(eintraege) + return raw + + set_p("Questions select…", step=_step_idx(topic, "Questions select")) + raw_by_title = _select_all(range(len(chunks))) + + # Phase "Questions clarify": 1 critic per chunk cleans up the tables (grouped by block). + async def _clarify(ci, idxs): + fp = final_path(ci) + if _question_pattern_chunk_schema(_json_file(fp)): + return # resume + block_texts = [] + for i in idxs: + t = blocks[i][0] + eintraege = raw_by_title.get(t) or [] + if not eintraege: + continue + lines = "\n".join(f"- ({e['subblock']}) {e['question']}" for e in eintraege) + block_texts.append(f"BLOCK: {t}\n{lines}") + if not block_texts: + return # nothing to clarify in this chunk + subs_total = sum(len(blocks[i][1]) for i in idxs) + status, _ = await run_single_slot( + ctx, f"Question-Pattern-Clarification {ci}", + key=f"blocks-{topic}-question-pattern-final-c{ci}", + prompt=_prompt("Question-Pattern-Critique", topic=topic, table="\n\n".join(block_texts), out_path=fp, extra=_extra(instructions)), + role="judge", capabilities="files", + payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)), + timeout=_timeout("question_pattern_check", subs_total), + ) + if status == FAILED: + _log(topic, f"Question pattern clarification chunk {ci} failed — raw pattern adopted") + + async def clarify_all(ci_list): + ci_list = list(ci_list) + await _gather_progress([_clarify(ci, chunks[ci]) for ci in ci_list], len(ci_list), _report_p(set_p, topic, "Questions clarify")) + + await clarify_all(range(len(chunks))) + if is_cancelled(): + return None + + # Clarified chunk table per block, fallback to raw pattern. Map titles loosely. + def _final_by_title(ci_list): + out = {} + for ci in ci_list: + idxs = chunks[ci] + ctitle = _chunk_title(idxs) + rel_by = {blocks[i][0]: blocks[i][1] for i in idxs} + for e in _question_pattern_chunk_schema(_json_file(final_path(ci))) or []: + title = _match_sub(e["block"], ctitle) + if title not in rel_by: + continue + out.setdefault(title, []).append( + {"subblock": _match_sub(e["subblock"], rel_by[title]), "question": e["question"]}) + return out + + final_by_title = _final_by_title(range(len(chunks))) + outcome = {t: (final_by_title.get(t) or raw_by_title.get(t) or []) for t, _ in blocks} + + # Phase "Questions check": per-sub completeness. Generators crash randomly (~15 %), + # 1 agent per chunk without retry → subs (whole blocks) fall through silently. Hence several + # rounds that re-request ONLY the missing subs (short packages, Question-Pattern-Research). + set_p("Questions check…", step=_step_idx(topic, "Questions check")) + + def _missing_subs() -> list[tuple[str, list[str]]]: + out = [] + for t, subs in blocks: + have_set = {_norm_title(e["subblock"]) for e in outcome.get(t) or []} + miss = [s for s in subs if _norm_title(s) not in have_set] + if miss: + out.append((t, miss)) + return out + + def _followup_block(items): # items: [(block_title, [missing sub_title])] + block_texts = [] + for title, subs in items: + lines = [] + for s in subs: + z = f"- {s}" + fk = facts_by.get((title, _norm_title(s))) + if fk and (ft := _facts_lines(fk)): + z += "\n" + "\n".join(" " + l for l in ft.split("\n")) + lines.append(z) + block_texts.append(f"BLOCK: {title}\nSUBBAUSTEINE:\n" + "\n".join(lines)) + return "\n\n".join(block_texts) + + async def _request_more(round_n, pi, items): + fp = work_dir / f"question-pattern-nach{round_n}-c{pi}.json" + if _question_pattern_chunk_schema(_json_file(fp)): + return # resume + subs_total = sum(len(s) for _, s in items) + await run_single_slot( + ctx, f"Question pattern catch-up R{round_n}/{pi}", + key=f"blocks-{topic}-question-pattern-nach{round_n}-c{pi}", + prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=_followup_block(items), + out_path=fp, extra=_extra(instructions)), + role="fast", capabilities="files", + payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)), + timeout=_timeout("question_pattern", subs_total), + ) + + for round_n in range(1, QUESTION_MAX_ROUNDS + 1): + missing_subs = _missing_subs() + if not missing_subs: + break + n_subs = sum(len(s) for _, s in missing_subs) + _log(topic, f"Question pattern round {round_n}: {n_subs} sub(s) in {len(missing_subs)} block(s) without a pattern — re-request") + packages = _lpt_chunks([len(s) for _, s in missing_subs], QUESTION_CHUNK_SUBS) + package_items = [[missing_subs[i] for i in idxs] for idxs in packages] + await _gather_progress( + [_request_more(round_n, pi, items) for pi, items in enumerate(package_items)], + len(package_items), _report_p(set_p, topic, "Questions check")) + if is_cancelled(): + return None + # parse output per package + merge newly gained subs (don't overwrite existing ones). + for pi, items in enumerate(package_items): + title_subs = {t: subs for t, subs in items} + ctitle = list(title_subs.keys()) + for e in _question_pattern_chunk_schema(_json_file(work_dir / f"question-pattern-nach{round_n}-c{pi}.json")) or []: + title = _match_sub(e["block"], ctitle) + if title not in title_subs: + continue + sub = _match_sub(e["subblock"], title_subs[title]) + have_set = {_norm_title(x["subblock"]) for x in outcome.get(title) or []} + if _norm_title(sub) in have_set: + continue + outcome.setdefault(title, []).append({"subblock": sub, "question": e["question"]}) + + rest = _missing_subs() + if rest: + n = sum(len(s) for _, s in rest) + _log(topic, f"Question pattern: {n} sub(s) in {len(rest)} block(s) remain empty after {QUESTION_MAX_ROUNDS} rounds: {[t for t, _ in rest][:5]}") + return outcome + + +# ── Inventory in the DB: research loop · consolidation · clarification ──────────── + + + +def _crawl_index(folder) -> dict[str, str]: + """Alias (filename OR QUELLE: URL, lowercase) → canonical page key (filename).""" + idx: dict[str, str] = {} + if not folder or not Path(folder).is_dir(): + return idx + for p in sorted(Path(folder).glob("*.txt")): + key = p.name + idx[key.lower()] = key + try: + first_line = p.read_text(encoding="utf-8").splitlines()[0] + except (OSError, IndexError): + first_line = "" + if first_line.startswith("QUELLE:"): + url = first_line[len("QUELLE:"):].strip() + if url: + idx[url.lower()] = key + idx[url.rstrip("/").lower()] = key + return idx + + + + + + +async def _set_inventory(topic: str, record: str, status: str) -> None: + """Write an inventory entry ('title — description') with status to the DB.""" + title = _title(record) + norm = _norm_title(title) + if not norm: + return + split_parts = [t.strip() for t in record.split(" — ")] + desc = split_parts[1] if len(split_parts) >= 2 else "" + await db.upsert_block(topic, norm, title, desc) + await db.set_block_status(topic, norm, status) + + +def _triage_rules(folder, pages: list[str]) -> tuple[list[str], list[str]]: + """Deterministic content/noise filter (config.CRAWL_*). Substring match (lowercase) against + URL + filename. Order: keep > noise > min_chars > keep. → (content, noise).""" + folder = Path(folder) + content, noise = [], [] + for fn in pages: + lines = _read(folder / fn).splitlines() + url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else "" + body = "\n".join(lines[1:]).strip() + hay = f"{url}\n{fn}".lower() + if any(p in hay for p in CRAWL_KEEP_PATTERNS): + content.append(fn) + elif any(p in hay for p in CRAWL_NOISE_PATTERNS): + noise.append(fn) + elif len(body) < CRAWL_MIN_CHARS: + noise.append(fn) + else: + content.append(fn) # default: keep — everything with content stays + return content, noise + + +def _page_snippet(folder, fn: str) -> tuple[str, str]: + """(url, snippet) of a crawl page for the relevance gate. url from the QUELLE: line; + snippet = body excerpt (navigation boilerplate is up front — the prompt ignores it). + The URL is the primary signal (meaningful slug), the snippet only supports it.""" + lines = _read(Path(folder) / fn).splitlines() + url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else "" + body = "\n".join(lines[1:]).strip() + snippet = " ".join(body.split())[:QUELLE_RELEVANZ_SNIPPET] + return (url or fn), snippet + + +async def _relevance_triage(ctx: GenContext, set_p, files: dict, folder, content: list[str], spec: str, instructions: str) -> tuple[list[str], list[str]]: + """LLM topic gate after the rule filter: each content page ja/nein against the spec. + Off-topic (different field) → out. Pattern like `_relevance_block`: small packages, 3 raters + (`fast`), 2-of-3 consensus. CONSERVATIVE: drop only on a clear "nein" majority; dispute/gap/ + race error → keep. SAFETY: if the gate would drop ≥80 % (or all), everything stays + (a spec mismatch/bug must not empty the source). → (kept, out) as filenames.""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + work_dir = files["arbeit"] + pages = sorted(content) + if not pages: + return content, [] + items = [_page_snippet(folder, fn) for fn in pages] # index aligns with `pages` + chunks = _chunk_nums(list(range(len(pages))), _n_chunks(len(pages), QUELLE_RELEVANZ_CHUNK)) + n = len(chunks) + + def rater_paths(c): + return [work_dir / f"source-relevance-c{c}-{i}.json" for i in (1, 2, 3)] + + def lset(idxs): + return set(range(1, len(idxs) + 1)) + + async def _rate(c, idxs): + local_set = lset(idxs) + paths = rater_paths(c) + existing = sum(1 for p in paths if _yesno_schema(_json_file(p), local_set)) + if existing >= 2: + return True + enum_lines = [] + for k, j in enumerate(idxs, 1): + url, snip = items[j] + enum_lines.append(f"{k}. {url}") + if snip: + enum_lines.append(f" {snip}") + enum = "\n".join(enum_lines) + pending = [(i, p) for i, p in enumerate(paths, 1) if not _yesno_schema(_json_file(p), local_set)] + slots = [{ + "key": f"blocks-{topic}-source-relevance-c{c}-{i}", + "prompt": _prompt("Source-Relevance", topic=topic, spec=spec, pages=enum, out_path=p, extra=_extra(instructions)), + "role": "fast", "capabilities": "files", + "payload": (lambda result, p=p, ids=local_set: _yesno_schema(_json_file(p), ids)), + } for i, p in pending] + new = await _race(topic, f"Relevance triage package {c}", slots, 2 - existing, _timeout("relevance", len(idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) + return not is_cancelled() and new is not None + + _qidx = _step_idx(topic, "Source prep") # gate runs in the source step (no own step) + set_p(f"Check relevance against spec ({n} packages)…", step=_qidx) + async def _report_triage(d, t): + set_p(f"Check relevance against spec {d}/{t}…", step=_qidx) + await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], n, _report_triage) + if is_cancelled(): + return content, [] # cancel → drop nothing (caller aborts) + + # Vote per page: only a clear "nein" majority (≥2 and more than "ja") throws it out. + dropped: list[str] = [] + for c, idxs in enumerate(chunks, 1): + local_set = lset(idxs) + rater = [d for p in rater_paths(c) if (d := _yesno_schema(_json_file(p), local_set))] + for k in range(1, len(idxs) + 1): + vote_list = [d[k] for d in rater if k in d] + nein, ja = vote_list.count("nein"), vote_list.count("ja") + if nein >= 2 and nein > ja: + dropped.append(pages[idxs[k - 1]]) + + if dropped and len(dropped) >= max(1, int(len(pages) * 0.8)): + _log(topic, f"Relevance triage: would drop {len(dropped)}/{len(pages)} — discarded (spec mismatch?), keeping all") + return content, [] + dropped_set = set(dropped) + keepers = [fn for fn in pages if fn not in dropped_set] + return keepers, dropped + + +async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool: + """Step "Source prep": crawl (link) + PDF convert + content/noise triage. + Persists the triage in the coverage table (content). → True (ok) / False (cancel/error). + thema: nothing. projekt/uni: only PDFs (curated folder, no triage).""" + topic, is_cancelled = ctx.topic, ctx.is_cancelled + if not folder: + return True # thema → no source to prepare + if q["type"] != "link": + await asyncio.to_thread(_convert_pdfs, folder) # projekt/uni: only PDFs, no triage + return True + if await db.get_step_status(topic, "Source prep") == "done": + return True + if not _crawl_done(topic): + set_p("Loading source (crawl)…", step=_step_idx(topic, "Source prep")) + n = await asyncio.to_thread(crawl, q["location"], folder, cancelled=is_cancelled) + if is_cancelled(): + return False + if not n: + _blocks_errors[topic] = "Crawl yielded no content — check link/domain" + return False + await asyncio.to_thread(_convert_pdfs, folder) + pages = sorted(set(_crawl_index(folder).values())) + if pages: + set_p("Triaging pages…", step=_step_idx(topic, "Source prep")) + await db.delete_coverage(topic) + content, noise = _triage_rules(folder, pages) # deterministic rule filter + if q.get("spec") and content: # topic gate: separates the field (rules can't) + content, dropped = await _relevance_triage(ctx, set_p, files, folder, content, q["spec"], instructions) + if is_cancelled(): + return False + if dropped: + noise = sorted(set(noise) | set(dropped)) + _log(topic, f"LLM relevance: {len(dropped)} pages off-topic → noise") + await db.mark_content(topic, sorted(content), sorted(noise)) + _log(topic, f"Triage: {len(content)} content / {len(noise)} noise of {len(pages)} (rules + LLM gate)") + await db.set_step_status(topic, "Source prep", "done") + return True + + +async def _research_batch(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool: + """Fills DB table `blocks` with candidates (+ mention counter). FIXED file batches: + each crawl page is assigned to exactly one batch and read by RESEARCH_READERS agents + (consensus ≥2 in the batch). All assigned pages are marked as read → 100 % coverage. + Without a crawl folder (source "thema") → free web research, one round. → True/False.""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + if await db.get_step_status(topic, "Research") == "done": + return True + work_dir = files["arbeit"] + await db.delete_blocks(topic) # coverage/content belongs to the triage — do NOT delete + await db.set_step_status(topic, "Research", "running") + + async def _ingest(reader_id: str, text: str) -> None: + seen_set = set() + for record in _parse_selection(text).values(): + title = _title(record) + norm = _norm_title(title) + if not norm or norm in seen_set: + continue + seen_set.add(norm) # one reader = one vote per concept + split_parts = [t.strip() for t in record.split(" — ")] + desc = split_parts[1] if len(split_parts) >= 2 else "" + source = [split_parts[2]] if len(split_parts) >= 3 and split_parts[2] else [] + await db.upsert_block(topic, norm, title, desc, source, reader=reader_id) + + pages = await db.list_content(topic) # pages marked as content by the triage + if not pages and folder: + pages = sorted(set(_crawl_index(folder).values())) # fallback (projekt/uni: no triage) + + if not pages: + # source "thema" (or no crawl): free web research, one round. + set_p("Research running…", step=_step_idx(topic, "Research")) + caps = "files" if folder else "full" + paths = [work_dir / f"research-{i}.md" for i in range(1, RESEARCH_THEMA_AGENTS + 1)] + for p in paths: + p.unlink(missing_ok=True) + slots = [{ + "key": f"blocks-{topic}-research-{i}", + "prompt": _build_research_prompt(topic, p, instructions, q["type"], folder), + "role": "quick", "capabilities": caps, + "payload": (lambda result, p=p, rid=f"t{i}": ((rid, t) if (t := _file_payload(p)) else None)), + } for i, p in enumerate(paths, 1)] + agent_texts = await _race(topic, "Research", slots, 3, _timeout("research"), provider, + cancelled=is_cancelled, grace=RESEARCH_GRACE) + if is_cancelled(): + return False + if not agent_texts: + _blocks_errors[topic] = "Research failed (minimum not reached)" + return False + for rid, text in agent_texts: + await _ingest(rid, text) + await db.set_step_status(topic, "Research", "done") + return True + + # uni/projekt: curated, often LARGE files (script). Instead of reading all at once + # (lost-in-the-middle), chunk into sections and have 2 readers thoroughly read EACH — + # text directly in the prompt (small context), mentions accumulate to consensus. + if q["type"] in ("uni", "projekt"): + eintraege: list[tuple[str, str]] = [] # (filename, section text) + for fn in sorted(pages): + for section_text in _text_sections(_read(folder / fn)): + eintraege.append((fn, section_text)) + if not eintraege: + _blocks_errors[topic] = "Research: source empty" + return False + set_p(f"Research ({len(eintraege)} sections)…", step=_step_idx(topic, "Research")) + + async def _read_section(ei: int, fn: str, section_text: str) -> None: + block = (f"ARBEITE AUSSCHLIESSLICH MIT DIESEM TEXTABSCHNITT (Source: {fn}). Lies ihn " + f"VOLLSTÄNDIG, überspringe nichts. Notiere `{fn}` als Source jedes Bausteins. " + f"Suche NICHT im Web — nur dieser Section zählt.\n\n-----\n{section_text}\n-----") + paths = [work_dir / f"research-a{ei}-{i}.md" for i in range(1, RESEARCH_READERS + 1)] + # reader file reuse: if all reader outputs are present and valid (resume / + # re-run without research change), re-ingest instead of spawning agents again. + existing = [(f"a{ei}-{i}", t) for i, p in enumerate(paths, 1) if (t := _file_payload(p))] + if len(existing) == len(paths): + for rid, text in existing: + await _ingest(rid, text) + return + for p in paths: + p.unlink(missing_ok=True) + if is_cancelled(): + return + slots = [{ + "key": f"blocks-{topic}-research-a{ei}-{i}", + "prompt": _build_research_prompt(topic, p, instructions, q["type"], folder, section=block), + "role": "quick", "capabilities": "files", + "payload": (lambda result, p=p, rid=f"a{ei}-{i}": ((rid, t) if (t := _file_payload(p)) else None)), + } for i, p in enumerate(paths, 1)] + # quorum 2: both readers per section should pass (more eyes = more concepts + + # real consensus); after timeout _race falls back to what exists. + agent_texts = await _race(topic, f"Research section {ei}", slots, 2, _timeout("research", 1), + provider, cancelled=is_cancelled, grace=RESEARCH_GRACE) + for rid, text in (agent_texts or []): + await _ingest(rid, text) + + await _gather_progress([_read_section(ei, fn, a) for ei, (fn, a) in enumerate(eintraege, 1)], + len(eintraege), _report_p(set_p, topic, "Research")) + if is_cancelled(): + return False + await db.mark_sources_read_done(topic, sorted(pages)) + total = len(await db.list_blocks(topic)) + _log(topic, f"Research (uni/projekt): {total} candidates from {len(eintraege)} sections ({len(pages)} files)") + if not total: + _blocks_errors[topic] = "Research failed (no blocks)" + return False + await db.set_step_status(topic, "Research", "done") + return True + + # Crawl/link: many small content pages (triage in the "Source prep" step). + # Fixed batches, RESEARCH_READERS readers per batch reading EXACTLY these files. + batches = _chunk_nums(sorted(pages), max(1, math.ceil(len(pages) / RESEARCH_BATCH))) + + async def _read_batch(bi: int, batch: list[str]) -> bool: + liste = "\n".join(f"- {p}" for p in batch) + fokus = ("WICHTIG — feste Assignment: Bearbeite AUSSCHLIESSLICH diese Dateien und lies JEDE " + f"vollständig. Ignoriere alle anderen Dateien im Ordner:\n{liste}") + paths = [work_dir / f"research-b{bi}-{i}.md" for i in range(1, RESEARCH_READERS + 1)] + for p in paths: + p.unlink(missing_ok=True) + if not is_cancelled(): + slots = [{ + "key": f"blocks-{topic}-research-b{bi}-{i}", + "prompt": _build_research_prompt(topic, p, instructions, q["type"], folder, focus=fokus), + "role": "quick", "capabilities": "files", + "payload": (lambda result, p=p, rid=f"b{bi}-{i}": ((rid, t) if (t := _file_payload(p)) else None)), + } for i, p in enumerate(paths, 1)] + agent_texts = await _race(topic, f"Research batch {bi}", slots, 1, _timeout("research", len(batch)), + provider, cancelled=is_cancelled, grace=RESEARCH_GRACE) + for rid, text in (agent_texts or []): + await _ingest(rid, text) + await db.mark_sources_read_done(topic, batch) # tick off all assigned pages (even without hits) + return not is_cancelled() + + await _gather_progress([_read_batch(bi, b) for bi, b in enumerate(batches, 1)], + len(batches), _report_p(set_p, topic, "Research")) + if is_cancelled(): + return False + total = len(await db.list_blocks(topic)) + coverage = len(await db.list_coverage(topic)) + _log(topic, f"Research: {total} candidates, coverage {coverage}/{len(pages)} pages ({len(batches)} batches)") + if not total: + _blocks_errors[topic] = "Research failed (no blocks)" + return False + await db.set_step_status(topic, "Research", "done") + return True + + +def _grp_schema(data, ids: set[int]): + """{"groups": [[1,3],[2], …]} → partition of `ids` as a list of index groups. + Tolerant: ignores foreign/duplicate numbers; forgotten candidates are added standalone + (singleton group). None only on structurally broken JSON.""" + if not isinstance(data, dict) or not isinstance(data.get("groups"), list): + return None + groups, seen_set = [], set() + for g in data["groups"]: + if not isinstance(g, list): + return None + grp = [] + for x in g: + try: + num = int(x) + except (ValueError, TypeError): + continue + if num in ids and num not in seen_set: + seen_set.add(num) + grp.append(num) + if grp: + groups.append(grp) + groups += [[r] for r in sorted(ids - seen_set)] # forgotten candidates stay standalone + return groups or None + + +_ASPECT_MARKER = ("∈ np", "∈np", " in np", "np-schwer", "np-vollständig", "verifizierer", + "zertifikat", "ndtm", "nicht-determ", "lower bound", "untere schranke", + "bzgl", "als sprache") + + +def _aspect_marker(title: str) -> int: + """Number of property markers in the title (∈NP, NP-hard, verifier, lower bound …). + 0 = generic main concept (the problem itself); >0 = a property of it.""" + t = title.casefold() + return sum(1 for m in _ASPECT_MARKER if m in t) + + +_REFERENCE_RE = re.compile(r'^(Satz|Lemma|Korollar|Bemerkung|Definition)\s*[\d.]+\s*(\([a-z]\)|[a-z])?\s*$', re.I) + + +def _is_reference(title: str) -> bool: + """True for pure reference/placeholder titles WITHOUT meaningful content: "Satz 7.18", "Lemma 6.2", + "Korollar 6.18" (number without a name) as well as marked spots "Bedingung (**)". NOT "Satz 6.24: + Cook/Levin" (has a name) and NOT short technical symbols like "P⊆NP"/"Σ*" (real concepts).""" + t = title.strip() + if _REFERENCE_RE.match(t): + return True + if re.search(r'\(\*+\)', t): # marked spot "(**)" / "(*)" + return True + return False + + +def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) -> dict: + """Representative of a cluster = the main concept (fewest property markers — the problem + itself, not "… ∈ NP"); tie → most frequent norm title → most readers. Title globally unique + (suffix ' (2)') so it works as a key.""" + by_norm: dict[str, list[int]] = {} + for k in idxs: + by_norm.setdefault(_norm_title(candidates[k]["title"]), []).append(k) + + def weight(nb: str): + ms = by_norm[nb] + reader = set().union(*[set(candidates[m]["reader"]) for m in ms]) if ms else set() + # reference/placeholder titles ("Satz 7.18") last — prefer a meaningful member. + is_real = not _is_reference(candidates[ms[0]]["title"]) + return (is_real, -_aspect_marker(nb), len(ms), len(reader)) + + best = max(by_norm, key=weight) + k = max(by_norm[best], key=lambda m: len(candidates[m]["description"])) + title = candidates[k]["title"] + n = 2 + while _norm_title(title) in seen_norm: + title = f"{candidates[k]['title']} ({n})" + n += 1 + seen_norm.add(_norm_title(title)) + return {"title": title, "description": candidates[k]["description"]} + + +async def _group_blocks(ctx: GenContext, set_p, work_dir: Path, candidates: list[dict], + blocks: list[list[int]], prefix: str = "consolidation", + step: str = "Consolidation") -> list[list[int]]: + """Per similarity block, a judge groups the titles into the real blocks (merge + paraphrases, split over-merges). Singletons directly. Error/timeout → conservatively each + candidate alone (avoids false over-merging). → final groups (global indices). + `praefix`/`step` separate consolidation and dedup (artefacts, race key, progress).""" + topic, is_cancelled = ctx.topic, ctx.is_cancelled + multi = [(bi, b) for bi, b in enumerate(blocks) if len(b) > 1] + outcome: list[list[int]] = [list(b) for b in blocks if len(b) == 1] # singletons directly + + def _line(k: int, g: int) -> str: + b = candidates[g] + return f"{k}. {b['title']}" + (f" — {b['description']}" if b["description"] else "") + + async def _grp(bi: int, block: list[int]) -> None: + ids = set(range(1, len(block) + 1)) + p = work_dir / f"{prefix}-block-c{bi}.json" + part = _grp_schema(_json_file(p), ids) + if part is None: # resume: don't recompute a valid file + p.unlink(missing_ok=True) + if is_cancelled(): + return + lines = [_line(k, block[k - 1]) for k in range(1, len(block) + 1)] + status, part = await run_single_slot( + ctx, f"Block grouping {bi}", + key=f"blocks-{topic}-{prefix}-block-c{bi}", + prompt=_prompt("Blocks-Block-Grouping", topic=topic, entries="\n".join(lines), out_path=p), + role="judge", capabilities="files", + payload=(lambda result, p=p, ids=ids: _grp_schema(_json_file(p), ids)), + timeout=_timeout("research_mapping", len(block)), + ) + part = part if status == OK else None + if part is None: # judge failed → individually (no over-merge) + outcome.extend([idx] for idx in block) + else: # local numbers → global candidate indices + outcome.extend([block[k - 1] for k in g] for g in part) + + await _gather_progress([_grp(bi, b) for bi, b in multi], + len(multi), _report_p(set_p, topic, step)) + return outcome + + +async def _consolidate_embedding(ctx: GenContext, set_p, files: dict, candidates: list[dict]) -> bool: + """Two-stage: embeddings → coarse capped blocks (high recall) → one judge per multi-block, + grouping the titles into the real blocks → reader union (≥2 = consensus).""" + topic, is_cancelled = ctx.topic, ctx.is_cancelled + work_dir = files["arbeit"] + texts = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in candidates] + sims = await asyncio.to_thread(embedding.embed_sims, texts) + if sims is None: # model not available after all → fallback + return await _consolidate_llm(ctx, set_p, files, candidates) + # Level 1: coarse similarity blocks (capped, no giant component). + blocks = await asyncio.to_thread(embedding.capped_blocks, sims, None, None) + # Level 2: one judge groups EACH multi-block into the real blocks. + groups = await _group_blocks(ctx, set_p, work_dir, candidates, blocks) + if is_cancelled(): + return False + + def _min_cos(idxs): # internal coherence as a check (chains would be ~0.3) + if len(idxs) < 2: + return 1.0 + return round(min(float(sims[i][j]) for n, i in enumerate(idxs) for j in idxs[n + 1:]), 3) + + # Consensus = ≥2 distinct readers per cluster. Legacy DBs without reader tracking (research ran + # before the migration, no re-ingest) have empty reader sets → fall back to a title heuristic + # (otherwise EVERYTHING would land in the rest). + hat_reader = any(b["reader"] for b in candidates) + consensus, rest, debug, seen_norm = [], [], [], set() + for idxs in groups: + reader = set().union(*[set(candidates[k]["reader"]) for k in idxs]) if idxs else set() + if hat_reader: + score = len(reader) + else: # without reader data: max(mentions, number of distinct title variants in the cluster) + score = max(max(candidates[k]["mentions"] for k in idxs), + len({candidates[k]["title_norm"] for k in idxs})) + rep = _canonical(candidates, idxs, seen_norm) + record = f"{rep['title']} — {rep['description']}" if rep["description"] else rep["title"] + (consensus if score >= 2 else rest).append(record) + debug.append({"title": rep["title"], "reader": sorted(reader), "score": score, + "consensus": score >= 2, "min_cos": _min_cos(idxs), + "mitglieder": [candidates[k]["title"] for k in idxs]}) + atomic_write_json(work_dir / "consolidation-cluster.json", debug, indent=1) + multi_blocks = sum(1 for b in blocks if len(b) > 1) + _log(topic, f"Consolidation (embedding): {len(blocks)} blocks ({multi_blocks} grouped via LLM) " + f"→ {len(groups)} clusters from {len(candidates)} candidates " + f"→ {len(consensus)} consensus / {len(rest)} rest") + + await db.delete_blocks(topic) + for t in consensus: + await _set_inventory(topic, t, "consensus") + for t in rest: + await _set_inventory(topic, t, "rest") + await db.set_step_status(topic, "Consolidation", "done") + return True + + +async def _consolidate(ctx: GenContext, set_p, files: dict) -> bool: + """Merges raw candidates into consensus (≥2 readers)/rest. Deterministic via embedding clustering; + if the model is missing → fall back to the LLM panel (`_consolidate_llm`). Status in DB.""" + topic = ctx.topic + if await db.get_step_status(topic, "Consolidation") == "done": + return True + set_p("Consolidating research…", step=_step_idx(topic, "Consolidation")) + candidates = await db.list_blocks(topic) + if not candidates: + _blocks_errors[topic] = "Consolidation: no candidates" + return False + if EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available): + return await _consolidate_embedding(ctx, set_p, files, candidates) + return await _consolidate_llm(ctx, set_p, files, candidates) + + +async def _consolidate_llm(ctx: GenContext, set_p, files: dict, candidates: list[dict]) -> bool: + """Fallback (only without an embedding model): a panel (KONSOLIDIERUNG_PANEL judges) merges + candidates semantically; a reconcile judge combines the panel outputs into the final + consensus (≥2)/rest (1×) list. Panel instead of a single judge: a single judge is bias-prone and unstable.""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + work_dir = files["arbeit"] + chunks = _chunk_nums(candidates, max(1, math.ceil(len(candidates) / CONSOLIDATION_CHUNK))) + + async def _map_panel(c: int, eintraege: str, amount: int): + """3 mapping judges over `eintraege` → reconcile judge → (consensus, rest). None on cancel/error.""" + paths = [work_dir / f"consolidation-c{c}-j{j}.json" for j in range(1, CONSOLIDATION_PANEL + 1)] + pending = [(j, p) for j, p in enumerate(paths, 1) if _mapping_schema(_json_file(p)) is None] + for _, p in pending: + p.unlink(missing_ok=True) + if pending: + slots = [{ + "key": f"blocks-{topic}-consolidation-c{c}-j{j}", + "prompt": _prompt("Blocks-Research-Mapping", topic=topic, n=RESEARCH_READERS, entries=eintraege, out_path=p), + "role": "judge", "capabilities": "files", + "payload": (lambda result, p=p: _mapping_schema(_json_file(p))), + } for j, p in pending] + existing = CONSOLIDATION_PANEL - len(pending) + await _race(topic, f"Consolidation {c}", slots, max(1, 2 - existing), + _timeout("research_mapping", amount), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) + if is_cancelled(): + return None + outs = [m for p in paths if (m := _mapping_schema(_json_file(p)))] + if not outs: + return None + # union of panel titles; per title count how many judges list it as consensus. + kvotes: dict[str, int] = {} + form: dict[str, str] = {} # norm → display title (first occurrence) + order: list[str] = [] + for kk, rr in outs: + for t in kk + rr: + nt = _norm_title(_title(t)) + if not nt: + continue + if nt not in form: + form[nt] = t + order.append(nt) + kvotes.setdefault(nt, 0) + for t in kk: + nt = _norm_title(_title(t)) + if nt: + kvotes[nt] = kvotes.get(nt, 0) + 1 + # Reconcile: one merge judge over the union, annotated with judge votes ("k× genannt"). + rp = work_dir / f"consolidation-c{c}-reconcile.json" + recon = _mapping_schema(_json_file(rp)) + if recon is None: + rp.unlink(missing_ok=True) + entries_r = "\n".join(f"{i}. {form[nt]} ({max(1, kvotes[nt])}× genannt)" for i, nt in enumerate(order, 1)) + status, recon = await run_single_slot( + ctx, f"Consolidation Reconcile {c}", + key=f"blocks-{topic}-consolidation-c{c}-reconcile", + prompt=_prompt("Blocks-Research-Mapping", topic=topic, n=CONSOLIDATION_PANEL, entries=entries_r, out_path=rp), + role="judge", capabilities="files", + payload=lambda result, p=rp: _mapping_schema(_json_file(p)), + timeout=_timeout("research_mapping", len(order)), + ) + if status == CANCELLED: + return None + recon = recon if status != FAILED else None + if recon: + return recon + # Fallback (reconcile failed): code majority — consensus if a majority of judges say consensus. + consensus = [form[nt] for nt in order if kvotes[nt] * 2 >= len(outs) and kvotes[nt] > 0] + kset = {_norm_title(_title(t)) for t in consensus} + return consensus, [form[nt] for nt in order if nt not in kset] + + consensus, rest = [], [] + for c, chunk in enumerate(chunks, 1): + eintraege = "\n".join( + f"{i}. {b['title']} — {b['description']} ({b['mentions']}× genannt)" for i, b in enumerate(chunk, 1) + ) + res = await _map_panel(c, eintraege, len(chunk)) + if res is None: + if is_cancelled(): + return False + _blocks_errors[topic] = "Research mapping failed" + return False + k, r = res + consensus += k + rest += r + # With multiple chunks: a global merge pass over the combined consensus entries, + # so duplicates across chunk boundaries (DAL×4, PHPUnit×5 …) merge. + if len(chunks) > 1 and consensus: + fp = work_dir / "consolidation-merge.json" + fp.unlink(missing_ok=True) + eintraege = "\n".join(f"{i}. {t} (2× genannt)" for i, t in enumerate(consensus, 1)) + status, mapping = await run_single_slot( + ctx, "Consolidation Merge", + key=f"blocks-{topic}-consolidation-merge", + prompt=_prompt("Blocks-Research-Mapping", topic=topic, n=RESEARCH_READERS, entries=eintraege, out_path=fp), + role="judge", capabilities="files", + payload=lambda result, p=fp: _mapping_schema(_json_file(p)), + timeout=_timeout("research_mapping", len(consensus)), + ) + if status == CANCELLED: + return False + if status != FAILED and mapping: + consensus, r2 = mapping + rest += r2 # entries downgraded by the merge into the rest + # Judge output is authoritative → re-set the inventory in the DB. + await db.delete_blocks(topic) + for t in consensus: + await _set_inventory(topic, t, "consensus") + for t in rest: + await _set_inventory(topic, t, "rest") + await db.set_step_status(topic, "Consolidation", "done") + return True + + +async def _clarify_inventory(ctx: GenContext, set_p, files: dict) -> bool: + """A panel (KONSOLIDIERUNG_PANEL judges) decides on the rest (1×-mentioned): majority `aufnehmen` + → consensus, otherwise discarded. Panel instead of a single judge — the rest cut is the sharpest + intervention; a single judge is too unstable here. Conservative tie → keep (never lose a concept).""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + if await db.get_step_status(topic, "Clarification") == "done": + return True + set_p("Clarification running…", step=_step_idx(topic, "Clarification")) + rest_rows = await db.list_blocks(topic, status="rest") + # Continuous gate (EDC "Define"): also check consensus blocks with reference/placeholder titles + # ("Satz 7.18", "Korollar 6.18", "Bedingung (**)") — otherwise they bypass every exam. + suspicious = [b for b in await db.list_blocks(topic, status="consensus") if _is_reference(b["title"])] + check_rows = rest_rows + suspicious + if check_rows: + work_dir = files["arbeit"] + paths = [work_dir / f"clarification-j{j}.json" for j in range(1, CONSOLIDATION_PANEL + 1)] + # final=False: a judge with an accidentally non-empty `rest` must not fail entirely + # (otherwise the panel collapses to 1 judge). Its `aufnehmen` counts; rest entries count as + # not-accepted. The "rest empty" requirement still stands in the prompt. + pending = [(j, p) for j, p in enumerate(paths, 1) if _runde_schema(_json_file(p)) is None] + for _, p in pending: + p.unlink(missing_ok=True) + if pending: + slots = [{ + "key": f"blocks-{topic}-clarification-j{j}", + "prompt": _prompt( + "Blocks-Klaerung", topic=topic, + rest="\n".join(f"- {b['title']} — {b['description']}" if b['description'] else f"- {b['title']}" + for b in check_rows), + final="\n- Entscheide JEDEN Eintrag. `rest` MUSS leer sein.", + out_path=p, + ), + "role": "judge", "capabilities": "files", + "payload": (lambda result, p=p: _runde_schema(_json_file(p))), + } for j, p in pending] + existing = CONSOLIDATION_PANEL - len(pending) + await _race(topic, "Clarification", slots, max(1, 2 - existing), + _timeout("selection_mapping", len(check_rows)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) + if is_cancelled(): + return False + outs = [r for p in paths if (r := _runde_schema(_json_file(p)))] + if not outs: + _blocks_errors[topic] = "Clarification failed" + return False + # Majority per rest entry (by norm title). Tie → keep (votes*2 >= n). + votes: dict[str, int] = {} + for accepted, _ in outs: + for nt in {_norm_title(_title(t)) for t in accepted}: + votes[nt] = votes.get(nt, 0) + 1 + # Rename suggestions (additive from the raw JSON — _runde_schema doesn't know the field): + # kept reference/placeholder titles → meaningful name from the content. Old title norm + # stays stable (doesn't break the votes match); per old title the most frequent suggestion. + renames: dict[str, dict[str, int]] = {} + for p in paths: + d = _json_file(p) + rename_raw = d.get("rename") if isinstance(d, dict) else None + if isinstance(rename_raw, dict): + for old, new in rename_raw.items(): + new = str(new).strip() + if new: + renames.setdefault(_norm_title(str(old)), {}).setdefault(new, 0) + renames[_norm_title(str(old))][new] += 1 + seen_norm = {b["title_norm"] for b in await db.list_blocks(topic, status="consensus")} + for b in check_rows: + accept = votes.get(b["title_norm"], 0) * 2 >= len(outs) + if not accept: + await db.set_block_status(topic, b["title_norm"], "discarded") + continue + new_title = None + if _is_reference(b["title"]) and (suggestions := renames.get(b["title_norm"])): + cands = max(suggestions, key=lambda k: (suggestions[k], len(k))) + if not _is_reference(cands): + new_title = cands + if new_title: + nn, t, n = _norm_title(new_title), new_title, 2 + while nn in seen_norm: + t, nn, n = f"{new_title} ({n})", _norm_title(f"{new_title} ({n})"), n + 1 + seen_norm.add(nn) + await db.set_block_status(topic, b["title_norm"], "consensus", title=t, neu_norm=nn) + else: + await db.set_block_status(topic, b["title_norm"], "consensus") + await db.set_step_status(topic, "Clarification", "done") + return True + + +def _pairs_schema(data) -> dict[int, bool] | None: + """{"pairs": {"1": "ja", "2": "nein", …}} → {pair_nr: True/False} · otherwise None.""" + if not isinstance(data, dict) or not isinstance(data.get("pairs"), dict): + return None + out: dict[int, bool] = {} + for k, v in data["pairs"].items(): + try: + nr = int(k) + except (ValueError, TypeError): + continue + out[nr] = str(v).strip().casefold() in ("ja", "yes", "true", "1") + return out or None + + +def _cliques(n: int, edge_list: list[tuple[int, int]]) -> list[list[int]]: + """Complete-link: greedy maximal cliques over the confirmed duplicate edges. A group + forms only if ALL its nodes are pairwise connected → no chaining (A=B + B=C forms + NO group {A,B,C} as long as A=C is missing). Only cliques ≥2 are returned.""" + adj: dict[int, set[int]] = {i: set() for i in range(n)} + for a, b in edge_list: + adj[a].add(b) + adj[b].add(a) + used: set[int] = set() + groups: list[list[int]] = [] + for v in sorted(range(n), key=lambda x: -len(adj[x])): + if v in used or not adj[v]: + continue + clique = {v} + for u in sorted(adj[v], key=lambda x: -len(adj[x])): + if u not in used and clique <= adj[u] | {u}: # u connected to ALL previous ones + clique.add(u) + if len(clique) >= 2: + groups.append(sorted(clique)) + used |= clique + return groups + + +async def _dedup_inventory(ctx: GenContext, set_p, files: dict) -> bool: + """Final dedup pass over the finished consensus list: pairwise verification (entity + resolution). Embedding yields candidate PAIRS (cosine ≥ DEDUP_PAAR_FLOOR), a judge + confirms EACH pair individually (ja = the same duplicate). ONLY confirmed pairs become + merge edges (union-find) — no chaining, no aspect over-merging like the block mixer. + Per group ONE representative (main concept) stays, the rest is discarded.""" + topic, is_cancelled = ctx.topic, ctx.is_cancelled + if await db.get_step_status(topic, "Dedup") == "done": + return True + if not (EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available)): + await db.set_step_status(topic, "Dedup", "done") # without a model: silently skip + return True + set_p("Dedup…", step=_step_idx(topic, "Dedup")) + work_dir = files["arbeit"] + consensus = await db.list_blocks(topic, status="consensus") + if len(consensus) >= 2: + import numpy as np + texts = [f"{b['title']} — {b['description']}" if b["description"] else b["title"] for b in consensus] + sims = await asyncio.to_thread(embedding.embed_sims, texts) + if sims is not None: + n = len(consensus) + iu = np.triu_indices(n, k=1) + cands = [(int(iu[0][m]), int(iu[1][m])) for m in np.where(sims[iu] >= DEDUP_PAIR_FLOOR)[0]] + _log(topic, f"Dedup: {len(cands)} candidate pairs (cosine ≥ {DEDUP_PAIR_FLOOR}) → pairwise filter") + packages = [cands[i:i + DEDUP_PAIRS_CHUNK] for i in range(0, len(cands), DEDUP_PAIRS_CHUNK)] + + def pair_path(pi): return work_dir / f"dedup-paar-c{pi}.json" + + async def _filt(pi, paare): + fp = pair_path(pi) + if _pairs_schema(_json_file(fp)): + return # resume + lines = "\n\n".join( + f"{j + 1}.\nA: {consensus[a]['title']} — {consensus[a]['description']}" + f"\nB: {consensus[b]['title']} — {consensus[b]['description']}" + for j, (a, b) in enumerate(paare)) + await run_single_slot( + ctx, f"Dedup pairs {pi}", + key=f"blocks-{topic}-dedup-paar-c{pi}", + prompt=_prompt("Blocks-Paar-Filter", topic=topic, pairs=lines, out_path=fp), + role="judge", capabilities="files", + payload=lambda result, p=fp: _pairs_schema(_json_file(p)), + timeout=_timeout("selection_mapping", len(paare)), + ) + + await _gather_progress([_filt(pi, p) for pi, p in enumerate(packages)], + len(packages), _report_p(set_p, topic, "Dedup")) + if is_cancelled(): + return False + # Collect confirmed "ja" edges, then COMPLETE-LINK (greedy cliques) instead of single-link + # union-find — prevents chaining (A=B + B=C does NOT merge A,C without a direct A=C). + edge_list, ja = [], 0 + for pi, paare in enumerate(packages): + verdict = _pairs_schema(_json_file(pair_path(pi))) or {} + for j, (a, b) in enumerate(paare): + if verdict.get(j + 1): + edge_list.append((a, b)) + ja += 1 + groups = _cliques(n, edge_list) + removed = 0 + for idxs in groups: + # representative = main concept (fewest property markers), then shortest title. + rep = min(idxs, key=lambda k: (_aspect_marker(consensus[k]["title"]), len(consensus[k]["title"]), k)) + for k in idxs: + if k != rep: + await db.set_block_status(topic, consensus[k]["title_norm"], "discarded") + removed += 1 + from collections import Counter + atomic_write_json(work_dir / "dedup-runde-1.json", + {"vorher": n, "entfernt": removed, "paare_geprueft": len(cands), "paare_ja": ja, + "clique_groessen": dict(sorted(Counter(len(g) for g in groups).items())), + "groups": [[consensus[k]["title"] for k in g] for g in groups]}, indent=1) + _log(topic, f"Dedup (pairwise): {n} → {n - removed} (−{removed}); {ja}/{len(cands)} pairs confirmed") + await db.set_step_status(topic, "Dedup", "done") + return True + + +def _filter_schema(data) -> dict[int, int] | None: + """{"fragments": {"3": 7, "12": 8}} → {block_nr: parent_nr} · None on invalid structure. + Empty dict = valid (nothing to degrade). Parent ≠ itself.""" + if not isinstance(data, dict) or not isinstance(data.get("fragments"), dict): + return None + out: dict[int, int] = {} + for k, v in data["fragments"].items(): + try: + nr, parent = int(k), int(v) + except (ValueError, TypeError): + continue + if nr != parent: + out[nr] = parent + return out + + +# Pure notation/symbols without a standalone concept — kept narrow (FP~0, checked against aak; +# "KNF"/"MST"/"NP" do NOT match). These are discarded autonomously (need no parent). +_FILTER_NOTATION = re.compile(r'^\s*\|.{1,6}\|\s*$|^Güte\s+\d+\s*$') +# Property/runtime suspicion — marks lines for the judge's verdict (NO auto-drop, FP too high: +# "NP-Schwere", reductions with "∈NP" are real blocks). Complements _aspekt_marker. +_FILTER_PREDICATE = re.compile( + r'ist NP-(vollständig|schwer)|NP-(Vollständigkeit|Schwere) von|ETH (Konsequenz|Lower Bound)' + r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP', re.I) + + +def _filter_suspect(b: dict) -> bool: + """Heuristic flag: could be a property/detail of another block.""" + return _aspect_marker(b["title"]) > 0 or bool(_FILTER_PREDICATE.search(f"{b['title']} {b['description'] or ''}")) + + +async def _filter_inventory(ctx: GenContext, set_p, files: dict) -> bool: + """Degrade pass (granularity): separates real blocks from fragments (properties, + proof gadgets, notation, runtime details). Each judge sees the FULL block list + (self-containment is relational) and marks fragments WITH a parent block from the list. + Fragment + parent-in-list → discarded (content comes back as a subblock of the parent). + No parent or in doubt → keep (no concept loss).""" + topic, is_cancelled = ctx.topic, ctx.is_cancelled + if await db.get_step_status(topic, "Blocks-Filter") == "done": + return True + set_p("Blocks-Filter…", step=_step_idx(topic, "Blocks-Filter")) + work_dir = files["arbeit"] + consensus_all = await db.list_blocks(topic, status="consensus") + # Safety net: discard pure notation autonomously (FP~0, no parent needed). The judge + # reliably overlooks such symbols (recall problem), hence deterministically beforehand. + consensus, notation_dropped = [], [] + for b in consensus_all: + if _FILTER_NOTATION.search(b["title"]): + await db.set_block_status(topic, b["title_norm"], "discarded") + notation_dropped.append(b["title"]) + else: + consensus.append(b) + if notation_dropped: + _log(topic, f"Blocks-Filter: {len(notation_dropped)} pure notation discarded: {notation_dropped[:6]}") + if len(consensus) < 2: + await db.set_step_status(topic, "Blocks-Filter", "done") + return True + n = len(consensus) + # ⚠ marks suspicious lines (property/runtime) — the judge MUST check them per entry. + def _line(i, b): + mark = "⚠ " if _filter_suspect(b) else "" + return f"{i}. {mark}{b['title']} — {b['description']}" if b["description"] else f"{i}. {mark}{b['title']}" + full_list = "\n".join(_line(i, b) for i, b in enumerate(consensus, 1)) + chunks = [list(range(i, min(i + FILTER_CHUNK, n + 1))) for i in range(1, n + 1, FILTER_CHUNK)] + + def filt_path(ci): return work_dir / f"inventar-filter-c{ci}.json" + + async def _assess(ci, numbers): + fp = filt_path(ci) + if _filter_schema(_json_file(fp)) is not None: + return # resume + await run_single_slot( + ctx, f"Blocks-Filter {ci}", + key=f"blocks-{topic}-inventar-filter-c{ci}", + prompt=_prompt("Blocks-Filter", topic=topic, list=full_list, + from_n=numbers[0], to_n=numbers[-1], out_path=fp), + role="judge", capabilities="files", + payload=lambda result, p=fp: _filter_schema(_json_file(p)), + timeout=_timeout("selection_mapping", len(numbers)), + ) + + await _gather_progress([_assess(ci, nm) for ci, nm in enumerate(chunks)], + len(chunks), _report_p(set_p, topic, "Blocks-Filter")) + if is_cancelled(): + return False + fragments: dict[int, int] = {} + for ci, numbers in enumerate(chunks): + verdict = _filter_schema(_json_file(filt_path(ci))) or {} + nset = set(numbers) + for nr, parent in verdict.items(): + if 1 <= parent <= n and nr in nset: + fragments[nr] = parent + # Chain protection: a block that is itself the parent of a fragment stays (its child needs the anchor). + parent_set = set(fragments.values()) + removed, debug = 0, [] + for nr, parent in fragments.items(): + if nr in parent_set: + continue + b = consensus[nr - 1] + await db.set_block_status(topic, b["title_norm"], "discarded") + removed += 1 + debug.append({"fragment": b["title"], "eltern": consensus[parent - 1]["title"]}) + atomic_write_json(work_dir / "inventar-filter.json", + {"vorher": n, "degradiert": removed, "fragments": debug}, indent=1) + _log(topic, f"Blocks-Filter: {n} → {n - removed} (−{removed} fragments → subblocks)") + await db.set_step_status(topic, "Blocks-Filter", "done") + return True + + +# --- Outline (blocks artifact: chapter structure, only read by the guide) --- + +def _outline_complete(files: dict) -> bool: + """Is the outline present (chapter list exists)?""" + d = _json_file(files["outline"]) + return isinstance(d, dict) and isinstance(d.get("chapters"), list) and bool(d.get("chapters")) + + +def _outline_schema(data, valid: set[int]): + """{"chapters":[{title,numbers}]} → cleaned (valid numbers, each exactly once) · + None at <80 % coverage (agent/judge omitted too much).""" + if not isinstance(data, dict) or not isinstance(data.get("chapters"), list): + return None + out, seen = [], set() + for ch in data["chapters"]: + if not isinstance(ch, dict): + continue + title = str(ch.get("title", "")).strip() or "Chapter" + nums = [] + for n in (ch.get("numbers") or []): + try: + n = int(n) + except (ValueError, TypeError): + continue + if n in valid and n not in seen: + seen.add(n) + nums.append(n) + if nums: + out.append({"title": title, "numbers": nums}) + if not out or len(seen) < 0.8 * len(valid): + return None + return {"chapters": out} + + +def _prereq_schema(data, valid: set[int]) -> dict[int, list[int]]: + """{"prereqs": {"3": [1, 7]}} → {num: [prereq nums]} · only numbers from `valid`, no self-edge. + Invalid/empty → {} (best-effort: then original order).""" + if not isinstance(data, dict) or not isinstance(data.get("prereqs"), dict): + return {} + out: dict[int, list[int]] = {} + for k, v in data["prereqs"].items(): + try: + num = int(k) + except (ValueError, TypeError): + continue + if num not in valid or not isinstance(v, list): + continue + pres = [] + for p in v: + try: + p = int(p) + except (ValueError, TypeError): + continue + if p in valid and p != num and p not in pres: + pres.append(p) + if pres: + out[num] = pres + return out + + +def _topo_order(nums: list[int], edges: dict[int, list[int]]) -> list[int]: + """Kahn topo sort: prerequisites first. `edges[num]` = numbers that must come BEFORE num. + Stable tie-break (original order of `nums`); cycles are broken (never deadlock).""" + pos = {n: i for i, n in enumerate(nums)} + # remaining in-degree over valid nodes only; self/foreign edges ignored. + pre = {n: [p for p in edges.get(n, []) if p in pos and p != n] for n in nums} + done: list[int] = [] + finished: set[int] = set() + rest = list(nums) + while rest: + ready_nodes = [n for n in rest if all(p in finished for p in pre[n])] + if not ready_nodes: # cycle → force the earliest remaining node in original order + ready_nodes = [min(rest, key=lambda n: pos[n])] + nxt = min(ready_nodes, key=lambda n: pos[n]) # stable: smallest original position first + done.append(nxt) + finished.add(nxt) + rest.remove(nxt) + return done + + +async def _learning_order(ctx: GenContext, set_p, files: dict, entries: dict, valid: set[int], instructions: str) -> dict: + """Put entries (num→title) into learning order: the LLM extracts prereq edges from the + extracted `prerequisites`, code solves via topo sort. Best-effort → otherwise entries unchanged.""" + if len(entries) < 3: + return entries + topic = ctx.topic + facts_map = _json_file(files["facts"]) + facts_map = facts_map if isinstance(facts_map, dict) else {} + + def _hint(title): + fm = facts_map.get(title) or {} + vs = [v for fk in fm.values() if isinstance(fk, dict) and (v := str(fk.get("prerequisites", "")).strip())] + return " · ".join(dict.fromkeys(vs)) + + pp = files["arbeit"] / "outline-prereqs.json" + + def _payload(result, p=pp): + d = _json_file(p) + return d if isinstance(d, dict) and "prereqs" in d else None + + existing = _json_file(pp) + if not (isinstance(existing, dict) and "prereqs" in existing): + lines = [f"{n}. {t}" + (f"\n braucht vorher: {h}" if (h := _hint(t)) else "") for n, t in entries.items()] + set_p("Outline — learning order…", step=_step_idx(topic, "Outline")) + await run_single_slot( + ctx, "Outline-Prerequisites", key=f"blocks-{topic}-outline-prereqs", + prompt=_prompt("Outline-Prerequisites", topic=topic, blocks="\n".join(lines), out_path=pp, extra=_extra(instructions)), + role="guide", capabilities="files", payload=_payload, timeout=_timeout("plan", len(entries))) + edges = _prereq_schema(_json_file(pp), valid) + if not edges: + return entries # no/invalid edges → original order (no regression) + ordered = _topo_order(list(entries), edges) + return {n: entries[n] for n in ordered} + + +async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict: + """Format-agnostic outline over ALL blocks — 3 proposals → judge merges. + Never aborts: 0 valid → one chapter with everything; missing blocks land in "Other". + → {"chapters":[{title,numbers}]} (also in files["outline"]).""" + topic, is_cancelled = ctx.topic, ctx.is_cancelled + valid = set(entries) + step = _step_idx(topic, "Outline") + + # Establish learning order (LLM-modulo): the LLM extracts prereq edges from the extracted + # `prerequisites`, code solves via topo sort. Best-effort → otherwise original order. + entries = await _learning_order(ctx, set_p, files, entries, valid, instructions) + liste = "\n".join(f"{n}. {t}" for n, t in entries.items()) + set_p("Outline — proposals…", step=step) + + async def _proposal(i, path): + if _outline_schema(_json_file(path), valid): + return True + await run_single_slot( + ctx, f"Outline {i}", key=f"blocks-{topic}-outline-{i}", + prompt=_prompt("Guide-Outline", topic=topic, blocks=liste, out_path=path, extra=_extra(instructions)), + role="guide", capabilities="files", + payload=lambda result, p=path: _outline_schema(_json_file(p), valid), + timeout=_timeout("plan", len(entries))) + return _outline_schema(_json_file(path), valid) is not None + + slots = files["outline_slots"] + await _gather_progress([_proposal(i, p) for i, p in enumerate(slots, 1)], len(slots), _report_p(set_p, topic, "Outline")) + if is_cancelled(): + return {} + proposals = [v for p in slots if (v := _outline_schema(_json_file(p), valid))] + + if not proposals: + plan = {"chapters": [{"title": "Contents", "numbers": list(entries)}]} + elif len(proposals) == 1: + plan = proposals[0] + else: + set_p("Outline merging…", step=step) + block_texts = "\n\n".join( + f"### Vorschlag {i}\n" + "\n".join( + f"KAPITEL: {ch['title']}\n Nummern: {', '.join(str(n) for n in ch['numbers'])}" for ch in v["chapters"]) + for i, v in enumerate(proposals, 1)) + await run_single_slot( + ctx, "Outline-Judge", key=f"blocks-{topic}-outline-judge", + prompt=_prompt("Guide-Outline-Judge", topic=topic, format_name="den Guide", + purpose="alle Blocks in einem roten Faden", n=len(proposals), + blocks=liste, outlines=block_texts, out_path=files["outline"], extra=_extra(instructions)), + role="judge", capabilities="files", + payload=lambda result: _outline_schema(_json_file(files["outline"]), valid), + timeout=_timeout("plan_judge", len(entries))) + plan = _outline_schema(_json_file(files["outline"]), valid) or proposals[0] + + # Completeness: every block appears — missing in "Other" (against omitting agents/judge). + included = {n for ch in plan["chapters"] for n in ch["numbers"]} + missing = [n for n in entries if n not in included] + if missing: + plan["chapters"].append({"title": "Other", "numbers": missing}) + atomic_write_json(files["outline"], plan, indent=1) + return plan + + +# --- Learning artefacts (flashcards/examples from the facts) --- + +def _cards_schema(data): + """{"cards":[{block,subblock,question,answer}]} → list (also empty) · None if broken.""" + if not isinstance(data, dict) or not isinstance(data.get("cards"), list): + return None + out = [] + for e in data["cards"]: + if isinstance(e, dict) and (f := str(e.get("question", "")).strip()) and (a := str(e.get("answer", "")).strip()): + out.append({"block": str(e.get("block", "")).strip(), "subblock": str(e.get("subblock", "")).strip(), + "question": f, "answer": a}) + return out + + +def _example_schema(data): + """{"examples":[{block,subblock,problem,steps,result}]} → list (also empty) · None if broken.""" + if not isinstance(data, dict) or not isinstance(data.get("examples"), list): + return None + out = [] + for e in data["examples"]: + if not isinstance(e, dict): + continue + problem = str(e.get("problem", "")).strip() + steps = [s for x in (e.get("steps") or []) if (s := str(x).strip())] + if problem and steps: + out.append({"block": str(e.get("block", "")).strip(), "subblock": str(e.get("subblock", "")).strip(), + "problem": problem, "steps": steps, "result": str(e.get("result", "")).strip()}) + return out + + +def _example_check_schema(data): + """Worked-example check → {"ok": true} → set() (all correct); {"problems":[{"index":N}]} → + {N, …} (1-based flagged indices); None if broken.""" + if not isinstance(data, dict): + return None + if data.get("ok") is True: + return set() + pr = data.get("problems") + if not isinstance(pr, list): + return None + out: set[int] = set() + for p in pr: + if isinstance(p, dict): + try: + out.add(int(p.get("index"))) + except (ValueError, TypeError): + continue + return out + + +_ARTEFACT_SCHEMA = {"flashcard": _cards_schema, "example": _example_schema} +_ARTEFACT_PROMPT = {"flashcard": "Artifact-Flashcard", "example": "Artifact-Example"} +_ARTEFACT_STEP = {"flashcard": "Flashcards", "example": "Examples"} + + +def _artefacts_complete(files: dict) -> bool: + """Artifact map present (all types generated)? Values may be empty (content-aware).""" + d = _json_file(files["artefakte"]) + return isinstance(d, dict) and all(t in d for t in ARTEFACT_TYPES) + + +async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str) -> dict | None: + """Generate learning artefacts per type from the stored facts — one generation pass + per type over chunks. Worked examples are verified against the facts (wrong ones discarded); + flashcards are low-risk and stay unchecked. → {type: [entries]} (also in files).""" + topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled + work_dir = files["arbeit"] + caps = "files" + # Blocks with subs + facts lines as input block (extract-once from the facts). + blocks = [] + for btitle, subs in sidecar.items(): + if not isinstance(subs, list): + continue + lines = [] + for s in subs: + if not isinstance(s, dict) or not (st := str(s.get("title", "")).strip()): + continue + fk = s.get("facts") if isinstance(s.get("facts"), dict) else {} + line = f"- {st}" + if fk and (fk_text := _facts_lines(fk)): + line += "\n" + "\n".join(" " + l for l in fk_text.split("\n")) + lines.append(line) + if lines: + blocks.append((btitle, lines)) + if not blocks: + empty_map = {t: [] for t in ARTEFACT_TYPES} + atomic_write_json(files["artefakte"], empty_map, indent=1) + return empty_map + + chunks = _lpt_chunks([len(z) for _, z in blocks], FACTS_CHUNK_SUBS) + def block_text(idxs): + return "\n\n".join(f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(blocks[i][1]) for i in idxs) + + # Check worked examples against the facts (panel majority) — discard wrong ones. CoT steps are + # error-prone; a wrong example imprints a faulty schema → no example > a wrong one. + async def _check_examples(ci, idxs, items): + if is_cancelled() or not items: + return items + def cpath(j): return work_dir / f"artifact-example-check-c{ci}-j{j}.json" + examples_txt = "\n\n".join( + f"{k}. PROBLEM: {e['problem']}\n SCHRITTE: " + " | ".join(e.get("steps", [])) + + (f"\n ERGEBNIS: {e['result']}" if e.get("result") else "") + for k, e in enumerate(items, 1)) + pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _example_check_schema(_json_file(cpath(j))) is None] + if pending: + await asyncio.gather(*[ + run_agent(f"blocks-{topic}-artifact-example-check-c{ci}-j{j}", + _prompt("Artifact-Example-Check", topic=topic, facts=block_text(idxs), examples=examples_txt, out_path=cpath(j), extra=_extra(instructions)), + _timeout("content_check", len(items)), provider=provider, role="judge", capabilities=caps) + for j in pending], return_exceptions=True) + outs = [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if (s := _example_check_schema(_json_file(cpath(j)))) is not None] + if not outs: + return items # no exam possible → keep (best-effort) + votes: dict[int, int] = {} + for s in outs: + for idx in s: + votes[idx] = votes.get(idx, 0) + 1 + threshold = len(outs) / 2 + dropped = {idx for idx, v in votes.items() if v > threshold} # majority (≥2 of 3) flagged → out + if dropped: + _log(topic, f"Worked-example check chunk {ci}: {len(dropped)}/{len(items)} discarded") + return [e for k, e in enumerate(items, 1) if k not in dropped] + + outcome: dict[str, list] = {} + for type in ARTEFACT_TYPES: + schema = _ARTEFACT_SCHEMA[type] + def apath(ci, t=type): return work_dir / f"artifact-{t}-c{ci}.json" + + async def _gen(ci, idxs, t=type, schema=schema): + p = apath(ci, t) + if schema(_json_file(p)) is not None: + return True + await run_single_slot( + ctx, f"{_ARTEFACT_STEP[t]} {ci}", key=f"blocks-{topic}-artifact-{t}-c{ci}", + prompt=_prompt(_ARTEFACT_PROMPT[t], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)), + role="guide", capabilities="files", + payload=lambda result, p=p, schema=schema: schema(_json_file(p)), + timeout=_timeout("content", sum(len(blocks[i][1]) for i in idxs))) + return schema(_json_file(p)) is not None + + await _gather_progress([_gen(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, _ARTEFACT_STEP[type])) + if is_cancelled(): + return None + eintraege: list = [] + for ci in range(len(chunks)): + chunk_items = schema(_json_file(apath(ci))) or [] + if type == "example" and chunk_items: + chunk_items = await _check_examples(ci, chunks[ci], chunk_items) + eintraege += chunk_items + outcome[type] = eintraege + atomic_write_json(files["artefakte"], outcome, indent=1) + return outcome + + +async def _mirror_artefacts_db(topic: str, sidecar: dict, artefacts: dict) -> None: + """Mirror artefacts into the DB. Flashcard/example per sub (sub_norm).""" + await db.delete_sub_artefakte(topic) + btitle_list = list(sidecar.keys()) + for type in ARTEFACT_TYPES: + for e in artefacts.get(type, []): + bt = _match_sub(e.get("block", ""), btitle_list) + bnorm, sn = _norm_title(bt), _norm_title(e.get("subblock", "")) + if not bnorm or not sn: + continue + data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")}, ensure_ascii=False) + await db.put_sub_artifact(topic, bnorm, sn, type, data, bt, e.get("subblock", "")) + + +async def _mirror_sidecar_db(topic: str, sidecar: dict) -> None: + """Mirror the sidecar {block title: [{title, level, relevance}]} into the DB table subblocks.""" + for btitle, subs in sidecar.items(): + bnorm = _norm_title(btitle) + if not bnorm or not isinstance(subs, list): + continue + for s in subs: + if not isinstance(s, dict): + continue + st = str(s.get("title", "")).strip() + sn = _norm_title(st) + if not sn: + continue + facts = json.dumps(s["facts"], ensure_ascii=False) if isinstance(s.get("facts"), dict) else None + await db.put_subblock(topic, bnorm, sn, btitle, st, + level=s.get("level"), relevance=s.get("relevance"), + facts=facts, status="consensus") + + +async def _mirror_question_pattern_db(topic: str, pattern: dict) -> None: + """Mirror question patterns {block title: [{subblock, question}]} into the DB table question_pattern.""" + await db.delete_question_pattern(topic) + for btitle, eintraege in pattern.items(): + bnorm = _norm_title(btitle) + if not bnorm or not isinstance(eintraege, list): + continue + for e in eintraege: + if not isinstance(e, dict): + continue + sub = str(e.get("subblock", "")).strip() + sn = _norm_title(sub) + question = str(e.get("question", "")).strip() + if not (sn and question): + continue + await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question) + + +async def _reset_db_from_phase(topic: str, label: str) -> None: + """Discard DB content of phases ≥ `label` (canonical order Source…Artefacts).""" + idx = _phase_idx(label) + if idx <= 8: # Artefacts (flashcards/examples) + await db.delete_sub_artefakte(topic) + if idx <= 7: # Questions + await db.delete_question_pattern(topic) + if idx <= 6: # Outline + await db.delete_outline(topic) + if idx <= 2: # Subblocks (facts/levels/relevance go through sidecar→mirror) + await db.delete_subblocks(topic) + if idx <= 1: # Inventory: inventory + research steps — triage stays + await db.delete_blocks(topic) + await db.delete_pipeline_state(topic, ["Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter"]) + if idx <= 0: # Source: redo triage (coverage/content + step) + await db.delete_coverage(topic) + await db.delete_pipeline_state(topic, ["Source prep"]) + + +async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, ab_phase: int | None = None, ab_step: int | None = None) -> None: + if topic in _blocks_progress: + return + _blocks_progress[topic] = "Waiting…" + _blocks_errors.pop(topic, None) + + files = _blocks_files(topic) + final_path = files["final"] + q = load_source(topic) + folder = source_folder(topic) # projekt/uni/link → folder, thema → None + instructions = q.get("spec") or instructions # prefer the persisted specification (also on resume) + + def set_p(msg: str, step: int | None = None) -> None: + _blocks_progress[topic] = msg + if step is not None: + _blocks_step[topic] = step + + def is_cancelled() -> bool: + return topic in _blocks_cancelled + + def aborted() -> None: + _blocks_errors[topic] = "Cancelled — progress is preserved" + + ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled) + + try: + async with _semaphore: + files["arbeit"].mkdir(parents=True, exist_ok=True) + # Re-run from the chosen phase: delete artefacts from there; the fresh-start block + # below is skipped (with a preserved sidecar it would otherwise wipe everything). + if ab_step is not None: # fine sub-step re-run (takes precedence over ab_phase) + await _reset_from_step(topic, ab_step) + elif ab_phase is not None: + phasen = _phases(topic) + label = phasen[ab_phase - 1][0] if 1 <= ab_phase <= len(phasen) else "Inventory" + _reset_from_phase(topic, label) + await _reset_db_from_phase(topic, label) + # A stage returning False ends generation; if it was a cancel, mark aborted first. + async def _stage(coro) -> bool: + ok = await coro + if not ok and is_cancelled(): + aborted() + return ok + + # Step "Source prep": crawl (link) + PDFs + content/noise triage. + if not await _stage(_prepare_source(ctx, set_p, files, q, folder, instructions)): + return + # "Create new": ONLY if truly everything is done (blocks.md AND + # sidecar) → complete fresh start. If blocks.md exists without the sidecar, + # it's a partial state (block B/C open) → resume, don't wipe. + # On an explicit re-run (ab_phase) _reset_ab_phase already handled that. + done = ab_phase is None and ab_step is None and final_path.exists() and _sidecar_schema(_json_file(files["sidecar"])) is not None + if done: + for p_old in _all_slot_files(files): + p_old.unlink(missing_ok=True) + await db.delete_pipeline_state(topic) + await db.delete_blocks(topic) + await db.delete_subblocks(topic) + await db.delete_question_pattern(topic) + await db.delete_coverage(topic) + await db.delete_outline(topic) + await db.delete_sub_artefakte(topic) + + # Inventory (DB): research loop → consolidation → clarification. + if not await _stage(_research_batch(ctx, set_p, files, q, folder, instructions)): + return + if not await _stage(_consolidate(ctx, set_p, files)): + return + if not await _stage(_clarify_inventory(ctx, set_p, files)): + return + if not await _stage(_dedup_inventory(ctx, set_p, files)): + return + if not await _stage(_filter_inventory(ctx, set_p, files)): + return + consensus_rows = await db.list_blocks(topic, status="consensus") + entries = { + i: (f"{b['title']} — {b['description']}" if b["description"] else b["title"]) + for i, b in enumerate(consensus_rows, 1) + } + + # Projects only: subject-field supplement — script/project is an excerpt, + # a web agent adds canonically missing blocks, marked with [Supplement]. + if q["type"] == "projekt": + set_p("Supplementing subject field…", step=_step_idx(topic, "Supplement")) + supp_path = files["ergaenzung"] + supplements = _supplement_schema(_json_file(supp_path)) + if supplements is None: + supp_path.unlink(missing_ok=True) + status, supplements = await run_single_slot( + ctx, "Supplement", + key=f"blocks-{topic}-ergaenzung-1", + prompt=_prompt( + "Blocks-Supplement", + topic=topic, blocks="\n".join(f"- {t}" for t in entries.values()), + out_path=supp_path, extra=_extra(instructions), + ), + role="quick", capabilities="full", + payload=lambda result: _supplement_schema(_json_file(supp_path)), + timeout=_timeout("ergaenzung"), + ) + if status == CANCELLED: + aborted() + return + if status == FAILED: + _blocks_errors[topic] = "Supplement failed (no valid result)" + return + idx = _title_index(entries) + new = [(t, b) for t, b in supplements if _resolve_title(idx, t) is None] + if new: + _log(topic, f"Supplement: {len(new)} block(s) added from the subject field") + start = max(entries, default=0) + 1 + for off, (t, b) in enumerate(new): + entries[start + off] = f"{t} — {b} [Supplement]" + + # Make titles unique and write the unsorted inventory + entries = _unique_title(entries) + atomic_write_text(final_path, "\n".join(f"{i}. {t}" for i, t in entries.items()) + "\n") + + # Block B + C: subblocks per block + levels → sidecar subblocks.json. + # Non-destructive: blocks.md already exists; if the sidecar is missing, only + # this part is retried on the next run. The guide falls back without the sidecar. + if _sidecar_schema(_json_file(files["sidecar"])) is None: + raw = _sub_raw_schema(_json_file(files["sub_roh"])) + if raw is None: + raw = await _subblocks_block(ctx, set_p, files, entries, instructions) + if is_cancelled(): + aborted() + return + if raw is None: + return # error is set + atomic_write_json(files["sub_roh"], raw, indent=1) + # Facts per sub (BEFORE the level): extract + verify source facts → facts.json. + # Extract-once grounding — level/relevance/questions/guide feed on it. + if not _facts_complete(files): + res = await _facts_block(ctx, set_p, files, raw, q, folder, instructions) + if is_cancelled(): + aborted() + return + if res is None: + return # error is set + facts_map, discarded = res + # Strike discarded (unsupportable) subs from raw — FIRST (resume-robust), then + # facts.json. This way levels/relevance/outline/questions/guide no longer see them. + if discarded: + for bt, sns in discarded.items(): + if bt in raw: + raw[bt] = [s for s in raw[bt] if _norm_title(s) not in sns] + raw = {bt: subs for bt, subs in raw.items() if subs} # drop empty blocks (_sub_roh_schema requires ≥1) + atomic_write_json(files["sub_roh"], raw, indent=1) + atomic_write_json(files["facts"], facts_map, indent=1) + sidecar = await _levels_block(ctx, set_p, files, raw, instructions) + if is_cancelled(): + aborted() + return + if sidecar is None: + return + # Merge facts into the sidecar subs (DB mirror + guide use). + facts_map = _json_file(files["facts"]) + if isinstance(facts_map, dict): + for btitle, subs in sidecar.items(): + fm = facts_map.get(btitle, {}) + for sub in subs: + if (fk := fm.get(_norm_title(sub["title"]))): + sub["facts"] = fk + atomic_write_json(files["sidecar"], sidecar, indent=1) + + # Block D: relevance per subblock (relevant/peripheral) → merge into the sidecar. + # Own phase after the levels; drives the ProGuide format (all blocks + # with ≥1 relevant subblock) and filters peripheral subs out of the guides. + sidecar = _json_file(files["sidecar"]) + if _sidecar_schema(sidecar) is not None and not _relevance_complete(sidecar): + relevance_by_id = await _relevance_block(ctx, set_p, files, sidecar, instructions) + if is_cancelled(): + aborted() + return + if relevance_by_id is None: + return # error is set + gid = 0 + for subs in sidecar.values(): + for sub in subs: + gid += 1 + sub["relevance"] = relevance_by_id.get(gid, "relevant") + atomic_write_json(files["sidecar"], sidecar, indent=1) + + # Block D.5: outline (blocks artifact) — chapter structure over ALL blocks, + # only read by the guide. Format-agnostic; the guide filters per format. + if not _outline_complete(files): + await _outline_block(ctx, set_p, files, entries, instructions) + if is_cancelled(): + aborted() + return + + # Block E: question pattern per relevant subblock × type → own sidecar. + # At exam time each agent draws a pattern without replacement and formulates + # a question from it — distinct seeding prevents the duplicate questions of live generation. + sidecar = _json_file(files["sidecar"]) + if _sidecar_schema(sidecar) is not None and _relevance_complete(sidecar) and not _question_pattern_complete(topic): + pattern = await _question_pattern_block(ctx, set_p, files, sidecar, instructions) + if is_cancelled(): + aborted() + return + if pattern is None: + return # cancel + atomic_write_json(files["question_pattern"], pattern, indent=1) + + # Block F: learning artefacts (flashcards/examples) from the facts — bonus, + # presented by the frontend. Does not abort the run (artefacts are optional). + sidecar = _json_file(files["sidecar"]) + if _sidecar_schema(sidecar) is not None and not _artefacts_complete(files): + artefacts = await _artefacts_block(ctx, set_p, files, sidecar, instructions) + if is_cancelled(): + aborted() + return + if artefacts is None: + return # cancel (error/cancel) + + # DB mirror (bridge): write the final sidecar + question-pattern state into the DB. + sidecar = _json_file(files["sidecar"]) + if _sidecar_schema(sidecar) is not None: + await _mirror_sidecar_db(topic, sidecar) + pattern = _json_file(files["question_pattern"]) + if isinstance(pattern, dict) and pattern: + await _mirror_question_pattern_db(topic, pattern) + # Outline (title-based, robust against number drift) → DB. + plan = _json_file(files["outline"]) + if isinstance(plan, dict) and plan.get("chapters"): + kapitel = [ + {"title": ch.get("title", "Chapter"), + "blocks": [_title(entries[n]) for n in ch.get("numbers", []) if n in entries]} + for ch in plan["chapters"] + ] + await db.set_outline(topic, json.dumps({"chapters": kapitel}, ensure_ascii=False)) + # Artefacts → DB (flashcard/example per sub, diagram per block). + artefacts = _json_file(files["artefakte"]) + if isinstance(artefacts, dict) and _sidecar_schema(sidecar) is not None: + await _mirror_artefacts_db(topic, sidecar, artefacts) + except Exception as e: + log.exception("[%s] Blocks generation failed", topic) + _blocks_errors[topic] = str(e)[:2000] + finally: + # No file cleanup: intermediate files stay for resume / traceability. + _blocks_progress.pop(topic, None) + _blocks_step.pop(topic, None) + _blocks_cancelled.discard(topic) + clear_scope(f"blocks-{topic}-") # clear the scope → restart isn't blocked diff --git a/backend/config.py b/backend/config.py index e972405..54959f4 100644 --- a/backend/config.py +++ b/backend/config.py @@ -10,122 +10,123 @@ UNI_DIR = PROJECT_ROOT / "uni" MAX_CONCURRENT_GENERATIONS = 10 -# Lesbarkeits-Gate: deterministischer Prüfer (kleines deutsches Komplexitäts-Modell, -# Skala 1–7). Zu schwere Sections gehen in die Lese-Prüfungs-Überarbeitung. -# Fehlen transformers/torch oder das Modell → Gate stumm aus. -LESBARKEIT_AKTIV = True -LESBARKEIT_MODELL = "MiriUll/distilbert-german-text-complexity" -# Anker auf der 1–7-Skala (TextComplexityDE): Leichte Sprache ~1,2; Wikipedia-Schnitt -# ~3,22; ab MOS > 4 gilt ein Satz als „echt komplex" (Vereinfachungs-Grenze des Papers). -LESBARKEIT_MAX = 3.5 # Section zu schwer, wenn der Satz-Schnitt darüber liegt -LESBARKEIT_HART = 4.0 # Einzelsatz ab hier „hart" -LESBARKEIT_HART_ANTEIL = 0.30 # … ODER wenn dieser Anteil der Sätze hart ist +# Readability gate: deterministic checker (small German complexity model, +# scale 1–7). Sections that are too hard go into the read-exam revision. +# If transformers/torch or the model are missing → gate silently off. +READABILITY_ACTIVE = True +READABILITY_MODEL = "MiriUll/distilbert-german-text-complexity" +# Anchors on the 1–7 scale (TextComplexityDE): plain language ~1.2; Wikipedia average +# ~3.22; from MOS > 4 a sentence counts as "truly complex" (the paper's simplification cutoff). +READABILITY_MAX = 3.5 # section too hard when the sentence average is above this +READABILITY_HARD = 4.0 # an individual sentence is "hard" from here on +READABILITY_HARD_SHARE = 0.30 # … OR when this share of sentences is hard -# Bausteine-Konsolidierung: semantisches Embedding-Clustering statt LLM-Listen-Merge. -# Ein kleines mehrsprachiges Satz-Embedding (mean-pool) bildet die Kandidaten-Cluster -# GLOBAL (kein Chunk-Verlust) per Cosine + Union-Find. Titel-Varianten desselben Konzepts -# ("Vertex Cover" / "Vertex Cover Definition") verschmelzen; der Konsens zählt danach die -# echten Reader pro Cluster (≥2 = Konsens). Fehlen transformers/torch oder lädt das Modell -# nicht → Embedding stumm aus, `_konsolidiere` fällt auf den alten Panel-Judge-Pfad zurück. +# Block consolidation: semantic embedding clustering instead of an LLM list merge. +# A small multilingual sentence embedding (mean-pool) builds the candidate clusters +# GLOBALLY (no chunk loss) via cosine + union-find. Title variants of the same concept +# ("Vertex Cover" / "Vertex Cover Definition") merge; the consensus then counts the +# real readers per cluster (≥2 = consensus). If transformers/torch are missing or the model +# won't load → embedding silently off, `_consolidate` falls back to the old panel-judge path. EMBEDDING_AKTIV = True -EMBEDDING_MODELL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" # CPU, mehrsprachig, ~470 MB -# Stärkere (größere) CPU-Alternative bei Bedarf: "BAAI/bge-m3". -# Konsolidierung = zweistufig: (1) Embedding bildet GROBE Ähnlichkeits-Blocks (High-Recall), -# (2) ein LLM-Judge gruppiert JEDEN Block in die echten Bausteine (merge Paraphrasen, split -# Über-Merges). Reines Threshold-Blocking erzeugt einen Giant-Component (alles verkettet) → -# darum „Capped-Blocking": greedy nach Cosine mergen, aber Blockgröße deckeln. So bleiben die -# LLM-Listen kurz und stabil (belegt: Embedding-Block + LLM-Judge ≈ 95 % Precision). -EMBEDDING_BLOCK_FLOOR = 0.5 # Mindest-Cosine, damit zwei Kandidaten in EINEN Block dürfen -EMBEDDING_BLOCK_CAP = 25 # max. Titel je Block (LLM-Liste kurz/stabil halten) -# Subbaustein-Dedup: rein deterministisch (kein LLM). Subbausteine sind kurze Aussagen IM SELBEN -# Baustein-Kontext — ab dieser Cosine sind zwei dieselbe Aussage (an aak geprüft: ≥0,88 ausnahmslos -# echte Dubletten). Konservativ 0,90, damit verschiedene Aspekte (∈NP ≠ NP-schwer) getrennt bleiben. +EMBEDDING_MODELL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" # CPU, multilingual, ~470 MB +# Stronger (larger) CPU alternative if needed: "BAAI/bge-m3". +# Consolidation = two-stage: (1) the embedding builds COARSE similarity blocks (high recall), +# (2) an LLM judge groups EACH block into the real blocks (merge paraphrases, split +# over-merges). Pure threshold blocking creates a giant component (everything chained) → +# hence "capped blocking": greedily merge by cosine, but cap the block size. This keeps the +# LLM lists short and stable (evidenced: embedding block + LLM judge ≈ 95% precision). +EMBEDDING_BLOCK_FLOOR = 0.5 # minimum cosine for two candidates to share ONE block +EMBEDDING_BLOCK_CAP = 25 # max. titles per block (keep the LLM list short/stable) +# Subblock dedup: purely deterministic (no LLM). Subblocks are short statements IN THE SAME +# block context — from this cosine on two are the same statement (checked on aak: ≥0.88 are +# without exception true duplicates). Conservative 0.90 so different aspects (∈NP ≠ NP-hard) stay separate. EMBEDDING_SUB_DUP = 0.90 -# Deckel für gleichzeitige CLI-Agenten-Prozesse (über alle Generierungen hinweg). -# Eigene Spur für interaktive Aufrufe (Chat, Elemente), damit sie nicht hinter -# laufenden Writern in der Warteschlange hängen. +# Cap for concurrent CLI agent processes (across all generations). +# Own lane for interactive calls (chat, elements) so they don't hang behind +# running writers in the queue. MAX_CONCURRENT_AGENTS = 10 MAX_CONCURRENT_INTERACTIVE = 8 -# Grace-Fenster der Konsens-Races (Bausteine, Guide, OnePager): Nach dem ersten -# gültigen Ergebnis dürfen die übrigen Agenten noch so viele Sekunden fertig -# werden (Kill nur, wenn das Minimum schon steht). -KONSENS_GRACE = 300 +# Grace window of the consensus races (blocks, guide, OnePager): after the first +# valid result the remaining agents may still become done for this many seconds +# (kill only once the minimum is already in). +CONSENSUS_GRACE = 300 -# Recherche-Race: längeres Grace-Fenster. Recherche treibt die ganze Bausteine-Anzahl; -# bei langsamen Providern (z.B. MiniMax) sollen ALLE 5 Agenten fertig werden, nicht nur -# das Quorum von 3. Pro-Agent-Timeout (TIMEOUTS["recherche"]=1800s) deckelt echte Hänger. -RECHERCHE_GRACE = 900 +# Research race: longer grace window. Research drives the whole block count; +# with slow providers (e.g. MiniMax) ALL 5 agents should become done, not just +# the quorum of 3. The per-agent timeout (TIMEOUTS["research"]=1800s) caps real hangs. +RESEARCH_GRACE = 900 -# Cap der Klärungs- und Prüf-Loops: maximale Runden, bis alles entschieden sein -# muss. In der letzten Runde MUSS der Mapping-Agent jeden Eintrag entscheiden; -# Prüf-Loops lassen Rest-Beanstandungen danach stehen. -KONSENS_MAX_RUNDEN = 3 +# Cap of the clarification and check loops: maximum rounds until everything must be +# decided. In the last round the mapping agent MUST decide every entry; +# check loops leave any remaining objections standing after that. +CONSENSUS_MAX_ROUNDS = 3 -# Crawler-Sichtung (Content/Noise) — deterministischer Regel-Filter statt LLM. -# Match: Substring (klein) gegen URL UND Dateiname. Reihenfolge: keep > noise > min_chars > behalten. -# Sonderregeln einfach hier ergänzen. -CRAWL_KEEP_PATTERNS = ["learn-unit", "learn-course"] # immer Content -CRAWL_NOISE_PATTERNS = [ # eindeutig themenfremd → raus +# Crawler triage (content/noise) — deterministic rule filter instead of an LLM. +# Match: substring (lowercase) against URL AND file name. Order: keep > noise > min_chars > keep. +# Just add special rules here. +CRAWL_KEEP_PATTERNS = ["learn-unit", "learn-course"] # always content +CRAWL_NOISE_PATTERNS = [ # clearly off-topic → out "clubs", "events", "podcasts", "resources", "-u-", "academy", "pricing", "/plans", "career", "newsletter", "impressum", "login", ] -CRAWL_MIN_CHARS = 400 # zu wenig Text → raus +CRAWL_MIN_CHARS = 400 # too little text → out -# LLM-Themen-Relevanz-Gate (nach dem Regel-Filter): je Content-Seite ja/nein gegen die Spec. -# Trennt das Fachgebiet (z.B. Backend vs Frontend), was die globalen CRAWL_*-Regeln nicht können. -QUELLE_RELEVANZ_CHUNK = 12 # Seiten je Rater-Paket (klein, da je Seite ein Snippet mitgeht) -QUELLE_RELEVANZ_SNIPPET = 800 # Body-Zeichen je Seite im Prompt (URL ist Primärsignal) +# LLM topic relevance gate (after the rule filter): per content page yes/no against the spec. +# Separates the subject area (e.g. backend vs frontend), which the global CRAWL_* rules can't. +QUELLE_RELEVANZ_CHUNK = 12 # pages per rater package (small, since a snippet ships per page) +QUELLE_RELEVANZ_SNIPPET = 800 # body characters per page in the prompt (URL is the primary signal) -# Timeouts pro Agenten-Schritt: (Basis-Sekunden, Sekunden pro Baustein/Section). -# Gilt für alle Provider gleich — wer zu langsam ist, wird neu gestartet bzw. überholt. +# Timeouts per agent step: (base seconds, seconds per block/section). +# Applies equally to all providers — whoever is too slow gets restarted or overtaken. TIMEOUTS = { - "recherche": (1800, 0), # fix 30 min - "recherche_mapping": (600, 3), # n = vorgemergte Einträge - "auswahl_mapping": (600, 2), # n = Rest-Einträge (Bausteine-Inventar) - "ergaenzung": (900, 0), # Themenfeld-Ergänzung bei Projekten (Web-Recherche) + "research": (1800, 0), # fixed 30 min + "research_mapping": (600, 3), # n = pre-merged entries + "selection_mapping": (600, 2), # n = remaining entries (block inventory) + "ergaenzung": (900, 0), # subject-field extension for projects (web research) "plan": (300, 5), - "plan_judge": (600, 5), # Judge liest bis zu 5 Gliederungen, n = Sections - "inhalt": (600, 90), # Inhalte je Baustein im Chunk identifizieren (Websuche) - "inhalt_check": (300, 10), # Inhalts-Prüfung je Baustein im Paket - "subbaustein": (900, 45), # Subbausteine je Baustein im Chunk finden (Websuche) - "subbaustein_check": (300, 15), # Judge entscheidet strittige Subbausteine im Chunk - "stufe": (300, 10), # Subbausteine einstufen je Chunk - "stufe_check": (300, 10), # Judge entscheidet strittige Stufen im Chunk - "relevanz": (300, 10), # Subbausteine relevant/rand je Chunk - "relevanz_check": (300, 10), # Judge entscheidet strittige Relevanz im Chunk - "frage_muster": (300, 15), # Frage-Muster je Baustein (Subbausteine × Typen) - "frage_muster_check": (300, 10), # Kritiker bereinigt die Muster-Tabelle je Baustein - "writer": (600, 120), # pro Section im Chunk - "lese_check": (300, 10), # pro Section im Paket + "plan_judge": (600, 5), # judge reads up to 5 outlines, n = sections + "content": (600, 90), # identify content per block in the chunk (web search) + "content_check": (300, 10), # content exam per block in the package + "subblock": (900, 45), # find subblocks per block in the chunk (web search) + "subblock_check": (300, 15), # judge decides contested subblocks in the chunk + "level": (300, 10), # classify subblocks per chunk + "level_check": (300, 10), # judge decides contested levels in the chunk + "relevance": (300, 10), # subblocks relevant/peripheral per chunk + "relevance_check": (300, 10), # judge decides contested relevance in the chunk + "question_pattern": (300, 15), # question patterns per block (subblocks × types) + "question_pattern_check": (300, 10), # critic cleans up the pattern table per block + "writer": (600, 120), # per section in the chunk + "lese_check": (300, 10), # per section in the package } -# Zweck je Format — fließt in den Gliederungs-Judge (was der Guide leisten soll). -FORMAT_ZWECK = { +# Purpose per format — flows into the outline judge (what the guide should achieve). +# German strings: these are inserted verbatim into the judge prompt → kept German on purpose. +FORMAT_PURPOSE = { "Guide": "einen fokussierten Guide — alles Relevante ohne Randthemen", "FullGuide": "einen Komplett-Guide — das ganze Thema inkl. Randthemen", "Rest": "einen Ergänzungs-Guide — nur die Randthemen", } -# Provider-Stacks: komplett unabhängig, einer kann jederzeit entfernt werden. -# Rollen: "quick" = Massenarbeit (Recherche, Einordnung), -# "fast" = Interaktion + Voten (Chat, Prüfung, Klärung, Elemente), -# "judge" = Mapping-/Judge-/Prüf-Agenten — kalt (niedrige Temperature, -# ohne Thinking) für stabile Urteile; Claude/Lokal mappen auf "fast", -# "guide" = große Generierung (Vorschläge, Writer). +# Provider stacks: completely independent, any one can be removed at any time. +# Roles: "quick" = bulk work (research, classification), +# "fast" = interaction + voting (chat, exam, clarification, elements), +# "judge" = mapping/judge/check agents — cold (low temperature, +# no thinking) for stable verdicts; Claude/local map to "fast", +# "guide" = large generation (proposals, writer). DEFAULT_PROVIDER = "claude" PROVIDERS = { "claude": { "cli": "claude", "guide": "claude-opus-4-8[1m]", "fast": "claude-sonnet-4-6", - "judge": "claude-sonnet-4-6", # CLI kennt keine Temperature + "judge": "claude-sonnet-4-6", # the CLI has no temperature setting "quick": "claude-sonnet-4-6", - "env_key": None, # Auth via CLAUDE_CODE_OAUTH_TOKEN oder ~/.claude + "env_key": None, # auth via CLAUDE_CODE_OAUTH_TOKEN or ~/.claude }, - # "minimax-kalt/…" ist KEIN eigener Stack, nur ein opencode-Provider-Eintrag - # (dev-ops/opencode.json) mit niedriger Temperature; M3 dort ohne Thinking. + # "minimax-kalt/…" is NOT its own stack, just an opencode provider entry + # (dev-ops/opencode.json) with low temperature; M3 there without thinking. "minimax": { "cli": "opencode", "guide": "minimax/MiniMax-M3", @@ -141,6 +142,6 @@ PROVIDERS = { "judge": "ollama/qwen3.5:9b", "quick": "ollama/qwen3.5:9b", "env_key": None, - "check_url": "http://localhost:11434/api/tags", # Ollama erreichbar? + "check_url": "http://localhost:11434/api/tags", # Ollama reachable? }, } diff --git a/backend/crawl.py b/backend/crawl.py index 03737b6..7b0e743 100644 --- a/backend/crawl.py +++ b/backend/crawl.py @@ -1,9 +1,9 @@ -"""Geboundeter Domain-Crawler für Link-Quellen — rendert JS via Playwright (Chromium). +"""Bounded domain crawler for link sources — renders JS via Playwright (Chromium). -Lädt ab einer Start-URL Seiten + PDFs — NUR dieselbe Domain, begrenzte Tiefe und -Seitenzahl. HTML-Seiten werden im Headless-Browser gerendert (für SPAs nötig), dann -Links + Text aus dem fertigen DOM gezogen. PDFs werden direkt als Bytes geladen. -Deterministisch, gebounded; läuft via asyncio.to_thread (Sync-API, kein Event-Loop). +Loads pages + PDFs starting from a start URL — ONLY the same domain, limited depth +and page count. HTML pages are rendered in a headless browser (needed for SPAs), then +links + text are pulled from the finished DOM. PDFs are loaded directly as bytes. +Deterministic, bounded; runs via asyncio.to_thread (sync API, no event loop). """ import hashlib @@ -17,23 +17,23 @@ from fsutil import atomic_write_text log = logging.getLogger("creator.crawl") -MAX_TIEFE = 3 -MAX_SEITEN = 500 -SEITE_TIMEOUT = 30 # Sekunden pro Seite (Render bzw. PDF-Download) -CRAWL_SETTLE_MS = 3000 # gedeckelter Settle nach domcontentloaded (SPA-Render); kein 30s-networkidle-Hang -MAX_BYTES = 10_000_000 # 10 MB Deckel pro PDF +MAX_DEPTH = 3 +MAX_PAGES = 500 +PAGE_TIMEOUT = 30 # seconds per page (render or PDF download) +CRAWL_SETTLE_MS = 3000 # capped settle after domcontentloaded (SPA render); no 30s networkidle hang +MAX_BYTES = 10_000_000 # 10 MB cap per PDF _UA = "Mozilla/5.0 (creator-lernbot)" def _fetch_bytes(url: str) -> bytes | None: - """PDF-Bytes per urllib laden (kein Rendering nötig). None bei Fehler/zu groß.""" + """Load PDF bytes via urllib (no rendering needed). None on error/too large.""" try: req = Request(url, headers={"User-Agent": _UA}) - with urlopen(req, timeout=SEITE_TIMEOUT) as resp: + with urlopen(req, timeout=PAGE_TIMEOUT) as resp: data = resp.read(MAX_BYTES + 1) return None if len(data) > MAX_BYTES else data except Exception as e: - log.debug("crawl: PDF-Fetch fehlgeschlagen %s: %s", url, e) + log.debug("crawl: PDF fetch failed %s: %s", url, e) return None @@ -48,25 +48,25 @@ def _is_pdf(url: str) -> bool: def _scope_prefix(start_url: str) -> str: - """Erstes nicht-leeres Pfad-Segment der Start-URL als Crawl-Scope, z.B. - `/learn/path/x` → `/learn`. Ohne Pfad-Segment → `""` (ganze Domain, kein Regress).""" + """First non-empty path segment of the start URL as the crawl scope, e.g. + `/learn/path/x` → `/learn`. No path segment → `""` (whole domain, no narrowing).""" seg = [s for s in urlparse(start_url).path.split("/") if s] return f"/{seg[0]}" if seg else "" def _in_scope(url: str, prefix: str) -> bool: - """Segment-genauer Prefix-Match (kein `/learn` ⊃ `/learning-x`). Leerer Prefix → alles erlaubt.""" + """Segment-exact prefix match (no `/learn` ⊃ `/learning-x`). Empty prefix → everything allowed.""" if not prefix: return True p = urlparse(url).path return p == prefix or p.startswith(prefix + "/") -def _seiten_text(page) -> str: - """Haupttext der gerenderten Seite — Nav/Footer/Boilerplate per trafilatura entfernt. - Fallback auf den rohen Body-Text, wenn die Extraktion leer/zu kurz ausfällt (Nicht-Artikel-Seiten).""" +def _page_text(page) -> str: + """Main text of the rendered page — nav/footer/boilerplate removed via trafilatura. + Falls back to the raw body text when extraction is empty/too short (non-article pages).""" try: - from trafilatura import extract # lazy: Backend startet auch ohne das Paket + from trafilatura import extract # lazy: the backend starts even without the package text = extract(page.content(), include_comments=False, include_tables=True) or "" except Exception: text = "" @@ -78,58 +78,58 @@ def _seiten_text(page) -> str: return text.strip() -def crawl(start_url: str, ziel: Path, *, max_tiefe: int = MAX_TIEFE, max_seiten: int = MAX_SEITEN, cancelled=None) -> int: - """Crawlt ab start_url (nur gleiche Domain), rendert JS und legt Seiten/PDFs in `ziel` ab. +def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages: int = MAX_PAGES, cancelled=None) -> int: + """Crawl from start_url (same domain only), render JS and store pages/PDFs in `target`. - BFS bis `max_tiefe` / `max_seiten`. Fehler einzelner Seiten werden übersprungen. - Schreibt am ENDE einen `.done`-Marker; ein Abbruch (`cancelled()` → True) lässt ihn weg, - sodass ein Neustart neu crawlt. Gibt die Zahl gespeicherter Quellen zurück. + BFS up to `max_depth` / `max_pages`. Errors on individual pages are skipped. + Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it, + so a restart crawls again. Returns the number of saved sources. """ - # Lazy: so startet das Backend auch ohne installiertes Playwright; nur das Crawlen schlägt dann fehl. + # Lazy: this way the backend starts even without Playwright installed; only crawling then fails. from playwright.sync_api import sync_playwright - ziel.mkdir(parents=True, exist_ok=True) + target.mkdir(parents=True, exist_ok=True) domain = urlparse(start_url).netloc - prefix = _scope_prefix(start_url) # nur Links unter diesem Pfad-Segment folgen - gesehen: set[str] = set() + prefix = _scope_prefix(start_url) # only follow links under this path segment + seen: set[str] = set() queue: list[tuple[str, int]] = [(urldefrag(start_url)[0], 0)] - gespeichert = 0 + saved = 0 with sync_playwright() as pw: browser = pw.chromium.launch(args=["--no-sandbox"]) # non-root (Docker user app) page = browser.new_page(user_agent=_UA) try: - while queue and gespeichert < max_seiten: + while queue and saved < max_pages: if cancelled and cancelled(): - return gespeichert # Abbruch → KEIN .done-Marker → Neustart crawlt neu - url, tiefe = queue.pop(0) - if url in gesehen: + return saved # abort → NO .done marker → restart crawls again + url, depth = queue.pop(0) + if url in seen: continue - gesehen.add(url) + seen.add(url) - # PDFs brauchen kein Rendering — direkt laden. + # PDFs need no rendering — load directly. if _is_pdf(url): data = _fetch_bytes(url) if data: - p = ziel / _name(url, ".pdf") + p = target / _name(url, ".pdf") if not p.exists(): p.write_bytes(data) - gespeichert += 1 + saved += 1 continue try: - page.goto(url, wait_until="domcontentloaded", timeout=SEITE_TIMEOUT * 1000) + page.goto(url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT * 1000) except Exception as e: - log.debug("crawl: goto unvollständig %s: %s", url, e) # trotzdem versuchen, Inhalt zu lesen + log.debug("crawl: goto incomplete %s: %s", url, e) # still try to read the content try: page.wait_for_load_state("networkidle", timeout=CRAWL_SETTLE_MS) except Exception: - pass # SPA mit Dauer-Traffic erreicht nie idle → nach Settle weiter, kein 30s-Hang - text = _seiten_text(page) # Haupttext, Nav/Footer entfernt (Fallback: roher Body) + pass # an SPA with constant traffic never reaches idle → continue after settle, no 30s hang + text = _page_text(page) # main text, nav/footer removed (fallback: raw body) if text: - atomic_write_text(ziel / _name(url, ".txt"), f"QUELLE: {url}\n\n{text}") - gespeichert += 1 - if tiefe < max_tiefe: + atomic_write_text(target / _name(url, ".txt"), f"QUELLE: {url}\n\n{text}") + saved += 1 + if depth < max_depth: try: hrefs = page.eval_on_selector_all("a[href]", "els => els.map(e => e.href)") except Exception: @@ -138,11 +138,11 @@ def crawl(start_url: str, ziel: Path, *, max_tiefe: int = MAX_TIEFE, max_seiten: nxt = urldefrag(href)[0] if (nxt.startswith(("http://", "https://")) and urlparse(nxt).netloc == domain and _in_scope(nxt, prefix) - and nxt not in gesehen): - queue.append((nxt, tiefe + 1)) + and nxt not in seen): + queue.append((nxt, depth + 1)) finally: browser.close() - (ziel / ".done").write_text("ok", encoding="utf-8") # sauber durchgelaufen - log.info("crawl %s → %d Quellen in %s", start_url, gespeichert, ziel) - return gespeichert + (target / ".done").write_text("ok", encoding="utf-8") # ran through cleanly + log.info("crawl %s → %d sources in %s", start_url, saved, target) + return saved diff --git a/backend/database.py b/backend/database.py index a388eea..2ec89c5 100644 --- a/backend/database.py +++ b/backend/database.py @@ -47,110 +47,110 @@ CREATE TABLE IF NOT EXISTS elements ( ) """ -CREATE_BAUSTEIN_TEXTE = """ -CREATE TABLE IF NOT EXISTS baustein_texte ( +CREATE_BLOCK_TEXTE = """ +CREATE TABLE IF NOT EXISTS block_texte ( topic TEXT NOT NULL, - baustein TEXT NOT NULL, - art TEXT NOT NULL, + block TEXT NOT NULL, + kind TEXT NOT NULL, md TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - PRIMARY KEY (topic, baustein, art) + PRIMARY KEY (topic, block, kind) ) """ -CREATE_BAUSTEIN_PROGRESS = """ -CREATE TABLE IF NOT EXISTS baustein_progress ( +CREATE_BLOCK_PROGRESS = """ +CREATE TABLE IF NOT EXISTS block_progress ( topic TEXT NOT NULL, - baustein TEXT NOT NULL, - gute_antworten INTEGER NOT NULL DEFAULT 0, + block TEXT NOT NULL, + good_answers INTEGER NOT NULL DEFAULT 0, streak INTEGER NOT NULL DEFAULT 0, - absolviert TEXT, - verstanden TEXT, - gemeistert TEXT, + completed TEXT, + understood TEXT, + mastered TEXT, updated_at TEXT NOT NULL, - PRIMARY KEY (topic, baustein) + PRIMARY KEY (topic, block) ) """ -# --- Bausteine-Pipeline-Inhalt (ersetzt Datei-Sidecars) --- +# --- Blocks pipeline content (replaces file sidecars) --- -# Inventar: ein Baustein je (topic, titel_norm). nennungen = Anzahl Agenten/Runden, -# die ihn nannten (≥2 = Konsens). status: kandidat/konsens/rest/verworfen. -CREATE_BAUSTEINE = """ -CREATE TABLE IF NOT EXISTS bausteine ( +# Inventory: one block per (topic, title_norm). mentions = number of agents/rounds +# that named it (≥2 = consensus). status: candidate/consensus/rest/discarded. +CREATE_BLOCKS = """ +CREATE TABLE IF NOT EXISTS blocks ( topic TEXT NOT NULL, - titel_norm TEXT NOT NULL, - titel TEXT NOT NULL, - beschreibung TEXT NOT NULL DEFAULT '', - nennungen INTEGER NOT NULL DEFAULT 1, - status TEXT NOT NULL DEFAULT 'kandidat', - quellen TEXT NOT NULL DEFAULT '[]', + title_norm TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + mentions INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'candidate', + sources TEXT NOT NULL DEFAULT '[]', reader TEXT NOT NULL DEFAULT '[]', updated_at TEXT NOT NULL, - PRIMARY KEY (topic, titel_norm) + PRIMARY KEY (topic, title_norm) ) """ -# Subbausteine je Baustein. stufe (einfach/mittel/schwer) + relevanz (relevant/rand) -# werden später gesetzt. nennungen analog zum Inventar. -CREATE_SUBBAUSTEINE = """ -CREATE TABLE IF NOT EXISTS subbausteine ( +# Subblocks per block. level (einfach/mittel/schwer) + relevance (relevant/rand) +# are set later. mentions analogous to the inventory. +CREATE_SUBBLOCKS = """ +CREATE TABLE IF NOT EXISTS subblocks ( topic TEXT NOT NULL, - baustein_norm TEXT NOT NULL, + block_norm TEXT NOT NULL, sub_norm TEXT NOT NULL, - baustein TEXT NOT NULL, - sub_titel TEXT NOT NULL, - nennungen INTEGER NOT NULL DEFAULT 1, - stufe TEXT, - relevanz TEXT, - fakten TEXT NOT NULL DEFAULT '', - status TEXT NOT NULL DEFAULT 'kandidat', + block TEXT NOT NULL, + sub_title TEXT NOT NULL, + mentions INTEGER NOT NULL DEFAULT 1, + level TEXT, + relevance TEXT, + facts TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'candidate', updated_at TEXT NOT NULL, - PRIMARY KEY (topic, baustein_norm, sub_norm) + PRIMARY KEY (topic, block_norm, sub_norm) ) """ -# Ein Frage-Muster je (Baustein, Subbaustein). Schwierigkeit kommt erst bei der Prüfung -# aus dem Lerner-Niveau, nicht aus dem Muster — daher kein Typ-Kreuzprodukt mehr. -CREATE_FRAGE_MUSTER = """ -CREATE TABLE IF NOT EXISTS frage_muster ( +# One question pattern per (block, subblock). Difficulty comes only at exam time +# from the learner tier, not from the pattern — hence no more type cross-product. +CREATE_QUESTION_PATTERN = """ +CREATE TABLE IF NOT EXISTS question_pattern ( topic TEXT NOT NULL, - baustein_norm TEXT NOT NULL, + block_norm TEXT NOT NULL, sub_norm TEXT NOT NULL, - baustein TEXT NOT NULL, - sub_titel TEXT NOT NULL, - frage TEXT NOT NULL, + block TEXT NOT NULL, + sub_title TEXT NOT NULL, + question TEXT NOT NULL, updated_at TEXT NOT NULL, - PRIMARY KEY (topic, baustein_norm, sub_norm) + PRIMARY KEY (topic, block_norm, sub_norm) ) """ -# Crawl-Seiten je Thema: inhalt (1=Content/0=Noise, von der Sichtung) + gelesen (von der Recherche). -CREATE_RECHERCHE_COVERAGE = """ -CREATE TABLE IF NOT EXISTS recherche_coverage ( +# Crawl pages per topic: content (1=content/0=noise, from the triage) + read_done (from the research). +CREATE_RESEARCH_COVERAGE = """ +CREATE TABLE IF NOT EXISTS research_coverage ( topic TEXT NOT NULL, - quelle TEXT NOT NULL, - gelesen INTEGER NOT NULL DEFAULT 0, - inhalt INTEGER, + source TEXT NOT NULL, + read_done INTEGER NOT NULL DEFAULT 0, + content INTEGER, updated_at TEXT NOT NULL, - PRIMARY KEY (topic, quelle) + PRIMARY KEY (topic, source) ) """ -# Schritt-Status der Pipeline (ersetzt Datei-Existenz-Resume + Reset-Globs). +# Step status of the pipeline (replaces file-existence resume + reset globs). CREATE_PIPELINE_STATE = """ CREATE TABLE IF NOT EXISTS pipeline_state ( topic TEXT NOT NULL, - schritt TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'offen', + step TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'open', updated_at TEXT NOT NULL, - PRIMARY KEY (topic, schritt) + PRIMARY KEY (topic, step) ) """ -# Fertiger Guide-Inhalt je (Thema, Format) als JSON-Blob (ersetzt die Guide-JSON-Datei). -# Geteilt über alle Guide-Läufe desselben Thema+Formats (wie zuvor die Content-Datei). +# Finished guide content per (topic, format) as a JSON blob (replaces the guide JSON file). +# Shared across all guide runs of the same topic+format (as the content file was before). CREATE_GUIDE_CONTENT = """ CREATE TABLE IF NOT EXISTS guide_content ( topic TEXT NOT NULL, @@ -161,41 +161,41 @@ CREATE TABLE IF NOT EXISTS guide_content ( ) """ -# Quellen-Wahl je Thema (ersetzt quelle.json). -CREATE_QUELLE = """ -CREATE TABLE IF NOT EXISTS quelle ( +# Source choice per topic (replaces source.json). +CREATE_SOURCE = """ +CREATE TABLE IF NOT EXISTS source ( topic TEXT PRIMARY KEY, type TEXT NOT NULL, - ort TEXT NOT NULL DEFAULT '', + location TEXT NOT NULL DEFAULT '', spec TEXT NOT NULL DEFAULT '', updated_at TEXT NOT NULL ) """ -# Gliederung (Kapitel → Baustein-Nummern) je Thema als JSON. In der Bausteine-Phase erzeugt, -# vom Guide nur noch gelesen + je Format gefiltert. Format-agnostisch (alle Bausteine). -CREATE_GUIDE_GLIEDERUNG = """ -CREATE TABLE IF NOT EXISTS guide_gliederung ( +# Outline (chapter → block numbers) per topic as JSON. Produced in the blocks phase, +# the guide only reads it + filters per format. Format-agnostic (all blocks). +CREATE_GUIDE_OUTLINE = """ +CREATE TABLE IF NOT EXISTS guide_outline ( topic TEXT PRIMARY KEY, json TEXT NOT NULL, updated_at TEXT NOT NULL ) """ -# Generischer Lern-Artefakt-Layer: je (Baustein, Subbaustein, Typ) ein Artefakt (JSON in `daten`). -# typ: karteikarte | beispiel. sub_norm='' = Baustein-Ebene (für künftige Baustein-weite Artefakte). -# In der Bausteine-Phase aus den Fakten erzeugt, vom Frontend präsentiert (Trennung Artefakt/Anzeige). +# Generic learning artifact layer: one artifact per (block, subblock, type) (JSON in `data`). +# type: flashcard | example. sub_norm='' = block level (for future block-wide artifacts). +# Produced from the facts in the blocks phase, presented by the frontend (artifact/display split). CREATE_SUB_ARTEFAKTE = """ CREATE TABLE IF NOT EXISTS sub_artefakte ( topic TEXT NOT NULL, - baustein_norm TEXT NOT NULL, + block_norm TEXT NOT NULL, sub_norm TEXT NOT NULL DEFAULT '', - typ TEXT NOT NULL, - baustein TEXT NOT NULL DEFAULT '', - sub_titel TEXT NOT NULL DEFAULT '', - daten TEXT NOT NULL DEFAULT '{}', + type TEXT NOT NULL, + block TEXT NOT NULL DEFAULT '', + sub_title TEXT NOT NULL DEFAULT '', + data TEXT NOT NULL DEFAULT '{}', updated_at TEXT NOT NULL, - PRIMARY KEY (topic, baustein_norm, sub_norm, typ) + PRIMARY KEY (topic, block_norm, sub_norm, type) ) """ @@ -212,75 +212,75 @@ async def get_db() -> aiosqlite.Connection: async def init_db(): db = await get_db() - # WAL übersteht Crashes deutlich besser; busy_timeout fängt kurze Locks ab. + # WAL survives crashes much better; busy_timeout absorbs short locks. await db.execute("PRAGMA journal_mode=WAL") await db.execute("PRAGMA busy_timeout=5000") await db.execute(CREATE_GUIDES) await db.execute(CREATE_PROGRESS) await db.execute(CREATE_TOPICS) await db.execute(CREATE_ELEMENTS) - await db.execute(CREATE_BAUSTEIN_TEXTE) - await db.execute(CREATE_BAUSTEIN_PROGRESS) - await db.execute(CREATE_BAUSTEINE) - await db.execute(CREATE_SUBBAUSTEINE) - await db.execute(CREATE_FRAGE_MUSTER) - await db.execute(CREATE_RECHERCHE_COVERAGE) + await db.execute(CREATE_BLOCK_TEXTE) + await db.execute(CREATE_BLOCK_PROGRESS) + await db.execute(CREATE_BLOCKS) + await db.execute(CREATE_SUBBLOCKS) + await db.execute(CREATE_QUESTION_PATTERN) + await db.execute(CREATE_RESEARCH_COVERAGE) await db.execute(CREATE_PIPELINE_STATE) await db.execute(CREATE_GUIDE_CONTENT) - await db.execute(CREATE_QUELLE) - await db.execute(CREATE_GUIDE_GLIEDERUNG) + await db.execute(CREATE_SOURCE) + await db.execute(CREATE_GUIDE_OUTLINE) await db.execute(CREATE_SUB_ARTEFAKTE) - try: # Migration für Bestands-DBs ohne step-Spalte + try: # migration for existing DBs without the step column await db.execute("ALTER TABLE guides ADD COLUMN step INTEGER") except aiosqlite.OperationalError: pass - try: # Migration: recherche_coverage.inhalt (Content/Noise aus der Sichtung) - await db.execute("ALTER TABLE recherche_coverage ADD COLUMN inhalt INTEGER") + try: # migration: research_coverage.content (content/noise from the triage) + await db.execute("ALTER TABLE research_coverage ADD COLUMN content INTEGER") except aiosqlite.OperationalError: pass - try: # Migration für Bestands-DBs ohne verstanden-Spalte (Mastery-Stufe) - await db.execute("ALTER TABLE baustein_progress ADD COLUMN verstanden TEXT") + try: # migration for existing DBs without the understood column (mastery level) + await db.execute("ALTER TABLE block_progress ADD COLUMN understood TEXT") except aiosqlite.OperationalError: pass - try: # Migration für Bestands-DBs ohne gemeistert-Spalte (Meisterpfad 25) - await db.execute("ALTER TABLE baustein_progress ADD COLUMN gemeistert TEXT") + try: # migration for existing DBs without the mastered column (master path 25) + await db.execute("ALTER TABLE block_progress ADD COLUMN mastered TEXT") except aiosqlite.OperationalError: pass - try: # Migration für Bestands-DBs ohne streak-Spalte (persistente Streak-Bonus-Folge) - await db.execute("ALTER TABLE baustein_progress ADD COLUMN streak INTEGER NOT NULL DEFAULT 0") + try: # migration for existing DBs without the streak column (persistent streak-bonus run) + await db.execute("ALTER TABLE block_progress ADD COLUMN streak INTEGER NOT NULL DEFAULT 0") except aiosqlite.OperationalError: pass - # Offene-Frage-Anker: Basis/Streak VOR der aktuell offenen Frage — macht die Bewertung - # serverseitig idempotent (Re-Bewertung) und driftfrei über Fragen hinweg. - for _spalte, _typ in (("offene_frage", "TEXT"), ("offene_basis", "INTEGER"), ("offene_streak", "INTEGER")): + # Open-question anchor: base/streak BEFORE the currently open question — makes the rating + # idempotent server-side (re-rating) and drift-free across questions. + for _col, _type in (("offene_question", "TEXT"), ("offene_basis", "INTEGER"), ("offene_streak", "INTEGER")): try: - await db.execute(f"ALTER TABLE baustein_progress ADD COLUMN {_spalte} {_typ}") + await db.execute(f"ALTER TABLE block_progress ADD COLUMN {_col} {_type}") except aiosqlite.OperationalError: pass - # Migration: frage_muster ohne typ-Spalte (1 Muster je Sub statt Sub×Typ-Kreuzprodukt). - # PK-Änderung → Tabelle einmalig neu bauen. Bestands-Muster gehen verloren (bewusst, kein Mapping). - cursor = await db.execute("PRAGMA table_info(frage_muster)") - if any(_r[1] == "typ" for _r in await cursor.fetchall()): - await db.execute("DROP TABLE frage_muster") - await db.execute(CREATE_FRAGE_MUSTER) - try: # Migration: subbausteine.fakten (Quell-Fakten je Sub, JSON-Blob) — Extract-once-Grounding. - await db.execute("ALTER TABLE subbausteine ADD COLUMN fakten TEXT NOT NULL DEFAULT ''") # DEFAULT '' nötig für NOT NULL beim ADD COLUMN + # Migration: question_pattern without a type column (1 pattern per sub instead of a sub×type cross-product). + # PK change → rebuild the table once. Existing patterns are lost (intentional, no mapping). + cursor = await db.execute("PRAGMA table_info(question_pattern)") + if any(_r[1] == "type" for _r in await cursor.fetchall()): + await db.execute("DROP TABLE question_pattern") + await db.execute(CREATE_QUESTION_PATTERN) + try: # migration: subblocks.facts (source facts per sub, JSON blob) — extract-once grounding. + await db.execute("ALTER TABLE subblocks ADD COLUMN facts TEXT NOT NULL DEFAULT ''") # DEFAULT '' needed for NOT NULL on ADD COLUMN except aiosqlite.OperationalError: pass - try: # Migration: bausteine.reader (Reader-Set je Kandidat, JSON) — exakte Konsens-Zählung (≥2 Reader) - await db.execute("ALTER TABLE bausteine ADD COLUMN reader TEXT NOT NULL DEFAULT '[]'") + try: # migration: blocks.reader (reader set per candidate, JSON) — exact consensus count (≥2 readers) + await db.execute("ALTER TABLE blocks ADD COLUMN reader TEXT NOT NULL DEFAULT '[]'") except aiosqlite.OperationalError: pass - # Migration: alte vertiefungen-Tabelle → baustein_texte (Bestand = lange Form, art 'deepdive') + # Migration: old vertiefungen table → block_texte (existing = long form, kind 'deepdive') cursor = await db.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'vertiefungen'") if await cursor.fetchone(): await db.execute( - "INSERT OR IGNORE INTO baustein_texte (topic, baustein, art, md, created_at, updated_at) " - "SELECT topic, baustein, 'deepdive', md, created_at, updated_at FROM vertiefungen" + "INSERT OR IGNORE INTO block_texte (topic, block, kind, md, created_at, updated_at) " + "SELECT topic, block, 'deepdive', md, created_at, updated_at FROM vertiefungen" ) await db.execute("DROP TABLE vertiefungen") await db.execute( - "UPDATE guides SET status = 'error', progress = NULL, error_msg = 'Server-Neustart' " + "UPDATE guides SET status = 'error', progress = NULL, error_msg = 'Server restart' " "WHERE status IN ('queued', 'generating')" ) await db.commit() @@ -325,14 +325,21 @@ async def list_guides() -> list[dict]: return [_row_to_dict(row, cursor) for row in rows] -async def update_guide(guide_id: str, **fields) -> None: +async def _update(table: str, fields: dict, where: dict) -> None: + """UPDATE SET WHERE (+ commit). WHERE params are aliased (`w_`) + so a field and a WHERE key of the same name don't collide — needed e.g. for a `title_norm` + rename (SET new norm WHERE old norm).""" sets = ", ".join(f"{k} = :{k}" for k in fields) - fields["id"] = guide_id + cond = " AND ".join(f"{k} = :w_{k}" for k in where) db = await get_db() - await db.execute(f"UPDATE guides SET {sets} WHERE id = :id", fields) + await db.execute(f"UPDATE {table} SET {sets} WHERE {cond}", {**fields, **{f"w_{k}": v for k, v in where.items()}}) await db.commit() +async def update_guide(guide_id: str, **fields) -> None: + await _update("guides", fields, {"id": guide_id}) + + async def delete_guide(guide_id: str) -> bool: db = await get_db() cursor = await db.execute("DELETE FROM guides WHERE id = ?", (guide_id,)) @@ -340,7 +347,7 @@ async def delete_guide(guide_id: str) -> bool: return cursor.rowcount > 0 -# --- Themen --- +# --- Topics --- async def create_topic(name: str) -> None: from datetime import datetime, timezone @@ -365,7 +372,7 @@ async def delete_topic(name: str) -> None: await db.commit() -# --- Elemente --- +# --- Elements --- def _element_row(row, cursor) -> dict: el = _row_to_dict(row, cursor) @@ -408,11 +415,7 @@ async def update_element(element_id: str, **fields) -> None: for key in ("examples", "hints"): if key in fields: fields[key] = json.dumps(fields[key], ensure_ascii=False) - sets = ", ".join(f"{k} = :{k}" for k in fields) - fields["id"] = element_id - db = await get_db() - await db.execute(f"UPDATE elements SET {sets} WHERE id = :id", fields) - await db.commit() + await _update("elements", fields, {"id": element_id}) async def delete_element(element_id: str) -> bool: @@ -422,10 +425,10 @@ async def delete_element(element_id: str) -> bool: return cursor.rowcount > 0 -# --- Kapitel-Fortschritt --- +# --- Chapter progress --- async def list_progress_all() -> dict[str, set[str]]: - """Kompletter Kapitel-Fortschritt in einem Query: guide_id → Kapitel-Titel.""" + """Complete chapter progress in one query: guide_id → chapter title.""" db = await get_db() cursor = await db.execute("SELECT guide_id, chapter FROM guide_progress") rows = await cursor.fetchall() @@ -465,421 +468,409 @@ async def delete_progress(guide_id: str) -> None: await db.commit() -# --- Baustein-Lernen: Vertiefungen + Prüfungs-Fortschritt --- +# --- Block learning: deep-dives + exam progress --- def _now() -> str: from datetime import datetime, timezone return datetime.now(timezone.utc).isoformat() -async def list_baustein_progress(topic: str) -> list[dict]: +async def list_block_progress(topic: str) -> list[dict]: db = await get_db() cursor = await db.execute( - "SELECT baustein, gute_antworten, streak, absolviert, verstanden, gemeistert FROM baustein_progress WHERE topic = ?", (topic,) + "SELECT block, good_answers, streak, completed, understood, mastered FROM block_progress WHERE topic = ?", (topic,) ) rows = await cursor.fetchall() - return [{"baustein": b, "gute_antworten": n, "streak": s, "absolviert": a, "verstanden": v, "gemeistert": m} for b, n, s, a, v, m in rows] + return [{"block": b, "good_answers": n, "streak": s, "completed": a, "understood": v, "mastered": m} for b, n, s, a, v, m in rows] -async def get_baustein_progress(topic: str, baustein: str) -> dict: - """Eine Baustein-Zeile inkl. Offene-Frage-Anker. Defaults, falls noch keine existiert.""" +async def get_block_progress(topic: str, block: str) -> dict: + """One block row incl. open-question anchor. Defaults if none exists yet.""" db = await get_db() cursor = await db.execute( - "SELECT gute_antworten, streak, absolviert, verstanden, gemeistert, " - "offene_frage, offene_basis, offene_streak FROM baustein_progress " - "WHERE topic = ? AND baustein = ?", - (topic, baustein), + "SELECT good_answers, streak, completed, understood, mastered, " + "offene_question, offene_basis, offene_streak FROM block_progress " + "WHERE topic = ? AND block = ?", + (topic, block), ) row = await cursor.fetchone() if row is None: - return {"gute_antworten": 0, "streak": 0, "absolviert": None, "verstanden": None, - "gemeistert": None, "offene_frage": None, "offene_basis": None, "offene_streak": None} - return {"gute_antworten": row[0], "streak": row[1], "absolviert": row[2], "verstanden": row[3], - "gemeistert": row[4], "offene_frage": row[5], "offene_basis": row[6], "offene_streak": row[7]} + return {"good_answers": 0, "streak": 0, "completed": None, "understood": None, + "mastered": None, "offene_question": None, "offene_basis": None, "offene_streak": None} + return {"good_answers": row[0], "streak": row[1], "completed": row[2], "understood": row[3], + "mastered": row[4], "offene_question": row[5], "offene_basis": row[6], "offene_streak": row[7]} -async def set_offene_frage(topic: str, baustein: str, frage: str, basis: int, streak: int) -> None: - """Friert Basis + Streak VOR der jetzt offenen Frage ein (Anker für idempotente Re-Bewertung).""" +async def set_open_question(topic: str, block: str, question: str, basis: int, streak: int) -> None: + """Freeze base + streak BEFORE the now-open question (anchor for idempotent re-rating).""" db = await get_db() now = _now() await db.execute( - """INSERT INTO baustein_progress (topic, baustein, offene_frage, offene_basis, offene_streak, updated_at) + """INSERT INTO block_progress (topic, block, offene_question, offene_basis, offene_streak, updated_at) VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(topic, baustein) DO UPDATE SET - offene_frage = excluded.offene_frage, offene_basis = excluded.offene_basis, + ON CONFLICT(topic, block) DO UPDATE SET + offene_question = excluded.offene_question, offene_basis = excluded.offene_basis, offene_streak = excluded.offene_streak, updated_at = excluded.updated_at""", - (topic, baustein, frage, basis, streak, now), + (topic, block, question, basis, streak, now), ) await db.commit() -async def set_baustein_score_and_streak(topic: str, baustein: str, score: int, streak: int) -> tuple[int, int]: - """Setzt Score + Streak atomar (vom Aufrufer geclampt). Liefert (score, streak).""" +async def set_block_score_and_streak(topic: str, block: str, score: int, streak: int) -> tuple[int, int]: + """Set score + streak atomically (clamped by the caller). Returns (score, streak).""" db = await get_db() await db.execute( - """INSERT INTO baustein_progress (topic, baustein, gute_antworten, streak, updated_at) + """INSERT INTO block_progress (topic, block, good_answers, streak, updated_at) VALUES (?, ?, ?, ?, ?) - ON CONFLICT(topic, baustein) DO UPDATE SET - gute_antworten = excluded.gute_antworten, streak = excluded.streak, + ON CONFLICT(topic, block) DO UPDATE SET + good_answers = excluded.good_answers, streak = excluded.streak, updated_at = excluded.updated_at""", - (topic, baustein, score, streak, _now()), + (topic, block, score, streak, _now()), ) await db.commit() return score, streak -async def delete_baustein_progress(topic: str, baustein: str) -> None: - """Fortschritt EINES Bausteins zurücksetzen: Zeile löschen (Score/Streak/Flags/offene Frage). - Fehlt die Zeile, liefert get_baustein_progress Defaults (0) — also voller Reset.""" +async def delete_block_progress(topic: str, block: str) -> None: + """Reset the progress of ONE block: delete the row (score/streak/flags/open question). + If the row is missing, get_block_progress returns defaults (0) — i.e. a full reset.""" db = await get_db() - await db.execute("DELETE FROM baustein_progress WHERE topic = ? AND baustein = ?", (topic, baustein)) + await db.execute("DELETE FROM block_progress WHERE topic = ? AND block = ?", (topic, block)) await db.commit() -async def set_baustein_absolviert(topic: str, baustein: str) -> bool: - """Markiert absolviert; True nur beim ersten Mal (steuert den Element-Task).""" +async def set_block_completed(topic: str, block: str) -> bool: + """Marks completed; True only the first time (drives the element task).""" db = await get_db() now = _now() await db.execute( - "INSERT OR IGNORE INTO baustein_progress (topic, baustein, gute_antworten, updated_at) VALUES (?, ?, 0, ?)", - (topic, baustein, now), + "INSERT OR IGNORE INTO block_progress (topic, block, good_answers, updated_at) VALUES (?, ?, 0, ?)", + (topic, block, now), ) cursor = await db.execute( - "UPDATE baustein_progress SET absolviert = ?, updated_at = ? " - "WHERE topic = ? AND baustein = ? AND absolviert IS NULL", - (now, now, topic, baustein), + "UPDATE block_progress SET completed = ?, updated_at = ? " + "WHERE topic = ? AND block = ? AND completed IS NULL", + (now, now, topic, block), ) await db.commit() return cursor.rowcount > 0 -# Sub-Ebene aus den zwei orthogonalen Spalten: rand → 4 (V), sonst Stufe (Lernpfad-Position): -# anfaenger/NULL → 1, fortgeschritten → 2, experte → 3. Alte Werte (einfach/mittel/schwer) werden -# abwärtskompatibel mitgemappt. Steuert Guide-Ansicht A/F/E/V + cap (freigeschaltete Subs × 25). -_EBENE_CASE = """CASE - WHEN relevanz = 'rand' THEN 4 - WHEN stufe IN ('fortgeschritten', 'mittel') THEN 2 - WHEN stufe IN ('experte', 'schwer') THEN 3 +# Sub-level from the two orthogonal columns: peripheral → 4 (V), otherwise level (learning-path position): +# beginner/NULL → 1, advanced → 2, expert → 3. Old values (einfach/mittel/schwer) are +# mapped in backward-compatibly. Drives the guide view A/F/E/V + cap (unlocked subs × 25). +_LEVEL_CASE = """CASE + WHEN relevance = 'peripheral' THEN 4 + WHEN level IN ('advanced', 'medium') THEN 2 + WHEN level IN ('expert', 'hard') THEN 3 ELSE 1 END""" -def _leere_ebenen() -> dict[int, int]: +def _empty_levels() -> dict[int, int]: return {1: 0, 2: 0, 3: 0, 4: 0} -async def subs_je_ebene(topic: str, baustein: str) -> dict[int, int]: - """Konsens-Subbausteine eines Bausteins je Ebene 1–4. roher Baustein-Titel rein.""" - from textkit import _norm_titel +async def subs_per_level(topic: str, block: str) -> dict[int, int]: + """Consensus subblocks of a block per level 1–4. Pass in the raw block title.""" + from textkit import _norm_title db = await get_db() cursor = await db.execute( - f"SELECT {_EBENE_CASE} AS ebene, COUNT(*) FROM subbausteine " - "WHERE topic = ? AND baustein_norm = ? AND status = 'konsens' GROUP BY ebene", - (topic, _norm_titel(baustein)), + f"SELECT {_LEVEL_CASE} AS level, COUNT(*) FROM subblocks " + "WHERE topic = ? AND block_norm = ? AND status = 'consensus' GROUP BY level", + (topic, _norm_title(block)), ) - out = _leere_ebenen() - for ebene, n in await cursor.fetchall(): - out[ebene] = n + out = _empty_levels() + for level, n in await cursor.fetchall(): + out[level] = n return out -async def subs_je_ebene_roh(topic: str) -> dict[str, dict[int, int]]: - """Subbausteine je Ebene, gruppiert nach ROHEM Baustein-Titel (= Guide-Section-Titel).""" +async def subs_per_level_raw(topic: str) -> dict[str, dict[int, int]]: + """Subblocks per level, grouped by RAW block title (= guide section title).""" db = await get_db() cursor = await db.execute( - f"SELECT baustein, {_EBENE_CASE} AS ebene, COUNT(*) FROM subbausteine " - "WHERE topic = ? AND status = 'konsens' GROUP BY baustein, ebene", + f"SELECT block, {_LEVEL_CASE} AS level, COUNT(*) FROM subblocks " + "WHERE topic = ? AND status = 'consensus' GROUP BY block, level", (topic,), ) out: dict[str, dict[int, int]] = {} - for b, ebene, n in await cursor.fetchall(): - out.setdefault(b, _leere_ebenen())[ebene] = n + for b, level, n in await cursor.fetchall(): + out.setdefault(b, _empty_levels())[level] = n return out -async def subs_je_ebene_alle() -> dict[tuple[str, str], dict[int, int]]: - """Subbausteine je Ebene je (topic, baustein_norm) — für die themenweite Stufen-Ableitung.""" +async def subs_per_level_all() -> dict[tuple[str, str], dict[int, int]]: + """Subblocks per level per (topic, block_norm) — for the topic-wide levels derivation.""" db = await get_db() cursor = await db.execute( - f"SELECT topic, baustein_norm, {_EBENE_CASE} AS ebene, COUNT(*) FROM subbausteine " - "WHERE status = 'konsens' GROUP BY topic, baustein_norm, ebene" + f"SELECT topic, block_norm, {_LEVEL_CASE} AS level, COUNT(*) FROM subblocks " + "WHERE status = 'consensus' GROUP BY topic, block_norm, level" ) out: dict[tuple[str, str], dict[int, int]] = {} - for t, bn, ebene, n in await cursor.fetchall(): - out.setdefault((t, bn), _leere_ebenen())[ebene] = n + for t, bn, level, n in await cursor.fetchall(): + out.setdefault((t, bn), _empty_levels())[level] = n return out -async def subs_mit_ebene(topic: str, baustein: str) -> list[dict]: - """Konsens-Subbausteine eines Bausteins mit Titel, sub_norm und Ebene 1–4.""" - from textkit import _norm_titel +async def subs_with_level(topic: str, block: str) -> list[dict]: + """Consensus subblocks of a block with title, sub_norm and level 1–4.""" + from textkit import _norm_title db = await get_db() cursor = await db.execute( - f"SELECT sub_titel, sub_norm, {_EBENE_CASE} AS ebene FROM subbausteine " - "WHERE topic = ? AND baustein_norm = ? AND status = 'konsens'", - (topic, _norm_titel(baustein)), + f"SELECT sub_title, sub_norm, {_LEVEL_CASE} AS level FROM subblocks " + "WHERE topic = ? AND block_norm = ? AND status = 'consensus'", + (topic, _norm_title(block)), ) - return [{"titel": t, "norm": sn, "ebene": e} for t, sn, e in await cursor.fetchall()] + return [{"title": t, "norm": sn, "level": e} for t, sn, e in await cursor.fetchall()] -async def list_baustein_scores_all() -> list[tuple[str, str, int]]: - """(topic, baustein, gute_antworten) je Baustein — Rohdaten für die Stufen-Ableitung.""" +async def list_block_scores_all() -> list[tuple[str, str, int]]: + """(topic, block, good_answers) per block — raw data for the levels derivation.""" db = await get_db() - cursor = await db.execute("SELECT topic, baustein, gute_antworten FROM baustein_progress") + cursor = await db.execute("SELECT topic, block, good_answers FROM block_progress") return [(t, b, n) for t, b, n in await cursor.fetchall()] -async def delete_baustein_daten(topic: str) -> None: +async def delete_block_data(topic: str) -> None: db = await get_db() - await db.execute("DELETE FROM baustein_texte WHERE topic = ?", (topic,)) - await db.execute("DELETE FROM baustein_progress WHERE topic = ?", (topic,)) + await db.execute("DELETE FROM block_texte WHERE topic = ?", (topic,)) + await db.execute("DELETE FROM block_progress WHERE topic = ?", (topic,)) await db.commit() -# --- Bausteine-Pipeline-Inhalt: Inventar / Subbausteine / Frage-Muster / Coverage / State / Quelle --- +# --- Blocks pipeline content: inventory / subblocks / question pattern / coverage / state / source --- -async def upsert_baustein(topic: str, titel_norm: str, titel: str, beschreibung: str = "", - quellen: list | None = None, reader: str | None = None) -> None: - """Kandidat einfügen oder Reader-Set vereinigen. Erst-Beschreibung bleibt erhalten. +async def upsert_block(topic: str, title_norm: str, title: str, description: str = "", + sources: list | None = None, reader: str | None = None) -> None: + """Insert a candidate or union the reader set. The first description is kept. - `reader` = ID des Recherche-Readers (z.B. "a5-1"). Die Vereinigung läuft race-frei in - EINEM Statement (json1), weil mehrere Reader-Coroutinen nebenläufig upserten — ein - read-modify-write über `await` würde Mitglieder verlieren. `nennungen` bleibt synchron - zu `len(reader)`. `reader=None` (z.B. aus `_set_inventar`) lässt das Set unverändert.""" + `reader` = ID of the research reader (e.g. "a5-1"). The union runs race-free in + ONE statement (json1), because several reader coroutines upsert concurrently — a + read-modify-write across `await` would lose members. `mentions` stays in sync + with `len(reader)`. `reader=None` (e.g. from `_set_inventar`) leaves the set unchanged.""" db = await get_db() rid = reader if isinstance(reader, str) and reader else None await db.execute( - """INSERT INTO bausteine (topic, titel_norm, titel, beschreibung, nennungen, status, quellen, reader, updated_at) - VALUES (?, ?, ?, ?, 1, 'kandidat', ?, ?, ?) - ON CONFLICT(topic, titel_norm) DO UPDATE SET + """INSERT INTO blocks (topic, title_norm, title, description, mentions, status, sources, reader, updated_at) + VALUES (?, ?, ?, ?, 1, 'candidate', ?, ?, ?) + ON CONFLICT(topic, title_norm) DO UPDATE SET reader = (SELECT json_group_array(v) FROM ( - SELECT value AS v FROM json_each(bausteine.reader) + SELECT value AS v FROM json_each(blocks.reader) UNION SELECT ? WHERE ? IS NOT NULL)), - nennungen = (SELECT count(*) FROM ( - SELECT value AS v FROM json_each(bausteine.reader) + mentions = (SELECT count(*) FROM ( + SELECT value AS v FROM json_each(blocks.reader) UNION SELECT ? WHERE ? IS NOT NULL)), - quellen = excluded.quellen, updated_at = excluded.updated_at""", - (topic, titel_norm, titel, beschreibung, - json.dumps(quellen or [], ensure_ascii=False), + sources = excluded.sources, updated_at = excluded.updated_at""", + (topic, title_norm, title, description, + json.dumps(sources or [], ensure_ascii=False), json.dumps([rid] if rid else [], ensure_ascii=False), _now(), rid, rid, rid, rid), ) await db.commit() -async def list_bausteine(topic: str, status: str | None = None) -> list[dict]: +async def list_blocks(topic: str, status: str | None = None) -> list[dict]: db = await get_db() if status is None: - cursor = await db.execute("SELECT * FROM bausteine WHERE topic = ? ORDER BY rowid", (topic,)) + cursor = await db.execute("SELECT * FROM blocks WHERE topic = ? ORDER BY rowid", (topic,)) else: - cursor = await db.execute("SELECT * FROM bausteine WHERE topic = ? AND status = ? ORDER BY rowid", (topic, status)) + cursor = await db.execute("SELECT * FROM blocks WHERE topic = ? AND status = ? ORDER BY rowid", (topic, status)) rows = await cursor.fetchall() out = [] for row in rows: d = _row_to_dict(row, cursor) - d["quellen"] = json.loads(d.get("quellen") or "[]") + d["sources"] = json.loads(d.get("sources") or "[]") d["reader"] = json.loads(d.get("reader") or "[]") out.append(d) return out -async def set_baustein_status(topic: str, titel_norm: str, status: str, titel: str | None = None, beschreibung: str | None = None, neu_norm: str | None = None) -> None: - """Status setzen; optional Titel/Beschreibung aktualisieren (z.B. nach semantischem Merge). - `neu_norm` benennt den Norm-Schlüssel um (Klärung: Verweis-Titel → sprechender Name). Nur - sicher, solange noch keine Subbausteine/Fakten am alten `titel_norm` hängen.""" - db = await get_db() +async def set_block_status(topic: str, title_norm: str, status: str, title: str | None = None, description: str | None = None, neu_norm: str | None = None) -> None: + """Set status; optionally update title/description (e.g. after a semantic merge). + `neu_norm` renames the norm key (clarification: reference title → meaningful name). Only + safe while no subblocks/facts are attached to the old `title_norm` yet.""" fields = {"status": status, "updated_at": _now()} - if titel is not None: - fields["titel"] = titel - if beschreibung is not None: - fields["beschreibung"] = beschreibung - if neu_norm is not None and neu_norm != titel_norm: - fields["titel_norm"] = neu_norm - sets = ", ".join(f"{k} = :{k}" for k in fields) + if title is not None: + fields["title"] = title + if description is not None: + fields["description"] = description + if neu_norm is not None and neu_norm != title_norm: + fields["title_norm"] = neu_norm + await _update("blocks", fields, {"topic": topic, "title_norm": title_norm}) + + +async def delete_blocks(topic: str) -> None: + db = await get_db() + await db.execute("DELETE FROM blocks WHERE topic = ?", (topic,)) + await db.commit() + + +async def upsert_subblock(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str) -> None: + db = await get_db() await db.execute( - f"UPDATE bausteine SET {sets} WHERE topic = :topic AND titel_norm = :titel_norm", - {**fields, "topic": topic, "titel_norm": titel_norm}, + """INSERT INTO subblocks (topic, block_norm, sub_norm, block, sub_title, mentions, status, updated_at) + VALUES (?, ?, ?, ?, ?, 1, 'candidate', ?) + ON CONFLICT(topic, block_norm, sub_norm) DO UPDATE SET + mentions = mentions + 1, updated_at = excluded.updated_at""", + (topic, block_norm, sub_norm, block, sub_title, _now()), ) await db.commit() -async def delete_bausteine(topic: str) -> None: - db = await get_db() - await db.execute("DELETE FROM bausteine WHERE topic = ?", (topic,)) - await db.commit() - - -async def upsert_subbaustein(topic: str, baustein_norm: str, sub_norm: str, baustein: str, sub_titel: str) -> None: +async def put_subblock(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str, + level: str | None = None, relevance: str | None = None, + facts: str | None = None, status: str = "consensus") -> None: + """Insert/update WITHOUT a mention counter (mirror from the sidecar). Overwrite + level/relevance/facts only when a new value is passed (COALESCE protects existing data).""" db = await get_db() await db.execute( - """INSERT INTO subbausteine (topic, baustein_norm, sub_norm, baustein, sub_titel, nennungen, status, updated_at) - VALUES (?, ?, ?, ?, ?, 1, 'kandidat', ?) - ON CONFLICT(topic, baustein_norm, sub_norm) DO UPDATE SET - nennungen = nennungen + 1, updated_at = excluded.updated_at""", - (topic, baustein_norm, sub_norm, baustein, sub_titel, _now()), - ) - await db.commit() - - -async def put_subbaustein(topic: str, baustein_norm: str, sub_norm: str, baustein: str, sub_titel: str, - stufe: str | None = None, relevanz: str | None = None, - fakten: str | None = None, status: str = "konsens") -> None: - """Insert/Update OHNE Nennungszähler (Spiegel aus dem Sidecar). stufe/relevanz/fakten nur - überschreiben, wenn ein neuer Wert übergeben wird (COALESCE schützt Bestehendes).""" - db = await get_db() - await db.execute( - """INSERT INTO subbausteine (topic, baustein_norm, sub_norm, baustein, sub_titel, nennungen, stufe, relevanz, fakten, status, updated_at) + """INSERT INTO subblocks (topic, block_norm, sub_norm, block, sub_title, mentions, level, relevance, facts, status, updated_at) VALUES (?, ?, ?, ?, ?, 1, ?, ?, COALESCE(?, ''), ?, ?) - ON CONFLICT(topic, baustein_norm, sub_norm) DO UPDATE SET - baustein = excluded.baustein, sub_titel = excluded.sub_titel, - stufe = COALESCE(excluded.stufe, subbausteine.stufe), - relevanz = COALESCE(excluded.relevanz, subbausteine.relevanz), - fakten = COALESCE(NULLIF(excluded.fakten, ''), subbausteine.fakten), + ON CONFLICT(topic, block_norm, sub_norm) DO UPDATE SET + block = excluded.block, sub_title = excluded.sub_title, + level = COALESCE(excluded.level, subblocks.level), + relevance = COALESCE(excluded.relevance, subblocks.relevance), + facts = COALESCE(NULLIF(excluded.facts, ''), subblocks.facts), status = excluded.status, updated_at = excluded.updated_at""", - (topic, baustein_norm, sub_norm, baustein, sub_titel, stufe, relevanz, fakten, status, _now()), + (topic, block_norm, sub_norm, block, sub_title, level, relevance, facts, status, _now()), ) await db.commit() -async def list_subbausteine(topic: str, baustein_norm: str | None = None) -> list[dict]: +async def list_subblocks(topic: str, block_norm: str | None = None) -> list[dict]: db = await get_db() - if baustein_norm is None: - cursor = await db.execute("SELECT * FROM subbausteine WHERE topic = ? ORDER BY rowid", (topic,)) + if block_norm is None: + cursor = await db.execute("SELECT * FROM subblocks WHERE topic = ? ORDER BY rowid", (topic,)) else: cursor = await db.execute( - "SELECT * FROM subbausteine WHERE topic = ? AND baustein_norm = ? ORDER BY rowid", (topic, baustein_norm) + "SELECT * FROM subblocks WHERE topic = ? AND block_norm = ? ORDER BY rowid", (topic, block_norm) ) rows = await cursor.fetchall() return [_row_to_dict(row, cursor) for row in rows] -async def set_subbaustein_felder(topic: str, baustein_norm: str, sub_norm: str, **fields) -> None: - """Setzt Felder (stufe/relevanz/status/sub_titel) einer Subbaustein-Zeile.""" +async def set_subblock_fields(topic: str, block_norm: str, sub_norm: str, **fields) -> None: + """Set fields (level/relevance/status/sub_title) of a subblock row.""" fields["updated_at"] = _now() - sets = ", ".join(f"{k} = :{k}" for k in fields) + await _update("subblocks", fields, {"topic": topic, "block_norm": block_norm, "sub_norm": sub_norm}) + + +async def delete_subblocks(topic: str) -> None: db = await get_db() - await db.execute( - f"UPDATE subbausteine SET {sets} WHERE topic = :topic AND baustein_norm = :baustein_norm AND sub_norm = :sub_norm", - {**fields, "topic": topic, "baustein_norm": baustein_norm, "sub_norm": sub_norm}, - ) + await db.execute("DELETE FROM subblocks WHERE topic = ?", (topic,)) await db.commit() -async def delete_subbausteine(topic: str) -> None: - db = await get_db() - await db.execute("DELETE FROM subbausteine WHERE topic = ?", (topic,)) - await db.commit() - - -async def upsert_frage_muster(topic: str, baustein_norm: str, sub_norm: str, baustein: str, sub_titel: str, frage: str) -> None: +async def upsert_question_pattern(topic: str, block_norm: str, sub_norm: str, block: str, sub_title: str, question: str) -> None: db = await get_db() await db.execute( - """INSERT INTO frage_muster (topic, baustein_norm, sub_norm, baustein, sub_titel, frage, updated_at) + """INSERT INTO question_pattern (topic, block_norm, sub_norm, block, sub_title, question, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(topic, baustein_norm, sub_norm) DO UPDATE SET - sub_titel = excluded.sub_titel, frage = excluded.frage, updated_at = excluded.updated_at""", - (topic, baustein_norm, sub_norm, baustein, sub_titel, frage, _now()), + ON CONFLICT(topic, block_norm, sub_norm) DO UPDATE SET + sub_title = excluded.sub_title, question = excluded.question, updated_at = excluded.updated_at""", + (topic, block_norm, sub_norm, block, sub_title, question, _now()), ) await db.commit() -async def list_frage_muster(topic: str, baustein_norm: str | None = None) -> list[dict]: +async def list_question_pattern(topic: str, block_norm: str | None = None) -> list[dict]: db = await get_db() - if baustein_norm is None: - cursor = await db.execute("SELECT * FROM frage_muster WHERE topic = ? ORDER BY rowid", (topic,)) + if block_norm is None: + cursor = await db.execute("SELECT * FROM question_pattern WHERE topic = ? ORDER BY rowid", (topic,)) else: cursor = await db.execute( - "SELECT * FROM frage_muster WHERE topic = ? AND baustein_norm = ? ORDER BY rowid", (topic, baustein_norm) + "SELECT * FROM question_pattern WHERE topic = ? AND block_norm = ? ORDER BY rowid", (topic, block_norm) ) rows = await cursor.fetchall() return [_row_to_dict(row, cursor) for row in rows] -async def delete_frage_muster(topic: str) -> None: +async def delete_question_pattern(topic: str) -> None: db = await get_db() - await db.execute("DELETE FROM frage_muster WHERE topic = ?", (topic,)) + await db.execute("DELETE FROM question_pattern WHERE topic = ?", (topic,)) await db.commit() -async def mark_quellen_gelesen(topic: str, quellen: list[str]) -> None: - """Markiert die zitierten Crawl-Seiten als gelesen (Recherche-Loop-Abdeckung).""" - if not quellen: +async def mark_sources_read_done(topic: str, sources: list[str]) -> None: + """Mark the cited crawl pages as read_done (research-loop coverage).""" + if not sources: return db = await get_db() now = _now() await db.executemany( - """INSERT INTO recherche_coverage (topic, quelle, gelesen, updated_at) VALUES (?, ?, 1, ?) - ON CONFLICT(topic, quelle) DO UPDATE SET gelesen = 1, updated_at = excluded.updated_at""", - [(topic, q, now) for q in quellen], + """INSERT INTO research_coverage (topic, source, read_done, updated_at) VALUES (?, ?, 1, ?) + ON CONFLICT(topic, source) DO UPDATE SET read_done = 1, updated_at = excluded.updated_at""", + [(topic, q, now) for q in sources], ) await db.commit() async def list_coverage(topic: str) -> dict[str, int]: db = await get_db() - cursor = await db.execute("SELECT quelle, gelesen FROM recherche_coverage WHERE topic = ?", (topic,)) + cursor = await db.execute("SELECT source, read_done FROM research_coverage WHERE topic = ?", (topic,)) rows = await cursor.fetchall() return {q: g for q, g in rows} -async def mark_inhalt(topic: str, content: list[str], noise: list[str]) -> None: - """Sichtungs-Ergebnis je Crawl-Seite ablegen: inhalt=1 (Content) bzw. 0 (Noise).""" +async def mark_content(topic: str, content: list[str], noise: list[str]) -> None: + """Store the triage result per crawl page: content=1 (content) or 0 (noise).""" db = await get_db() now = _now() rows = [(topic, q, 1, now) for q in content] + [(topic, q, 0, now) for q in noise] if not rows: return await db.executemany( - """INSERT INTO recherche_coverage (topic, quelle, inhalt, updated_at) VALUES (?, ?, ?, ?) - ON CONFLICT(topic, quelle) DO UPDATE SET inhalt = excluded.inhalt, updated_at = excluded.updated_at""", + """INSERT INTO research_coverage (topic, source, content, updated_at) VALUES (?, ?, ?, ?) + ON CONFLICT(topic, source) DO UPDATE SET content = excluded.content, updated_at = excluded.updated_at""", rows, ) await db.commit() async def list_content(topic: str) -> list[str]: - """Crawl-Seiten, die die Sichtung als Content markiert hat (inhalt=1).""" + """Crawl pages the triage marked as content (content=1).""" db = await get_db() cursor = await db.execute( - "SELECT quelle FROM recherche_coverage WHERE topic = ? AND inhalt = 1 ORDER BY quelle", (topic,) + "SELECT source FROM research_coverage WHERE topic = ? AND content = 1 ORDER BY source", (topic,) ) return [r[0] for r in await cursor.fetchall()] async def delete_coverage(topic: str) -> None: db = await get_db() - await db.execute("DELETE FROM recherche_coverage WHERE topic = ?", (topic,)) + await db.execute("DELETE FROM research_coverage WHERE topic = ?", (topic,)) await db.commit() -async def set_step_status(topic: str, schritt: str, status: str) -> None: +async def set_step_status(topic: str, step: str, status: str) -> None: db = await get_db() await db.execute( - """INSERT INTO pipeline_state (topic, schritt, status, updated_at) VALUES (?, ?, ?, ?) - ON CONFLICT(topic, schritt) DO UPDATE SET status = excluded.status, updated_at = excluded.updated_at""", - (topic, schritt, status, _now()), + """INSERT INTO pipeline_state (topic, step, status, updated_at) VALUES (?, ?, ?, ?) + ON CONFLICT(topic, step) DO UPDATE SET status = excluded.status, updated_at = excluded.updated_at""", + (topic, step, status, _now()), ) await db.commit() -async def get_step_status(topic: str, schritt: str) -> str: +async def get_step_status(topic: str, step: str) -> str: db = await get_db() cursor = await db.execute( - "SELECT status FROM pipeline_state WHERE topic = ? AND schritt = ?", (topic, schritt) + "SELECT status FROM pipeline_state WHERE topic = ? AND step = ?", (topic, step) ) row = await cursor.fetchone() - return row[0] if row else "offen" + return row[0] if row else "open" -async def delete_pipeline_state(topic: str, schritte: list[str] | None = None) -> None: +async def delete_pipeline_state(topic: str, steps: list[str] | None = None) -> None: db = await get_db() - if schritte is None: + if steps is None: await db.execute("DELETE FROM pipeline_state WHERE topic = ?", (topic,)) - elif schritte: - marks = ",".join("?" for _ in schritte) - await db.execute(f"DELETE FROM pipeline_state WHERE topic = ? AND schritt IN ({marks})", (topic, *schritte)) + elif steps: + marks = ",".join("?" for _ in steps) + await db.execute(f"DELETE FROM pipeline_state WHERE topic = ? AND step IN ({marks})", (topic, *steps)) await db.commit() @@ -887,14 +878,14 @@ async def delete_pipeline_state(topic: str, schritte: list[str] | None = None) - -async def delete_quelle(topic: str) -> None: +async def delete_source(topic: str) -> None: db = await get_db() - await db.execute("DELETE FROM quelle WHERE topic = ?", (topic,)) + await db.execute("DELETE FROM source WHERE topic = ?", (topic,)) await db.commit() async def set_guide_content(topic: str, format: str, content_json: str) -> None: - """Fertigen Guide-Inhalt (JSON-Blob) je Thema+Format speichern.""" + """Store finished guide content (JSON blob) per topic+format.""" db = await get_db() await db.execute( """INSERT INTO guide_content (topic, format, json, updated_at) VALUES (?, ?, ?, ?) @@ -920,52 +911,52 @@ async def delete_guide_content(topic: str, format: str | None = None) -> None: await db.commit() -async def set_gliederung(topic: str, gliederung_json: str) -> None: - """Gliederung (Kapitel→Nummern, JSON) je Thema speichern — Bausteine-Artefakt für den Guide.""" +async def set_outline(topic: str, outline_json: str) -> None: + """Store the outline (chapter→numbers, JSON) per topic — blocks artifact for the guide.""" db = await get_db() await db.execute( - """INSERT INTO guide_gliederung (topic, json, updated_at) VALUES (?, ?, ?) + """INSERT INTO guide_outline (topic, json, updated_at) VALUES (?, ?, ?) ON CONFLICT(topic) DO UPDATE SET json = excluded.json, updated_at = excluded.updated_at""", - (topic, gliederung_json, _now()), + (topic, outline_json, _now()), ) await db.commit() -async def get_gliederung(topic: str) -> str | None: +async def get_outline(topic: str) -> str | None: db = await get_db() - cursor = await db.execute("SELECT json FROM guide_gliederung WHERE topic = ?", (topic,)) + cursor = await db.execute("SELECT json FROM guide_outline WHERE topic = ?", (topic,)) row = await cursor.fetchone() return row[0] if row else None -async def delete_gliederung(topic: str) -> None: +async def delete_outline(topic: str) -> None: db = await get_db() - await db.execute("DELETE FROM guide_gliederung WHERE topic = ?", (topic,)) + await db.execute("DELETE FROM guide_outline WHERE topic = ?", (topic,)) await db.commit() -async def put_sub_artefakt(topic: str, baustein_norm: str, sub_norm: str, typ: str, - daten: str, baustein: str = "", sub_titel: str = "") -> None: - """Ein Lern-Artefakt (karteikarte/beispiel) als JSON in `daten` speichern.""" +async def put_sub_artifact(topic: str, block_norm: str, sub_norm: str, type: str, + data: str, block: str = "", sub_title: str = "") -> None: + """Store one learning artifact (flashcard/example) as JSON in `data`.""" db = await get_db() await db.execute( - """INSERT INTO sub_artefakte (topic, baustein_norm, sub_norm, typ, baustein, sub_titel, daten, updated_at) + """INSERT INTO sub_artefakte (topic, block_norm, sub_norm, type, block, sub_title, data, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(topic, baustein_norm, sub_norm, typ) DO UPDATE SET - baustein = excluded.baustein, sub_titel = excluded.sub_titel, - daten = excluded.daten, updated_at = excluded.updated_at""", - (topic, baustein_norm, sub_norm, typ, baustein, sub_titel, daten, _now()), + ON CONFLICT(topic, block_norm, sub_norm, type) DO UPDATE SET + block = excluded.block, sub_title = excluded.sub_title, + data = excluded.data, updated_at = excluded.updated_at""", + (topic, block_norm, sub_norm, type, block, sub_title, data, _now()), ) await db.commit() -async def get_sub_artefakte(topic: str, typ: str | None = None) -> list[dict]: +async def get_sub_artefakte(topic: str, type: str | None = None) -> list[dict]: db = await get_db() - if typ is None: + if type is None: cursor = await db.execute("SELECT * FROM sub_artefakte WHERE topic = ? ORDER BY rowid", (topic,)) else: cursor = await db.execute( - "SELECT * FROM sub_artefakte WHERE topic = ? AND typ = ? ORDER BY rowid", (topic, typ) + "SELECT * FROM sub_artefakte WHERE topic = ? AND type = ? ORDER BY rowid", (topic, type) ) rows = await cursor.fetchall() return [_row_to_dict(row, cursor) for row in rows] @@ -977,33 +968,33 @@ async def delete_sub_artefakte(topic: str) -> None: await db.commit() -async def get_baustein_huerden(topic: str, baustein_norm: str) -> list[str]: - """Typische Irrtümer (huerden) aller Subs eines Bausteins — als Distraktor-Pool fürs Quiz. - Liest aus subbausteine.fakten (JSON-Blob); leere/fehlende werden übersprungen.""" +async def get_block_hurdles(topic: str, block_norm: str) -> list[str]: + """Typical misconceptions (hurdles) of all subs of a block — as a distractor pool for the quiz. + Reads from subblocks.facts (JSON blob); empty/missing ones are skipped.""" db = await get_db() cursor = await db.execute( - "SELECT fakten FROM subbausteine WHERE topic = ? AND baustein_norm = ?", (topic, baustein_norm) + "SELECT facts FROM subblocks WHERE topic = ? AND block_norm = ?", (topic, block_norm) ) rows = await cursor.fetchall() - huerden = [] - for (fakten,) in rows: - if not fakten: + hurdles = [] + for (facts,) in rows: + if not facts: continue try: - fk = json.loads(fakten) + fk = json.loads(facts) except (ValueError, TypeError): continue - h = (fk.get("huerden") or "").strip() if isinstance(fk, dict) else "" + h = (fk.get("hurdles") or "").strip() if isinstance(fk, dict) else "" if h: - huerden.append(h) - return huerden + hurdles.append(h) + return hurdles async def delete_topic_pipeline(topic: str) -> None: - """Bausteine-Bereich eines Themas verwerfen (Inventar/Subs/Muster/Coverage/State/Artefakte). - NICHT die Themen-Config `quelle` — die wird separat verwaltet (delete_quelle).""" + """Discard the blocks area of a topic (inventory/subs/pattern/coverage/state/artifacts). + NOT the topic config `source` — that is managed separately (delete_source).""" db = await get_db() - for tab in ("bausteine", "subbausteine", "frage_muster", "recherche_coverage", - "pipeline_state", "guide_gliederung", "sub_artefakte"): + for tab in ("blocks", "subblocks", "question_pattern", "research_coverage", + "pipeline_state", "guide_outline", "sub_artefakte"): await db.execute(f"DELETE FROM {tab} WHERE topic = ?", (topic,)) await db.commit() diff --git a/backend/elements.py b/backend/elements.py index d10a496..4be2f01 100644 --- a/backend/elements.py +++ b/backend/elements.py @@ -1,4 +1,4 @@ -"""Elemente (persönliche Zusammenfassung) und Tutor-Chat zum Guide.""" +"""Elements (personal summary) and tutor chat for the guide.""" import json import logging @@ -6,25 +6,25 @@ import uuid from agents import run_agent from config import DEFAULT_PROVIDER -from jsonio import parse_json_text as _parse_json_text, read_json_file as _json_datei -from paths import bausteine_path, guide_content_path +from jsonio import parse_json_text as _parse_json_text, read_json_file as _read_json_file +from paths import blocks_path, guide_content_path from pipeline import _prompt log = logging.getLogger("creator.elements") -# --- Tutor-Chat --- +# --- Tutor chat --- def _build_guide_chat_prompt(topic: str, format_name: str, section: str, outline: str, messages: list[dict]) -> str: transcript = "\n".join( - f"{'Nutzer' if m.get('role') == 'user' else 'Assistent'}: {m.get('content', '')}" + f"{'User' if m.get('role') == 'user' else 'Assistant'}: {m.get('content', '')}" for m in messages ) return _prompt( "Chat", topic=topic, format_name=format_name, - outline_block=outline.strip() or "(keine)", - section_block=section.strip() or "(kein Abschnitt erkannt)", + outline_block=outline.strip() or "(none)", + section_block=section.strip() or "(no section detected)", transcript=transcript, ) @@ -36,62 +36,62 @@ async def chat_with_guide(topic: str, format_name: str, section: str, outline: s "chat-" + str(uuid.uuid4()), prompt, 240, provider=provider, role="fast", capabilities="none", lane="interactive" ) if returncode != 0: - return "Entschuldigung, das hat nicht geklappt. Bitte versuche es erneut." + return "Sorry, that didn't work. Please try again." reply = stdout.strip() - return reply or "Entschuldigung, ich habe keine Antwort erhalten." + return reply or "Sorry, I didn't get a response." except Exception: - log.warning("[%s] Guide-Chat fehlgeschlagen", topic, exc_info=True) - return "Entschuldigung, das hat nicht geklappt. Bitte versuche es erneut." + log.warning("[%s] Guide chat failed", topic, exc_info=True) + return "Sorry, that didn't work. Please try again." -# --- Elemente --- +# --- Elements --- def _element_fields(data: dict) -> dict | None: - """Validiert KI-Element-JSON und normalisiert auf die DB-Felder.""" + """Validate AI element JSON and normalize it onto the DB fields.""" if not isinstance(data, dict): return None title = str(data.get("title", "")).strip() if not title: return None - listen = {} + lists = {} for key in ("examples", "hints"): raw = data.get(key, []) - listen[key] = [str(e).strip() for e in raw if str(e).strip()] if isinstance(raw, list) else [] + lists[key] = [str(e).strip() for e in raw if str(e).strip()] if isinstance(raw, list) else [] return { "title": title[:200], "description": str(data.get("description", "")).strip(), - "examples": listen["examples"], - "hints": listen["hints"], + "examples": lists["examples"], + "hints": lists["hints"], } def _topic_context(topic: str, limit: int = 12000) -> str: - """Bausteine + Guide-Inhalte des Themas als Kontext-Text (gekürzt).""" + """Blocks + guide content of the topic as context text (truncated).""" parts: list[str] = [] - bp = bausteine_path(topic) + bp = blocks_path(topic) if bp.exists(): parts.append(bp.read_text(encoding="utf-8")) - for fmt in ("Guide", "FullGuide"): # bester verfügbarer Prosa-Guide als Chat-Kontext - content = _json_datei(guide_content_path(topic, fmt)) + for fmt in ("Guide", "FullGuide"): # best available prose guide as chat context + content = _read_json_file(guide_content_path(topic, fmt)) if content: for ch in content.get("chapters", []): for sec in ch.get("sections", []): parts.append(sec if isinstance(sec, str) else json.dumps(sec, ensure_ascii=False)) - break # bester verfügbarer Guide reicht + break # the best available guide is enough text = "\n\n".join(parts).strip() - return text[:limit] if text else "(kein Material vorhanden)" + return text[:limit] if text else "(no material available)" async def generate_element(topic: str, hint: str, provider: str = DEFAULT_PROVIDER, extra_context: str = "") -> dict: - """Erstellt Element-Felder per KI. Fallback: nur Titel aus dem Stichwort.""" - fallback = {"title": hint.strip() or "Neues Element", "description": "", "examples": [], "hints": []} + """Create element fields via AI. Fallback: only the title from the keyword.""" + fallback = {"title": hint.strip() or "New element", "description": "", "examples": [], "hints": []} try: context = _topic_context(topic) if extra_context.strip(): context = (extra_context.strip() + "\n\n" + context)[:12000] prompt = _prompt( "Element-Create", - topic=topic, hint=hint.strip() or "(keins — wähle selbst ein Kernkonzept)", + topic=topic, hint=hint.strip() or "(none — pick a core concept yourself)", context=context, ) returncode, stdout, _ = await run_agent( @@ -101,12 +101,12 @@ async def generate_element(topic: str, hint: str, provider: str = DEFAULT_PROVID return fallback return _element_fields(_parse_json_text(stdout)) or fallback except Exception: - log.warning("[%s] Element-Erstellung fehlgeschlagen", topic, exc_info=True) + log.warning("[%s] Element creation failed", topic, exc_info=True) return fallback def _parse_suggestions(stdout: str) -> list[dict] | None: - """Validiert Vorschlags-JSON aus KI-Output. None bei ungültigem JSON.""" + """Validate suggestion JSON from AI output. None on invalid JSON.""" data = _parse_json_text(stdout) if not isinstance(data, dict): return None @@ -123,7 +123,7 @@ def _parse_suggestions(stdout: str) -> list[dict] | None: async def check_element(element: dict, provider: str = DEFAULT_PROVIDER) -> list[dict] | None: - """Zweischrittige Prüfung auf fehlende Infos: Recherche → Verifizieren. None bei Fehler.""" + """Two-step check for missing info: research → verify. None on error.""" try: element_json = json.dumps( {k: element[k] for k in ("title", "description", "examples", "hints")}, @@ -131,7 +131,7 @@ async def check_element(element: dict, provider: str = DEFAULT_PROVIDER) -> list ) context = _topic_context(element["topic"]) - # Schritt 1: Recherche — breit Kandidaten sammeln + # Step 1: research — collect candidates broadly prompt = _prompt("Element-Check", topic=element["topic"], element_json=element_json, context=context) returncode, stdout, _ = await run_agent( "element-check-" + str(uuid.uuid4()), prompt, 240, provider=provider, role="fast", capabilities="none", lane="interactive" @@ -144,7 +144,7 @@ async def check_element(element: dict, provider: str = DEFAULT_PROVIDER) -> list if not candidates: return [] - # Schritt 2: Verifizieren — nur Wichtiges, nicht Redundantes durchlassen + # Step 2: verify — only let important, non-redundant items through prompt = _prompt( "Element-Verify", topic=element["topic"], element_json=element_json, @@ -158,7 +158,7 @@ async def check_element(element: dict, provider: str = DEFAULT_PROVIDER) -> list return None return _parse_suggestions(stdout) except Exception: - log.warning("[%s] Element-Prüfung fehlgeschlagen", element.get("topic", "?"), exc_info=True) + log.warning("[%s] Element check failed", element.get("topic", "?"), exc_info=True) return None @@ -170,7 +170,7 @@ def _element_json(element: dict) -> str: def _validate_change(c, element: dict) -> dict | None: - """Validiert einen Änderungs-Vorschlag aus KI-Output gegen das Element.""" + """Validate a change suggestion from AI output against the element.""" if not isinstance(c, dict): return None text = str(c.get("text", "")).strip() @@ -178,16 +178,16 @@ def _validate_change(c, element: dict) -> dict | None: target = c.get("target") index = c.get("index") content = str(c.get("content", "")).strip() - if not text or action not in ("entfernen", "anpassen", "hinzufuegen"): + if not text or action not in ("remove", "adjust", "add"): return None if target not in ("title", "description", "examples", "hints"): return None - if action in ("anpassen", "hinzufuegen") and not content: + if action in ("adjust", "add") and not content: return None - if action == "entfernen" and target not in ("examples", "hints"): + if action == "remove" and target not in ("examples", "hints"): return None - # Index nur für anpassen/entfernen in Listen-Feldern; muss existieren - if target in ("examples", "hints") and action in ("anpassen", "entfernen"): + # Index only for adjust/remove on list fields; must exist + if target in ("examples", "hints") and action in ("adjust", "remove"): if not isinstance(index, int) or not (0 <= index < len(element[target])): return None else: @@ -196,11 +196,11 @@ def _validate_change(c, element: dict) -> dict | None: async def chat_with_element(element: dict, messages: list[dict], provider: str = DEFAULT_PROVIDER) -> tuple[str, list[dict]]: - """Chat zum Element. Gibt (Antwort, Änderungs-Vorschläge) zurück — ändert nichts direkt.""" - fehler = "Entschuldigung, das hat nicht geklappt. Bitte versuche es erneut." + """Chat about the element. Returns (reply, change suggestions) — changes nothing directly.""" + error = "Sorry, that didn't work. Please try again." try: transcript = "\n".join( - f"{'Nutzer' if m.get('role') == 'user' else 'Assistent'}: {m.get('content', '')}" + f"{'User' if m.get('role') == 'user' else 'Assistant'}: {m.get('content', '')}" for m in messages ) prompt = _prompt("Element-Chat", topic=element["topic"], element_json=_element_json(element), transcript=transcript) @@ -208,22 +208,22 @@ async def chat_with_element(element: dict, messages: list[dict], provider: str = "element-chat-" + str(uuid.uuid4()), prompt, 240, provider=provider, role="fast", capabilities="none", lane="interactive" ) if returncode != 0: - return fehler, [] + return error, [] data = _parse_json_text(stdout) if not isinstance(data, dict): - return fehler, [] + return error, [] changes = [v for c in data.get("changes", []) if (v := _validate_change(c, element))] - reply = str(data.get("reply", "")).strip() or ("Vorschläge erstellt." if changes else fehler) + reply = str(data.get("reply", "")).strip() or ("Suggestions created." if changes else error) return reply, changes except Exception: - log.warning("[%s] Element-Chat fehlgeschlagen", element.get("topic", "?"), exc_info=True) - return fehler, [] + log.warning("[%s] Element chat failed", element.get("topic", "?"), exc_info=True) + return error, [] async def style_element(element: dict, provider: str = DEFAULT_PROVIDER) -> list[dict] | None: - """Prüft ein Element auf die Stil-Regeln und schlägt Änderungen vor. None bei Fehler.""" + """Check an element against the style rules and suggest changes. None on error.""" try: - prompt = _prompt("Element-Stil", topic=element["topic"], element_json=_element_json(element)) + prompt = _prompt("Element-Style", topic=element["topic"], element_json=_element_json(element)) returncode, stdout, _ = await run_agent( "element-stil-" + str(uuid.uuid4()), prompt, 240, provider=provider, role="fast", capabilities="none", lane="interactive" ) @@ -234,12 +234,12 @@ async def style_element(element: dict, provider: str = DEFAULT_PROVIDER) -> list return None return [v for c in data.get("changes", []) if (v := _validate_change(c, element))] except Exception: - log.warning("[%s] Stil-Prüfung fehlgeschlagen", element.get("topic", "?"), exc_info=True) + log.warning("[%s] Style check failed", element.get("topic", "?"), exc_info=True) return None async def refine_suggestion(element: dict, suggestion: dict, instruction: str, provider: str = DEFAULT_PROVIDER) -> dict | None: - """Überarbeitet einen einzelnen Vorschlag nach Nutzer-Anweisung. None bei Fehler.""" + """Revise a single suggestion per user instruction. None on error.""" try: prompt = _prompt( "Element-Refine", @@ -257,5 +257,5 @@ async def refine_suggestion(element: dict, suggestion: dict, instruction: str, p return None return _validate_change(data.get("change"), element) except Exception: - log.warning("[%s] Vorschlags-Überarbeitung fehlgeschlagen", element.get("topic", "?"), exc_info=True) + log.warning("[%s] Suggestion revision failed", element.get("topic", "?"), exc_info=True) return None diff --git a/backend/embedding.py b/backend/embedding.py index 0e8bffb..3b8b6f9 100644 --- a/backend/embedding.py +++ b/backend/embedding.py @@ -1,14 +1,14 @@ -"""Semantisches Embedding-Clustering für die Baustein-Konsolidierung. +"""Semantic embedding clustering for block consolidation. -Mean-Pool-Embeddings eines mehrsprachigen Satz-Modells bilden über Cosine-Blocking + -Union-Find GLOBALE Kandidaten-Cluster (kein Chunk-Verlust). Sichere Paare (Ähnlichkeit -≥ HART) werden ohne LLM gemergt; Grenz-Paare im Band [BAND_LOW, HART) gibt der Aufrufer -einem LLM-Judge zur ja/nein-Entscheidung. Fehlen `transformers`/`torch` oder lädt das -Modell nicht → `embed_sims()` liefert `None`, der Aufrufer fällt auf den alten -Panel-Judge-Pfad zurück (silente Deaktivierung, wie das Lesbarkeits-Gate). +Mean-pool embeddings of a multilingual sentence model build GLOBAL candidate +clusters via cosine blocking + union-find (no chunk loss). Safe pairs (similarity +≥ HARD) are merged without an LLM; borderline pairs in the band [BAND_LOW, HARD) are +handed by the caller to an LLM judge for a yes/no decision. If `transformers`/`torch` +are missing or the model won't load → `embed_sims()` returns `None`, and the caller +falls back to the old panel-judge path (silent deactivation, like the readability gate). -CPU genügt; der Aufrufer wrappt die blockierende Inferenz in `asyncio.to_thread`. -`numpy` ist transitiv über torch vorhanden (bewusst nicht in requirements.txt, analog torch). +CPU is enough; the caller wraps the blocking inference in `asyncio.to_thread`. +`numpy` comes in transitively via torch (deliberately not in requirements.txt, like torch). """ import logging @@ -19,19 +19,19 @@ from config import EMBEDDING_AKTIV, EMBEDDING_MODELL, EMBEDDING_BLOCK_FLOOR, EMB log = logging.getLogger("creator.embedding") -_modell_cache = None # (tokenizer, model, torch) — Singleton -_ladeversuch = False # schon versucht zu laden? +_model_cache = None # (tokenizer, model, torch) — singleton +_load_attempt = False # already tried to load? -EMBEDDING_BATCH = 32 # Inferenz-Batchgröße (CPU) -EMBEDDING_MAX_LEN = 128 # Titel + Kurzbeschreibung sind kurz → kleiner Truncation-Cap genügt +EMBEDDING_BATCH = 32 # inference batch size (CPU) +EMBEDDING_MAX_LEN = 128 # title + short description are short → a small truncation cap suffices -def _modell(): - """Lädt das Modell einmalig. None = Clustering aus (deaktiviert oder Lade-Fehler).""" - global _modell_cache, _ladeversuch - if _ladeversuch: - return _modell_cache - _ladeversuch = True +def _model(): + """Load the model once. None = clustering off (disabled or load error).""" + global _model_cache, _load_attempt + if _load_attempt: + return _model_cache + _load_attempt = True if not EMBEDDING_AKTIV: return None try: @@ -40,24 +40,24 @@ def _modell(): tok = AutoTokenizer.from_pretrained(EMBEDDING_MODELL) model = AutoModel.from_pretrained(EMBEDDING_MODELL) model.eval() - _modell_cache = (tok, model, torch) - log.info("Embedding-Modell geladen: %s", EMBEDDING_MODELL) + _model_cache = (tok, model, torch) + log.info("embedding model loaded: %s", EMBEDDING_MODELL) except Exception as e: - log.warning("Embedding-Clustering deaktiviert (Modell nicht ladbar): %s", e) - _modell_cache = None - return _modell_cache + log.warning("embedding clustering disabled (model not loadable): %s", e) + _model_cache = None + return _model_cache -def verfuegbar() -> bool: - """True, wenn das Modell geladen werden konnte. Lädt beim ersten Aufruf (blockierend).""" - return _modell() is not None +def available() -> bool: + """True if the model could be loaded. Loads on the first call (blocking).""" + return _model() is not None def embed(texts: list[str]) -> "np.ndarray | None": - """Texte → (n, d) L2-normalisierte, mean-gepoolte Embeddings. None = Modell aus.""" - if _modell() is None: + """Texts → (n, d) L2-normalized, mean-pooled embeddings. None = model off.""" + if _model() is None: return None - tok, model, torch = _modell_cache + tok, model, torch = _model_cache out = [] for i in range(0, len(texts), EMBEDDING_BATCH): batch = texts[i:i + EMBEDDING_BATCH] @@ -65,8 +65,8 @@ def embed(texts: list[str]) -> "np.ndarray | None": with torch.no_grad(): hidden = model(**enc).last_hidden_state # (b, t, d) mask = enc["attention_mask"].unsqueeze(-1).type_as(hidden) - vec = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1e-9) # mean-pool ohne Padding - vec = torch.nn.functional.normalize(vec, p=2, dim=1) # L2 → Cosine = Skalarprodukt + vec = (hidden * mask).sum(1) / mask.sum(1).clamp(min=1e-9) # mean-pool without padding + vec = torch.nn.functional.normalize(vec, p=2, dim=1) # L2 → cosine = dot product out.append(vec.cpu().numpy()) return np.vstack(out).astype(np.float32) @@ -81,24 +81,24 @@ def _find(parent: list[int], x: int) -> int: def _union(parent: list[int], a: int, b: int) -> None: ra, rb = _find(parent, a), _find(parent, b) if ra != rb: - parent[max(ra, rb)] = min(ra, rb) # kleinster Index = Wurzel (deterministisch) + parent[max(ra, rb)] = min(ra, rb) # smallest index = root (deterministic) def embed_sims(texts: list[str]): - """Texte → (n, n) Cosine-Matrix · None = Modell nicht verfügbar (Fallback).""" + """Texts → (n, n) cosine matrix · None = model not available (fallback).""" embs = embed(texts) if embs is None: return None - return embs @ embs.T # (n, n) Cosine, float32 (~2 MB bei n=700) + return embs @ embs.T # (n, n) cosine, float32 (~2 MB at n=700) def capped_blocks(sims, floor: float | None = None, cap: int | None = None) -> list[list[int]]: - """Grobe Ähnlichkeits-Blocks für den LLM — High-Recall, aber Größe gedeckelt. + """Coarse similarity blocks for the LLM — high recall, but size-capped. - Greedy: alle Paare mit Cosine ≥ `floor` nach Cosine absteigend; zwei Blocks werden nur - verschmolzen, wenn der resultierende Block ≤ `cap` bleibt. Verhindert den Giant-Component - (reines Threshold-Blocking verkettet sonst fast alles) und hält die LLM-Listen kurz. - → Liste von Blocks (Index-Listen), jeder Knoten in genau einem Block. + Greedy: all pairs with cosine ≥ `floor` in descending cosine order; two blocks are merged + only if the resulting block stays ≤ `cap`. Prevents the giant component (pure threshold + blocking would otherwise chain almost everything together) and keeps the LLM lists short. + → list of blocks (index lists), each node in exactly one block. """ fl = EMBEDDING_BLOCK_FLOOR if floor is None else floor cp = EMBEDDING_BLOCK_CAP if cap is None else cap @@ -109,7 +109,7 @@ def capped_blocks(sims, floor: float | None = None, cap: int | None = None) -> l iu = np.triu_indices(n, k=1) s = sims[iu] kept = np.where(s >= fl)[0] - # höchste Cosine zuerst → engste Paare bilden zuerst Blocks + # highest cosine first → the tightest pairs form blocks first for k in kept[np.argsort(-s[kept])]: i, j = int(iu[0][k]), int(iu[1][k]) ri, rj = _find(parent, i), _find(parent, j) diff --git a/backend/fsutil.py b/backend/fsutil.py index 9c99f0d..b649264 100644 --- a/backend/fsutil.py +++ b/backend/fsutil.py @@ -1,7 +1,7 @@ -"""Atomare Datei-Writes: erst .tmp im selben Verzeichnis, dann os.replace. +"""Atomic file writes: first a .tmp in the same directory, then os.replace. -Ein Crash hinterlässt höchstens eine .tmp-Datei — nie eine halb geschriebene -Zieldatei. Die .tmp wird beim nächsten erfolgreichen Write überschrieben. +A crash leaves at most a .tmp file behind — never a half-written target +file. The .tmp is overwritten on the next successful write. """ import json diff --git a/backend/guide.py b/backend/guide.py index 799b87a..f653512 100644 --- a/backend/guide.py +++ b/backend/guide.py @@ -1,11 +1,11 @@ -"""Guide-Generierung als Konsens-Pipeline. +"""Guide generation as a consensus pipeline. -Gliederung: Auswahl der Bausteine (deterministisch je Format) → 3 Vorschläge -(Grace), die Bausteine NUMMERN-basiert in Kapitel ordnen → ein Judge merged die -Vorschläge zu einer kohärenten Reihenfolge. -Schreiben: Writer je Baustein. Lese-Prüfung: Check→Fix (eine Runde), -Folgerunden prüfen nur ersetzte Sections; danach bleiben Beanstandungen stehen. -Schritt-Dateien bleiben liegen → Abbruch erhält Fortschritt, ▶ setzt am offenen Schritt fort. +Outline: select the blocks (deterministic per format) → 3 proposals +(Grace) that order the blocks into chapters by NUMBER → a judge merges the +proposals into one coherent order. +Writing: one writer per block. Reading exam: Check→Fix (one round), +follow-up rounds check only replaced sections; remaining complaints then stand. +Step files are kept → an abort preserves progress, ▶ resumes at the open step. """ import asyncio @@ -18,166 +18,166 @@ from pathlib import Path import uuid from agents import run_agent -from bausteine import _pdfs_konvertieren, quelle_ordner +from blocks import _convert_pdfs, source_folder from config import ( - DEFAULT_PROVIDER, FORMAT_ZWECK, KONSENS_GRACE, - LESBARKEIT_AKTIV, TEMPLATES_DIR, + DEFAULT_PROVIDER, FORMAT_PURPOSE, CONSENSUS_GRACE, + READABILITY_ACTIVE, TEMPLATES_DIR, ) -import lesbarkeit -from database import list_guides, update_guide, list_bausteine, list_subbausteine, set_guide_content, get_guide_content, get_gliederung +import readability +from database import list_guides, update_guide, list_blocks, list_subblocks, set_guide_content, get_guide_content, get_outline from fsutil import atomic_write_json, atomic_write_text -from jsonio import read_json_file as _json_datei, parse_json_text as _parse_json_text -from paths import bausteine_path, guide_content_path, project_dir, subbausteine_path +from jsonio import read_json_file as _json_file, parse_json_text as _parse_json_text +from paths import blocks_path, guide_content_path, project_dir, subblocks_path from pipeline import ( CANCELLED, FAILED, GenContext, _claude_error, _extra, - _fail, _gather_error, _gather_fortschritt, _log, _prompt, _race, + _fail, _gather_error, _gather_progress, _log, _prompt, _race, _semaphore, _set_progress, _set_step, _timeout, clear_guide_cancelled, is_guide_cancelled, run_single_slot, ) from textkit import ( - _eindeutige_titel, _lade_bausteine, _norm_titel, _parse_fragment, _split_chunks, - _titel, _titel_aufloesen, _titel_index, + _unique_title, _load_blocks, _norm_title, _parse_fragment, _split_chunks, + _title, _resolve_title, _title_index, ) log = logging.getLogger("creator.guide") -GUIDE_STEPS = ("Gliederung", "Inhalte", "Inhalts-Check", "Schreiben", "Lese-Prüfung") +GUIDE_STEPS = ("Outline", "Content", "Content-Check", "Writing", "Reading-Exam") -# Inhalte/Inhalts-Check/Lese-Prüfung laufen in Paketen von ~GUIDE_CHUNK Bausteinen je Agent. -# Nur der Writer (Schreiben) bleibt 1 Agent je Baustein (variable Längen, kein Kürzen, keine -# Längen-Angleichung zwischen Bausteinen). +# Content/Content-Check/Reading-Exam run in packets of ~GUIDE_CHUNK blocks per agent. +# Only the writer (Writing) stays at 1 agent per block (variable lengths, no trimming, no +# length alignment between blocks). GUIDE_CHUNK = 10 -# Prüf-Schritte als Panel: CHECK_PANEL Judges je Chunk, Section beanstandet bei Mehrheit. -# Ein einzelner Judge ist bias-/sampling-anfällig; ein kleines Panel ist stabiler. +# Check steps as a panel: CHECK_PANEL judges per chunk, section flagged on a majority. +# A single judge is bias/sampling prone; a small panel is more stable. CHECK_PANEL = 3 -# Lese-Prüfung: nur EINE Runde (Check + Fix). Folgerunden brachten kaum Mehrwert -# (1 Agent je Baustein prüft ohnehin fein), kosten aber extra Agenten. -LESE_RUNDEN = 1 +# Reading exam: only ONE round (Check + Fix). Follow-up rounds added little value +# (1 agent per block checks finely anyway) but cost extra agents. +READING_ROUNDS = 1 -# Gültige Stufen-Werte: neu (Lernpfad) + alt (Schwierigkeit) abwärtskompatibel. -_STUFEN_OK = ("anfaenger", "fortgeschritten", "experte", "einfach", "mittel", "schwer") +# Valid level values: new (learning path) + old (difficulty) backward-compatible. +_LEVELS_OK = ("beginner", "advanced", "expert", "easy", "medium", "hard") -async def _load_subbausteine(topic: str) -> dict[str, list[dict]]: - """Subbausteine je Baustein — DB-first ({titel, stufe, relevanz}), Fallback Sidecar-Datei. - Fehlt beides → {} (Guide nimmt alles).""" +async def _load_subblocks(topic: str) -> dict[str, list[dict]]: + """Subblocks per block — DB-first ({title, level, relevance}), fallback sidecar file. + Both missing → {} (guide takes everything).""" out: dict[str, list[dict]] = {} - for r in await list_subbausteine(topic): - if r["status"] == "konsens" and r["sub_titel"] and r["stufe"] in _STUFEN_OK: + for r in await list_subblocks(topic): + if r["status"] == "consensus" and r["sub_title"] and r["level"] in _LEVELS_OK: try: - fakten = json.loads(r["fakten"]) if r.get("fakten") else {} + facts = json.loads(r["facts"]) if r.get("facts") else {} except (ValueError, TypeError): - fakten = {} - out.setdefault(r["baustein"], []).append( - {"titel": r["sub_titel"], "stufe": r["stufe"], "relevanz": r["relevanz"], "fakten": fakten}) + facts = {} + out.setdefault(r["block"], []).append( + {"title": r["sub_title"], "level": r["level"], "relevance": r["relevance"], "facts": facts}) if out: return out - data = _json_datei(subbausteine_path(topic)) + data = _json_file(subblocks_path(topic)) if not isinstance(data, dict): return {} - for titel, subs in data.items(): + for title, subs in data.items(): if not isinstance(subs, list): continue - gut = [s for s in subs if isinstance(s, dict) and str(s.get("titel", "")).strip() - and s.get("stufe") in _STUFEN_OK] - if gut: - out[titel] = gut + good = [s for s in subs if isinstance(s, dict) and str(s.get("title", "")).strip() + and s.get("level") in _LEVELS_OK] + if good: + out[title] = good return out -def _ebene_label(s: dict) -> str: - """Ansichts-Ebene eines Subbausteins: rand → 'rand' (Ebene 4), sonst die Stufe (1–3).""" - return "rand" if s.get("relevanz") == "rand" else (s.get("stufe") or "anfaenger") +def _level_label(s: dict) -> str: + """View level of a subblock: peripheral → 'peripheral' (level 4), otherwise the level (1–3).""" + return "peripheral" if s.get("relevance") == "peripheral" else (s.get("level") or "beginner") -def _zuteilung_subs(chunk: list[dict], entries: dict[int, str], subs_by_titel: dict[str, list[dict]]) -> str: - """Listet je Kapitel die Bausteine, darunter ihre Subbausteine mit Ebenen-Label.""" +def _assignment_subs(chunk: list[dict], entries: dict[int, str], subs_by_title: dict[str, list[dict]]) -> str: + """Lists the blocks per chapter, with their subblocks and level labels beneath.""" lines: list[str] = [] for ch in chunk: - lines.append(f"KAPITEL: {ch['title']}") + lines.append(f"CHAPTER: {ch['title']}") for num in ch["nums"]: lines.append(f"- {entries[num]}") - for s in subs_by_titel.get(_titel(entries[num]), []): - lines.append(f" [{_ebene_label(s)}] {s['titel']}") + for s in subs_by_title.get(_title(entries[num]), []): + lines.append(f" [{_level_label(s)}] {s['title']}") return "\n".join(lines) def _guide_files(content_path: Path) -> dict: d, stem = content_path.parent, content_path.stem return { - "gliederung_slots": [d / f"{stem}.gliederung-{i}.json" for i in (1, 2, 3)], - "gliederung": d / f"{stem}.gliederung.json", # Judge-Ausgabe - # chunk-/lese-check-/fix-Dateien sind dynamisch: + "outline_slots": [d / f"{stem}.outline-{i}.json" for i in (1, 2, 3)], + "outline": d / f"{stem}.outline.json", # judge output + # chunk/reading-check/fix files are dynamic: # {stem}.chunk-i.md, {stem}.lese-check-r{n}-{i}.json, {stem}.fix-r{n}-{i}.md } -def guide_slot_dateien(content_path: Path) -> list[Path]: - """Alle Schritt-Dateien eines Guides (für den Frischstart).""" +def guide_slot_files(content_path: Path) -> list[Path]: + """All step files of a guide (for a fresh start).""" return [p for p in content_path.parent.glob(f"{content_path.stem}.*") if p != content_path] -def _fertig_path(content_path: Path) -> Path: - return content_path.parent / f"{content_path.stem}.fertig" +def _done_path(content_path: Path) -> Path: + return content_path.parent / f"{content_path.stem}.done" -def guide_fertig_step(content_path: Path) -> int: - """Höchster VOLL abgeschlossener Schritt-Index (Marker je Thema+Format). -1 = keiner. - Existiert die Content-Datei, sind alle Schritte fertig.""" +def guide_done_step(content_path: Path) -> int: + """Highest FULLY completed step index (marker per topic+format). -1 = none. + If the content file exists, all steps are done.""" if content_path.exists(): return len(GUIDE_STEPS) - 1 try: - return int(_fertig_path(content_path).read_text(encoding="utf-8").strip()) + return int(_done_path(content_path).read_text(encoding="utf-8").strip()) except (OSError, ValueError): return -1 -def _set_fertig(content_path: Path, step: int) -> None: - """Marker auf `step` setzen — monoton (nur erhöhen), außer beim Re-Run-Reset (force).""" - if step > guide_fertig_step(content_path): - atomic_write_text(_fertig_path(content_path), str(step)) +def _set_done(content_path: Path, step: int) -> None: + """Set marker to `step` — monotone (only increase), except on the re-run reset (force).""" + if step > guide_done_step(content_path): + atomic_write_text(_done_path(content_path), str(step)) -def _reset_fertig(content_path: Path, step: int) -> None: - """Marker hart auf `step` setzen (für Re-Run ab Schritt; step kann sinken).""" +def _reset_done(content_path: Path, step: int) -> None: + """Set marker hard to `step` (for re-run from step; step may decrease).""" if step < 0: - _fertig_path(content_path).unlink(missing_ok=True) + _done_path(content_path).unlink(missing_ok=True) else: - atomic_write_text(_fertig_path(content_path), str(step)) + atomic_write_text(_done_path(content_path), str(step)) -# Slot-Datei-Globs je Schritt (Index = GUIDE_STEPS). Stem-verankert, kollisionsfrei. +# Slot-file globs per step (index = GUIDE_STEPS). Stem-anchored, collision-free. _STEP_GLOBS = ( - ("gliederung*",), # 0 Gliederung (inkl. Auswahl-Filter) - ("inhalt-chunk-*", "inhalt-nach-*"), # 1 Inhalte (inkl. Nachrunde) - ("inhalt-check-*", "inhalt-fix-*"), # 2 Inhalts-Check - ("chunk-*",), # 3 Schreiben (chunk-* matcht auch chunk-nach-*) - ("lese-check-*", "fix-r*"), # 4 Lese-Prüfung + ("outline*",), # 0 Outline (incl. selection filter) + ("content-chunk-*", "content-nach-*"), # 1 Content (incl. follow-up round) + ("content-check-*", "content-fix-*"), # 2 Content-Check + ("chunk-*",), # 3 Writing (chunk-* also matches chunk-nach-*) + ("lese-check-*", "fix-r*"), # 4 Reading-Exam ) -def _reset_guide_ab_step(content_path: Path, step: int) -> None: - """Re-Run ab Schritt: Content + alle Slot-Dateien der Schritte ≥ step löschen. - Frühere Schritte bleiben → der Resume baut ab `step` neu (alles darunter wiederverwendet).""" - content_path.unlink(missing_ok=True) # nicht mehr „done" → kein Frischstart-Wipe +def _reset_guide_from_step(content_path: Path, step: int) -> None: + """Re-run from step: delete content + all slot files of steps ≥ step. + Earlier steps stay → the resume rebuilds from `step` (everything below is reused).""" + content_path.unlink(missing_ok=True) # no longer "done" → no fresh-start wipe d, stem = content_path.parent, content_path.stem for globs in _STEP_GLOBS[step:]: for pat in globs: for p in d.glob(f"{stem}.{pat}"): p.unlink(missing_ok=True) - _reset_fertig(content_path, step - 1) # Schritte < step gelten als fertig + _reset_done(content_path, step - 1) # steps < step count as done -def _lese_probleme_schema(data): - """{"ok": true} → [] · {"probleme": [{"section", "problem"}]} → Liste · sonst None.""" +def _read_problems_schema(data): + """{"ok": true} → [] · {"problems": [{"section", "problem"}]} → list · else None.""" if not isinstance(data, dict): return None if data.get("ok") is True: return [] - p = data.get("probleme") + p = data.get("problems") if not isinstance(p, list) or not p: return None out = [] @@ -188,139 +188,139 @@ def _lese_probleme_schema(data): return out or None -def _panel_probleme(judge_paths: list[Path], geltung: set[int], idx: dict[str, int]) -> dict[int, str]: - """Panel-Aggregation: mehrere Judge-Outputs eines Chunks → beanstandete {num: problem}. +def _panel_problems(judge_paths: list[Path], valid: set[int], idx: dict[str, int]) -> dict[int, str]: + """Panel aggregation: several judge outputs of a chunk → flagged {num: problem}. - Eine Stimme je Judge, der eine Section nennt. Beanstandet, wenn > Hälfte der - GELIEFERTEN (valid geparsten) Judges sie nennt (3→≥2, 2→≥2, 1→≥1). Robust gegen - Einzel-Ausfall: fehlende Dateien zählen nicht mit. Problem-Text vom erstnennenden Judge. + One vote per judge that names a section. Flagged when more than half of the + DELIVERED (validly parsed) judges name it (3→≥2, 2→≥2, 1→≥1). Robust against a + single failure: missing files do not count. Problem text from the first naming judge. """ - outputs = [p for p in (_lese_probleme_schema(_json_datei(j)) for j in judge_paths) if p is not None] + outputs = [p for p in (_read_problems_schema(_json_file(j)) for j in judge_paths) if p is not None] if not outputs: return {} votes: dict[int, int] = {} problem: dict[int, str] = {} for out in outputs: - gesehen: set[int] = set() + seen: set[int] = set() for item in out: - num = _titel_aufloesen(idx, item["section"]) - if num is None or num not in geltung or num in gesehen: + num = _resolve_title(idx, item["section"]) + if num is None or num not in valid or num in seen: continue - gesehen.add(num) + seen.add(num) votes[num] = votes.get(num, 0) + 1 problem.setdefault(num, item["problem"]) - schwelle = len(outputs) / 2 - return {num: problem[num] for num, v in votes.items() if v > schwelle} + threshold = len(outputs) / 2 + return {num: problem[num] for num, v in votes.items() if v > threshold} -def _resolve_gliederung(data, entries: dict[int, str], soll_min: int, soll_max: int) -> list[dict] | None: - """{"kapitel": [{"titel", "nummern": [1, 3, 7]}]} → [{"title", "nums"}]. +def _resolve_outline(data, entries: dict[int, str], target_min: int, target_max: int) -> list[dict] | None: + """{"chapters": [{"title", "numbers": [1, 3, 7]}]} → [{"title", "nums"}]. - Nummern sind die IDs aus `entries` (1-basiert, wie dem Agenten präsentiert). - `soll_min`/`soll_max` = erlaubte Spanne gewählter Bausteine (mit kleiner Toleranz). + Numbers are the IDs from `entries` (1-based, as presented to the agent). + `target_min`/`target_max` = allowed range of selected blocks (with a small tolerance). """ - if not isinstance(data, dict) or not isinstance(data.get("kapitel"), list): + if not isinstance(data, dict) or not isinstance(data.get("chapters"), list): return None - gueltig = set(entries) + valid = set(entries) chapters: list[dict] = [] seen: set[int] = set() total = unknown = 0 - for ch in data["kapitel"]: - if not isinstance(ch, dict) or not isinstance(ch.get("nummern"), list): + for ch in data["chapters"]: + if not isinstance(ch, dict) or not isinstance(ch.get("numbers"), list): return None nums = [] - for t in ch["nummern"]: + for t in ch["numbers"]: total += 1 num = t if isinstance(t, int) and not isinstance(t, bool) else None - if num is None or num not in gueltig: + if num is None or num not in valid: unknown += 1 elif num not in seen: nums.append(num) seen.add(num) if nums: - chapters.append({"title": str(ch.get("titel", "")).strip() or "Kapitel", "nums": nums}) + chapters.append({"title": str(ch.get("title", "")).strip() or "Chapter", "nums": nums}) if not chapters or total == 0: return None if (total - unknown) / total < 0.85: return None - if len(seen) < 0.9 * soll_min or len(seen) > 1.1 * soll_max: + if len(seen) < 0.9 * target_min or len(seen) > 1.1 * target_max: return None return chapters -def _fallback_gliederung(entries: dict[int, str]) -> list[dict]: - """Deterministische Gliederung, wenn die Agenten keine liefern: ein Kapitel mit - allen gewählten Bausteinen in Reihenfolge. Garantiert vollständige Abdeckung.""" - return [{"title": "Inhalte", "nums": list(entries)}] +def _fallback_outline(entries: dict[int, str]) -> list[dict]: + """Deterministic outline when the agents deliver none: one chapter with + all selected blocks in order. Guarantees full coverage.""" + return [{"title": "Contents", "nums": list(entries)}] -def _mit_resten(plan: list[dict], entries: dict[int, str]) -> list[dict]: - """Stellt sicher, dass JEDER gewählte Baustein im Plan steht — fehlende landen in - einem Kapitel „Weitere" (gegen Agenten/Judge, die Bausteine weglassen).""" - drin = {num for ch in plan for num in ch.get("nums", [])} - fehlen = [num for num in entries if num not in drin] - return [*plan, {"title": "Weitere", "nums": fehlen}] if fehlen else plan +def _with_remainder(plan: list[dict], entries: dict[int, str]) -> list[dict]: + """Ensures that EVERY selected block is in the plan — missing ones land in + an "Additional" chapter (against agents/judges that drop blocks).""" + present = {num for ch in plan for num in ch.get("nums", [])} + missing = [num for num in entries if num not in present] + return [*plan, {"title": "Additional", "nums": missing}] if missing else plan -def _fakten_grounding(subs_raw: dict[str, list[dict]]) -> str: - """Verifizierte Sub-Fakten (extract-once aus der Bausteine-Phase) als Grounding-Block für den - Inhalts-Agent. Leer, wenn keine Fakten gespeichert (Altbestand → Fallback auf Quelle-Hinweis).""" - bloecke = [] - for titel, subs in subs_raw.items(): - zeilen = [] +def _facts_grounding(subs_raw: dict[str, list[dict]]) -> str: + """Verified sub-facts (extract-once from the blocks phase) as a grounding block for the + content agent. Empty if no facts are stored (legacy data → fallback to a source hint).""" + blocks = [] + for title, subs in subs_raw.items(): + lines = [] for s in subs: - fk = s.get("fakten") if isinstance(s.get("fakten"), dict) else None + fk = s.get("facts") if isinstance(s.get("facts"), dict) else None if not fk: continue - teile = [] - if fk.get("kernpunkte"): - teile.append("Kern: " + " · ".join(fk["kernpunkte"])) - for bf in fk.get("belegte_fakten", []): - teile.append(f"FAKT[{bf.get('quelle', '?')}]: {bf.get('text', '')}") - if fk.get("voraussetzungen"): - teile.append("Voraussetzung: " + fk["voraussetzungen"]) - if fk.get("huerden"): - teile.append("Hürde: " + fk["huerden"]) - if fk.get("beispiel_idee"): - teile.append("Beispiel: " + fk["beispiel_idee"]) - if teile: - zeilen.append(f"- {s['titel']}: " + " | ".join(teile)) - if zeilen: - bloecke.append(f"BAUSTEIN: {titel}\n" + "\n".join(zeilen)) - if not bloecke: + parts = [] + if fk.get("key_points"): + parts.append("Core: " + " · ".join(fk["key_points"])) + for bf in fk.get("cited_facts", []): + parts.append(f"FACT[{bf.get('source', '?')}]: {bf.get('text', '')}") + if fk.get("prerequisites"): + parts.append("Prerequisite: " + fk["prerequisites"]) + if fk.get("hurdles"): + parts.append("Hurdle: " + fk["hurdles"]) + if fk.get("example_idea"): + parts.append("Example: " + fk["example_idea"]) + if parts: + lines.append(f"- {s['title']}: " + " | ".join(parts)) + if lines: + blocks.append(f"BLOCK: {title}\n" + "\n".join(lines)) + if not blocks: return "" - return ("VERIFIZIERTE FAKTEN je Subbaustein — verbindliche Grundlage. Belegte Fakten (FAKT[Quelle]) " - "WÖRTLICH übernehmen, nichts dazu erfinden, NICHT neu recherchieren. Beispiele als Beispiel " - "nutzen, nie als Fakt.\n\n" + "\n\n".join(bloecke)) + return ("VERIFIED FACTS per subblock — binding basis. Quote cited facts (FACT[Source]) " + "VERBATIM, invent nothing extra, do NOT re-research. Use examples as examples, " + "never as fact.\n\n" + "\n\n".join(blocks)) -async def _gliederung_aus_db(topic: str, sel_entries: dict[int, str]) -> list[dict] | None: - """Gliederung aus dem Bausteine-Artefakt (DB) lesen und auf die gewählten Bausteine mappen. - Titel-basiert (robust gegen Nummern-Drift): Bausteine außerhalb der Auswahl werden ignoriert - (Format-Filter), fehlende ergänzt später _mit_resten. None → kein Artefakt (Altbestand).""" - raw = await get_gliederung(topic) +async def _outline_from_db(topic: str, sel_entries: dict[int, str]) -> list[dict] | None: + """Read the outline from the blocks artifact (DB) and map it onto the selected blocks. + Title-based (robust against number drift): blocks outside the selection are ignored + (format filter), missing ones are added later by _with_remainder. None → no artifact (legacy).""" + raw = await get_outline(topic) if not raw: return None try: data = json.loads(raw) except (ValueError, TypeError): return None - kapitel = data.get("kapitel") if isinstance(data, dict) else None - if not isinstance(kapitel, list): + chapters = data.get("chapters") if isinstance(data, dict) else None + if not isinstance(chapters, list): return None - norm_to_num = {_norm_titel(_titel(t)): num for num, t in sel_entries.items()} + norm_to_num = {_norm_title(_title(t)): num for num, t in sel_entries.items()} plan, seen = [], set() - for ch in kapitel: + for ch in chapters: if not isinstance(ch, dict): continue nums = [] - for bt in ch.get("bausteine", []): - num = norm_to_num.get(_norm_titel(str(bt))) + for bt in ch.get("blocks", []): + num = norm_to_num.get(_norm_title(str(bt))) if num is not None and num not in seen: seen.add(num) nums.append(num) if nums: - plan.append({"title": str(ch.get("titel", "")).strip() or "Kapitel", "nums": nums}) + plan.append({"title": str(ch.get("title", "")).strip() or "Chapter", "nums": nums}) return plan or None @@ -335,257 +335,257 @@ async def _generate_sections( ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled, guide_id=guide_id) spec = (TEMPLATES_DIR / "Format" / "Section.md").read_text(encoding="utf-8") files = _guide_files(content_path) - zweck = FORMAT_ZWECK[format_name] + zweck = FORMAT_PURPOSE[format_name] - # Subbausteine je Baustein (DB-first) — früh geladen: steuert Auswahl + Sub-Filter je Format. - # Fehlt sie → {} (Fallback: Guide nimmt alles). - subs_raw = await _load_subbausteine(topic) - # Extract-once-Grounding: gespeicherte, verifizierte Fakten ersetzen den generischen - # Quelle-Hinweis. Der Inhalts-Agent formuliert daraus, statt die Quelle neu zu lesen. - if (fakten_block := _fakten_grounding(subs_raw)): - facts = fakten_block + # Subblocks per block (DB-first) — loaded early: drives selection + sub-filter per format. + # Missing → {} (fallback: guide takes everything). + subs_raw = await _load_subblocks(topic) + # Extract-once grounding: stored, verified facts replace the generic source hint. + # The content agent phrases from them instead of reading the source again. + if (facts_block := _facts_grounding(subs_raw)): + facts = facts_block - def _hat_relevanz(num, art): - return any(isinstance(s, dict) and s.get("relevanz") == art for s in subs_raw.get(_titel(entries[num]), [])) + def _has_relevance(num, kind): + return any(isinstance(s, dict) and s.get("relevance") == kind for s in subs_raw.get(_title(entries[num]), [])) - # Auswahl: EIN Voll-Dokument mit ALLEN Bausteinen (inkl. Rand). Die Ansichten E/M/S/F - # filtern später pro Subbaustein-Ebene. (FullGuide/Rest bleiben als Alt-Zweige bestehen.) + # Selection: ONE full document with ALL blocks (incl. peripheral). The views E/M/S/F + # filter later per subblock level. (FullGuide/Rest remain as legacy branches.) if format_name == "Rest": - auswahl = [num for num in entries if not _hat_relevanz(num, "relevant")] - else: # Guide / FullGuide → alle Bausteine - auswahl = list(entries) - if not auswahl: - await _fail(guide_id, "Keine passenden Bausteine für dieses Format") + selection = [num for num in entries if not _has_relevance(num, "relevant")] + else: # Guide / FullGuide → all blocks + selection = list(entries) + if not selection: + await _fail(guide_id, "No matching blocks for this format") return None - sel_entries = {num: entries[num] for num in auswahl} - soll = len(sel_entries) - # Nummerierte Liste (ID = Baustein-Nummer aus entries) — Agenten/Judge ordnen per Nummer. - sel_liste = "\n".join(f"{num}. {t}" for num, t in sel_entries.items()) + sel_entries = {num: entries[num] for num in selection} + target = len(sel_entries) + # Numbered list (ID = block number from entries) — agents/judge order by number. + sel_list = "\n".join(f"{num}. {t}" for num, t in sel_entries.items()) - # Schritt 0: Gliederung. Bevorzugt das Bausteine-Artefakt (DB) — der Guide präsentiert nur, - # gliedert nicht mehr selbst. Fehlt es (Altbestand) → bisherige Agenten/Judge-Logik als Fallback. - # 0 gültige → Code-Fallback, 1 → direkt, ≥2 → Judge (mit Vorschlag als Rückfall). - plan = await _gliederung_aus_db(topic, sel_entries) + # Step 0: outline. Prefers the blocks artifact (DB) — the guide only presents, + # no longer structures itself. Missing (legacy) → previous agents/judge logic as fallback. + # 0 valid → code fallback, 1 → direct, ≥2 → judge (with proposal as fallback). + plan = await _outline_from_db(topic, sel_entries) if plan is not None: - _log(topic, f"Gliederung aus Bausteine-Artefakt ({len(plan)} Kapitel)") + _log(topic, f"Outline from blocks artifact ({len(plan)} chapters)") if plan is None: - plan = _resolve_gliederung(_json_datei(files["gliederung"]), sel_entries, soll, soll) + plan = _resolve_outline(_json_file(files["outline"]), sel_entries, target, target) if plan is None: - await _set_step(guide_id, 0, "Gliederungs-Vorschläge (3 Agenten)…") - files["gliederung"].unlink(missing_ok=True) - vorschlaege: list[list[dict]] = [] - offen = [] - for i, path in enumerate(files["gliederung_slots"], 1): - res = _resolve_gliederung(_json_datei(path), sel_entries, soll, soll) + await _set_step(guide_id, 0, "Outline proposals (3 agents)…") + files["outline"].unlink(missing_ok=True) + proposals: list[list[dict]] = [] + pending = [] + for i, path in enumerate(files["outline_slots"], 1): + res = _resolve_outline(_json_file(path), sel_entries, target, target) if res is not None: - vorschlaege.append(res) + proposals.append(res) else: - offen.append((i, path)) - if len(vorschlaege) < 3 and offen: + pending.append((i, path)) + if len(proposals) < 3 and pending: slots = [ { - "key": f"{guide_id}-gliederung-{i}", + "key": f"{guide_id}-outline-{i}", "prompt": _prompt( - "Guide-Gliederung", - topic=topic, format_name=format_name, bausteine=sel_liste, + "Guide-Outline", + topic=topic, format_name=format_name, blocks=sel_list, out_path=path, extra=_extra(instructions), ), "role": "guide", "capabilities": "files", - "payload": (lambda result, p=path: _resolve_gliederung(_json_datei(p), sel_entries, soll, soll)), + "payload": (lambda result, p=path: _resolve_outline(_json_file(p), sel_entries, target, target)), } - for i, path in offen + for i, path in pending ] - # Quorum 1: nimm, was kommt — kein Mindestzwang, kein Abbruch. - neue = await _race( - topic, "Gliederung", slots, 1, _timeout("plan", soll), - provider, cancelled=is_cancelled, grace=KONSENS_GRACE, + # Quorum 1: take whatever comes — no minimum requirement, no abort. + new = await _race( + topic, "Outline", slots, 1, _timeout("plan", target), + provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE, ) if is_cancelled(): return None - vorschlaege += neue or [] + proposals += new or [] - if not vorschlaege: - _log(topic, "Gliederung: kein gültiger Vorschlag — deterministischer Fallback") - plan = _fallback_gliederung(sel_entries) - elif len(vorschlaege) == 1: - plan = vorschlaege[0] # ein Vorschlag → kein Judge nötig + if not proposals: + _log(topic, "Outline: no valid proposal — deterministic fallback") + plan = _fallback_outline(sel_entries) + elif len(proposals) == 1: + plan = proposals[0] # one proposal → no judge needed else: - await _set_step(guide_id, 0, "Gliederungen zusammenführen…") - bloecke = "\n\n".join( - f"### Vorschlag {i}\n" - + "\n".join(f"KAPITEL: {ch['title']}\n Nummern: {', '.join(str(num) for num in ch['nums'])}" for ch in v) - for i, v in enumerate(vorschlaege, 1) + await _set_step(guide_id, 0, "Merging outlines…") + proposals_text = "\n\n".join( + f"### Proposal {i}\n" + + "\n".join(f"CHAPTER: {ch['title']}\n Numbers: {', '.join(str(num) for num in ch['nums'])}" for ch in v) + for i, v in enumerate(proposals, 1) ) status, plan = await run_single_slot( - ctx, "Gliederungs-Judge", - key=f"{guide_id}-gliederung-judge", + ctx, "Outline-Judge", + key=f"{guide_id}-outline-judge", prompt=_prompt( - "Guide-Gliederung-Judge", - topic=topic, format_name=format_name, zweck=zweck, n=len(vorschlaege), - bausteine=sel_liste, gliederungen=bloecke, - out_path=files["gliederung"], extra=_extra(instructions), + "Guide-Outline-Judge", + topic=topic, format_name=format_name, purpose=zweck, n=len(proposals), + blocks=sel_list, outlines=proposals_text, + out_path=files["outline"], extra=_extra(instructions), ), role="judge", capabilities="files", - payload=lambda result: _resolve_gliederung(_json_datei(files["gliederung"]), sel_entries, soll, soll), - timeout=_timeout("plan_judge", soll), + payload=lambda result: _resolve_outline(_json_file(files["outline"]), sel_entries, target, target), + timeout=_timeout("plan_judge", target), ) if status == CANCELLED: return None if status == FAILED or plan is None: - _log(topic, "Gliederung-Judge ohne Ergebnis — bester Vorschlag bleibt") - plan = vorschlaege[0] + _log(topic, "Outline judge produced no result — best proposal kept") + plan = proposals[0] - # Garantie: jeder gewählte Baustein steht im Plan (gegen weglassende Agenten/Judges). - plan = _mit_resten(plan, sel_entries) - _set_fertig(content_path, 0) # Gliederung steht + # Guarantee: every selected block is in the plan (against dropping agents/judges). + plan = _with_remainder(plan, sel_entries) + _set_done(content_path, 0) # outline ready - # Grobe Chunks (~GUIDE_CHUNK Bausteine je Agent) für Inhalte, Inhalts-Check und Lese-Prüfung. - # Der Writer baut darunter pro Baustein (eigene feine Chunks, s.u.) → variable Längen. + # Coarse chunks (~GUIDE_CHUNK blocks per agent) for content, content-check and reading-exam. + # The writer builds per block beneath (its own fine chunks, see below) → variable lengths. total_sections = sum(len(c["nums"]) for c in plan) chunks = _split_chunks(plan, max(1, math.ceil(total_sections / GUIDE_CHUNK))) - # Subbausteine je Baustein: Guide/FullGuide nehmen ALLE (inkl. Rand → Ebene 4 in der Ansicht); - # nur der Alt-Zweig Rest filtert auf Rand. So trägt das eine Dokument alle Ebenen. + # Subblocks per block: Guide/FullGuide take ALL (incl. peripheral → level 4 in the view); + # only the legacy Rest branch filters to peripheral. So the one document carries all levels. if format_name == "Rest": - subs_by_titel = {t: [s for s in subs if s.get("relevanz") == "rand"] for t, subs in subs_raw.items()} + subs_by_title = {t: [s for s in subs if s.get("relevance") == "peripheral"] for t, subs in subs_raw.items()} else: # Guide / FullGuide - subs_by_titel = {t: list(subs) for t, subs in subs_raw.items()} - subs_by_titel = {t: subs for t, subs in subs_by_titel.items() if subs} - zuteilungen = [_zuteilung_subs(chunk, entries, subs_by_titel) for chunk in chunks] + subs_by_title = {t: list(subs) for t, subs in subs_raw.items()} + subs_by_title = {t: subs for t, subs in subs_by_title.items() if subs} + assignments = [_assignment_subs(chunk, entries, subs_by_title) for chunk in chunks] chunk_sizes = [sum(len(c["nums"]) for c in chunk) for chunk in chunks] writer_count = len(chunks) - idx = _titel_index(entries) + idx = _title_index(entries) - # Schritt 2: Inhalte je Baustein identifizieren — pro Chunk ein Agent (Marker-Output, Resume). - inhalt_paths = [content_path.parent / f"{content_path.stem}.inhalt-chunk-{i}.md" for i in range(1, writer_count + 1)] - offen = [i for i, p in enumerate(inhalt_paths) if not p.exists()] - if offen: - async def melde(d, t): await _set_step(guide_id, 1, f"Sammle Inhalte {d}/{t}…") - results = await _gather_fortschritt([ + # Step 2: identify content per block — one agent per chunk (marker output, resume). + content_paths = [content_path.parent / f"{content_path.stem}.content-chunk-{i}.md" for i in range(1, writer_count + 1)] + pending = [i for i, p in enumerate(content_paths) if not p.exists()] + if pending: + async def report(d, t): await _set_step(guide_id, 1, f"Gathering content {d}/{t}…") + results = await _gather_progress([ run_agent( - f"{guide_id}-inhalt-{i + 1}", + f"{guide_id}-content-{i + 1}", _prompt( - "Guide-Inhalt", - topic=topic, zuteilung=zuteilungen[i], facts=facts, - out_path=inhalt_paths[i], extra=_extra(instructions), + "Guide-Content", + topic=topic, assignment=assignments[i], facts=facts, + out_path=content_paths[i], extra=_extra(instructions), ), - _timeout("inhalt", chunk_sizes[i]), provider=provider, role="guide", capabilities="full", + _timeout("content", chunk_sizes[i]), provider=provider, role="guide", capabilities="full", ) - for i in offen - ], writer_count, melde, start=writer_count - len(offen)) + for i in pending + ], writer_count, report, start=writer_count - len(pending)) if is_cancelled(): return None - if not any(p.exists() for p in inhalt_paths): - await _fail(guide_id, _gather_error("Inhalts-Fehler", list(results))) + if not any(p.exists() for p in content_paths): + await _fail(guide_id, _gather_error("Content error", list(results))) return None - inhalt_by_num: dict[int, str] = {} - for p in inhalt_paths: + content_by_num: dict[int, str] = {} + for p in content_paths: if not p.exists(): continue for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _titel_aufloesen(idx, sec["titel"]) - if num is not None and num not in inhalt_by_num and sec["md"].strip(): - inhalt_by_num[num] = sec["md"] - if not inhalt_by_num: - await _fail(guide_id, "Keine Inhalte identifiziert") + num = _resolve_title(idx, sec["title"]) + if num is not None and num not in content_by_num and sec["md"].strip(): + content_by_num[num] = sec["md"] + if not content_by_num: + await _fail(guide_id, "No content identified") return None - # Nachrunde: fehlende Bausteine (Chunk-Ausfall oder Lazy-Output) gezielt nachziehen — eine Runde. - geplant_nums = [num for ch in plan for num in ch["nums"]] - fehlend = [num for num in geplant_nums if num not in inhalt_by_num] - if fehlend: - _log(topic, f"Inhalte: {len(fehlend)} Baustein(e) fehlen — Nachrunde…") - nach_chunks = [[{"title": "Weitere", "nums": fehlend[k:k + GUIDE_CHUNK]}] for k in range(0, len(fehlend), GUIDE_CHUNK)] - nach_paths = [content_path.parent / f"{content_path.stem}.inhalt-nach-{k}.md" for k in range(1, len(nach_chunks) + 1)] - nach_offen = [k for k, p in enumerate(nach_paths) if not p.exists()] - if nach_offen: - async def melde_n(d, t): await _set_step(guide_id, 1, f"Sammle fehlende Inhalte {d}/{t}…") - await _gather_fortschritt([ + # Follow-up round: pull missing blocks (chunk failure or lazy output) deliberately — one round. + planned_nums = [num for ch in plan for num in ch["nums"]] + missing = [num for num in planned_nums if num not in content_by_num] + if missing: + _log(topic, f"Content: {len(missing)} block(s) missing — follow-up round…") + followup_chunks = [[{"title": "Additional", "nums": missing[k:k + GUIDE_CHUNK]}] for k in range(0, len(missing), GUIDE_CHUNK)] + followup_paths = [content_path.parent / f"{content_path.stem}.content-nach-{k}.md" for k in range(1, len(followup_chunks) + 1)] + followup_pending = [k for k, p in enumerate(followup_paths) if not p.exists()] + if followup_pending: + async def report_n(d, t): await _set_step(guide_id, 1, f"Gathering missing content {d}/{t}…") + await _gather_progress([ run_agent( - f"{guide_id}-inhalt-nach-{k + 1}", + f"{guide_id}-content-nach-{k + 1}", _prompt( - "Guide-Inhalt", - topic=topic, zuteilung=_zuteilung_subs(nach_chunks[k], entries, subs_by_titel), - facts=facts, out_path=nach_paths[k], extra=_extra(instructions), + "Guide-Content", + topic=topic, assignment=_assignment_subs(followup_chunks[k], entries, subs_by_title), + facts=facts, out_path=followup_paths[k], extra=_extra(instructions), ), - _timeout("inhalt", len(nach_chunks[k][0]["nums"])), provider=provider, role="guide", capabilities="full", + _timeout("content", len(followup_chunks[k][0]["nums"])), provider=provider, role="guide", capabilities="full", ) - for k in nach_offen - ], len(nach_chunks), melde_n, start=len(nach_chunks) - len(nach_offen)) + for k in followup_pending + ], len(followup_chunks), report_n, start=len(followup_chunks) - len(followup_pending)) if is_cancelled(): return None - for p in nach_paths: + for p in followup_paths: if not p.exists(): continue for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _titel_aufloesen(idx, sec["titel"]) - if num is not None and num not in inhalt_by_num and sec["md"].strip(): - inhalt_by_num[num] = sec["md"] + num = _resolve_title(idx, sec["title"]) + if num is not None and num not in content_by_num and sec["md"].strip(): + content_by_num[num] = sec["md"] - if all(p.exists() for p in inhalt_paths): - _set_fertig(content_path, 1) # Inhalte vollständig + if all(p.exists() for p in content_paths): + _set_done(content_path, 1) # content complete - inhalt_chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in inhalt_by_num] for chunk in chunks] + content_chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in content_by_num] for chunk in chunks] - # Schritt 3: Inhalte prüfen — CHECK_PANEL Judges je Chunk, Mehrheit beanstandet. - # + beanstandete einmal überarbeiten. Resume: nur fehlende Judge-Dateien neu starten. + # Step 3: check content — CHECK_PANEL judges per chunk, majority flags. + # + revise flagged ones once. Resume: only restart missing judge files. check_judge_paths = [ - [content_path.parent / f"{content_path.stem}.inhalt-check-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)] + [content_path.parent / f"{content_path.stem}.content-check-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)] for i in range(1, writer_count + 1) ] - offen_slots = [ - (i, j) for i in range(writer_count) if inhalt_chunk_nums[i] - for j in range(CHECK_PANEL) if _lese_probleme_schema(_json_datei(check_judge_paths[i][j])) is None + pending_slots = [ + (i, j) for i in range(writer_count) if content_chunk_nums[i] + for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None ] - if offen_slots: - await _set_step(guide_id, 2, "Prüfe Inhalte…") - sections_je_chunk = { - i: "\n\n".join(f"SECTION: {_titel(entries[num])}\n{inhalt_by_num[num]}" for num in inhalt_chunk_nums[i]) - for i, _ in offen_slots + if pending_slots: + await _set_step(guide_id, 2, "Checking content…") + sections_per_chunk = { + i: "\n\n".join(f"SECTION: {_title(entries[num])}\n{content_by_num[num]}" for num in content_chunk_nums[i]) + for i, _ in pending_slots } slots = [{ - "key": f"{guide_id}-inhalt-check-{i + 1}-j{j + 1}", + "key": f"{guide_id}-content-check-{i + 1}-j{j + 1}", "prompt": _prompt( - "Guide-Inhalt-Check", - topic=topic, format_name=format_name, sections=sections_je_chunk[i], + "Guide-Content-Check", + topic=topic, format_name=format_name, sections=sections_per_chunk[i], out_path=check_judge_paths[i][j], extra=_extra(instructions), ), "role": "judge", "capabilities": "files", - "payload": (lambda result, p=check_judge_paths[i][j]: _lese_probleme_schema(_json_datei(p))), - } for i, j in offen_slots] + "payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))), + } for i, j in pending_slots] n_checks = len(slots) - upd = lambda n: asyncio.create_task(_set_step(guide_id, 2, f"Prüfe Inhalte {n}/{n_checks}…")) - await _race(topic, "Inhalts-Prüfung", slots, len(slots), _timeout("inhalt_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=KONSENS_GRACE) + upd = lambda n: asyncio.create_task(_set_step(guide_id, 2, f"Checking content {n}/{n_checks}…")) + await _race(topic, "Content-Exam", slots, len(slots), _timeout("content_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE) if is_cancelled(): return None - probleme_by_num: dict[int, str] = {} + problems_by_num: dict[int, str] = {} for i in range(writer_count): - if inhalt_chunk_nums[i]: - probleme_by_num.update(_panel_probleme(check_judge_paths[i], set(inhalt_chunk_nums[i]), idx)) + if content_chunk_nums[i]: + problems_by_num.update(_panel_problems(check_judge_paths[i], set(content_chunk_nums[i]), idx)) - if probleme_by_num: - _log(topic, f"Inhalts-Prüfung: {len(probleme_by_num)} Baustein(e) beanstandet") - await _set_step(guide_id, 2, f"Überarbeite {len(probleme_by_num)} Inhalt(e)…") - fix_chunks = [[num for num in nums if num in probleme_by_num] for nums in inhalt_chunk_nums] - fix_paths = [content_path.parent / f"{content_path.stem}.inhalt-fix-{i + 1}.md" for i in range(writer_count)] - fix_offen = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()] + if problems_by_num: + _log(topic, f"Content exam: {len(problems_by_num)} block(s) flagged") + await _set_step(guide_id, 2, f"Revising {len(problems_by_num)} content(s)…") + fix_chunks = [[num for num in nums if num in problems_by_num] for nums in content_chunk_nums] + fix_paths = [content_path.parent / f"{content_path.stem}.content-fix-{i + 1}.md" for i in range(writer_count)] + fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()] results = await asyncio.gather(*[ run_agent( - f"{guide_id}-inhalt-fix-{i + 1}", + f"{guide_id}-content-fix-{i + 1}", _prompt( - "Guide-Inhalt-Fix", + "Guide-Content-Fix", topic=topic, facts=facts, - auftraege="\n\n".join( - f"SECTION: {_titel(entries[num])}\nPROBLEM: {probleme_by_num[num]}\nAKTUELL:\n{inhalt_by_num[num]}" + tasks="\n\n".join( + f"SECTION: {_title(entries[num])}\nPROBLEM: {problems_by_num[num]}\nCURRENT:\n{content_by_num[num]}" for num in fix_chunks[i] ), out_path=fix_paths[i], extra=_extra(instructions), ), - _timeout("inhalt", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full", + _timeout("content", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full", ) - for i in fix_offen + for i in fix_pending ], return_exceptions=True) if is_cancelled(): return None @@ -593,48 +593,48 @@ async def _generate_sections( if not p.exists(): continue for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _titel_aufloesen(idx, sec["titel"]) - if num in probleme_by_num and sec["md"].strip(): - inhalt_by_num[num] = sec["md"] + num = _resolve_title(idx, sec["title"]) + if num in problems_by_num and sec["md"].strip(): + content_by_num[num] = sec["md"] - _set_fertig(content_path, 2) # Inhalts-Check durch + _set_done(content_path, 2) # content check done - # Schritt 4: Schreiben — Writer formuliert die geprüften Inhalte aus (Resume). - # FEINE Chunks: genau 1 Baustein je Writer → variable Längen, kein Budget-Rationieren. - def inhalte_text(chunk) -> str: - nums = [num for ch in chunk for num in ch["nums"] if num in inhalt_by_num] - return "\n\n".join(f"\n{inhalt_by_num[num]}" for num in nums) + # Step 4: writing — the writer phrases out the checked content (resume). + # FINE chunks: exactly 1 block per writer → variable lengths, no budget rationing. + def content_text(chunk) -> str: + nums = [num for ch in chunk for num in ch["nums"] if num in content_by_num] + return "\n\n".join(f"\n{content_by_num[num]}" for num in nums) w_chunks = [[{"title": ch["title"], "nums": [num]}] for ch in plan for num in ch["nums"]] - w_zuteil = [_zuteilung_subs(c, entries, subs_by_titel) for c in w_chunks] + w_assignments = [_assignment_subs(c, entries, subs_by_title) for c in w_chunks] paths = [content_path.parent / f"{content_path.stem}.chunk-{i}.md" for i in range(1, len(w_chunks) + 1)] - offen = [i for i, p in enumerate(paths) if not p.exists()] - if offen: - async def melde(d, t): await _set_step(guide_id, 3, f"Schreibe Sections {d}/{t}…") - results = await _gather_fortschritt([ + pending = [i for i, p in enumerate(paths) if not p.exists()] + if pending: + async def report(d, t): await _set_step(guide_id, 3, f"Writing sections {d}/{t}…") + results = await _gather_progress([ run_agent( f"{guide_id}-w{i + 1}", _prompt( "Guide-Writer", - topic=topic, format_name=format_name, zuteilung=w_zuteil[i], - inhalte=inhalte_text(w_chunks[i]), + topic=topic, format_name=format_name, assignment=w_assignments[i], + contents=content_text(w_chunks[i]), spec=spec, out_path=paths[i], extra=_extra(instructions), ), _timeout("writer", 1), provider=provider, role="guide", capabilities="files", ) - for i in offen - ], len(w_chunks), melde, start=len(w_chunks) - len(offen)) + for i in pending + ], len(w_chunks), report, start=len(w_chunks) - len(pending)) if is_cancelled(): return None - for i, r in zip(offen, results): + for i, r in zip(pending, results): if isinstance(r, BaseException): _log(topic, f"Writer {i + 1}: {type(r).__name__}: {r}") elif r[0] != 0: - _log(topic, f"Writer {i + 1}: {_claude_error('Fehler', *r)}") + _log(topic, f"Writer {i + 1}: {_claude_error('Error', *r)}") elif not paths[i].exists(): - _log(topic, f"Writer {i + 1}: keine Ausgabedatei erstellt") + _log(topic, f"Writer {i + 1}: no output file created") if not any(p.exists() for p in paths): - await _fail(guide_id, _gather_error("Writer-Fehler", list(results))) + await _fail(guide_id, _gather_error("Writer error", list(results))) return None by_num: dict[int, dict] = {} @@ -642,277 +642,277 @@ async def _generate_sections( if not p.exists(): continue for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _titel_aufloesen(idx, sec["titel"]) + num = _resolve_title(idx, sec["title"]) if num is None: - _log(topic, f"Writer lieferte unbekannte Section '{sec['titel'][:40]}' (ignoriert)") + _log(topic, f"Writer produced unknown section '{sec['title'][:40]}' (ignored)") elif num not in by_num: by_num[num] = sec if not by_num: - await _fail(guide_id, "Keine Sections in der Writer-Ausgabe gefunden") + await _fail(guide_id, "No sections found in writer output") return None - # Nachrunde: fehlende Sections (Writer-Ausfall) gezielt nachschreiben — eine Runde. - nach_fehlend = [num for num in geplant_nums if num not in by_num] - if nach_fehlend: - _log(topic, f"Schreiben: {len(nach_fehlend)} Section(s) fehlen — Nachrunde…") - nw_chunks = [[{"title": "Weitere", "nums": [num]}] for num in nach_fehlend] + # Follow-up round: write missing sections (writer failure) deliberately — one round. + missing_after = [num for num in planned_nums if num not in by_num] + if missing_after: + _log(topic, f"Writing: {len(missing_after)} section(s) missing — follow-up round…") + nw_chunks = [[{"title": "Additional", "nums": [num]}] for num in missing_after] nw_paths = [content_path.parent / f"{content_path.stem}.chunk-nach-{k}.md" for k in range(1, len(nw_chunks) + 1)] - nw_offen = [k for k, p in enumerate(nw_paths) if not p.exists()] - if nw_offen: - async def melde_nw(d, t): await _set_step(guide_id, 3, f"Schreibe fehlende Sections {d}/{t}…") - await _gather_fortschritt([ + nw_pending = [k for k, p in enumerate(nw_paths) if not p.exists()] + if nw_pending: + async def report_nw(d, t): await _set_step(guide_id, 3, f"Writing missing sections {d}/{t}…") + await _gather_progress([ run_agent( f"{guide_id}-w-nach-{k + 1}", _prompt( "Guide-Writer", - topic=topic, format_name=format_name, zuteilung=_zuteilung_subs(nw_chunks[k], entries, subs_by_titel), - inhalte=inhalte_text(nw_chunks[k]), spec=spec, out_path=nw_paths[k], extra=_extra(instructions), + topic=topic, format_name=format_name, assignment=_assignment_subs(nw_chunks[k], entries, subs_by_title), + contents=content_text(nw_chunks[k]), spec=spec, out_path=nw_paths[k], extra=_extra(instructions), ), _timeout("writer", 1), provider=provider, role="guide", capabilities="files", ) - for k in nw_offen - ], len(nw_chunks), melde_nw, start=len(nw_chunks) - len(nw_offen)) + for k in nw_pending + ], len(nw_chunks), report_nw, start=len(nw_chunks) - len(nw_pending)) if is_cancelled(): return None for p in nw_paths: if not p.exists(): continue for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _titel_aufloesen(idx, sec["titel"]) + num = _resolve_title(idx, sec["title"]) if num is not None and num not in by_num and sec["md"].strip(): by_num[num] = sec if all(p.exists() for p in paths): - _set_fertig(content_path, 3) # Schreiben vollständig + _set_done(content_path, 3) # writing complete - # Schritt 3: Lese-Prüfungs-Loop — Check pro Writer-Paket, Fix nur für - # beanstandete Sections; Folgerunden prüfen NUR die ersetzten Sections. - # Nach dem Runden-Cap bleiben offene Beanstandungen stehen. + # Step 3: reading-exam loop — check per writer packet, fix only for + # flagged sections; follow-up rounds check ONLY the replaced sections. + # After the round cap, open complaints stand. chunk_nums = [[num for ch in chunk for num in ch["nums"] if num in by_num] for chunk in chunks] def sections_text(nums: list[int]) -> str: - return "\n\n".join(f"SECTION: {_titel(entries[num])}\n{by_num[num]['md']}" for num in nums) + return "\n\n".join(f"SECTION: {_title(entries[num])}\n{by_num[num]['md']}" for num in nums) - def _sub_liste(num: int) -> str: - subs = subs_by_titel.get(_titel(entries[num]), []) - return "\n".join(f"- [{_ebene_label(s)}] {s['titel']}" for s in subs) or "(keine)" + def _sub_list(num: int) -> str: + subs = subs_by_title.get(_title(entries[num]), []) + return "\n".join(f"- [{_level_label(s)}] {s['title']}" for s in subs) or "(none)" - def auftraege_text(nums: list[int], probleme: dict[int, str]) -> str: + def tasks_text(nums: list[int], problems: dict[int, str]) -> str: return "\n\n".join( - f"SECTION: {_titel(entries[num])}\n" - f"SUBBAUSTEINE (je einen ``-Marker setzen, Label/Reihenfolge wie hier):\n{_sub_liste(num)}\n" - f"PROBLEM: {probleme[num]}\nAKTUELLER INHALT:\n{by_num[num]['md']}" + f"SECTION: {_title(entries[num])}\n" + f"SUBBLOCKS (set one `` marker each, label/order as here):\n{_sub_list(num)}\n" + f"PROBLEM: {problems[num]}\nCURRENT CONTENT:\n{by_num[num]['md']}" for num in nums ) scope = chunk_nums - for runde in range(1, LESE_RUNDEN + 1): - # CHECK_PANEL Judges je Paket; Mehrheit beanstandet. Aggregation robust gegen Einzel-Ausfall. + for round_no in range(1, READING_ROUNDS + 1): + # CHECK_PANEL judges per packet; majority flags. Aggregation robust against a single failure. check_judge_paths = [ - [content_path.parent / f"{content_path.stem}.lese-check-r{runde}-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)] + [content_path.parent / f"{content_path.stem}.lese-check-r{round_no}-{i}-j{j}.json" for j in range(1, CHECK_PANEL + 1)] for i in range(1, writer_count + 1) ] - offen_slots = [ + pending_slots = [ (i, j) for i in range(writer_count) if scope[i] - for j in range(CHECK_PANEL) if _lese_probleme_schema(_json_datei(check_judge_paths[i][j])) is None + for j in range(CHECK_PANEL) if _read_problems_schema(_json_file(check_judge_paths[i][j])) is None ] - if offen_slots: - await _set_step(guide_id, 4, "Prüfe Lesbarkeit…") - sections_je_chunk = {i: sections_text(scope[i]) for i, _ in offen_slots} + if pending_slots: + await _set_step(guide_id, 4, "Checking readability…") + sections_per_chunk = {i: sections_text(scope[i]) for i, _ in pending_slots} slots = [{ - "key": f"{guide_id}-lese-check-r{runde}-{i + 1}-j{j + 1}", + "key": f"{guide_id}-lese-check-r{round_no}-{i + 1}-j{j + 1}", "prompt": _prompt( "Guide-Lese-Check", topic=topic, format_name=format_name, spec=spec, - sections=sections_je_chunk[i], + sections=sections_per_chunk[i], out_path=check_judge_paths[i][j], extra=_extra(instructions), ), "role": "judge", "capabilities": "files", - "payload": (lambda result, p=check_judge_paths[i][j]: _lese_probleme_schema(_json_datei(p))), - } for i, j in offen_slots] + "payload": (lambda result, p=check_judge_paths[i][j]: _read_problems_schema(_json_file(p))), + } for i, j in pending_slots] n_checks = len(slots) - upd = lambda n: asyncio.create_task(_set_step(guide_id, 4, f"Prüfe Lesbarkeit {n}/{n_checks}…")) - res = await _race(topic, f"Lese-Prüfung r{runde}", slots, len(slots), _timeout("lese_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=KONSENS_GRACE) + upd = lambda n: asyncio.create_task(_set_step(guide_id, 4, f"Checking readability {n}/{n_checks}…")) + res = await _race(topic, f"Reading-Exam r{round_no}", slots, len(slots), _timeout("lese_check", max(chunk_sizes)), provider, on_update=upd, cancelled=is_cancelled, grace=CONSENSUS_GRACE) if is_cancelled(): return None if res is None: - _log(topic, f"Lese-Prüfung Runde {runde}: kein volles Quorum — vorhandene Judges aggregiert") + _log(topic, f"Reading exam round {round_no}: no full quorum — aggregated available judges") - probleme_by_num: dict[int, str] = {} + problems_by_num: dict[int, str] = {} for i in range(writer_count): if scope[i]: - probleme_by_num.update(_panel_probleme(check_judge_paths[i], set(scope[i]), idx)) + problems_by_num.update(_panel_problems(check_judge_paths[i], set(scope[i]), idx)) - # Deterministisches Lesbarkeits-Gate: zu schwere Sections in dieselbe - # Überarbeitung einreihen (LLM-Beanstandung hat Vorrang). Gate aus → no-op. - if LESBARKEIT_AKTIV: + # Deterministic readability gate: queue too-hard sections into the same + # revision (LLM complaint takes precedence). Gate off → no-op. + if READABILITY_ACTIVE: md_by_num = {num: by_num[num]["md"] for nums in scope for num in nums if num in by_num} - hinweise = await asyncio.to_thread(lesbarkeit.bewerte_sections, md_by_num) - if hinweise: - _log(topic, f"Lesbarkeit: {len(hinweise)} Section(s) zu schwer") - for num, hinweis in hinweise.items(): - probleme_by_num.setdefault(num, hinweis) + hints = await asyncio.to_thread(readability.rate_sections, md_by_num) + if hints: + _log(topic, f"Readability: {len(hints)} section(s) too hard") + for num, hint in hints.items(): + problems_by_num.setdefault(num, hint) - if not probleme_by_num: + if not problems_by_num: break - _log(topic, f"Lese-Prüfung Runde {runde}: {len(probleme_by_num)} Section(s) beanstandet") - await _set_step(guide_id, 4, f"Überarbeite {len(probleme_by_num)} Section(s) (Runde {runde})…") - fix_chunks = [[num for num in nums if num in probleme_by_num] for nums in chunk_nums] - fix_paths = [content_path.parent / f"{content_path.stem}.fix-r{runde}-{i + 1}.md" for i in range(writer_count)] - fix_offen = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()] + _log(topic, f"Reading exam round {round_no}: {len(problems_by_num)} section(s) flagged") + await _set_step(guide_id, 4, f"Revising {len(problems_by_num)} section(s) (round {round_no})…") + fix_chunks = [[num for num in nums if num in problems_by_num] for nums in chunk_nums] + fix_paths = [content_path.parent / f"{content_path.stem}.fix-r{round_no}-{i + 1}.md" for i in range(writer_count)] + fix_pending = [i for i, nums in enumerate(fix_chunks) if nums and not fix_paths[i].exists()] results = await asyncio.gather(*[ run_agent( - f"{guide_id}-fix-r{runde}-w{i + 1}", + f"{guide_id}-fix-r{round_no}-w{i + 1}", _prompt( "Guide-Sections-Fix", topic=topic, format_name=format_name, facts=facts, spec=spec, - auftraege=auftraege_text(fix_chunks[i], probleme_by_num), + tasks=tasks_text(fix_chunks[i], problems_by_num), out_path=fix_paths[i], extra=_extra(instructions), ), _timeout("writer", len(fix_chunks[i])), provider=provider, role="guide", capabilities="full", ) - for i in fix_offen + for i in fix_pending ], return_exceptions=True) if is_cancelled(): return None - for i, r in zip(fix_offen, results): + for i, r in zip(fix_pending, results): if isinstance(r, BaseException) or (not isinstance(r, BaseException) and r[0] != 0): - _log(topic, f"Sections-Fix {i + 1} (Runde {runde}) fehlgeschlagen — Original bleibt") - ersetzt: set[int] = set() + _log(topic, f"Sections fix {i + 1} (round {round_no}) failed — original kept") + replaced: set[int] = set() for p in fix_paths: if not p.exists(): continue for sec in _parse_fragment(p.read_text(encoding="utf-8")): - num = _titel_aufloesen(idx, sec["titel"]) - if num not in probleme_by_num or not sec["md"].strip(): + num = _resolve_title(idx, sec["title"]) + if num not in problems_by_num or not sec["md"].strip(): continue - # Marker-Invariante: Verliert der Fix die Sub-Marker, obwohl das Original welche - # hatte, wird er verworfen — sonst stirbt der Stufen-Filter (E/M/S/F) still. + # Marker invariant: if the fix loses the sub markers although the original had + # some, it is discarded — otherwise the level filter (E/M/S/F) dies silently. if by_num[num].get("subs") and not sec.get("subs"): - _log(topic, f"Lese-Fix für '{sec['titel']}' ohne Sub-Marker — verworfen, getaggtes Original bleibt") + _log(topic, f"Reading fix for '{sec['title']}' without sub markers — discarded, tagged original kept") continue by_num[num] = sec - ersetzt.add(num) - _log(topic, f"Lese-Prüfung Runde {runde}: {len(ersetzt)} Section(s) überarbeitet") - if not ersetzt: + replaced.add(num) + _log(topic, f"Reading exam round {round_no}: {len(replaced)} section(s) revised") + if not replaced: break - if runde == LESE_RUNDEN: - _log(topic, f"Lese-Prüfung: 1 Runde — Überarbeitung bleibt ungeprüft") + if round_no == READING_ROUNDS: + _log(topic, f"Reading exam: 1 round — revision stays unchecked") break - scope = [[num for num in nums if num in ersetzt] for nums in chunk_nums] - _set_fertig(content_path, 4) # Lese-Prüfung durch + scope = [[num for num in nums if num in replaced] for nums in chunk_nums] + _set_done(content_path, 4) # reading exam done - # Prüfbar = Format hat Prüfung UND Baustein hat ≥1 relevanten Subbaustein. - # Guide ist immer prüfbar (auch ohne Relevanz-Daten, Fallback = alles). - def _pruefbar(num): + # Checkable = format has an exam AND the block has ≥1 relevant subblock. + # Guide is always checkable (even without relevance data, fallback = everything). + def _checkable(num): if format_name == "Guide": return True if format_name == "FullGuide": - return any(isinstance(s, dict) and s.get("relevanz") == "relevant" - for s in subs_raw.get(_titel(entries[num]), [])) - return False # Rest u.a. → reine Lese-Sections + return any(isinstance(s, dict) and s.get("relevance") == "relevant" + for s in subs_raw.get(_title(entries[num]), [])) + return False # Rest etc. → pure reading sections - await _set_progress(guide_id, "Setze zusammen…") + await _set_progress(guide_id, "Assembling…") chapters: list[dict] = [] for ch in plan: sections = [ - {"num": num, "title": _titel(entries[num]), "md": by_num[num]["md"], - "kompakt": by_num[num].get("kompakt", ""), - "anker": by_num[num].get("anker", ""), "anker_kompakt": by_num[num].get("anker_kompakt", ""), - "subs": by_num[num].get("subs", []), "pruefbar": _pruefbar(num)} + {"num": num, "title": _title(entries[num]), "md": by_num[num]["md"], + "compact": by_num[num].get("compact", ""), + "anchor": by_num[num].get("anchor", ""), "anker_compact": by_num[num].get("anker_compact", ""), + "subs": by_num[num].get("subs", []), "checkable": _checkable(num)} for num in ch["nums"] if num in by_num ] if sections: chapters.append({"title": ch["title"], "sections": sections}) - geplant = {num for ch in plan for num in ch["nums"]} - missing = sorted(geplant - set(by_num)) + planned = {num for ch in plan for num in ch["nums"]} + missing = sorted(planned - set(by_num)) if missing: - _log(topic, f"Sections fehlen in der Writer-Ausgabe: {[_titel(entries[n]) for n in missing]}") + _log(topic, f"Sections missing from writer output: {[_title(entries[n]) for n in missing]}") if not chapters: - await _fail(guide_id, "Keine Sections in der Writer-Ausgabe gefunden") + await _fail(guide_id, "No sections found in writer output") return None return chapters -_STUFE_EBENE = {"anfaenger": 1, "fortgeschritten": 2, "experte": 3, "rand": 4, - "einfach": 1, "mittel": 2, "schwer": 3} # alte Werte abwärtskompatibel +_LEVEL_RANK = {"beginner": 1, "advanced": 2, "expert": 3, "peripheral": 4, + "easy": 1, "medium": 2, "hard": 3} # old values backward-compatible -def _section_fuer_ebene(sec: dict, ebene: int) -> dict: - """md/kompakt einer Section auf Subbausteine bis zur Ebene rekonstruieren (Anker bleibt).""" +def _section_for_level(sec: dict, level: int) -> dict: + """Reconstruct md/compact of a section from subblocks up to the level (anchor stays).""" subs = sec.get("subs") or [] if not subs: - return sec # keine Sub-Tags (Altbestand) → unverändert sichtbar - sichtbar = [s for s in subs if _STUFE_EBENE.get(s.get("stufe"), 1) <= ebene] - md = "\n\n".join(t for t in [sec.get("anker", ""), *(s.get("md", "") for s in sichtbar)] if t).strip() - kompakt = "\n".join(t for t in [sec.get("anker_kompakt", ""), *(s.get("kompakt", "") for s in sichtbar)] if t).strip() - return {**sec, "md": md, "kompakt": kompakt, "leer": not sichtbar} + return sec # no sub tags (legacy) → unchanged, visible + visible = [s for s in subs if _LEVEL_RANK.get(s.get("level"), 1) <= level] + md = "\n\n".join(t for t in [sec.get("anchor", ""), *(s.get("md", "") for s in visible)] if t).strip() + compact = "\n".join(t for t in [sec.get("anker_compact", ""), *(s.get("compact", "") for s in visible)] if t).strip() + return {**sec, "md": md, "compact": compact, "leer": not visible} -def content_fuer_ebene(content: dict, ebene: int) -> dict: - """Guide-Content auf eine Ansichts-Ebene (1=A · 2=F · 3=E · 4=V) filtern. Ebene 4 = Vollfassung. - Sections ohne sichtbare Subs werden ausgeblendet, leere Kapitel entfallen.""" - if not isinstance(content, dict) or ebene >= 4: +def content_fuer_level(content: dict, level: int) -> dict: + """Filter guide content to a view level (1=A · 2=F · 3=E · 4=V). Level 4 = full version. + Sections without visible subs are hidden, empty chapters drop out.""" + if not isinstance(content, dict) or level >= 4: return content - kapitel = [] + chapters = [] for ch in content.get("chapters", []): - secs = [s for s in (_section_fuer_ebene(x, ebene) for x in ch.get("sections", [])) if not s.get("leer")] + secs = [s for s in (_section_for_level(x, level) for x in ch.get("sections", [])) if not s.get("leer")] if secs: - kapitel.append({**ch, "sections": secs}) - return {**content, "chapters": kapitel} + chapters.append({**ch, "sections": secs}) + return {**content, "chapters": chapters} async def reconcile_guides() -> None: - """DB↔Dateisystem abgleichen: status=done ohne Content-Datei → error. + """Reconcile DB↔filesystem: status=done without content file → error. - Läuft beim Server-Start (nach init_db) — fängt Crashes zwischen - Datei-Write und Status-Update ab. + Runs at server start (after init_db) — catches crashes between + file write and status update. """ for g in await list_guides(): if g["status"] == "done" and not guide_content_path(g["topic"], g["format"]).exists(): - log.warning("[%s] Guide %s: done ohne Content-Datei — auf error gesetzt", g["topic"], g["id"]) + log.warning("[%s] Guide %s: done without content file — set to error", g["topic"], g["id"]) now = datetime.now(timezone.utc).isoformat() - await update_guide(g["id"], status="error", error_msg="Inhalt fehlt — neu generieren", updated_at=now) + await update_guide(g["id"], status="error", error_msg="Content missing — regenerate", updated_at=now) async def generate_guide(guide_id: str, topic: str, format_name: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, ab_step: int | None = None) -> None: async with _semaphore: now = datetime.now(timezone.utc).isoformat() - await update_guide(guide_id, status="generating", progress="Starte…", updated_at=now) + await update_guide(guide_id, status="generating", progress="Starting…", updated_at=now) content_path = guide_content_path(topic, format_name) content_path.parent.mkdir(parents=True, exist_ok=True) - project = quelle_ordner(topic) # Ordner-Quelle (projekt/uni/link) → Pfad, sonst None + project = source_folder(topic) # folder source (project/uni/link) → path, else None try: if is_guide_cancelled(guide_id): return if project: - await asyncio.to_thread(_pdfs_konvertieren, project) + await asyncio.to_thread(_convert_pdfs, project) - # Re-Run ab Schritt: Content + Slots ab `ab_step` löschen, Rest bleibt → Resume baut ab dort. - # Sonst „Neu erstellen": fertiger Guide → kompletter Frischstart. - # Sonst sind Schritt-Dateien Reste eines Abbruchs/Fehlers → Resume. + # Re-run from step: delete content + slots from `ab_step`, rest stays → resume rebuilds from there. + # Otherwise "recreate": a finished guide → complete fresh start. + # Otherwise step files are leftovers of an abort/error → resume. if ab_step is not None: - _reset_guide_ab_step(content_path, ab_step) + _reset_guide_from_step(content_path, ab_step) elif content_path.exists(): - for p_alt in guide_slot_dateien(content_path): + for p_alt in guide_slot_files(content_path): p_alt.unlink(missing_ok=True) - bs = await list_bausteine(topic, status="konsens") + bs = await list_blocks(topic, status="consensus") if bs: - alle = {i: (f"{b['titel']} — {b['beschreibung']}" if b["beschreibung"] else b["titel"]) + alle = {i: (f"{b['title']} — {b['description']}" if b["description"] else b["title"]) for i, b in enumerate(bs, 1)} - else: # Fallback: bausteine.md (Alt-Themen) - bp = bausteine_path(topic) - alle = _lade_bausteine(bp.read_text(encoding="utf-8")) if bp.exists() else {} + else: # fallback: blocks.md (legacy topics) + bp = blocks_path(topic) + alle = _load_blocks(bp.read_text(encoding="utf-8")) if bp.exists() else {} if not alle: - await _fail(guide_id, "Keine Bausteine gefunden") + await _fail(guide_id, "No blocks found") return - entries = _eindeutige_titel(alle) - facts = _prompt("Guide-Fakten-Projekt", project=project) if project else _prompt("Guide-Fakten-Thema") + entries = _unique_title(alle) + facts = _prompt("Guide-Facts-Projekt", project=project) if project else _prompt("Guide-Facts-Thema") chapters = await _generate_sections( guide_id, topic, format_name, entries, facts, instructions, provider, content_path, @@ -921,26 +921,26 @@ async def generate_guide(guide_id: str, topic: str, format_name: str, instructio return content = {"topic": topic, "format": format_name, "chapters": chapters} - atomic_write_json(content_path, content, indent=1) # Brücke (Resume/Fallback) + atomic_write_json(content_path, content, indent=1) # bridge (resume/fallback) await set_guide_content(topic, format_name, json.dumps(content, ensure_ascii=False)) now = datetime.now(timezone.utc).isoformat() await update_guide(guide_id, status="done", progress=None, step=None, updated_at=now) except asyncio.TimeoutError: - await _fail(guide_id, "Timeout bei der Generierung") + await _fail(guide_id, "Timeout during generation") except FileNotFoundError: - await _fail(guide_id, "Bausteine fehlen") + await _fail(guide_id, "Blocks missing") except Exception as e: - log.exception("[%s] Guide-Generierung fehlgeschlagen (%s)", topic, guide_id) + log.exception("[%s] Guide generation failed (%s)", topic, guide_id) await _fail(guide_id, str(e)[:2000]) finally: clear_guide_cancelled(guide_id) -# --- On-Demand: eine Section prüfen / beheben / neu schreiben (Fokus, interaktiv) --- +# --- On-demand: check / fix / rewrite one section (focus, interactive) --- -SECTION_PRUEFEN_TIMEOUT = 300 +SECTION_CHECK_TIMEOUT = 300 def _section_spec() -> str: @@ -948,25 +948,25 @@ def _section_spec() -> str: def _section_facts(topic: str) -> str: - project = quelle_ordner(topic) - return _prompt("Guide-Fakten-Projekt", project=project) if project else _prompt("Guide-Fakten-Thema") + project = source_folder(topic) + return _prompt("Guide-Facts-Projekt", project=project) if project else _prompt("Guide-Facts-Thema") -def _hinweis_block(hinweis: str) -> str: - hinweis = (hinweis or "").strip() - return f"HINWEIS DES NUTZERS (besonders beachten):\n{hinweis}" if hinweis else "" +def _hint_block(hint: str) -> str: + hint = (hint or "").strip() + return f"NOTE FROM THE USER (pay special attention):\n{hint}" if hint else "" -async def _subs_text(topic: str, baustein: str) -> str: - """Relevante Subbausteine eines Bausteins mit Stufe — als Checkliste für die Agenten.""" - subs_raw = await _load_subbausteine(topic) - subs = [s for s in subs_raw.get(_titel(baustein), []) if s.get("relevanz") != "rand"] +async def _subs_text(topic: str, block: str) -> str: + """Relevant subblocks of a block with level — as a checklist for the agents.""" + subs_raw = await _load_subblocks(topic) + subs = [s for s in subs_raw.get(_title(block), []) if s.get("relevance") != "peripheral"] if not subs: - return "(keine Subbausteine hinterlegt — 3–7 knappe Punkte abdecken)" - return "\n".join(f"- [{s['stufe']}] {s['titel']}" for s in subs) + return "(no subblocks recorded — cover 3–7 concise points)" + return "\n".join(f"- [{s['level']}] {s['title']}" for s in subs) -async def _guide_content_laden(topic: str, format_name: str) -> dict | None: +async def _load_guide_content(topic: str, format_name: str) -> dict | None: js = await get_guide_content(topic, format_name) if not js: return None @@ -976,57 +976,57 @@ async def _guide_content_laden(topic: str, format_name: str) -> dict | None: return None -def _section_finden(content: dict, baustein: str) -> dict | None: +def _find_section(content: dict, block: str) -> dict | None: for ch in content.get("chapters", []): for s in ch.get("sections", []): - if s.get("title") == baustein: + if s.get("title") == block: return s return None -async def block_pruefen(topic: str, format_name: str, baustein: str, stelle: str, block: str, hinweis: str = "", provider: str = DEFAULT_PROVIDER) -> str | None: - """Einen Abschnitt (Markdown-Block) gegen die Guide-Regeln prüfen → korrigierte - Block-Version als Markdown. None = Fehler/Section fehlt.""" - content = await _guide_content_laden(topic, format_name) - sec = _section_finden(content, baustein) if content else None +async def block_pruefen(topic: str, format_name: str, block: str, spot: str, snippet: str, hint: str = "", provider: str = DEFAULT_PROVIDER) -> str | None: + """Check one section (Markdown block) against the guide rules → corrected + block version as Markdown. None = error/section missing.""" + content = await _load_guide_content(topic, format_name) + sec = _find_section(content, block) if content else None if sec is None: return None - ganze = sec.get("kompakt", "") if str(stelle).startswith("kompakt") else sec.get("md", "") + whole = sec.get("compact", "") if str(spot).startswith("compact") else sec.get("md", "") prompt = _prompt( "Block-Pruefen", topic=topic, spec=_section_spec(), facts=_section_facts(topic), - subbausteine=await _subs_text(topic, baustein), kontext=ganze, block=block, hinweis=_hinweis_block(hinweis), + subblocks=await _subs_text(topic, block), context=whole, snippet=snippet, hint=_hint_block(hint), ) rc, stdout, _ = await run_agent( - f"block-pruefen-{uuid.uuid4()}", prompt, SECTION_PRUEFEN_TIMEOUT, + f"block-pruefen-{uuid.uuid4()}", prompt, SECTION_CHECK_TIMEOUT, provider=provider, role="judge", capabilities="none", lane="interactive", ) - neu = stdout.strip() if rc == 0 else "" - return neu or None + new = stdout.strip() if rc == 0 else "" + return new or None -async def block_uebernehmen(topic: str, format_name: str, baustein: str, stelle: str, alt: str, neu: str) -> dict | None: - """Einen Block (alt→neu) im Feld kompakt/ausführlich ersetzen + persistieren. - → {kompakt, md, gefunden}; None = Section fehlt.""" - content = await _guide_content_laden(topic, format_name) - sec = _section_finden(content, baustein) if content else None +async def block_adopt(topic: str, format_name: str, block: str, spot: str, old: str, new: str) -> dict | None: + """Replace one block (old→new) in the compact/detailed field + persist. + → {compact, md, found}; None = section missing.""" + content = await _load_guide_content(topic, format_name) + sec = _find_section(content, block) if content else None if sec is None: return None - ist_kompakt = str(stelle).startswith("kompakt") - feld = "kompakt" if ist_kompakt else "md" - aktuell = sec.get(feld, "") or "" - gefunden = alt in aktuell - if gefunden: - sec[feld] = aktuell.replace(alt, neu, 1) - # Auch in Anker + Subs ersetzen (Quellen der gefilterten E/M/S-Ansicht), sonst zeigt - # die gestufte Ansicht weiter den alten Block. - anker_feld = "anker_kompakt" if ist_kompakt else "anker" - if alt in (sec.get(anker_feld) or ""): - sec[anker_feld] = sec[anker_feld].replace(alt, neu, 1) + is_compact = str(spot).startswith("compact") + field = "compact" if is_compact else "md" + current = sec.get(field, "") or "" + found = old in current + if found: + sec[field] = current.replace(old, new, 1) + # Also replace in anchor + subs (sources of the filtered E/M/S view), otherwise the + # leveled view keeps showing the old block. + anchor_field = "anker_compact" if is_compact else "anchor" + if old in (sec.get(anchor_field) or ""): + sec[anchor_field] = sec[anchor_field].replace(old, new, 1) for sub in sec.get("subs", []): - if alt in (sub.get(feld, "") or ""): - sub[feld] = sub[feld].replace(alt, neu, 1) + if old in (sub.get(field, "") or ""): + sub[field] = sub[field].replace(old, new, 1) break js = json.dumps(content, ensure_ascii=False) await set_guide_content(topic, format_name, js) atomic_write_json(guide_content_path(topic, format_name), content, indent=1) - return {"kompakt": sec.get("kompakt", ""), "md": sec.get("md", ""), "gefunden": gefunden} + return {"compact": sec.get("compact", ""), "md": sec.get("md", ""), "found": found} diff --git a/backend/jsonio.py b/backend/jsonio.py index 3a2ea61..cdc56b7 100644 --- a/backend/jsonio.py +++ b/backend/jsonio.py @@ -1,8 +1,8 @@ -"""Toleranter JSON-Parser für KI-Output — als Text oder aus Dateien. +"""Tolerant JSON parser for AI output — from text or from files. -Verkraftet Code-Fences, Drumherum-Text und unescapte Anführungszeichen in -Strings (z. B. MiniMax: "Titel „p" geändert"): das letzte `"` vor der -Fehlerstelle wird escapet und erneut geparst. +Copes with code fences, surrounding prose and unescaped quotes inside +strings (e.g. MiniMax: "Title „p" changed"): the last `"` before the +error position is escaped and parsing is retried. """ import json @@ -14,7 +14,7 @@ log = logging.getLogger("creator.jsonio") def parse_json_text(text: str): - """Parst JSON aus KI-Output; None bei nicht reparierbarem Input.""" + """Parse JSON from AI output; None for input that can't be repaired.""" text = re.sub(r"^```(?:json)?\s*|\s*```$", "", (text or "").strip()) start, end = text.find("{"), text.rfind("}") if start == -1 or end <= start: @@ -36,14 +36,14 @@ def parse_json_text(text: str): def read_json_file(path: Path): - """Liest eine JSON-Datei mit derselben Toleranz; None bei fehlend/ungültig.""" + """Read a JSON file with the same tolerance; None if missing/invalid.""" if not path.exists(): return None try: data = parse_json_text(path.read_text(encoding="utf-8")) except Exception as e: - log.debug("JSON-Datei nicht lesbar: %s (%s)", path, e) + log.debug("JSON file not readable: %s (%s)", path, e) return None if data is None: - log.debug("JSON-Datei ungültig: %s", path) + log.debug("JSON file invalid: %s", path) return data diff --git a/backend/learning.py b/backend/learning.py new file mode 100644 index 0000000..52f6bb3 --- /dev/null +++ b/backend/learning.py @@ -0,0 +1,657 @@ +"""Block learning: deep-dive, block chat and exam for individual guide sections. + +All calls are interactive (stdout response, lane "interactive") and stateless — +the chat/exam history comes from the frontend; only the exam counter (DB) and +the deep-dive (DB) are persisted. +""" + +import logging +import random +import re +import uuid +from datetime import datetime, timezone + +from agents import run_agent +from config import DEFAULT_PROVIDER +from database import create_element, list_elements, get_block_hurdles +from elements import generate_element +from jsonio import parse_json_text as _parse_json_text +from pipeline import _prompt, _problems_schema +from textkit import _norm_title + +log = logging.getLogger("creator.learning") + +# Learning levels per block — relative to the cap (floor as % of the max score): +# green=beginner 20% · blue=advanced 40% · purple=expert 60% · gold=master 100%. +# Exam form is always random (5 forms); the cap scales with the amount of material. +LEVELS = (("beginner", 0.2), ("advanced", 0.4), ("expert", 0.6), ("master", 1.0)) + + +POINTS_BASE = 25 # Points per subblock. Master cap = (all subs) × 25. + + +def _levels(n_je_level: dict[int, int]) -> list[int]: + return [n_je_level.get(k, 0) for k in (1, 2, 3, 4)] + + +def thresholds(n_je_level: dict[int, int]) -> list[int]: + """Cumulative sub-level thresholds [S_1, S_2, S_3, S_4] = (n_1+…+n_k) × 25. + S_k is the score at which sub-level k+1 unlocks; S_4 = cap_final.""" + out, acc = [], 0 + for n in _levels(n_je_level): + acc += n + out.append(acc * POINTS_BASE) + return out + + +def cap_final(n_je_level: dict[int, int]) -> int: + """Max score (master) = all subblocks × 25.""" + return thresholds(n_je_level)[-1] + + +def freie_level(score: int, n_je_level: dict[int, int]) -> int: + """Highest unlocked sub-level 1–4. Level k+1 unlocks once score ≥ S_k. + Empty levels (n_k=0) are skipped automatically (S_k == S_{k-1}).""" + s = thresholds(n_je_level) + e = 1 + for k in range(3): # S_1..S_3 unlock levels 2..4 + if score >= s[k]: + e = k + 2 + return e + + +def cap_aktuell(score: int, n_je_level: dict[int, int]) -> int: + """Reachable cap of the currently unlocked level = unlocked subs × 25.""" + return thresholds(n_je_level)[freie_level(score, n_je_level) - 1] + + +def _threshold(p: float, cap: int) -> int: + return round(p * cap) + + +def level_from_score(score: int, cap_final_value: int) -> str | None: + """Highest reached learning level (None below 20%), relative to cap_final.""" + reached = None + for key, p in LEVELS: + if score >= _threshold(p, cap_final_value): + reached = key + return reached + + +def progressive_malus(basis: int, cap_akt: int) -> int: + """Error penalty by progress within the current level (against cap_aktuell): + ≤25%→−5 · ≤50%→−10 · ≤75%→−15 · >75%→−20.""" + pct = (basis / cap_akt) if cap_akt else 0.0 + if pct <= 0.25: + return -5 + if pct <= 0.5: + return -10 + if pct <= 0.75: + return -15 + return -20 +CHAT_TIMEOUT = 240 +EXAM_TIMEOUT = 120 # short JSON turns; caps the serial latency per exam step +THOROUGH_TIMEOUT = 600 # "thorough check": strong model (role guide) takes longer +CRITIC_MAX_ROUNDS = 2 # Generator → Critic → maybe Regenerate, at most this many times + +# Question types for active recall — one per question, chosen at random. Creates variety. +QUESTION_TYPES = { + "abruf": "Free Recall: have the learner explain the core idea freely from memory (open comprehension question).", + "punkt": "Cued Recall: ask for ONE specific detail or distinction.", + "warum": "Why-question: ask for the reason/mechanism — why does this work or hold?", + "anwendung": "Application: have the concept applied to ONE short, new example/scenario.", + "pruefen": "For code/tool topics: show a small snippet — predict the output OR find the bug. No code topic → an application question instead.", +} + + +# Answer tier → base points (new 25-scale). "barely" = −1 is only the signal for the +# progressive malus (the real value comes from progressive_malus). Positive values are +# modulated up on a streak and clamped to [10, 40]. +TIERS = { + "unanswerable": 0, # question itself broken → no change + "barely": -1, # < 25% correct → malus + "partial": 0, # 25–49% → neutral + "solid": 16, # 50–74% + "strong": 24, # 75–99% (quiz/gap hit) + "complete": 30, # 100% (only reachable by free explanation) +} + +# Order weak→strong (for the follow-up cap). +_TIER_RANK = ("barely", "partial", "solid", "strong", "complete") + + +def cap_followup(tier: str, asked_again: bool) -> str: + """With a follow-up (hint received) at most "solid" — no full score by cheating.""" + if asked_again and tier in ("strong", "complete"): + return "solid" + return tier + + +def streak_points(basis_delta: int, streak_basis: int) -> int: + """Modulate a positive base delta up by streak, clamped to [10, 40].""" + factor = min(1.33, 1 + 0.066 * min(streak_basis, 5)) + return max(10, min(40, round(basis_delta * factor))) + + +def points_delta(tier: str, streak_basis: int, basis: int, cap_akt: int) -> tuple[int, int]: + """Answer tier → (points delta, new streak). Positive: streak-modulated, streak +1. + Neutral (0): no change, streak stays. Negative: progressive malus, streak reset to 0.""" + basis_delta = TIERS.get(tier, 0) + if basis_delta > 0: + return streak_points(basis_delta, streak_basis), streak_basis + 1 + if basis_delta == 0: + return 0, streak_basis + return progressive_malus(basis, cap_akt), 0 + + +def compute_score(basis: int, delta: int, floor: int, cap_akt: int, cap_fin: int) -> int: + """New score · drift-free from the base. Clamps up against `cap_akt` (cap of the + currently unlocked level) and down against `floor`. Frozen ONLY at the absolute + maximum (`basis ≥ cap_fin`) — otherwise it would block at every level threshold.""" + if basis >= cap_fin: + return basis + return max(floor, min(cap_akt, basis + delta)) + + +def floor_from_score(basis: int, cap_fin: int, s_thresholds: list[int]) -> int: + """Lower bound (no fallback): highest reached learning-level threshold (over cap_final) + AND highest reached level-unlock threshold S_k. max of both axes.""" + floor = 0 + for _, p in LEVELS: + s = _threshold(p, cap_fin) + if basis >= s: + floor = max(floor, s) + for s in s_thresholds: + if basis >= s: + floor = max(floor, s) + return floor + + +def _transcript(messages: list[dict]) -> str: + return "\n".join( + f"{'User' if m.get('role') == 'user' else 'Assistant'}: {m.get('content', '')}" + for m in messages + ) or "(empty)" + + +async def block_chat(topic: str, block: str, section: str, compact: str | None, messages: list[dict], provider: str = DEFAULT_PROVIDER) -> str: + try: + prompt = _prompt( + "Block-Chat", + topic=topic, block=block, + section_block=section.strip() or "(no guide version provided)", + compact_block=(compact or "").strip() or "(none)", + transcript=_transcript(messages), + ) + returncode, stdout, _ = await run_agent( + "blockchat-" + str(uuid.uuid4()), prompt, CHAT_TIMEOUT, + provider=provider, role="fast", capabilities="none", lane="interactive", + ) + if returncode != 0: + return "Sorry, that didn't work. Please try again." + reply = stdout.strip() + return reply or "Sorry, I didn't get a response." + except Exception: + log.warning("[%s] Block chat failed (%s)", topic, block, exc_info=True) + return "Sorry, that didn't work. Please try again." + + +def _question_schema(data) -> dict | None: + """{"question": str} · else None.""" + if not isinstance(data, dict): + return None + question = str(data.get("question", "")).strip() + return {"question": question} if question else None + + +def _rating_schema(data) -> dict | None: + """{"feedback": str, "tier": ∈ TIERS} · else None.""" + if not isinstance(data, dict): + return None + feedback = str(data.get("feedback", "")).strip() + tier = data.get("tier") + if not feedback or tier not in TIERS: + return None + return {"feedback": feedback, "tier": tier} + + +async def _gen_call(name: str, role: str, schema, provider: str, timeout: int = EXAM_TIMEOUT, lane: str = "interactive", **kwargs) -> dict | None: + """Generator agent: fill the template, run it, parse via schema · None on error. + lane="batch" for background (preloading, thorough rating) → its own slot queue.""" + returncode, stdout, _ = await run_agent( + name.lower() + "-" + str(uuid.uuid4()), _prompt(name, **kwargs), timeout, + provider=provider, role=role, capabilities="none", lane=lane, + ) + return schema(_parse_json_text(stdout)) if returncode == 0 else None + + +async def _critique_call(name: str, provider: str, role: str = "judge", timeout: int = EXAM_TIMEOUT, lane: str = "interactive", **kwargs) -> list[str]: + """Critic agent (default role judge): empty list = fine. Fail-open: a critic failure + must not block the turn, so it returns an empty list then as well.""" + returncode, stdout, _ = await run_agent( + name.lower() + "-" + str(uuid.uuid4()), _prompt(name, **kwargs), timeout, + provider=provider, role=role, capabilities="none", lane=lane, + ) + if returncode != 0: + return [] + return _problems_schema(_parse_json_text(stdout)) or [] + + +def _critique_block(prev_version: str, problems: list[str]) -> str: + points = "\n".join(f"- {p}" for p in problems) + return ( + f"Your previous version was:\n«{prev_version}»\n\n" + f"The examiner objects:\n{points}\n\nFix these points." + ) + + +def _rating_text(rating: dict) -> str: + return f"Tier: {rating['tier']}\nFeedback: {rating['feedback']}" + + +# Deterministic guard against double questions — the AI critic misses "…, and which…". +_QUESTION_WORD = r"(was|welche[rsnm]?|wie|wieso|warum|wofür|wozu|wann|wo|wer|wem|wen|nenne)" +_DOUBLE_RE = re.compile(r"[,;]?\s+(und|sowie|außerdem|bzw\.?)\s+" + _QUESTION_WORD + r"\b", re.IGNORECASE) + + +def _double_question_flaw(question: str) -> str | None: + """Detects two chained questions. None = ok. Flags ONLY 'und/sowie' + question word.""" + if question.count("?") > 1: + return "More than one question mark — ask EXACTLY ONE question." + if _DOUBLE_RE.search(question): + return "Two questions chained with 'und'/'sowie' — ask EXACTLY ONE question, one thing." + return None + + +async def _question_with_critique( + topic: str, block: str, section_block: str, compact_block: str, + transcript: str, avoid_block: str, type_block: str, fokus_block: str, + tier_block: str, provider: str, +) -> str | None: + """Generate a question, have the critic check it, regenerate on flaws (max CRITIC_MAX_ROUNDS).""" + kritik_block = "(none)" + question = None + for _ in range(CRITIC_MAX_ROUNDS): + data = await _gen_call( + "Block-Question", "guide", _question_schema, provider, lane="batch", + topic=topic, block=block, section_block=section_block, + compact_block=compact_block, transcript=transcript, avoid_block=avoid_block, + type_block=type_block, fokus_block=fokus_block, tier_block=tier_block, kritik_block=kritik_block, + ) + if data is None: + return None + question = data["question"] + problems = await _critique_call( + "Block-Question-Critique", provider, role="guide", lane="batch", # strong AI checks the rules + topic=topic, block=block, section_block=section_block, + compact_block=compact_block, transcript=transcript, avoid_block=avoid_block, + type_block=type_block, fokus_block=fokus_block, question=question, + ) + hard = _double_question_flaw(question) # forces regeneration even if the AI critic missed it + if hard: + problems = [hard, *(problems or [])] + if not problems: + return question + kritik_block = _critique_block(question, problems) + return question # best-effort after the last round + + +async def _rating_with_critique( + topic: str, block: str, section_block: str, compact_block: str, + question: str, transcript: str, reason_block: str, provider: str, role: str = "judge", +) -> dict | None: + """Rate an answer (tier), have the critic check it, redo on misjudgment. + + `question` anchors the checked question; the dialog (transcript) provides answer + discussion. + `reason_block` = optional learner dissatisfaction (only for "thorough check"). + `role` = "judge" (fast) or "guide" (thorough, strong model with thinking). + """ + timeout = THOROUGH_TIMEOUT if role == "guide" else EXAM_TIMEOUT + # Thorough (role guide) = user is waiting → interactive. Background-thorough (judge) → batch. + lane = "interactive" if role == "guide" else "batch" + kritik_block = "(none)" + rating = None + for _ in range(CRITIC_MAX_ROUNDS): + rating = await _gen_call( + "Block-Rating", role, _rating_schema, provider, timeout, lane=lane, + topic=topic, block=block, section_block=section_block, + compact_block=compact_block, question=question, transcript=transcript, + reason_block=reason_block, kritik_block=kritik_block, + ) + if rating is None: + return None + problems = await _critique_call( + "Block-Rating-Critique", provider, role=role, timeout=timeout, lane=lane, + topic=topic, block=block, section_block=section_block, + compact_block=compact_block, question=question, transcript=transcript, + rating_block=_rating_text(rating), + ) + if not problems: + return rating + kritik_block = _critique_block(_rating_text(rating), problems) + return rating # best-effort after the last round + + +def _section_blocks(section: str, compact: str | None) -> tuple[str, str]: + return ( + section.strip() or "(no guide version provided)", + (compact or "").strip() or "(none)", + ) + + +def _avoid_block(avoid: list[str] | None) -> str: + entries = [f.strip() for f in (avoid or []) if f and f.strip()] + return "\n".join(f"- {f}" for f in entries) or "(none)" + + +# Learner tier (derived from the score) → addressee role for the question. This is how the +# difficulty arises: not "make it extra hard", but "ask questions for a beginner/expert". +# Per level: addressee role + cognitive demand (Bloom) + "ask like this" cue. Without explicit levels +# the model takes the easy path (mere recall) — the cues lift higher tiers to apply/analyze/transfer. +TIER_ROLE = { + "beginner": "The learner is a BEGINNER. Cognitive: REMEMBER/UNDERSTAND. Ask about the basic understanding — the core concept, simple and direct.", + "advanced": "The learner is ADVANCED. Cognitive: APPLY. Pose a small concrete situation and have the concept applied to it — don't just ask for the definition.", + "expert": "The learner is an EXPERT. Cognitive: ANALYZE. Have them distinguish/compare, classify a special case or uncover a typical pitfall (hurdle) — don't quiz textbook knowledge.", + "master": "The learner is at MASTER level. Cognitive: EVALUATE/TRANSFER. Have the concept transferred to a NEW problem, justify a decision or weigh a trade-off.", +} + + +def _tier_block(tier: str | None) -> str: + return TIER_ROLE.get(tier or "", TIER_ROLE["beginner"]) + + +async def exam_question( + topic: str, block: str, section: str, compact: str | None, + messages: list[dict], subblocks: list[str] | None = None, + avoid: list[str] | None = None, tier: str = "beginner", provider: str = DEFAULT_PROVIDER, +) -> str | None: + """Action 'question': generate a question — random type for a random subblock, + in the addressee role of the tier, then critic (sequential) · None on error.""" + try: + section_block, compact_block = _section_blocks(section, compact) + transcript = _transcript(messages) if messages else "(empty)" + type_block = QUESTION_TYPES[random.choice(list(QUESTION_TYPES))] + subs = [s for s in (subblocks or []) if s and s.strip()] + focus = random.choice(subs) if subs else "" + fokus_block = ( + f"Focus the question on this subblock: „{focus}\"" if focus + else "(whole block — no specific subblock)" + ) + return await _question_with_critique( + topic, block, section_block, compact_block, transcript, + _avoid_block(avoid), type_block, fokus_block, _tier_block(tier), provider, + ) + except Exception: + log.warning("[%s] Question failed (%s)", topic, block, exc_info=True) + return None + + +async def exam_question_variant( + topic: str, block: str, section: str, compact: str | None, + pattern: str, tier: str = "beginner", provider: str = DEFAULT_PROVIDER, +) -> str | None: + """Action 'question' with a pattern: from a predefined pattern, phrase a concrete question in + the addressee role of the tier. No critic (the pattern is build-checked). + The style guard stays as a cheap protection against double questions · None on error.""" + try: + section_block, compact_block = _section_blocks(section, compact) + data = await _gen_call( + "Block-Question-Variante", "guide", _question_schema, provider, lane="batch", + topic=topic, block=block, section_block=section_block, + compact_block=compact_block, pattern=pattern, tier_block=_tier_block(tier), + ) + if data is None: + return None + return data["question"] + except Exception: + log.warning("[%s] Question variant failed (%s)", topic, block, exc_info=True) + return None + + +def _options_schema(opts) -> list[dict] | None: + """[{text, correct}]×4 → validated list · else None.""" + if not isinstance(opts, list) or len(opts) != 4: + return None + out = [] + for o in opts: + if not isinstance(o, dict): + return None + text = str(o.get("text", "")).strip() + correct = o.get("correct") + if not text or not isinstance(correct, bool): + return None + out.append({"text": text, "correct": correct}) + return out + + +def _quiz_schema(data) -> dict | None: + """{"question": str, "options": [{text, correct}]×4} → validated · else None. + Single choice: exactly 1 correct. The difficulty is in the tier, not in the count.""" + if not isinstance(data, dict): + return None + question = str(data.get("question", "")).strip() + out = _options_schema(data.get("options")) + if not question or out is None: + return None + if sum(o["correct"] for o in out) != 1: + return None + return {"question": question, "options": out} + + +def _gapchoice_schema(data) -> dict | None: + """{"sentence": str (with ___), "options": [{text, correct}]×4} → exactly 1 correct · else None.""" + if not isinstance(data, dict): + return None + sentence = str(data.get("sentence", "")).strip() + out = _options_schema(data.get("options")) + if not sentence or "___" not in sentence or out is None or sum(o["correct"] for o in out) != 1: + return None + return {"sentence": sentence, "options": out} + + +async def hurdles_distractor_block(topic: str, block: str) -> str: + """Typical misconceptions (facts hurdles) of the block as a distractor source for quiz/gap choice. + Empty if none exist (legacy) → the prompt placeholder disappears without a trace.""" + try: + hurdles = await get_block_hurdles(topic, _norm_title(block)) + except Exception: + return "" + if not hurdles: + return "" + lines = "\n".join(f"- {h}" for h in hurdles[:8]) + return ("TYPICAL MISCONCEPTIONS for this block (use them as distractors when they fit the question):\n" + + lines + "\n") + + +async def generate_quiz( + topic: str, block: str, section: str, compact: str | None, + pattern: str, tier: str = "beginner", provider: str = DEFAULT_PROVIDER, + distractor_block: str = "", +) -> dict | None: + """From a pattern, a single-choice question (exactly 1 correct), at the tier's level. + Strong model (role guide) for correct flags. → {question, options} · None on error. + distractor_block: optional typical misconceptions (from the facts hurdles) as a distractor source.""" + try: + section_block, compact_block = _section_blocks(section, compact) + return await _gen_call( + "Block-Quiz", "guide", _quiz_schema, provider, lane="batch", + topic=topic, block=block, section_block=section_block, + compact_block=compact_block, pattern=pattern, tier_block=_tier_block(tier), + distractor_block=distractor_block, + ) + except Exception: + log.warning("[%s] Quiz question failed (%s)", topic, block, exc_info=True) + return None + + +async def generate_gapchoice( + topic: str, block: str, section: str, compact: str | None, + pattern: str, tier: str = "beginner", provider: str = DEFAULT_PROVIDER, + distractor_block: str = "", +) -> dict | None: + """Gap text with choices: sentence with ___ + 4 terms, exactly 1 correct — at the tier's level. + → {sentence, options:[{text,correct}]} · None on error. + distractor_block: optional typical misconceptions (from the facts hurdles) as a distractor source.""" + try: + section_block, compact_block = _section_blocks(section, compact) + return await _gen_call( + "Block-Gapchoice", "guide", _gapchoice_schema, provider, lane="batch", + topic=topic, block=block, section_block=section_block, + compact_block=compact_block, pattern=pattern, tier_block=_tier_block(tier), + distractor_block=distractor_block, + ) + except Exception: + log.warning("[%s] Gap-text choice failed (%s)", topic, block, exc_info=True) + return None + + +def _gap_schema(data) -> dict | None: + """{"sentence": str (with ___), "solution": str, "alternatives": [str]} → validated · else None.""" + if not isinstance(data, dict): + return None + sentence = str(data.get("sentence", "")).strip() + solution = str(data.get("solution", "")).strip() + alt = data.get("alternatives", []) + if not sentence or "___" not in sentence or not solution: + return None + alternatives = [str(a).strip() for a in alt if isinstance(a, str) and str(a).strip()] if isinstance(alt, list) else [] + return {"sentence": sentence, "solution": solution, "alternatives": alternatives} + + +async def generate_gaptext( + topic: str, block: str, section: str, compact: str | None, + pattern: str, tier: str = "beginner", provider: str = DEFAULT_PROVIDER, +) -> dict | None: + """From a pattern, a gap-text task (sentence with ___, solution, synonyms), at the + tier's level. → {sentence, solution, alternatives} · None on error.""" + try: + section_block, compact_block = _section_blocks(section, compact) + return await _gen_call( + "Block-Gaptext", "guide", _gap_schema, provider, lane="batch", + topic=topic, block=block, section_block=section_block, + compact_block=compact_block, pattern=pattern, tier_block=_tier_block(tier), + ) + except Exception: + log.warning("[%s] Gap-text question failed (%s)", topic, block, exc_info=True) + return None + + +def _norm_term(t: str) -> str: + return re.sub(r"[^\wäöüß]", "", str(t or "").lower()) + + +def _correct_schema(data) -> dict | None: + if not isinstance(data, dict) or not isinstance(data.get("correct"), bool): + return None + return {"correct": data["correct"]} + + +async def check_gaptext( + topic: str, block: str, sentence: str, solution: str, alternatives: list[str], + input: str, provider: str = DEFAULT_PROVIDER, +) -> bool: + """Check a gap-text answer: first a normalized comparison (solution + synonyms), + otherwise 1 AI call for synonym tolerance. Fail-open to CORRECT only on an exact match.""" + if not input.strip(): + return False + norm = _norm_term(input) + if norm and norm in {_norm_term(solution), *(_norm_term(a) for a in alternatives)}: + return True + data = await _gen_call( + "Block-Gaptext-Exam", "fast", _correct_schema, provider, + topic=topic, block=block, sentence=sentence, solution=solution, + alternatives=", ".join(alternatives) or "(none)", input=input, + ) + return bool(data and data["correct"]) + + +async def exam_rating_fast( + topic: str, block: str, section: str, compact: str | None, + question: str, messages: list[dict], provider: str = DEFAULT_PROVIDER, +) -> dict | None: + """Action 'answer' (Agent 1, fast): evaluator only, no critic. → {feedback, tier}.""" + try: + section_block, compact_block = _section_blocks(section, compact) + transcript = _transcript(messages) if messages else "(empty)" + return await _gen_call( + "Block-Rating", "judge", _rating_schema, provider, + topic=topic, block=block, section_block=section_block, compact_block=compact_block, + question=question.strip() or "(no question provided)", transcript=transcript, + reason_block="(none)", kritik_block="(none)", + ) + except Exception: + log.warning("[%s] Fast rating failed (%s)", topic, block, exc_info=True) + return None + + +async def exam_rating( + topic: str, block: str, section: str, compact: str | None, + question: str, messages: list[dict], provider: str = DEFAULT_PROVIDER, + role: str = "judge", reason: str = "", +) -> dict | None: + """Action 'answer_check' (Agent 2, thorough): evaluator + critic. → {feedback, tier}. + + `role` = "guide" for "thorough check" (strong model). `reason` = optional + learner dissatisfaction with an earlier rating. + """ + try: + section_block, compact_block = _section_blocks(section, compact) + transcript = _transcript(messages) if messages else "(empty)" + reason_block = reason.strip() or "(none)" + return await _rating_with_critique( + topic, block, section_block, compact_block, + question.strip() or "(no question provided)", transcript, reason_block, provider, role, + ) + except Exception: + log.warning("[%s] Rating failed (%s)", topic, block, exc_info=True) + return None + + +async def block_discussion( + topic: str, block: str, section: str, compact: str | None, + question: str, last_rating: str | None, messages: list[dict], provider: str = DEFAULT_PROVIDER, +) -> str | None: + """Action 'discussion': tutor explains/discusses the question or a rating. + + No rating, no critic — here the human is the examiner. None on error. + """ + try: + section_block, compact_block = _section_blocks(section, compact) + prompt = _prompt( + "Block-Exam-Discussion", + topic=topic, block=block, + section_block=section_block, compact_block=compact_block, + question=question.strip() or "(no question provided)", + last_rating_block=(last_rating or "").strip() or "(none yet)", + transcript=_transcript(messages) if messages else "(empty)", + ) + returncode, stdout, _ = await run_agent( + "examdiscussion-" + str(uuid.uuid4()), prompt, CHAT_TIMEOUT, + provider=provider, role="fast", capabilities="none", lane="interactive", + ) + if returncode != 0: + return None + return stdout.strip() or None + except Exception: + log.warning("[%s] Exam discussion failed (%s)", topic, block, exc_info=True) + return None + + +async def create_block_element(topic: str, block: str, section: str, provider: str = DEFAULT_PROVIDER) -> None: + """Background task after completion: register the block as an element. + + Dedup via normalized title — if an element for the block already exists, + nothing happens. Must never raise an exception to the outside. + """ + try: + existing = {_norm_title(e["title"]) for e in await list_elements(topic)} + if _norm_title(block) in existing: + return + fields = await generate_element(topic, hint=block, provider=provider, extra_context=section) + if _norm_title(fields["title"]) in existing: + return + now = datetime.now(timezone.utc).isoformat() + await create_element({"id": str(uuid.uuid4()), "topic": topic, **fields, "created_at": now, "updated_at": now}) + log.info("[%s] Block registered as element: %s", topic, fields["title"]) + except Exception: + log.warning("[%s] Element registration after exam failed (%s)", topic, block, exc_info=True) diff --git a/backend/lernen.py b/backend/lernen.py deleted file mode 100644 index af94ced..0000000 --- a/backend/lernen.py +++ /dev/null @@ -1,657 +0,0 @@ -"""Baustein-Lernen: Vertiefung, Bausteinchat und Prüfung zu einzelnen Guide-Sections. - -Alle Aufrufe sind interaktiv (stdout-Antwort, lane "interactive") und stateless — -der Chat-/Prüfungs-Verlauf kommt vom Frontend, persistiert wird nur der -Prüfungs-Zähler (DB) und die Vertiefung (DB). -""" - -import logging -import random -import re -import uuid -from datetime import datetime, timezone - -from agents import run_agent -from config import DEFAULT_PROVIDER -from database import create_element, list_elements, get_baustein_huerden -from elements import generate_element -from jsonio import parse_json_text as _parse_json_text -from pipeline import _prompt, _probleme_schema -from textkit import _norm_titel - -log = logging.getLogger("creator.lernen") - -# Lernstufen je Baustein — relativ zum cap (Floor in % des Maximal-Scores): -# grün=Anfänger 20% · blau=Fortgeschritten 40% · lila=Experte 60% · gold=Meister 100%. -# Prüfungsform ist immer zufällig (5 Formen); der cap skaliert mit der Stoffmenge. -STUFEN = (("anfaenger", 0.2), ("fortgeschritten", 0.4), ("experte", 0.6), ("meister", 1.0)) - - -PUNKT_BASIS = 25 # Punkte je Subbaustein. Meister-cap = (alle Subs) × 25. - - -def _ebenen(n_je_ebene: dict[int, int]) -> list[int]: - return [n_je_ebene.get(k, 0) for k in (1, 2, 3, 4)] - - -def schwellen(n_je_ebene: dict[int, int]) -> list[int]: - """Kumulative Ebenen-Schwellen [S_1, S_2, S_3, S_4] = (n_1+…+n_k) × 25. - S_k ist der Score, ab dem Ebene k+1 (E→M→S→F) freigeschaltet ist; S_4 = cap_final.""" - out, akk = [], 0 - for n in _ebenen(n_je_ebene): - akk += n - out.append(akk * PUNKT_BASIS) - return out - - -def cap_final(n_je_ebene: dict[int, int]) -> int: - """Maximal-Score (Meister) = alle Subbausteine × 25.""" - return schwellen(n_je_ebene)[-1] - - -def freie_ebene(score: int, n_je_ebene: dict[int, int]) -> int: - """Höchste freigeschaltete Sub-Ebene 1–4. Ebene k+1 frei, sobald score ≥ S_k. - Leere Ebenen (n_k=0) werden automatisch übersprungen (S_k == S_{k-1}).""" - s = schwellen(n_je_ebene) - e = 1 - for k in range(3): # S_1..S_3 schalten Ebene 2..4 frei - if score >= s[k]: - e = k + 2 - return e - - -def cap_aktuell(score: int, n_je_ebene: dict[int, int]) -> int: - """Erreichbarer cap der aktuell freigeschalteten Ebene = freigeschaltete Subs × 25.""" - return schwellen(n_je_ebene)[freie_ebene(score, n_je_ebene) - 1] - - -def _schwelle(p: float, cap: int) -> int: - return round(p * cap) - - -def stufe_aus_score(score: int, cap_final_wert: int) -> str | None: - """Höchste erreichte Lernstufe (None unter 20 %), relativ zum cap_final.""" - erreicht = None - for key, p in STUFEN: - if score >= _schwelle(p, cap_final_wert): - erreicht = key - return erreicht - - -def progressiver_malus(basis: int, cap_akt: int) -> int: - """Fehler-Strafe nach Fortschritt in der aktuellen Ebene (gegen cap_aktuell): - ≤25 %→−5 · ≤50 %→−10 · ≤75 %→−15 · >75 %→−20.""" - pct = (basis / cap_akt) if cap_akt else 0.0 - if pct <= 0.25: - return -5 - if pct <= 0.5: - return -10 - if pct <= 0.75: - return -15 - return -20 -CHAT_TIMEOUT = 240 -PRUEFUNG_TIMEOUT = 120 # kurze JSON-Turns; deckelt die Serien-Latenz pro Prüfungs-Schritt -GRUENDLICH_TIMEOUT = 600 # „Gründlich prüfen": starkes Modell (role guide) braucht länger -KRITIK_MAX_RUNDEN = 2 # Generator → Kritiker → ggf. Neu, höchstens so oft - -# Fragetypen für Active Recall — pro Frage einer, zufällig gewählt. Schafft Vielfalt. -FRAGETYPEN = { - "abruf": "Free Recall: Lass den Lerner die Kernidee frei aus dem Kopf erklären (offene Verständnisfrage).", - "punkt": "Cued Recall: Frag gezielt EIN konkretes Detail oder eine Abgrenzung ab.", - "warum": "Warum-Frage: Frag nach dem Grund/Mechanismus — warum funktioniert oder gilt das so?", - "anwendung": "Anwendung: Lass das Konzept auf EIN kurzes, neues Beispiel/Szenario anwenden.", - "pruefen": "Bei Code-/Tool-Themen: kleinen Schnipsel zeigen — Output vorhersagen ODER den Fehler finden. Kein Code-Thema → stattdessen eine Anwendungsfrage.", -} - - -# Antwort-Niveau → Basis-Punkte (neue 25er-Skala). „kaum" = −1 ist nur das Signal für den -# progressiven Malus (echter Wert kommt aus progressiver_malus). Positive Werte werden bei -# Streak hochmoduliert und auf [10, 40] geklemmt. -NIVEAUS = { - "unbeantwortbar": 0, # Frage selbst kaputt → keine Änderung - "kaum": -1, # < 25 % richtig → Malus - "teilweise": 0, # 25–49 % → neutral - "solide": 16, # 50–74 % - "stark": 24, # 75–99 % (Quiz/Lücke-Treffer) - "komplett": 30, # 100 % (nur freies Erklären erreichbar) -} - -# Reihenfolge schwach→stark (für den Nachfrage-Deckel). -_NIVEAU_RANG = ("kaum", "teilweise", "solide", "stark", "komplett") - - -def deckel_nachfrage(niveau: str, nachgefragt: bool) -> str: - """Mit Nachfrage (Hinweis erhalten) höchstens „solide" — kein Voll-Score erschummeln.""" - if nachgefragt and niveau in ("stark", "komplett"): - return "solide" - return niveau - - -def streak_punkte(basis_delta: int, streak_basis: int) -> int: - """Positives Basis-Delta mit Streak hochmodulieren, auf [10, 40] geklemmt.""" - faktor = min(1.33, 1 + 0.066 * min(streak_basis, 5)) - return max(10, min(40, round(basis_delta * faktor))) - - -def punkte_delta(niveau: str, streak_basis: int, basis: int, cap_akt: int) -> tuple[int, int]: - """Antwort-Niveau → (Punkt-Delta, neue Streak). Positiv: streak-moduliert, Streak +1. - Neutral (0): keine Änderung, Streak bleibt. Negativ: progressiver Malus, Streak-Reset 0.""" - basis_delta = NIVEAUS.get(niveau, 0) - if basis_delta > 0: - return streak_punkte(basis_delta, streak_basis), streak_basis + 1 - if basis_delta == 0: - return 0, streak_basis - return progressiver_malus(basis, cap_akt), 0 - - -def score_berechnen(basis: int, delta: int, floor: int, cap_akt: int, cap_fin: int) -> int: - """Neuer Score · driftfrei aus der Basis. Klemmt nach oben gegen `cap_akt` (Deckel der - aktuell freigeschalteten Ebene) und nach unten gegen `floor`. Eingefroren NUR am - absoluten Maximum (`basis ≥ cap_fin`) — sonst würde an jeder Ebenen-Schwelle blockiert.""" - if basis >= cap_fin: - return basis - return max(floor, min(cap_akt, basis + delta)) - - -def floor_aus_score(basis: int, cap_fin: int, s_schwellen: list[int]) -> int: - """Untergrenze (kein Rückfall): höchste erreichte Lernstufen-Schwelle (über cap_final) - UND höchste erreichte Ebenen-Freischalt-Schwelle S_k. max beider Achsen.""" - floor = 0 - for _, p in STUFEN: - s = _schwelle(p, cap_fin) - if basis >= s: - floor = max(floor, s) - for s in s_schwellen: - if basis >= s: - floor = max(floor, s) - return floor - - -def _transcript(messages: list[dict]) -> str: - return "\n".join( - f"{'Nutzer' if m.get('role') == 'user' else 'Assistent'}: {m.get('content', '')}" - for m in messages - ) or "(leer)" - - -async def baustein_chat(topic: str, baustein: str, section: str, kompakt: str | None, messages: list[dict], provider: str = DEFAULT_PROVIDER) -> str: - try: - prompt = _prompt( - "Baustein-Chat", - topic=topic, baustein=baustein, - section_block=section.strip() or "(keine Guide-Fassung übergeben)", - kompakt_block=(kompakt or "").strip() or "(keine)", - transcript=_transcript(messages), - ) - returncode, stdout, _ = await run_agent( - "bausteinchat-" + str(uuid.uuid4()), prompt, CHAT_TIMEOUT, - provider=provider, role="fast", capabilities="none", lane="interactive", - ) - if returncode != 0: - return "Entschuldigung, das hat nicht geklappt. Bitte versuche es erneut." - reply = stdout.strip() - return reply or "Entschuldigung, ich habe keine Antwort erhalten." - except Exception: - log.warning("[%s] Baustein-Chat fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return "Entschuldigung, das hat nicht geklappt. Bitte versuche es erneut." - - -def _frage_schema(data) -> dict | None: - """{"frage": str} · sonst None.""" - if not isinstance(data, dict): - return None - frage = str(data.get("frage", "")).strip() - return {"frage": frage} if frage else None - - -def _bewertung_schema(data) -> dict | None: - """{"feedback": str, "niveau": ∈ NIVEAUS} · sonst None.""" - if not isinstance(data, dict): - return None - feedback = str(data.get("feedback", "")).strip() - niveau = data.get("niveau") - if not feedback or niveau not in NIVEAUS: - return None - return {"feedback": feedback, "niveau": niveau} - - -async def _gen_call(name: str, role: str, schema, provider: str, timeout: int = PRUEFUNG_TIMEOUT, lane: str = "interactive", **kwargs) -> dict | None: - """Generator-Agent: Template füllen, laufen lassen, per schema parsen · None bei Fehler. - lane="batch" für Hintergrund (Vorladen, Genau-Bewertung) → eigene Slot-Schlange.""" - returncode, stdout, _ = await run_agent( - name.lower() + "-" + str(uuid.uuid4()), _prompt(name, **kwargs), timeout, - provider=provider, role=role, capabilities="none", lane=lane, - ) - return schema(_parse_json_text(stdout)) if returncode == 0 else None - - -async def _kritik_call(name: str, provider: str, role: str = "judge", timeout: int = PRUEFUNG_TIMEOUT, lane: str = "interactive", **kwargs) -> list[str]: - """Kritiker-Agent (Default role judge): leere Liste = in Ordnung. Fail-open: Ausfall des - Kritikers darf den Turn nicht blockieren, also dann ebenfalls leere Liste.""" - returncode, stdout, _ = await run_agent( - name.lower() + "-" + str(uuid.uuid4()), _prompt(name, **kwargs), timeout, - provider=provider, role=role, capabilities="none", lane=lane, - ) - if returncode != 0: - return [] - return _probleme_schema(_parse_json_text(stdout)) or [] - - -def _kritik_block(vorversion: str, probleme: list[str]) -> str: - punkte = "\n".join(f"- {p}" for p in probleme) - return ( - f"Deine vorige Fassung war:\n«{vorversion}»\n\n" - f"Der Prüfer bemängelt:\n{punkte}\n\nBehebe diese Punkte." - ) - - -def _bewertung_text(bew: dict) -> str: - return f"Niveau: {bew['niveau']}\nFeedback: {bew['feedback']}" - - -# Deterministischer Guard gegen Doppelfragen — der KI-Kritiker übersieht „…, und welchen…". -_FRAGEWORT = r"(was|welche[rsnm]?|wie|wieso|warum|wofür|wozu|wann|wo|wer|wem|wen|nenne)" -_DOPPEL_RE = re.compile(r"[,;]?\s+(und|sowie|außerdem|bzw\.?)\s+" + _FRAGEWORT + r"\b", re.IGNORECASE) - - -def _doppelfrage_mangel(frage: str) -> str | None: - """Erkennt zwei verkettete Fragen. None = ok. Flaggt NUR 'und/sowie' + Fragewort.""" - if frage.count("?") > 1: - return "Mehr als ein Fragezeichen — stelle GENAU EINE Frage." - if _DOPPEL_RE.search(frage): - return "Zwei Fragen mit 'und'/'sowie' verkettet — stelle GENAU EINE Frage, eine Sache." - return None - - -async def _frage_mit_kritik( - topic: str, baustein: str, section_block: str, kompakt_block: str, - transcript: str, vermeide_block: str, typ_block: str, fokus_block: str, - niveau_block: str, provider: str, -) -> str | None: - """Frage generieren, vom Kritiker prüfen lassen, bei Mängeln neu (max KRITIK_MAX_RUNDEN).""" - kritik_block = "(keine)" - frage = None - for _ in range(KRITIK_MAX_RUNDEN): - data = await _gen_call( - "Baustein-Frage", "guide", _frage_schema, provider, lane="batch", - topic=topic, baustein=baustein, section_block=section_block, - kompakt_block=kompakt_block, transcript=transcript, vermeide_block=vermeide_block, - typ_block=typ_block, fokus_block=fokus_block, niveau_block=niveau_block, kritik_block=kritik_block, - ) - if data is None: - return None - frage = data["frage"] - probleme = await _kritik_call( - "Baustein-Frage-Kritik", provider, role="guide", lane="batch", # starke KI prüft die Regeln - topic=topic, baustein=baustein, section_block=section_block, - kompakt_block=kompakt_block, transcript=transcript, vermeide_block=vermeide_block, - typ_block=typ_block, fokus_block=fokus_block, frage=frage, - ) - hart = _doppelfrage_mangel(frage) # erzwingt Neugenerierung, auch wenn der KI-Kritiker es übersah - if hart: - probleme = [hart, *(probleme or [])] - if not probleme: - return frage - kritik_block = _kritik_block(frage, probleme) - return frage # best-effort nach der letzten Runde - - -async def _bewertung_mit_kritik( - topic: str, baustein: str, section_block: str, kompakt_block: str, - frage: str, transcript: str, begruendung_block: str, provider: str, role: str = "judge", -) -> dict | None: - """Antwort bewerten (Niveau), vom Kritiker prüfen lassen, bei Fehlurteil neu. - - `frage` ankert die geprüfte Frage; der Dialog (transcript) liefert Antwort + Diskussion. - `begruendung_block` = optionale Unzufriedenheit des Lerners (nur bei „Gründlich prüfen"). - `role` = "judge" (schnell) oder "guide" (gründlich, starkes Modell mit Thinking). - """ - timeout = GRUENDLICH_TIMEOUT if role == "guide" else PRUEFUNG_TIMEOUT - # Gründlich (role guide) = Nutzer wartet → interaktiv. Hintergrund-Genau (judge) → batch. - lane = "interactive" if role == "guide" else "batch" - kritik_block = "(keine)" - bew = None - for _ in range(KRITIK_MAX_RUNDEN): - bew = await _gen_call( - "Baustein-Bewertung", role, _bewertung_schema, provider, timeout, lane=lane, - topic=topic, baustein=baustein, section_block=section_block, - kompakt_block=kompakt_block, frage=frage, transcript=transcript, - begruendung_block=begruendung_block, kritik_block=kritik_block, - ) - if bew is None: - return None - probleme = await _kritik_call( - "Baustein-Bewertung-Kritik", provider, role=role, timeout=timeout, lane=lane, - topic=topic, baustein=baustein, section_block=section_block, - kompakt_block=kompakt_block, frage=frage, transcript=transcript, - bewertung_block=_bewertung_text(bew), - ) - if not probleme: - return bew - kritik_block = _kritik_block(_bewertung_text(bew), probleme) - return bew # best-effort nach der letzten Runde - - -def _bloecke(section: str, kompakt: str | None) -> tuple[str, str]: - return ( - section.strip() or "(keine Guide-Fassung übergeben)", - (kompakt or "").strip() or "(keine)", - ) - - -def _vermeide_block(vermeide: list[str] | None) -> str: - eintraege = [f.strip() for f in (vermeide or []) if f and f.strip()] - return "\n".join(f"- {f}" for f in eintraege) or "(keine)" - - -# Lerner-Niveau (aus dem Score abgeleitet) → Adressaten-Rolle für die Frage. So entsteht die -# Schwierigkeit: nicht „extra schwer machen", sondern „für einen Anfänger/Experten fragen". -# Je Stufe: Adressaten-Rolle + kognitive Anforderung (Bloom) + „frag so"-Cue. Ohne explizite Stufe nimmt -# das Modell den leichten Pfad (bloßer Abruf) — die Cues heben höhere Niveaus auf Anwenden/Analysieren/Transfer. -NIVEAU_ROLLE = { - "anfaenger": "Der Lerner ist ANFÄNGER. Kognitiv: ERINNERN/VERSTEHEN. Frage nach dem Grundverständnis — das Kernkonzept, einfach und direkt.", - "fortgeschritten": "Der Lerner ist FORTGESCHRITTEN. Kognitiv: ANWENDEN. Stelle eine kleine konkrete Situation und lass das Konzept darauf anwenden — frage nicht bloß die Definition ab.", - "experte": "Der Lerner ist EXPERTE. Kognitiv: ANALYSIEREN. Lass abgrenzen/vergleichen, einen Sonderfall einordnen oder eine typische Tücke (Hürde) aufdecken — nicht das Lehrbuch-Wissen abfragen.", - "meister": "Der Lerner ist auf MEISTER-Niveau. Kognitiv: BEWERTEN/TRANSFER. Lass das Konzept auf ein NEUES Problem übertragen, eine Entscheidung begründen oder einen Trade-off abwägen.", -} - - -def _niveau_block(niveau: str | None) -> str: - return NIVEAU_ROLLE.get(niveau or "", NIVEAU_ROLLE["anfaenger"]) - - -async def pruefung_frage( - topic: str, baustein: str, section: str, kompakt: str | None, - messages: list[dict], subbausteine: list[str] | None = None, - vermeide: list[str] | None = None, niveau: str = "anfaenger", provider: str = DEFAULT_PROVIDER, -) -> str | None: - """Aktion 'frage': eine Frage generieren — zufälliger Typ zu einem zufälligen Subbaustein, - in der Adressaten-Rolle des Niveaus, dann Kritiker (sequenziell) · None bei Fehler.""" - try: - section_block, kompakt_block = _bloecke(section, kompakt) - transcript = _transcript(messages) if messages else "(leer)" - typ_block = FRAGETYPEN[random.choice(list(FRAGETYPEN))] - subs = [s for s in (subbausteine or []) if s and s.strip()] - fokus = random.choice(subs) if subs else "" - fokus_block = ( - f"Konzentriere die Frage auf diesen Subbaustein: „{fokus}\"" if fokus - else "(ganzer Baustein — kein bestimmter Subbaustein)" - ) - return await _frage_mit_kritik( - topic, baustein, section_block, kompakt_block, transcript, - _vermeide_block(vermeide), typ_block, fokus_block, _niveau_block(niveau), provider, - ) - except Exception: - log.warning("[%s] Frage fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return None - - -async def pruefung_frage_variante( - topic: str, baustein: str, section: str, kompakt: str | None, - muster: str, niveau: str = "anfaenger", provider: str = DEFAULT_PROVIDER, -) -> str | None: - """Aktion 'frage' mit Muster: aus einem vordefinierten Muster eine konkrete Frage in der - Adressaten-Rolle des Niveaus formulieren. Kein Kritiker (Muster ist build-geprüft). - Stil-Guard bleibt als billiger Schutz gegen Doppelfragen · None bei Fehler.""" - try: - section_block, kompakt_block = _bloecke(section, kompakt) - data = await _gen_call( - "Baustein-Frage-Variante", "guide", _frage_schema, provider, lane="batch", - topic=topic, baustein=baustein, section_block=section_block, - kompakt_block=kompakt_block, muster=muster, niveau_block=_niveau_block(niveau), - ) - if data is None: - return None - return data["frage"] - except Exception: - log.warning("[%s] Frage-Variante fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return None - - -def _optionen_schema(opts) -> list[dict] | None: - """[{text, korrekt}]×4 → validierte Liste · sonst None.""" - if not isinstance(opts, list) or len(opts) != 4: - return None - out = [] - for o in opts: - if not isinstance(o, dict): - return None - text = str(o.get("text", "")).strip() - korrekt = o.get("korrekt") - if not text or not isinstance(korrekt, bool): - return None - out.append({"text": text, "korrekt": korrekt}) - return out - - -def _quiz_schema(data) -> dict | None: - """{"frage": str, "optionen": [{text, korrekt}]×4} → validiert · sonst None. - Single-Choice: genau 1 richtig. Die Schwierigkeit steckt im Niveau, nicht in der Anzahl.""" - if not isinstance(data, dict): - return None - frage = str(data.get("frage", "")).strip() - out = _optionen_schema(data.get("optionen")) - if not frage or out is None: - return None - if sum(o["korrekt"] for o in out) != 1: - return None - return {"frage": frage, "optionen": out} - - -def _lueckwahl_schema(data) -> dict | None: - """{"satz": str (mit ___), "optionen": [{text, korrekt}]×4} → genau 1 korrekt · sonst None.""" - if not isinstance(data, dict): - return None - satz = str(data.get("satz", "")).strip() - out = _optionen_schema(data.get("optionen")) - if not satz or "___" not in satz or out is None or sum(o["korrekt"] for o in out) != 1: - return None - return {"satz": satz, "optionen": out} - - -async def huerden_distraktor_block(topic: str, baustein: str) -> str: - """Typische Irrtümer (Fakten-Hürden) des Bausteins als Distraktor-Quelle für Quiz/Lückenwahl. - Leer, wenn keine vorhanden (Altbestand) → der Prompt-Platzhalter verschwindet rückstandslos.""" - try: - huerden = await get_baustein_huerden(topic, _norm_titel(baustein)) - except Exception: - return "" - if not huerden: - return "" - zeilen = "\n".join(f"- {h}" for h in huerden[:8]) - return ("TYPISCHE IRRTÜMER zu diesem Baustein (nutze sie als Distraktoren, wenn sie zur Frage passen):\n" - + zeilen + "\n") - - -async def quiz_generieren( - topic: str, baustein: str, section: str, kompakt: str | None, - muster: str, niveau: str = "anfaenger", provider: str = DEFAULT_PROVIDER, - distraktor_block: str = "", -) -> dict | None: - """Aus einem Muster eine Single-Choice-Frage (genau 1 richtig), im Anspruch des Niveaus. - Starkes Modell (role guide) für korrekte Flags. → {frage, optionen} · None bei Fehler. - distraktor_block: optionale typische Irrtümer (aus den Fakten-Hürden) als Distraktor-Quelle.""" - try: - section_block, kompakt_block = _bloecke(section, kompakt) - return await _gen_call( - "Baustein-Quiz", "guide", _quiz_schema, provider, lane="batch", - topic=topic, baustein=baustein, section_block=section_block, - kompakt_block=kompakt_block, muster=muster, niveau_block=_niveau_block(niveau), - distraktor_block=distraktor_block, - ) - except Exception: - log.warning("[%s] Quiz-Frage fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return None - - -async def lueckwahl_generieren( - topic: str, baustein: str, section: str, kompakt: str | None, - muster: str, niveau: str = "anfaenger", provider: str = DEFAULT_PROVIDER, - distraktor_block: str = "", -) -> dict | None: - """Lückentext mit Auswahl: Satz mit ___ + 4 Begriffe, genau 1 richtig — im Anspruch des Niveaus. - → {satz, optionen:[{text,korrekt}]} · None bei Fehler. - distraktor_block: optionale typische Irrtümer (aus den Fakten-Hürden) als Distraktor-Quelle.""" - try: - section_block, kompakt_block = _bloecke(section, kompakt) - return await _gen_call( - "Baustein-Lueckwahl", "guide", _lueckwahl_schema, provider, lane="batch", - topic=topic, baustein=baustein, section_block=section_block, - kompakt_block=kompakt_block, muster=muster, niveau_block=_niveau_block(niveau), - distraktor_block=distraktor_block, - ) - except Exception: - log.warning("[%s] Lückentext-Auswahl fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return None - - -def _lueck_schema(data) -> dict | None: - """{"satz": str (mit ___), "loesung": str, "alternativen": [str]} → validiert · sonst None.""" - if not isinstance(data, dict): - return None - satz = str(data.get("satz", "")).strip() - loesung = str(data.get("loesung", "")).strip() - alt = data.get("alternativen", []) - if not satz or "___" not in satz or not loesung: - return None - alternativen = [str(a).strip() for a in alt if isinstance(a, str) and str(a).strip()] if isinstance(alt, list) else [] - return {"satz": satz, "loesung": loesung, "alternativen": alternativen} - - -async def lueckentext_generieren( - topic: str, baustein: str, section: str, kompakt: str | None, - muster: str, niveau: str = "anfaenger", provider: str = DEFAULT_PROVIDER, -) -> dict | None: - """Aus einem Muster eine Lückentext-Aufgabe (Satz mit ___, Lösung, Synonyme), im Anspruch - des Niveaus. → {satz, loesung, alternativen} · None bei Fehler.""" - try: - section_block, kompakt_block = _bloecke(section, kompakt) - return await _gen_call( - "Baustein-Lueckentext", "guide", _lueck_schema, provider, lane="batch", - topic=topic, baustein=baustein, section_block=section_block, - kompakt_block=kompakt_block, muster=muster, niveau_block=_niveau_block(niveau), - ) - except Exception: - log.warning("[%s] Lückentext-Frage fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return None - - -def _norm_begriff(t: str) -> str: - return re.sub(r"[^\wäöüß]", "", str(t or "").lower()) - - -def _richtig_schema(data) -> dict | None: - if not isinstance(data, dict) or not isinstance(data.get("richtig"), bool): - return None - return {"richtig": data["richtig"]} - - -async def lueckentext_pruefen( - topic: str, baustein: str, satz: str, loesung: str, alternativen: list[str], - eingabe: str, provider: str = DEFAULT_PROVIDER, -) -> bool: - """Lückentext-Antwort prüfen: erst normalisierter Vergleich (Lösung + Synonyme), - sonst 1 KI-Call für Synonym-Toleranz. Fail-open zu RICHTIG nur bei exaktem Match.""" - if not eingabe.strip(): - return False - norm = _norm_begriff(eingabe) - if norm and norm in {_norm_begriff(loesung), *(_norm_begriff(a) for a in alternativen)}: - return True - data = await _gen_call( - "Baustein-Lueckentext-Pruefung", "fast", _richtig_schema, provider, - topic=topic, baustein=baustein, satz=satz, loesung=loesung, - alternativen=", ".join(alternativen) or "(keine)", eingabe=eingabe, - ) - return bool(data and data["richtig"]) - - -async def pruefung_bewertung_schnell( - topic: str, baustein: str, section: str, kompakt: str | None, - frage: str, messages: list[dict], provider: str = DEFAULT_PROVIDER, -) -> dict | None: - """Aktion 'antwort' (Agent 1, schnell): nur Evaluator, kein Kritiker. → {feedback, niveau}.""" - try: - section_block, kompakt_block = _bloecke(section, kompakt) - transcript = _transcript(messages) if messages else "(leer)" - return await _gen_call( - "Baustein-Bewertung", "judge", _bewertung_schema, provider, - topic=topic, baustein=baustein, section_block=section_block, kompakt_block=kompakt_block, - frage=frage.strip() or "(keine Frage übergeben)", transcript=transcript, - begruendung_block="(keine)", kritik_block="(keine)", - ) - except Exception: - log.warning("[%s] Schnell-Bewertung fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return None - - -async def pruefung_bewertung( - topic: str, baustein: str, section: str, kompakt: str | None, - frage: str, messages: list[dict], provider: str = DEFAULT_PROVIDER, - role: str = "judge", begruendung: str = "", -) -> dict | None: - """Aktion 'antwort_pruefen' (Agent 2, genau): Evaluator + Kritiker. → {feedback, niveau}. - - `role` = "guide" für „Gründlich prüfen" (starkes Modell). `begruendung` = optionale - Unzufriedenheit des Lerners mit einer früheren Bewertung. - """ - try: - section_block, kompakt_block = _bloecke(section, kompakt) - transcript = _transcript(messages) if messages else "(leer)" - begruendung_block = begruendung.strip() or "(keine)" - return await _bewertung_mit_kritik( - topic, baustein, section_block, kompakt_block, - frage.strip() or "(keine Frage übergeben)", transcript, begruendung_block, provider, role, - ) - except Exception: - log.warning("[%s] Bewertung fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return None - - -async def baustein_diskussion( - topic: str, baustein: str, section: str, kompakt: str | None, - frage: str, letzte_bewertung: str | None, messages: list[dict], provider: str = DEFAULT_PROVIDER, -) -> str | None: - """Aktion 'diskussion': Tutor erklärt/diskutiert die Frage oder eine Bewertung. - - Kein Bewerten, kein Kritiker — hier ist der Mensch der Prüfer. None bei Fehler. - """ - try: - section_block, kompakt_block = _bloecke(section, kompakt) - prompt = _prompt( - "Baustein-Pruefung-Diskussion", - topic=topic, baustein=baustein, - section_block=section_block, kompakt_block=kompakt_block, - frage=frage.strip() or "(keine Frage übergeben)", - letzte_bewertung_block=(letzte_bewertung or "").strip() or "(noch keine)", - transcript=_transcript(messages) if messages else "(leer)", - ) - returncode, stdout, _ = await run_agent( - "pruefungdiskussion-" + str(uuid.uuid4()), prompt, CHAT_TIMEOUT, - provider=provider, role="fast", capabilities="none", lane="interactive", - ) - if returncode != 0: - return None - return stdout.strip() or None - except Exception: - log.warning("[%s] Prüfungs-Diskussion fehlgeschlagen (%s)", topic, baustein, exc_info=True) - return None - - -async def baustein_element_anlegen(topic: str, baustein: str, section: str, provider: str = DEFAULT_PROVIDER) -> None: - """Hintergrund-Task nach dem Absolvieren: Baustein als Element anlegen. - - Dedup über normalisierte Titel — existiert schon ein Element zum Baustein, - passiert nichts. Darf nie eine Exception nach außen werfen. - """ - try: - vorhanden = {_norm_titel(e["title"]) for e in await list_elements(topic)} - if _norm_titel(baustein) in vorhanden: - return - fields = await generate_element(topic, hint=baustein, provider=provider, extra_context=section) - if _norm_titel(fields["title"]) in vorhanden: - return - now = datetime.now(timezone.utc).isoformat() - await create_element({"id": str(uuid.uuid4()), "topic": topic, **fields, "created_at": now, "updated_at": now}) - log.info("[%s] Baustein als Element angelegt: %s", topic, fields["title"]) - except Exception: - log.warning("[%s] Element-Anlage nach Prüfung fehlgeschlagen (%s)", topic, baustein, exc_info=True) diff --git a/backend/lesbarkeit.py b/backend/lesbarkeit.py deleted file mode 100644 index b13db48..0000000 --- a/backend/lesbarkeit.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Deterministisches Lesbarkeits-Gate für Guide-Sections. - -Ein kleines deutsches Komplexitäts-Modell (DistilBERT, GermEval 2022, Skala 1–7) -bewertet die Verständlichkeit der Fließtext-Prosa. guide.py meldet zu schwere -Sections in die bestehende Lese-Prüfungs-/Überarbeitungs-Schleife — kein Prompt, -kein Raten. - -Optional: fehlen `transformers`/`torch` oder lädt das Modell nicht, ist das Gate -stumm deaktiviert (das Backend läuft unverändert weiter). CPU genügt; der Aufrufer -wrappt die Bewertung in `asyncio.to_thread` (blockierende Modell-Inferenz). -""" - -import logging -import re - -from config import ( - LESBARKEIT_AKTIV, LESBARKEIT_HART, LESBARKEIT_HART_ANTEIL, LESBARKEIT_MAX, LESBARKEIT_MODELL, -) - -log = logging.getLogger("creator.lesbarkeit") - -_modell_cache = None # (tokenizer, model, torch) — Singleton -_ladeversuch = False # schon versucht zu laden? - -# Markup raus → reiner Fließtext (Code zählt nicht zur Lesbarkeit). -_CODE_FENCE = re.compile(r"```.*?```", re.DOTALL) -_COMMENT = re.compile(r"", re.DOTALL) -_INLINE_CODE = re.compile(r"`[^`]*`") -_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)") -_MD_MARK = re.compile(r"^[ \t]*([#>]+|[-*+]\s)|[*_~|]", re.MULTILINE) -_WS = re.compile(r"\s+") -_SATZ = re.compile(r"(?<=[.!?])\s+") - - -def _modell(): - """Lädt das Modell einmalig. None = Gate aus (deaktiviert oder Lade-Fehler).""" - global _modell_cache, _ladeversuch - if _ladeversuch: - return _modell_cache - _ladeversuch = True - if not LESBARKEIT_AKTIV: - return None - try: - import torch - from transformers import AutoModelForSequenceClassification, AutoTokenizer - tok = AutoTokenizer.from_pretrained(LESBARKEIT_MODELL) - model = AutoModelForSequenceClassification.from_pretrained(LESBARKEIT_MODELL) - model.eval() - _modell_cache = (tok, model, torch) - log.info("Lesbarkeits-Modell geladen: %s (num_labels=%d)", LESBARKEIT_MODELL, model.config.num_labels) - except Exception as e: - log.warning("Lesbarkeits-Gate deaktiviert (Modell nicht ladbar): %s", e) - _modell_cache = None - return _modell_cache - - -def _prosa(md: str) -> str: - """Markdown/Code strippen → reiner Fließtext für die Bewertung.""" - t = _CODE_FENCE.sub(" ", md) - t = _COMMENT.sub(" ", t) - t = _INLINE_CODE.sub(" ", t) - t = _LINK.sub(r"\1", t) - t = _MD_MARK.sub(" ", t) - return _WS.sub(" ", t).strip() - - -def _saetze(text: str) -> list[str]: - """Fließtext in Sätze splitten; sehr kurze Fragmente verwerfen.""" - return [s.strip() for s in _SATZ.split(text) if len(s.strip()) >= 15] - - -def _scores(saetze: list[str]) -> list[float]: - """Komplexität je Satz (1–7). Regression (num_labels=1) oder Erwartungswert über Klassen.""" - tok, model, torch = _modell_cache - werte: list[float] = [] - n = model.config.num_labels - for i in range(0, len(saetze), 16): - batch = saetze[i:i + 16] - enc = tok(batch, return_tensors="pt", truncation=True, max_length=256, padding=True) - with torch.no_grad(): - logits = model(**enc).logits - if n == 1: - vals = logits.reshape(-1).tolist() - else: - probs = torch.softmax(logits, dim=-1) - stufen = torch.arange(1, n + 1, dtype=probs.dtype) - vals = (probs * stufen).sum(-1).reshape(-1).tolist() - werte.extend(vals) - return werte - - -def bewerte_sections(md_by_num: dict[int, str]) -> dict[int, str]: - """{num: section_md} → {num: Hinweis} nur für zu schwere Sections. - - Leeres dict, wenn das Gate aus ist. Blockierend (CPU) — in to_thread aufrufen. - """ - if _modell() is None: - return {} - out: dict[int, str] = {} - for num, md in md_by_num.items(): - saetze = _saetze(_prosa(md or "")) - if len(saetze) < 2: # fast nur Code / zu kurz → überspringen - continue - werte = _scores(saetze) - if not werte: - continue - schnitt = sum(werte) / len(werte) - hart = sum(1 for w in werte if w > LESBARKEIT_HART) / len(werte) - # Zu schwer = hoher Schnitt ODER zu viele harte Einzelsätze (Ausreißer-Nester). - if schnitt > LESBARKEIT_MAX or hart >= LESBARKEIT_HART_ANTEIL: - out[num] = ( - f"Zu schwer lesbar (Ø {schnitt:.1f}/7, {hart * 100:.0f}% harte Sätze): " - "kürzere Sätze, einfachere Wörter, weniger Schachtelsätze, mehr Beispiele." - ) - return out diff --git a/backend/logsetup.py b/backend/logsetup.py index 3288303..c306665 100644 --- a/backend/logsetup.py +++ b/backend/logsetup.py @@ -1,4 +1,4 @@ -"""Zentrales Logging-Setup — einmal in main.py aufrufen, bevor die App entsteht.""" +"""Central logging setup — call once in main.py before the app is created.""" import logging import os diff --git a/backend/main.py b/backend/main.py index c3e2ff8..08e33fc 100644 --- a/backend/main.py +++ b/backend/main.py @@ -16,7 +16,7 @@ from routes import router @asynccontextmanager async def lifespan(app: FastAPI): - (STORAGE_DIR / "themen").mkdir(parents=True, exist_ok=True) + (STORAGE_DIR / "topics").mkdir(parents=True, exist_ok=True) await init_db() await reconcile_guides() yield @@ -24,8 +24,8 @@ async def lifespan(app: FastAPI): class CachedStatic(StaticFiles): - """StaticFiles mit Cache-Control: gehashte Assets dauerhaft (immutable), - index.html nie cachen (verweist immer auf die aktuellen Asset-Hashes).""" + """StaticFiles with Cache-Control: hashed assets forever (immutable), + index.html never cached (it always points at the current asset hashes).""" async def get_response(self, path, scope): resp = await super().get_response(path, scope) if path.startswith("assets/"): @@ -37,7 +37,7 @@ class CachedStatic(StaticFiles): app = FastAPI(title="Creator", lifespan=lifespan) -# gzip für JS/CSS-Bundle + große JSON-Antworten (~1,39 MB JS → ~400 KB). +# gzip for the JS/CSS bundle + large JSON responses (~1.39 MB JS → ~400 KB). app.add_middleware(GZipMiddleware, minimum_size=500) app.include_router(router) diff --git a/backend/models.py b/backend/models.py index 8edce2e..1244cfa 100644 --- a/backend/models.py +++ b/backend/models.py @@ -17,47 +17,47 @@ class GuideCreateRequest(BaseModel): format: FormatType instructions: str = Field(default="", max_length=2000) provider: ProviderType = "claude" - ab_step: int | None = Field(default=None, ge=0, le=4) # Re-Run ab Guide-Schritt (0 Gliederung … 4 Lese-Prüfung); None = voll/Resume + ab_step: int | None = Field(default=None, ge=0, le=4) # re-run from guide step (0 outline … 4 read-exam); None = full/resume class TopicCreateRequest(BaseModel): name: str = Field(min_length=1, max_length=100) -class BausteineCreateRequest(BaseModel): +class BlocksCreateRequest(BaseModel): topic: str = Field(min_length=1, max_length=100) instructions: str = Field(default="", max_length=2000) provider: ProviderType = "claude" source_type: SourceType = "thema" - source_ort: str = Field(default="", max_length=2000) - ab_phase: int | None = Field(default=None, ge=1, le=9) # Re-Run ab grober Phase (Position in _phasen(topic), 1-based; bis 9: …Gliederung/Fragen/Artefakte); None = Resume/Fortsetzen ohne Löschen - ab_step: int | None = Field(default=None, ge=0) # Re-Run ab feinem Teilschritt (0-basierter Index in _bausteine_steps); hat Vorrang vor ab_phase + source_location: str = Field(default="", max_length=2000) + ab_phase: int | None = Field(default=None, ge=1, le=9) # re-run from a coarse phase (position in _phasen(topic), 1-based; up to 9: …outline/questions/artifacts); None = resume/continue without deleting + ab_step: int | None = Field(default=None, ge=0) # re-run from a fine sub-step (0-based index into _blocks_steps); takes precedence over ab_phase -class BausteineResetStepRequest(BaseModel): +class BlocksResetStepRequest(BaseModel): topic: str = Field(min_length=1, max_length=100) - ab_step: int = Field(ge=0) # NUR zurücksetzen ab diesem Teilschritt (kein Neu-Generieren) + ab_step: int = Field(ge=0) # ONLY reset from this sub-step (no regeneration) -class BausteineStep(BaseModel): +class BlocksStep(BaseModel): label: str state: Literal["done", "active", "pending"] -class BausteineFeinStep(BaseModel): +class BlocksFineStep(BaseModel): label: str phase: str = "" state: Literal["done", "active", "pending"] -class BausteineStatusResponse(BaseModel): +class BlocksStatusResponse(BaseModel): ready: bool generating: bool progress: str | None = None error: str | None = None partial: bool = False - steps: list[BausteineStep] = [] - feine_steps: list[BausteineFeinStep] = [] + steps: list[BlocksStep] = [] + feine_steps: list[BlocksFineStep] = [] class ProjectResponse(BaseModel): @@ -66,33 +66,33 @@ class ProjectResponse(BaseModel): class FolderResponse(BaseModel): name: str - ort: str # relativer Pfad ab Repo-Root (z.B. "projects/foo") + location: str # path relative to the repo root (e.g. "projects/foo") -class BausteineQuelleUpdate(BaseModel): +class BlocksSourceUpdate(BaseModel): topic: str = Field(min_length=1, max_length=100) type: SourceType = "thema" - ort: str = Field(default="", max_length=2000) + location: str = Field(default="", max_length=2000) spec: str = Field(default="", max_length=2000) -class BausteineQuelleResponse(BaseModel): +class BlocksSourceResponse(BaseModel): type: SourceType - ort: str + location: str spec: str -class SubbausteinInfo(BaseModel): - titel: str - stufe: Literal["anfaenger", "fortgeschritten", "experte", "einfach", "mittel", "schwer"] - relevanz: Literal["relevant", "rand"] | None = None +class SubblockInfo(BaseModel): + title: str + level: Literal["beginner", "advanced", "expert", "easy", "medium", "hard"] + relevance: Literal["relevant", "peripheral"] | None = None -class BausteinUebersicht(BaseModel): +class BlockOverview(BaseModel): num: int - titel: str - beschreibung: str = "" - subbausteine: list[SubbausteinInfo] = [] + title: str + description: str = "" + subblocks: list[SubblockInfo] = [] class ProviderInfo(BaseModel): @@ -168,7 +168,7 @@ class ElementCheckResponse(BaseModel): class ElementStyleChange(BaseModel): text: str - action: Literal["entfernen", "anpassen", "hinzufuegen"] + action: Literal["remove", "adjust", "add"] target: Literal["title", "description", "examples", "hints"] index: int | None = None content: str = "" @@ -207,104 +207,104 @@ class ProgressResponse(BaseModel): chapters: list[str] -# --- Baustein-Lernen --- +# --- Block learning --- -class BausteinChatRequest(BaseModel): +class BlockChatRequest(BaseModel): topic: str = Field(min_length=1, max_length=100) - baustein: str = Field(min_length=1, max_length=200) - section: str = Field(default="", max_length=20000) # ausführliche Fassung - section_kompakt: str = Field(default="", max_length=20000) # kompakte Fassung (Merksätze) + block: str = Field(min_length=1, max_length=200) + section: str = Field(default="", max_length=20000) # detailed version + section_compact: str = Field(default="", max_length=20000) # compact version (mnemonics) messages: list[ChatMessage] = Field(min_length=1) provider: ProviderType = "claude" -class BausteinChatResponse(BaseModel): +class BlockChatResponse(BaseModel): reply: str -class BausteinPruefungRequest(BaseModel): +class BlockExamRequest(BaseModel): topic: str = Field(min_length=1, max_length=100) - baustein: str = Field(min_length=1, max_length=200) - section: str = Field(default="", max_length=20000) # ausführliche Fassung - section_kompakt: str = Field(default="", max_length=20000) # kompakte Fassung (Merksätze) - aktion: Literal[ - "frage", "diskussion", "antwort", "antwort_pruefen", - "quiz_frage", "quiz_antwort", "lueck_frage", "lueck_antwort", - ] = "frage" - frage: str = Field(default="", max_length=2000) # aktuell geprüfte Frage (für diskussion/antwort); Anker der Basis - auswahl: list[int] = [] # Quiz/Lückentext-Auswahl: vom Lerner gewählte Options-Indizes - korrekt: list[int] = [] # Quiz/Lückentext-Auswahl: korrekte Indizes (Client hält sie aus der Generierung) - loesung: str = Field(default="", max_length=500) # Lückentext frei: erwarteter Begriff - alternativen: list[str] = [] # Lückentext frei: akzeptierte Synonyme - eingabe: str = Field(default="", max_length=500) # Lückentext frei: getippter Begriff - schwer: bool = False # Variante: leicht (+1/−1) vs schwer (+3/−1) - letzte_bewertung: str = Field(default="", max_length=2000) # Feedback der letzten Bewertung (Kontext für diskussion) - vermeide: list[str] = [] # schon gestellte + vorgemerkte Fragen — sinngemäß nicht wiederholen - nachgefragt: bool = False # für diese Frage wurde nachgefragt → Gewinn auf +1 gedeckelt - begruendung: str = Field(default="", max_length=2000) # „Gründlich prüfen": warum mit der Bewertung unzufrieden - muster: str = Field(default="", max_length=2000) # gezogenes Frage-Muster (Saat); leer → Live-Generierung (Fallback) - # Basis + cap werden serverseitig geführt (Anker bzw. Subs×25) — Client-cap nur Hinweis. - cap: int = Field(default=10, ge=1, le=10000) # Score-Deckel = freigeschaltete Subbausteine × 25 - messages: list[ChatMessage] = [] # Dialog bisher; leer = erste Frage + block: str = Field(min_length=1, max_length=200) + section: str = Field(default="", max_length=20000) # detailed version + section_compact: str = Field(default="", max_length=20000) # compact version (mnemonics) + action: Literal[ + "question", "discussion", "answer", "answer_check", + "quiz_question", "quiz_answer", "gap_question", "gap_answer", + ] = "question" + question: str = Field(default="", max_length=2000) # currently checked question (for discussion/answer); base anchor + selection: list[int] = [] # quiz/gap-text choice: option indices picked by the learner + correct: list[int] = [] # quiz/gap-text choice: correct indices (the client keeps them from generation) + solution: str = Field(default="", max_length=500) # gap text free: expected term + alternatives: list[str] = [] # gap text free: accepted synonyms + input: str = Field(default="", max_length=500) # gap text free: typed term + schwer: bool = False # variant: easy (+1/−1) vs hard (+3/−1) + last_rating: str = Field(default="", max_length=2000) # feedback of the last rating (context for discussion) + avoid: list[str] = [] # already-asked + earmarked questions — don't repeat them in substance + asked_again: bool = False # asked_again was used for this question → gain capped at +1 + reason: str = Field(default="", max_length=2000) # "thorough check": why dissatisfied with the rating + pattern: str = Field(default="", max_length=2000) # drawn question pattern (seed); empty → live generation (fallback) + # Base + cap are kept server-side (anchor / subs×25) — the client cap is only a hint. + cap: int = Field(default=10, ge=1, le=10000) # score cap = unlocked subblocks × 25 + messages: list[ChatMessage] = [] # dialog so far; empty = first question provider: ProviderType = "claude" - gruendlich: bool = False # „Gründlich prüfen": Bewertung mit starkem Modell (role guide) + thorough: bool = False # "thorough check": rating with a strong model (role guide) class QuizOption(BaseModel): text: str - korrekt: bool + correct: bool -class BausteinPruefungResponse(BaseModel): - frage: str | None = None +class BlockExamResponse(BaseModel): + question: str | None = None reply: str | None = None feedback: str | None = None - punkte: int | None = None # Punkt-Delta dieser Antwort (−2 … +3); schnell = voraussichtlich - bewertung: Literal["gut", "neutral", "schlecht"] | None = None # aus Vorzeichen, fürs Einfärben - optionen: list[QuizOption] | None = None # Quiz: 4 Optionen + Korrekt-Flags - satz: str | None = None # Lückentext: Satz mit Lücke (___) - loesung: str | None = None # Lückentext: erwarteter Begriff - alternativen: list[str] | None = None # Lückentext: akzeptierte Synonyme - gute_antworten: int - streak: int = 0 # aktuelle Serie korrekter Antworten (je Baustein) - cap: int = 10 # cap_final = alle Subs × 25 — Frontend leitet die Lernstufe ab + points: int | None = None # points delta of this answer (−2 … +3); fast = expected + rating: Literal["gut", "neutral", "schlecht"] | None = None # from the sign, for coloring + options: list[QuizOption] | None = None # quiz: 4 options + correct flags + sentence: str | None = None # gap text: sentence with a gap (___) + solution: str | None = None # gap text: expected term + alternatives: list[str] | None = None # gap text: accepted synonyms + good_answers: int + streak: int = 0 # current run of correct answers (per block) + cap: int = 10 # cap_final = all subs × 25 — the frontend derives the learning level -class BausteinLernstand(BaseModel): - gute_antworten: int +class BlockLearnState(BaseModel): + good_answers: int streak: int = 0 - cap: int = 0 # cap_final = alle Subbausteine × 25 - cap_aktuell: int = 0 # erreichbarer cap der aktuell freigeschalteten Ebene - freie_ebene: int = 1 # 1=A · 2=F · 3=E · 4=V + cap: int = 0 # cap_final = all subblocks × 25 + cap_aktuell: int = 0 # reachable cap of the currently unlocked level + freie_level: int = 1 # 1=A · 2=F · 3=E · 4=V -class BausteinLernstandResponse(BaseModel): - bausteine: dict[str, BausteinLernstand] +class BlockLearnStateResponse(BaseModel): + blocks: dict[str, BlockLearnState] -# --- Block-Inhalt: einen Abschnitt on-demand prüfen + übernehmen (Fokus, Rechtsklick) --- +# --- Block content: check + apply one section on demand (focus, right-click) --- class BlockPruefenRequest(BaseModel): - baustein: str = Field(min_length=1, max_length=200) - stelle: str = "ausführlich" # "kompakt" | "ausführlich" (angezeigtes Feld) - block: str = Field(min_length=1, max_length=20000) # roher Markdown-Block - hinweis: str = Field(default="", max_length=2000) # optionaler Zusatz (✏️) + block: str = Field(min_length=1, max_length=200) + spot: str = "ausführlich" # "compact" | "ausführlich" (displayed field) + snippet: str = Field(min_length=1, max_length=20000) # raw markdown block + hint: str = Field(default="", max_length=2000) # optional addition (✏️) provider: ProviderType = "claude" class BlockPruefenResponse(BaseModel): - neu: str # korrigierter Block als Markdown + revised: str # corrected block as markdown class BlockUebernehmenRequest(BaseModel): - baustein: str = Field(min_length=1, max_length=200) - stelle: str = "ausführlich" + block: str = Field(min_length=1, max_length=200) + spot: str = "ausführlich" alt: str = Field(min_length=1, max_length=20000) - neu: str = Field(default="", max_length=20000) + revised: str = Field(default="", max_length=20000) provider: ProviderType = "claude" class BlockUebernehmenResponse(BaseModel): - kompakt: str + compact: str md: str - gefunden: bool + found: bool diff --git a/backend/paths.py b/backend/paths.py index 50b394f..4c9be43 100644 --- a/backend/paths.py +++ b/backend/paths.py @@ -2,7 +2,7 @@ from pathlib import Path from config import STORAGE_DIR, PROJECTS_DIR, PROJECT_ROOT -THEMEN_DIR = STORAGE_DIR / "themen" +TOPICS_DIR = STORAGE_DIR / "topics" def _safe(name: str) -> str: @@ -10,42 +10,42 @@ def _safe(name: str) -> str: def topic_dir(topic: str) -> Path: - return THEMEN_DIR / _safe(topic) + return TOPICS_DIR / _safe(topic) def arbeit_dir(topic: str) -> Path: return topic_dir(topic) / "arbeit" -def bausteine_path(topic: str) -> Path: - return topic_dir(topic) / "bausteine.md" +def blocks_path(topic: str) -> Path: + return topic_dir(topic) / "blocks.md" -def subbausteine_path(topic: str) -> Path: - """Sidecar: pro Baustein die Subbausteine mit Stufe (von allen Guides geteilt).""" - return topic_dir(topic) / "subbausteine.json" +def subblocks_path(topic: str) -> Path: + """Sidecar: the subblocks with level per block (shared by all guides).""" + return topic_dir(topic) / "subblocks.json" -def frage_muster_path(topic: str) -> Path: - """Sidecar: pro Baustein vordefinierte Frage-Muster (Subbaustein × Typ → Beispielfrage).""" - return topic_dir(topic) / "frage_muster.json" +def question_pattern_path(topic: str) -> Path: + """Sidecar: predefined question patterns per block (subblock × type → example question).""" + return topic_dir(topic) / "question_pattern.json" -def quelle_path(topic: str) -> Path: - """Persistierte Quellen-Wahl pro Thema: {type, ort, spec}.""" - return topic_dir(topic) / "quelle.json" +def source_path(topic: str) -> Path: + """Persisted source choice per topic: {type, location, spec}.""" + return topic_dir(topic) / "source.json" -def quelle_crawl_dir(topic: str) -> Path: - """Zielordner für gecrawlte Link-Quellen (Seiten + PDF-.txt).""" - return topic_dir(topic) / "quelle" +def source_crawl_dir(topic: str) -> Path: + """Target folder for crawled link sources (pages + PDF .txt).""" + return topic_dir(topic) / "source" -def safe_ordner(ort: str) -> Path | None: - """Ordnerpfad relativ zum Repo-Root, gesandboxt. None bei leer/Ausbruch (../, absolut außerhalb).""" - if not ort or not ort.strip(): +def safe_folder(location: str) -> Path | None: + """Folder path relative to the repo root, sandboxed. None if empty/escaping (../, absolute outside).""" + if not location or not location.strip(): return None - p = (PROJECT_ROOT / ort.strip()).resolve() + p = (PROJECT_ROOT / location.strip()).resolve() try: p.relative_to(PROJECT_ROOT) except ValueError: @@ -57,11 +57,11 @@ def guide_content_path(topic: str, format_name: str) -> Path: return topic_dir(topic) / "guides" / f"{format_name}.json" -def bausteine_topics() -> list[str]: - """Themen, für die ein Themen-Ordner existiert.""" - if not THEMEN_DIR.is_dir(): +def blocks_topics() -> list[str]: + """Topics for which a topic folder exists.""" + if not TOPICS_DIR.is_dir(): return [] - return [d.name for d in THEMEN_DIR.iterdir() if d.is_dir()] + return [d.name for d in TOPICS_DIR.iterdir() if d.is_dir()] def project_dir(name: str) -> Path: diff --git a/backend/pipeline.py b/backend/pipeline.py index ea22957..ecfdea6 100644 --- a/backend/pipeline.py +++ b/backend/pipeline.py @@ -1,8 +1,8 @@ -"""Pipeline-Grundbausteine: Agent-Races (mit Grace), Single-Slot, Schemata, Prompts, Guide-Status. +"""Pipeline building blocks: agent races (with grace), single-slot, schemas, prompts, guide status. -Hält den mutablen Pipeline-Zustand (Generierungs-Semaphore, Cancel-Set). -Zugriff auf das Cancel-Set NUR über die Funktionen hier — kopierte Referenzen -in anderen Modulen würden bei einem Re-Assign auseinanderlaufen. +Holds the mutable pipeline state (generation semaphore, cancel set). +Access the cancel set ONLY through the functions here — copied references +in other modules would diverge on a re-assign. """ import asyncio @@ -15,7 +15,7 @@ from typing import Callable from agents import run_agent, kill_process, cancel_scope, clear_scope from config import MAX_CONCURRENT_GENERATIONS, TEMPLATES_DIR, TIMEOUTS from database import update_guide -from jsonio import read_json_file as _json_datei +from jsonio import read_json_file as _json_file from textkit import _STUFEN log = logging.getLogger("creator.pipeline") @@ -26,10 +26,10 @@ _cancelled: set[str] = set() async def cancel_guide(guide_id: str) -> bool: _cancelled.add(guide_id) - cancel_scope(f"{guide_id}-") # wartende Agenten bailen vorm Spawn - kill_process(guide_id) # laufende Subprozesse killen + cancel_scope(f"{guide_id}-") # waiting agents bail before spawn + kill_process(guide_id) # kill running subprocesses now = datetime.now(timezone.utc).isoformat() - await update_guide(guide_id, status="error", progress=None, error_msg="Abgebrochen — Fortschritt bleibt erhalten", updated_at=now) + await update_guide(guide_id, status="error", progress=None, error_msg="Cancelled — progress is preserved", updated_at=now) return True @@ -39,7 +39,7 @@ def is_guide_cancelled(guide_id: str) -> bool: def clear_guide_cancelled(guide_id: str) -> None: _cancelled.discard(guide_id) - clear_scope(f"{guide_id}-") # Scope leeren → Neustart blockiert nicht + clear_scope(f"{guide_id}-") # clear scope → restart not blocked async def _set_progress(guide_id: str, progress: str) -> None: @@ -63,7 +63,7 @@ def _prompt(name: str, **kwargs) -> str: def _extra(instructions: str) -> str: - return f"\n\nZUSÄTZLICHE ANWEISUNGEN VOM NUTZER:\n{instructions}\n" if instructions else "" + return f"\n\nADDITIONAL INSTRUCTIONS FROM THE USER:\n{instructions}\n" if instructions else "" def _log(topic: str, msg: str) -> None: @@ -76,8 +76,8 @@ def _claude_error(label: str, returncode: int, stdout: str, stderr: str) -> str: return f"{label}: {stderr[:1000]}" tail = (stdout or "").strip()[-500:] if tail: - return f"{label} (exit {returncode}, stderr leer): …{tail}" - return f"{label} (exit {returncode}, ohne Ausgabe)" + return f"{label} (exit {returncode}, stderr empty): …{tail}" + return f"{label} (exit {returncode}, no output)" def _gather_error(label: str, results: list) -> str: @@ -87,7 +87,7 @@ def _gather_error(label: str, results: list) -> str: returncode, stdout, stderr = r if returncode != 0: return _claude_error(label, returncode, stdout, stderr) - return f"{label}: kein verwertbares Ergebnis" + return f"{label}: no usable result" def _timeout(step: str, n: int = 0) -> int: @@ -95,21 +95,21 @@ def _timeout(step: str, n: int = 0) -> int: return base + per * n -def _probleme_schema(data): - """{"ok": true} → [] · {"probleme": [str]} → Liste · sonst None.""" +def _problems_schema(data): + """{"ok": true} → [] · {"problems": [str]} → list · else None.""" if not isinstance(data, dict): return None if data.get("ok") is True: return [] - p = data.get("probleme") + p = data.get("problems") if not isinstance(p, list) or not p: return None out = [str(x).strip() for x in p if str(x).strip()] return out or None -def _str_liste(val) -> list[str] | None: - """Liste nicht-leerer Strings → gestrippte Liste (leer erlaubt) · sonst None.""" +def _str_list(val) -> list[str] | None: + """List of non-empty strings → stripped list (empty allowed) · else None.""" if not isinstance(val, list) or not all(isinstance(x, str) for x in val): return None out = [x.strip() for x in val] @@ -119,110 +119,67 @@ def _str_liste(val) -> list[str] | None: def _runde_schema(data, final: bool = False): - """{"aufnehmen": [str], "rest": [str]} → (aufnehmen, rest) · sonst None. + """{"keep": [str], "rest": [str]} → (include, rest) · else None. - final=True: letzte Klärungs-Runde — ein nicht-leerer Rest ist ungültig. + final=True: last clarification round — a non-empty rest is invalid. """ if not isinstance(data, dict): return None - aufnehmen = _str_liste(data.get("aufnehmen")) - rest = _str_liste(data.get("rest")) - if aufnehmen is None or rest is None or (final and rest): + include = _str_list(data.get("keep")) + rest = _str_list(data.get("rest")) + if include is None or rest is None or (final and rest): return None - return aufnehmen, rest + return include, rest -def _stufen_schema(data, ids: set[int] | None = None): - """{"stufen": {"1": "anfaenger", …}} → {id: stufe} · sonst None. +_RELEVANCE = ("relevant", "peripheral") +_YESNO = ("ja", "nein") - Stufe ∈ {anfaenger, fortgeschritten, experte} (alte Werte abwärtskompatibel). Sind `ids` - gegeben, müssen mindestens diese abgedeckt sein (Extras erlaubt); der Aufrufer filtert auf `ids`. - """ - if not isinstance(data, dict) or not isinstance(data.get("stufen"), dict) or not data["stufen"]: - return None - out: dict[int, str] = {} - for k, v in data["stufen"].items(): - try: - num = int(k) - except (ValueError, TypeError): + +def _enum_map_schema(key: str, allowed): + """Factory for `{"": {"1": value, …}}` → `{id: value}` parsers; value ∈ `allowed` + (casefolded). If `ids` are given, at least these must be covered (extras allowed). None + on any invalid id/value or wrong shape. The caller filters the result to `ids`.""" + def parse(data, ids: set[int] | None = None): + if not isinstance(data, dict) or not isinstance(data.get(key), dict) or not data[key]: return None - stufe = str(v).strip().casefold() - if stufe not in _STUFEN: + out: dict[int, str] = {} + for k, v in data[key].items(): + try: + num = int(k) + except (ValueError, TypeError): + return None + value = str(v).strip().casefold() + if value not in allowed: + return None + out[num] = value + if ids is not None and not ids <= set(out): return None - out[num] = stufe - if ids is not None and not ids <= set(out): - return None - return out + return out + return parse -_RELEVANZ = ("relevant", "rand") - - -def _relevanz_schema(data, ids: set[int] | None = None): - """{"relevanz": {"1": "relevant", …}} → {id: relevanz} · sonst None. - - Relevanz ∈ {relevant, rand} (binär). Wie `_stufen_schema`: sind `ids` gegeben, - müssen mindestens diese abgedeckt sein (Extras erlaubt). - """ - if not isinstance(data, dict) or not isinstance(data.get("relevanz"), dict) or not data["relevanz"]: - return None - out: dict[int, str] = {} - for k, v in data["relevanz"].items(): - try: - num = int(k) - except (ValueError, TypeError): - return None - wert = str(v).strip().casefold() - if wert not in _RELEVANZ: - return None - out[num] = wert - if ids is not None and not ids <= set(out): - return None - return out - - -_JANEIN = ("ja", "nein") - - -def _janein_schema(data, ids: set[int] | None = None): - """{"relevant": {"1": "ja", …}} → {id: ja/nein} · sonst None. - - Binäres ja/nein — das Themen-Relevanz-Gate der Sichtung. Wie `_relevanz_schema`: - sind `ids` gegeben, müssen mindestens diese abgedeckt sein (Extras erlaubt). - """ - if not isinstance(data, dict) or not isinstance(data.get("relevant"), dict) or not data["relevant"]: - return None - out: dict[int, str] = {} - for k, v in data["relevant"].items(): - try: - num = int(k) - except (ValueError, TypeError): - return None - wert = str(v).strip().casefold() - if wert not in _JANEIN: - return None - out[num] = wert - if ids is not None and not ids <= set(out): - return None - return out +_levels_schema = _enum_map_schema("levels", _STUFEN) # level ∈ beginner/advanced/expert +_relevance_schema = _enum_map_schema("relevance", _RELEVANCE) # relevance ∈ relevant/peripheral +_yesno_schema = _enum_map_schema("relevant", _YESNO) # triage gate ∈ ja/nein _MAX_RESTARTS = 2 async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: int, provider: str, on_update=None, cancelled=None, *, grace: int | None = None) -> list | None: - """Startet alle Slots parallel und sammelt `quorum` gültige Ergebnisse. + """Starts all slots in parallel and collects `quorum` valid results. - Slot-Spec: {key, prompt, role, capabilities, payload}. `payload(result)` - prüft die Gültigkeit und liefert das Slot-Ergebnis oder None. - Fehler/Timeout/ungültig → Slot-Neustart (max. _MAX_RESTARTS). Sobald das - Quorum steht, werden die übrigen Agenten gekillt. None = Quorum verfehlt. - `cancelled()` → True bricht ab (keine Restarts, Rückgabe None). + Slot spec: {key, prompt, role, capabilities, payload}. `payload(result)` + checks validity and returns the slot result or None. + Error/timeout/invalid → slot restart (max. _MAX_RESTARTS). As soon as the + quorum stands, the remaining agents are killed. None = quorum missed. + `cancelled()` → True aborts (no restarts, returns None). - Mit `grace` wird `quorum` zum Minimum: Das erste gültige Ergebnis startet - einen Timer von `grace` Sekunden. Nach dessen Ablauf werden laufende - Agenten nur gekillt, wenn das Minimum steht — sonst läuft das Race samt - Restarts weiter, bis es steht. Rückgabe: `quorum` bis `len(slots)` Ergebnisse. + With `grace`, `quorum` becomes the minimum: the first valid result starts + a timer of `grace` seconds. After it expires, running agents are only + killed if the minimum stands — otherwise the race, including restarts, + keeps running until it stands. Returns: `quorum` to `len(slots)` results. """ attempts = {i: 0 for i in range(len(slots))} tasks: dict[asyncio.Task, int] = {} @@ -247,7 +204,7 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: return None if deadline is not None and len(results) >= quorum and loop.time() >= deadline: return results - # Grace gesetzt und Minimum erreicht → nur bis zum Deadline-Rest warten + # Grace set and minimum reached → only wait for the remaining deadline wait_timeout = None if deadline is not None and len(results) >= quorum: wait_timeout = max(0.0, deadline - loop.time()) @@ -260,13 +217,13 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: try: result = task.result() if result[0] != 0: - err = _claude_error("Fehler", *result) + err = _claude_error("Error", *result) else: payload = slots[i]["payload"](result) if payload is None: - err = "Ergebnis ungültig/nicht parsebar" + err = "result invalid/not parseable" except asyncio.TimeoutError: - err = f"Timeout nach {timeout}s" + err = f"Timeout after {timeout}s" except Exception as e: err = f"{type(e).__name__}: {e}" @@ -274,23 +231,23 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: results.append(payload) if grace is not None and deadline is None: deadline = loop.time() + grace - _log(topic, f"{label}: erstes Ergebnis — Grace {grace}s läuft") + _log(topic, f"{label}: first result — grace {grace}s running") if on_update: on_update(len(results)) if len(results) >= quorum and (grace is None or loop.time() >= deadline): return results continue - _log(topic, f"{label} {i + 1} (Versuch {attempts[i] + 1}): {err}") + _log(topic, f"{label} {i + 1} (attempt {attempts[i] + 1}): {err}") attempts[i] += 1 - # Steht das Minimum schon, sind Restarts sinnlos — der Neustart - # würde am Grace-Ende ohnehin gekillt. - satt = grace is not None and len(results) >= quorum - if attempts[i] <= _MAX_RESTARTS and not satt and not (cancelled and cancelled()): + # If the minimum already stands, restarts are pointless — the restart + # would be killed at the grace end anyway. + enough = grace is not None and len(results) >= quorum + if attempts[i] <= _MAX_RESTARTS and not enough and not (cancelled and cancelled()): spawn(i) - if len(results) >= quorum: # alle Slots durch, Minimum steht (nur mit grace erreichbar) + if len(results) >= quorum: # all slots done, minimum stands (only reachable with grace) return results - _log(topic, f"{label}: Quorum {quorum} nicht erreicht ({len(results)} gültig)") + _log(topic, f"{label}: quorum {quorum} not reached ({len(results)} valid)") return None finally: for task, i in tasks.items(): @@ -302,14 +259,14 @@ async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: @dataclass class GenContext: - """Durchgereichte Pipeline-Parameter — erspart lange Argument-Signaturen.""" + """Pipeline parameters passed through — saves long argument signatures.""" topic: str provider: str is_cancelled: Callable[[], bool] guide_id: str | None = None -# Ergebnis-Status von run_single_slot +# Result status of run_single_slot OK, CANCELLED, FAILED = "ok", "cancelled", "failed" @@ -317,9 +274,9 @@ async def run_single_slot( ctx: GenContext, label: str, *, key: str, prompt: str, role: str, capabilities: str, payload, timeout: int, ) -> tuple[str, object]: - """Ein Agent, ein gültiges Ergebnis (Race mit Quorum 1). + """One agent, one valid result (race with quorum 1). - → (OK, wert) | (CANCELLED, None) | (FAILED, None) + → (OK, value) | (CANCELLED, None) | (FAILED, None) """ slots = [{"key": key, "prompt": prompt, "role": role, "capabilities": capabilities, "payload": payload}] res = await _race(ctx.topic, label, slots, 1, timeout, ctx.provider, cancelled=ctx.is_cancelled) @@ -330,9 +287,9 @@ async def run_single_slot( return OK, res[0] -async def _gather_fortschritt(coros, total, melde, start=0): - """Läuft `coros` nebenläufig und meldet Live-Fortschritt: `await melde(fertig, total)` - nach jedem Abschluss (und einmal initial). Ergebnisse in Reihenfolge, return_exceptions=True.""" +async def _gather_progress(coros, total, report, start=0): + """Runs `coros` concurrently and reports live progress: `await report(done, total)` + after each completion (and once initially). Results in order, return_exceptions=True.""" done = start async def wrap(c): @@ -341,9 +298,7 @@ async def _gather_fortschritt(coros, total, melde, start=0): return await c finally: done += 1 - await melde(done, total) + await report(done, total) - await melde(done, total) + await report(done, total) return await asyncio.gather(*[wrap(c) for c in coros], return_exceptions=True) - - diff --git a/backend/readability.py b/backend/readability.py new file mode 100644 index 0000000..419114d --- /dev/null +++ b/backend/readability.py @@ -0,0 +1,115 @@ +"""Deterministic readability gate for guide sections. + +A small German complexity model (DistilBERT, GermEval 2022, scale 1–7) rates the +readability of the prose. guide.py feeds sections that are too hard into the existing +read-exam/revision loop — no prompt, no guessing. + +Optional: if `transformers`/`torch` are missing or the model won't load, the gate is +silently disabled (the backend keeps running unchanged). CPU is enough; the caller +wraps the scoring in `asyncio.to_thread` (blocking model inference). +""" + +import logging +import re + +from config import ( + READABILITY_ACTIVE, READABILITY_HARD, READABILITY_HARD_SHARE, READABILITY_MAX, READABILITY_MODEL, +) + +log = logging.getLogger("creator.readability") + +_model_cache = None # (tokenizer, model, torch) — singleton +_load_attempt = False # already tried to load? + +# Strip markup → plain prose (code does not count toward readability). +_CODE_FENCE = re.compile(r"```.*?```", re.DOTALL) +_COMMENT = re.compile(r"", re.DOTALL) +_INLINE_CODE = re.compile(r"`[^`]*`") +_LINK = re.compile(r"\[([^\]]*)\]\([^)]*\)") +_MD_MARK = re.compile(r"^[ \t]*([#>]+|[-*+]\s)|[*_~|]", re.MULTILINE) +_WS = re.compile(r"\s+") +_SENTENCE = re.compile(r"(?<=[.!?])\s+") + + +def _model(): + """Load the model once. None = gate off (disabled or load error).""" + global _model_cache, _load_attempt + if _load_attempt: + return _model_cache + _load_attempt = True + if not READABILITY_ACTIVE: + return None + try: + import torch + from transformers import AutoModelForSequenceClassification, AutoTokenizer + tok = AutoTokenizer.from_pretrained(READABILITY_MODEL) + model = AutoModelForSequenceClassification.from_pretrained(READABILITY_MODEL) + model.eval() + _model_cache = (tok, model, torch) + log.info("readability model loaded: %s (num_labels=%d)", READABILITY_MODEL, model.config.num_labels) + except Exception as e: + log.warning("readability gate disabled (model not loadable): %s", e) + _model_cache = None + return _model_cache + + +def _prose(md: str) -> str: + """Strip markdown/code → plain prose for scoring.""" + t = _CODE_FENCE.sub(" ", md) + t = _COMMENT.sub(" ", t) + t = _INLINE_CODE.sub(" ", t) + t = _LINK.sub(r"\1", t) + t = _MD_MARK.sub(" ", t) + return _WS.sub(" ", t).strip() + + +def _sentences(text: str) -> list[str]: + """Split prose into sentences; discard very short fragments.""" + return [s.strip() for s in _SENTENCE.split(text) if len(s.strip()) >= 15] + + +def _scores(sentences: list[str]) -> list[float]: + """Complexity per sentence (1–7). Regression (num_labels=1) or expectation over classes.""" + tok, model, torch = _model_cache + values: list[float] = [] + n = model.config.num_labels + for i in range(0, len(sentences), 16): + batch = sentences[i:i + 16] + enc = tok(batch, return_tensors="pt", truncation=True, max_length=256, padding=True) + with torch.no_grad(): + logits = model(**enc).logits + if n == 1: + vals = logits.reshape(-1).tolist() + else: + probs = torch.softmax(logits, dim=-1) + levels = torch.arange(1, n + 1, dtype=probs.dtype) + vals = (probs * levels).sum(-1).reshape(-1).tolist() + values.extend(vals) + return values + + +def rate_sections(md_by_num: dict[int, str]) -> dict[int, str]: + """{num: section_md} → {num: hint} only for sections that are too hard. + + Empty dict if the gate is off. Blocking (CPU) — call inside to_thread. + """ + if _model() is None: + return {} + out: dict[int, str] = {} + for num, md in md_by_num.items(): + sentences = _sentences(_prose(md or "")) + if len(sentences) < 2: # almost only code / too short → skip + continue + values = _scores(sentences) + if not values: + continue + mean = sum(values) / len(values) + hard = sum(1 for w in values if w > READABILITY_HARD) / len(values) + # Too hard = high mean OR too many hard individual sentences (outlier nests). + if mean > READABILITY_MAX or hard >= READABILITY_HARD_SHARE: + # German revision hint fed to the (German-writing) writer agent — kept German on purpose. + out[num] = ( + f"Zu schwer lesbar (Ø {mean:.1f}/7, {hard * 100:.0f}% harte Sätze): " + "kürzere Sätze, einfachere Wörter, weniger Schachtelsätze, mehr Examples." + ) + return out diff --git a/backend/regeln.py b/backend/regeln.py deleted file mode 100644 index 91c2e55..0000000 --- a/backend/regeln.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Lernschulden-Regeln: Progression und Deckel für offene Guides — die EINZIGE Quelle. - -Regeln (nur Neu-Erstellungen; Themen + Bausteine unbegrenzt): -- Format „Guide": höchstens 3 erstellte, nicht absolvierte Guides -- Keine Progression/Vorstufe mehr (nur ein Guide-Format). -- Absolviert: ALLE Bausteine (Section-Titel) des neuesten fertigen Guides haben - eine bestandene Prüfung. Rest ist read-only (kein Fortschritt, keine Prüfung). -Alle Funktionen arbeiten auf einmal geladenen Daten (lade_lernstand) — keine -Query-Schleifen mehr pro Guide. -""" - -import json - -from database import list_baustein_scores_all, subs_je_ebene_alle, list_guides, list_progress_all -from guide import guide_slot_dateien -from lernen import cap_final, STUFEN, _schwelle -from paths import bausteine_path, guide_content_path -from textkit import _norm_titel - -MAX_OFFENE_GUIDES = 3 -# Nur noch EIN Format „Guide" (alle relevanten Bausteine, Prüfung 0–cap). Keine Progression, -# keine Vorstufe → der Guide ist immer freischaltbar. „Rest"/FullGuide separat. -VORSTUFE: dict[str, str] = {} -FREISCHALT_LEVEL: dict[str, str] = {} -FORMATE = ("Guide",) - -# 4 Lernstufen (Floor in % des cap) — Schlüssel aus lernen.STUFEN. -_LEVEL_WORT = { - "anfaenger": "auf Anfänger (20 %)", - "fortgeschritten": "auf Fortgeschritten (40 %)", - "experte": "auf Experte (60 %)", - "meister": "meistern (100 %)", -} - - -async def lade_lernstand() -> tuple[list[dict], dict[str, set[str]], dict[str, dict[str, set[str]]]]: - """Guides + Kapitel-Fortschritt + Bausteine je Stufe. - - levels: {"anfaenger"/"fortgeschritten"/"experte"/"meister": {topic → normalisierte Titel}}. - Stufe je Baustein wird aus Score + cap (4×relevante Subs) abgeleitet. - """ - scores = await list_baustein_scores_all() - ebenen = await subs_je_ebene_alle() - levels: dict[str, dict[str, set[str]]] = {key: {} for key, _ in STUFEN} - for topic, baustein, score in scores: - cf = cap_final(ebenen.get((topic, _norm_titel(baustein)), {})) - for key, p in STUFEN: - if cf and score >= _schwelle(p, cf): - levels[key].setdefault(topic, set()).add(_norm_titel(baustein)) - return await list_guides(), await list_progress_all(), levels - - -def _content_json(topic: str, fmt: str) -> dict | None: - path = guide_content_path(topic, fmt) - if not path.exists(): - return None - try: - return json.loads(path.read_text(encoding="utf-8")) - except ValueError: - return None - - - - -def _section_titel(topic: str, fmt: str) -> set[str] | None: - """Normalisierte Baustein-Titel (Sections) aus dem Guide-Content.""" - content = _content_json(topic, fmt) - if content is None: - return None - return { - _norm_titel(s.get("title", "")) - for ch in content.get("chapters", []) - for s in ch.get("sections", []) - } - - -def _neueste_done(guides: list[dict], fmt: str) -> dict[str, dict]: - """Pro Thema der neueste fertige Guide dieses Formats.""" - neueste: dict[str, dict] = {} - for g in guides: - if g["format"] == fmt and g["status"] == "done": - if g["topic"] not in neueste or g["created_at"] > neueste[g["topic"]]["created_at"]: - neueste[g["topic"]] = g - return neueste - - -def _guide_alle(g: dict, progress: dict[str, set[str]], levelset: dict[str, set[str]]) -> bool: - """Sind ALLE Bausteine des Guides auf dem geforderten Niveau?""" - sections = _section_titel(g["topic"], g["format"]) - return bool(sections) and sections <= levelset.get(g["topic"], set()) - - -def ist_level(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levelset: dict[str, set[str]]) -> bool: - """Neuester fertiger Guide (Thema+Format): alle Bausteine auf dem Niveau von levelset?""" - g = _neueste_done(guides, fmt).get(topic) - return g is not None and _guide_alle(g, progress, levelset) - - -def ist_absolviert(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> bool: - """Alle Bausteine des neuesten fertigen Guides mindestens Anfänger (≥20 %)?""" - return ist_level(topic, fmt, guides, progress, levels["anfaenger"]) - - -def thema_abgeschlossen(topic: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> bool: - """Thema fertig: neuester fertiger Guide, alle Bausteine auf Meister (100 %)?""" - return ist_level(topic, "Guide", guides, progress, levels["meister"]) - - -def formate_stats(guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> dict: - """Pro Format erstellt/absolviert — pro Thema zählt nur der neueste fertige Guide.""" - formate = {} - for fmt in FORMATE: - neueste = _neueste_done(guides, fmt) - absolviert = sum(1 for g in neueste.values() if _guide_alle(g, progress, levels["anfaenger"])) - formate[fmt] = {"erstellt": len(neueste), "absolviert": absolviert} - return formate - - -def guide_lock(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> str | None: - """Grund, warum ein Neu-Start für Thema+Format gesperrt ist — None = erlaubt. - - Exakt die Regeln aus POST /guides: Bausteine nötig, kein Duplikat-Start, - Lernschulden nur für echte Neu-Erstellungen (Resume/Regenerieren frei). - """ - if not bausteine_path(topic).exists(): - return "Erst Bausteine erstellen" - for g in guides: - if g["topic"] == topic and g["format"] == fmt and g["status"] in ("queued", "generating"): - return "Generierung läuft bereits" - content = guide_content_path(topic, fmt) - if not content.exists() and not guide_slot_dateien(content): - vorstufe = VORSTUFE.get(fmt) - if vorstufe: - stufe = FREISCHALT_LEVEL[fmt] # absolviert=10 · verstanden=20 · gemeistert=30 - if not ist_level(topic, vorstufe, guides, progress, levels[stufe]): - return f"Erst den {vorstufe} dieses Themas {_LEVEL_WORT[stufe]}" - stat = formate_stats(guides, progress, levels).get(fmt, {"erstellt": 0, "absolviert": 0}) - offen = stat["erstellt"] - stat["absolviert"] - if offen >= MAX_OFFENE_GUIDES: - return f"Erst {fmt}s absolvieren — maximal {MAX_OFFENE_GUIDES} offene erlaubt ({offen} offen)" - return None diff --git a/backend/routes.py b/backend/routes.py index 7d0ce6f..325ed9f 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -14,33 +14,33 @@ from database import ( create_topic, list_topics as db_list_topics, delete_topic, list_progress, set_progress, delete_progress, create_element, list_elements, get_element, update_element, delete_element, - list_baustein_progress, get_baustein_progress, set_offene_frage, - set_baustein_score_and_streak, set_baustein_absolviert, - delete_baustein_daten, delete_baustein_progress, subs_je_ebene, subs_je_ebene_roh, - delete_topic_pipeline, delete_quelle, get_guide_content, delete_guide_content, + list_block_progress, get_block_progress, set_open_question, + set_block_score_and_streak, set_block_completed, + delete_block_data, delete_block_progress, subs_per_level, subs_per_level_raw, + delete_topic_pipeline, delete_source, get_guide_content, delete_guide_content, get_sub_artefakte, ) -from bausteine import generate_bausteine, cancel_bausteine, bausteine_status, active_bausteine, reset_bausteine, reset_bausteine_ab_step, lade_quelle, lade_uebersicht, subbausteine_titel, subbausteine_frei, lade_frage_muster, lade_frage_muster_frei +from blocks import generate_blocks, cancel_blocks, blocks_status, active_blocks, reset_blocks, reset_blocks_ab_step, load_source, load_overview, subblocks_title, subblocks_frei, load_question_pattern, load_question_pattern_free from elements import generate_element, chat_with_guide, chat_with_element, check_element, style_element, refine_suggestion -from lernen import baustein_chat, baustein_diskussion, baustein_element_anlegen, pruefung_bewertung, pruefung_bewertung_schnell, pruefung_frage, pruefung_frage_variante, quiz_generieren, lueckwahl_generieren, lueckentext_generieren, lueckentext_pruefen, huerden_distraktor_block, score_berechnen, floor_aus_score, stufe_aus_score, cap_final, cap_aktuell, freie_ebene, schwellen, punkte_delta, deckel_nachfrage -from guide import generate_guide, guide_slot_dateien, guide_fertig_step, block_pruefen, block_uebernehmen, content_fuer_ebene +from learning import block_chat, block_discussion, create_block_element, exam_rating, exam_rating_fast, exam_question, exam_question_variant, generate_quiz, generate_gapchoice, generate_gaptext, check_gaptext, hurdles_distractor_block, compute_score, floor_from_score, level_from_score, cap_final, cap_aktuell, freie_level, thresholds, points_delta, cap_followup +from guide import generate_guide, guide_slot_files, guide_done_step, block_pruefen, block_adopt, content_fuer_level from pipeline import cancel_guide -from regeln import FORMATE, formate_stats, guide_lock, ist_absolviert, lade_lernstand, thema_abgeschlossen +from rules import FORMATE, formats_stats, guide_lock, ist_completed, load_learnstate, topic_completed from models import ( GuideCreateRequest, GuideResponse, TopicCreateRequest, - BausteineCreateRequest, BausteineResetStepRequest, BausteineStatusResponse, + BlocksCreateRequest, BlocksResetStepRequest, BlocksStatusResponse, GuideChatRequest, GuideChatResponse, ElementCreateRequest, ElementChatRequest, ElementChatResponse, ElementResponse, ElementUpdateRequest, ElementCheckRequest, ElementCheckResponse, ElementStyleResponse, ElementRefineRequest, ElementRefineResponse, ProgressUpdate, ProgressResponse, ProjectResponse, ProviderInfo, - FolderResponse, BausteineQuelleUpdate, BausteineQuelleResponse, BausteinUebersicht, - BausteinChatRequest, BausteinChatResponse, - BausteinPruefungRequest, BausteinPruefungResponse, BausteinLernstandResponse, + FolderResponse, BlocksSourceUpdate, BlocksSourceResponse, BlockOverview, + BlockChatRequest, BlockChatResponse, + BlockExamRequest, BlockExamResponse, BlockLearnStateResponse, BlockPruefenRequest, BlockPruefenResponse, BlockUebernehmenRequest, BlockUebernehmenResponse, ) -from paths import bausteine_topics, guide_content_path, project_dir, topic_dir, quelle_path, safe_ordner +from paths import blocks_topics, guide_content_path, project_dir, topic_dir, source_path, safe_folder from fsutil import atomic_write_json router = APIRouter(prefix="/api") @@ -56,28 +56,28 @@ async def get_topics(): db_topics = await db_list_topics() guides = await list_guides() derived = {g["topic"] for g in guides} - derived.update(bausteine_topics()) - derived.update(job["topic"] for job in active_bausteine()) - # DB ist führend (Reihenfolge: neueste zuerst); Abgeleitetes ohne DB-Eintrag hinten anhängen + derived.update(blocks_topics()) + derived.update(job["topic"] for job in active_blocks()) + # DB is authoritative (order: newest first); append derived entries without a DB row at the end return db_topics + sorted(derived - set(db_topics)) @router.get("/stats") async def get_stats(): - """Tracker: Themen-Anzahl + pro Format erstellt/absolviert.""" - guides, progress, levels = await lade_lernstand() - themen = set(await db_list_topics()) | {g["topic"] for g in guides} | set(bausteine_topics()) + """Tracker: number of topics + per format created/completed.""" + guides, progress, levels = await load_learnstate() + topics = set(await db_list_topics()) | {g["topic"] for g in guides} | set(blocks_topics()) if PROJECTS_DIR.is_dir(): - themen |= {e.name for e in PROJECTS_DIR.iterdir() if e.is_dir()} - return {"themen": len(themen), "formate": formate_stats(guides, progress, levels)} + topics |= {e.name for e in PROJECTS_DIR.iterdir() if e.is_dir()} + return {"topics": len(topics), "formats": formats_stats(guides, progress, levels)} -@router.get("/topics/fortschritt") -async def topic_fortschritt(topic: str): - """Absolviert-Status pro Format + Themen-Abschluss — fürs Freischalten der nächsten Ausbaustufe.""" - guides, progress, levels = await lade_lernstand() - status = {fmt: ist_absolviert(topic, fmt, guides, progress, levels) for fmt in FORMATE} - status["abgeschlossen"] = thema_abgeschlossen(topic, guides, progress, levels) +@router.get("/topics/progress") +async def topic_progress(topic: str): + """Completion status per format + topic completion — for unlocking the next expansion stage.""" + guides, progress, levels = await load_learnstate() + status = {fmt: ist_completed(topic, fmt, guides, progress, levels) for fmt in FORMATE} + status["completed"] = topic_completed(topic, guides, progress, levels) return status @@ -90,9 +90,9 @@ async def add_topic(req: TopicCreateRequest): @router.delete("/topics") async def remove_topic(topic: str): await delete_topic(topic) - await delete_baustein_daten(topic) + await delete_block_data(topic) await delete_topic_pipeline(topic) - await delete_quelle(topic) # Themen-Config (DB) — beim Thema-Löschen mit weg + await delete_source(topic) # topic config (DB) — removed together with the topic await delete_guide_content(topic) shutil.rmtree(topic_dir(topic), ignore_errors=True) return {"ok": True} @@ -100,7 +100,7 @@ async def remove_topic(topic: str): def _safe_project_name(name: str) -> str: if not name or "/" in name or "\\" in name or ".." in name or "\x00" in name: - raise HTTPException(400, "Ungültiger Projektname") + raise HTTPException(400, "Invalid project name") return name @@ -116,356 +116,356 @@ async def remove_project(name: str): _safe_project_name(name) pdir = project_dir(name) if not pdir.is_dir(): - raise HTTPException(404, "Projekt nicht gefunden") + raise HTTPException(404, "Project not found") shutil.rmtree(pdir) return {"ok": True} @router.get("/folders", response_model=list[FolderResponse]) async def list_folders(kind: str): - """Ordner für die Quellen-Auswahl: kind=projekt → projects/, kind=uni → uni/.""" + """Folders for the source selection: kind=projekt → projects/, kind=uni → uni/.""" base = {"projekt": (PROJECTS_DIR, "projects"), "uni": (UNI_DIR, "uni")}.get(kind) if base is None: - raise HTTPException(400, "kind muss 'projekt' oder 'uni' sein") + raise HTTPException(400, "kind must be 'projekt' or 'uni'") root, prefix = base if not root.is_dir(): return [] - return [{"name": e.name, "ort": f"{prefix}/{e.name}"} for e in sorted(root.iterdir()) if e.is_dir()] + return [{"name": e.name, "location": f"{prefix}/{e.name}"} for e in sorted(root.iterdir()) if e.is_dir()] -# --- Bausteine --- +# --- Blocks --- -@router.get("/bausteine/status", response_model=BausteineStatusResponse) -async def get_bausteine_status(topic: str): - return bausteine_status(topic) +@router.get("/blocks/status", response_model=BlocksStatusResponse) +async def get_blocks_status(topic: str): + return await blocks_status(topic) -@router.get("/bausteine/active") -async def get_active_bausteine(): - return active_bausteine() +@router.get("/blocks/active") +async def get_active_blocks(): + return active_blocks() -@router.post("/bausteine") -async def create_bausteine(req: BausteineCreateRequest): +@router.post("/blocks") +async def create_blocks(req: BlocksCreateRequest): topic = req.topic.strip() - if bausteine_status(topic)["generating"]: + if (await blocks_status(topic))["generating"]: return {"ok": True, "status": "already_generating"} await create_topic(topic) - qp = quelle_path(topic) - # Quelle nur beim ERSTEN Mal festschreiben; ▶/Resume erhält die bestehende Wahl. + qp = source_path(topic) + # Persist the source only the FIRST time; ▶/Resume keeps the existing choice. if not qp.exists(): - typ, ort = req.source_type, req.source_ort.strip() - if typ in ("projekt", "uni"): - ordner = safe_ordner(ort) - if ordner is None or not ordner.is_dir(): - raise HTTPException(400, "Ordner ungültig oder nicht gefunden (Pfad relativ zum Projekt-Root, kein ../).") - elif typ == "link": - if not ort.lower().startswith(("http://", "https://")): - raise HTTPException(400, "Link muss mit http:// oder https:// beginnen.") + type, location = req.source_type, req.source_location.strip() + if type in ("projekt", "uni"): + folder = safe_folder(location) + if folder is None or not folder.is_dir(): + raise HTTPException(400, "Folder invalid or not found (path relative to the project root, no ../).") + elif type == "link": + if not location.lower().startswith(("http://", "https://")): + raise HTTPException(400, "Link must start with http:// or https://.") qp.parent.mkdir(parents=True, exist_ok=True) - atomic_write_json(qp, {"type": typ, "ort": ort, "spec": req.instructions.strip()}) - asyncio.create_task(generate_bausteine(topic, req.instructions.strip(), req.provider, ab_phase=req.ab_phase, ab_step=req.ab_step)) + atomic_write_json(qp, {"type": type, "location": location, "spec": req.instructions.strip()}) + asyncio.create_task(generate_blocks(topic, req.instructions.strip(), req.provider, ab_phase=req.ab_phase, ab_step=req.ab_step)) return {"ok": True} -@router.post("/bausteine/cancel") -async def cancel_bausteine_route(topic: str): - if not cancel_bausteine(topic): - raise HTTPException(404, "Keine laufende Generierung") +@router.post("/blocks/cancel") +async def cancel_blocks_route(topic: str): + if not cancel_blocks(topic): + raise HTTPException(404, "No running generation") return {"ok": True} -@router.delete("/bausteine") -async def remove_bausteine(topic: str): - reset_bausteine(topic) # Dateien: Crawl + Sichtung + Inventar…Fragen weg; quelle.json bleibt - await delete_topic_pipeline(topic) # DB: Bausteine-Bereich weg; Themen-Config (quelle) bleibt +@router.delete("/blocks") +async def remove_blocks(topic: str): + reset_blocks(topic) # Files: crawl + triage + inventory…questions gone; source.json stays + await delete_topic_pipeline(topic) # DB: blocks area gone; topic config (source) stays return {"ok": True} -@router.post("/bausteine/reset-step") -async def reset_bausteine_step(req: BausteineResetStepRequest): +@router.post("/blocks/reset-step") +async def reset_blocks_step(req: BlocksResetStepRequest): topic = req.topic.strip() - if bausteine_status(topic)["generating"]: - return {"ok": True, "status": "generating"} # nicht in laufende Generierung eingreifen - await reset_bausteine_ab_step(topic, req.ab_step) + if (await blocks_status(topic))["generating"]: + return {"ok": True, "status": "generating"} # don't interfere with a running generation + await reset_blocks_ab_step(topic, req.ab_step) return {"ok": True} -@router.delete("/bausteine/fortschritt") -async def reset_baustein_fortschritt(topic: str, baustein: str): - """Lern-Fortschritt EINES Bausteins auf null (Score/Streak/Flags/offene Frage).""" - await delete_baustein_progress(topic, baustein) +@router.delete("/blocks/progress") +async def reset_block_progress(topic: str, block: str): + """Reset learning progress of ONE block to zero (score/streak/flags/open question).""" + await delete_block_progress(topic, block) return {"ok": True} -def _validate_quelle(typ: str, ort: str) -> None: - """Quellen-Eingabe prüfen (gleiche Regeln wie beim Erstellen).""" - if typ in ("projekt", "uni"): - ordner = safe_ordner(ort) - if ordner is None or not ordner.is_dir(): - raise HTTPException(400, "Ordner ungültig oder nicht gefunden (Pfad relativ zum Projekt-Root, kein ../).") - elif typ == "link": - if not ort.lower().startswith(("http://", "https://")): - raise HTTPException(400, "Link muss mit http:// oder https:// beginnen.") +def _validate_source(type: str, location: str) -> None: + """Check source input (same rules as on creation).""" + if type in ("projekt", "uni"): + folder = safe_folder(location) + if folder is None or not folder.is_dir(): + raise HTTPException(400, "Folder invalid or not found (path relative to the project root, no ../).") + elif type == "link": + if not location.lower().startswith(("http://", "https://")): + raise HTTPException(400, "Link must start with http:// or https://.") -@router.get("/bausteine/quelle", response_model=BausteineQuelleResponse) -async def get_bausteine_quelle(topic: str): - return lade_quelle(topic) +@router.get("/blocks/source", response_model=BlocksSourceResponse) +async def get_blocks_source(topic: str): + return load_source(topic) -@router.put("/bausteine/quelle", response_model=BausteineQuelleResponse) -async def update_bausteine_quelle(req: BausteineQuelleUpdate): - """Nur speichern — KEINE Neugenerierung. Quellen-/Spec-Wahl überschreiben.""" - topic, typ, ort = req.topic.strip(), req.type, req.ort.strip() - _validate_quelle(typ, ort) - qp = quelle_path(topic) +@router.put("/blocks/source", response_model=BlocksSourceResponse) +async def update_blocks_source(req: BlocksSourceUpdate): + """Only save — NO regeneration. Overwrite the source/spec choice.""" + topic, type, location = req.topic.strip(), req.type, req.location.strip() + _validate_source(type, location) + qp = source_path(topic) qp.parent.mkdir(parents=True, exist_ok=True) - daten = {"type": typ, "ort": ort, "spec": req.spec.strip()} - atomic_write_json(qp, daten) - return daten + data = {"type": type, "location": location, "spec": req.spec.strip()} + atomic_write_json(qp, data) + return data -@router.get("/bausteine/uebersicht", response_model=list[BausteinUebersicht]) -async def get_bausteine_uebersicht(topic: str): - return await lade_uebersicht(topic) +@router.get("/blocks/overview", response_model=list[BlockOverview]) +async def get_blocks_uebersicht(topic: str): + return await load_overview(topic) -@router.get("/bausteine/frage-muster") -async def get_frage_muster(topic: str, baustein: str): - """Freigeschaltete Frage-Muster eines Bausteins (bis zur aktuellen Ebene; leer = Live).""" - stand = await get_baustein_progress(topic, baustein) - fe = freie_ebene(stand["gute_antworten"], await subs_je_ebene(topic, baustein)) - return {"muster": await lade_frage_muster_frei(topic, baustein, fe)} +@router.get("/blocks/question-pattern") +async def get_question_pattern(topic: str, block: str): + """Unlocked question patterns of a block (up to the current level; empty = live).""" + state = await get_block_progress(topic, block) + fe = freie_level(state["good_answers"], await subs_per_level(topic, block)) + return {"pattern": await load_question_pattern_free(topic, block, fe)} -@router.get("/bausteine/artefakte") -async def get_artefakte(topic: str, typ: str | None = None): - """Lern-Artefakte (Karteikarten/Beispiele) je Thema, gruppiert nach Baustein-Norm — je Subbaustein.""" - rows = await get_sub_artefakte(topic, typ) +@router.get("/blocks/artefakte") +async def get_artefakte(topic: str, type: str | None = None): + """Learning artifacts (flashcards/examples) per topic, grouped by block norm — per subblock.""" + rows = await get_sub_artefakte(topic, type) out: dict[str, dict] = {} for r in rows: - b = out.setdefault(r["baustein_norm"], {"baustein": r["baustein"], "karteikarte": [], "beispiel": []}) - if r["baustein"] and not b["baustein"]: - b["baustein"] = r["baustein"] + b = out.setdefault(r["block_norm"], {"block": r["block"], "flashcard": [], "example": []}) + if r["block"] and not b["block"]: + b["block"] = r["block"] try: - daten = json.loads(r["daten"]) + data = json.loads(r["data"]) except (ValueError, TypeError): continue - if r["typ"] in ("karteikarte", "beispiel"): - b[r["typ"]].append({"subbaustein": r["sub_titel"], **daten}) + if r["type"] in ("flashcard", "example"): + b[r["type"]].append({"subblock": r["sub_title"], **data}) return {"artefakte": out} -# --- Baustein-Lernen: Chat, Prüfung --- +# --- Block learning: chat, exam --- -@router.get("/bausteine/lernstand", response_model=BausteinLernstandResponse) -async def baustein_lernstand(topic: str): - """Prüfungs-Stand pro Baustein (roher Titel als Key). cap_final = alle Subs × 25; - cap_aktuell + freie_ebene aus dem Score — für ALLE Bausteine (auch ungeprüfte).""" - progress = {p["baustein"]: p for p in await list_baustein_progress(topic)} - ebenen = await subs_je_ebene_roh(topic) +@router.get("/blocks/learnstate", response_model=BlockLearnStateResponse) +async def block_learnstate(topic: str): + """Exam state per block (raw title as key). cap_final = all subs × 25; + cap_aktuell + freie_level from the score — for ALL blocks (even unexamined).""" + progress = {p["block"]: p for p in await list_block_progress(topic)} + levels = await subs_per_level_raw(topic) - def _stand(score: int, streak: int, n_je_ebene: dict[int, int]) -> dict: + def _state(score: int, streak: int, n_je_level: dict[int, int]) -> dict: return { - "gute_antworten": score, "streak": streak, - "cap": cap_final(n_je_ebene), - "cap_aktuell": cap_aktuell(score, n_je_ebene), - "freie_ebene": freie_ebene(score, n_je_ebene), + "good_answers": score, "streak": streak, + "cap": cap_final(n_je_level), + "cap_aktuell": cap_aktuell(score, n_je_level), + "freie_level": freie_level(score, n_je_level), } - bausteine = { - b: _stand(progress[b]["gute_antworten"] if b in progress else 0, + blocks = { + b: _state(progress[b]["good_answers"] if b in progress else 0, progress[b]["streak"] if b in progress else 0, n) - for b, n in ebenen.items() + for b, n in levels.items() } - # Altbestand-Bausteine mit Prüfung, aber ohne Subs → leere Ebenen (cap 0). + # Legacy blocks with an exam but without subs → empty levels (cap 0). for b, p in progress.items(): - if b not in bausteine: - bausteine[b] = _stand(p["gute_antworten"], p["streak"], {}) - return {"bausteine": bausteine} + if b not in blocks: + blocks[b] = _state(p["good_answers"], p["streak"], {}) + return {"blocks": blocks} -@router.post("/bausteine/chat", response_model=BausteinChatResponse) -async def baustein_chat_route(req: BausteinChatRequest): - reply = await baustein_chat( - req.topic, req.baustein, req.section, req.section_kompakt, +@router.post("/blocks/chat", response_model=BlockChatResponse) +async def block_chat_route(req: BlockChatRequest): + reply = await block_chat( + req.topic, req.block, req.section, req.section_compact, [m.model_dump() for m in req.messages], provider=req.provider, ) return {"reply": reply} -# Bewertungen je (topic, baustein) serialisieren — sonst überschreiben zwei -# gleichzeitige Bewertungen den absoluten Score mit veralteter Basis (Race). -_pruef_locks: dict[tuple[str, str], asyncio.Lock] = {} +# Serialize ratings per (topic, block) — otherwise two simultaneous ratings would +# overwrite the absolute score with a stale base (race). +_check_locks: dict[tuple[str, str], asyncio.Lock] = {} -def _pruef_lock(topic: str, baustein: str) -> asyncio.Lock: - key = (topic, baustein) - lock = _pruef_locks.get(key) +def _check_lock(topic: str, block: str) -> asyncio.Lock: + key = (topic, block) + lock = _check_locks.get(key) if lock is None: - lock = _pruef_locks[key] = asyncio.Lock() + lock = _check_locks[key] = asyncio.Lock() return lock -def _basis(stand: dict, frage: str) -> tuple[int, bool]: - """Score-Basis VOR der Frage. Gleiche offene Frage → Re-Bewertung auf derselben Basis - (idempotent); sonst neue Frage auf dem aktuellen Stand. → (basis, re_bewertung).""" - re_bewertung = stand["offene_frage"] == frage and stand["offene_basis"] is not None - return (stand["offene_basis"] if re_bewertung else stand["gute_antworten"]), re_bewertung +def _basis(state: dict, question: str) -> tuple[int, bool]: + """Score base BEFORE the question. Same open question → re-rating on the same base + (idempotent); otherwise a new question on the current state. → (basis, re_rating).""" + re_rating = state["offene_question"] == question and state["offene_basis"] is not None + return (state["offene_basis"] if re_rating else state["good_answers"]), re_rating -def _farbe(punkte: int) -> str: - """Punkte-Delta → grobe Einfärbung der Bubble.""" - return "gut" if punkte > 0 else ("neutral" if punkte == 0 else "schlecht") +def _color(points: int) -> str: + """Points delta → rough bubble coloring.""" + return "gut" if points > 0 else ("neutral" if points == 0 else "schlecht") -async def _buche(req, frage: str, niveau: str, n_je_ebene: dict[int, int]) -> dict: - """Score+Streak driftfrei buchen (Lock + offene_frage/offene_streak-Anker). Niveau → - Punkt-Delta (streak-moduliert) bzw. progressiver Malus bei Fehler. cap_aktuell wird aus - der Basis abgeleitet (verzögerte Freischaltung an der Ebenen-Schwelle); Element einmalig - ab Anfänger-Stufe. Re-Bewertung derselben Frage nutzt den offenen Streak-Anker → idempotent.""" - async with _pruef_lock(req.topic, req.baustein): - stand = await get_baustein_progress(req.topic, req.baustein) - war_stufe = stand["absolviert"] is not None # Element-Guard: schon je angelegt? - basis, re_bewertung = _basis(stand, frage) - streak_basis = stand["offene_streak"] if re_bewertung else stand["streak"] - if not re_bewertung: - await set_offene_frage(req.topic, req.baustein, frage, basis, stand["streak"]) - s = schwellen(n_je_ebene) +async def _book_score(req, question: str, tier: str, n_je_level: dict[int, int]) -> dict: + """Book score+streak drift-free (lock + open-question/open-streak anchor). Tier → + points delta (streak-modulated) or progressive malus on error. cap_aktuell is derived + from the base (delayed unlock at the level threshold); element once from beginner level. + Re-rating of the same question uses the open streak anchor → idempotent.""" + async with _check_lock(req.topic, req.block): + state = await get_block_progress(req.topic, req.block) + was_level = state["completed"] is not None # element guard: ever created already? + basis, re_rating = _basis(state, question) + streak_basis = state["offene_streak"] if re_rating else state["streak"] + if not re_rating: + await set_open_question(req.topic, req.block, question, basis, state["streak"]) + s = thresholds(n_je_level) cf = s[-1] - ca = cap_aktuell(basis, n_je_ebene) - floor = floor_aus_score(basis, cf, s) - d, neue_streak = punkte_delta(niveau, streak_basis, basis, ca) - score = score_berechnen(basis, d, floor, ca, cf) - punkte = score - basis - gute, streak = await set_baustein_score_and_streak(req.topic, req.baustein, score, neue_streak) - # Lern-Element einmalig anlegen, sobald die erste Stufe (Anfänger) erreicht ist. - if not war_stufe and stufe_aus_score(score, cf) is not None: - if await set_baustein_absolviert(req.topic, req.baustein): - asyncio.create_task(baustein_element_anlegen(req.topic, req.baustein, req.section, req.provider)) - return {"punkte": punkte, "bewertung": _farbe(punkte), "gute_antworten": gute, "streak": streak, "cap": cf} + ca = cap_aktuell(basis, n_je_level) + floor = floor_from_score(basis, cf, s) + d, new_streak = points_delta(tier, streak_basis, basis, ca) + score = compute_score(basis, d, floor, ca, cf) + points = score - basis + good, streak = await set_block_score_and_streak(req.topic, req.block, score, new_streak) + # Create the learning element once, as soon as the first level (beginner) is reached. + if not was_level and level_from_score(score, cf) is not None: + if await set_block_completed(req.topic, req.block): + asyncio.create_task(create_block_element(req.topic, req.block, req.section, req.provider)) + return {"points": points, "rating": _color(points), "good_answers": good, "streak": streak, "cap": cf} -@router.post("/bausteine/pruefung", response_model=BausteinPruefungResponse) -async def baustein_pruefung_route(req: BausteinPruefungRequest): - stand = await get_baustein_progress(req.topic, req.baustein) - gute = stand["gute_antworten"] - n_je_ebene = await subs_je_ebene(req.topic, req.baustein) - cap = cap_final(n_je_ebene) - niveau = stufe_aus_score(gute, cap) or "anfaenger" # Adressaten-Rolle der Frage - fe = freie_ebene(gute, n_je_ebene) # nur freigeschaltete Subs prüfen - kompakt = req.section_kompakt +@router.post("/blocks/exam", response_model=BlockExamResponse) +async def block_exam_route(req: BlockExamRequest): + state = await get_block_progress(req.topic, req.block) + good = state["good_answers"] + n_je_level = await subs_per_level(req.topic, req.block) + cap = cap_final(n_je_level) + tier = level_from_score(good, cap) or "beginner" # addressee role of the question + fe = freie_level(good, n_je_level) # only check unlocked subs + compact = req.section_compact msgs = [m.model_dump() for m in req.messages] - if req.aktion == "frage": - if req.muster.strip(): - # Aus gezogenem Muster eine konkrete Frage im Niveau formulieren (kein Dedup nötig). - frage = await pruefung_frage_variante(req.topic, req.baustein, req.section, kompakt, req.muster, niveau=niveau, provider=req.provider) + if req.action == "question": + if req.pattern.strip(): + # From a drawn pattern, phrase a concrete question at the tier (no dedup needed). + question = await exam_question_variant(req.topic, req.block, req.section, compact, req.pattern, tier=tier, provider=req.provider) else: - # Fallback (kein Muster-Sidecar): Live-Generierung, Fokus nur auf freigeschaltete Subs. - subs = await subbausteine_frei(req.topic, req.baustein, fe) - frage = await pruefung_frage(req.topic, req.baustein, req.section, kompakt, msgs, subbausteine=subs, vermeide=req.vermeide, niveau=niveau, provider=req.provider) - if frage is None: - raise HTTPException(502, "Frage fehlgeschlagen — bitte erneut versuchen") - return {"frage": frage, "gute_antworten": gute, "cap": cap} + # Fallback (no pattern sidecar): live generation, focus only on unlocked subs. + subs = await subblocks_frei(req.topic, req.block, fe) + question = await exam_question(req.topic, req.block, req.section, compact, msgs, subblocks=subs, avoid=req.avoid, tier=tier, provider=req.provider) + if question is None: + raise HTTPException(502, "Question failed — please try again") + return {"question": question, "good_answers": good, "cap": cap} - if req.aktion == "diskussion": - if not req.frage.strip(): - raise HTTPException(400, "Diskussion braucht eine laufende Frage") - reply = await baustein_diskussion( - req.topic, req.baustein, req.section, kompakt, - req.frage, req.letzte_bewertung or None, msgs, provider=req.provider, + if req.action == "discussion": + if not req.question.strip(): + raise HTTPException(400, "Discussion needs an active question") + reply = await block_discussion( + req.topic, req.block, req.section, compact, + req.question, req.last_rating or None, msgs, provider=req.provider, ) if reply is None: - raise HTTPException(502, "Diskussion fehlgeschlagen — bitte erneut versuchen") - return {"reply": reply, "gute_antworten": gute, "cap": cap} + raise HTTPException(502, "Discussion failed — please try again") + return {"reply": reply, "good_answers": good, "cap": cap} - # --- Quiz: leicht (1 von 4) +1/−1 · schwer (x von 4) +3/−1 — deterministisch --- - if req.aktion == "quiz_frage": - if not req.muster.strip(): - raise HTTPException(400, "Quiz braucht ein Muster") - distraktoren = await huerden_distraktor_block(req.topic, req.baustein) - quiz = await quiz_generieren(req.topic, req.baustein, req.section, kompakt, req.muster, niveau=niveau, provider=req.provider, distraktor_block=distraktoren) + # --- Quiz: easy (1 of 4) +1/−1 · hard (x of 4) +3/−1 — deterministic --- + if req.action == "quiz_question": + if not req.pattern.strip(): + raise HTTPException(400, "Quiz needs a pattern") + distractors = await hurdles_distractor_block(req.topic, req.block) + quiz = await generate_quiz(req.topic, req.block, req.section, compact, req.pattern, tier=tier, provider=req.provider, distractor_block=distractors) if quiz is None: - raise HTTPException(502, "Quiz-Frage fehlgeschlagen — bitte erneut versuchen") - return {"frage": quiz["frage"], "optionen": quiz["optionen"], - "gute_antworten": gute, "cap": cap} + raise HTTPException(502, "Quiz question failed — please try again") + return {"question": quiz["question"], "options": quiz["options"], + "good_answers": good, "cap": cap} - if req.aktion == "quiz_antwort": - if not req.frage.strip(): - raise HTTPException(400, "Quiz-Antwort braucht eine Frage") - getroffen = set(req.auswahl) == set(req.korrekt) # exakt die richtige Menge - res = await _buche(req, req.frage, "stark" if getroffen else "kaum", n_je_ebene) - res["feedback"] = "Richtig — alle korrekten getroffen." if getroffen else "Nicht ganz — die markierten waren richtig." + if req.action == "quiz_answer": + if not req.question.strip(): + raise HTTPException(400, "Quiz answer needs a question") + hit = set(req.selection) == set(req.correct) # exactly the correct set + res = await _book_score(req, req.question, "strong" if hit else "barely", n_je_level) + res["feedback"] = "Correct — all correct ones hit." if hit else "Not quite — the marked ones were correct." return res - # --- Lückentext: leicht (Begriff aus 4) +1/−1 · schwer (frei tippen) +3/−1 --- - if req.aktion == "lueck_frage": - if not req.muster.strip(): - raise HTTPException(400, "Lückentext braucht ein Muster") + # --- Gap text: easy (term from 4) +1/−1 · hard (free typing) +3/−1 --- + if req.action == "gap_question": + if not req.pattern.strip(): + raise HTTPException(400, "Gap text needs a pattern") if req.schwer: - lt = await lueckentext_generieren(req.topic, req.baustein, req.section, kompakt, req.muster, niveau=niveau, provider=req.provider) + lt = await generate_gaptext(req.topic, req.block, req.section, compact, req.pattern, tier=tier, provider=req.provider) if lt is None: - raise HTTPException(502, "Lückentext fehlgeschlagen — bitte erneut versuchen") - return {"satz": lt["satz"], "loesung": lt["loesung"], "alternativen": lt["alternativen"], - "gute_antworten": gute, "cap": cap} - distraktoren = await huerden_distraktor_block(req.topic, req.baustein) - lw = await lueckwahl_generieren(req.topic, req.baustein, req.section, kompakt, req.muster, niveau=niveau, provider=req.provider, distraktor_block=distraktoren) + raise HTTPException(502, "Gap text failed — please try again") + return {"sentence": lt["sentence"], "solution": lt["solution"], "alternatives": lt["alternatives"], + "good_answers": good, "cap": cap} + distractors = await hurdles_distractor_block(req.topic, req.block) + lw = await generate_gapchoice(req.topic, req.block, req.section, compact, req.pattern, tier=tier, provider=req.provider, distractor_block=distractors) if lw is None: - raise HTTPException(502, "Lückentext fehlgeschlagen — bitte erneut versuchen") - return {"satz": lw["satz"], "optionen": lw["optionen"], - "gute_antworten": gute, "cap": cap} + raise HTTPException(502, "Gap text failed — please try again") + return {"sentence": lw["sentence"], "options": lw["options"], + "good_answers": good, "cap": cap} - if req.aktion == "lueck_antwort": - if not req.frage.strip(): - raise HTTPException(400, "Lückentext-Antwort braucht einen Satz") - if req.schwer: # frei getippt → Synonym-tolerante KI-Prüfung - ok = await lueckentext_pruefen(req.topic, req.baustein, req.frage, req.loesung, req.alternativen, req.eingabe, provider=req.provider) - feedback = "Richtig!" if ok else f"Nicht ganz — erwartet war „{req.loesung}“." - else: # Begriff aus 4 gewählt → deterministisch - ok = set(req.auswahl) == set(req.korrekt) - feedback = "Richtig!" if ok else "Nicht ganz — der markierte Begriff war richtig." - res = await _buche(req, req.frage, "stark" if ok else "kaum", n_je_ebene) + if req.action == "gap_answer": + if not req.question.strip(): + raise HTTPException(400, "Gap-text answer needs a sentence") + if req.schwer: # free typed → synonym-tolerant AI check + ok = await check_gaptext(req.topic, req.block, req.question, req.solution, req.alternatives, req.input, provider=req.provider) + feedback = "Correct!" if ok else f"Not quite — expected „{req.solution}“." + else: # term chosen from 4 → deterministic + ok = set(req.selection) == set(req.correct) + feedback = "Correct!" if ok else "Not quite — the marked term was correct." + res = await _book_score(req, req.question, "strong" if ok else "barely", n_je_level) res["feedback"] = feedback return res - # aktion "antwort" (Agent 1 schnell) / "antwort_pruefen" (Agent 2 genau). + # action "answer" (Agent 1 fast) / "answer_check" (Agent 2 thorough). if not any(m.get("role") == "user" for m in msgs): - raise HTTPException(400, "Antwort braucht eine Nutzer-Antwort") - if not req.frage.strip(): - raise HTTPException(400, "Antwort braucht eine laufende Frage") + raise HTTPException(400, "Answer needs a user answer") + if not req.question.strip(): + raise HTTPException(400, "Answer needs an active question") - if req.aktion == "antwort": - # Agent 1: nur Vorschau — Niveau + voraussichtliche Punkte, NICHTS persistieren, kein Anker. - data = await pruefung_bewertung_schnell( - req.topic, req.baustein, req.section, kompakt, req.frage, msgs, provider=req.provider, + if req.action == "answer": + # Agent 1: preview only — tier + expected points, persist NOTHING, no anchor. + data = await exam_rating_fast( + req.topic, req.block, req.section, compact, req.question, msgs, provider=req.provider, ) if data is None: - raise HTTPException(502, "Bewertung fehlgeschlagen — bitte erneut versuchen") - basis, re_bew = _basis(stand, req.frage) - streak_basis = stand["offene_streak"] if re_bew else stand["streak"] - s = schwellen(n_je_ebene) - ca = cap_aktuell(basis, n_je_ebene) - floor = floor_aus_score(basis, s[-1], s) - niveau = deckel_nachfrage(data["niveau"], req.nachgefragt) - d, _ = punkte_delta(niveau, streak_basis, basis, ca) - score = score_berechnen(basis, d, floor, ca, s[-1]) - punkte = score - basis - return {"feedback": data["feedback"], "punkte": punkte, "bewertung": _farbe(punkte), - "gute_antworten": gute, "cap": cap} + raise HTTPException(502, "Rating failed — please try again") + basis, re_rating = _basis(state, req.question) + streak_basis = state["offene_streak"] if re_rating else state["streak"] + s = thresholds(n_je_level) + ca = cap_aktuell(basis, n_je_level) + floor = floor_from_score(basis, s[-1], s) + tier = cap_followup(data["tier"], req.asked_again) + d, _ = points_delta(tier, streak_basis, basis, ca) + score = compute_score(basis, d, floor, ca, s[-1]) + points = score - basis + return {"feedback": data["feedback"], "points": points, "rating": _color(points), + "good_answers": good, "cap": cap} - # aktion "antwort_pruefen" (Agent 2 genau): verbindlich, persistiert. NUR hier ändert sich der Score. - # LLM läuft OHNE Lock; gebucht wird kurz über _buche (Anker + Score), wie bei Quiz/Lück. - # So blockiert die lange KI-Bewertung keine folgende (deterministische) Antwort desselben Bausteins. - data = await pruefung_bewertung( - req.topic, req.baustein, req.section, kompakt, req.frage, msgs, provider=req.provider, - role="guide" if req.gruendlich else "judge", begruendung=req.begruendung, + # action "answer_check" (Agent 2 thorough): binding, persisted. ONLY here does the score change. + # The LLM runs WITHOUT a lock; booking is done briefly via _book_score (anchor + score), as with quiz/gap. + # This way the long AI rating doesn't block a following (deterministic) answer of the same block. + data = await exam_rating( + req.topic, req.block, req.section, compact, req.question, msgs, provider=req.provider, + role="guide" if req.thorough else "judge", reason=req.reason, ) if data is None: - raise HTTPException(502, "Bewertung fehlgeschlagen — bitte erneut versuchen") - niveau = deckel_nachfrage(data["niveau"], req.nachgefragt) - res = await _buche(req, req.frage, niveau, n_je_ebene) # kurzer Lock: Basis driftfrei über Anker + raise HTTPException(502, "Rating failed — please try again") + tier = cap_followup(data["tier"], req.asked_again) + res = await _book_score(req, req.question, tier, n_je_level) # short lock: drift-free base via anchor res["feedback"] = data["feedback"] return res @@ -474,10 +474,10 @@ async def baustein_pruefung_route(req: BausteinPruefungRequest): @router.post("/guides", response_model=GuideResponse) async def create(req: GuideCreateRequest): - guides, progress, levels = await lade_lernstand() - grund = guide_lock(req.topic.strip(), req.format, guides, progress, levels) - if grund: - raise HTTPException(400 if grund == "Erst Bausteine erstellen" else 409, grund) + guides, progress, levels = await load_learnstate() + reason = guide_lock(req.topic.strip(), req.format, guides, progress, levels) + if reason: + raise HTTPException(400 if reason == "Erst Blocks erstellen" else 409, reason) # string matches rules.py contract await create_topic(req.topic.strip()) now = datetime.now(timezone.utc).isoformat() guide = { @@ -502,45 +502,45 @@ async def list_all(): @router.get("/guides/locks") async def guide_locks(topic: str): - """Sperr-Gründe pro Format für den ▶-Button — None = erstellbar.""" - guides, progress, levels = await lade_lernstand() + """Lock reasons per format for the ▶ button — None = creatable.""" + guides, progress, levels = await load_learnstate() return {fmt: guide_lock(topic, fmt, guides, progress, levels) for fmt in ("FullGuide", "Rest", *FORMATE)} @router.get("/guides/steps") async def guide_steps(topic: str): - """Höchster voll abgeschlossener Schritt-Index je Format (artefakt-basiert, -1 = keiner). - Treibt die klickbaren Schritt-Kugeln (wie die Bausteine-Phasen).""" - return {fmt: guide_fertig_step(guide_content_path(topic, fmt)) for fmt in ("Guide", "FullGuide", "Rest")} + """Highest fully completed step index per format (artifact-based, -1 = none). + Drives the clickable step bubbles (like the blocks phases).""" + return {fmt: guide_done_step(guide_content_path(topic, fmt)) for fmt in ("Guide", "FullGuide", "Rest")} @router.get("/guides/{guide_id}", response_model=GuideResponse) async def get_one(guide_id: str): guide = await get_guide(guide_id) if guide is None: - raise HTTPException(404, "Guide nicht gefunden") + raise HTTPException(404, "Guide not found") return guide @router.get("/guides/{guide_id}/content") -async def guide_content(guide_id: str, ebene: int = 4): - """Guide-Inhalt. `ebene` (1=A · 2=F · 3=E · 4=V) filtert auf Subbausteine bis zu dieser - Ebene; 4 = Vollfassung (roh, unverändert).""" +async def guide_content(guide_id: str, level: int = 4): + """Guide content. `level` (1=A · 2=F · 3=E · 4=V) filters to subblocks up to this + level; 4 = full version (raw, unchanged).""" guide = await get_guide(guide_id) if guide is None: - raise HTTPException(404, "Guide nicht gefunden") + raise HTTPException(404, "Guide not found") if guide["status"] != "done": - raise HTTPException(404, "Inhalt nicht verfügbar") + raise HTTPException(404, "Content not available") stored = await get_guide_content(guide["topic"], guide["format"]) # DB-first if stored is None: - path = guide_content_path(guide["topic"], guide["format"]) # Fallback: Datei (Alt-Themen) + path = guide_content_path(guide["topic"], guide["format"]) # fallback: file (legacy topics) if not path.exists(): - raise HTTPException(404, "Datei nicht gefunden") + raise HTTPException(404, "File not found") stored = path.read_text(encoding="utf-8") - if ebene >= 4: - return Response(content=stored, media_type="application/json") # Vollfassung roh + if level >= 4: + return Response(content=stored, media_type="application/json") # full version, raw try: - return content_fuer_ebene(json.loads(stored), ebene) + return content_fuer_level(json.loads(stored), level) except ValueError: return Response(content=stored, media_type="application/json") @@ -549,7 +549,7 @@ async def guide_content(guide_id: str, ebene: int = 4): async def guide_chat(guide_id: str, req: GuideChatRequest): guide = await get_guide(guide_id) if guide is None: - raise HTTPException(404, "Guide nicht gefunden") + raise HTTPException(404, "Guide not found") reply = await chat_with_guide( guide["topic"], guide["format"], req.section, req.outline, [m.model_dump() for m in req.messages], @@ -561,29 +561,29 @@ async def guide_chat(guide_id: str, req: GuideChatRequest): async def _guide_tf(guide_id: str) -> tuple[str, str]: guide = await get_guide(guide_id) if guide is None: - raise HTTPException(404, "Guide nicht gefunden") + raise HTTPException(404, "Guide not found") return guide["topic"], guide["format"] @router.post("/guides/{guide_id}/block/pruefen", response_model=BlockPruefenResponse) async def block_pruefen_route(guide_id: str, req: BlockPruefenRequest): topic, fmt = await _guide_tf(guide_id) - neu = await block_pruefen(topic, fmt, req.baustein, req.stelle, req.block, req.hinweis, provider=req.provider) - if neu is None: - raise HTTPException(502, "Prüfung fehlgeschlagen — bitte erneut versuchen") - return {"neu": neu} + new = await block_pruefen(topic, fmt, req.block, req.spot, req.snippet, req.hint, provider=req.provider) + if new is None: + raise HTTPException(502, "Check failed — please try again") + return {"revised": new} @router.post("/guides/{guide_id}/block/uebernehmen", response_model=BlockUebernehmenResponse) -async def block_uebernehmen_route(guide_id: str, req: BlockUebernehmenRequest): +async def block_adopt_route(guide_id: str, req: BlockUebernehmenRequest): topic, fmt = await _guide_tf(guide_id) - res = await block_uebernehmen(topic, fmt, req.baustein, req.stelle, req.alt, req.neu) + res = await block_adopt(topic, fmt, req.block, req.spot, req.alt, req.revised) if res is None: - raise HTTPException(404, "Section nicht gefunden") + raise HTTPException(404, "Section not found") return res -# --- Elemente (persönliche Zusammenfassung) --- +# --- Elements (personal summary) --- @router.get("/elements", response_model=list[ElementResponse]) async def get_elements(topic: str): @@ -603,7 +603,7 @@ async def post_element(req: ElementCreateRequest): async def element_chat(element_id: str, req: ElementChatRequest): element = await get_element(element_id) if element is None: - raise HTTPException(404, "Element nicht gefunden") + raise HTTPException(404, "Element not found") reply, changes = await chat_with_element(element, [m.model_dump() for m in req.messages], provider=req.provider) return {"reply": reply, "changes": changes} @@ -612,17 +612,17 @@ async def element_chat(element_id: str, req: ElementChatRequest): async def element_refine(element_id: str, req: ElementRefineRequest): element = await get_element(element_id) if element is None: - raise HTTPException(404, "Element nicht gefunden") + raise HTTPException(404, "Element not found") change = await refine_suggestion(element, req.suggestion.model_dump(), req.instruction, provider=req.provider) if change is None: - raise HTTPException(502, "Überarbeitung fehlgeschlagen — bitte erneut versuchen") + raise HTTPException(502, "Revision failed — please try again") return {"change": change} @router.put("/elements/{element_id}", response_model=ElementResponse) async def put_element(element_id: str, req: ElementUpdateRequest): if await get_element(element_id) is None: - raise HTTPException(404, "Element nicht gefunden") + raise HTTPException(404, "Element not found") fields = req.model_dump(exclude_unset=True, exclude_none=True) if fields: now = datetime.now(timezone.utc).isoformat() @@ -634,10 +634,10 @@ async def put_element(element_id: str, req: ElementUpdateRequest): async def element_style(element_id: str, req: ElementCheckRequest): element = await get_element(element_id) if element is None: - raise HTTPException(404, "Element nicht gefunden") + raise HTTPException(404, "Element not found") changes = await style_element(element, provider=req.provider) if changes is None: - raise HTTPException(502, "Stil-Prüfung fehlgeschlagen — bitte erneut versuchen") + raise HTTPException(502, "Style check failed — please try again") return {"changes": changes} @@ -645,17 +645,17 @@ async def element_style(element_id: str, req: ElementCheckRequest): async def element_check(element_id: str, req: ElementCheckRequest): element = await get_element(element_id) if element is None: - raise HTTPException(404, "Element nicht gefunden") + raise HTTPException(404, "Element not found") suggestions = await check_element(element, provider=req.provider) if suggestions is None: - raise HTTPException(502, "Prüfung fehlgeschlagen — bitte erneut versuchen") + raise HTTPException(502, "Check failed — please try again") return {"suggestions": suggestions} @router.delete("/elements/{element_id}") async def remove_element(element_id: str): if not await delete_element(element_id): - raise HTTPException(404, "Element nicht gefunden") + raise HTTPException(404, "Element not found") return {"ok": True} @@ -663,7 +663,7 @@ async def remove_element(element_id: str): async def cancel(guide_id: str): cancelled = await cancel_guide(guide_id) if not cancelled: - raise HTTPException(404, "Kein aktiver Prozess gefunden") + raise HTTPException(404, "No active process found") return {"ok": True} @@ -671,18 +671,18 @@ async def cancel(guide_id: str): async def remove(guide_id: str, slots: bool = False): guide = await get_guide(guide_id) if guide is None: - raise HTTPException(404, "Guide nicht gefunden") + raise HTTPException(404, "Guide not found") await delete_progress(guide_id) await delete_guide(guide_id) - # Content-/Schritt-Dateien teilen sich alle Läufe eines Thema+Formats — erst löschen, - # wenn kein Eintrag sie mehr braucht. Teilfortschritt (Schritt-Dateien ohne fertigen - # Content) bleibt fürs Resume erhalten, außer es wird explizit verlangt (slots=1). + # Content/step files are shared by all runs of a topic+format — only delete them + # once no entry needs them anymore. Partial progress (step files without finished + # content) is kept for resume, unless explicitly requested (slots=1). rest = [g for g in await list_guides() if g["topic"] == guide["topic"] and g["format"] == guide["format"]] if not rest: await delete_guide_content(guide["topic"], guide["format"]) content = guide_content_path(guide["topic"], guide["format"]) if slots or content.exists(): - for p in guide_slot_dateien(content): + for p in guide_slot_files(content): p.unlink(missing_ok=True) content.unlink(missing_ok=True) return {"ok": True} @@ -692,7 +692,7 @@ async def remove(guide_id: str, slots: bool = False): async def get_progress(guide_id: str): guide = await get_guide(guide_id) if guide is None: - raise HTTPException(404, "Guide nicht gefunden") + raise HTTPException(404, "Guide not found") return {"chapters": await list_progress(guide_id)} @@ -700,6 +700,6 @@ async def get_progress(guide_id: str): async def update_progress(guide_id: str, req: ProgressUpdate): guide = await get_guide(guide_id) if guide is None: - raise HTTPException(404, "Guide nicht gefunden") + raise HTTPException(404, "Guide not found") await set_progress(guide_id, req.chapter, req.done) return {"chapters": await list_progress(guide_id)} diff --git a/backend/rules.py b/backend/rules.py new file mode 100644 index 0000000..c669039 --- /dev/null +++ b/backend/rules.py @@ -0,0 +1,141 @@ +"""Learning-debt rules: progression and cap for open guides — the ONLY source. + +Rules (new creations only; topics + blocks unlimited): +- Format "Guide": at most 3 created, not-yet-completed guides +- No more progression/prerequisite (only a single guide format). +- Completed: ALL blocks (section titles) of the latest finished guide have + a passed exam. The rest is read-only (no progress, no exam). +All functions work on data loaded once (load_learnstate) — no more +query loops per guide. +""" + +import json + +from database import list_block_scores_all, subs_per_level_all, list_guides, list_progress_all +from guide import guide_slot_files +from learning import cap_final, LEVELS, _threshold +from paths import blocks_path, guide_content_path +from textkit import _norm_title + +MAX_OFFENE_GUIDES = 3 +# Only ONE format "Guide" left (all relevant blocks, exam 0–cap). No progression, +# no prerequisite → the guide is always unlockable. "Rest"/FullGuide are separate. +PRESTAGE: dict[str, str] = {} +FREISCHALT_LEVEL: dict[str, str] = {} +FORMATE = ("Guide",) + +# 4 learning levels (floor in % of the cap) — keys from learning.LEVELS. +_LEVEL_WORT = { + "beginner": "to beginner (20%)", + "advanced": "to advanced (40%)", + "expert": "to expert (60%)", + "master": "to mastery (100%)", +} + + +async def load_learnstate() -> tuple[list[dict], dict[str, set[str]], dict[str, dict[str, set[str]]]]: + """Guides + chapter progress + blocks per level. + + levels: {"beginner"/"advanced"/"expert"/"master": {topic → normalized title}}. + The level per block is derived from score + cap (4×relevant subs). + """ + scores = await list_block_scores_all() + subs_by_level = await subs_per_level_all() + levels: dict[str, dict[str, set[str]]] = {key: {} for key, _ in LEVELS} + for topic, block, score in scores: + cf = cap_final(subs_by_level.get((topic, _norm_title(block)), {})) + for key, p in LEVELS: + if cf and score >= _threshold(p, cf): + levels[key].setdefault(topic, set()).add(_norm_title(block)) + return await list_guides(), await list_progress_all(), levels + + +def _content_json(topic: str, fmt: str) -> dict | None: + path = guide_content_path(topic, fmt) + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except ValueError: + return None + + + + +def _section_title(topic: str, fmt: str) -> set[str] | None: + """Normalized block titles (sections) from the guide content.""" + content = _content_json(topic, fmt) + if content is None: + return None + return { + _norm_title(s.get("title", "")) + for ch in content.get("chapters", []) + for s in ch.get("sections", []) + } + + +def _latest_done(guides: list[dict], fmt: str) -> dict[str, dict]: + """Per topic, the latest finished guide of this format.""" + latest: dict[str, dict] = {} + for g in guides: + if g["format"] == fmt and g["status"] == "done": + if g["topic"] not in latest or g["created_at"] > latest[g["topic"]]["created_at"]: + latest[g["topic"]] = g + return latest + + +def _guide_all(g: dict, progress: dict[str, set[str]], levelset: dict[str, set[str]]) -> bool: + """Are ALL blocks of the guide at the required level?""" + sections = _section_title(g["topic"], g["format"]) + return bool(sections) and sections <= levelset.get(g["topic"], set()) + + +def is_level(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levelset: dict[str, set[str]]) -> bool: + """Latest finished guide (topic+format): all blocks at the level of levelset?""" + g = _latest_done(guides, fmt).get(topic) + return g is not None and _guide_all(g, progress, levelset) + + +def ist_completed(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> bool: + """All blocks of the latest finished guide at least beginner (≥20%)?""" + return is_level(topic, fmt, guides, progress, levels["beginner"]) + + +def topic_completed(topic: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> bool: + """Topic done: latest finished guide, all blocks at master (100%)?""" + return is_level(topic, "Guide", guides, progress, levels["master"]) + + +def formats_stats(guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> dict: + """Per format created/completed — per topic only the latest finished guide counts.""" + formats = {} + for fmt in FORMATE: + latest = _latest_done(guides, fmt) + completed = sum(1 for g in latest.values() if _guide_all(g, progress, levels["beginner"])) + formats[fmt] = {"created": len(latest), "completed": completed} + return formats + + +def guide_lock(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> str | None: + """Reason why a fresh start for topic+format is locked — None = allowed. + + Exactly the rules from POST /guides: blocks required, no duplicate start, + learning debt only for genuine new creations (resume/regenerate are free). + """ + if not blocks_path(topic).exists(): + return "Create blocks first" + for g in guides: + if g["topic"] == topic and g["format"] == fmt and g["status"] in ("queued", "generating"): + return "Generation already running" + content = guide_content_path(topic, fmt) + if not content.exists() and not guide_slot_files(content): + prereq = PRESTAGE.get(fmt) + if prereq: + level = FREISCHALT_LEVEL[fmt] # completed=10 · understood=20 · mastered=30 + if not is_level(topic, prereq, guides, progress, levels[level]): + return f"First take the {prereq} of this topic {_LEVEL_WORT[level]}" + stat = formats_stats(guides, progress, levels).get(fmt, {"created": 0, "completed": 0}) + open_count = stat["created"] - stat["completed"] + if open_count >= MAX_OFFENE_GUIDES: + return f"Complete {fmt}s first — at most {MAX_OFFENE_GUIDES} open allowed ({open_count} open)" + return None diff --git a/backend/textkit.py b/backend/textkit.py index d177d97..c8df3ae 100644 --- a/backend/textkit.py +++ b/backend/textkit.py @@ -1,19 +1,19 @@ -"""Reine Text-Helfer: Titel-Normalisierung, Listen-Parser, Chunk-Aufteilung. +"""Pure text helpers: title normalization, list parsers, chunk splitting. -Kein Zustand, keine IO — überall gefahrlos importierbar. +No state, no IO — safe to import anywhere. """ import re import unicodedata -_CATEGORIES = ("KERN", "WICHTIG", "REST") # nur noch für den Altformat-Reader +_CATEGORIES = ("KERN", "WICHTIG", "REST") # only for the legacy-format reader now -def _norm_titel(s: str) -> str: - """Normalisiert einen Titel für den Schlüssel-Vergleich. +def _norm_title(s: str) -> str: + """Normalize a title for key comparison. - NFKC + casefold fangen Unicode-Varianten; Anführungszeichen, Markdown- - Emphasis und Dash-Varianten kommen aus KI-Output in allen Spielarten. + NFKC + casefold catch Unicode variants; quotes, markdown emphasis + and dash variants come out of AI output in every shape. """ s = unicodedata.normalize("NFKC", s) s = re.sub(r"[`'\"<>„“”‚’«»*_]", "", s) @@ -22,49 +22,49 @@ def _norm_titel(s: str) -> str: return s.casefold() -def _titel(entry: str) -> str: +def _title(entry: str) -> str: return entry.split(" — ")[0].strip() or entry -def _eindeutige_titel(entries: dict[int, str]) -> dict[int, str]: - """Macht Titel eindeutig (Suffix " (2)", " (3)" …), damit sie als Schlüssel taugen.""" +def _unique_title(entries: dict[int, str]) -> dict[int, str]: + """Make titles unique (suffix " (2)", " (3)" …) so they work as keys.""" seen: dict[str, int] = {} out: dict[int, str] = {} for num, text in entries.items(): - titel = _titel(text) - key = _norm_titel(titel) + title = _title(text) + key = _norm_title(title) seen[key] = seen.get(key, 0) + 1 if seen[key] > 1: rest = text.split(" — ", 1) - text = f"{titel} ({seen[key]})" + (f" — {rest[1]}" if len(rest) == 2 else "") - # zweiter Durchlauf nicht nötig: Suffixe kollidieren praktisch nicht + text = f"{title} ({seen[key]})" + (f" — {rest[1]}" if len(rest) == 2 else "") + # a second pass isn't needed: suffixes practically never collide out[num] = text return out -def _titel_index(entries: dict[int, str]) -> dict[str, int]: - return {_norm_titel(_titel(text)): num for num, text in entries.items()} +def _title_index(entries: dict[int, str]) -> dict[str, int]: + return {_norm_title(_title(text)): num for num, text in entries.items()} -def _titel_aufloesen(idx: dict[str, int], t: str) -> int | None: - """Titel → Nummer; toleriert mitgeschleppte Beschreibungen ("Titel — …").""" +def _resolve_title(idx: dict[str, int], t: str) -> int | None: + """Title → number; tolerates trailing descriptions ("Title — …").""" if not isinstance(t, str): return None - return idx.get(_norm_titel(t)) or idx.get(_norm_titel(_titel(t))) + return idx.get(_norm_title(t)) or idx.get(_norm_title(_title(t))) def _norm_dash(s: str) -> str: - """Space-umgebene Dash-Varianten (en/em/figure/bar/hyphen) → einheitlicher Trenner ' — '. - Manche Modelle (v.a. nicht-westliche) setzen statt des Em-Dashs einen En-Dash „–"; ohne - Normalisierung scheitert der ` — `-Split komplett und der ganze Eintrag wird zum Titel. - ASCII-Bindestrich „-" bleibt unangetastet (sonst zerlegt es Formeln wie „n - 1").""" + """Space-surrounded dash variants (en/em/figure/bar/hyphen) → uniform separator ' — '. + Some models (especially non-western ones) use an en-dash "–" instead of the em-dash; without + normalization the ` — ` split fails entirely and the whole entry becomes the title. + The ASCII hyphen "-" is left untouched (otherwise it would split formulas like "n - 1").""" return re.sub(r"\s+[‒–—―‐]\s+", " — ", s) -def _parse_auswahl(text: str) -> dict[int, str]: - """Parst eine Baustein-Liste: `N. Titel — Kurzbeschreibung` pro Zeile.""" +def _parse_selection(text: str) -> dict[int, str]: + """Parse a block list: `N. Title — short description` per line.""" entries: dict[int, str] = {} last = None for line in text.splitlines(): @@ -77,8 +77,8 @@ def _parse_auswahl(text: str) -> dict[int, str]: return entries -def _parse_kategorien(text: str) -> dict[str, list[str]]: - """Altformat-Reader: finale Baustein-Datei mit ## KERN/WICHTIG/REST-Abschnitten.""" +def _parse_categories(text: str) -> dict[str, list[str]]: + """Legacy-format reader: final block file with ## KERN/WICHTIG/REST sections.""" cats: dict[str, list[str]] = {} current = None for line in text.splitlines(): @@ -94,40 +94,40 @@ def _parse_kategorien(text: str) -> dict[str, list[str]]: return cats -def _lade_bausteine(text: str) -> dict[int, str]: - """Lädt die finale Baustein-Datei — sortierte Liste (neu) oder Kategorien (Altformat).""" +def _load_blocks(text: str) -> dict[int, str]: + """Load the final block file — sorted list (new) or categories (legacy format).""" if re.search(r"^#+\s*KERN\b", text, re.IGNORECASE | re.MULTILINE): - cats = _parse_kategorien(text) + cats = _parse_categories(text) texts = [t for cat in _CATEGORIES for t in cats.get(cat, [])] return {i: t for i, t in enumerate(texts, 1)} - return _parse_auswahl(text) + return _parse_selection(text) _FRAGMENT_KAPITEL_RE = re.compile(r"", re.IGNORECASE) _FRAGMENT_SECTION_RE = re.compile(r"", re.IGNORECASE) _FRAGMENT_SUB_RE = re.compile(r"", re.IGNORECASE) -_FRAGMENT_BAUSTEIN_RE = re.compile(r"", re.IGNORECASE) -# Zwei Lese-Schichten je Section: kompakt (Merksätze) + ausführlich (Erklärung). -_FRAGMENT_KOMPAKT_RE = re.compile(r"", re.IGNORECASE) +_FRAGMENT_BAUSTEIN_RE = re.compile(r"", re.IGNORECASE) +# Two reading layers per section: compact (key sentences) + detailed (explanation). +_FRAGMENT_KOMPAKT_RE = re.compile(r"", re.IGNORECASE) _FRAGMENT_AUSF_RE = re.compile(r"", re.IGNORECASE) -# Lernpfad-Stufen + Rand; alte Schwierigkeits-Werte abwärtskompatibel akzeptiert. -_STUFEN = ("anfaenger", "fortgeschritten", "experte", "rand", "einfach", "mittel", "schwer") +# Learning-path levels + peripheral; old difficulty values accepted for backward compatibility. +_STUFEN = ("beginner", "advanced", "expert", "peripheral", "easy", "medium", "hard") def _parse_fragment(text: str) -> list[dict]: - """Parst eine Writer-Datei → [{kapitel, titel, md, kompakt, anker, anker_kompakt, subs}]. + """Parse a writer file → [{kapitel, title, md, compact, anker, anker_compact, subs}]. - Zwei Lese-Schichten je Section über `` / ``. Innerhalb - beider markieren ``-Marker je Subbaustein einen Block; gleicher - Sub-Titel in beiden Schichten wird gemergt → `sec["subs"] = [{stufe, titel, md, kompakt}]`. - Text VOR dem ersten Sub-Marker ist der Anker (Einordnung) → `anker`/`anker_kompakt`. - `md`/`kompakt` bleiben die VOLLE Fassung (Anker + alle Subs) — rückwärtskompatibel. + Two reading layers per section via `` / ``. Within + both, `` markers mark a block per subblock; the same + sub title in both layers is merged → `sec["subs"] = [{level, title, md, compact}]`. + Text BEFORE the first sub marker is the anchor (framing) → `anker`/`anker_compact`. + `md`/`compact` stay the FULL version (anchor + all subs) — backward compatible. """ sections: list[dict] = [] kapitel = None current = None cur_sub = None - cur_layer = "md" # Default: alles ohne Schicht-Marker ist die ausführliche Fassung + cur_layer = "md" # default: anything without a layer marker is the detailed version for line in text.splitlines(): s = line.strip() m = _FRAGMENT_KAPITEL_RE.match(s) @@ -138,14 +138,14 @@ def _parse_fragment(text: str) -> list[dict]: continue m = _FRAGMENT_SECTION_RE.match(s) if m: - current = {"kapitel": kapitel, "titel": m.group(1), "md": [], "kompakt": [], - "anker_md": [], "anker_kompakt": [], "_submap": {}, "_suborder": []} + current = {"chapters": kapitel, "title": m.group(1), "md": [], "compact": [], + "anker_md": [], "anker_compact": [], "_submap": {}, "_suborder": []} cur_sub = None cur_layer = "md" sections.append(current) continue if current is not None and _FRAGMENT_KOMPAKT_RE.match(s): - cur_layer = "kompakt" + cur_layer = "compact" cur_sub = None continue if current is not None and _FRAGMENT_AUSF_RE.match(s): @@ -154,17 +154,17 @@ def _parse_fragment(text: str) -> list[dict]: continue m = _FRAGMENT_SUB_RE.match(s) if m and current is not None: - teil = m.group(1).split("|", 1) - stufe = teil[0].strip().casefold() - titel = teil[1].strip() if len(teil) == 2 else "" - key = titel.casefold() or f"_pos{len(current['_suborder'])}" + parts = m.group(1).split("|", 1) + level = parts[0].strip().casefold() + title = parts[1].strip() if len(parts) == 2 else "" + key = title.casefold() or f"_pos{len(current['_suborder'])}" cur_sub = current["_submap"].get(key) if cur_sub is None: - cur_sub = {"stufe": stufe if stufe in _STUFEN else "anfaenger", "titel": titel, "md": [], "kompakt": []} + cur_sub = {"level": level if level in _STUFEN else "beginner", "title": title, "md": [], "compact": []} current["_submap"][key] = cur_sub current["_suborder"].append(key) - elif stufe in _STUFEN: - cur_sub["stufe"] = stufe + elif level in _STUFEN: + cur_sub["level"] = level continue if current is not None: current[cur_layer].append(line) @@ -178,24 +178,24 @@ def _parse_fragment(text: str) -> list[dict]: for key in sec["_suborder"]: sub = sec["_submap"][key] sub["md"] = "\n".join(sub["md"]).strip() - sub["kompakt"] = "\n".join(sub["kompakt"]).strip() - if sub["md"] or sub["kompakt"]: + sub["compact"] = "\n".join(sub["compact"]).strip() + if sub["md"] or sub["compact"]: subs.append(sub) out.append({ - "kapitel": sec["kapitel"], "titel": sec["titel"], + "chapters": sec["chapters"], "title": sec["title"], "md": "\n".join(sec["md"]).strip(), - "kompakt": "\n".join(sec["kompakt"]).strip(), - "anker": "\n".join(sec["anker_md"]).strip(), - "anker_kompakt": "\n".join(sec["anker_kompakt"]).strip(), + "compact": "\n".join(sec["compact"]).strip(), + "anchor": "\n".join(sec["anker_md"]).strip(), + "anker_compact": "\n".join(sec["anker_compact"]).strip(), "subs": subs, }) return out -def _parse_subbausteine(text: str) -> dict[str, list[str]]: - """Parst eine Subbaustein-Datei → {Baustein-Titel: [Subbaustein, …]} in Reihenfolge. +def _parse_subblocks(text: str) -> dict[str, list[str]]: + """Parse a subblock file → {block title: [subblock, …]} in order. - Format: `` gefolgt von Listenzeilen `- Subbaustein`. + Format: `` followed by list lines `- Subblock`. """ out: dict[str, list[str]] = {} current = None @@ -215,7 +215,7 @@ def _parse_subbausteine(text: str) -> dict[str, list[str]]: def _split_chunks(chapters: list[dict], n: int) -> list[list[dict]]: - """Teilt Kapitel in bis zu n zusammenhängende Chunks, balanciert nach Section-Anzahl.""" + """Split chapters into up to n contiguous chunks, balanced by section count.""" n = max(1, min(n, len(chapters))) chunks: list[list[dict]] = [] current: list[dict] = [] diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 0a03024..5449bb4 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -1,10 +1,10 @@