This commit is contained in:
team3
2026-07-03 11:45:27 +02:00
parent 285317927d
commit abcadd145d
44 changed files with 1909 additions and 292 deletions

View File

@@ -106,8 +106,8 @@ _topic_sems: dict[str, _PrioritySemaphore] = {}
# Within board 2 the LATE stages win (outline → artefacts → … → subblocks): finish cards
# instead of opening new WIP, so the makespan tail block gets slots before fresh work.
_STAGE_PRIORITY = ("research", "ingest", "cluster", "pair", "clarify", "naming", "filter",
"grouping", "supplement", "outline", "artifact", "question", "relevance",
"level", "facts", "subblock")
"dedup", "grouping", "gruppierung", "supplement", "outline", "artifact",
"question", "relevance", "level", "facts", "subblock")
def _agent_priority(key: str) -> int:

View File

@@ -24,9 +24,9 @@ from pathlib import Path
import database as db
import embedding
from agents import kill_process, cancel_scope, clear_scope, run_agent
from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP, GROUP_RECONCILE_FLOOR, GROUP_MIN_COS_FLOOR, SUB_VARIANT_COS, SEED_COVER_COS
from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP, GROUP_RECONCILE_FLOOR, GROUP_MIN_COS_FLOOR, SUB_VARIANT_COS, SEED_COVER_COS, EVIDENCE_BUDGET_CHARS, EVIDENCE_CTX_LINES
from fsutil import atomic_write_text, atomic_write_json
from jsonio import read_json_file as _json_file
from jsonio import parse_json_text, read_json_file as _json_file
from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder
from crawl import crawl
from pipeline import (
@@ -61,6 +61,8 @@ CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (dedups everything); ab
DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair (complete-link aggregates → no chaining)
DEDUP_PAIRS_CHUNK = 40 # pairs per judge package (pairwise verification instead of a block mixer)
DEDUP_TITLE_AUTO = 0.95 # near-identical TITLE cosine ⇒ same entity → merge without the judge (recall net)
DEDUP_GLOBAL_FLOOR = 0.65 # global post-naming dedup stage: candidate floor above the 0.5-0.65
# same-domain noise band, below the sibling zone (~0.85) — the judge decides there
FILTER_CHUNK = 35 # blocks to assess per judge in the degrade pass (full list as context)
# Balance question-pattern chunks by sub load via LPT (makespan), not by block count.
QUESTION_CHUNK_SUBS = 50 # target sum of relevant subs per chunk
@@ -308,9 +310,11 @@ def cancel_blocks(topic: str) -> bool:
async def blocks_status(topic: str) -> dict:
"""Kanban-based status: `generating` from the run registry, progress from the card
counts. `partial` = cards sit in non-terminal columns while nothing runs (continue-able)."""
ready = blocks_path(topic).exists() # inventory written → block overview available
generating = topic in _blocks_progress
counts = await db.kanban_stage_counts(topic)
# ready = finished inventory. The DB is the source of truth (a synced topic may lack
# blocks.md, the file is a legacy mirror) — either signal counts.
ready = blocks_path(topic).exists() or counts.get("inventory", {}).get("done_block", 0) > 0
terminal = {"clustered", "done_cluster", "grouped", "rejected", "done_block", "done_artefact", "dead"}
open_cards = sum(n for stages in counts.values()
for stage, n in stages.items() if stage not in terminal)
@@ -366,28 +370,92 @@ def _supplement_schema(data):
return out
def _convert_pdfs(project: Path) -> None:
"""Convert PDFs in the project to .txt (pdftotext) — agents read text instead of page images.
def _ocr_languages() -> str | None:
"""Installierte Tesseract-Sprachen (deu/eng), None → OCR aus."""
try:
import pymupdf
base = Path(pymupdf.get_tessdata())
except Exception:
return None
langs = [l for l in ("deu", "eng") if (base / f"{l}.traineddata").exists()]
return "+".join(langs) or None
Called before every project generation; converts only if the
.txt is missing or older than the PDF. The original is left untouched.
If pdftotext is missing and the project contains PDFs → hard error instead of
an unreliable direct-read mode (MiniMax image limit, vision cost).
"""
def _pdf_markdown(pdf: Path) -> str | None:
"""pymupdf4llm → Markdown string (None if the lib is missing or it fails)."""
try:
import pymupdf4llm
except ImportError:
return None
try:
# OCR nur, wenn Tesseract-Sprachdaten wirklich vorhanden sind —
# sonst wirft der OCR-Pfad und die ganze Datei faellt auf pdftotext.
langs = _ocr_languages()
kwargs = {"use_ocr": True, "ocr_language": langs} if langs else {"use_ocr": False}
return pymupdf4llm.to_markdown(str(pdf), show_progress=False, **kwargs)
except Exception:
log.warning("pymupdf4llm failed for %s", pdf.name, exc_info=True)
return None
def _pdf_plaintext(pdf: Path) -> str | None:
"""pdftotext -layout → plain string (None if missing/fails)."""
if shutil.which("pdftotext") is None:
return None
try:
out = subprocess.run(["pdftotext", "-layout", str(pdf), "-"],
check=True, timeout=120, capture_output=True)
return out.stdout.decode("utf-8", errors="replace")
except Exception:
log.warning("pdftotext failed for %s", pdf.name, exc_info=True)
return None
# Content-fidelity guard between the two converters (topic-NEUTRAL: measures loss, not domain).
# pymupdf4llm yields structured Markdown but silently DROPS rendered display formulas and can
# splinter combining diacritics (measured on a LaTeX script: „P = {L …}" gone, „h¨aufig").
# pdftotext is structure-poor but faithful. Take the Markdown only when it preserves the bulk
# of the content; otherwise the faithful plaintext wins.
_PDF_FIDELITY_SYMBOLS = "≤≥∈∉⊆∪∧∨¬→Σδα{}="
def _pick_conversion(md: str | None, plain: str | None) -> tuple[str, str] | None:
if md is None and plain is None:
return None
if md is None:
return plain, "pdftotext"
if plain is None:
return md, "pymupdf4llm"
ok_len = len(md) >= 0.7 * len(plain)
sym_plain = sum(plain.count(c) for c in _PDF_FIDELITY_SYMBOLS)
ok_sym = sym_plain == 0 or sum(md.count(c) for c in _PDF_FIDELITY_SYMBOLS) >= 0.8 * sym_plain
ok_diakritik = md.count("\u00a8") <= plain.count("\u00a8") + 2 # standalone ¨ = splintered umlauts
if ok_len and ok_sym and ok_diakritik:
return md, "pymupdf4llm"
return plain, "pdftotext"
def _convert_pdfs(project: Path) -> None:
"""Convert PDFs in the project to .txt — agents read text instead of page images.
Called before every project generation; converts only if the .txt is missing or
older than the PDF. Both converters run; the fidelity guard picks the better result
per file. Neither available → hard error instead of an unreliable direct-read mode
(MiniMax image limit, vision cost)."""
pdfs = list(project.rglob("*.pdf"))
if not pdfs:
return
if shutil.which("pdftotext") is None:
raise RuntimeError("pdftotext missing (install poppler-utils) — PDFs in the project cannot be read")
for pdf in pdfs:
txt = pdf.with_suffix(".txt")
if txt.exists() and txt.stat().st_mtime >= pdf.stat().st_mtime:
continue
try:
subprocess.run(["pdftotext", "-layout", str(pdf), str(txt)], check=True, timeout=120)
_log(project.name, f"PDF converted: {pdf.name}{txt.name}")
except Exception as e:
raise RuntimeError(f"PDF conversion failed ({pdf.name}): {e}") from e
picked = _pick_conversion(_pdf_markdown(pdf), _pdf_plaintext(pdf))
if picked is None:
raise RuntimeError(f"PDF conversion failed ({pdf.name}): weder pymupdf4llm noch "
"pdftotext verfügbar/erfolgreich (pip install pymupdf4llm oder poppler-utils)")
text, tool = picked
txt.write_text(text, encoding="utf-8")
_log(project.name, f"PDF konvertiert ({tool}): {pdf.name}{txt.name}")
_SOURCE_TEMPLATE = {"projekt": "Blocks-Source-Projekt", "uni": "Blocks-Source-Uni", "link": "Blocks-Source-Link"}
@@ -429,6 +497,142 @@ def _text_sections(text: str, goal: int = RESEARCH_SECTION_CHARS) -> list[str]:
return sections
# ── Inline evidence for judges ──────────────────────────────────────────────────────
# Judges used to re-search the corpus per session ({source} → "ls/find … read", ~10 tool
# turns each). The corpus excerpts now go INTO the prompt; the agent answers as text.
def _corpus_files(folder: Path | None, sources: list[str] | None) -> list[Path]:
"""The block's named source .txt files; fallback: every .txt in the folder."""
if folder is None or not folder.is_dir():
return []
if sources:
named = [folder / Path(s).with_suffix(".txt").name for s in sources if s]
named = [p for p in named if p.is_file()]
if named:
return named
return sorted(p for p in folder.glob("*.txt") if p.is_file())
def _q_tokens(text: str) -> set[str]:
return set(re.findall(r"\w{3,}", text.casefold()))
def _evidence_pack(folder: Path | None, sources: list[str] | None, queries: list[str],
budget: int = EVIDENCE_BUDGET_CHARS) -> str:
"""Keyword-selected corpus excerpts for a judge prompt. Sections are ranked by token
overlap with `queries` (block title + candidates); every query with any match gets its
best section (coverage guarantee), the rest of the budget takes the global top. Empty
string when there is no corpus — the caller keeps the old self-research source then."""
parts: list[tuple[str, int, str, set[str]]] = [] # (file, idx, text, tokens)
for f in _corpus_files(folder, sources):
try:
text = f.read_text(encoding="utf-8")
except OSError:
continue
for i, sec in enumerate(_text_sections(text), 1):
parts.append((f.name, i, sec, _q_tokens(sec)))
if not parts:
return ""
qtoks = [(_q_tokens(q)) for q in queries if q]
score = [sum(len(qt & p[3]) for qt in qtoks) for p in parts]
chosen: set[int] = set()
for qt in qtoks: # coverage guarantee: best section per query
best = max(range(len(parts)), key=lambda k: len(qt & parts[k][3]), default=None)
if best is not None and qt & parts[best][3]:
chosen.add(best)
used = sum(len(parts[k][2]) for k in chosen)
for k in sorted(range(len(parts)), key=lambda k: -score[k]): # top-up to budget
if k in chosen or score[k] <= 0:
continue
if used + len(parts[k][2]) > budget:
continue
chosen.add(k)
used += len(parts[k][2])
out, total = [], 0
for k in sorted(chosen): # document order for readability
fname, i, sec, _t = parts[k]
if total + len(sec) > max(budget, used): # hard cap incl. guarantee overshoot
break
out.append(f"── {fname} · Abschnitt {i} ──\n{sec}")
total += len(sec)
return "\n\n".join(out)
_CITE_POS = re.compile(r"\b(?:Z(?:eilen?)?|lines?)\.?\s*(\d+)(?:\s*[-]\s*(\d+))?", re.I)
def _cite_ref(cite: str, files: list[Path]) -> tuple[Path, int, int] | None:
"""(file, line_lo, line_hi) from a cited_facts source string
(„Skript.txt, Übung 6.47, Z.1341-1344") — None when file or position is missing."""
c = (cite or "").casefold()
f = next((p for p in files if p.name.casefold() in c or p.stem.casefold() in c), None)
m = _CITE_POS.search(cite or "")
if f is None or m is None:
return None
lo, hi = int(m.group(1)), int(m.group(2) or m.group(1))
return (f, min(lo, hi), max(lo, hi))
def _cited_evidence(folder: Path | None, sources: list[str] | None, cites: list[str],
fallback_queries: list[str], budget: int = EVIDENCE_BUDGET_CHARS) -> str:
"""Evidence for the facts check: the EXACT cited regions (±EVIDENCE_CTX_LINES, merged,
line numbers in the header) — precise and tiny. Cites without a parseable position fall
back to the keyword pack. Empty string without a corpus."""
files = _corpus_files(folder, sources)
if not files:
return ""
ranges: dict[Path, list[tuple[int, int]]] = {}
unresolved = False
for c in cites:
ref = _cite_ref(c, files)
if ref is None:
unresolved = True
continue
f, lo, hi = ref
ranges.setdefault(f, []).append((max(1, lo - EVIDENCE_CTX_LINES), hi + EVIDENCE_CTX_LINES))
out, total = [], 0
for f in files:
if f not in ranges:
continue
try:
lines = f.read_text(encoding="utf-8").splitlines()
except OSError:
continue
merged: list[list[int]] = []
for lo, hi in sorted(ranges[f]):
hi = min(hi, len(lines))
if merged and lo <= merged[-1][1] + 1:
merged[-1][1] = max(merged[-1][1], hi)
else:
merged.append([lo, hi])
for lo, hi in merged:
sec = "\n".join(lines[lo - 1:hi])
if not sec.strip() or total + len(sec) > budget:
continue
out.append(f"── {f.name} · Z. {lo}-{hi} ──\n{sec}")
total += len(sec)
if unresolved or not out:
pack = _evidence_pack(folder, sources, fallback_queries, max(0, budget - total))
if pack:
out.append(pack)
return "\n\n".join(out)
def _reply_text(result) -> str:
"""Assistant text of a no-tool agent call ((rc, stdout, stderr) from run_agent)."""
return (result[1] or "") if result else ""
def _sink_json(result, path: Path, schema):
"""Payload validator for no-tool agents: the JSON comes as reply TEXT; the engine
persists it to `path`, so resume guards and audit files keep working unchanged."""
data = parse_json_text(_reply_text(result))
val = schema(data)
if val is not None:
atomic_write_json(path, data)
return val
def _build_research_prompt(topic: str, out_path: Path, instructions: str, type: str, folder: Path | None, fokus: str = "", section: str = "") -> str:
if section:
# Section mode (uni/projekt): text directly in the prompt → small context, no file reading.
@@ -603,7 +807,7 @@ def _variant_clusters(titles: list[str], mentions: list[int], sims) -> list[dict
async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str,
wipe: bool = True, ns: str = "", seeds: list[str] | None = None,
lbl: str = "") -> dict | None:
lbl: str = "", sources: list[str] | None = None) -> dict | None:
"""Block B (DB + loop): per package, find subblocks in rounds (3 finders, until 0 new/cap),
collect in the DB (variant-clustered mentions ≥2 = consensus), a judge panel cleans up per
package; blocks below SUBBLOCK_MIN get focused catch-up rounds; `seeds` (demoted fragment
@@ -832,11 +1036,25 @@ async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, i
for _, p in pending:
p.unlink(missing_ok=True)
if pending:
# Inline evidence: corpus excerpts in the prompt (no self-research); the judge
# answers as TEXT, the engine persists the j-file (resume + majority unchanged).
ev = _evidence_pack(folder, sources,
[title_by_num[num] for num in chunk]
+ [s for num in chunk for s in shown_by_num.get(num, [])]) if folder else ""
j_source = _prompt("Blocks-Source-Inline", excerpts=ev) if ev else source
def _sink(result, p):
text = _reply_text(result).strip()
d = _parse_subblocks(text)
if d:
atomic_write_text(p, text)
return d or None
slots = [{
"key": f"blocks-{topic}-{ns}subblock-final-c{c}{tag}-j{j}",
"prompt": _prompt("Subblock-Mapping", topic=topic, source=source, blocks="\n\n".join(block_texts), out_path=p, extra=_extra(instructions)),
"role": "judge", "capabilities": caps,
"payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None),
"prompt": _prompt("Subblock-Mapping", topic=topic, source=j_source, blocks="\n\n".join(block_texts), out_path=p, extra=_extra(instructions)),
"role": "judge", "capabilities": "none" if ev else caps,
"payload": (lambda result, p=p: _sink(result, p)),
} for j, p in pending]
existing = SUBBLOCK_PANEL - len(pending)
await _race(topic, f"{lbl}Subblock-Clarification {c}", slots, max(1, 2 - existing),
@@ -1102,8 +1320,8 @@ async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instruct
slots = [{
"key": f"blocks-{topic}-{ns}level-c{c}-{i}",
"prompt": _prompt("Levels-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)),
"role": "quick", "capabilities": "files",
"payload": (lambda result, p=p, ids=local_set: _levels_schema(_json_file(p), ids)),
"role": "quick", "capabilities": "none", # pure mapping, everything inline → text reply
"payload": (lambda result, p=p, ids=local_set: _sink_json(result, p, lambda d: _levels_schema(d, ids))),
} for i, p in pending]
new = await _race(topic, f"{lbl}Levels package {c}", slots, 2 - existing, _timeout("level", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
return not is_cancelled() and new is not None
@@ -1135,8 +1353,8 @@ async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instruct
ctx, f"{lbl}Levels-Clarification {c}",
key=f"blocks-{topic}-{ns}level-final-c{c}",
prompt=_prompt("Levels-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)),
role="judge", capabilities="files",
payload=lambda result, p=judge_path, ids=set(strittig): _levels_schema(_json_file(p), ids),
role="judge", capabilities="none",
payload=lambda result, p=judge_path, ids=set(strittig): _sink_json(result, p, lambda d: _levels_schema(d, ids)),
timeout=_timeout("level_check", len(strittig)),
)
if status == FAILED:
@@ -1245,7 +1463,7 @@ def _facts_complete(files: dict) -> bool:
return isinstance(d, dict) and bool(d)
async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str, ns: str = "", lbl: str = "") -> tuple | None:
async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str, ns: str = "", lbl: str = "", sources: list[str] | None = None) -> tuple | None:
"""Block: per sub extract source facts (find) → verify (check) → correct/discard (fix).
Extract-once grounding: the result feeds level/relevance/questions/guide.
→ (facts_map, discarded_map) — facts_map {block: {sub_norm: facts}}, discarded_map
@@ -1367,13 +1585,24 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
if not per:
return ci, set(), set()
facts_text = "\n\n".join(f"SUBBAUSTEIN: {fk['sub']}\n{_facts_lines(fk)}" for fm in per.values() for fk in fm.values())
# Inline evidence: the EXACT cited regions (falls back to keyword excerpts) go into
# the prompt; the judge answers as TEXT, the engine persists the check file.
cites = [bf.get("source", "") for fm in per.values() for fk in fm.values()
for bf in fk.get("cited_facts", [])]
fallback = [blocks[i][0] for i in idxs] + [fk["sub"] for fm in per.values() for fk in fm.values()]
ev = _cited_evidence(folder, sources, cites, fallback) if folder else ""
c_source = _prompt("Blocks-Source-Inline", excerpts=ev) if ev else source
pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _facts_check_schema(_json_file(chk_path(ci, j))) is None]
await asyncio.gather(*[
rs = await asyncio.gather(*[
run_agent(f"blocks-{topic}-{ns}facts-check-c{ci}-j{j}",
_prompt("Facts-Check", topic=topic, source=source, facts=facts_text, out_path=chk_path(ci, j), extra=_extra(instructions)),
_timeout("content_check", len(per)), provider=provider, role="judge", capabilities=caps,
_prompt("Facts-Check", topic=topic, source=c_source, facts=facts_text, out_path=chk_path(ci, j), extra=_extra(instructions)),
_timeout("content_check", len(per)), provider=provider, role="judge",
capabilities="none" if ev else caps,
scope=topic, label=f"{lbl}Facts check {ci}/{j}")
for j in pending], return_exceptions=True)
for j, r in zip(pending, rs):
if isinstance(r, tuple):
_sink_json(r, chk_path(ci, j), _facts_check_schema)
outs = [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if (s := _facts_check_schema(_json_file(chk_path(ci, j)))) is not None]
bvotes: dict[str, int] = {}
vvotes: dict[str, int] = {}
@@ -1484,8 +1713,8 @@ async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i
slots = [{
"key": f"blocks-{topic}-{ns}relevance-c{c}-{i}",
"prompt": _prompt("Relevance-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)),
"role": "quick", "capabilities": "files",
"payload": (lambda result, p=p, ids=local_set: _relevance_schema(_json_file(p), ids)),
"role": "quick", "capabilities": "none", # pure mapping, everything inline → text reply
"payload": (lambda result, p=p, ids=local_set: _sink_json(result, p, lambda d: _relevance_schema(d, ids))),
} for i, p in pending]
new = await _race(topic, f"{lbl}Relevance package {c}", slots, 2 - existing, _timeout("relevance", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
return not is_cancelled() and new is not None
@@ -1517,8 +1746,8 @@ async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i
ctx, f"{lbl}Relevance-Clarification {c}",
key=f"blocks-{topic}-{ns}relevance-final-c{c}",
prompt=_prompt("Relevance-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)),
role="judge", capabilities="files",
payload=lambda result, p=judge_path, ids=set(strittig): _relevance_schema(_json_file(p), ids),
role="judge", capabilities="none",
payload=lambda result, p=judge_path, ids=set(strittig): _sink_json(result, p, lambda d: _relevance_schema(d, ids)),
timeout=_timeout("relevance_check", len(strittig)),
)
if status == FAILED:
@@ -1677,8 +1906,8 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar:
ctx, f"{lbl}Question-Pattern-Clarification {ci}",
key=f"blocks-{topic}-{ns}question-pattern-final-c{ci}",
prompt=_prompt("Question-Pattern-Critique", topic=topic, table="\n\n".join(block_texts), out_path=fp, extra=_extra(instructions)),
role="judge", capabilities="files",
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
role="judge", capabilities="none", # pure review, everything inline → text reply
payload=lambda result, p=fp: _sink_json(result, p, _question_pattern_chunk_schema),
timeout=_timeout("question_pattern_check", subs_total),
)
if status == FAILED:
@@ -2087,11 +2316,22 @@ _CANON_STOP = re.compile(
r'von|of|für|for|und|and|zum|zur|im)\b', re.I)
# Catalogue references ("Definition 6.19", "Satz 7.8") are scaffolding INCLUDING their number —
# stripped as a phrase, so the digits don't pollute the key. Variant digits ("3-SAT") have no
# scaffolding word in front and survive.
_CANON_CATALOGUE = re.compile(
r'\b(?:definition|def|satz|lemma|korollar|corollary|theorem|proposition'
r'|kapitel|chapter|abschnitt|section)\s*\d+(?:\.\d+)*\b', re.I)
def _canonical_key(title: str) -> str:
"""Order-independent canonical key of a title (scaffolding stripped, relation operators normalized).
Two titles with the same key denote the same entity with ~100% precision (ER blocking). Empty string
if nothing survives (never auto-merged)."""
s = unicodedata.normalize("NFKC", title).casefold()
s = unicodedata.normalize("NFKC", title)
s = re.sub(r'([a-zäöüß])([A-ZÄÖÜ])', r'\1 \2', s) # CamelCase → two tokens
s = s.casefold()
s = _CANON_CATALOGUE.sub(' ', s)
s = re.sub(r'≟|\bversus\b|\bvs\.?\b|=', ' opeq ', s) # equality / "vs" → one token
s = re.sub(r'≤|⪯|→|⇒|⟹|\breduces?\s+to\b|\breduziert\b', ' opred ', s) # reduction → one token
s = _CANON_STOP.sub(' ', s)
@@ -2103,14 +2343,20 @@ def _canonical_key(title: str) -> str:
# operands AND direction (RDF-triple identity / SKOS narrowMatch — a subset/restriction is NOT the same).
# So "SAT ≤ Clique" ≠ "3-SAT ≤ Clique" (source differs) and "A → B" ≠ "B → A" (direction). Two DIFFERENT
# relations must never merge, even if a judge or a high title-cosine says so.
_REL_STRIP = re.compile(r'^\s*(?:satz|lemma|korollar|theorem)\s*[\d.]*\s*:?\s*|^\s*reduktion(?:en)?\s*:?\s*', re.I)
_REL_OPERATOR = re.compile(r'[≤⪯≥⊆⊊→⇒⟹⇔←]|=>|<=|->')
_REL_STRIP = re.compile(r'^\s*(?:satz|lemma|korollar|corollary|theorem|proposition)\s*[\d.]*\s*:?\s*'
r'|^\s*redu[ck]tion(?:en|s)?\s*:?\s*', re.I)
# trailing scaffolding ("SetCover ≤ HittingSet Reduktion") is NOT part of the target operand —
# without this strip the guard false-alarms and blocks the correct merge with the bare relation
_REL_STRIP_TAIL = re.compile(r'[\s\-]*(?:redu[ck]tion(?:en|s)?|transformation(?:en|s)?)\s*$', re.I)
# an attached p/m marker ("≤p", "≤m") is operator notation, not part of the right operand
_REL_OPERATOR = re.compile(r'[≤⪯≥⊆⊊→⇒⟹⇔←][pm]?|=>|<=|->', re.I)
def _relation_operands(title: str) -> tuple[str, str] | None:
"""(canonical_source, canonical_target) of a relation/reduction title, else None (not a relation).
Operands canonicalized (lowercased, non-alphanumerics stripped) so spacing/hyphenation don't matter."""
t = _REL_STRIP.sub('', title, count=1)
t = _REL_STRIP_TAIL.sub('', t, count=1)
m = _REL_OPERATOR.search(t)
if not m:
return None
@@ -2702,12 +2948,16 @@ async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i
for k, e in enumerate(items, 1))
pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _example_check_schema(_json_file(cpath(j))) is None]
if pending:
await asyncio.gather(*[
# ground truth (facts) is fully inline → no tools, text reply, engine persists
rs = await asyncio.gather(*[
run_agent(f"blocks-{topic}-{ns}artifact-example-check-c{ci}-j{j}",
_prompt("Artifact-Example-Check", topic=topic, facts=block_text(idxs), examples=examples_txt, out_path=cpath(j), extra=_extra(instructions)),
_timeout("content_check", len(items)), provider=provider, role="judge", capabilities=caps,
_timeout("content_check", len(items)), provider=provider, role="judge", capabilities="none",
scope=topic, label=f"{lbl}Beispiel-Check {ci}/{j}")
for j in pending], return_exceptions=True)
for j, r in zip(pending, rs):
if isinstance(r, tuple):
_sink_json(r, cpath(j), _example_check_schema)
outs = [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if (s := _example_check_schema(_json_file(cpath(j)))) is not None]
if not outs:
return items # no exam possible → keep (best-effort)

View File

@@ -81,6 +81,7 @@ def make_spawner(topic: str, files: dict):
await db.kanban_upsert_card(topic, BOARD, norm, "ablock", "subblocks", {
"title": payload.get("title", ""),
"description": payload.get("description", ""),
"n_size": payload.get("n_size", 0), # LPT estimate until subs_n exists
})
return spawn
@@ -165,7 +166,8 @@ async def _proc_subblocks(ctx: GenContext, flow: Flow, files: dict, instructions
+ "\n".join(f"- {s}" for s in sd))
raw = await _subblocks_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm),
{1: _entry_line(p)}, instr, wipe=False, ns=f"{_safe(norm)}-",
seeds=sd or None, lbl=f"{p.get('title', norm)} · ")
seeds=sd or None, lbl=f"{p.get('title', norm)} · ",
sources=p.get("sources"))
if raw is None:
return _fail_or_cancel(ctx, f"Subblocks {p.get('title', norm)}")
p["raw"] = raw
@@ -186,7 +188,7 @@ async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
raw = p.get("raw") or {}
res = await _facts_block(ctx, _card_set_p(flow, norm), _pfiles(files, norm), raw, q,
folder, instructions, ns=f"{_safe(norm)}-",
lbl=f"{p.get('title', norm)} · ")
lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
if res is None:
return _fail_or_cancel(ctx, f"Facts {p.get('title', norm)}")
facts_map, discarded = res

View File

@@ -14,6 +14,7 @@ Stages (cards):
naming cluster judge picks the canonical member title
naming_check cluster second judge verifies → spawns the block card
fragment_filter block BARRIER/drain: global re-merge + degrade pass (full list)
dedup block BARRIER/drain: global judge-verified pair dedup (incl. context)
grouping block BARRIER/drain: umbrella grouping (type gate, reconcile)
gap_check block BARRIER/drain: one supplement round (web) → feeds ingest
done block mirror into the legacy `blocks` table → done_block
@@ -34,7 +35,7 @@ import kanban
from kanban import Flow, Stage, chain_stages
import blocks
from blocks import (
DEDUP_PAIR_FLOOR, DEDUP_PAIRS_CHUNK, DEDUP_TITLE_AUTO, FILTER_CHUNK,
DEDUP_GLOBAL_FLOOR, DEDUP_PAIR_FLOOR, DEDUP_PAIRS_CHUNK, DEDUP_TITLE_AUTO, FILTER_CHUNK,
FILTER_RECHECK_PANEL, CONSOLIDATION_PANEL, RESEARCH_BATCH, RESEARCH_READERS,
RESEARCH_THEMA_AGENTS, _FILTER_NOTATION, _GROUP_STANDALONE,
_build_research_prompt, _canonical, _canonical_key, _chunk_nums, _cliques,
@@ -425,7 +426,7 @@ async def _proc_consensus_gate(ctx: GenContext, flow: Flow, cards):
rep = _rep(rows)
p = c["payload"]
p.update(title=rep["title"], description=rep["description"], readers=sorted(readers),
supplement=supplement)
supplement=supplement, n_size=len(readers)) # LPT: big evidence first
if supplement or len(readers) >= 2:
if _is_reference(rep["title"]) and not supplement:
p["quorum"] = "majority" # consensus reference title: rename/exam, not the hard bar
@@ -598,7 +599,7 @@ async def _namecheck_one(ctx: GenContext, flow: Flow, c):
sources = sorted(set().union(*[set(r.get("sources") or []) for r in rows])) if rows else []
await db.kanban_upsert_card(topic, BOARD, f"b-{cid}", "block", "fragment_filter", {
"title": p.get("title", ""), "description": p.get("description", ""),
"readers": readers, "sources": sources,
"readers": readers, "sources": sources, "n_size": len(readers),
"supplement": bool(p.get("supplement")), "cluster": cid,
})
await db.kanban_advance(topic, BOARD, cid, "done_cluster")
@@ -864,7 +865,162 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards):
"floor_veto": [allrows[nr - 1]["title"] for nr in floor_veto]}, indent=1)
_log(topic, f"Fragment-Filter: {n_dem}{n_dem - len(journal)} ({len(journal)})")
demoted = {cid for cid, _ in moves}
moves += [(r["card_id"], "grouping") for r in rows if r["card_id"] not in demoted]
moves += [(r["card_id"], "dedup") for r in rows if r["card_id"] not in demoted]
await db.kanban_advance_many(topic, BOARD, moves)
flow.wake.set()
async def _proc_dedup(ctx: GenContext, flow: Flow, cards):
"""BARRIER/drain — global pair dedup over the NAMED blocks: filter's re-merge only
catches key-exact/≥0.95 titles inside the stage; here embedding candidates (mean OR
title cosine ≥ DEDUP_GLOBAL_FLOOR) + canonical-key blocking go to a TWO-judge panel
(merge needs unanimity — single judges conflate variants with their base entity),
auto edges (title ≥0.95, exact key) with relation guard, complete-link cliques merge
into the champion (readers/sources union). The second wave (supplement) compares
against the already-confirmed context blocks. Journal carries every pair verdict."""
topic = flow.topic
work_dir = flow.work_dir
rows = [{"card_id": c["card_id"], "payload": c["payload"],
"title": c["payload"].get("title", ""),
"description": c["payload"].get("description") or ""} for c in cards]
context = await _context_blocks(topic, exclude={r["card_id"] for r in rows})
allrows = rows + context
n_dem, n_all = len(rows), len(allrows)
async def _pass_through():
await db.kanban_advance_many(topic, BOARD, [(r["card_id"], "grouping") for r in rows])
flow.wake.set()
if n_all < 2 or not await _emb_ok(flow):
await _pass_through()
return
vf = await _vec_rows(flow, [_t_text(r) for r in allrows])
# titles casefolded: ALL-CAPS variants ("VERTEX COVER" vs "Vertex Cover (VC)") tank the
# raw title cosine below the candidate floor
vt = await _vec_rows(flow, [r["title"].casefold() for r in allrows])
if vf is None or vt is None:
await _pass_through()
return
sims_title = vt @ vt.T
sims = (vf @ vf.T + sims_title) / 2
def _demotable_pair(a: int, b: int) -> bool:
return a < n_dem or b < n_dem # confirmed context blocks never merge among themselves
pairs: set[tuple[int, int]] = set()
for i in range(n_all):
for j in range(i + 1, n_all):
# title-only cosine as second candidate source: descriptions of the same
# entity often stress different facets and dilute the mean below the floor
if _demotable_pair(i, j) and (float(sims[i][j]) >= DEDUP_GLOBAL_FLOOR
or float(sims_title[i][j]) >= DEDUP_GLOBAL_FLOOR):
pairs.add((i, j))
keys: dict[str, list[int]] = {}
for i, r in enumerate(allrows):
if (k := _canonical_key(r["title"])):
keys.setdefault(k, []).append(i)
for grp in keys.values():
for x in range(len(grp)):
for y in range(x + 1, len(grp)):
if _demotable_pair(grp[x], grp[y]):
pairs.add((grp[x], grp[y]))
ordered = sorted(pairs)
h = _h(*[r["card_id"] for r in allrows])
if not ordered:
atomic_write_json(work_dir / f"inventar-dedup-{h}.json",
{"vorher": n_dem, "paare": 0, "merged": []}, indent=1)
await _pass_through()
return
chunks = [ordered[k:k + DEDUP_PAIRS_CHUNK] for k in range(0, len(ordered), DEDUP_PAIRS_CHUNK)]
async def _judge(ci, chunk, jj):
path = work_dir / f"dedup-{h}-c{ci}-j{jj}.json"
if _pairs_schema(_json_file(path)) is not None:
return # resume
lines = "\n\n".join(f"{j + 1}.\nA: {_t_text(allrows[a])}\nB: {_t_text(allrows[b])}"
for j, (a, b) in enumerate(chunk))
status, _v = await run_single_slot(
ctx, f"Dedup {ci} j{jj}", key=f"blocks-{topic}-dedup-{h}-c{ci}-j{jj}",
prompt=_prompt("Blocks-Dedup", topic=topic, pairs=lines, out_path=path),
role="judge", capabilities="files",
payload=lambda result, p=path: _pairs_schema(_json_file(p)),
timeout=_timeout("selection_mapping", len(chunk)))
if status == FAILED:
raise RuntimeError(f"Dedup chunk {ci} j{jj} ohne Ergebnis")
# two-judge panel per chunk (one wave): a merge needs UNANIMITY — the observed
# failure mode is a single judge conflating a variant with its base entity
results = await asyncio.gather(*[_judge(ci, c, jj) for ci, c in enumerate(chunks)
for jj in (1, 2)], return_exceptions=True)
errs = [r for r in results if isinstance(r, Exception)]
if errs:
raise errs[0]
if ctx.is_cancelled():
return
edges: list[tuple[int, int]] = []
detail: dict[tuple[int, int], str] = {}
def _edge(a, b, kanal):
if _relation_conflict(allrows[a]["title"], allrows[b]["title"]):
detail[(a, b)] = "guard_veto"
else:
edges.append((a, b))
detail[(a, b)] = kanal
for ci, chunk in enumerate(chunks):
v1 = _pairs_schema(_json_file(work_dir / f"dedup-{h}-c{ci}-j1.json")) or {}
v2 = _pairs_schema(_json_file(work_dir / f"dedup-{h}-c{ci}-j2.json")) or {}
for j, (a, b) in enumerate(chunk):
ja1, ja2 = bool(v1.get(j + 1)), bool(v2.get(j + 1))
if ja1 and ja2:
_edge(a, b, "ja")
else:
detail[(a, b)] = "nein" if not (ja1 or ja2) else "uneinig"
for a, b in ordered: # auto recall net: near-identical titles merge without the judges
if float(sims_title[a][b]) >= DEDUP_TITLE_AUTO:
_edge(a, b, "auto_titel")
for grp in keys.values(): # exact-canonical-key auto-merge
for x in range(len(grp)):
for y in range(x + 1, len(grp)):
if _demotable_pair(grp[x], grp[y]):
_edge(grp[x], grp[y], "auto_key")
moves: list[tuple[str, str]] = []
journal: list[dict] = []
gone: set[int] = set()
for g in _cliques(n_all, edges):
ctx_members = [k for k in g if k >= n_dem]
if ctx_members:
rep = ctx_members[0] # confirmed block always wins
else:
rep = max(g, key=lambda k: (-_aspect_marker(allrows[k]["title"]),
len(allrows[k]["description"]), len(allrows[k]["title"]), -k))
rp = allrows[rep]["payload"]
changed = False
for k in g:
if k == rep or k >= n_dem:
continue # context members stay untouched
lp = allrows[k]["payload"]
rp["readers"] = sorted(set(rp.get("readers") or []) | set(lp.get("readers") or []))
rp["sources"] = sorted(set(rp.get("sources") or []) | set(lp.get("sources") or []))
lp.update(reason="merged", merged_into=allrows[rep]["title"])
await db.kanban_set_payload(topic, BOARD, allrows[k]["card_id"], lp)
moves.append((allrows[k]["card_id"], "grouped"))
gone.add(k)
journal.append({"dublette": allrows[k]["title"], "in": allrows[rep]["title"]})
changed = True
if changed:
await db.kanban_set_payload(topic, BOARD, allrows[rep]["card_id"], rp)
if rep >= n_dem and rp.get("mirrored_norm"): # mirror refresh (sources only)
await db.upsert_block(topic, rp["mirrored_norm"], allrows[rep]["title"],
allrows[rep]["description"], rp.get("sources") or [])
moves += [(r["card_id"], "grouping") for i, r in enumerate(rows) if i not in gone]
atomic_write_json(work_dir / f"inventar-dedup-{h}.json",
{"vorher": n_dem, "paare": len(ordered), "merged": journal,
"paare_detail": [{"a": allrows[a]["title"], "b": allrows[b]["title"],
"verdict": v} for (a, b), v in sorted(detail.items())]},
indent=1)
if journal:
_log(topic, f"Dedup: {len(journal)} Dublette(n) zusammengelegt")
await db.kanban_advance_many(topic, BOARD, moves)
flow.wake.set()
@@ -1202,6 +1358,8 @@ def inventory_stages(ctx: GenContext, flow: Flow) -> list[Stage]:
Stage(BOARD, "naming_check", lambda cs: _proc_naming_check(ctx, flow, cs)),
Stage(BOARD, "fragment_filter", lambda cs: _proc_fragment_filter(ctx, flow, cs),
barrier=True, drain=True, gate=research_done),
Stage(BOARD, "dedup", lambda cs: _proc_dedup(ctx, flow, cs),
barrier=True, drain=True, gate=research_done),
Stage(BOARD, "grouping", lambda cs: _proc_grouping(ctx, flow, cs),
barrier=True, drain=True, gate=research_done),
Stage(BOARD, "gap_check", lambda cs: _proc_gap_check(ctx, flow, cs),
@@ -1305,6 +1463,7 @@ COLUMNS = [
("inventory", "naming", "Naming", "cluster"),
("inventory", "naming_check", "Naming-Check", "cluster"),
("inventory", "fragment_filter", "Fragment-Filter", "block"),
("inventory", "dedup", "Dubletten", "block"),
("inventory", "grouping", "Gruppierung", "block"),
("inventory", "gap_check", "Lücken-Check", "block"),
("inventory", "done", "Spiegeln", "block"),
@@ -1324,7 +1483,7 @@ COLUMNS = [
_TITLE_STAGES = ["ingest", "cluster"]
_CLUSTER_STAGES = ["pair_check", "consensus_gate", "clarify", "naming", "naming_check"]
_BLOCK_STAGES = ["fragment_filter", "grouping", "gap_check", "done"]
_BLOCK_STAGES = ["fragment_filter", "dedup", "grouping", "gap_check", "done"]
_ART_STAGES = ["subblocks", "facts", "levels", "relevance", "question_pattern",
"artefacts", "finalize", "outline"]
# where a requeued dead card restarts, by kind

View File

@@ -139,6 +139,12 @@ CRAWL_MIN_CHARS = 400 # too little te
QUELLE_RELEVANZ_CHUNK = 12 # pages per rater package (small, since a snippet ships per page)
QUELLE_RELEVANZ_SNIPPET = 800 # body characters per page in the prompt (URL is the primary signal)
# Inline evidence for judge agents: corpus excerpts go INTO the prompt instead of letting
# every judge re-search the source folder (measured: ~10 tool turns/judge, 82 % of the
# run's tokens were cache reads from those loops).
EVIDENCE_BUDGET_CHARS = 48_000 # max excerpt characters per judge prompt
EVIDENCE_CTX_LINES = 15 # context lines around a cited source position (facts check)
# Timeouts per agent step: (base seconds, seconds per block/section).
# Applies equally to all providers — whoever is too slow gets restarted or overtaken.
TIMEOUTS = {

View File

@@ -186,6 +186,20 @@ CREATE TABLE IF NOT EXISTS sub_artefakte (
)
"""
# Leitner learning state per flashcard — IDENTITY-keyed (no content): survives the
# sub_artefakte wipe on regeneration; orphaned rows simply never match in the deck join.
CREATE_PRACTICE_PROGRESS = """
CREATE TABLE IF NOT EXISTS practice_progress (
topic TEXT NOT NULL,
block_norm TEXT NOT NULL,
sub_norm TEXT NOT NULL,
box INTEGER NOT NULL DEFAULT 1,
due_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (topic, block_norm, sub_norm)
)
"""
# Kanban dataflow (boards 'inventory' + 'artefacts'): ONE generic card table for all card kinds
# (title/cluster/block). `stage` is the queue key — a worker pulls WHERE stage = <its input stage>.
# `payload` is a JSON blob (title, description, sources, readers, mentions, parent_norm, journal …);
@@ -307,6 +321,7 @@ async def init_db():
await db.execute(CREATE_SOURCE)
await db.execute(CREATE_GUIDE_OUTLINE)
await db.execute(CREATE_SUB_ARTEFAKTE)
await db.execute(CREATE_PRACTICE_PROGRESS)
await db.execute(CREATE_KANBAN_CARDS)
await db.execute(CREATE_EVENTS)
await db.execute(CREATE_EVENTS_INDEX)
@@ -592,8 +607,10 @@ async def subs_per_level(topic: str, block: str) -> dict[int, int]:
from textkit import _norm_title
db = await get_db()
cursor = await db.execute(
f"SELECT {_LEVEL_CASE} AS level, COUNT(*) FROM subblocks "
"WHERE topic = ? AND block_norm = ? AND status = 'consensus' GROUP BY level",
# alias must NOT be named `level`: SQLite resolves an ambiguous GROUP BY name to
# the source COLUMN, which silently miscounts peripheral subs
f"SELECT {_LEVEL_CASE} AS lv, COUNT(*) FROM subblocks "
"WHERE topic = ? AND block_norm = ? AND status = 'consensus' GROUP BY lv",
(topic, _norm_title(block)),
)
out = _empty_levels()
@@ -606,8 +623,8 @@ async def subs_per_level_raw(topic: str) -> dict[str, dict[int, int]]:
"""Subblocks per level, grouped by RAW block title (= guide section title)."""
db = await get_db()
cursor = await db.execute(
f"SELECT block, {_LEVEL_CASE} AS level, COUNT(*) FROM subblocks "
"WHERE topic = ? AND status = 'consensus' GROUP BY block, level",
f"SELECT block, {_LEVEL_CASE} AS lv, COUNT(*) FROM subblocks "
"WHERE topic = ? AND status = 'consensus' GROUP BY block, lv",
(topic,),
)
out: dict[str, dict[int, int]] = {}
@@ -620,8 +637,8 @@ async def subs_per_level_all() -> dict[tuple[str, str], dict[int, int]]:
"""Subblocks per level per (topic, block_norm) — for the topic-wide levels derivation."""
db = await get_db()
cursor = await db.execute(
f"SELECT topic, block_norm, {_LEVEL_CASE} AS level, COUNT(*) FROM subblocks "
"WHERE status = 'consensus' GROUP BY topic, block_norm, level"
f"SELECT topic, block_norm, {_LEVEL_CASE} AS lv, COUNT(*) FROM subblocks "
"WHERE status = 'consensus' GROUP BY topic, block_norm, lv"
)
out: dict[tuple[str, str], dict[int, int]] = {}
for t, bn, level, n in await cursor.fetchall():
@@ -741,11 +758,13 @@ def _card(row, cursor) -> dict:
async def kanban_pull(topic: str, board: str, stage: str, limit: int) -> list[dict]:
"""`limit` ready cards of `stage` (backoff expired). LPT: cards carrying a `subs_n`
payload field (board 2, set after subblocks) are pulled BIGGEST first — the longest
block starts earliest and stops dominating the makespan tail. Others stay FIFO."""
block starts earliest and stops dominating the makespan tail. `n_size` (board 1,
reader count) is the coarser fallback estimate. Others stay FIFO."""
db = await get_db()
cursor = await db.execute(
"""SELECT * FROM kanban_cards WHERE topic = ? AND board = ? AND stage = ? AND not_before <= ?
ORDER BY COALESCE(json_extract(payload, '$.subs_n'), 0) DESC, updated_at LIMIT ?""",
ORDER BY COALESCE(json_extract(payload, '$.subs_n'),
json_extract(payload, '$.n_size'), 0) DESC, updated_at LIMIT ?""",
(topic, board, stage, _now(), limit))
return [_card(row, cursor) for row in await cursor.fetchall()]
@@ -1211,6 +1230,30 @@ async def list_question_pattern(topic: str, block_norm: str | None = None) -> li
return [_row_to_dict(row, cursor) for row in rows]
async def count_question_pattern_blocks(topic: str) -> int:
"""Blocks that have at least one exam question pattern."""
db = await get_db()
cur = await db.execute("SELECT COUNT(DISTINCT block_norm) FROM question_pattern WHERE topic = ?", (topic,))
return (await cur.fetchone())[0]
async def count_sub_artefakte(topic: str) -> int:
db = await get_db()
cur = await db.execute("SELECT COUNT(*) FROM sub_artefakte WHERE topic = ?", (topic,))
return (await cur.fetchone())[0]
async def event_span(topic: str) -> int:
"""Minutes between first and last pipeline event of the topic (0 if none)."""
db = await get_db()
cur = await db.execute("SELECT MIN(ts), MAX(ts) FROM events WHERE topic = ?", (topic,))
lo, hi = await cur.fetchone()
if not lo or not hi:
return 0
from datetime import datetime
return int((datetime.fromisoformat(hi) - datetime.fromisoformat(lo)).total_seconds() // 60)
async def delete_question_pattern(topic: str, block_norm: str | None = None) -> None:
db = await get_db()
if block_norm is None:
@@ -1378,18 +1421,64 @@ async def put_sub_artifact(topic: str, block_norm: str, sub_norm: str, type: str
await db.commit()
async def get_sub_artefakte(topic: str, type: str | None = None) -> list[dict]:
async def get_sub_artefakte(topic: str, type: str | None = None,
block_norm: str | None = None) -> list[dict]:
db = await get_db()
if type is None:
cursor = await db.execute("SELECT * FROM sub_artefakte WHERE topic = ? ORDER BY rowid", (topic,))
else:
cursor = await db.execute(
"SELECT * FROM sub_artefakte WHERE topic = ? AND type = ? ORDER BY rowid", (topic, type)
)
sql = "SELECT * FROM sub_artefakte WHERE topic = ?"
args: list = [topic]
if type is not None:
sql += " AND type = ?"
args.append(type)
if block_norm is not None:
sql += " AND block_norm = ?"
args.append(block_norm)
cursor = await db.execute(sql + " ORDER BY rowid", args)
rows = await cursor.fetchall()
return [_row_to_dict(row, cursor) for row in rows]
async def get_practice_progress(topic: str) -> list[dict]:
db = await get_db()
cursor = await db.execute(
"SELECT block_norm, sub_norm, box, due_at FROM practice_progress WHERE topic = ?", (topic,))
rows = await cursor.fetchall()
return [_row_to_dict(row, cursor) for row in rows]
async def upsert_practice_progress(topic: str, block_norm: str, sub_norm: str,
box: int, due_at: str) -> None:
db = await get_db()
await db.execute(
"""INSERT INTO practice_progress (topic, block_norm, sub_norm, box, due_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(topic, block_norm, sub_norm)
DO UPDATE SET box = excluded.box, due_at = excluded.due_at,
updated_at = excluded.updated_at""",
(topic, block_norm, sub_norm, box, due_at, _now()))
await db.commit()
async def sub_levels_norm(topic: str) -> dict[tuple[str, str], int]:
"""(block_norm, sub_norm) → level 1-4 for the consensus subs — the practice deck's gate."""
db = await get_db()
cursor = await db.execute(
f"SELECT block_norm, sub_norm, {_LEVEL_CASE} AS level FROM subblocks "
"WHERE topic = ? AND status = 'consensus'", (topic,))
return {(bn, sn): lv for bn, sn, lv in await cursor.fetchall()}
async def subs_per_level_norm(topic: str) -> dict[str, dict[int, int]]:
"""Subblocks per level, grouped by block_norm (sub_artefakte is norm-keyed)."""
db = await get_db()
cursor = await db.execute(
f"SELECT block_norm, {_LEVEL_CASE} AS lv, COUNT(*) FROM subblocks "
"WHERE topic = ? AND status = 'consensus' GROUP BY block_norm, lv", (topic,))
out: dict[str, dict[int, int]] = {}
for bn, level, n in await cursor.fetchall():
out.setdefault(bn, _empty_levels())[level] = n
return out
async def delete_sub_artefakte(topic: str, block_norm: str | None = None) -> None:
db = await get_db()
if block_norm is None:
@@ -1426,6 +1515,6 @@ async def delete_topic_pipeline(topic: str) -> None:
NOT the topic config `source` — that is managed separately (delete_source)."""
db = await get_db()
for tab in ("blocks", "subblocks", "question_pattern", "research_coverage",
"pipeline_state", "guide_outline", "sub_artefakte", "events"):
"pipeline_state", "guide_outline", "sub_artefakte", "practice_progress", "events"):
await db.execute(f"DELETE FROM {tab} WHERE topic = ?", (topic,))
await db.commit()

View File

@@ -137,6 +137,38 @@ def _card_facts(env: _Env, block_title: str) -> str:
return grounding or env.fallback_facts
async def _card_examples(env: _Env, block_norm: str, subs: list[dict],
include_unmatched: bool = True) -> str:
"""Verified worked examples of the block as writer input, matched to `subs` via
sub_norm (a split half gets only its own). Rows whose sub does not match (generation
mismatch) go to the full writer / split part 1 so they never vanish silently."""
rows = await db.get_sub_artefakte(env.topic, type="example", block_norm=block_norm)
if not rows:
return ""
wanted = {_norm_title(s["title"]) for s in subs}
out = []
for r in rows:
matched = r["sub_norm"] in wanted
if not matched and not include_unmatched:
continue
data = json.loads(r["data"]) if isinstance(r["data"], str) else (r["data"] or {})
steps = " ".join(f"{i}) {s}" for i, s in enumerate(data.get("steps") or [], 1))
where = (f"Subbaustein „{r['sub_title']}" if matched
else "Subbaustein unklar — dort einweben, wo es fachlich passt")
out.append(f"- {where}:\n Problem: {data.get('problem', '')}\n"
f" Schritte: {steps}\n Ergebnis: {data.get('result', '')}")
if not out:
return ""
return ("VERIFIED WORKED EXAMPLES (already fact-checked; each belongs to ONE subblock):\n"
+ "\n".join(out) + "\n"
"Weave each example into the ausführlich text of EXACTLY its subblock, right "
"after the concept it applies has been explained — as a short worked-through "
"passage (problem → steps → result recognizable, flowing prose or a compact "
"numbered list). Take all values and results over VERBATIM, never recompute "
"or alter them. NEVER put examples into the compact layer. Subblocks without "
"an example get none.")
def _card_assignment(env: _Env, card: dict) -> str:
from guide import _level_label
lines = [f"- {card['block']}"]
@@ -254,7 +286,10 @@ async def _write_split(env: _Env, card: dict, ziele_text: str):
prompt=_prompt("Guide-Writer-Board", topic=env.topic, format_name=env.format,
chapter=card.get("chapter") or "Inhalte",
assignment=assignment, ziele=ziele_text,
facts=_card_facts(env, card["block"]), gaps="\n" + hints[i] + "\n",
facts=_card_facts(env, card["block"]),
examples=await _card_examples(env, norm, parts[i],
include_unmatched=(i == 0)),
gaps="\n" + hints[i] + "\n",
spec=env.spec, out_path=path, extra=_extra(env.instructions)),
role="guide", capabilities="files", payload=_payload,
timeout=_timeout("writer", 1))
@@ -300,7 +335,9 @@ async def _stage_writer(env: _Env, card: dict) -> bool:
prompt=_prompt("Guide-Writer-Board", topic=env.topic, format_name=env.format,
chapter=card.get("chapter") or "Inhalte",
assignment=_card_assignment(env, card), ziele=ziele_text,
facts=_card_facts(env, card["block"]), gaps=gaps, spec=env.spec,
facts=_card_facts(env, card["block"]),
examples=await _card_examples(env, norm, env.subs_by_title.get(card["block"], [])),
gaps=gaps, spec=env.spec,
out_path=path, extra=_extra(env.instructions)),
role="guide", capabilities="files", payload=_payload,
timeout=_timeout("writer", 1))
@@ -320,6 +357,9 @@ async def _stage_fakten_gate(env: _Env, card: dict) -> bool:
await _set(env, card, status="error", gate_info="Writer-Fragment unlesbar")
return False
facts = _card_facts(env, card["block"])
ex = await _card_examples(env, norm, env.subs_by_title.get(card["block"], []))
if ex: # the fix agent sees the same facts variable — examples survive the fix pass
facts += "\n\nVERIFIED WORKED EXAMPLES (count as verified facts for this check):\n" + ex
path = env.slot(f"gate-{_safe(norm)}-r{card['writer_rounds']}.json")
status, claims = await run_single_slot(
env.ctx, f"Fakten-Gate {card['block']}", key=f"{env.guide_id}-gate-{_safe(norm)}-r{card['writer_rounds']}",

View File

@@ -28,6 +28,25 @@ LEVELS = (("beginner", 0.2), ("advanced", 0.4), ("expert", 0.6), ("master", 1.0)
POINTS_BASE = 25 # Points per subblock. Master cap = (all subs) × 25.
# Leitner boxes for the flashcard practice deck: roughly doubling intervals cover
# session → day → week → month. Box 1 with interval 0 = a failed card stays due in
# the running session. Absolute UTC times, no day-boundary semantics (timezone-free).
LEITNER_INTERVALS = {1: 0, 2: 1, 3: 3, 4: 7, 5: 21} # days per box
LEITNER_MAX_BOX = 5
PRACTICE_NEW_PER_SESSION = 10 # new cards offered per deck fetch
def leitner_step(box: int | None, correct: bool) -> tuple[int, int]:
"""(new box, interval in days). New card + correct → box 2; wrong → box 1 (due now);
correct → one box up, capped at LEITNER_MAX_BOX."""
if not correct:
new = 1
elif box is None:
new = 2
else:
new = min(box + 1, LEITNER_MAX_BOX)
return new, LEITNER_INTERVALS[new]
def _levels(n_je_level: dict[int, int]) -> list[int]:
return [n_je_level.get(k, 0) for k in (1, 2, 3, 4)]

View File

@@ -44,6 +44,18 @@ class BlocksCardRestartRequest(BaseModel):
card_id: str = Field(min_length=1, max_length=200)
class GuideFormatRequest(BaseModel):
topic: str = Field(min_length=1)
format: str = Field(min_length=1)
class PracticeAnswerRequest(BaseModel):
topic: str = Field(min_length=1)
block_norm: str = Field(min_length=1, max_length=300)
sub_norm: str = Field(max_length=300)
correct: bool
class GuideCardResetRequest(BaseModel):
topic: str = Field(min_length=1)
format: str = Field(min_length=1)

View File

@@ -3,6 +3,7 @@ uvicorn[standard]
aiosqlite
playwright
trafilatura
pymupdf4llm
transformers
# torch NICHT hier listen — sonst zieht pip die CUDA-Variante (~2,5 GB).
# Es wird separat als CPU-Build installiert (Dockerfile + Makefile-Target `install`).

View File

@@ -2,7 +2,7 @@ import asyncio
import json
import shutil
import uuid
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from fastapi import APIRouter, HTTPException
from fastapi.responses import Response
@@ -18,10 +18,12 @@ from database import (
delete_block_data, delete_block_progress, subs_per_level, subs_per_level_raw,
delete_topic_pipeline, delete_source, get_guide_content, delete_guide_content,
get_sub_artefakte, kanban_reset, delete_guide_board,
get_practice_progress, upsert_practice_progress, sub_levels_norm, subs_per_level_norm,
)
from textkit import _norm_title
from blocks import generate_blocks, cancel_blocks, blocks_status, active_blocks, reset_blocks, load_source, load_overview, subblocks_title, subblocks_frei, load_question_pattern, load_question_pattern_free, _blocks_files
from board_inventory import add_research_agent, board_snapshot, requeue_dead, reset_board_from_stage, restart_artefact_card
from learning import block_chat, block_discussion, exam_rating, exam_rating_fast, exam_question, exam_question_variant, generate_quiz, generate_gapchoice, generate_gaptext, check_gaptext, hurdles_distractor_block, compute_score, floor_from_score, level_from_score, cap_final, cap_aktuell, freie_level, thresholds, points_delta, cap_followup
from learning import block_chat, block_discussion, exam_rating, exam_rating_fast, exam_question, exam_question_variant, generate_quiz, generate_gapchoice, generate_gaptext, check_gaptext, hurdles_distractor_block, compute_score, floor_from_score, level_from_score, cap_final, cap_aktuell, freie_level, thresholds, points_delta, cap_followup, leitner_step, PRACTICE_NEW_PER_SESSION
from guide import chat_with_guide, generate_guide, guide_slot_files, block_pruefen, block_adopt, content_fuer_level
from pipeline import cancel_guide
from rules import FORMATE, formats_stats, guide_lock, ist_completed, load_learnstate, topic_completed
@@ -29,13 +31,14 @@ from models import (
GuideCreateRequest, GuideResponse,
TopicCreateRequest,
BlocksCreateRequest, BlocksResetStageRequest, BlocksCardRestartRequest, BlocksStatusResponse,
GuideCardResetRequest,
GuideCardResetRequest, GuideFormatRequest,
GuideBoardResetRequest, GuideChatRequest, GuideChatResponse,
ProgressUpdate, ProgressResponse, ProjectResponse, ProviderInfo,
FolderResponse, BlocksSourceUpdate, BlocksSourceResponse, BlockOverview,
BlockChatRequest, BlockChatResponse,
BlockExamRequest, BlockExamResponse, BlockLearnStateResponse,
BlockPruefenRequest, BlockPruefenResponse, BlockUebernehmenRequest, BlockUebernehmenResponse,
PracticeAnswerRequest,
)
from paths import blocks_topics, guide_content_path, project_dir, topic_dir, source_path, safe_folder
from fsutil import atomic_write_json
@@ -282,6 +285,43 @@ async def update_blocks_source(req: BlocksSourceUpdate):
return data
@router.get("/blocks/completeness")
async def blocks_completeness(topic: str):
"""Beleg der Themen-Zerlegung: Bestand, Filter-Bilanz, Lernziele, Artefakte, Laufzeit."""
import glob as _glob
from pathlib import Path as _Path
from paths import arbeit_dir
from database import (kanban_stage_counts, list_blocks, list_subblocks, list_lernziele,
count_question_pattern_blocks, count_sub_artefakte, event_span)
counts = await kanban_stage_counts(topic)
inv = counts.get("inventory", {})
blocks = await list_blocks(topic, status="consensus")
subs = 0
for b in blocks:
subs += sum(1 for s in await list_subblocks(topic, b["title_norm"]) if s["status"] == "consensus")
ziele = await list_lernziele(topic)
dead = sum(v for board in counts.values() for s, v in board.items() if s == "dead")
degradiert = ueberstimmt = 0
for p in _glob.glob(str(arbeit_dir(topic) / "inventar-filter*.json")):
try:
d = json.loads(_Path(p).read_text(encoding="utf-8"))
degradiert += d.get("degradiert", 0)
ueberstimmt += len(d.get("ueberstimmt", []))
except Exception:
continue
status = await blocks_status(topic)
return {
"bloecke": len(blocks), "subs": subs,
"verworfen": inv.get("rejected", 0), "zusammengelegt": inv.get("grouped", 0),
"degradiert_geprueft": degradiert, "panel_gerettet": ueberstimmt,
"ziele_total": len(ziele), "ziele_covered": sum(1 for z in ziele if z["covered"]),
"frage_bloecke": await count_question_pattern_blocks(topic),
"lernkarten": await count_sub_artefakte(topic),
"dead": dead, "lauf_minuten": await event_span(topic),
"vollstaendig": bool(status.get("ready")) and dead == 0,
}
@router.get("/blocks/overview", response_model=list[BlockOverview])
async def get_blocks_uebersicht(topic: str):
return await load_overview(topic)
@@ -313,6 +353,68 @@ async def get_artefakte(topic: str, type: str | None = None):
return {"artefakte": out}
# --- Practice deck: Leitner flashcard pool per topic ---
async def build_practice_deck(topic: str) -> dict:
"""ONE stack per topic (spacing beats per-block mini-stacks): due cards first
(oldest due_at), then up to PRACTICE_NEW_PER_SESSION new ones. Level gate via the
block's exam score (freie_level) — locked cards are counted for transparency."""
cards = await get_sub_artefakte(topic, "flashcard")
levels = await sub_levels_norm(topic)
n_je = await subs_per_level_norm(topic)
progress = {_norm_title(p["block"]): p["good_answers"] for p in await list_block_progress(topic)}
pp = {(p["block_norm"], p["sub_norm"]): p for p in await get_practice_progress(topic)}
now = datetime.now(timezone.utc).isoformat()
due, new, future, gesperrt = [], [], [], 0
for r in cards:
bn, sn = r["block_norm"], r["sub_norm"]
n_block = n_je.get(bn)
if n_block is not None: # legacy blocks without level data pass unfiltered
if levels.get((bn, sn), 1) > freie_level(progress.get(bn, 0), n_block):
gesperrt += 1
continue
try:
data = json.loads(r["data"])
except (ValueError, TypeError):
continue
card = {"block": r["block"], "block_norm": bn, "sub_norm": sn,
"subblock": r["sub_title"], "question": data.get("question", ""),
"answer": data.get("answer", "")}
p = pp.get((bn, sn))
if p is None:
card.update(box=None, status="new")
new.append(card)
elif p["due_at"] <= now:
card.update(box=p["box"], status="due", due_at=p["due_at"])
due.append(card)
else:
future.append(p["due_at"])
due.sort(key=lambda c: c["due_at"])
new_total = len(new)
new = new[:PRACTICE_NEW_PER_SESSION]
return {"cards": due + new,
"counts": {"due": len(due), "new": len(new), "new_total": new_total,
"gesperrt": gesperrt},
"next_due_at": min(future) if future else None}
@router.get("/practice/deck")
async def practice_deck(topic: str):
return await build_practice_deck(topic)
@router.post("/practice/answer")
async def practice_answer(req: PracticeAnswerRequest):
"""Book a Leitner step. Deliberately NO existence check against sub_artefakte:
an answer during regeneration books instead of failing (worst case an orphan row)."""
pp = {(p["block_norm"], p["sub_norm"]): p for p in await get_practice_progress(req.topic)}
prev = pp.get((req.block_norm, req.sub_norm))
box, days = leitner_step(prev["box"] if prev else None, req.correct)
due_at = (datetime.now(timezone.utc) + timedelta(days=days)).isoformat()
await upsert_practice_progress(req.topic, req.block_norm, req.sub_norm, box, due_at)
return {"box": box, "due_at": due_at}
# --- Block learning: chat, exam ---
@router.get("/blocks/learnstate", response_model=BlockLearnStateResponse)
@@ -679,6 +781,25 @@ async def cancel(guide_id: str):
return {"ok": True}
@router.post("/guides/board/remove")
async def remove_guide_format(req: GuideFormatRequest):
"""Board-Remove: discard ALL runs of topic+format — old error rows pile up, and the
per-guide delete keeps the board cards until the LAST row is gone (measured: 8 rows)."""
doomed = [g for g in await list_guides() if g["topic"] == req.topic and g["format"] == req.format]
if any(g["status"] in ("queued", "generating") for g in doomed):
return {"ok": True, "status": "generating"}
for g in doomed:
await delete_progress(g["id"])
await delete_guide(g["id"])
await delete_guide_content(req.topic, req.format)
await delete_guide_board(req.topic, req.format)
content = guide_content_path(req.topic, req.format)
for p in guide_slot_files(content):
p.unlink(missing_ok=True)
content.unlink(missing_ok=True)
return {"ok": True, "removed": len(doomed)}
@router.delete("/guides/{guide_id}")
async def remove(guide_id: str, slots: bool = False):
guide = await get_guide(guide_id)

View File

@@ -25,6 +25,9 @@ def _fake_single_slot(tmp_path):
if "-pair-" in key:
pairs = prompt.count("\nA: ")
out = {"pairs": {str(i + 1): "ja" for i in range(pairs)}}
elif "-dedup-" in key:
pairs = prompt.count("\nA: ")
out = {"pairs": {str(i + 1): "nein" for i in range(pairs)}}
elif "-clarify-" in key:
keep = [line[2:].split("")[0] for line in prompt.splitlines()
if line.startswith("- ")]
@@ -57,11 +60,11 @@ async def board_env(testdb, tmp_path, monkeypatch):
return False
monkeypatch.setattr(bi, "_emb_ok", no_emb)
async def fake_subblocks(ctx, set_p, files, entries, instructions, wipe=True, ns="", seeds=None, lbl=""):
async def fake_subblocks(ctx, set_p, files, entries, instructions, wipe=True, ns="", seeds=None, lbl="", sources=None):
title = list(entries.values())[0].split("")[0]
return {title: ["Sub Eins", "Sub Zwei"]}
async def fake_facts(ctx, set_p, files, raw, q, folder, instructions, ns="", lbl=""):
async def fake_facts(ctx, set_p, files, raw, q, folder, instructions, ns="", lbl="", sources=None):
facts = {t: {_norm_title(s): {"key_points": [f"Fakt zu {s}"], "cited_facts": []}
for s in subs} for t, subs in raw.items()}
return facts, {}
@@ -134,6 +137,7 @@ async def test_board1_full_flow(board_env):
# reader union survived the pipeline (consensus evidence on the block card)
alpha = next(c for c in done if c["payload"]["title"] == "Alpha")
assert set(alpha["payload"]["readers"]) == {"r1", "r2"}
assert alpha["payload"]["n_size"] == 2 # LPT estimate travels with the card
# board 2: one artefact card per block ran through to done_artefact (+ outline singleton)
art_done = await db.kanban_cards(TOPIC, board="artefacts", stage="done_artefact")
assert len(art_done) == 5 # 4 blocks + outline card
@@ -185,7 +189,7 @@ async def test_empty_subblocks_completes_without_deadletter(board_env, monkeypat
import blocks as blx
db, ctx, files = board_env
async def empty_subs(ctx, set_p, files, entries, instructions, wipe=True, ns="", seeds=None, lbl=""):
async def empty_subs(ctx, set_p, files, entries, instructions, wipe=True, ns="", seeds=None, lbl="", sources=None):
return {}
monkeypatch.setattr(ba, "_subblocks_block", empty_subs)
await db.kanban_upsert_card(TOPIC, "artefacts", "leer", "ablock", "subblocks",
@@ -264,7 +268,7 @@ async def test_panel_confirms_demote(board_env, tmp_path, monkeypatch):
assert c1["stage"] == "rejected"
assert c1["payload"]["reason"] == "fragment"
assert c1["payload"]["parent_norm"] == "codeblock"
assert (await db.kanban_get_card(TOPIC, B, "b-2"))["stage"] == "grouping"
assert (await db.kanban_get_card(TOPIC, B, "b-2"))["stage"] == "dedup"
async def test_panel_overrules_single_vote(board_env, tmp_path, monkeypatch):
@@ -278,7 +282,7 @@ async def test_panel_overrules_single_vote(board_env, tmp_path, monkeypatch):
("b-1", {"title": "Blockzitat", "description": "Zitat mit >"}),
("b-2", {"title": "Codeblock", "description": "Code mit Einrückung"}),
])
assert (await db.kanban_get_card(TOPIC, B, "b-1"))["stage"] == "grouping"
assert (await db.kanban_get_card(TOPIC, B, "b-1"))["stage"] == "dedup"
journal = json.loads(next(tmp_path.glob("inventar-filter-*.json")).read_text(encoding="utf-8"))
assert journal["ueberstimmt"] == ["Blockzitat"]
assert journal["degradiert"] == 0
@@ -329,7 +333,7 @@ async def test_floor_vetoes_structureless_demote(board_env, tmp_path, monkeypatc
("b-3", {"title": "Aufgabenlisten (Task Lists)", "description": "Checkboxen"}),
("b-4", {"title": "Aufgabenlisten", "description": "GFM-Listen mit Checkbox"}),
])
assert (await db.kanban_get_card(TOPIC, B, "b-1"))["stage"] == "grouping" # floor veto
assert (await db.kanban_get_card(TOPIC, B, "b-1"))["stage"] == "dedup" # floor veto
assert (await db.kanban_get_card(TOPIC, B, "b-3"))["stage"] == "rejected" # containment holds
journal = json.loads(next(tmp_path.glob("inventar-filter-*.json")).read_text(encoding="utf-8"))
assert journal["floor_veto"] == ["Blockzitat"]
@@ -388,19 +392,268 @@ async def test_supplement_reopens_dead_lineage(board_env, tmp_path, monkeypatch)
assert fresh and fresh["stage"] == "ingest" and fresh["payload"]["supplement"] is True
# ── Dedup-Stage: globaler Paar-Abgleich nach dem Naming ─────────────────────────────
def _angle_vecs(mapping):
"""Vector fake with controllable cosine: mapped substring → angle (degrees) in a
shared 2D plane; unmapped texts get their own orthogonal axis (cos 0 to everything)."""
import math as m
import numpy as np
async def fake(flow, texts):
dim = 2 + len(texts)
rows = []
for i, t in enumerate(texts):
v = np.zeros(dim)
for key, deg in mapping.items():
if key in t:
v[0], v[1] = m.cos(m.radians(deg)), m.sin(m.radians(deg))
break
else:
v[2 + i] = 1.0
rows.append(v)
return np.vstack(rows)
return fake
async def _run_dedup(db, ctx, tmp_path, cards):
"""Seed block cards into dedup and run ONE barrier pass over them."""
for cid, p in cards:
await db.kanban_upsert_card(TOPIC, B, cid, "block", "dedup", p)
rows = [{"card_id": cid, "payload": dict(p)} for cid, p in cards]
await bi._proc_dedup(ctx, _mk_flow(tmp_path), rows)
@pytest.fixture
def emb_on(monkeypatch):
async def yes(flow):
return True
monkeypatch.setattr(bi, "_emb_ok", yes)
async def test_dedup_merges_confirmed_pair(board_env, tmp_path, monkeypatch, emb_on):
"""Judge-„ja" merged: Verlierer → grouped (reason/merged_into), Champion sammelt reader."""
# Titel-Cos ~0.77 (unter Auto-0.95) → Kandidat, der Judge entscheidet
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({"satisfiability": 0, "sat": 40}))
counter = {}
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", {"pairs": {"1": "ja"}}),
], counter))
await _run_dedup(db_ := board_env[0], ctx := board_env[1], tmp_path, [
("b-1", {"title": "SAT", "description": "kurz", "readers": ["r1"]}),
("b-2", {"title": "SAT (Satisfiability Problem)",
"description": "Erfüllbarkeit Boolescher Ausdrücke", "readers": ["r2"]}),
])
loser = await db_.kanban_get_card(TOPIC, B, "b-1")
champ = await db_.kanban_get_card(TOPIC, B, "b-2")
assert loser["stage"] == "grouped"
assert loser["payload"]["reason"] == "merged"
assert loser["payload"]["merged_into"] == "SAT (Satisfiability Problem)"
assert champ["stage"] == "grouping"
assert set(champ["payload"]["readers"]) == {"r1", "r2"}
journal = json.loads(next(tmp_path.glob("inventar-dedup-*.json")).read_text(encoding="utf-8"))
assert journal["merged"] == [{"dublette": "SAT", "in": "SAT (Satisfiability Problem)"}]
assert journal["paare_detail"][0]["verdict"] == "ja"
assert counter["-dedup-"] == 2 # Zwei-Judge-Panel
async def test_dedup_judge_nein_keeps_both(board_env, tmp_path, monkeypatch, emb_on):
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({"modifizierter": 0, "greedy": 40}))
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", {"pairs": {"1": "nein"}}),
]))
await _run_dedup(board_env[0], board_env[1], tmp_path, [
("b-1", {"title": "Greedy-Algorithmus", "description": "Basisverfahren"}),
("b-2", {"title": "Modifizierter Greedy-Algorithmus", "description": "Variante"}),
])
assert (await board_env[0].kanban_get_card(TOPIC, B, "b-1"))["stage"] == "grouping"
assert (await board_env[0].kanban_get_card(TOPIC, B, "b-2"))["stage"] == "grouping"
async def test_dedup_relation_guard_blocks_identical_tokens(board_env, tmp_path, monkeypatch, emb_on):
"""Gleiche Tokens, andere Richtung: Judge sagt „ja", Titel-Cos 1.0 (Auto-Kante) —
der Relation-Guard blockt beides, beide Karten überleben."""
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({"hamiltonian": 0}))
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", lambda key: {"pairs": {"1": "ja"}}),
]))
await _run_dedup(board_env[0], board_env[1], tmp_path, [
("b-1", {"title": "Hamiltonian Cycle ≤ Hamiltonian Path", "description": "Reduktion"}),
("b-2", {"title": "Hamiltonian Path ≤ Hamiltonian Cycle", "description": "Reduktion"}),
])
assert (await board_env[0].kanban_get_card(TOPIC, B, "b-1"))["stage"] == "grouping"
assert (await board_env[0].kanban_get_card(TOPIC, B, "b-2"))["stage"] == "grouping"
journal = json.loads(next(tmp_path.glob("inventar-dedup-*.json")).read_text(encoding="utf-8"))
assert journal["paare_detail"][0]["verdict"] == "guard_veto"
async def test_dedup_panel_disagreement_keeps_both(board_env, tmp_path, monkeypatch, emb_on):
"""Merge braucht Einstimmigkeit: j1 ja + j2 nein → beide überleben (Journal: uneinig)."""
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({"satisfiability": 0, "sat": 40}))
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", lambda key: {"pairs": {"1": "ja" if "-j1" in key else "nein"}}),
]))
await _run_dedup(board_env[0], board_env[1], tmp_path, [
("b-1", {"title": "SAT", "description": "d1"}),
("b-2", {"title": "SAT (Satisfiability Problem)", "description": "d2"}),
])
assert (await board_env[0].kanban_get_card(TOPIC, B, "b-1"))["stage"] == "grouping"
assert (await board_env[0].kanban_get_card(TOPIC, B, "b-2"))["stage"] == "grouping"
journal = json.loads(next(tmp_path.glob("inventar-dedup-*.json")).read_text(encoding="utf-8"))
assert journal["paare_detail"][0]["verdict"] == "uneinig"
async def test_dedup_title_only_candidate(board_env, tmp_path, monkeypatch, emb_on):
"""Titel-Cos über dem Floor reicht als Kandidat — auch wenn der Mittelwert
(verschiedene Beschreibungs-Facetten) darunter liegt."""
# Beschreibungen fast orthogonal (0° vs 80°), Titel ähnlich (0° vs 40°)
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs(
{"völlig": 0, "ganz": 80, "alpha kern": 0, "alpha zentrum": 40}))
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", {"pairs": {"1": "ja"}}),
]))
await _run_dedup(board_env[0], board_env[1], tmp_path, [
("b-1", {"title": "Alpha Kern", "description": "völlig anderes"}),
("b-2", {"title": "Alpha Zentrum", "description": "ganz anders zwei"}),
])
stages = sorted([(await board_env[0].kanban_get_card(TOPIC, B, c))["stage"]
for c in ("b-1", "b-2")])
assert stages == ["grouped", "grouping"] # Paar wurde gejudged und merged
def test_canonical_key_camel_and_catalogue():
"""CamelCase-Split + Katalog-Phrasen-Strip; Varianten-Ziffern bleiben erhalten."""
from blocks import _canonical_key as k
assert k("Set Cover") == k("SetCover-Problem") != ""
assert k("Definition 6.19 (NP)") == "np"
assert k("SAT") != k("3-SAT") # Varianten-Ziffer ist Signal, keine Katalognummer
assert k("ModifiedGreedy") == k("Modified Greedy")
# englische Katalog-Phrasen gleichwertig (Quellen sind nicht immer deutsch)
assert k("Corollary 3.2 (VC)") == "vc"
assert k("Chapter 7: Vertex Cover") == k("Vertex Cover")
assert k("Section 2.1 Matching") == k("Matching")
def test_relation_guard_ignores_trailing_scaffolding():
"""Trailing „Reduktion/Transformation" ist kein Operand — sonst blockt der Guard
den korrekten Merge; Richtungs-Konflikte bleiben erkannt."""
from blocks import _relation_conflict as c
assert not c("SetCover ≤ HittingSet", "SetCover ≤ HittingSet Reduktion")
assert not c("A → B", "A → B Transformation")
assert c("Hamiltonian Cycle ≤ Hamiltonian Path", "Hamiltonian Path ≤ Hamiltonian Cycle")
def test_relation_guard_english_and_operator_suffix():
"""Englisches „Reduction" ist Scaffolding wie „Reduktion"; ein angehängtes
p/m am Operator („≤p") gehört zum Operator, nicht zum Operanden.
Beides waren Fehl-Vetos im aak-Lauf. Varianten-Konflikte bleiben."""
from blocks import _relation_conflict as c
assert not c("3-Exact Cover ≤ SubSet Sum", "Reduction 3-EXACT COVER ≤ SUBSET SUM")
assert not c("k-CLIQUE ≤ k-INDEPENDENT SET", "Reduction k-CLIQUE ≤ k-INDEPENDENT SET")
assert not c("3-SAT ≤ 3-Färbung", "3-SAT ≤p 3-Färbung")
assert c("SAT ≤ Clique", "3-SAT ≤ Clique") # Variante als Operand bleibt Konflikt
async def test_dedup_casefold_title_candidate(board_env, tmp_path, monkeypatch, emb_on):
"""GROSSSCHREIBUNG darf den Titel-Kanal nicht brechen: Titel werden casefolded
eingebettet („VERTEX COVER" vs. „Vertex Cover (VC)" lag real bei Cos 0.55)."""
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({"vertex cover": 0}))
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", {"pairs": {"1": "ja"}}),
]))
await _run_dedup(board_env[0], board_env[1], tmp_path, [
("b-1", {"title": "VERTEX COVER", "description": "knapp"}),
("b-2", {"title": "Vertex Cover (VC)", "description": "Knotenüberdeckung ausführlich"}),
])
stages = sorted([(await board_env[0].kanban_get_card(TOPIC, B, c))["stage"]
for c in ("b-1", "b-2")])
assert stages == ["grouped", "grouping"]
async def test_dedup_no_embedding_passes_through(board_env, tmp_path, monkeypatch):
"""Ohne Embedding-Modell winkt die Stage durch — 0 Agent-Calls (kein n²-Fallback)."""
counter = {}
monkeypatch.setattr(bi, "run_single_slot", _slot_router([("-dedup-", {"pairs": {}})], counter))
await _run_dedup(board_env[0], board_env[1], tmp_path, [
("b-1", {"title": "SAT", "description": "d"}),
("b-2", {"title": "SAT (Satisfiability Problem)", "description": "d"}),
])
assert (await board_env[0].kanban_get_card(TOPIC, B, "b-1"))["stage"] == "grouping"
assert (await board_env[0].kanban_get_card(TOPIC, B, "b-2"))["stage"] == "grouping"
assert counter == {}
async def test_dedup_second_wave_merges_into_context(board_env, tmp_path, monkeypatch, emb_on):
"""Supplement-Welle: Neuling merged in den bestätigten Block; der bleibt unberührt."""
db, ctx, files = board_env
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({"satisfiability": 0, "sat": 40}))
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", {"pairs": {"1": "ja"}}),
]))
await db.kanban_upsert_card(TOPIC, B, "b-old", "block", "done_block",
{"title": "SAT", "description": "Erfüllbarkeitsproblem",
"readers": ["r1"], "mirrored_norm": "sat"})
await db.upsert_block(TOPIC, "sat", "SAT", "Erfüllbarkeitsproblem", [])
await _run_dedup(db, ctx, tmp_path, [
("b-new", {"title": "SAT (Satisfiability Problem)", "description": "kurz", "readers": ["r9"]}),
])
new = await db.kanban_get_card(TOPIC, B, "b-new")
old = await db.kanban_get_card(TOPIC, B, "b-old")
assert new["stage"] == "grouped"
assert new["payload"]["merged_into"] == "SAT"
assert old["stage"] == "done_block" # context never demoted
assert set(old["payload"]["readers"]) == {"r1", "r9"}
async def test_dedup_resume_no_new_calls(board_env, tmp_path, monkeypatch, emb_on):
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({"satisfiability": 0, "sat": 40}))
counter = {}
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", {"pairs": {"1": "nein"}}),
], counter))
cards = [("b-1", {"title": "SAT", "description": "d1"}),
("b-2", {"title": "SAT (Satisfiability Problem)", "description": "d2"})]
await _run_dedup(board_env[0], board_env[1], tmp_path, cards)
assert counter["-dedup-"] == 2 # zwei Panel-Judges
await _run_dedup(board_env[0], board_env[1], tmp_path, cards)
assert counter["-dedup-"] == 2 # judge files reused
async def test_dedup_complete_link_no_chaining(board_env, tmp_path, monkeypatch, emb_on):
"""A≈B ja, B≈C ja, AC kein Kandidat → complete-link merged nur ein Paar."""
# Winkel 0/30/60: A-B und B-C sind Kandidaten (cos .87), A-C nicht (cos .5 < Floor)
monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({"alpha": 0, "beta": 30, "gamma": 60}))
monkeypatch.setattr(bi, "run_single_slot", _slot_router([
("-dedup-", {"pairs": {"1": "ja", "2": "ja"}}),
]))
await _run_dedup(board_env[0], board_env[1], tmp_path, [
("b-1", {"title": "Konzept Alpha", "description": "a"}),
("b-2", {"title": "Konzept Beta", "description": "bb"}),
("b-3", {"title": "Konzept Gamma", "description": "c"}),
])
db = board_env[0]
stages = {cid: (await db.kanban_get_card(TOPIC, B, cid))["stage"]
for cid in ("b-1", "b-2", "b-3")}
assert sorted(stages.values()) == ["grouped", "grouping", "grouping"]
# ── Makespan: Slot-Priorität, vorgezogene Gliederung ────────────────────────────────
def test_agent_priority_order():
"""Board 1 zuerst; in Board 2 gewinnen späte Stages (Restarbeit vor Nachschub)."""
from agents import _agent_priority as p
t = "blocks-Markdown"
assert p(f"{t}-research-1") < p(f"{t}-filter-abc-c0") < p(f"{t}-supplement")
assert p(f"{t}-research-1") < p(f"{t}-filter-abc-c0") < p(f"{t}-dedup-abc-c0") < p(f"{t}-supplement")
# Gruppierung-Keys heißen "gruppierung": ohne eigenen Eintrag fielen sie ans Ende
# und verhungerten hinter Board 2 (aak: 211 min Slot-Wartezeit)
assert p(f"{t}-dedup-abc-c0") < p(f"{t}-gruppierung-xyz-cTOP") < p(f"{t}-supplement")
assert p(f"{t}-gruppierung-xyz-cTOP") < p(f"{t}-ns-subblock-c1-r2-1")
assert (p(f"{t}-outline-judge") < p(f"{t}-ns-artifact-example-c0")
< p(f"{t}-ns-question-pattern-c0") < p(f"{t}-ns-relevance-final-c0")
< p(f"{t}-ns-level-final-c0") < p(f"{t}-ns-facts-erg-c0")
< p(f"{t}-ns-subblock-c1-r2-1"))
assert p(f"{t}-supplement") < p(f"{t}-outline-1")
assert p("guide-t-writer-k1") == 16 # unmatched → after everything
assert p("guide-t-writer-k1") == 18 # unmatched → after everything
async def test_outline_runs_before_artefacts_finish(board_env, monkeypatch):
@@ -509,3 +762,5 @@ def test_per_block_functions_accept_wrapper_kwargs():
params = inspect.signature(getattr(blx, fn)).parameters
assert "ns" in params and "lbl" in params, fn
assert "seeds" in inspect.signature(blx._subblocks_block).parameters
for fn in ("_subblocks_block", "_facts_block"): # Board 2 reicht die Block-Quellen durch
assert "sources" in inspect.signature(getattr(blx, fn)).parameters, fn

View File

@@ -104,6 +104,13 @@ async def test_pull_prefers_bigger_blocks(testdb):
await db.kanban_upsert_card(TOPIC, "inventory", "b", "block", "s1")
pulled = await db.kanban_pull(TOPIC, "inventory", "s1", 10)
assert [c["card_id"] for c in pulled] == ["a", "b"]
# n_size (Board 1) ist der Fallback-Schätzer; subs_n behält Vorrang
await db.kanban_upsert_card(TOPIC, "inventory", "n-klein", "block", "s2", {"n_size": 2})
await db.kanban_upsert_card(TOPIC, "inventory", "n-gross", "block", "s2", {"n_size": 9})
await db.kanban_upsert_card(TOPIC, "inventory", "n-ohne", "block", "s2")
await db.kanban_upsert_card(TOPIC, "inventory", "n-subs", "block", "s2", {"subs_n": 3, "n_size": 1})
pulled = await db.kanban_pull(TOPIC, "inventory", "s2", 10)
assert [c["card_id"] for c in pulled] == ["n-gross", "n-subs", "n-klein", "n-ohne"]
async def test_learnstate_smoke(testdb):
@@ -172,3 +179,50 @@ async def test_guide_reset_card_single(testdb):
assert await gb.reset_card(TOPIC, "Guide", "beta", 3) is True
cards = {c["block_norm"]: c for c in await db.list_guide_cards(TOPIC, "Guide")}
assert cards["beta"]["stage"] == "fakten_gate" and cards["beta"]["md"]
async def test_completeness_route(testdb, tmp_path, monkeypatch):
import routes, paths
db = testdb
monkeypatch.setattr(paths, "arbeit_dir", lambda t: tmp_path)
await db.upsert_block(TOPIC, "alpha", "Alpha", "d", "[]")
await db.set_block_status(TOPIC, "alpha", "consensus")
await db.upsert_subblock(TOPIC, "alpha", "s1", "Alpha", "Sub Eins")
await db.set_subblock_fields(TOPIC, "alpha", "s1", status="consensus")
await db.upsert_question_pattern(TOPIC, "alpha", "s1", "Alpha", "Sub Eins", "Frage?")
await db.put_sub_artifact(TOPIC, "alpha", "s1", "flashcard", "Alpha", "Sub Eins", "{}")
await db.put_lernziel(TOPIC, "alpha", "z1", "Ziel")
await db.set_ziel_covered(TOPIC, "alpha", "z1", True)
(tmp_path / "inventar-filter-x.json").write_text(
'{"degradiert": 3, "ueberstimmt": ["A"], "floor_veto": []}', encoding="utf-8")
res = await routes.blocks_completeness(TOPIC)
assert res["bloecke"] == 1 and res["subs"] == 1
assert res["frage_bloecke"] == 1 and res["lernkarten"] == 1
assert res["ziele_total"] == 1 and res["ziele_covered"] == 1
assert res["degradiert_geprueft"] == 3 and res["panel_gerettet"] == 1
assert res["dead"] == 0
async def test_blocks_ready_from_db(testdb, monkeypatch):
"""Regression: gesynctes Topic ohne blocks.md muss trotzdem ready sein (DB zählt)."""
import blocks as blx
db = testdb
await db.kanban_upsert_card(TOPIC, "inventory", "b-1", "block", "done_block", {"title": "Alpha"})
st = await blx.blocks_status(TOPIC)
assert st["ready"] is True and st["partial"] is False
async def test_remove_guide_format_clears_everything(testdb, monkeypatch):
"""Board-Remove räumt ALLE Läufe eines Formats + Karten (8 error-Zeilen stapelten sich)."""
import routes
from models import GuideFormatRequest
db = testdb
for i in range(3):
await db.create_guide({"id": f"g{i}", "topic": TOPIC, "format": "Guide",
"instructions": "", "status": "error", "progress": None,
"created_at": "2026-01-01", "updated_at": "2026-01-01"})
await db.upsert_guide_card(TOPIC, "Guide", "alpha", "Alpha")
res = await routes.remove_guide_format(GuideFormatRequest(topic=TOPIC, format="Guide"))
assert res["removed"] == 3
assert await db.list_guides() == [] or all(g["topic"] != TOPIC for g in await db.list_guides())
assert await db.list_guide_cards(TOPIC, "Guide") == []

View File

@@ -122,6 +122,69 @@ TOML ausführlich.""")[0]
assert "TOML ausführlich." in sec["md"] and "YAML kompakt." in sec["compact"]
async def test_card_examples_filters_and_formats(testdb):
"""_card_examples: Norm-Matching auf die übergebenen Subs; unmatchte Beispiele nur
beim Voll-Writer/Teil 1 (include_unmatched) — nie stillschweigend weg."""
import json as _json
import guide_board as gb
from types import SimpleNamespace
db = testdb
await db.put_sub_artifact("t", "gross", "sub eins", "example",
_json.dumps({"problem": "P1", "steps": ["a", "b"], "result": "R1"}),
"Gross", "Sub Eins")
await db.put_sub_artifact("t", "gross", "verwaist", "example",
_json.dumps({"problem": "P2", "steps": ["x"], "result": "R2"}),
"Gross", "Verwaister Sub")
env = SimpleNamespace(topic="t")
subs = [{"title": "Sub Eins", "level": "beginner"}]
full = await gb._card_examples(env, "gross", subs)
assert "Sub Eins" in full and "P1" in full and "1) a 2) b" in full and "R1" in full
assert "Subbaustein unklar" in full and "P2" in full # orphan attached with hint
half = await gb._card_examples(env, "gross", subs, include_unmatched=False)
assert "P1" in half and "P2" not in half # split half: only its own subs
assert await gb._card_examples(env, "leer", subs) == ""
def test_writer_template_has_examples_placeholder():
"""Smoke: alle Platzhalter versorgt — ein fehlender Kwarg stürbe als KeyError."""
from pipeline import _prompt
text = _prompt("Guide-Writer-Board", topic="t", format_name="Guide", chapter="K1",
assignment="- B", ziele="- z", facts="F", examples="", gaps="",
spec="", out_path="/tmp/x.md", extra="")
assert "VERIFIED FACTS" in text
async def test_fakten_gate_counts_examples_as_facts(testdb, monkeypatch, tmp_path):
"""Gate-Prompt enthält die Beispiele als verifizierte Fakten — sonst fliegen
gerechnete Beispielwerte als „nicht belegt" raus."""
import json as _json
import guide_board as gb
from types import SimpleNamespace
db = testdb
await db.upsert_guide_card("t", "Guide", "gross", "Gross")
await db.put_sub_artifact("t", "gross", "sub eins", "example",
_json.dumps({"problem": "P1", "steps": ["a"], "result": "R1"}),
"Gross", "Sub Eins")
captured = {}
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
captured["prompt"] = prompt
return "ok", []
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
monkeypatch.setattr(gb, "_card_facts", lambda e, b: "FAKT X")
env = SimpleNamespace(ctx=SimpleNamespace(topic="t", provider="p", is_cancelled=lambda: False),
guide_id="g", topic="t", format="Guide", instructions="",
subs_by_title={"Gross": [{"title": "Sub Eins", "level": "beginner"}]},
spec="", slot=lambda name: tmp_path / name)
card = {"block_norm": "gross", "block": "Gross", "stage": "fakten_gate", "status": "open",
"writer_rounds": 0, "gate_info": "",
"md": "<!-- section: Gross -->\n<!-- ausführlich -->\nText."}
ok = await gb._stage_fakten_gate(env, card)
assert ok is True
assert "VERIFIED WORKED EXAMPLES" in captured["prompt"] and "P1" in captured["prompt"]
async def test_writer_splits_oversized_first_draft(testdb, monkeypatch, tmp_path):
import guide_board as gb
from types import SimpleNamespace

View File

@@ -0,0 +1,71 @@
"""PDF→Text-Konvertierung: pymupdf4llm primär, pdftotext-Fallback, mtime-Cache."""
import os
import time
import fitz # PyMuPDF
import pytest
import blocks as blx
def _mini_pdf(path, text="Approximationsalgorithmen sind wichtig."):
doc = fitz.open()
page = doc.new_page()
page.insert_text((72, 72), text, fontsize=12)
doc.save(str(path))
doc.close()
def test_convert_writes_markdown_txt(tmp_path):
_mini_pdf(tmp_path / "skript.pdf")
blx._convert_pdfs(tmp_path)
out = (tmp_path / "skript.txt").read_text(encoding="utf-8")
assert "Approximationsalgorithmen" in out
def test_cache_skips_fresh_txt(tmp_path):
_mini_pdf(tmp_path / "a.pdf")
marker = tmp_path / "a.txt"
marker.write_text("MARKER", encoding="utf-8")
now = time.time() + 60
os.utime(marker, (now, now))
blx._convert_pdfs(tmp_path)
assert marker.read_text(encoding="utf-8") == "MARKER" # nicht neu konvertiert
def test_fallback_to_pdftotext(tmp_path, monkeypatch):
_mini_pdf(tmp_path / "b.pdf")
monkeypatch.setattr(blx, "_pdf_markdown", lambda p: None)
monkeypatch.setattr(blx, "_pdf_plaintext", lambda p: "fallback")
blx._convert_pdfs(tmp_path)
assert (tmp_path / "b.txt").read_text(encoding="utf-8") == "fallback"
def test_ocr_languages_from_tessdata(tmp_path, monkeypatch):
import pymupdf
monkeypatch.setattr(pymupdf, "get_tessdata", lambda: str(tmp_path))
assert blx._ocr_languages() is None # keine Sprachdaten → OCR aus
(tmp_path / "eng.traineddata").touch()
assert blx._ocr_languages() == "eng"
(tmp_path / "deu.traineddata").touch()
assert blx._ocr_languages() == "deu+eng"
monkeypatch.setattr(pymupdf, "get_tessdata", lambda: (_ for _ in ()).throw(RuntimeError()))
assert blx._ocr_languages() is None
def test_fidelity_guard_prefers_faithful_plaintext():
plain = "Definition. P = {L ⊆ Σ | A ∈ L} und ≤ sowie häufig über. " * 20
# Markdown verlor die Formeln (Symbole weg) → plain gewinnt
md_lossy = "Definition. und sowie h¨aufig ¨uber. " * 20
text, tool = blx._pick_conversion(md_lossy, plain)
assert tool == "pdftotext"
# Markdown treu (Symbole + Länge da) → md gewinnt
md_ok = "# Def\n" + plain
text, tool = blx._pick_conversion(md_ok, plain)
assert tool == "pymupdf4llm"
# nur eine Quelle verfügbar
assert blx._pick_conversion(None, plain)[1] == "pdftotext"
assert blx._pick_conversion(md_ok, None)[1] == "pymupdf4llm"
assert blx._pick_conversion(None, None) is None

View File

@@ -0,0 +1,146 @@
"""Flashcard-Übungspool: Leitner-Schritte, Deck-Bau (Level-Gate, fällig/neu), Persistenz."""
import json
from datetime import datetime, timedelta, timezone
from learning import LEITNER_MAX_BOX, PRACTICE_NEW_PER_SESSION, leitner_step
TOPIC = "t"
def _iso(days: float = 0) -> str:
return (datetime.now(timezone.utc) + timedelta(days=days)).isoformat()
async def _card(db, bn, sn, sub_title="Sub", q="Q?", block="Block"):
await db.put_sub_artifact(TOPIC, bn, sn, "flashcard",
json.dumps({"question": q, "answer": "A"}), block, sub_title)
# ── Leitner rein funktional ──────────────────────────────────────────────────────────
def test_leitner_step_transitions():
assert leitner_step(None, True) == (2, 1) # neue Karte gewusst → Box 2, morgen
assert leitner_step(None, False) == (1, 0) # neue Karte falsch → Box 1, sofort
assert leitner_step(2, True) == (3, 3)
assert leitner_step(LEITNER_MAX_BOX, True) == (LEITNER_MAX_BOX, 21) # Cap
assert leitner_step(4, False) == (1, 0) # falsch → zurück auf Anfang
# ── Persistenz ───────────────────────────────────────────────────────────────────────
async def test_progress_upsert_roundtrip(testdb):
db = testdb
await db.upsert_practice_progress(TOPIC, "b", "s", 2, _iso(1))
await db.upsert_practice_progress(TOPIC, "b", "s", 3, _iso(3))
rows = await db.get_practice_progress(TOPIC)
assert len(rows) == 1 and rows[0]["box"] == 3
async def test_progress_survives_artefakte_wipe(testdb):
db = testdb
await _card(db, "b", "s")
await db.upsert_practice_progress(TOPIC, "b", "s", 4, _iso(7))
await db.delete_sub_artefakte(TOPIC) # Regenerations-Wipe
assert (await db.get_practice_progress(TOPIC))[0]["box"] == 4
async def test_delete_topic_pipeline_clears_progress(testdb):
db = testdb
await db.upsert_practice_progress(TOPIC, "b", "s", 2, _iso(1))
await db.delete_topic_pipeline(TOPIC)
assert await db.get_practice_progress(TOPIC) == []
async def test_sub_levels_norm_and_counts(testdb):
db = testdb
await db.put_subblock(TOPIC, "b", "s1", "Block", "S1", level="beginner")
await db.put_subblock(TOPIC, "b", "s2", "Block", "S2", level="expert")
await db.put_subblock(TOPIC, "b", "s3", "Block", "S3", level="beginner", relevance="peripheral")
await db.put_subblock(TOPIC, "b", "s4", "Block", "S4", level="beginner", status="variant")
levels = await db.sub_levels_norm(TOPIC)
assert levels[("b", "s1")] == 1 and levels[("b", "s2")] == 3 and levels[("b", "s3")] == 4
assert ("b", "s4") not in levels # non-consensus ausgeschlossen
counts = await db.subs_per_level_norm(TOPIC)
assert counts["b"] == {1: 1, 2: 0, 3: 1, 4: 1}
# ── Deck-Bau ─────────────────────────────────────────────────────────────────────────
async def test_deck_level_gate_and_unlock(testdb):
from routes import build_practice_deck
db = testdb
# block_norm muss _norm_title(Roh-Titel) sein — so entsteht er auch in der Pipeline
await db.put_subblock(TOPIC, "block", "s1", "Block", "S1", level="beginner")
await db.put_subblock(TOPIC, "block", "s2", "Block", "S2", level="expert")
await _card(db, "block", "s1", "S1")
await _card(db, "block", "s2", "S2")
deck = await build_practice_deck(TOPIC)
assert [c["sub_norm"] for c in deck["cards"]] == ["s1"] # expert gesperrt
assert deck["counts"]["gesperrt"] == 1
# Score über S1+S2-Schwelle (2 Subs × 25 = 50) → expert (Level 3) frei
await db.set_block_score_and_streak(TOPIC, "Block", 50, 0)
deck = await build_practice_deck(TOPIC)
assert {c["sub_norm"] for c in deck["cards"]} == {"s1", "s2"}
async def test_deck_due_before_new_oldest_first(testdb):
from routes import build_practice_deck
db = testdb
for sn in ("s1", "s2", "s3"):
await db.put_subblock(TOPIC, "b", sn, "Block", sn.upper(), level="beginner")
await _card(db, "b", sn, sn.upper())
await db.upsert_practice_progress(TOPIC, "b", "s2", 2, _iso(-1))
await db.upsert_practice_progress(TOPIC, "b", "s3", 2, _iso(-5))
deck = await build_practice_deck(TOPIC)
assert [c["sub_norm"] for c in deck["cards"]] == ["s3", "s2", "s1"] # älteste fällige zuerst
assert [c["status"] for c in deck["cards"]] == ["due", "due", "new"]
assert deck["counts"] == {"due": 2, "new": 1, "new_total": 1, "gesperrt": 0}
async def test_deck_caps_new_and_reports_total(testdb):
from routes import build_practice_deck
db = testdb
for i in range(PRACTICE_NEW_PER_SESSION + 5):
sn = f"s{i:02d}"
await db.put_subblock(TOPIC, "b", sn, "Block", sn, level="beginner")
await _card(db, "b", sn, sn)
deck = await build_practice_deck(TOPIC)
assert deck["counts"]["new"] == PRACTICE_NEW_PER_SESSION
assert deck["counts"]["new_total"] == PRACTICE_NEW_PER_SESSION + 5
async def test_deck_future_due_sets_next_due_at(testdb):
from routes import build_practice_deck
db = testdb
await db.put_subblock(TOPIC, "b", "s1", "Block", "S1", level="beginner")
await _card(db, "b", "s1", "S1")
await db.upsert_practice_progress(TOPIC, "b", "s1", 3, _iso(3))
deck = await build_practice_deck(TOPIC)
assert deck["cards"] == [] and deck["counts"]["due"] == 0
assert deck["next_due_at"] is not None
async def test_deck_orphan_progress_and_legacy_block(testdb):
from routes import build_practice_deck
db = testdb
# Orphan: Progress ohne Karte → unschädlich, taucht nicht auf
await db.upsert_practice_progress(TOPIC, "weg", "s0", 2, _iso(-1))
# Legacy: Karte ohne subblocks-Zeilen → ungefiltert durchlassen
await _card(db, "leg", "sx", "SX")
deck = await build_practice_deck(TOPIC)
assert [c["block_norm"] for c in deck["cards"]] == ["leg"]
async def test_answer_books_without_card(testdb):
"""Antwort während Regeneration: bucht immer, kein Fehlerpfad."""
from models import PracticeAnswerRequest
from routes import practice_answer
db = testdb
res = await practice_answer(PracticeAnswerRequest(
topic=TOPIC, block_norm="b", sub_norm="s", correct=True))
assert res["box"] == 2
res = await practice_answer(PracticeAnswerRequest(
topic=TOPIC, block_norm="b", sub_norm="s", correct=False))
assert res["box"] == 1
assert (await db.get_practice_progress(TOPIC))[0]["box"] == 1

View File

@@ -62,24 +62,27 @@ def _mk_race(finder_by_agent):
for slot in slots:
key, prompt = slot["key"], slot["prompt"]
prompts.append((key, prompt))
text = None
fake_race.slots_seen.append(slot)
if "-subblock-final-" in key:
# no-tool judges reply as TEXT; the payload sink writes the j-file itself
kons = re.search(r"Konsens \(≥2 finders\):\n(.*?)\nUnsicher", prompt, re.S)
subs = [l[2:] for l in (kons.group(1).splitlines() if kons else [])
if l.startswith("- ") and l != "- (keiner)"]
if subs:
text = "<!-- block: Alpha -->\n" + "\n".join(f"- {s}" for s in subs)
elif "-r1-" in key:
outs.append(slot["payload"]((0, text, "")))
continue
if "-r1-" in key:
agent = int(key.rsplit("-", 1)[1])
subs = finder_by_agent.get(agent) or []
if subs:
if subs and (m := _MD_PATH.search(prompt)):
text = "<!-- block: Alpha -->\n" + "\n".join(f"- {s}" for s in subs)
if text is not None and (m := _MD_PATH.search(prompt)):
with open(m.group(1), "w", encoding="utf-8") as f:
f.write(text)
outs.append(slot["payload"](None))
with open(m.group(1), "w", encoding="utf-8") as f:
f.write(text)
outs.append(slot["payload"](None))
outs = [o for o in outs if o]
return outs or None
fake_race.slots_seen = []
return fake_race, prompts
@@ -293,3 +296,124 @@ async def test_round_cap_stops_endless_finders(sub_env, monkeypatch):
"", wipe=False, ns="x-")
max_round = max(int(k.split("-r")[1].split("-")[0]) for k, _ in prompts if "-r" in k and "-subblock-c" in k)
assert max_round == blx.SUBBLOCK_MAX_ROUNDS
# ── Inline-Evidenz für Judges (Token-Umbau) ──────────────────────────────────────────
def _corpus(tmp_path):
d = tmp_path / "korpus"
d.mkdir()
(d / "Skript.txt").write_text(
"Kapitel 1\nAlpha Grundlagen: der Kernbegriff.\nMehr Text dazu.\n\n"
"Kapitel 2\nGamma Randnotiz ohne Bezug.\n", encoding="utf-8")
(d / "Aufgaben.txt").write_text("Übung 1\nAlpha Vertiefung der Konzepte.\n", encoding="utf-8")
return d
def test_evidence_pack_selects_matching_sections(tmp_path):
d = _corpus(tmp_path)
pack = blx._evidence_pack(d, None, ["Alpha Grundlagen"])
assert "── Skript.txt" in pack and "Kernbegriff" in pack
pack2 = blx._evidence_pack(d, ["Aufgaben.txt"], ["Alpha"]) # genannte Quellen engen ein
assert "Skript.txt" not in pack2 and "Aufgaben.txt" in pack2
assert blx._evidence_pack(None, None, ["x"]) == "" # kein Korpus → Selbst-Recherche bleibt
def test_evidence_pack_budget_and_guarantee(tmp_path):
d = tmp_path / "korpus"
d.mkdir()
(d / "A.txt").write_text("Alpha wichtig. " * 50, encoding="utf-8")
(d / "B.txt").write_text("Beta anderes Thema. " * 50, encoding="utf-8")
pack = blx._evidence_pack(d, None, ["Alpha"], budget=10)
assert "Alpha" in pack # Abdeckungs-Garantie schlägt das Budget
assert "Beta" not in pack # Top-up respektiert das Budget
def test_cite_ref_parses_positions(tmp_path):
d = _corpus(tmp_path)
files = blx._corpus_files(d, None)
f, lo, hi = blx._cite_ref("Skript.txt, Übung 6.47, Z.2-3", files)
assert f.name == "Skript.txt" and (lo, hi) == (2, 3)
f2, lo2, hi2 = blx._cite_ref("Aufgaben.txt Zeile 2", files)
assert f2.name == "Aufgaben.txt" and lo2 == hi2 == 2
assert blx._cite_ref("Skript.txt, Übung 6.47", files) is None # keine Zeilenangabe
assert blx._cite_ref("Z.5 irgendwo", files) is None # keine Datei
# englische Zitierformen (Quellen sind nicht immer deutsch)
f3, lo3, hi3 = blx._cite_ref("Skript.txt, line 2", files)
assert f3.name == "Skript.txt" and lo3 == hi3 == 2
f4, lo4, hi4 = blx._cite_ref("Aufgaben.txt, lines 1-2", files)
assert f4.name == "Aufgaben.txt" and (lo4, hi4) == (1, 2)
def test_cited_evidence_lines_and_fallback(tmp_path):
d = _corpus(tmp_path)
ev = blx._cited_evidence(d, None, ["Skript.txt, Z.2"], ["Alpha"])
assert "── Skript.txt · Z." in ev and "Kernbegriff" in ev
ev2 = blx._cited_evidence(d, None, ["ohne Position"], ["Alpha Grundlagen"])
assert "Kernbegriff" in ev2 # Keyword-Fallback
def test_sink_json_writes_only_valid(tmp_path):
p = tmp_path / "level-final-c1.json"
ok = blx._sink_json((0, 'Vorab {"levels": {"1": "beginner"}} nach', ""), p,
lambda d: blx._levels_schema(d, {1}))
assert ok == {1: "beginner"}
assert json.loads(p.read_text(encoding="utf-8"))["levels"]["1"] == "beginner"
bad = blx._sink_json((0, "kein json", ""), tmp_path / "x.json", lambda d: d)
assert bad is None and not (tmp_path / "x.json").exists()
async def test_clarify_inline_evidence_no_tools(sub_env, monkeypatch, tmp_path):
"""Mit Korpus: Judges bekommen Auszüge inline und laufen ohne Tools (Text-Antwort);
die j-Datei schreibt die Engine. Finder bleiben unverändert bei capabilities=files."""
db, ctx, files = sub_env
d = _corpus(tmp_path)
monkeypatch.setattr(blx, "source_folder", lambda t: d)
monkeypatch.setattr(blx, "load_source", lambda t: {"type": "uni"})
fake, prompts = _mk_race({1: ["Alpha Grundlagen"], 2: ["Alpha Grundlagen"]})
monkeypatch.setattr(blx, "_race", fake)
raw = await blx._subblocks_block(ctx, lambda *a, **k: None, files, {1: "Alpha — Grundkonzept"},
"", wipe=False, ns="x-", sources=["Skript.txt"])
assert raw == {"Alpha": ["Alpha Grundlagen"]}
judges = [s for s in fake.slots_seen if "-subblock-final-" in s["key"]]
finders = [s for s in fake.slots_seen if "-r1-" in s["key"]]
assert judges and all(s["capabilities"] == "none" for s in judges)
assert "── Skript.txt" in judges[0]["prompt"]
assert "ls/find" not in judges[0]["prompt"] # keine Selbst-Recherche-Anweisung mehr
assert finders and all(s["capabilities"] == "files" for s in finders)
assert list(tmp_path.glob("subblock-final-*-j*.md")) # Engine persistiert die Judge-Antwort
async def test_facts_check_inline_cited_evidence(sub_env, monkeypatch, tmp_path):
"""Facts-Check: zitierte Zeilenbereiche gehen inline mit, Judge läuft ohne Tools,
die Check-Datei schreibt die Engine aus der Text-Antwort."""
db, ctx, files = sub_env
d = _corpus(tmp_path)
facts = {"facts": [{"block": "Alpha", "subblock": "Sub Eins", "key_points": ["k"],
"prerequisites": "", "hurdles": "",
"cited_facts": [{"text": "T", "source": "Skript.txt, Z.2"}],
"example_idea": ""}]}
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None):
if "-facts-erg-" in key:
return blx.FAILED, None
(tmp_path / "facts-c0.json").write_text(json.dumps(facts), encoding="utf-8")
return blx.OK, None
seen = []
async def fake_agent(key, prompt, timeout, provider="", role="", capabilities="", scope=None, label="", **kw):
seen.append((key, capabilities, prompt))
return (0, '{"ok": true}', "")
monkeypatch.setattr(blx, "run_single_slot", fake_slot)
monkeypatch.setattr(blx, "run_agent", fake_agent)
res = await blx._facts_block(ctx, lambda *a, **k: None, {"arbeit": tmp_path},
{"Alpha": ["Sub Eins"]}, {"type": "uni"}, d, "", ns="x-")
assert res is not None
facts_map, discarded = res
assert "Alpha" in facts_map and not discarded
assert len(seen) == blx.FACTS_CHECK_PANEL
key, caps, prompt = seen[0]
assert caps == "none" and "── Skript.txt · Z." in prompt
assert (tmp_path / "facts-check-c0-j1.json").exists() # Engine persistiert die Antwort