1717 lines
79 KiB
Python
1717 lines
79 KiB
Python
"""Blocks pipeline: research consensus + clarification loop — pure inventory, unsorted.
|
||
|
||
5x research (min. 3, grace) → mapping (consensus/rest) → clarification loop (max.
|
||
CONSENSUS_MAX_ROUNDS rounds): 3 selection agents (min. 2, grace) decide
|
||
on the disputed rest, a mapping agent sorts into accept/discard/
|
||
still disputed. An empty rest ends the loop; the last round must decide
|
||
everything. Races use a grace window instead of "first N win": after the
|
||
first valid result, the remaining agents get CONSENSUS_GRACE seconds to
|
||
finish. The consensus is accumulated in code — no agent re-emits
|
||
the full list.
|
||
"""
|
||
|
||
import asyncio
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import math
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
import database as db
|
||
import embedding
|
||
from agents import kill_process, cancel_scope, clear_scope
|
||
from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, SUB_VARIANT_COS, SUBBLOCK_MAX, EVIDENCE_BUDGET_CHARS, EVIDENCE_CTX_LINES
|
||
from fsutil import atomic_write_json
|
||
from jsonio import parse_json_text, read_json_file as _json_file
|
||
from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder
|
||
from crawl import load_pages
|
||
from pipeline import (
|
||
GenContext, _extra, _gather_progress, _yesno_schema, _log, _prompt, _race,
|
||
_semaphore, _timeout, run_single_slot,
|
||
)
|
||
from textkit import _load_blocks, _norm_title, _parse_selection, _title
|
||
|
||
# Pipeline-Tuning-Konstanten liegen zentral in config.py (tunebar via CREATOR_PARAMS).
|
||
from config import RESEARCH_SECTION_CHARS # noqa: E402
|
||
|
||
log = logging.getLogger("creator.blocks")
|
||
|
||
_blocks_progress: dict[str, str] = {}
|
||
_blocks_errors: dict[str, str] = {}
|
||
_blocks_cancelled: set[str] = set()
|
||
_blocks_step: dict[str, int] = {}
|
||
|
||
ARTEFACT_TYPES = ("flashcard", "example")
|
||
|
||
|
||
def load_source(topic: str) -> dict:
|
||
"""Read the persisted source choice. Fallback (legacy topics without source.json):
|
||
if projects/<topic> exists → projekt, otherwise thema."""
|
||
q = _json_file(source_path(topic))
|
||
if isinstance(q, dict) and q.get("type") in ("thema", "projekt", "uni", "link"):
|
||
return q
|
||
if project_dir(topic).is_dir():
|
||
return {"type": "projekt", "location": f"projects/{topic}", "spec": ""}
|
||
return {"type": "thema", "location": "", "spec": ""}
|
||
|
||
|
||
def source_folder(topic: str) -> Path | None:
|
||
"""Folder source (projekt/uni → path, link → crawl folder) — otherwise None (thema)."""
|
||
q = load_source(topic)
|
||
if q["type"] == "link":
|
||
return source_crawl_dir(topic)
|
||
if q["type"] in ("projekt", "uni"):
|
||
return safe_folder(q.get("location", ""))
|
||
return None
|
||
|
||
|
||
def _crawl_done(topic: str) -> bool:
|
||
return (source_crawl_dir(topic) / ".done").exists() # marker only on clean completion
|
||
|
||
|
||
# Learning-path levels (beginner/advanced/expert); old difficulty values are backward-compatible.
|
||
_LEVELS = ("beginner", "advanced", "expert", "easy", "medium", "hard")
|
||
|
||
|
||
async def subblocks_title(topic: str, block: str) -> list[str]:
|
||
"""Subblock titles of a block — DB-first (consensus), fallback to the sidecar file."""
|
||
rows = [s["sub_title"] for s in await db.list_subblocks(topic, _norm_title(block))
|
||
if s["status"] == "consensus" and s["sub_title"]]
|
||
if rows:
|
||
return rows
|
||
sc = _json_file(subblocks_path(topic))
|
||
if not isinstance(sc, dict):
|
||
return []
|
||
return [
|
||
t for s in (sc.get(block) or [])
|
||
if isinstance(s, dict) and (t := str(s.get("title", "")).strip())
|
||
]
|
||
|
||
|
||
async def load_question_pattern(topic: str, block: str) -> list[dict]:
|
||
"""Predefined question patterns of a block — DB-first, fallback to sidecar (empty = live)."""
|
||
rows = await db.list_question_pattern(topic, _norm_title(block))
|
||
if rows:
|
||
return [{"subblock": r["sub_title"], "question": r["question"]} for r in rows if r["question"]]
|
||
fm = _json_file(question_pattern_path(topic))
|
||
if not isinstance(fm, dict):
|
||
return []
|
||
return [
|
||
{"subblock": str(e.get("subblock", "")).strip(), "question": question}
|
||
for e in (fm.get(block) or [])
|
||
if isinstance(e, dict) and (question := str(e.get("question", "")).strip())
|
||
]
|
||
|
||
|
||
async def subblocks_frei(topic: str, block: str, max_level: int) -> list[str]:
|
||
"""Subblock titles up to the unlocked level (≤ max_level). Fallback without
|
||
level knowledge (legacy/sidecar): all subblock titles."""
|
||
rows = await db.subs_with_level(topic, block)
|
||
if not rows:
|
||
return await subblocks_title(topic, block)
|
||
return [s["title"] for s in rows if s["level"] <= max_level and s["title"]]
|
||
|
||
|
||
async def load_question_pattern_free(topic: str, block: str, max_level: int) -> list[dict]:
|
||
"""Question patterns, filtered to subblocks up to the unlocked level. Without
|
||
level knowledge (legacy/sidecar), unfiltered."""
|
||
rows = await db.subs_with_level(topic, block)
|
||
if not rows:
|
||
return await load_question_pattern(topic, block)
|
||
unlocked = {s["norm"] for s in rows if s["level"] <= max_level}
|
||
return [m for m in await load_question_pattern(topic, block) if _norm_title(m["subblock"]) in unlocked]
|
||
|
||
|
||
async def load_overview(topic: str) -> list[dict]:
|
||
"""Structured block list for the overview — DB-first (consensus + subs/levels/relevance),
|
||
fallback to blocks.md + sidecar (legacy topics)."""
|
||
bs = await db.list_blocks(topic, status="consensus")
|
||
if bs:
|
||
out = []
|
||
for num, b in enumerate(bs, 1):
|
||
subs = [s for s in await db.list_subblocks(topic, b["title_norm"]) if s["status"] == "consensus"]
|
||
out.append({
|
||
"num": num, "title": b["title"], "description": b["description"],
|
||
"subblocks": [
|
||
{"title": s["sub_title"],
|
||
"level": s["level"] if s["level"] in _LEVELS else "advanced",
|
||
"relevance": s["relevance"] if s["relevance"] in ("relevant", "peripheral") else None}
|
||
for s in subs if s["sub_title"]
|
||
],
|
||
})
|
||
return out
|
||
entries = _load_blocks(_read(blocks_path(topic)))
|
||
sidecar = _json_file(subblocks_path(topic))
|
||
sidecar = sidecar if isinstance(sidecar, dict) else {}
|
||
out = []
|
||
for num, entry in entries.items():
|
||
title = _title(entry)
|
||
split_parts = entry.split(" — ", 1)
|
||
description = split_parts[1].strip() if len(split_parts) == 2 else ""
|
||
subblocks = [
|
||
{
|
||
"title": t,
|
||
"level": s.get("level") if s.get("level") in _LEVELS else "advanced",
|
||
"relevance": s.get("relevance") if s.get("relevance") in ("relevant", "peripheral") else None,
|
||
}
|
||
for s in (sidecar.get(title) or [])
|
||
if isinstance(s, dict) and (t := str(s.get("title", "")).strip())
|
||
]
|
||
out.append({"num": num, "title": title, "description": description, "subblocks": subblocks})
|
||
return out
|
||
|
||
|
||
def _blocks_steps(topic: str) -> tuple:
|
||
"""Steps per source: link gets "Source laden" up front, projekt additionally "Supplement".
|
||
|
||
Subblocks + levels are three phases each (find, select, clarify). Per phase
|
||
all packages run in parallel; the step remains until the last package is done.
|
||
"""
|
||
q = load_source(topic)
|
||
base = ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung")
|
||
rest = (
|
||
"Generate", "Verify", "Fix",
|
||
"Outline",
|
||
"Artefakte gen", "Artefakte check",
|
||
)
|
||
middle = base + (("Supplement",) if q["type"] == "projekt" else ()) + rest
|
||
return (("Source prep",) if q["type"] == "link" else ()) + middle
|
||
|
||
|
||
def _step_idx(topic: str, name: str) -> int:
|
||
return _blocks_steps(topic).index(name)
|
||
|
||
|
||
def _report_p(set_p, topic: str, step: str):
|
||
"""Async report callback for _gather_progress: sets "<step> d/t…" + step index."""
|
||
idx = _step_idx(topic, step)
|
||
async def report(d, t):
|
||
set_p(f"{step} {d}/{t}…", step=idx)
|
||
return report
|
||
|
||
|
||
# Coarse display phases: bundle the fine steps (internally everything stays fine-grained).
|
||
# Special steps (Source laden, Supplement) belong to the "Inventory" phase.
|
||
PHASEN = (
|
||
("Source", ("Source prep",)),
|
||
("Inventory", ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung", "Supplement")),
|
||
("Generate", ("Generate",)),
|
||
("Verify", ("Verify", "Fix")),
|
||
("Outline", ("Outline",)),
|
||
("Artefakte", ("Artefakte gen", "Artefakte check")),
|
||
)
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _blocks_files(topic: str) -> dict:
|
||
work_dir = arbeit_dir(topic)
|
||
rounds = range(1, CONSENSUS_MAX_ROUNDS + 1)
|
||
return {
|
||
"final": blocks_path(topic),
|
||
"arbeit": work_dir,
|
||
"research": [work_dir / f"research-{i}.md" for i in (1, 2, 3, 4, 5)],
|
||
"research_mapping": work_dir / "research-mapping.json",
|
||
"selection": {n: [work_dir / f"selection-r{n}-{i}.json" for i in (1, 2, 3)] for n in rounds},
|
||
"mapping": {n: work_dir / f"selection-mapping-r{n}.json" for n in rounds},
|
||
"ergaenzung": work_dir / "ergaenzung.json",
|
||
"sub_roh": work_dir / "subblocks-roh.json",
|
||
"facts": work_dir / "subblocks-facts.json",
|
||
"sidecar": subblocks_path(topic),
|
||
"question_pattern": question_pattern_path(topic),
|
||
"outline": work_dir / "outline.json",
|
||
"outline_slots": [work_dir / f"outline-{i}.json" for i in (1, 2, 3)],
|
||
"artefakte": work_dir / "artefakte.json",
|
||
}
|
||
|
||
|
||
def cancel_blocks(topic: str) -> bool:
|
||
if topic not in _blocks_progress:
|
||
return False
|
||
_blocks_cancelled.add(topic)
|
||
cancel_scope(f"blocks-{topic}-") # waiting agents bail before spawning
|
||
kill_process(f"blocks-{topic}-") # kill running subprocesses
|
||
return True
|
||
|
||
|
||
|
||
|
||
|
||
|
||
async def blocks_status(topic: str) -> dict:
|
||
"""Kanban-based status: `generating` from the run registry, progress from the card
|
||
counts. `partial` = cards sit in non-terminal columns while nothing runs (continue-able)."""
|
||
generating = topic in _blocks_progress
|
||
counts = await db.kanban_stage_counts(topic)
|
||
# ready = finished inventory. The DB is the source of truth (a synced topic may lack
|
||
# blocks.md, the file is a legacy mirror) — either signal counts.
|
||
ready = blocks_path(topic).exists() or counts.get("inventory", {}).get("done_block", 0) > 0
|
||
terminal = {"clustered", "done_cluster", "grouped", "rejected", "done_block", "done_artefact", "dead"}
|
||
open_cards = sum(n for stages in counts.values()
|
||
for stage, n in stages.items() if stage not in terminal)
|
||
return {
|
||
"ready": ready,
|
||
"generating": generating,
|
||
"progress": _blocks_progress.get(topic),
|
||
"error": _blocks_errors.get(topic),
|
||
"partial": not generating and open_cards > 0,
|
||
}
|
||
|
||
|
||
def active_blocks() -> list[dict]:
|
||
return [{"topic": t, "progress": p} for t, p in _blocks_progress.items()]
|
||
|
||
|
||
def reset_blocks(topic: str) -> None:
|
||
""""Remove": deletes the ENTIRE blocks area — crawl, triage, inventory … questions.
|
||
KEEPS only the topic config `source.json` (type/link/spec). Re-generating crawls anew.
|
||
(Crawl/triage belong to the blocks; only the config is the "topic".)"""
|
||
files = _blocks_files(topic)
|
||
files["final"].unlink(missing_ok=True)
|
||
files["sidecar"].unlink(missing_ok=True)
|
||
files["question_pattern"].unlink(missing_ok=True)
|
||
shutil.rmtree(source_crawl_dir(topic), ignore_errors=True) # crawl belongs to the blocks
|
||
shutil.rmtree(files["arbeit"], ignore_errors=True)
|
||
_blocks_errors.pop(topic, None)
|
||
# source.json intentionally stays — that is the topic config.
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _supplement_schema(data):
|
||
"""{"blocks": [{"title", "description"}]} → list (empty allowed) · otherwise None."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("blocks"), list):
|
||
return None
|
||
out = []
|
||
for b in data["blocks"]:
|
||
if not isinstance(b, dict) or not isinstance(b.get("title"), str) or not isinstance(b.get("description"), str):
|
||
return None
|
||
title, description = b["title"].strip(), b["description"].strip()
|
||
if not title:
|
||
return None
|
||
out.append((title, description))
|
||
return out
|
||
|
||
|
||
def _ocr_languages() -> str | None:
|
||
"""Installierte Tesseract-Sprachen (deu/eng), None → OCR aus."""
|
||
try:
|
||
import pymupdf
|
||
base = Path(pymupdf.get_tessdata())
|
||
except Exception:
|
||
return None
|
||
langs = [l for l in ("deu", "eng") if (base / f"{l}.traineddata").exists()]
|
||
return "+".join(langs) or None
|
||
|
||
|
||
def _pdf_markdown(pdf: Path) -> str | None:
|
||
"""pymupdf4llm → Markdown string (None if the lib is missing or it fails)."""
|
||
try:
|
||
import pymupdf4llm
|
||
except ImportError:
|
||
return None
|
||
try:
|
||
# OCR nur, wenn Tesseract-Sprachdaten wirklich vorhanden sind —
|
||
# sonst wirft der OCR-Pfad und die ganze Datei faellt auf pdftotext.
|
||
langs = _ocr_languages()
|
||
kwargs = {"use_ocr": True, "ocr_language": langs} if langs else {"use_ocr": False}
|
||
return pymupdf4llm.to_markdown(str(pdf), show_progress=False, **kwargs)
|
||
except Exception:
|
||
log.warning("pymupdf4llm failed for %s", pdf.name, exc_info=True)
|
||
return None
|
||
|
||
|
||
def _pdf_plaintext(pdf: Path) -> str | None:
|
||
"""pdftotext -layout → plain string (None if missing/fails)."""
|
||
if shutil.which("pdftotext") is None:
|
||
return None
|
||
try:
|
||
out = subprocess.run(["pdftotext", "-layout", str(pdf), "-"],
|
||
check=True, timeout=120, capture_output=True)
|
||
return out.stdout.decode("utf-8", errors="replace")
|
||
except Exception:
|
||
log.warning("pdftotext failed for %s", pdf.name, exc_info=True)
|
||
return None
|
||
|
||
|
||
# Content-fidelity guard between the two converters (topic-NEUTRAL: measures loss, not domain).
|
||
# pymupdf4llm yields structured Markdown but silently DROPS rendered display formulas and can
|
||
# splinter combining diacritics (measured on a LaTeX script: „P = {L …}" gone, „h¨aufig").
|
||
# pdftotext is structure-poor but faithful. Take the Markdown only when it preserves the bulk
|
||
# of the content; otherwise the faithful plaintext wins.
|
||
_PDF_FIDELITY_SYMBOLS = "≤≥∈∉⊆∪∧∨¬→Σδα{}="
|
||
|
||
|
||
def _pick_conversion(md: str | None, plain: str | None) -> tuple[str, str] | None:
|
||
if md is None and plain is None:
|
||
return None
|
||
if md is None:
|
||
return plain, "pdftotext"
|
||
if plain is None:
|
||
return md, "pymupdf4llm"
|
||
ok_len = len(md) >= 0.7 * len(plain)
|
||
sym_plain = sum(plain.count(c) for c in _PDF_FIDELITY_SYMBOLS)
|
||
ok_sym = sym_plain == 0 or sum(md.count(c) for c in _PDF_FIDELITY_SYMBOLS) >= 0.8 * sym_plain
|
||
ok_diakritik = md.count("\u00a8") <= plain.count("\u00a8") + 2 # standalone ¨ = splintered umlauts
|
||
if ok_len and ok_sym and ok_diakritik:
|
||
return md, "pymupdf4llm"
|
||
return plain, "pdftotext"
|
||
|
||
|
||
def _convert_pdfs(project: Path) -> None:
|
||
"""Convert PDFs in the project to .txt — agents read text instead of page images.
|
||
|
||
Called before every project generation; converts only if the .txt is missing or
|
||
older than the PDF. Both converters run; the fidelity guard picks the better result
|
||
per file. Neither available → hard error instead of an unreliable direct-read mode
|
||
(MiniMax image limit, vision cost)."""
|
||
pdfs = list(project.rglob("*.pdf"))
|
||
if not pdfs:
|
||
return
|
||
for pdf in pdfs:
|
||
txt = pdf.with_suffix(".txt")
|
||
if txt.exists() and txt.stat().st_mtime >= pdf.stat().st_mtime:
|
||
continue
|
||
picked = _pick_conversion(_pdf_markdown(pdf), _pdf_plaintext(pdf))
|
||
if picked is None:
|
||
raise RuntimeError(f"PDF conversion failed ({pdf.name}): weder pymupdf4llm noch "
|
||
"pdftotext verfügbar/erfolgreich (pip install pymupdf4llm oder poppler-utils)")
|
||
text, tool = picked
|
||
txt.write_text(text, encoding="utf-8")
|
||
_log(project.name, f"PDF konvertiert ({tool}): {pdf.name} → {txt.name}")
|
||
|
||
|
||
_SOURCE_TEMPLATE = {"projekt": "Blocks-Source-Projekt", "uni": "Blocks-Source-Uni", "link": "Blocks-Source-Link"}
|
||
|
||
|
||
def _text_sections(text: str, goal: int = RESEARCH_SECTION_CHARS) -> list[str]:
|
||
"""Split text at paragraph/line boundaries into sections of ~`ziel` chars (against lost-in-the-middle
|
||
on large documents). Small text stays ONE section. Content stays complete — only
|
||
separating whitespace is dropped."""
|
||
text = text.strip()
|
||
if len(text) <= goal:
|
||
return [text] if text else []
|
||
sections: list[str] = []
|
||
buf = ""
|
||
|
||
def flush():
|
||
nonlocal buf
|
||
if buf.strip():
|
||
sections.append(buf.strip())
|
||
buf = ""
|
||
|
||
for block in re.split(r"\n\s*\n", text): # at paragraph boundaries
|
||
block = block.strip()
|
||
if not block:
|
||
continue
|
||
if len(block) > goal: # single huge paragraph → hard-cut at lines
|
||
flush()
|
||
for line in block.split("\n"):
|
||
if buf and len(buf) + len(line) + 1 > goal:
|
||
flush()
|
||
buf += line + "\n"
|
||
flush()
|
||
elif buf and len(buf) + len(block) + 2 > goal:
|
||
flush()
|
||
buf = block
|
||
else:
|
||
buf = (buf + "\n\n" + block) if buf else block
|
||
flush()
|
||
return sections
|
||
|
||
|
||
# ── Inline evidence for judges ──────────────────────────────────────────────────────
|
||
# Judges used to re-search the corpus per session ({source} → "ls/find … read", ~10 tool
|
||
# turns each). The corpus excerpts now go INTO the prompt; the agent answers as text.
|
||
|
||
def _corpus_files(folder: Path | None, sources: list[str] | None) -> list[Path]:
|
||
"""The block's named source .txt files; fallback: every .txt in the folder."""
|
||
if folder is None or not folder.is_dir():
|
||
return []
|
||
if sources:
|
||
named = [folder / Path(s).with_suffix(".txt").name for s in sources if s]
|
||
named = [p for p in named if p.is_file()]
|
||
if named:
|
||
return named
|
||
return sorted(p for p in folder.glob("*.txt") if p.is_file())
|
||
|
||
|
||
def _q_tokens(text: str) -> set[str]:
|
||
return set(re.findall(r"\w{3,}", text.casefold()))
|
||
|
||
|
||
def _evidence_pack(folder: Path | None, sources: list[str] | None, queries: list[str],
|
||
budget: int = EVIDENCE_BUDGET_CHARS) -> str:
|
||
"""Keyword-selected corpus excerpts for a judge prompt. Sections are ranked by token
|
||
overlap with `queries` (block title + candidates); every query with any match gets its
|
||
best section (coverage guarantee), the rest of the budget takes the global top. Empty
|
||
string when there is no corpus — the caller keeps the old self-research source then."""
|
||
parts: list[tuple[str, int, str, set[str]]] = [] # (file, idx, text, tokens)
|
||
for f in _corpus_files(folder, sources):
|
||
try:
|
||
text = f.read_text(encoding="utf-8")
|
||
except OSError:
|
||
continue
|
||
for i, sec in enumerate(_text_sections(text), 1):
|
||
parts.append((f.name, i, sec, _q_tokens(sec)))
|
||
if not parts:
|
||
return ""
|
||
qtoks = [(_q_tokens(q)) for q in queries if q]
|
||
score = [sum(len(qt & p[3]) for qt in qtoks) for p in parts]
|
||
chosen: set[int] = set()
|
||
for qt in qtoks: # coverage guarantee: best section per query
|
||
best = max(range(len(parts)), key=lambda k: len(qt & parts[k][3]), default=None)
|
||
if best is not None and qt & parts[best][3]:
|
||
chosen.add(best)
|
||
used = sum(len(parts[k][2]) for k in chosen)
|
||
for k in sorted(range(len(parts)), key=lambda k: -score[k]): # top-up to budget
|
||
if k in chosen or score[k] <= 0:
|
||
continue
|
||
if used + len(parts[k][2]) > budget:
|
||
continue
|
||
chosen.add(k)
|
||
used += len(parts[k][2])
|
||
out, total = [], 0
|
||
for k in sorted(chosen): # document order for readability
|
||
fname, i, sec, _t = parts[k]
|
||
if total + len(sec) > max(budget, used): # hard cap incl. guarantee overshoot
|
||
break
|
||
out.append(f"── {fname} · Abschnitt {i} ──\n{sec}")
|
||
total += len(sec)
|
||
return "\n\n".join(out)
|
||
|
||
|
||
_CITE_POS = re.compile(r"\b(?:Z(?:eilen?)?|lines?)\.?\s*(\d+)(?:\s*[-–]\s*(\d+))?", re.I)
|
||
|
||
|
||
def _cite_ref(cite: str, files: list[Path]) -> tuple[Path, int, int] | None:
|
||
"""(file, line_lo, line_hi) from a cited_facts source string
|
||
(„Skript.txt, Übung 6.47, Z.1341-1344") — None when file or position is missing."""
|
||
c = (cite or "").casefold()
|
||
f = next((p for p in files if p.name.casefold() in c or p.stem.casefold() in c), None)
|
||
m = _CITE_POS.search(cite or "")
|
||
if f is None or m is None:
|
||
return None
|
||
lo, hi = int(m.group(1)), int(m.group(2) or m.group(1))
|
||
return (f, min(lo, hi), max(lo, hi))
|
||
|
||
|
||
def _cited_evidence(folder: Path | None, sources: list[str] | None, cites: list[str],
|
||
fallback_queries: list[str], budget: int = EVIDENCE_BUDGET_CHARS) -> str:
|
||
"""Evidence for the facts check: the EXACT cited regions (±EVIDENCE_CTX_LINES, merged,
|
||
line numbers in the header) — precise and tiny. Cites without a parseable position fall
|
||
back to the keyword pack. Empty string without a corpus."""
|
||
files = _corpus_files(folder, sources)
|
||
if not files:
|
||
return ""
|
||
ranges: dict[Path, list[tuple[int, int]]] = {}
|
||
unresolved = False
|
||
for c in cites:
|
||
ref = _cite_ref(c, files)
|
||
if ref is None:
|
||
unresolved = True
|
||
continue
|
||
f, lo, hi = ref
|
||
ranges.setdefault(f, []).append((max(1, lo - EVIDENCE_CTX_LINES), hi + EVIDENCE_CTX_LINES))
|
||
out, total = [], 0
|
||
for f in files:
|
||
if f not in ranges:
|
||
continue
|
||
try:
|
||
lines = f.read_text(encoding="utf-8").splitlines()
|
||
except OSError:
|
||
continue
|
||
merged: list[list[int]] = []
|
||
for lo, hi in sorted(ranges[f]):
|
||
hi = min(hi, len(lines))
|
||
if merged and lo <= merged[-1][1] + 1:
|
||
merged[-1][1] = max(merged[-1][1], hi)
|
||
else:
|
||
merged.append([lo, hi])
|
||
for lo, hi in merged:
|
||
sec = "\n".join(lines[lo - 1:hi])
|
||
if not sec.strip() or total + len(sec) > budget:
|
||
continue
|
||
out.append(f"── {f.name} · Z. {lo}-{hi} ──\n{sec}")
|
||
total += len(sec)
|
||
if unresolved or not out:
|
||
pack = _evidence_pack(folder, sources, fallback_queries, max(0, budget - total))
|
||
if pack:
|
||
out.append(pack)
|
||
return "\n\n".join(out)
|
||
|
||
|
||
def _reply_text(result) -> str:
|
||
"""Assistant text of a no-tool agent call ((rc, stdout, stderr) from run_agent)."""
|
||
return (result[1] or "") if result else ""
|
||
|
||
|
||
def _sink_or_file(result, path: Path, schema):
|
||
"""Text reply preferred, file fallback. Research agents used to WRITE their big JSON —
|
||
measured: one facts call spent 40 of 64 turns in a write/validate/repair loop (9 min).
|
||
As text, parse_json_text repairs the escaping in one pass; tools stay for research."""
|
||
val = _sink_json(result, path, schema)
|
||
if val is not None:
|
||
return val
|
||
return schema(_json_file(path))
|
||
|
||
|
||
def _sink_json(result, path: Path, schema):
|
||
"""Payload validator for no-tool agents: the JSON comes as reply TEXT; the engine
|
||
persists it to `path`, so resume guards and audit files keep working unchanged."""
|
||
data = parse_json_text(_reply_text(result))
|
||
val = schema(data)
|
||
if val is not None:
|
||
atomic_write_json(path, data)
|
||
return val
|
||
|
||
|
||
def material_folder(topic: str) -> Path | None:
|
||
"""Korpus fürs Inline-Material: echte Quelle (uni/projekt/link) oder bei thema die
|
||
Fundstellen der Research-Reader (arbeit/material/*.txt). Nur für Evidence-Packs —
|
||
Konsens-Gate und QA messen unverändert gegen die konfigurierte Quelle (Messinvarianz)."""
|
||
f = source_folder(topic)
|
||
if f is not None:
|
||
return f
|
||
d = arbeit_dir(topic) / "material"
|
||
return d if d.is_dir() and any(d.glob("*.txt")) else None
|
||
|
||
|
||
def _build_research_prompt(topic: str, out_path: Path, instructions: str, type: str, folder: Path | None, fokus: str = "", section: str = "", material_path: Path | None = None) -> str:
|
||
if section:
|
||
# Section mode (uni/projekt): text directly in the prompt → small context, no file reading.
|
||
source = section
|
||
elif type in _SOURCE_TEMPLATE:
|
||
source = _prompt(_SOURCE_TEMPLATE[type], project=folder)
|
||
else:
|
||
source = _prompt("Blocks-Source-Thema", topic=topic)
|
||
if material_path is not None:
|
||
# thema: Fundstellen sichern — sie werden das Inline-Material der Folgeschritte
|
||
# (Finder/Facts liefen sonst mit eigener Websuche → Reasoning-Schleifen, Retries)
|
||
source += ("\n\nALSO write the file " + str(material_path) + " as you research: for every "
|
||
"source you use, append one line with the URL followed by the relevant excerpt "
|
||
"(plain text). Later steps back all facts ONLY against this material — "
|
||
"an excerpt you skip here cannot back anything later.")
|
||
return _prompt(
|
||
"Blocks-Research",
|
||
topic=topic, source=source, blocks_path=out_path, focus=fokus, extra=_extra(instructions),
|
||
)
|
||
|
||
|
||
def _file_payload(path: Path):
|
||
"""Valid if the slot file exists and contains numbered entries."""
|
||
if not path.exists():
|
||
return None
|
||
text = path.read_text(encoding="utf-8")
|
||
return text if _parse_selection(text) else None
|
||
|
||
|
||
|
||
|
||
def _read(p: Path) -> str:
|
||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
||
|
||
|
||
def _chunk_nums(items: list, n: int) -> list[list]:
|
||
"""Splits a flat list into n chunks as equal in size as possible."""
|
||
n = max(1, n)
|
||
size = max(1, math.ceil(len(items) / n))
|
||
return [items[i:i + size] for i in range(0, len(items), size)]
|
||
|
||
|
||
def _n_chunks(count: int, size: int) -> int:
|
||
return min(SUBBLOCK_MAX, max(1, math.ceil(count / size)))
|
||
|
||
|
||
_NEG_LEMMA = {"nicht": "nicht", "ohne": "ohne", "nie": "nie", "niemals": "nie",
|
||
"kein": "kein", "keine": "kein", "keinen": "kein", "keiner": "kein",
|
||
"keinem": "kein", "keines": "kein"}
|
||
|
||
|
||
def _neg_set(title: str) -> frozenset:
|
||
"""Lemmatized negation tokens of a title — antonym statements measure cos 0.91–0.95
|
||
(above any usable variant threshold), so equal negation sets are a hard merge precondition."""
|
||
return frozenset(l for t in re.findall(r"\w+", _norm_title(title)) if (l := _NEG_LEMMA.get(t)))
|
||
|
||
|
||
def _sub_tokens(title: str) -> set:
|
||
return set(re.findall(r"\w+", _norm_title(title)))
|
||
|
||
|
||
def _variant_clusters(titles: list[str], mentions: list[int], sims) -> list[dict]:
|
||
"""Fold phrasing VARIANTS of one concept BEFORE the consensus count: finders rephrase per
|
||
round, so exact-norm counting starves real concepts. Union-find over cos ≥ SUB_VARIANT_COS
|
||
with the negation guard. → [{"rep": idx, "members": [idx…], "mentions": sum}]."""
|
||
n = len(titles)
|
||
parent = list(range(n))
|
||
|
||
def find(x):
|
||
while parent[x] != x:
|
||
parent[x] = parent[parent[x]]
|
||
x = parent[x]
|
||
return x
|
||
|
||
negs = [_neg_set(t) for t in titles]
|
||
for i in range(n):
|
||
for j in range(i + 1, n):
|
||
if float(sims[i][j]) >= SUB_VARIANT_COS and negs[i] == negs[j]:
|
||
parent[find(i)] = find(j)
|
||
groups: dict[int, list[int]] = {}
|
||
for i in range(n):
|
||
groups.setdefault(find(i), []).append(i)
|
||
return [{"rep": max(g, key=lambda k: (len(titles[k]), -k)), "members": g,
|
||
"mentions": sum(mentions[k] for k in g)} for g in groups.values()]
|
||
|
||
|
||
async def _dedup_subblocks(topic: str, raw: dict[str, list[str]]) -> None:
|
||
"""Deterministic near-duplicate filter per block: subblocks with cosine ≥
|
||
EMBEDDING_SUB_DUP are the same statement (reliable in the narrow block context — no LLM
|
||
needed). Per duplicate group keeps the most informative (longest); rest → DB discarded + out of `roh`.
|
||
Model missing → silently skip (like the rest of the embedding fallback)."""
|
||
if not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
|
||
return
|
||
for title, subs in list(raw.items()):
|
||
if len(subs) < 2:
|
||
continue
|
||
sims = await asyncio.to_thread(embedding.embed_sims, subs)
|
||
if sims is None:
|
||
return
|
||
keepers: list[int] = []
|
||
discarded: list[int] = []
|
||
negs = [_neg_set(s) for s in subs]
|
||
for i in sorted(range(len(subs)), key=lambda x: (-len(subs[x]), x)): # most informative first
|
||
if any(float(sims[i][j]) >= EMBEDDING_SUB_DUP and negs[i] == negs[j] for j in keepers):
|
||
discarded.append(i)
|
||
else:
|
||
keepers.append(i)
|
||
if not discarded:
|
||
continue
|
||
bnorm = _norm_title(title)
|
||
for i in discarded:
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(subs[i]), status="discarded")
|
||
raw[title] = [subs[i] for i in sorted(keepers)] # original order of the kept ones
|
||
|
||
|
||
def _facts_union(wf: dict, lf: dict) -> None:
|
||
"""Merge a folded sub's facts into the winner's: key_points/cited_facts union
|
||
(exact-duplicate-free), scalar fields only fill gaps."""
|
||
for feld in ("key_points", "cited_facts"):
|
||
have = wf.get(feld) or []
|
||
seen = {json.dumps(e, sort_keys=True, ensure_ascii=False) for e in have}
|
||
fresh = [e for e in (lf.get(feld) or [])
|
||
if json.dumps(e, sort_keys=True, ensure_ascii=False) not in seen]
|
||
if fresh:
|
||
wf[feld] = have + fresh
|
||
for feld in ("prerequisites", "hurdles", "example_idea"):
|
||
if not wf.get(feld) and lf.get(feld):
|
||
wf[feld] = lf[feld]
|
||
|
||
|
||
def _agreed_cliques(pair_sets: list[set], negs: list, n: int) -> list[list[int]]:
|
||
"""Union-find over the UNANIMOUS pairs (both judges grouped them), negation-guarded."""
|
||
agreed = {(a, b) for a, b in pair_sets[0] & pair_sets[1] if negs[a - 1] == negs[b - 1]}
|
||
parent = list(range(n + 1))
|
||
|
||
def find(x):
|
||
while parent[x] != x:
|
||
parent[x] = parent[parent[x]]
|
||
x = parent[x]
|
||
return x
|
||
|
||
for a, b in agreed:
|
||
parent[find(a)] = find(b)
|
||
groups: dict[int, list[int]] = {}
|
||
for k in range(1, n + 1):
|
||
groups.setdefault(find(k), []).append(k)
|
||
return [g for g in groups.values() if len(g) >= 2]
|
||
|
||
|
||
def _pairs_of(groups) -> set:
|
||
ps: set[tuple[int, int]] = set()
|
||
for ids in groups:
|
||
ps |= {(a, b) for x, a in enumerate(ids) for b in ids[x + 1:]}
|
||
return ps
|
||
|
||
|
||
_LUECKEN_CAP = 3 # the gap list feeds ONE finder round — an uncapped list doubled the decomposition
|
||
|
||
|
||
def _luecken_schnitt(l1: list[str], l2: list[str], cap: int = _LUECKEN_CAP) -> list[str]:
|
||
"""Gaps BOTH judges name — exact strings never match across paraphrases, so a gap
|
||
survives when the other judge names one sharing a distinctive token (≥4 chars).
|
||
j1's phrasing wins. The measured union produced 107 'gaps' on 216 subs."""
|
||
def toks(s):
|
||
return {t for t in _sub_tokens(s) if len(t) >= 4}
|
||
toks2 = [toks(l) for l in l2]
|
||
out = [l for l in l1 if toks(l) and any(toks(l) & t2 for t2 in toks2)]
|
||
return out[:cap]
|
||
|
||
|
||
def _match_sub(agent_sub: str, rel: list[str]) -> str:
|
||
"""Map the agent's subblock title to the matching relevant title — exact,
|
||
then normalized, then substring (the agent drops e.g. the prefix "Question: ").
|
||
No match → keep the agent title. This way NO pattern is lost to a title mismatch."""
|
||
if agent_sub in rel:
|
||
return agent_sub
|
||
an = _norm_title(agent_sub)
|
||
for r in rel:
|
||
rn = _norm_title(r)
|
||
if an and rn and (an == rn or an in rn or rn in an):
|
||
return r
|
||
return agent_sub
|
||
|
||
|
||
def _subs_hash(sidecar_or_raw: dict) -> str:
|
||
"""Sub-set identity for the resume files of the sub-CONSUMING stages (levels/relevance/
|
||
questions/artefacts). Without it a re-run with a recut sub set adopted the stale stage
|
||
results (measured: 626 orphans — artefacts of the old 425-sub set re-imported)."""
|
||
parts: list[str] = []
|
||
for title, subs in sidecar_or_raw.items():
|
||
parts.append(str(title))
|
||
for s in subs:
|
||
parts.append(s["title"] if isinstance(s, dict) else str(s))
|
||
return hashlib.md5("\n".join(parts).encode()).hexdigest()[:8]
|
||
|
||
|
||
_FACTS_FIELDS = ("key_points", "prerequisites", "hurdles", "cited_facts", "example_idea")
|
||
|
||
|
||
def _facts_schema(data) -> list[dict] | None:
|
||
"""{"facts": [{block, subblock, …}]} → valid list · otherwise None.
|
||
Strictly separates belegte_facts (with source) from example_idee (generative)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("facts"), list):
|
||
return None
|
||
out = []
|
||
for e in data["facts"]:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
blk = str(e.get("block", "")).strip()
|
||
sub = str(e.get("subblock", "")).strip()
|
||
if not blk or not sub:
|
||
continue
|
||
bf = [{"text": t, "source": str(f.get("source", "")).strip()}
|
||
for f in (e.get("cited_facts") or []) if isinstance(f, dict) and (t := str(f.get("text", "")).strip())]
|
||
out.append({
|
||
"block": blk, "subblock": sub,
|
||
"key_points": [k for x in (e.get("key_points") or []) if (k := str(x).strip())],
|
||
"prerequisites": str(e.get("prerequisites", "")).strip(),
|
||
"hurdles": str(e.get("hurdles", "")).strip(),
|
||
"cited_facts": bf,
|
||
"example_idea": str(e.get("example_idea", "")).strip(),
|
||
})
|
||
return out or None
|
||
|
||
|
||
def _facts_lines(fk: dict) -> str:
|
||
z = []
|
||
if fk.get("key_points"):
|
||
z.append("Kernpunkte: " + " · ".join(str(k) for k in fk["key_points"]))
|
||
if fk.get("prerequisites"):
|
||
z.append("Voraussetzung: " + fk["prerequisites"])
|
||
if fk.get("hurdles"):
|
||
z.append("Hürde: " + fk["hurdles"])
|
||
for bf in fk.get("cited_facts", []):
|
||
z.append(f"FAKT: {bf['text']} (Source: {bf.get('source', '?')})")
|
||
if fk.get("example_idea"):
|
||
z.append("Example: " + fk["example_idea"])
|
||
return "\n".join(z)
|
||
|
||
|
||
# ── Inventory in the DB: research loop · consolidation · clarification ────────────
|
||
|
||
|
||
|
||
def _crawl_index(folder) -> dict[str, str]:
|
||
"""Alias (filename OR QUELLE: URL, lowercase) → canonical page key (filename)."""
|
||
idx: dict[str, str] = {}
|
||
if not folder or not Path(folder).is_dir():
|
||
return idx
|
||
for p in sorted(Path(folder).glob("*.txt")):
|
||
key = p.name
|
||
idx[key.lower()] = key
|
||
try:
|
||
first_line = p.read_text(encoding="utf-8").splitlines()[0]
|
||
except (OSError, IndexError):
|
||
first_line = ""
|
||
if first_line.startswith("QUELLE:"):
|
||
url = first_line[len("QUELLE:"):].strip()
|
||
if url:
|
||
idx[url.lower()] = key
|
||
idx[url.rstrip("/").lower()] = key
|
||
return idx
|
||
|
||
|
||
async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool:
|
||
"""Step "Source prep": load link pages + PDF convert. → True (ok) / False (cancel/error).
|
||
link: load each user-supplied URL (one page, no following); ALL pages count as content
|
||
(curated — no triage). thema: nothing. projekt/uni: only PDFs (curated folder)."""
|
||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||
if not folder:
|
||
return True # thema → no source to prepare
|
||
if q["type"] != "link":
|
||
await asyncio.to_thread(_convert_pdfs, folder) # projekt/uni: only PDFs, no triage
|
||
return True
|
||
if await db.get_step_status(topic, "Source prep") == "done":
|
||
return True
|
||
if not _crawl_done(topic):
|
||
urls = [ln.strip() for ln in q["location"].splitlines() if ln.strip()]
|
||
set_p("Loading pages…", step=_step_idx(topic, "Source prep"))
|
||
n = await asyncio.to_thread(load_pages, urls, folder, cancelled=is_cancelled)
|
||
if is_cancelled():
|
||
return False
|
||
if not n:
|
||
_blocks_errors[topic] = "Links yielded no content — check the URLs"
|
||
return False
|
||
await asyncio.to_thread(_convert_pdfs, folder)
|
||
pages = sorted(set(_crawl_index(folder).values()))
|
||
if pages:
|
||
# Curated links: the user picked them — keep every page, no rule/LLM triage.
|
||
await db.delete_coverage(topic)
|
||
await db.mark_content(topic, pages, [])
|
||
_log(topic, f"Links: {len(pages)} pages as content (curated, no triage)")
|
||
await db.set_step_status(topic, "Source prep", "done")
|
||
return True
|
||
|
||
|
||
|
||
|
||
|
||
|
||
_ASPECT_MARKER = ("∈ np", "∈np", " in np", "np-schwer", "np-vollständig", "verifizierer",
|
||
"zertifikat", "ndtm", "nicht-determ", "lower bound", "untere schranke",
|
||
"bzgl", "als sprache", "beweis", "güte", "austausch",
|
||
# generic bound/limit/runtime property stems (a "…-Grenze"/"…-Schranke"/"…-Laufzeit"
|
||
# is a property OF a concept, not the concept — MDM survivorship must never pick it as
|
||
# the representative, so it scores >0 here like the other aspect markers):
|
||
"grenze", "schranke", "laufzeit")
|
||
|
||
|
||
def _aspect_marker(title: str) -> int:
|
||
"""Number of property markers in the title (∈NP, NP-hard, verifier, lower bound …).
|
||
0 = generic main concept (the problem itself); >0 = a property of it."""
|
||
t = title.casefold()
|
||
return sum(1 for m in _ASPECT_MARKER if m in t)
|
||
|
||
|
||
_REFERENCE_RE = re.compile(r'^(Satz|Lemma|Korollar|Bemerkung|Definition)\s*[\d.]+\s*(\([a-z]\)|[a-z])?\s*$', re.I)
|
||
|
||
# Catalogue scaffolding around a concept name: a leading label+number ("Satz 7.13", "Satz 7.6:")
|
||
# or a trailing "(Definition 6.19)". _reference_strip peels it off and returns the concept remainder;
|
||
# _is_reference is then just "nothing survives the strip". Generic — no author/domain whitelist. The
|
||
# remainder feeds the rename step (a card must be titled by its concept, not its catalogue number).
|
||
_REF_WORDS = (r'(?:Satz|Lemma|Korollar|Corollary|Bemerkung|Beobachtung|Definition|Def'
|
||
r'|Theorem|Proposition|Folgerung|Kapitel|Abschnitt)')
|
||
_REF_PREFIX_RE = re.compile(rf'^\s*{_REF_WORDS}\s*\d+(?:\.\d+)*[a-z]?\s*[:.\-–—]?\s*', re.I)
|
||
_REF_SUFFIX_RE = re.compile(rf'\s*[\(\[]\s*{_REF_WORDS}\s*\d+(?:\.\d+)*[a-z]?\s*[\)\]]\s*$', re.I)
|
||
_WHOLE_PAREN_RE = re.compile(r'^\s*[\(\[]\s*(.{2,}?)\s*[\)\]]\s*$')
|
||
|
||
|
||
def _reference_strip(title: str) -> str:
|
||
"""Concept remainder of a catalogue-referenced title, scaffolding removed. '' for a pure reference
|
||
without a concept ('Bemerkung 7.22' → ''); the concept otherwise ('Satz 7.13 (Christofides)' →
|
||
'Christofides', 'Satz 7.6: Kriterium für X' → 'Kriterium für X', 'N P … (Definition 6.19)' →
|
||
'N P …'). A title without any catalogue reference comes back unchanged."""
|
||
t = (title or "").strip()
|
||
t = _REF_SUFFIX_RE.sub("", t).strip()
|
||
m = _REF_PREFIX_RE.match(t)
|
||
if m:
|
||
t = t[m.end():].strip()
|
||
if (w := _WHOLE_PAREN_RE.match(t)): # leading label left a lone '(Christofides)'
|
||
t = w.group(1).strip()
|
||
return t
|
||
|
||
|
||
def _is_reference(title: str) -> bool:
|
||
"""True for a pure reference/placeholder title WITHOUT concept content: "Satz 7.18", "Lemma 6.2",
|
||
"Korollar 6.18" (number, no name) and marked spots "Bedingung (**)". False once a concept survives
|
||
the catalogue strip ("Satz 7.13 (Christofides)", "Satz 6.24: Cook/Levin") and for short technical
|
||
symbols like "P⊆NP"/"Σ*" (real concepts, no reference scaffolding)."""
|
||
t = title.strip()
|
||
if re.search(r'\(\*+\)', t): # marked spot "(**)" / "(*)"
|
||
return True
|
||
return bool(_REFERENCE_RE.match(t)) or not _reference_strip(t)
|
||
|
||
|
||
def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) -> dict:
|
||
"""Representative of a cluster = the main concept (fewest property markers — the problem
|
||
itself, not "… ∈ NP"); tie → most frequent norm title → most readers. Title globally unique
|
||
(suffix ' (2)') so it works as a key."""
|
||
by_norm: dict[str, list[int]] = {}
|
||
for k in idxs:
|
||
by_norm.setdefault(_norm_title(candidates[k]["title"]), []).append(k)
|
||
|
||
def weight(nb: str):
|
||
ms = by_norm[nb]
|
||
reader = set().union(*[set(candidates[m]["reader"]) for m in ms]) if ms else set()
|
||
# reference/placeholder titles ("Satz 7.18") last — prefer a meaningful member.
|
||
is_real = not _is_reference(candidates[ms[0]]["title"])
|
||
return (is_real, -_aspect_marker(nb), len(ms), len(reader))
|
||
|
||
best = max(by_norm, key=weight)
|
||
k = max(by_norm[best], key=lambda m: len(candidates[m]["description"]))
|
||
title = candidates[k]["title"]
|
||
n = 2
|
||
while _norm_title(title) in seen_norm:
|
||
title = f"{candidates[k]['title']} ({n})"
|
||
n += 1
|
||
seen_norm.add(_norm_title(title))
|
||
return {"title": title, "description": candidates[k]["description"]}
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _pairs_schema(data) -> dict[int, bool] | None:
|
||
"""{"pairs": {"1": "ja", "2": "nein", …}} → {pair_nr: True/False} · otherwise None."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("pairs"), dict):
|
||
return None
|
||
out: dict[int, bool] = {}
|
||
for k, v in data["pairs"].items():
|
||
try:
|
||
nr = int(k)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
out[nr] = str(v).strip().casefold() in ("ja", "yes", "true", "1")
|
||
return out or None
|
||
|
||
|
||
def _cliques(n: int, edge_list: list[tuple[int, int]]) -> list[list[int]]:
|
||
"""Complete-link: greedy maximal cliques over the confirmed duplicate edges. A group
|
||
forms only if ALL its nodes are pairwise connected → no chaining (A=B + B=C forms
|
||
NO group {A,B,C} as long as A=C is missing). Only cliques ≥2 are returned."""
|
||
adj: dict[int, set[int]] = {i: set() for i in range(n)}
|
||
for a, b in edge_list:
|
||
adj[a].add(b)
|
||
adj[b].add(a)
|
||
used: set[int] = set()
|
||
groups: list[list[int]] = []
|
||
for v in sorted(range(n), key=lambda x: -len(adj[x])):
|
||
if v in used or not adj[v]:
|
||
continue
|
||
clique = {v}
|
||
for u in sorted(adj[v], key=lambda x: -len(adj[x])):
|
||
if u not in used and clique <= adj[u] | {u}: # u connected to ALL previous ones
|
||
clique.add(u)
|
||
if len(clique) >= 2:
|
||
groups.append(sorted(clique))
|
||
used |= clique
|
||
return groups
|
||
|
||
|
||
# Canonical-name blocking (dedup recall): entity resolution's recall ceiling is set by candidate
|
||
# generation — a pair that never shares a candidate can never be merged. Normalize a title to a
|
||
# scaffolding-free, operator-class-normalized, order-independent key so surface variants of ONE entity
|
||
# collapse ("Offenes Problem P=NP?" ≡ "P vs NP" ≡ "P=NP"). Generic (no course terms): strip a small
|
||
# stoplist of catalogue/wrapper words, fold the relation operators into class tokens, sort content tokens.
|
||
_CANON_STOP = re.compile(
|
||
r'\b(?:offenes?|open|problem|frage|question|algorithmus|algorithm|verfahren|method|methode|'
|
||
r'satz|theorem|lemma|korollar|definition|def|das|der|die|the|ein|eine|einen|a|an|'
|
||
r'von|of|für|for|und|and|zum|zur|im)\b', re.I)
|
||
|
||
|
||
# German compounds are head-final: "Cliquenproblem" = stem + optional Fuge + head noun "problem".
|
||
# The standalone wrapper "problem" is already in _CANON_STOP, but a GLUED head hides the stem from ER
|
||
# blocking ("Cliquenproblem" would never share a key with "Clique"). Strip the glued "problem" head so
|
||
# the stem surfaces — a generic suffix rule, not a term list. The Fuge is only a consonant n/s
|
||
# ("Cliquen-", "Entscheidungs-"): matching a vowel "e" would eat the stem's own "e" ("clique" → "cliqu").
|
||
# The ≥3-char stem lookbehind keeps a card literally titled "Problem"/"Probleme" (→ _CANON_STOP) intact.
|
||
_CANON_GLUED_HEAD = re.compile(r'(?<=\w{3})[ns]?problem(?:en|e|s)?\b', re.I)
|
||
|
||
|
||
# Catalogue references ("Definition 6.19", "Satz 7.8") are scaffolding INCLUDING their number —
|
||
# stripped as a phrase, so the digits don't pollute the key. Variant digits ("3-SAT") have no
|
||
# scaffolding word in front and survive.
|
||
_CANON_CATALOGUE = re.compile(
|
||
r'\b(?:definition|def|satz|lemma|korollar|corollary|theorem|proposition'
|
||
r'|kapitel|chapter|abschnitt|section)\s*\d+(?:\.\d+)*\b', re.I)
|
||
|
||
|
||
_PAREN_GROUP = re.compile(r'^\s*(.*?)\s*\(([^()]{2,60})\)\s*$')
|
||
|
||
|
||
def _title_variants(title: str) -> set[str]:
|
||
"""Acronym/expansion variants of a "X (Y)" title — normalized outer part and paren
|
||
content. „Satisfiability Problem (SAT)" → {'satisfiability problem', 'sat'}: matched
|
||
against another card's norm/key this makes acronym↔expansion pairs dedup CANDIDATES
|
||
(measured: title cosine 'sat' vs the long form is 0.53, far below the floor).
|
||
Titles without exactly one trailing paren group → empty set."""
|
||
from textkit import _norm_title
|
||
m = _PAREN_GROUP.match(title or "")
|
||
if not m:
|
||
return set()
|
||
return {v for v in (_norm_title(m.group(1)), _norm_title(m.group(2))) if v}
|
||
|
||
|
||
def _canonical_key(title: str) -> str:
|
||
"""Order-independent canonical key of a title (scaffolding stripped, relation operators normalized).
|
||
Two titles with the same key denote the same entity with ~100% precision (ER blocking). Empty string
|
||
if nothing survives (never auto-merged)."""
|
||
s = unicodedata.normalize("NFKC", title)
|
||
s = re.sub(r'([a-zäöüß])([A-ZÄÖÜ])', r'\1 \2', s) # CamelCase → two tokens
|
||
s = s.casefold()
|
||
s = _CANON_CATALOGUE.sub(' ', s)
|
||
s = re.sub(r'≟|\bversus\b|\bvs\.?\b|=', ' opeq ', s) # equality / "vs" → one token
|
||
s = re.sub(r'≤|⪯|→|⇒|⟹|\breduces?\s+to\b|\breduziert\b', ' opred ', s) # reduction → one token
|
||
s = _CANON_GLUED_HEAD.sub(' ', s) # glued "…problem" head → stem
|
||
s = _CANON_STOP.sub(' ', s)
|
||
s = re.sub(r'[^\w ]', ' ', s) # drop punctuation/symbols
|
||
return " ".join(sorted(t for t in s.split() if t))
|
||
|
||
|
||
# Relation-triple individuation (dedup precision): a reduction/relation "A ≤ B" is identified by BOTH
|
||
# operands AND direction (RDF-triple identity / SKOS narrowMatch — a subset/restriction is NOT the same).
|
||
# So "SAT ≤ Clique" ≠ "3-SAT ≤ Clique" (source differs) and "A → B" ≠ "B → A" (direction). Two DIFFERENT
|
||
# relations must never merge, even if a judge or a high title-cosine says so.
|
||
_REL_STRIP = re.compile(r'^\s*(?:satz|lemma|korollar|corollary|theorem|proposition)\s*[\d.]*\s*:?\s*'
|
||
r'|^\s*redu[ck]tion(?:en|s)?\s*:?\s*', re.I)
|
||
# trailing scaffolding ("SetCover ≤ HittingSet Reduktion") is NOT part of the target operand —
|
||
# without this strip the guard false-alarms and blocks the correct merge with the bare relation
|
||
_REL_STRIP_TAIL = re.compile(r'[\s\-–]*(?:redu[ck]tion(?:en|s)?|transformation(?:en|s)?)\s*$', re.I)
|
||
# an attached p/m marker ("≤p", "≤m") is operator notation, not part of the right operand
|
||
_REL_OPERATOR = re.compile(r'[≤⪯≥⊆⊊→⇒⟹⇔←][pm]?|=>|<=|->', re.I)
|
||
|
||
|
||
def _relation_operands(title: str) -> tuple[str, str] | None:
|
||
"""(canonical_source, canonical_target) of a relation/reduction title, else None (not a relation).
|
||
Operands canonicalized (lowercased, non-alphanumerics stripped) so spacing/hyphenation don't matter."""
|
||
t = _REL_STRIP.sub('', title, count=1)
|
||
t = _REL_STRIP_TAIL.sub('', t, count=1)
|
||
m = _REL_OPERATOR.search(t)
|
||
if not m:
|
||
return None
|
||
left = re.sub(r'[\W_]', '', t[:m.start()].casefold()) # keep unicode letters/digits (umlauts), drop the rest
|
||
right = re.sub(r'[\W_]', '', t[m.end():].casefold())
|
||
if not left or not right:
|
||
return None
|
||
return (left, right)
|
||
|
||
|
||
def _relation_conflict(title_a: str, title_b: str) -> bool:
|
||
"""True if BOTH titles are relations/reductions but denote DIFFERENT ones (operands or direction
|
||
differ) → they must NOT be merged. False if either is not a relation, or they are the same relation."""
|
||
a, b = _relation_operands(title_a), _relation_operands(title_b)
|
||
return a is not None and b is not None and a != b
|
||
|
||
|
||
def _direction_conflict(title_a: str, title_b: str) -> bool:
|
||
"""Nur der DETERMINISTISCH sichere Teil des Relation-Guards: dieselben Operanden in
|
||
GETAUSCHTER Richtung („A ≤ B" vs „B ≤ A") — immer eine andere Reduktion. Ein Operand-Unterschied
|
||
(z. B. „CLIQUE" vs „k-CLIQUE" — Synonym? oder echt verschieden?) ist eine Bedeutungsfrage und
|
||
bleibt dem (belegten) Judge überlassen, statt per String-Vergleich blind blockiert zu werden."""
|
||
a, b = _relation_operands(title_a), _relation_operands(title_b)
|
||
return a is not None and b is not None and a != b and set(a) == set(b)
|
||
|
||
|
||
|
||
|
||
def _filter_schema(data) -> dict[int, int] | None:
|
||
"""{"fragments": {"3": 7, "12": 8}} → {block_nr: parent_nr} · None on invalid structure.
|
||
Empty dict = valid (nothing to degrade). Parent ≠ itself."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("fragments"), dict):
|
||
return None
|
||
out: dict[int, int] = {}
|
||
for k, v in data["fragments"].items():
|
||
try:
|
||
nr, parent = int(k), int(v)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if nr != parent:
|
||
out[nr] = parent
|
||
return out
|
||
|
||
|
||
# Pure notation/symbols without a standalone concept — kept narrow (FP~0, checked against aak;
|
||
# "KNF"/"MST"/"NP" do NOT match). These are discarded autonomously (need no parent).
|
||
_FILTER_NOTATION = re.compile(r'^\s*\|.{1,6}\|\s*$|^Güte\s+\d+\s*$')
|
||
# Exercise-sheet / cross-reference artefacts — the SECOND gate for a judge `drop` verdict: a hard-drop
|
||
# (parentless removal) only fires when the judge lists the number in `drop` AND `_is_artifact(title)`.
|
||
# A judge-drop without a match degrades to "keep" (logged), never deleted (FP~0 discipline like
|
||
# _FILTER_NOTATION). Shape-matching on a NORMALIZED, title-side string (see _is_artifact): four branches —
|
||
# (1) lettered/Roman/numeric sub-claims "(Aussage i)"/"Aussage (a)"/"Teil (b)"/"Fall (2)" in ANY
|
||
# parenthesization; (2) sheet refs "Blatt 10"/"Aufgabe 3"/"Übung"; (3) worked-example/table/figure refs
|
||
# "Beispiel Tab. 7.1" (a NUMBER is required — guards polysemes "Hash-Tabelle"/"Bijektive Abbildung");
|
||
# (4) parenthesized "(Variante)". Theorem words (Satz/Definition/Lemma) are deliberately NOT in the
|
||
# vocabulary, so "Satz 6.24 Cook/Levin — SAT ist NP-vollständig" is kept. Word boundaries guard prefixes
|
||
# (Aussagenlogik/Teilmenge/Blattknoten/Fallunterscheidung).
|
||
_FILTER_ARTIFACT = re.compile(r"""
|
||
\b(?:aussage|teil|behauptung|fall)\b[\s(]*(?:[ivx]{1,4}|[a-z]|\d{1,2})\)?(?![a-zäöüß])
|
||
| \b(?:blatt|aufgabe|serie|hausaufgabe)\s*\d+(?:[.,]\d+)*
|
||
| \b(?:übungsblatt|übung|uebung)\b
|
||
| \b(?:beispiel|abbildung|abb|tabelle|tab|bild|grafik|diagramm|figur|skizze)\b\.?\s*\d+(?:[.,]\d+)*
|
||
| \(\s*(?:variante|variation|spezialfall|sonderfall)\s*\)
|
||
""", re.VERBOSE)
|
||
|
||
|
||
def _is_artifact(title: str) -> bool:
|
||
"""True if the TITLE looks like exercise-sheet / cross-reference scaffolding (P1 hard-drop gate).
|
||
NFKC + casefold normalization (position/case/umlaut/Unicode-invariant: folds Ⅱ→ii, full-width, NBSP),
|
||
then match only the title side of the em-dash — a real concept whose *description* merely cites a
|
||
sheet ("… — vgl. Aufgabe 3") is never dropped. Normalize for matching only; never store the result."""
|
||
norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip()
|
||
head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0] # spaced dash only → "3-SAT"/"np-schwer" unsplit
|
||
return bool(_FILTER_ARTIFACT.search(head))
|
||
# Property/runtime suspicion — marks lines for the judge's verdict (NO auto-drop, FP too high:
|
||
# "NP-Schwere", reductions with "∈NP" are real blocks). Complements _aspekt_marker.
|
||
_FILTER_PREDICATE = re.compile(
|
||
r'ist NP-(vollständig|schwer)|NP-(Vollständigkeit|Schwere) von|ETH (Konsequenz|Lower Bound)'
|
||
r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP'
|
||
# fragment families that the aspect substrings miss (all title/desc, advisory ⚠ only):
|
||
r'|\bSatz\s+\d|\bLemma\s+\d|\bKorollar\s+\d' # bare theorem/proof references (with number)
|
||
r'|^\s*(?:Remark|Bemerkung|Anmerkung|Note|Notiz|Beobachtung|Observation)\b' # EN+DE remark labels
|
||
r'|\bSatz\s*:|\bSatz\s+[A-Z]\b' # "Satz:" (colon, no number) / "Satz D*" letter label
|
||
r'|\bGegenbeispiel\b|\bWorst[- ]?Case\b|Schärfe\s+der\b' # proof-example / sharpness facets
|
||
r'|2\s*\^\s*[{(]?\s*[Ωωoο]\s*\(' # ETH exponential bound 2^Ω(…)/2^o(…)
|
||
r'|\d\s*[−–-]\s*1\s*/\s*m' # approximation-güte ratio "2 − 1/m"
|
||
r'|Variablenungleichung|[α-ωΑ-Ω][a-z]?-?Variablen', re.I) # proof variables (αu-Variablen …)
|
||
|
||
|
||
def _filter_suspect(b: dict) -> bool:
|
||
"""Heuristic flag: could be a property/detail of another block."""
|
||
return _aspect_marker(b["title"]) > 0 or bool(_FILTER_PREDICATE.search(f"{b['title']} {b['description'] or ''}"))
|
||
|
||
|
||
def _root(nr: int, fragments: dict[int, int]) -> tuple[int, bool]:
|
||
"""Follow the fragment→parent chain to the first ancestor that is NOT itself a fragment.
|
||
Returns (root, cyclic). This walk IS the fixpoint (no separate iteration): a mid-tier
|
||
fragment whose own parent is also a fragment resolves to the top-level real block, so all
|
||
chain levels collapse in one pass (unlike the old single-level parent_set shield).
|
||
Cycle guard: on revisiting a node returns (node, True) → caller keeps both (no annihilation)."""
|
||
seen: set[int] = set()
|
||
cur = nr
|
||
while cur in fragments:
|
||
if cur in seen:
|
||
return cur, True
|
||
seen.add(cur)
|
||
cur = fragments[cur]
|
||
return cur, False
|
||
|
||
|
||
def _containment_parent(frag_norm: str, others: list[tuple[int, str]]) -> int | None:
|
||
"""Deterministic parent-by-name-containment for a ⚠-flagged survivor: a fragment whose title NAMES
|
||
another block ("Lower Bound … für VERTEX COVER" → Vertex Cover; "List Scheduling Güte …" → List
|
||
Scheduling). `others` = (nr, title_norm) of the OTHER blocks. A parent is a block whose normalized
|
||
title occurs as a WHOLE-WORD span inside `frag_norm`, is SIGNIFICANT (≥2 tokens or ≥6 chars — never
|
||
"p"/"np"/"sat") and strictly shorter than the fragment. Returns the parent nr on EXACTLY ONE match,
|
||
else None (0 or ≥2 → leave to the judge). Whole-word + significance + exactly-one → near-FP-0."""
|
||
hits = []
|
||
for nr, ptitle in others:
|
||
if not ptitle or ptitle == frag_norm or len(ptitle) >= len(frag_norm):
|
||
continue
|
||
if len(ptitle) < 6 and ptitle.count(" ") < 1: # reject short single-token names (P/NP/SAT)
|
||
continue
|
||
if re.search(r'(?<!\w)' + re.escape(ptitle) + r'(?!\w)', frag_norm):
|
||
hits.append(nr)
|
||
return hits[0] if len(hits) == 1 else None
|
||
|
||
|
||
# Two provably-noise, parent-less fragment classes safe to hard-drop (precision ~≥0.95): a bare
|
||
# label reference with an empty/thin residual ("Remark 7.28", "Satz D*", "Lemma 3.2") and a single-
|
||
# variable notation assignment ("r = n + m"). Matched on the normalized title HEAD (before the em-dash).
|
||
_PARENTLESS_NOISE = re.compile(
|
||
r'^\s*(?:remark|bemerkung|anmerkung|note|satz|lemma|korollar|proposition|folgerung)\s*[\d.]*\s*[a-z]?\*?\s*$'
|
||
# single-var arithmetic assignment ("r = n + m") — NOT a tuple/set/language/cardinality definition
|
||
# ("T = (Q,…)", "L = {…}", "n = |V|"): the RHS must not open with a bracket/pipe.
|
||
r'|^\s*[a-z](?:_?[a-z0-9])?\s*:?=\s*(?![({\[|])\S', re.I)
|
||
# KEEP-guards: never drop a named theorem WITH an author, a complexity-class (in)equality, or a Definition.
|
||
_PARENTLESS_KEEP = re.compile(
|
||
r'\b(?:cook|levin|karp|savitch|ladner|immerman|rice|håstad|hastad|christofides|dijkstra|bellman|'
|
||
r'ford|kruskal|prim|edmonds|blum|sipser|papadimitriou)\b'
|
||
r'|\b(?:p|np|conp|nl|pspace|exp|nexp|bpp|rp|zpp)\b|⇔|⇒|\bdefinition\b', re.I)
|
||
|
||
|
||
def _is_parentless_noise(title: str) -> bool:
|
||
"""True for the narrow, parent-less noise classes safe to hard-drop (subject to KEEP-guards)."""
|
||
norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip()
|
||
head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0]
|
||
return bool(_PARENTLESS_NOISE.search(head)) and not _PARENTLESS_KEEP.search(norm)
|
||
|
||
|
||
# F1 — statement-gate keep-guard (Knowledge-Component / OMDoc theory): a self-contained ASSERTION is its
|
||
# own learning unit, not a whole-part fragment. Protect two general classes from demotion:
|
||
# (a) a named reduction between two PROBLEMS ("3-SAT ≤ Clique", "Clique → Vertex Cover"), and
|
||
# (b) a LABELED or ATTRIBUTED theorem carrying its own biconditional/implication ("Satz 6.37: … ⇔ …").
|
||
# NOT protected: a bare label ("Satz 7.18", "Remark 7.28"), unary status ("X ist NP-vollständig", "X ∈ NP"),
|
||
# a proof-size step ("Strikte Reduktion |A| = O(m)"), or a güte/bound facet — those carry neither a
|
||
# two-sided reduction operator NOR a ⇔/⇒ assertion. General: attribution is structural, no author whitelist.
|
||
_STMT_LABEL = re.compile(r'^\s*(?:satz|lemma|korollar|theorem|proposition|prop|folgerung)\b', re.I)
|
||
_STMT_ATTRIB = re.compile(r'\b(?:satz|lemma|theorem|korollar)\s+von\s+[A-ZÄÖÜ]') # "Satz von Cook/Levin"
|
||
_STMT_ASSERT = re.compile(r'⇔|⇒|⟺|⟹|\bgdw\.?\b|\bgenau dann\b', re.I)
|
||
_REDUCTION_ONLY_OP = re.compile(r'[≤⪯]|→|⇒|⟹|->') # genuine reduction operators (NOT plain "=")
|
||
|
||
|
||
def _is_reduction_statement(title: str) -> bool:
|
||
"""True if the title is a reduction between two NAMED problems (both sides carry ≥3 letters and
|
||
neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP", proof-size "|A| = O(m)", and a
|
||
construction/assignment suffix after the target ("3-SAT ≤ K-COLOR: G=(V,E) Konstruktion" is a
|
||
construction fragment, not a standalone statement — a bare target carries no ':' or '=')."""
|
||
t = _REL_STRIP.sub('', title, count=1)
|
||
m = _REDUCTION_ONLY_OP.search(t)
|
||
if not m:
|
||
return False
|
||
right = t[m.end():]
|
||
if ':' in right or '=' in right: # construction/assignment detail after the target → fragment
|
||
return False
|
||
|
||
def _named(s: str) -> bool:
|
||
return len(re.findall(r'[a-zäöüß]', s, re.I)) >= 3 and not re.match(r'\s*[Oo]\s*\(', s)
|
||
|
||
return _named(t[:m.start()]) and _named(right)
|
||
|
||
|
||
def _is_named_statement(title: str, desc: str = "") -> bool:
|
||
"""Statement-gate keep-guard: a named reduction (a) or a labeled/attributed theorem WITH its own
|
||
⇔/⇒ assertion (b). Bare labels / unary status / proof-size steps return False (stay demotable)."""
|
||
if _is_reduction_statement(title):
|
||
return True
|
||
if (_STMT_LABEL.match(title) or _STMT_ATTRIB.search(title)) and _STMT_ASSERT.search(f"{title} {desc or ''}"):
|
||
return True
|
||
return False
|
||
|
||
|
||
|
||
|
||
def _umbrella_schema(data, ids: set[int]):
|
||
"""{"umbrellas":[{"title":str,"description":str,"members":[int,…]}, …]}
|
||
→ [(title, description, [member ids])]. [] = valid (no umbrella); None ONLY on broken JSON
|
||
(so resume treats a valid-but-empty file as done, like _filter_schema's {} vs None). Each id
|
||
used at most once across all umbrellas (first wins); members filtered to `ids`; an umbrella
|
||
needs ≥2 surviving members; `description` required + non-empty — it MUST enumerate the children,
|
||
else the source-scoped subblock step can't re-derive them (no web on uni)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("umbrellas"), list):
|
||
return None
|
||
out, used = [], set()
|
||
for u in data["umbrellas"]:
|
||
if not isinstance(u, dict):
|
||
continue
|
||
title = str(u.get("title", "")).strip()
|
||
desc = str(u.get("description", "")).strip()
|
||
raw_members = u.get("members")
|
||
if not title or not desc or not isinstance(raw_members, list):
|
||
continue
|
||
members = []
|
||
for x in raw_members:
|
||
try:
|
||
m = int(x)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if m in ids and m not in used:
|
||
used.add(m)
|
||
members.append(m)
|
||
if len(members) >= 2:
|
||
out.append((title, desc, members))
|
||
return out
|
||
|
||
|
||
def _completion_schema(data, n_umbrellas: int, ids: set[int]):
|
||
"""{"additions":[{"umbrella":int,"members":[int,…]}, …]} → [(umbrella_idx, [member ids])].
|
||
[] = valid (nothing to absorb); None only on broken JSON. umbrella idx in range; members drawn
|
||
from `ids` (the still-standalone leftovers), de-duped within an addition."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("additions"), list):
|
||
return None
|
||
out = []
|
||
for a in data["additions"]:
|
||
if not isinstance(a, dict):
|
||
continue
|
||
try:
|
||
k = int(a.get("umbrella"))
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if not (0 <= k < n_umbrellas):
|
||
continue
|
||
mem = []
|
||
for x in (a.get("members") or []):
|
||
try:
|
||
m = int(x)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if m in ids and m not in mem:
|
||
mem.append(m)
|
||
if mem:
|
||
out.append((k, mem))
|
||
return out
|
||
|
||
|
||
# Deterministic backstop to the grouping judge's TEST 1 (type gate): an umbrella may bundle ONLY
|
||
# constituent sub-definitions of ONE definition. If a member title carries a standalone-unit signal
|
||
# (a named algorithm / problem / reduction / theorem), the umbrella is dissolved — those stay their own
|
||
# blocks. Kept narrow so real definition-parts (Konfiguration, Übergangsfunktion δ, Literale, Makespan,
|
||
# m Maschinen) never match; checked against the aak over-merge (member „Greedy-Algorithmus GA" hits).
|
||
# Suffix-anchored head nouns (German compounds are head-final: „Approximations+algorithmus" has NO word
|
||
# boundary before „algorithmus", so \bAlgorithmus\b misses it → the MAX-SAT over-merge). \w* absorbs the
|
||
# modifier; the head noun stays the discriminator. FP-safe: no real TM/KNF definition-part ends in these
|
||
# heads (Berechnung is deliberately NOT a head → „Akzeptierende Berechnung" stays a valid member).
|
||
_GROUP_STANDALONE = re.compile(
|
||
r'\w*algorithm(?:us|en)\b|\w*problem(?:e|s|en)?\b|\w*reduktion(?:en)?\b|\bscheduling\b|[≤⪯]'
|
||
r'|^\s*(?:Satz|Lemma|Korollar|Theorem|Bemerkung|Beobachtung)\s*\d'
|
||
# atomicity: a named COMPLEXITY CLASS / a "…-Vollständigkeit(completeness)" / a "…Transformation" is a
|
||
# self-contained concept (learning-object / atomic-KC), never a sub-definition — a bundle of ≥1 such
|
||
# member is siblings, not one model → dissolve (catches the P/NP/NP-Vollständigkeit over-merge that NO
|
||
# cosine floor separates). Head-final compounds (\w*klasse absorbs "Komplexitäts+klasse"); FP-safe —
|
||
# no real TM/KNF/TSP/Scheduling definition-part carries these heads.
|
||
r'|\w*vollständigkeit\b|\w*completeness\b|\w*transformation(?:en)?\b|\w*klasse[nr]?\b', re.I)
|
||
|
||
|
||
|
||
|
||
# --- Outline (blocks artifact: chapter structure, only read by the guide) ---
|
||
|
||
def _outline_review_schema(data, valid: set[int], n_chapters: int, n_blocks: int):
|
||
"""{"moves": {"<blocknr>": <chapter-idx>}} → {nr: idx} (may be {}) · None if broken/invalid.
|
||
A mass rewrite (more than a third of all blocks) is rejected — the reviewer's job is
|
||
spotting misplacements, not re-designing the outline."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("moves"), dict):
|
||
return None
|
||
out: dict[int, int] = {}
|
||
for k, v in data["moves"].items():
|
||
try:
|
||
nr, ch = int(k), int(v)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
if nr not in valid or not (1 <= ch <= n_chapters):
|
||
return None
|
||
out[nr] = ch
|
||
if len(out) * 3 > n_blocks:
|
||
return None
|
||
return out
|
||
|
||
|
||
def _outline_schema(data, valid: set[int]):
|
||
"""{"chapters":[{title,numbers}]} → cleaned (valid numbers, each exactly once) ·
|
||
None at <80 % coverage (agent/judge omitted too much)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("chapters"), list):
|
||
return None
|
||
out, seen = [], set()
|
||
for ch in data["chapters"]:
|
||
if not isinstance(ch, dict):
|
||
continue
|
||
title = str(ch.get("title", "")).strip() or "Chapter"
|
||
nums = []
|
||
for n in (ch.get("numbers") or []):
|
||
try:
|
||
n = int(n)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if n in valid and n not in seen:
|
||
seen.add(n)
|
||
nums.append(n)
|
||
if nums:
|
||
out.append({"title": title, "numbers": nums})
|
||
if not out or len(seen) < 0.8 * len(valid):
|
||
return None
|
||
return {"chapters": out}
|
||
|
||
|
||
def _prereq_schema(data, valid: set[int]) -> dict[int, list[int]]:
|
||
"""{"prereqs": {"3": [1, 7]}} → {num: [prereq nums]} · only numbers from `valid`, no self-edge.
|
||
Invalid/empty → {} (best-effort: then original order)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("prereqs"), dict):
|
||
return {}
|
||
out: dict[int, list[int]] = {}
|
||
for k, v in data["prereqs"].items():
|
||
try:
|
||
num = int(k)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if num not in valid or not isinstance(v, list):
|
||
continue
|
||
pres = []
|
||
for p in v:
|
||
try:
|
||
p = int(p)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if p in valid and p != num and p not in pres:
|
||
pres.append(p)
|
||
if pres:
|
||
out[num] = pres
|
||
return out
|
||
|
||
|
||
def _topo_order(nums: list[int], edges: dict[int, list[int]]) -> list[int]:
|
||
"""Kahn topo sort: prerequisites first. `edges[num]` = numbers that must come BEFORE num.
|
||
Stable tie-break (original order of `nums`); cycles are broken (never deadlock)."""
|
||
pos = {n: i for i, n in enumerate(nums)}
|
||
# remaining in-degree over valid nodes only; self/foreign edges ignored.
|
||
pre = {n: [p for p in edges.get(n, []) if p in pos and p != n] for n in nums}
|
||
done: list[int] = []
|
||
finished: set[int] = set()
|
||
rest = list(nums)
|
||
while rest:
|
||
ready_nodes = [n for n in rest if all(p in finished for p in pre[n])]
|
||
if not ready_nodes: # cycle → force the earliest remaining node in original order
|
||
ready_nodes = [min(rest, key=lambda n: pos[n])]
|
||
nxt = min(ready_nodes, key=lambda n: pos[n]) # stable: smallest original position first
|
||
done.append(nxt)
|
||
finished.add(nxt)
|
||
rest.remove(nxt)
|
||
return done
|
||
|
||
|
||
async def _learning_order(ctx: GenContext, set_p, files: dict, entries: dict, valid: set[int], instructions: str) -> dict:
|
||
"""Put entries (num→title) into learning order: the LLM extracts prereq edges from the
|
||
extracted `prerequisites`, code solves via topo sort. Best-effort → otherwise entries unchanged."""
|
||
if len(entries) < 3:
|
||
return entries
|
||
topic = ctx.topic
|
||
facts_map = _json_file(files["facts"])
|
||
facts_map = facts_map if isinstance(facts_map, dict) else {}
|
||
|
||
def _hint(title):
|
||
fm = facts_map.get(title) or {}
|
||
vs = [v for fk in fm.values() if isinstance(fk, dict) and (v := str(fk.get("prerequisites", "")).strip())]
|
||
return " · ".join(dict.fromkeys(vs))
|
||
|
||
pp = files["arbeit"] / "outline-prereqs.json"
|
||
|
||
def _payload(result, p=pp):
|
||
d = _sink_or_file(result, p, lambda x: x if isinstance(x, dict) and "prereqs" in x else None)
|
||
return d
|
||
|
||
existing = _json_file(pp)
|
||
if not (isinstance(existing, dict) and "prereqs" in existing):
|
||
lines = [f"{n}. {t}" + (f"\n braucht vorher: {h}" if (h := _hint(t)) else "") for n, t in entries.items()]
|
||
set_p("Outline — learning order…", step=_step_idx(topic, "Outline"))
|
||
await run_single_slot(
|
||
ctx, "Outline-Prerequisites", key=f"blocks-{topic}-outline-prereqs",
|
||
prompt=_prompt("Outline-Prerequisites", topic=topic, blocks="\n".join(lines), out_path=pp, extra=_extra(instructions)),
|
||
role="guide", capabilities="files", payload=_payload, timeout=_timeout("plan", len(entries)))
|
||
edges = _prereq_schema(_json_file(pp), valid)
|
||
if not edges:
|
||
return entries # no/invalid edges → original order (no regression)
|
||
ordered = _topo_order(list(entries), edges)
|
||
return {n: entries[n] for n in ordered}
|
||
|
||
|
||
async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict:
|
||
"""Format-agnostic outline over ALL blocks — 3 proposals → judge merges.
|
||
Never aborts: 0 valid → one chapter with everything; missing blocks land in "Other".
|
||
→ {"chapters":[{title,numbers}]} (also in files["outline"])."""
|
||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||
valid = set(entries)
|
||
step = _step_idx(topic, "Outline")
|
||
|
||
# Establish learning order (LLM-modulo): the LLM extracts prereq edges from the extracted
|
||
# `prerequisites`, code solves via topo sort. Best-effort → otherwise original order.
|
||
entries = await _learning_order(ctx, set_p, files, entries, valid, instructions)
|
||
liste = "\n".join(f"{n}. {t}" for n, t in entries.items())
|
||
set_p("Outline — proposals…", step=step)
|
||
|
||
async def _proposal(i, path):
|
||
if _outline_schema(_json_file(path), valid):
|
||
return True
|
||
await run_single_slot(
|
||
ctx, f"Outline {i}", key=f"blocks-{topic}-outline-{i}",
|
||
prompt=_prompt("Guide-Outline", topic=topic, blocks=liste, out_path=path, extra=_extra(instructions)),
|
||
role="guide", capabilities="files",
|
||
payload=lambda result, p=path: _sink_or_file(result, p, lambda d: _outline_schema(d, valid)),
|
||
timeout=_timeout("plan", len(entries)))
|
||
return _outline_schema(_json_file(path), valid) is not None
|
||
|
||
slots = files["outline_slots"]
|
||
await _gather_progress([_proposal(i, p) for i, p in enumerate(slots, 1)], len(slots), _report_p(set_p, topic, "Outline"))
|
||
if is_cancelled():
|
||
return {}
|
||
proposals = [v for p in slots if (v := _outline_schema(_json_file(p), valid))]
|
||
|
||
if not proposals:
|
||
plan = {"chapters": [{"title": "Contents", "numbers": list(entries)}]}
|
||
elif len(proposals) == 1:
|
||
plan = proposals[0]
|
||
else:
|
||
set_p("Outline merging…", step=step)
|
||
block_texts = "\n\n".join(
|
||
f"### Vorschlag {i}\n" + "\n".join(
|
||
f"KAPITEL: {ch['title']}\n Nummern: {', '.join(str(n) for n in ch['numbers'])}" for ch in v["chapters"])
|
||
for i, v in enumerate(proposals, 1))
|
||
await run_single_slot(
|
||
ctx, "Outline-Judge", key=f"blocks-{topic}-outline-judge",
|
||
prompt=_prompt("Guide-Outline-Judge", topic=topic, format_name="den Guide",
|
||
purpose="alle Blocks in einem roten Faden", n=len(proposals),
|
||
blocks=liste, outlines=block_texts, out_path=files["outline"], extra=_extra(instructions)),
|
||
role="judge", capabilities="files",
|
||
payload=lambda result: _sink_or_file(result, files["outline"],
|
||
lambda d: _outline_schema(d, valid)),
|
||
timeout=_timeout("plan_judge", len(entries)))
|
||
plan = _outline_schema(_json_file(files["outline"]), valid) or proposals[0]
|
||
|
||
# Placement review (best-effort): ONE judge checks every block→chapter assignment and
|
||
# reports ONLY misplacements as moves. Invalid/mass output → plan unchanged.
|
||
if proposals and len(plan["chapters"]) >= 2 and not is_cancelled():
|
||
rp = files["arbeit"] / "outline-review.json"
|
||
moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
|
||
if moves is None:
|
||
chapter_text = "\n\n".join(
|
||
f"KAPITEL {k}: {ch['title']}\n" + "\n".join(f" {n}. {_title(entries[n])}" for n in ch["numbers"])
|
||
for k, ch in enumerate(plan["chapters"], 1))
|
||
set_p("Outline review…", step=step)
|
||
await run_single_slot(
|
||
ctx, "Outline-Review", key=f"blocks-{topic}-outline-review",
|
||
prompt=_prompt("Guide-Outline-Review", topic=topic, chapters=chapter_text,
|
||
out_path=rp, extra=_extra(instructions)),
|
||
role="judge", capabilities="files",
|
||
payload=lambda result: _sink_or_file(result, rp, lambda d: _outline_review_schema(
|
||
d, valid, len(plan["chapters"]), len(entries))),
|
||
timeout=_timeout("plan_judge", len(entries)))
|
||
moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
|
||
for nr, target in (moves or {}).items():
|
||
for ch in plan["chapters"]:
|
||
if nr in ch["numbers"]:
|
||
ch["numbers"].remove(nr)
|
||
plan["chapters"][target - 1]["numbers"].append(nr)
|
||
if moves:
|
||
plan["chapters"] = [ch for ch in plan["chapters"] if ch["numbers"]]
|
||
_log(topic, f"Outline-Review: {len(moves)} Block/Blöcke umsortiert")
|
||
|
||
# Completeness: every block appears — missing in "Other" (against omitting agents/judge).
|
||
included = {n for ch in plan["chapters"] for n in ch["numbers"]}
|
||
missing = [n for n in entries if n not in included]
|
||
if missing:
|
||
plan["chapters"].append({"title": "Other", "numbers": missing})
|
||
atomic_write_json(files["outline"], plan, indent=1)
|
||
return plan
|
||
|
||
|
||
# --- Learning artefacts (flashcards/examples from the facts) ---
|
||
|
||
async def _mirror_sidecar_db(topic: str, sidecar: dict) -> None:
|
||
"""Mirror the sidecar {block title: [{title, level, relevance}]} into the DB table subblocks."""
|
||
for btitle, subs in sidecar.items():
|
||
bnorm = _norm_title(btitle)
|
||
if not bnorm or not isinstance(subs, list):
|
||
continue
|
||
for s in subs:
|
||
if not isinstance(s, dict):
|
||
continue
|
||
st = str(s.get("title", "")).strip()
|
||
sn = _norm_title(st)
|
||
if not sn:
|
||
continue
|
||
facts = json.dumps(s["facts"], ensure_ascii=False) if isinstance(s.get("facts"), dict) else None
|
||
await db.put_subblock(topic, bnorm, sn, btitle, st,
|
||
level=s.get("level"), relevance=s.get("relevance"),
|
||
facts=facts, status="consensus")
|
||
|
||
|
||
|
||
|
||
async def _guide_ebene(topic: str, instructions: str, provider: str, is_cancelled, set_p) -> None:
|
||
"""Guide-Ebene der Auto-Sequenz: generieren → QA → „Befunde beheben"-Loop bis
|
||
100 %/Stillstand/10×. Fail-open: Fehler überspringen den Guide, blockieren nichts."""
|
||
import uuid
|
||
from datetime import datetime, timezone
|
||
try:
|
||
import guide, guide_qa, guide_board
|
||
from auto_loop import auto_repair_loop
|
||
fmt = "Guide"
|
||
now = datetime.now(timezone.utc).isoformat()
|
||
guide_id = str(uuid.uuid4())
|
||
await db.create_guide({"id": guide_id, "topic": topic, "format": fmt,
|
||
"instructions": instructions, "status": "queued",
|
||
"progress": None, "created_at": now, "updated_at": now})
|
||
set_p("Guide wird generiert…")
|
||
await guide.generate_guide(guide_id, topic, fmt, instructions, provider)
|
||
if is_cancelled():
|
||
return
|
||
|
||
async def _messen():
|
||
rep = await guide_qa.guide_qa_report(topic, llm=True)
|
||
return float(rep["note_guide"]) if rep else 10.0
|
||
|
||
erst = await _messen()
|
||
|
||
async def _reparieren():
|
||
set_p("Befunde beheben (Guide)…")
|
||
await guide_board.repair_karten(topic, fmt) # befundtragende Karten → pruefer
|
||
await guide.generate_guide(topic=topic, format_name=fmt, guide_id=guide_id,
|
||
instructions=instructions, provider=provider) # resumt offene Karten
|
||
return await _messen()
|
||
|
||
res = await auto_repair_loop("Guide", erst, _reparieren)
|
||
if res["grund"] != "fertig":
|
||
msg = (f"Guide bleibt bei {round(res['note'] * 10)} % ({res['grund']}, "
|
||
f"{res['runden']} Runden) — «Befunde beheben» hat keinen Pfad (Systemfehler)")
|
||
_blocks_errors[topic] = msg
|
||
set_p(msg)
|
||
else:
|
||
set_p(f"Guide: 100 % nach {res['runden']} Runde(n)")
|
||
except Exception as e:
|
||
log.exception("[%s] Guide-Ebene fehlgeschlagen", topic)
|
||
_blocks_errors[topic] = f"Guide-Ebene: {str(e)[:500]}"
|
||
|
||
|
||
async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER,
|
||
research: bool = True, qa_force: bool = False,
|
||
auto_inventory: bool = True, auto_artefacts: bool = True,
|
||
auto_guide: bool = True, only_artefacts: bool = False) -> None:
|
||
"""Kanban entry point: source prep, then both boards (inventory + artefacts) until
|
||
quiescence. research=False = Continue (drain the existing queue, no new search).
|
||
A run on a finished topic ADDS research (live extension) — full rebuild = DELETE /blocks.
|
||
auto_* per level: after each level's QA, loop „Befunde beheben" until 100 % / stall / 10×;
|
||
if off, stop after that level's QA (control point)."""
|
||
if topic in _blocks_progress:
|
||
return
|
||
_blocks_progress[topic] = "Warten…"
|
||
_blocks_errors.pop(topic, None)
|
||
|
||
files = _blocks_files(topic)
|
||
q = load_source(topic)
|
||
folder = source_folder(topic) # projekt/uni/link → folder, thema → None
|
||
instructions = q.get("spec") or instructions # prefer the persisted specification (also on resume)
|
||
|
||
def set_p(msg: str, step: int | None = None) -> None:
|
||
_blocks_progress[topic] = msg
|
||
if step is not None:
|
||
_blocks_step[topic] = step
|
||
|
||
def is_cancelled() -> bool:
|
||
return topic in _blocks_cancelled
|
||
|
||
ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled)
|
||
try:
|
||
async with _semaphore:
|
||
files["arbeit"].mkdir(parents=True, exist_ok=True)
|
||
# Step "Source prep": crawl (link) + PDFs + content/noise triage.
|
||
if not await _prepare_source(ctx, set_p, files, q, folder, instructions):
|
||
if is_cancelled():
|
||
_blocks_errors[topic] = "Cancelled — progress is preserved"
|
||
return
|
||
import board_inventory # lazy: the boards import blocks
|
||
status: dict = {}
|
||
ok = await board_inventory.run_boards(ctx, set_p, files, q, folder, instructions,
|
||
research=research, qa_force=qa_force,
|
||
auto_inventory=auto_inventory, auto_artefacts=auto_artefacts,
|
||
inventory=not only_artefacts, status_out=status)
|
||
if not ok and is_cancelled():
|
||
_blocks_errors[topic] = "Cancelled — progress is preserved"
|
||
elif ok and auto_guide and status.get("weiter") and not is_cancelled():
|
||
# Auto-Sequenz: Inventar + Artefakte liefen sauber durch → Guide-Ebene.
|
||
await _guide_ebene(topic, instructions, provider, is_cancelled, set_p)
|
||
except Exception as e:
|
||
log.exception("[%s] Blocks generation failed", topic)
|
||
_blocks_errors[topic] = str(e)[:2000]
|
||
finally:
|
||
# No file cleanup: intermediate files stay for resume / traceability.
|
||
_blocks_progress.pop(topic, None)
|
||
_blocks_step.pop(topic, None)
|
||
_blocks_cancelled.discard(topic)
|
||
clear_scope(f"blocks-{topic}-") # clear the scope → restart isn't blocked
|