3465 lines
168 KiB
Python
3465 lines
168 KiB
Python
"""Blocks pipeline: research consensus + clarification loop — pure inventory, unsorted.
|
||
|
||
5x research (min. 3, grace) → mapping (consensus/rest) → clarification loop (max.
|
||
CONSENSUS_MAX_ROUNDS rounds): 3 selection agents (min. 2, grace) decide
|
||
on the disputed rest, a mapping agent sorts into accept/discard/
|
||
still disputed. An empty rest ends the loop; the last round must decide
|
||
everything. Races use a grace window instead of "first N win": after the
|
||
first valid result, the remaining agents get CONSENSUS_GRACE seconds to
|
||
finish. The consensus is accumulated in code — no agent re-emits
|
||
the full list.
|
||
"""
|
||
|
||
import asyncio
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import math
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import time
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
import database as db
|
||
import embedding
|
||
from agents import kill_process, cancel_scope, clear_scope, run_agent
|
||
from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP, GROUP_RECONCILE_FLOOR, GROUP_MIN_COS_FLOOR, SUB_VARIANT_COS, SEED_COVER_COS, EVIDENCE_BUDGET_CHARS, EVIDENCE_CTX_LINES
|
||
from fsutil import atomic_write_text, atomic_write_json
|
||
from jsonio import parse_json_text, read_json_file as _json_file
|
||
from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder
|
||
from crawl import crawl
|
||
from pipeline import (
|
||
CANCELLED, FAILED, OK, GenContext, _detached, _extra, _gather_progress, _yesno_schema, _log, _prompt, _race,
|
||
_relevance_schema, _runde_schema, _semaphore, _str_list, _levels_schema, _timeout, run_single_slot,
|
||
)
|
||
from textkit import (
|
||
_unique_title, _load_blocks, _norm_title, _parse_selection, _parse_subblocks, _title,
|
||
_resolve_title, _title_index, clean_title,
|
||
)
|
||
|
||
# Pipeline-Tuning-Konstanten liegen zentral in config.py (tunebar via CREATOR_PARAMS).
|
||
from config import ( # noqa: E402
|
||
ARTEFACT_CHUNK_SUBS, CONSOLIDATION_CHUNK, CONSOLIDATION_PANEL, DEDUP_GLOBAL_FLOOR,
|
||
DEDUP_PAIR_FLOOR, DEDUP_PAIRS_CHUNK, DEDUP_TITLE_AUTO, FACTS_CHECK_PANEL, FACTS_CHUNK_SUBS,
|
||
FILTER_CHUNK, FILTER_RECHECK_PANEL, LEVEL_CHUNK, QUESTION_CHUNK_SUBS, QUESTION_MAX_ROUNDS,
|
||
RESEARCH_BATCH, RESEARCH_READERS, RESEARCH_SECTION_CHARS, RESEARCH_THEMA_AGENTS,
|
||
SUBBLOCK_CAP, SUBBLOCK_CHUNK, SUBBLOCK_EXTRA_ROUNDS, SUBBLOCK_MAX, SUBBLOCK_MAX_ROUNDS,
|
||
SUBBLOCK_MIN, SUBBLOCK_PANEL)
|
||
|
||
log = logging.getLogger("creator.blocks")
|
||
|
||
_blocks_progress: dict[str, str] = {}
|
||
_blocks_errors: dict[str, str] = {}
|
||
_blocks_cancelled: set[str] = set()
|
||
_blocks_step: dict[str, int] = {}
|
||
|
||
ARTEFACT_TYPES = ("flashcard", "example")
|
||
|
||
|
||
def load_source(topic: str) -> dict:
|
||
"""Read the persisted source choice. Fallback (legacy topics without source.json):
|
||
if projects/<topic> exists → projekt, otherwise thema."""
|
||
q = _json_file(source_path(topic))
|
||
if isinstance(q, dict) and q.get("type") in ("thema", "projekt", "uni", "link"):
|
||
return q
|
||
if project_dir(topic).is_dir():
|
||
return {"type": "projekt", "location": f"projects/{topic}", "spec": ""}
|
||
return {"type": "thema", "location": "", "spec": ""}
|
||
|
||
|
||
def source_folder(topic: str) -> Path | None:
|
||
"""Folder source (projekt/uni → path, link → crawl folder) — otherwise None (thema)."""
|
||
q = load_source(topic)
|
||
if q["type"] == "link":
|
||
return source_crawl_dir(topic)
|
||
if q["type"] in ("projekt", "uni"):
|
||
return safe_folder(q.get("location", ""))
|
||
return None
|
||
|
||
|
||
def _crawl_done(topic: str) -> bool:
|
||
return (source_crawl_dir(topic) / ".done").exists() # marker only on clean completion
|
||
|
||
|
||
# Learning-path levels (beginner/advanced/expert); old difficulty values are backward-compatible.
|
||
_LEVELS = ("beginner", "advanced", "expert", "easy", "medium", "hard")
|
||
|
||
|
||
async def subblocks_title(topic: str, block: str) -> list[str]:
|
||
"""Subblock titles of a block — DB-first (consensus), fallback to the sidecar file."""
|
||
rows = [s["sub_title"] for s in await db.list_subblocks(topic, _norm_title(block))
|
||
if s["status"] == "consensus" and s["sub_title"]]
|
||
if rows:
|
||
return rows
|
||
sc = _json_file(subblocks_path(topic))
|
||
if not isinstance(sc, dict):
|
||
return []
|
||
return [
|
||
t for s in (sc.get(block) or [])
|
||
if isinstance(s, dict) and (t := str(s.get("title", "")).strip())
|
||
]
|
||
|
||
|
||
async def load_question_pattern(topic: str, block: str) -> list[dict]:
|
||
"""Predefined question patterns of a block — DB-first, fallback to sidecar (empty = live)."""
|
||
rows = await db.list_question_pattern(topic, _norm_title(block))
|
||
if rows:
|
||
return [{"subblock": r["sub_title"], "question": r["question"]} for r in rows if r["question"]]
|
||
fm = _json_file(question_pattern_path(topic))
|
||
if not isinstance(fm, dict):
|
||
return []
|
||
return [
|
||
{"subblock": str(e.get("subblock", "")).strip(), "question": question}
|
||
for e in (fm.get(block) or [])
|
||
if isinstance(e, dict) and (question := str(e.get("question", "")).strip())
|
||
]
|
||
|
||
|
||
async def subblocks_frei(topic: str, block: str, max_level: int) -> list[str]:
|
||
"""Subblock titles up to the unlocked level (≤ max_level). Fallback without
|
||
level knowledge (legacy/sidecar): all subblock titles."""
|
||
rows = await db.subs_with_level(topic, block)
|
||
if not rows:
|
||
return await subblocks_title(topic, block)
|
||
return [s["title"] for s in rows if s["level"] <= max_level and s["title"]]
|
||
|
||
|
||
async def load_question_pattern_free(topic: str, block: str, max_level: int) -> list[dict]:
|
||
"""Question patterns, filtered to subblocks up to the unlocked level. Without
|
||
level knowledge (legacy/sidecar), unfiltered."""
|
||
rows = await db.subs_with_level(topic, block)
|
||
if not rows:
|
||
return await load_question_pattern(topic, block)
|
||
unlocked = {s["norm"] for s in rows if s["level"] <= max_level}
|
||
return [m for m in await load_question_pattern(topic, block) if _norm_title(m["subblock"]) in unlocked]
|
||
|
||
|
||
async def load_overview(topic: str) -> list[dict]:
|
||
"""Structured block list for the overview — DB-first (consensus + subs/levels/relevance),
|
||
fallback to blocks.md + sidecar (legacy topics)."""
|
||
bs = await db.list_blocks(topic, status="consensus")
|
||
if bs:
|
||
out = []
|
||
for num, b in enumerate(bs, 1):
|
||
subs = [s for s in await db.list_subblocks(topic, b["title_norm"]) if s["status"] == "consensus"]
|
||
out.append({
|
||
"num": num, "title": b["title"], "description": b["description"],
|
||
"subblocks": [
|
||
{"title": s["sub_title"],
|
||
"level": s["level"] if s["level"] in _LEVELS else "advanced",
|
||
"relevance": s["relevance"] if s["relevance"] in ("relevant", "peripheral") else None}
|
||
for s in subs if s["sub_title"]
|
||
],
|
||
})
|
||
return out
|
||
entries = _load_blocks(_read(blocks_path(topic)))
|
||
sidecar = _json_file(subblocks_path(topic))
|
||
sidecar = sidecar if isinstance(sidecar, dict) else {}
|
||
out = []
|
||
for num, entry in entries.items():
|
||
title = _title(entry)
|
||
split_parts = entry.split(" — ", 1)
|
||
description = split_parts[1].strip() if len(split_parts) == 2 else ""
|
||
subblocks = [
|
||
{
|
||
"title": t,
|
||
"level": s.get("level") if s.get("level") in _LEVELS else "advanced",
|
||
"relevance": s.get("relevance") if s.get("relevance") in ("relevant", "peripheral") else None,
|
||
}
|
||
for s in (sidecar.get(title) or [])
|
||
if isinstance(s, dict) and (t := str(s.get("title", "")).strip())
|
||
]
|
||
out.append({"num": num, "title": title, "description": description, "subblocks": subblocks})
|
||
return out
|
||
|
||
|
||
def _blocks_steps(topic: str) -> tuple:
|
||
"""Steps per source: link gets "Source laden" up front, projekt additionally "Supplement".
|
||
|
||
Subblocks + levels are three phases each (find, select, clarify). Per phase
|
||
all packages run in parallel; the step remains until the last package is done.
|
||
"""
|
||
q = load_source(topic)
|
||
base = ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung")
|
||
rest = (
|
||
"Subblocks find", "Subblocks select", "Subblocks clarify",
|
||
"Facts find", "Facts check", "Facts fix",
|
||
"Levels find", "Levels select", "Levels clarify",
|
||
"Relevance find", "Relevance select", "Relevance clarify",
|
||
"Outline",
|
||
"Questions find", "Questions select", "Questions clarify", "Questions check",
|
||
"Flashcards", "Examples",
|
||
)
|
||
middle = base + (("Supplement",) if q["type"] == "projekt" else ()) + rest
|
||
return (("Source prep",) if q["type"] == "link" else ()) + middle
|
||
|
||
|
||
def _step_idx(topic: str, name: str) -> int:
|
||
return _blocks_steps(topic).index(name)
|
||
|
||
|
||
def _report_p(set_p, topic: str, step: str):
|
||
"""Async report callback for _gather_progress: sets "<step> d/t…" + step index."""
|
||
idx = _step_idx(topic, step)
|
||
async def report(d, t):
|
||
set_p(f"{step} {d}/{t}…", step=idx)
|
||
return report
|
||
|
||
|
||
# Coarse display phases: bundle the fine steps (internally everything stays fine-grained).
|
||
# Special steps (Source laden, Supplement) belong to the "Inventory" phase.
|
||
PHASEN = (
|
||
("Source", ("Source prep",)),
|
||
("Inventory", ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung", "Supplement")),
|
||
("Subblocks", ("Subblocks find", "Subblocks select", "Subblocks clarify")),
|
||
("Facts", ("Facts find", "Facts check", "Facts fix")),
|
||
("Levels", ("Levels find", "Levels select", "Levels clarify")),
|
||
("Relevance", ("Relevance find", "Relevance select", "Relevance clarify")),
|
||
("Outline", ("Outline",)),
|
||
("Questions", ("Questions find", "Questions select", "Questions clarify", "Questions check")),
|
||
("Artefacts", ("Flashcards", "Examples")),
|
||
)
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _blocks_files(topic: str) -> dict:
|
||
work_dir = arbeit_dir(topic)
|
||
rounds = range(1, CONSENSUS_MAX_ROUNDS + 1)
|
||
return {
|
||
"final": blocks_path(topic),
|
||
"arbeit": work_dir,
|
||
"research": [work_dir / f"research-{i}.md" for i in (1, 2, 3, 4, 5)],
|
||
"research_mapping": work_dir / "research-mapping.json",
|
||
"selection": {n: [work_dir / f"selection-r{n}-{i}.json" for i in (1, 2, 3)] for n in rounds},
|
||
"mapping": {n: work_dir / f"selection-mapping-r{n}.json" for n in rounds},
|
||
"ergaenzung": work_dir / "ergaenzung.json",
|
||
"sub_roh": work_dir / "subblocks-roh.json",
|
||
"facts": work_dir / "subblocks-facts.json",
|
||
"sidecar": subblocks_path(topic),
|
||
"question_pattern": question_pattern_path(topic),
|
||
"outline": work_dir / "outline.json",
|
||
"outline_slots": [work_dir / f"outline-{i}.json" for i in (1, 2, 3)],
|
||
"artefakte": work_dir / "artefakte.json",
|
||
}
|
||
|
||
|
||
def cancel_blocks(topic: str) -> bool:
|
||
if topic not in _blocks_progress:
|
||
return False
|
||
_blocks_cancelled.add(topic)
|
||
cancel_scope(f"blocks-{topic}-") # waiting agents bail before spawning
|
||
kill_process(f"blocks-{topic}-") # kill running subprocesses
|
||
return True
|
||
|
||
|
||
|
||
|
||
|
||
|
||
async def blocks_status(topic: str) -> dict:
|
||
"""Kanban-based status: `generating` from the run registry, progress from the card
|
||
counts. `partial` = cards sit in non-terminal columns while nothing runs (continue-able)."""
|
||
generating = topic in _blocks_progress
|
||
counts = await db.kanban_stage_counts(topic)
|
||
# ready = finished inventory. The DB is the source of truth (a synced topic may lack
|
||
# blocks.md, the file is a legacy mirror) — either signal counts.
|
||
ready = blocks_path(topic).exists() or counts.get("inventory", {}).get("done_block", 0) > 0
|
||
terminal = {"clustered", "done_cluster", "grouped", "rejected", "done_block", "done_artefact", "dead"}
|
||
open_cards = sum(n for stages in counts.values()
|
||
for stage, n in stages.items() if stage not in terminal)
|
||
return {
|
||
"ready": ready,
|
||
"generating": generating,
|
||
"progress": _blocks_progress.get(topic),
|
||
"error": _blocks_errors.get(topic),
|
||
"partial": not generating and open_cards > 0,
|
||
"steps": [], # legacy phase pills — replaced by the live board
|
||
"feine_steps": [],
|
||
}
|
||
|
||
|
||
def active_blocks() -> list[dict]:
|
||
return [{"topic": t, "progress": p} for t, p in _blocks_progress.items()]
|
||
|
||
|
||
def reset_blocks(topic: str) -> None:
|
||
""""Remove": deletes the ENTIRE blocks area — crawl, triage, inventory … questions.
|
||
KEEPS only the topic config `source.json` (type/link/spec). Re-generating crawls anew.
|
||
(Crawl/triage belong to the blocks; only the config is the "topic".)"""
|
||
files = _blocks_files(topic)
|
||
files["final"].unlink(missing_ok=True)
|
||
files["sidecar"].unlink(missing_ok=True)
|
||
files["question_pattern"].unlink(missing_ok=True)
|
||
shutil.rmtree(source_crawl_dir(topic), ignore_errors=True) # crawl belongs to the blocks
|
||
shutil.rmtree(files["arbeit"], ignore_errors=True)
|
||
_blocks_errors.pop(topic, None)
|
||
# source.json intentionally stays — that is the topic config.
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _supplement_schema(data):
|
||
"""{"blocks": [{"title", "description"}]} → list (empty allowed) · otherwise None."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("blocks"), list):
|
||
return None
|
||
out = []
|
||
for b in data["blocks"]:
|
||
if not isinstance(b, dict) or not isinstance(b.get("title"), str) or not isinstance(b.get("description"), str):
|
||
return None
|
||
title, description = b["title"].strip(), b["description"].strip()
|
||
if not title:
|
||
return None
|
||
out.append((title, description))
|
||
return out
|
||
|
||
|
||
def _ocr_languages() -> str | None:
|
||
"""Installierte Tesseract-Sprachen (deu/eng), None → OCR aus."""
|
||
try:
|
||
import pymupdf
|
||
base = Path(pymupdf.get_tessdata())
|
||
except Exception:
|
||
return None
|
||
langs = [l for l in ("deu", "eng") if (base / f"{l}.traineddata").exists()]
|
||
return "+".join(langs) or None
|
||
|
||
|
||
def _pdf_markdown(pdf: Path) -> str | None:
|
||
"""pymupdf4llm → Markdown string (None if the lib is missing or it fails)."""
|
||
try:
|
||
import pymupdf4llm
|
||
except ImportError:
|
||
return None
|
||
try:
|
||
# OCR nur, wenn Tesseract-Sprachdaten wirklich vorhanden sind —
|
||
# sonst wirft der OCR-Pfad und die ganze Datei faellt auf pdftotext.
|
||
langs = _ocr_languages()
|
||
kwargs = {"use_ocr": True, "ocr_language": langs} if langs else {"use_ocr": False}
|
||
return pymupdf4llm.to_markdown(str(pdf), show_progress=False, **kwargs)
|
||
except Exception:
|
||
log.warning("pymupdf4llm failed for %s", pdf.name, exc_info=True)
|
||
return None
|
||
|
||
|
||
def _pdf_plaintext(pdf: Path) -> str | None:
|
||
"""pdftotext -layout → plain string (None if missing/fails)."""
|
||
if shutil.which("pdftotext") is None:
|
||
return None
|
||
try:
|
||
out = subprocess.run(["pdftotext", "-layout", str(pdf), "-"],
|
||
check=True, timeout=120, capture_output=True)
|
||
return out.stdout.decode("utf-8", errors="replace")
|
||
except Exception:
|
||
log.warning("pdftotext failed for %s", pdf.name, exc_info=True)
|
||
return None
|
||
|
||
|
||
# Content-fidelity guard between the two converters (topic-NEUTRAL: measures loss, not domain).
|
||
# pymupdf4llm yields structured Markdown but silently DROPS rendered display formulas and can
|
||
# splinter combining diacritics (measured on a LaTeX script: „P = {L …}" gone, „h¨aufig").
|
||
# pdftotext is structure-poor but faithful. Take the Markdown only when it preserves the bulk
|
||
# of the content; otherwise the faithful plaintext wins.
|
||
_PDF_FIDELITY_SYMBOLS = "≤≥∈∉⊆∪∧∨¬→Σδα{}="
|
||
|
||
|
||
def _pick_conversion(md: str | None, plain: str | None) -> tuple[str, str] | None:
|
||
if md is None and plain is None:
|
||
return None
|
||
if md is None:
|
||
return plain, "pdftotext"
|
||
if plain is None:
|
||
return md, "pymupdf4llm"
|
||
ok_len = len(md) >= 0.7 * len(plain)
|
||
sym_plain = sum(plain.count(c) for c in _PDF_FIDELITY_SYMBOLS)
|
||
ok_sym = sym_plain == 0 or sum(md.count(c) for c in _PDF_FIDELITY_SYMBOLS) >= 0.8 * sym_plain
|
||
ok_diakritik = md.count("\u00a8") <= plain.count("\u00a8") + 2 # standalone ¨ = splintered umlauts
|
||
if ok_len and ok_sym and ok_diakritik:
|
||
return md, "pymupdf4llm"
|
||
return plain, "pdftotext"
|
||
|
||
|
||
def _convert_pdfs(project: Path) -> None:
|
||
"""Convert PDFs in the project to .txt — agents read text instead of page images.
|
||
|
||
Called before every project generation; converts only if the .txt is missing or
|
||
older than the PDF. Both converters run; the fidelity guard picks the better result
|
||
per file. Neither available → hard error instead of an unreliable direct-read mode
|
||
(MiniMax image limit, vision cost)."""
|
||
pdfs = list(project.rglob("*.pdf"))
|
||
if not pdfs:
|
||
return
|
||
for pdf in pdfs:
|
||
txt = pdf.with_suffix(".txt")
|
||
if txt.exists() and txt.stat().st_mtime >= pdf.stat().st_mtime:
|
||
continue
|
||
picked = _pick_conversion(_pdf_markdown(pdf), _pdf_plaintext(pdf))
|
||
if picked is None:
|
||
raise RuntimeError(f"PDF conversion failed ({pdf.name}): weder pymupdf4llm noch "
|
||
"pdftotext verfügbar/erfolgreich (pip install pymupdf4llm oder poppler-utils)")
|
||
text, tool = picked
|
||
txt.write_text(text, encoding="utf-8")
|
||
_log(project.name, f"PDF konvertiert ({tool}): {pdf.name} → {txt.name}")
|
||
|
||
|
||
_SOURCE_TEMPLATE = {"projekt": "Blocks-Source-Projekt", "uni": "Blocks-Source-Uni", "link": "Blocks-Source-Link"}
|
||
|
||
|
||
def _text_sections(text: str, goal: int = RESEARCH_SECTION_CHARS) -> list[str]:
|
||
"""Split text at paragraph/line boundaries into sections of ~`ziel` chars (against lost-in-the-middle
|
||
on large documents). Small text stays ONE section. Content stays complete — only
|
||
separating whitespace is dropped."""
|
||
text = text.strip()
|
||
if len(text) <= goal:
|
||
return [text] if text else []
|
||
sections: list[str] = []
|
||
buf = ""
|
||
|
||
def flush():
|
||
nonlocal buf
|
||
if buf.strip():
|
||
sections.append(buf.strip())
|
||
buf = ""
|
||
|
||
for block in re.split(r"\n\s*\n", text): # at paragraph boundaries
|
||
block = block.strip()
|
||
if not block:
|
||
continue
|
||
if len(block) > goal: # single huge paragraph → hard-cut at lines
|
||
flush()
|
||
for line in block.split("\n"):
|
||
if buf and len(buf) + len(line) + 1 > goal:
|
||
flush()
|
||
buf += line + "\n"
|
||
flush()
|
||
elif buf and len(buf) + len(block) + 2 > goal:
|
||
flush()
|
||
buf = block
|
||
else:
|
||
buf = (buf + "\n\n" + block) if buf else block
|
||
flush()
|
||
return sections
|
||
|
||
|
||
# ── Inline evidence for judges ──────────────────────────────────────────────────────
|
||
# Judges used to re-search the corpus per session ({source} → "ls/find … read", ~10 tool
|
||
# turns each). The corpus excerpts now go INTO the prompt; the agent answers as text.
|
||
|
||
def _corpus_files(folder: Path | None, sources: list[str] | None) -> list[Path]:
|
||
"""The block's named source .txt files; fallback: every .txt in the folder."""
|
||
if folder is None or not folder.is_dir():
|
||
return []
|
||
if sources:
|
||
named = [folder / Path(s).with_suffix(".txt").name for s in sources if s]
|
||
named = [p for p in named if p.is_file()]
|
||
if named:
|
||
return named
|
||
return sorted(p for p in folder.glob("*.txt") if p.is_file())
|
||
|
||
|
||
def _q_tokens(text: str) -> set[str]:
|
||
return set(re.findall(r"\w{3,}", text.casefold()))
|
||
|
||
|
||
def _evidence_pack(folder: Path | None, sources: list[str] | None, queries: list[str],
|
||
budget: int = EVIDENCE_BUDGET_CHARS) -> str:
|
||
"""Keyword-selected corpus excerpts for a judge prompt. Sections are ranked by token
|
||
overlap with `queries` (block title + candidates); every query with any match gets its
|
||
best section (coverage guarantee), the rest of the budget takes the global top. Empty
|
||
string when there is no corpus — the caller keeps the old self-research source then."""
|
||
parts: list[tuple[str, int, str, set[str]]] = [] # (file, idx, text, tokens)
|
||
for f in _corpus_files(folder, sources):
|
||
try:
|
||
text = f.read_text(encoding="utf-8")
|
||
except OSError:
|
||
continue
|
||
for i, sec in enumerate(_text_sections(text), 1):
|
||
parts.append((f.name, i, sec, _q_tokens(sec)))
|
||
if not parts:
|
||
return ""
|
||
qtoks = [(_q_tokens(q)) for q in queries if q]
|
||
score = [sum(len(qt & p[3]) for qt in qtoks) for p in parts]
|
||
chosen: set[int] = set()
|
||
for qt in qtoks: # coverage guarantee: best section per query
|
||
best = max(range(len(parts)), key=lambda k: len(qt & parts[k][3]), default=None)
|
||
if best is not None and qt & parts[best][3]:
|
||
chosen.add(best)
|
||
used = sum(len(parts[k][2]) for k in chosen)
|
||
for k in sorted(range(len(parts)), key=lambda k: -score[k]): # top-up to budget
|
||
if k in chosen or score[k] <= 0:
|
||
continue
|
||
if used + len(parts[k][2]) > budget:
|
||
continue
|
||
chosen.add(k)
|
||
used += len(parts[k][2])
|
||
out, total = [], 0
|
||
for k in sorted(chosen): # document order for readability
|
||
fname, i, sec, _t = parts[k]
|
||
if total + len(sec) > max(budget, used): # hard cap incl. guarantee overshoot
|
||
break
|
||
out.append(f"── {fname} · Abschnitt {i} ──\n{sec}")
|
||
total += len(sec)
|
||
return "\n\n".join(out)
|
||
|
||
|
||
_CITE_POS = re.compile(r"\b(?:Z(?:eilen?)?|lines?)\.?\s*(\d+)(?:\s*[-–]\s*(\d+))?", re.I)
|
||
|
||
|
||
def _cite_ref(cite: str, files: list[Path]) -> tuple[Path, int, int] | None:
|
||
"""(file, line_lo, line_hi) from a cited_facts source string
|
||
(„Skript.txt, Übung 6.47, Z.1341-1344") — None when file or position is missing."""
|
||
c = (cite or "").casefold()
|
||
f = next((p for p in files if p.name.casefold() in c or p.stem.casefold() in c), None)
|
||
m = _CITE_POS.search(cite or "")
|
||
if f is None or m is None:
|
||
return None
|
||
lo, hi = int(m.group(1)), int(m.group(2) or m.group(1))
|
||
return (f, min(lo, hi), max(lo, hi))
|
||
|
||
|
||
def _cited_evidence(folder: Path | None, sources: list[str] | None, cites: list[str],
|
||
fallback_queries: list[str], budget: int = EVIDENCE_BUDGET_CHARS) -> str:
|
||
"""Evidence for the facts check: the EXACT cited regions (±EVIDENCE_CTX_LINES, merged,
|
||
line numbers in the header) — precise and tiny. Cites without a parseable position fall
|
||
back to the keyword pack. Empty string without a corpus."""
|
||
files = _corpus_files(folder, sources)
|
||
if not files:
|
||
return ""
|
||
ranges: dict[Path, list[tuple[int, int]]] = {}
|
||
unresolved = False
|
||
for c in cites:
|
||
ref = _cite_ref(c, files)
|
||
if ref is None:
|
||
unresolved = True
|
||
continue
|
||
f, lo, hi = ref
|
||
ranges.setdefault(f, []).append((max(1, lo - EVIDENCE_CTX_LINES), hi + EVIDENCE_CTX_LINES))
|
||
out, total = [], 0
|
||
for f in files:
|
||
if f not in ranges:
|
||
continue
|
||
try:
|
||
lines = f.read_text(encoding="utf-8").splitlines()
|
||
except OSError:
|
||
continue
|
||
merged: list[list[int]] = []
|
||
for lo, hi in sorted(ranges[f]):
|
||
hi = min(hi, len(lines))
|
||
if merged and lo <= merged[-1][1] + 1:
|
||
merged[-1][1] = max(merged[-1][1], hi)
|
||
else:
|
||
merged.append([lo, hi])
|
||
for lo, hi in merged:
|
||
sec = "\n".join(lines[lo - 1:hi])
|
||
if not sec.strip() or total + len(sec) > budget:
|
||
continue
|
||
out.append(f"── {f.name} · Z. {lo}-{hi} ──\n{sec}")
|
||
total += len(sec)
|
||
if unresolved or not out:
|
||
pack = _evidence_pack(folder, sources, fallback_queries, max(0, budget - total))
|
||
if pack:
|
||
out.append(pack)
|
||
return "\n\n".join(out)
|
||
|
||
|
||
def _reply_text(result) -> str:
|
||
"""Assistant text of a no-tool agent call ((rc, stdout, stderr) from run_agent)."""
|
||
return (result[1] or "") if result else ""
|
||
|
||
|
||
def _sink_or_file(result, path: Path, schema):
|
||
"""Text reply preferred, file fallback. Research agents used to WRITE their big JSON —
|
||
measured: one facts call spent 40 of 64 turns in a write/validate/repair loop (9 min).
|
||
As text, parse_json_text repairs the escaping in one pass; tools stay for research."""
|
||
val = _sink_json(result, path, schema)
|
||
if val is not None:
|
||
return val
|
||
return schema(_json_file(path))
|
||
|
||
|
||
def _sink_json(result, path: Path, schema):
|
||
"""Payload validator for no-tool agents: the JSON comes as reply TEXT; the engine
|
||
persists it to `path`, so resume guards and audit files keep working unchanged."""
|
||
data = parse_json_text(_reply_text(result))
|
||
val = schema(data)
|
||
if val is not None:
|
||
atomic_write_json(path, data)
|
||
return val
|
||
|
||
|
||
async def _panel_2of3(tasks: dict, sink, outs_now, norm) -> None:
|
||
"""Panel-Welle „first 2 agree": kehrt zurück, sobald zwei vorliegende Verdicts
|
||
übereinstimmen — die dritte Stimme kann die Mehrheit dann nicht mehr kippen. Sonst
|
||
(Dissens) wird weiter gewartet. Der Langsamste bestimmte jede Welle (gemessen: 98 s
|
||
bei ok-p50 ~50 s). Nachzügler laufen detached weiter; ihr File dient nur dem Resume.
|
||
tasks: {Task: judge_nr} · sink(j, result) persistiert · outs_now() liest Verdicts ·
|
||
norm(verdict) macht sie vergleichbar."""
|
||
offen = dict(tasks)
|
||
while offen:
|
||
done, _rest = await asyncio.wait(list(offen), return_when=asyncio.FIRST_COMPLETED)
|
||
for t in done:
|
||
j = offen.pop(t)
|
||
try:
|
||
r = t.result()
|
||
except Exception: # noqa: BLE001 — Panel ist fail-open, Ausfall = fehlende Stimme
|
||
continue
|
||
if isinstance(r, tuple):
|
||
sink(j, r)
|
||
outs = [norm(s) for s in outs_now()]
|
||
if len(outs) >= 2 and any(outs[a] == outs[b]
|
||
for a in range(len(outs)) for b in range(a + 1, len(outs))):
|
||
break
|
||
for t, j in offen.items(): # Dritter läuft weiter — sein File zählt fürs Resume
|
||
async def _warte(t=t, j=j):
|
||
try:
|
||
r = await t
|
||
if isinstance(r, tuple):
|
||
sink(j, r)
|
||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||
pass
|
||
_detached(asyncio.create_task(_warte()))
|
||
|
||
|
||
def _sink_subs(result, path: Path):
|
||
"""Finder reply as TEXT (marker format), persisted to `path` for audit/diagnosis.
|
||
File fallback: a tool-capable agent (thema web mode) that wrote the file despite the
|
||
text instruction still counts — same tolerance as _sink_or_file."""
|
||
text = _reply_text(result).strip()
|
||
d = _parse_subblocks(text)
|
||
if d:
|
||
atomic_write_text(path, text)
|
||
return d
|
||
return _parse_subblocks(_read(path)) or None
|
||
|
||
|
||
def _finder_material(folder, sources: list[str] | None, queries: list[str]) -> tuple[str, str, str]:
|
||
"""→ (material, backing, caps) for the Subblock-Research prompt. uni/projekt: corpus
|
||
excerpts INLINE — the finder had neither path nor excerpts and hunted the material per
|
||
call via glob/grep/bash (measured: 3–13 tool rounds, 52–141 s vs 15–35 s single-shot).
|
||
thema (or excerpt miss): web research as before, fail-open."""
|
||
ev = _evidence_pack(folder, sources, queries) if folder else ""
|
||
if ev:
|
||
material = "\n" + _prompt("Blocks-Source-Inline", excerpts=ev) + "\n"
|
||
backing = ("Backed by the SOURCE EXCERPTS above, not invented — leave out any "
|
||
"sub-point the excerpts do not support.")
|
||
return material, backing, "none"
|
||
return "", "Backed, not invented. Verify uncertain points via web search.", ("files" if folder else "full")
|
||
|
||
|
||
def _build_research_prompt(topic: str, out_path: Path, instructions: str, type: str, folder: Path | None, fokus: str = "", section: str = "") -> str:
|
||
if section:
|
||
# Section mode (uni/projekt): text directly in the prompt → small context, no file reading.
|
||
source = section
|
||
elif type in _SOURCE_TEMPLATE:
|
||
source = _prompt(_SOURCE_TEMPLATE[type], project=folder)
|
||
else:
|
||
source = _prompt("Blocks-Source-Thema", topic=topic)
|
||
return _prompt(
|
||
"Blocks-Research",
|
||
topic=topic, source=source, blocks_path=out_path, focus=fokus, extra=_extra(instructions),
|
||
)
|
||
|
||
|
||
def _file_payload(path: Path):
|
||
"""Valid if the slot file exists and contains numbered entries."""
|
||
if not path.exists():
|
||
return None
|
||
text = path.read_text(encoding="utf-8")
|
||
return text if _parse_selection(text) else None
|
||
|
||
|
||
|
||
|
||
def _question_pattern_chunk_schema(data) -> list[dict] | None:
|
||
"""{"pattern": [{block, subblock, question}, …]} → list of valid entries · otherwise None.
|
||
|
||
One pattern per subblock (no type cross-product — the difficulty only comes at
|
||
exam time from the learner's tier). Invalid individual entries are skipped."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("pattern"), list):
|
||
return None
|
||
out = []
|
||
for e in data["pattern"]:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
blk = str(e.get("block", "")).strip()
|
||
sub = str(e.get("subblock", "")).strip()
|
||
question = str(e.get("question", "")).strip()
|
||
if not blk or not sub or not question:
|
||
continue
|
||
out.append({"block": blk, "subblock": sub, "question": question})
|
||
return out or None
|
||
|
||
|
||
def _read(p: Path) -> str:
|
||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
||
|
||
|
||
def _chunk_nums(items: list, n: int) -> list[list]:
|
||
"""Splits a flat list into n chunks as equal in size as possible."""
|
||
n = max(1, n)
|
||
size = max(1, math.ceil(len(items) / n))
|
||
return [items[i:i + size] for i in range(0, len(items), size)]
|
||
|
||
|
||
def _n_chunks(count: int, size: int = SUBBLOCK_CHUNK) -> int:
|
||
return min(SUBBLOCK_MAX, max(1, math.ceil(count / size)))
|
||
|
||
|
||
def _lpt_chunks(weights: list[int], target: int) -> list[list[int]]:
|
||
"""Distribute indices across chunks load-balanced (LPT, makespan-minimal). Weight = cost per index.
|
||
K = ceil(total weight/target); heaviest first into the currently lightest bin. → index lists."""
|
||
if not weights:
|
||
return []
|
||
K = max(1, math.ceil(sum(weights) / max(1, target)))
|
||
bins: list[list[int]] = [[] for _ in range(K)]
|
||
last = [0] * K
|
||
for i in sorted(range(len(weights)), key=lambda x: weights[x], reverse=True):
|
||
j = min(range(K), key=lambda b: last[b])
|
||
bins[j].append(i)
|
||
last[j] += weights[i]
|
||
return [b for b in bins if b]
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
# lemmatized: 'kein Syntaxfehler' vs 'keine Syntax-Fehlermeldung' are the SAME statement —
|
||
# unlemmatized token sets ({kein} ≠ {keine}) blocked that fold at cos 0.974 (measured).
|
||
_NEG_LEMMA = {"nicht": "nicht", "ohne": "ohne", "nie": "nie", "niemals": "nie",
|
||
"kein": "kein", "keine": "kein", "keinen": "kein", "keiner": "kein",
|
||
"keinem": "kein", "keines": "kein"}
|
||
|
||
|
||
def _neg_set(title: str) -> frozenset:
|
||
"""Lemmatized negation tokens of a title — antonym statements measure cos 0.91–0.95
|
||
(above any usable variant threshold), so equal negation sets are a hard merge precondition."""
|
||
return frozenset(l for t in re.findall(r"\w+", _norm_title(title)) if (l := _NEG_LEMMA.get(t)))
|
||
|
||
|
||
def _sub_tokens(title: str) -> set:
|
||
return set(re.findall(r"\w+", _norm_title(title)))
|
||
|
||
|
||
def _variant_clusters(titles: list[str], mentions: list[int], sims) -> list[dict]:
|
||
"""Fold phrasing VARIANTS of one concept BEFORE the consensus count: finders rephrase per
|
||
round, so exact-norm counting starves real concepts. Union-find over cos ≥ SUB_VARIANT_COS
|
||
with the negation guard. → [{"rep": idx, "members": [idx…], "mentions": sum}]."""
|
||
n = len(titles)
|
||
parent = list(range(n))
|
||
|
||
def find(x):
|
||
while parent[x] != x:
|
||
parent[x] = parent[parent[x]]
|
||
x = parent[x]
|
||
return x
|
||
|
||
negs = [_neg_set(t) for t in titles]
|
||
for i in range(n):
|
||
for j in range(i + 1, n):
|
||
if float(sims[i][j]) >= SUB_VARIANT_COS and negs[i] == negs[j]:
|
||
parent[find(i)] = find(j)
|
||
groups: dict[int, list[int]] = {}
|
||
for i in range(n):
|
||
groups.setdefault(find(i), []).append(i)
|
||
return [{"rep": max(g, key=lambda k: (len(titles[k]), -k)), "members": g,
|
||
"mentions": sum(mentions[k] for k in g)} for g in groups.values()]
|
||
|
||
|
||
async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str,
|
||
wipe: bool = True, ns: str = "", seeds: list[str] | None = None,
|
||
lbl: str = "", sources: list[str] | None = None) -> dict | None:
|
||
"""Block B (DB + loop): per package, find subblocks in rounds (3 finders, until 0 new/cap),
|
||
collect in the DB (variant-clustered mentions ≥2 = consensus), a judge panel cleans up per
|
||
package; blocks below SUBBLOCK_MIN get focused catch-up rounds; `seeds` (demoted fragment
|
||
titles, single-block kanban calls) are guaranteed to reach the facts evidence gate.
|
||
→ {block title: [subblock, …]} (consensus) or None. Fills DB table `subblocks`.
|
||
wipe=False (kanban board: one call per block) keeps the other blocks' rows."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
folder = source_folder(topic)
|
||
caps = "files" if folder else "full"
|
||
# Source for the evidence exam in the clarify step (discards invented/unsupportable subs).
|
||
_type = load_source(topic).get("type", "thema")
|
||
source = _prompt(_SOURCE_TEMPLATE[_type], project=folder) if _type in _SOURCE_TEMPLATE else _prompt("Blocks-Source-Thema", topic=topic)
|
||
nums = list(entries)
|
||
chunks = _chunk_nums(nums, _n_chunks(len(nums)))
|
||
n = len(chunks)
|
||
title_by_num = {num: _title(entries[num]) for num in nums}
|
||
norm_by_num = {num: _norm_title(title_by_num[num]) for num in nums}
|
||
emb_on = EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available)
|
||
if wipe:
|
||
await db.delete_subblocks(topic) # fresh start of the block (idempotent counter)
|
||
else:
|
||
for num in nums: # per-block wipe: a re-spawned card must not accumulate mentions
|
||
await db.delete_subblocks(topic, norm_by_num[num])
|
||
|
||
async def _known_block(chunk):
|
||
known = []
|
||
for num in chunk:
|
||
subs = [s["sub_title"] for s in await db.list_subblocks(topic, norm_by_num[num])]
|
||
if subs:
|
||
known.append(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in subs))
|
||
if not known:
|
||
return ""
|
||
# Do NOT list known items again (otherwise re-confirmation inflates the mention count,
|
||
# self-bias/echo) — only add what's missing. This keeps the counter an honest consensus signal.
|
||
return ("\n\nBEREITS ERFASST — liste diese NICHT erneut. Finde nur, was FEHLT:\n" + "\n".join(known))
|
||
|
||
# ONE finder round (3 slots, quorum 2) → count of NEW sub norms; None = no result/cancel.
|
||
async def _one_round(label, subset, assignment, paths, keys, known, extra_instr, material, backing, round_caps):
|
||
chunk_idx = _title_index({num: title_by_num[num] for num in subset})
|
||
for p in paths:
|
||
p.unlink(missing_ok=True)
|
||
slots = [{
|
||
"key": k,
|
||
"prompt": _prompt("Subblock-Research", topic=topic, assignment=assignment, known=known, material=material, backing=backing, extra=_extra(extra_instr)),
|
||
"role": "quick", "capabilities": round_caps,
|
||
"payload": (lambda result, p=p: _sink_subs(result, p)),
|
||
} for k, p in zip(keys, paths)]
|
||
|
||
async def _fold_late(d: dict) -> None:
|
||
"""Dritte Stimme nachbuchen statt warten (ersetzte grace=300, gemessen 73 s/Runde):
|
||
Mentions sind additiv; ein Fund, den nur der Nachzügler hat, bleibt Einzelfund
|
||
und läuft durchs Clarify-Quellen-Gate — verfälscht wird nichts."""
|
||
for marker, subs in d.items():
|
||
num = _resolve_title(chunk_idx, marker)
|
||
if num is None:
|
||
continue
|
||
seen_late = set()
|
||
for sub in subs:
|
||
sn = _norm_title(sub)
|
||
if sn and sn not in seen_late:
|
||
seen_late.add(sn)
|
||
await db.upsert_subblock(topic, norm_by_num[num], sn, title_by_num[num], sub)
|
||
|
||
agent_texts = await _race(topic, label, slots, 2, _timeout("subblock", len(subset)), provider, cancelled=is_cancelled, late=_fold_late)
|
||
if is_cancelled() or not agent_texts:
|
||
return None
|
||
rows_before = {num: await db.list_subblocks(topic, norm_by_num[num]) for num in subset}
|
||
existing = {num: {s["sub_norm"] for s in rows_before[num]} for num in subset}
|
||
fresh: dict[int, list[str]] = {}
|
||
for d in agent_texts:
|
||
for marker, subs in d.items():
|
||
num = _resolve_title(chunk_idx, marker)
|
||
if num is None:
|
||
continue
|
||
seen_set = set()
|
||
for sub in subs:
|
||
sn = _norm_title(sub)
|
||
if not sn or sn in seen_set:
|
||
continue
|
||
seen_set.add(sn)
|
||
if sn not in existing[num]:
|
||
existing[num].add(sn)
|
||
fresh.setdefault(num, []).append(sub)
|
||
await db.upsert_subblock(topic, norm_by_num[num], sn, title_by_num[num], sub)
|
||
# "New" is variant-robust: a paraphrase of an existing sub (or of another fresh find)
|
||
# still gets stored above (its mention feeds the cluster consensus), but it must not
|
||
# keep the saturation loop spinning — finders rephrase every round (measured: 5–9
|
||
# rounds without this fold). Model off → exact counting (status quo).
|
||
new = 0
|
||
for num, cands in fresh.items():
|
||
sims = None
|
||
base = [s["sub_title"] for s in rows_before[num]]
|
||
if emb_on and base + cands:
|
||
sims = await asyncio.to_thread(embedding.embed_sims, base + cands)
|
||
if sims is None:
|
||
new += len(cands)
|
||
continue
|
||
negs = [_neg_set(t) for t in base + cands]
|
||
nb = len(base)
|
||
kept: list[int] = []
|
||
for i in range(nb, nb + len(cands)):
|
||
dup = any(float(sims[i][j]) >= SUB_VARIANT_COS and negs[i] == negs[j]
|
||
for j in [*range(nb), *kept])
|
||
if not dup:
|
||
kept.append(i)
|
||
new += len(kept)
|
||
return new
|
||
|
||
# Phase "Subblocks find": per package loop until 0 new subs / time cap.
|
||
async def _find(c, chunk):
|
||
assignment = "\n".join(f"- {entries[num]}" for num in chunk)
|
||
# Material once per package (excerpts are round-invariant; one query per block
|
||
# keeps the coverage guarantee of _evidence_pack).
|
||
material, backing, round_caps = await asyncio.to_thread(
|
||
_finder_material, folder, sources, [str(entries[num]) for num in chunk])
|
||
start = time.monotonic()
|
||
round_n = 0
|
||
while not is_cancelled():
|
||
round_n += 1
|
||
bekannt = await _known_block(chunk) if round_n > 1 else ""
|
||
paths = [work_dir / f"subblock-c{c}-r{round_n}-{i}.md" for i in (1, 2, 3)]
|
||
keys = [f"blocks-{topic}-{ns}subblock-c{c}-r{round_n}-{i}" for i in (1, 2, 3)]
|
||
new = await _one_round(f"{lbl}Subblocks package {c} R{round_n}", chunk, assignment, paths, keys, bekannt, instructions, material, backing, round_caps)
|
||
if new is None:
|
||
if is_cancelled():
|
||
return False
|
||
return round_n > 1 # round 1 without result = error; later = simply the end
|
||
if new == 0:
|
||
break
|
||
if round_n >= SUBBLOCK_MAX_ROUNDS:
|
||
_log(topic, f"Subblocks package {c}: round cap reached ({round_n})")
|
||
break
|
||
if time.monotonic() - start > SUBBLOCK_CAP:
|
||
_log(topic, f"Subblocks package {c}: time cap reached (round {round_n})")
|
||
break
|
||
return True
|
||
|
||
oks = await _gather_progress([_find(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _report_p(set_p, topic, "Subblocks find"))
|
||
if is_cancelled():
|
||
return None
|
||
if not all(ok is True for ok in oks):
|
||
_blocks_errors[topic] = "Subblocks failed (research)"
|
||
return None
|
||
|
||
# Phase "Subblocks select": variant-clustered mentions ≥2 = consensus — the cluster
|
||
# representative carries the status, folded members become `variant` (NOT discarded:
|
||
# the clarify panel's uncertain group must not re-list them). Model off → exact counter.
|
||
async def _select(subset, keep_consensus=False):
|
||
for num in subset:
|
||
rows = await db.list_subblocks(topic, norm_by_num[num])
|
||
clusters = None
|
||
if emb_on and len(rows) >= 2:
|
||
sims = await asyncio.to_thread(embedding.embed_sims, [r["sub_title"] for r in rows])
|
||
if sims is not None:
|
||
clusters = _variant_clusters([r["sub_title"] for r in rows],
|
||
[r["mentions"] for r in rows], sims)
|
||
if clusters is None:
|
||
for r in rows:
|
||
if keep_consensus and r["status"] == "consensus":
|
||
continue
|
||
await db.set_subblock_fields(topic, norm_by_num[num], r["sub_norm"],
|
||
status=("consensus" if r["mentions"] >= 2 else "discarded"))
|
||
continue
|
||
for cl in clusters:
|
||
# a re-select (catch-up) never demotes panel-confirmed subs — an existing
|
||
# consensus member stays the representative, new variants fold under it.
|
||
kept = [k for k in cl["members"] if keep_consensus and rows[k]["status"] == "consensus"]
|
||
for k in cl["members"]:
|
||
if kept:
|
||
st = "consensus" if k in kept else "variant"
|
||
elif cl["mentions"] >= 2:
|
||
st = "consensus" if k == cl["rep"] else "variant"
|
||
else:
|
||
st = "discarded"
|
||
if keep_consensus and rows[k]["status"] == "consensus" and st != "consensus":
|
||
continue
|
||
await db.set_subblock_fields(topic, norm_by_num[num], rows[k]["sub_norm"], status=st)
|
||
|
||
set_p(f"Subblocks select ({n} packages)…", step=_step_idx(topic, "Subblocks select"))
|
||
await _select(nums)
|
||
|
||
# Judge formulation → shown candidate (best cos ≥ SUB_VARIANT_COS, negation-guarded).
|
||
# Judges demonstrably paraphrase; without canonicalizing, the exact-norm majority vote
|
||
# splinters across formulations (measured: 0.993-duplicates in a final list).
|
||
async def _canon_map(shown: list[str], judge_titles: list[str]) -> dict[str, tuple[str, str]]:
|
||
if not emb_on or not shown or not judge_titles:
|
||
return {}
|
||
texts = shown + list(judge_titles)
|
||
sims = await asyncio.to_thread(embedding.embed_sims, texts)
|
||
if sims is None:
|
||
return {}
|
||
negs = [_neg_set(t) for t in texts]
|
||
m: dict[str, tuple[str, str]] = {}
|
||
for a in range(len(shown), len(texts)):
|
||
best, bv = None, 0.0
|
||
for b in range(len(shown)):
|
||
v = float(sims[a][b])
|
||
if v >= SUB_VARIANT_COS and v > bv and negs[a] == negs[b]:
|
||
best, bv = b, v
|
||
if best is not None:
|
||
m[_norm_title(texts[a])] = (_norm_title(shown[best]), shown[best])
|
||
return m
|
||
|
||
# Phase "Subblocks clarify": source panel (SUBBAUSTEIN_PANEL judges) checks consensus + uncertain (1×)
|
||
# against the source; code majority per sub. External, multi-voice gate against single-judge bias + echo.
|
||
async def _clarify(c, chunk, tag=""):
|
||
fp = work_dir / f"subblock-final-c{c}{tag}.md"
|
||
if _parse_subblocks(_read(fp)):
|
||
return
|
||
block_texts, has_any = [], False
|
||
consensus_by_num: dict[int, list[str]] = {}
|
||
shown_by_num: dict[int, list[str]] = {}
|
||
for num in chunk:
|
||
rows = await db.list_subblocks(topic, norm_by_num[num])
|
||
consensus_subs = [s["sub_title"] for s in rows if s["status"] == "consensus"]
|
||
# folded variants (status `variant`) are already counted — only true singles are uncertain
|
||
uncertain = [s["sub_title"] for s in rows if s["status"] == "discarded" and s["mentions"] == 1]
|
||
consensus_by_num[num] = consensus_subs
|
||
shown_by_num[num] = consensus_subs + uncertain
|
||
if not consensus_subs and not uncertain:
|
||
continue
|
||
has_any = True
|
||
k_lines = "\n".join(f"- {s}" for s in consensus_subs) if consensus_subs else "- (keiner)"
|
||
u_lines = "\n".join(f"- {s}" for s in uncertain) if uncertain else "- (keiner)"
|
||
band = ""
|
||
shown = shown_by_num[num]
|
||
if emb_on and len(shown) >= 2: # near-dup pairs BELOW the fold threshold → explicit panel hint
|
||
sims = await asyncio.to_thread(embedding.embed_sims, shown)
|
||
if sims is not None:
|
||
pairs = [f"- „{shown[i]}“ ↔ „{shown[j]}“"
|
||
for i in range(len(shown)) for j in range(i + 1, len(shown))
|
||
if 0.75 <= float(sims[i][j]) < SUB_VARIANT_COS]
|
||
if pairs:
|
||
band = ("\nMögliche Duplikate — prüfen und ggf. zu EINEM Eintrag zusammenführen:\n"
|
||
+ "\n".join(pairs[:12]))
|
||
block_texts.append(f"BLOCK: {title_by_num[num]}\nKonsens (≥2 finders):\n{k_lines}\nUnsicher (1× — streng gegen Source check):\n{u_lines}{band}")
|
||
if not has_any:
|
||
return
|
||
|
||
chunk_idx = _title_index({num: title_by_num[num] for num in chunk})
|
||
paths = [work_dir / f"subblock-final-c{c}{tag}-j{j}.md" for j in range(1, SUBBLOCK_PANEL + 1)]
|
||
# truthiness, NOT `is None`: a missing file parses to {} — with `is None` the whole
|
||
# panel silently never ran (fallback adopted the raw consensus unchecked).
|
||
pending = [(j, p) for j, p in enumerate(paths, 1) if not _parse_subblocks(_read(p))]
|
||
for _, p in pending:
|
||
p.unlink(missing_ok=True)
|
||
if pending:
|
||
# Inline evidence: corpus excerpts in the prompt (no self-research); the judge
|
||
# answers as TEXT, the engine persists the j-file (resume + majority unchanged).
|
||
ev = _evidence_pack(folder, sources,
|
||
[title_by_num[num] for num in chunk]
|
||
+ [s for num in chunk for s in shown_by_num.get(num, [])]) if folder else ""
|
||
j_source = _prompt("Blocks-Source-Inline", excerpts=ev) if ev else source
|
||
|
||
def _sink(result, p):
|
||
text = _reply_text(result).strip()
|
||
d = _parse_subblocks(text)
|
||
if d:
|
||
atomic_write_text(p, text)
|
||
return d or None
|
||
|
||
slots = [{
|
||
"key": f"blocks-{topic}-{ns}subblock-final-c{c}{tag}-j{j}",
|
||
"prompt": _prompt("Subblock-Mapping", topic=topic, source=j_source, blocks="\n\n".join(block_texts), out_path=p, extra=_extra(instructions)),
|
||
"role": "judge", "capabilities": "none" if ev else caps,
|
||
"payload": (lambda result, p=p: _sink(result, p)),
|
||
} for j, p in pending]
|
||
existing = SUBBLOCK_PANEL - len(pending)
|
||
await _race(topic, f"{lbl}Subblock-Clarification {c}", slots, max(1, 2 - existing),
|
||
_timeout("subblock_check", len(chunk)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
if is_cancelled():
|
||
return
|
||
outs = [d for p in paths if (d := _parse_subblocks(_read(p)))]
|
||
if not outs: # panel fully failed → adopt consensus (the fallback as before)
|
||
_log(topic, f"Subblock clarification package {c} failed — consensus adopted")
|
||
text = "\n\n".join(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in consensus_by_num[num])
|
||
for num in chunk if consensus_by_num[num])
|
||
atomic_write_text(fp, text)
|
||
return
|
||
|
||
# code majority per block/sub-norm: keep if a majority of judges list it (tie → keep).
|
||
# Votes are canonicalized onto the shown candidates first (paraphrase-robust).
|
||
block_texts_out = []
|
||
for num in chunk:
|
||
raw_votes: list[list[str]] = []
|
||
for d in outs:
|
||
subs_of_num: list[str] = []
|
||
for marker, subs in d.items():
|
||
if _resolve_title(chunk_idx, marker) == num:
|
||
subs_of_num.extend(subs)
|
||
raw_votes.append(subs_of_num)
|
||
judge_titles = list(dict.fromkeys(s for subs in raw_votes for s in subs
|
||
if _norm_title(s) not in {_norm_title(t) for t in shown_by_num[num]}))
|
||
canon = await _canon_map(shown_by_num[num], judge_titles)
|
||
votes: dict[str, int] = {}
|
||
form: dict[str, str] = {}
|
||
for subs_of_num in raw_votes:
|
||
seen = set()
|
||
for sub in subs_of_num:
|
||
sn = _norm_title(sub)
|
||
if not sn:
|
||
continue
|
||
if sn in canon:
|
||
sn, sub = canon[sn]
|
||
if sn in seen:
|
||
continue
|
||
seen.add(sn)
|
||
form.setdefault(sn, sub)
|
||
votes[sn] = votes.get(sn, 0) + 1
|
||
kept = [form[sn] for sn in form if votes[sn] * 2 >= len(outs)]
|
||
if kept:
|
||
block_texts_out.append(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in kept))
|
||
atomic_write_text(fp, "\n\n".join(block_texts_out))
|
||
|
||
await _gather_progress([_clarify(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _report_p(set_p, topic, "Subblocks clarify"))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# Final list per block: judge output, otherwise consensus fallback. Reconcile DB + build raw.
|
||
raw: dict[str, list[str]] = {}
|
||
|
||
async def _align(c, chunk, tag=""):
|
||
final = _parse_subblocks(_read(work_dir / f"subblock-final-c{c}{tag}.md")) or {}
|
||
chunk_idx = _title_index({num: title_by_num[num] for num in chunk})
|
||
final_by_num = {_resolve_title(chunk_idx, m): subs for m, subs in final.items() if _resolve_title(chunk_idx, m) is not None}
|
||
for num in chunk:
|
||
title = title_by_num[num]
|
||
consensus = [s["sub_title"] for s in await db.list_subblocks(topic, norm_by_num[num]) if s["status"] == "consensus"]
|
||
subs = final_by_num.get(num) or consensus
|
||
if not subs:
|
||
continue
|
||
raw[title] = subs
|
||
# align DB to the final list: final = consensus, rest discarded, add new ones.
|
||
final_norms = {_norm_title(s) for s in subs}
|
||
have = {s["sub_norm"] for s in await db.list_subblocks(topic, norm_by_num[num])}
|
||
for s in await db.list_subblocks(topic, norm_by_num[num]):
|
||
if s["sub_norm"] in final_norms:
|
||
st = "consensus"
|
||
elif s["status"] == "variant":
|
||
st = "variant" # folded members stay marked — a catch-up clarify must not
|
||
else: # re-list them as "uncertain singles"
|
||
st = "discarded"
|
||
await db.set_subblock_fields(topic, norm_by_num[num], s["sub_norm"], status=st)
|
||
for s in subs:
|
||
sn = _norm_title(s)
|
||
if sn and sn not in have:
|
||
await db.upsert_subblock(topic, norm_by_num[num], sn, title, s)
|
||
await db.set_subblock_fields(topic, norm_by_num[num], sn, status="consensus")
|
||
|
||
for c, chunk in enumerate(chunks, 1):
|
||
await _align(c, chunk)
|
||
|
||
# Minimum catch-up: a block below SUBBLOCK_MIN gets up to SUBBLOCK_EXTRA_ROUNDS focused
|
||
# finder rounds. Saturation stop stays — a thin block REMAINS thin if nothing verifiable.
|
||
async def _catchup(c, chunk):
|
||
for k in range(1, SUBBLOCK_EXTRA_ROUNDS + 1):
|
||
lacking = [num for num in chunk if len(raw.get(title_by_num[num]) or []) < SUBBLOCK_MIN]
|
||
if not lacking or is_cancelled():
|
||
return
|
||
assignment = "\n".join(f"- {entries[num]}" for num in lacking)
|
||
known = await _known_block(lacking) # ALL rows of the block, incl. variants/discarded
|
||
focus = (instructions + "\n\nDieser Block hat bisher nur sehr wenige belegte "
|
||
"Subbausteine. Suche gezielt nach WEITEREN belegbaren Kernaspekten, die "
|
||
"oben fehlen. Nimm NUR auf, was die Quellen wirklich hergeben — nicht aufblähen.")
|
||
material, backing, round_caps = await asyncio.to_thread(
|
||
_finder_material, folder, sources, [str(entries[num]) for num in lacking])
|
||
paths = [work_dir / f"subblock-x{k}-c{c}-{i}.md" for i in (1, 2, 3)]
|
||
keys = [f"blocks-{topic}-{ns}subblock-x{k}-c{c}-{i}" for i in (1, 2, 3)]
|
||
new = await _one_round(f"{lbl}Subblocks catch-up {c} X{k}", lacking, assignment, paths, keys, known, focus, material, backing, round_caps)
|
||
if not new:
|
||
return
|
||
await _select(lacking, keep_consensus=True)
|
||
await _clarify(c, chunk, tag=f"-x{k}")
|
||
if is_cancelled():
|
||
return
|
||
await _align(c, chunk, tag=f"-x{k}")
|
||
|
||
# progress reuses the clarify step label — catch-up has no own registry entry
|
||
await _gather_progress([_catchup(c, chunk) for c, chunk in enumerate(chunks, 1)], n,
|
||
_report_p(set_p, topic, "Subblocks clarify"))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# Seed guarantee (single-block kanban calls): every demoted-fragment seed must reach the
|
||
# facts evidence gate — covered by a consensus sub, promoted from a single find, or
|
||
# inserted as its own sub. Unverifiable seeds die at the facts discard, not silently here.
|
||
for num in (nums if seeds else []):
|
||
title = title_by_num[num]
|
||
for seed in dict.fromkeys(s for s in seeds if s):
|
||
st = _sub_tokens(seed)
|
||
have = raw.get(title) or []
|
||
if not st or any(st <= _sub_tokens(s) for s in have):
|
||
continue
|
||
if emb_on and have: # embedding backup for rephrased covers (lexical is primary)
|
||
sims = await asyncio.to_thread(embedding.embed_sims, [seed] + have)
|
||
if sims is not None and max(float(sims[0][j]) for j in range(1, len(have) + 1)) >= SEED_COVER_COS:
|
||
continue
|
||
rows = [r for r in await db.list_subblocks(topic, norm_by_num[num]) if r["status"] != "consensus"]
|
||
cand = next((r for r in rows if st <= _sub_tokens(r["sub_title"])), None)
|
||
if cand is None and emb_on and rows:
|
||
sims = await asyncio.to_thread(embedding.embed_sims, [seed] + [r["sub_title"] for r in rows])
|
||
if sims is not None:
|
||
j = max(range(1, len(rows) + 1), key=lambda x: float(sims[0][x]))
|
||
if float(sims[0][j]) >= SEED_COVER_COS and _neg_set(seed) == _neg_set(rows[j - 1]["sub_title"]):
|
||
cand = rows[j - 1]
|
||
if cand is not None:
|
||
await db.set_subblock_fields(topic, norm_by_num[num], cand["sub_norm"], status="consensus")
|
||
raw.setdefault(title, []).append(cand["sub_title"])
|
||
_log(topic, f"Seed „{seed}“: Einzelfund „{cand['sub_title']}“ übernommen ({title})")
|
||
elif (sn := _norm_title(seed)):
|
||
await db.put_subblock(topic, norm_by_num[num], sn, title, seed, status="consensus")
|
||
raw.setdefault(title, []).append(seed)
|
||
_log(topic, f"Seed „{seed}“ als Subbaustein eingefügt ({title}) — Facts-Gate prüft")
|
||
|
||
# AFTER the seed guarantee: promoted/inserted seeds must not bypass the near-dup filter
|
||
await _dedup_subblocks(topic, raw) # near-dup filter per block (deterministic, no LLM)
|
||
|
||
if not raw:
|
||
# Finders ran but nothing survived the consensus/evidence gates: a legitimately
|
||
# thin block (e.g. a bare named reduction). {} = done-without-subs — the guide
|
||
# writes it from description+facts. Agent FAILURES return None elsewhere (retry).
|
||
_log(topic, "Subblocks: nichts Belegbares gefunden — Block bleibt ohne Subbausteine")
|
||
return {}
|
||
return raw
|
||
|
||
|
||
async def _dedup_subblocks(topic: str, raw: dict[str, list[str]]) -> None:
|
||
"""Deterministic near-duplicate filter per block: subblocks with cosine ≥
|
||
EMBEDDING_SUB_DUP are the same statement (reliable in the narrow block context — no LLM
|
||
needed). Per duplicate group keeps the most informative (longest); rest → DB discarded + out of `roh`.
|
||
Model missing → silently skip (like the rest of the embedding fallback)."""
|
||
if not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
|
||
return
|
||
for title, subs in list(raw.items()):
|
||
if len(subs) < 2:
|
||
continue
|
||
sims = await asyncio.to_thread(embedding.embed_sims, subs)
|
||
if sims is None:
|
||
return
|
||
keepers: list[int] = []
|
||
discarded: list[int] = []
|
||
negs = [_neg_set(s) for s in subs]
|
||
for i in sorted(range(len(subs)), key=lambda x: (-len(subs[x]), x)): # most informative first
|
||
if any(float(sims[i][j]) >= EMBEDDING_SUB_DUP and negs[i] == negs[j] for j in keepers):
|
||
discarded.append(i)
|
||
else:
|
||
keepers.append(i)
|
||
if not discarded:
|
||
continue
|
||
bnorm = _norm_title(title)
|
||
for i in discarded:
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(subs[i]), status="discarded")
|
||
raw[title] = [subs[i] for i in sorted(keepers)] # original order of the kept ones
|
||
|
||
|
||
_KONSOLIDIERUNG_PANEL = 2 # merge needs unanimity — single judges over-merge (blocks-dedup lesson)
|
||
|
||
|
||
def _kons_id(x, n: int) -> int | None:
|
||
"""Judge id → int in 1..n, else None (bools are not ids)."""
|
||
if isinstance(x, bool):
|
||
return None
|
||
if isinstance(x, str) and x.isdigit():
|
||
x = int(x)
|
||
return x if isinstance(x, int) and 1 <= x <= n else None
|
||
|
||
|
||
def _konsolidierung_schema(data, n: int) -> dict | None:
|
||
"""Judge output → normalized dict, else None. gruppen accepts the {"haupt": 1,
|
||
"weitere": [4]} form AND the legacy plain-list form [1, 4] (resume files of the
|
||
first template version). kataloge/fremd/luecken are optional."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("gruppen"), list):
|
||
return None
|
||
|
||
def _ids(lst):
|
||
return sorted({i for x in (lst or []) if (i := _kons_id(x, n)) is not None})
|
||
|
||
gruppen = []
|
||
for g in data["gruppen"]:
|
||
if isinstance(g, dict):
|
||
haupt = _kons_id(g.get("haupt"), n)
|
||
ids = _ids(([haupt] if haupt else []) + list(g.get("weitere") or []))
|
||
elif isinstance(g, list):
|
||
haupt, ids = None, _ids(g)
|
||
else:
|
||
return None
|
||
if len(ids) >= 2:
|
||
gruppen.append({"haupt": haupt if haupt in ids else None, "ids": ids})
|
||
kataloge = []
|
||
for k in data.get("kataloge") or []:
|
||
if not isinstance(k, dict):
|
||
continue
|
||
ids = _ids(k.get("mitglieder"))
|
||
titel = str(k.get("titel") or "").strip()
|
||
if len(ids) >= 2 and titel:
|
||
kataloge.append({"titel": titel, "ids": ids})
|
||
return {"gruppen": gruppen, "kataloge": kataloge, "fremd": set(_ids(data.get("fremd"))),
|
||
"luecken": [s.strip() for s in data.get("luecken") or [] if isinstance(s, str) and s.strip()]}
|
||
|
||
|
||
def _facts_union(wf: dict, lf: dict) -> None:
|
||
"""Merge a folded sub's facts into the winner's: key_points/cited_facts union
|
||
(exact-duplicate-free), scalar fields only fill gaps."""
|
||
for feld in ("key_points", "cited_facts"):
|
||
have = wf.get(feld) or []
|
||
seen = {json.dumps(e, sort_keys=True, ensure_ascii=False) for e in have}
|
||
fresh = [e for e in (lf.get(feld) or [])
|
||
if json.dumps(e, sort_keys=True, ensure_ascii=False) not in seen]
|
||
if fresh:
|
||
wf[feld] = have + fresh
|
||
for feld in ("prerequisites", "hurdles", "example_idea"):
|
||
if not wf.get(feld) and lf.get(feld):
|
||
wf[feld] = lf[feld]
|
||
|
||
|
||
def _agreed_cliques(pair_sets: list[set], negs: list, n: int) -> list[list[int]]:
|
||
"""Union-find over the UNANIMOUS pairs (both judges grouped them), negation-guarded."""
|
||
agreed = {(a, b) for a, b in pair_sets[0] & pair_sets[1] if negs[a - 1] == negs[b - 1]}
|
||
parent = list(range(n + 1))
|
||
|
||
def find(x):
|
||
while parent[x] != x:
|
||
parent[x] = parent[parent[x]]
|
||
x = parent[x]
|
||
return x
|
||
|
||
for a, b in agreed:
|
||
parent[find(a)] = find(b)
|
||
groups: dict[int, list[int]] = {}
|
||
for k in range(1, n + 1):
|
||
groups.setdefault(find(k), []).append(k)
|
||
return [g for g in groups.values() if len(g) >= 2]
|
||
|
||
|
||
def _pairs_of(groups) -> set:
|
||
ps: set[tuple[int, int]] = set()
|
||
for ids in groups:
|
||
ps |= {(a, b) for x, a in enumerate(ids) for b in ids[x + 1:]}
|
||
return ps
|
||
|
||
|
||
_LUECKEN_CAP = 3 # the gap list feeds ONE finder round — an uncapped list doubled the decomposition
|
||
|
||
|
||
def _luecken_schnitt(l1: list[str], l2: list[str], cap: int = _LUECKEN_CAP) -> list[str]:
|
||
"""Gaps BOTH judges name — exact strings never match across paraphrases, so a gap
|
||
survives when the other judge names one sharing a distinctive token (≥4 chars).
|
||
j1's phrasing wins. The measured union produced 107 'gaps' on 216 subs."""
|
||
def toks(s):
|
||
return {t for t in _sub_tokens(s) if len(t) >= 4}
|
||
toks2 = [toks(l) for l in l2]
|
||
out = [l for l in l1 if toks(l) and any(toks(l) & t2 for t2 in toks2)]
|
||
return out[:cap]
|
||
|
||
|
||
async def _konsolidiere_subblocks(ctx: GenContext, files: dict, raw: dict, facts_map: dict,
|
||
instructions: str = "", ns: str = "", lbl: str = "") -> dict:
|
||
"""In-block consolidation AFTER the facts stage: a two-judge panel sees the subs WITH
|
||
their key points and applies the 100%-decomposition test — the embedding paths only
|
||
catch cos ≥ 0.90, real paraphrase duplicates measure down to 0.61, and only the facts
|
||
reveal a subset. Every action needs UNANIMITY of both judges:
|
||
gruppen — same-statement/subset entries fold into the judge-named `haupt` (base
|
||
before detail; heuristic fallback), facts union, losers → `variant`
|
||
kataloge — pure enumeration entries of one kind bundle into a NEW named sub row
|
||
(members → `variant`); runs before levels/relevance, so the new row
|
||
gets classified normally
|
||
fremd — statements off-topic for the TOPIC → `discarded` (removal test)
|
||
Questions/artefacts do not exist yet — no orphans. Gaps are returned per block so the
|
||
caller can run the single follow-up finder round (`_luecken_runde`).
|
||
Judge replies persist as j-files keyed by a subs-list hash (resume-safe).
|
||
→ {block title: [luecken]}"""
|
||
topic = ctx.topic
|
||
work_dir = files["arbeit"]
|
||
luecken_by_title: dict[str, list[str]] = {}
|
||
for title, subs in list(raw.items()):
|
||
if ctx.is_cancelled():
|
||
return luecken_by_title
|
||
n = len(subs)
|
||
if n < 2:
|
||
continue
|
||
bnorm = _norm_title(title)
|
||
bfacts = facts_map.setdefault(title, {})
|
||
|
||
def _kp(s):
|
||
return (bfacts.get(_norm_title(s)) or {}).get("key_points") or []
|
||
|
||
# prompt shows max 3 key points per sub — full lists blew past the judge timeout
|
||
# (measured: 15 % timeouts at 585 s); the facts UNION on merge stays complete
|
||
lines = "\n".join(f"{k}. {s}" + "".join(f"\n - {p}" for p in _kp(s)[:3])
|
||
for k, s in enumerate(subs, 1))
|
||
h = hashlib.md5("\n".join(subs).encode()).hexdigest()[:8]
|
||
paths = [work_dir / f"sub-konsolidierung-{ns}{h}-j{j}.json" for j in (1, 2)]
|
||
|
||
async def _judge(j, path):
|
||
if _konsolidierung_schema(_json_file(path), n) is not None:
|
||
return # resume
|
||
status, _v = await run_single_slot(
|
||
ctx, f"{lbl}Sub-Konsolidierung j{j}",
|
||
key=f"blocks-{topic}-{ns}sub-konsolidierung-{h}-j{j}",
|
||
prompt=_prompt("Subblock-Konsolidierung", topic=topic, block=title, subs=lines, extra=_extra(instructions)),
|
||
role="judge", capabilities="none",
|
||
payload=lambda result, p=path: _sink_json(result, p, lambda d: _konsolidierung_schema(d, n)),
|
||
timeout=_timeout("konsolidierung", n))
|
||
if status == FAILED:
|
||
_log(topic, f"Sub-Konsolidierung {title} j{j} ohne Ergebnis — fail-open")
|
||
|
||
await asyncio.gather(*[_judge(j, p) for j, p in zip((1, 2), paths)])
|
||
if ctx.is_cancelled():
|
||
return luecken_by_title
|
||
outs = [o for p in paths if (o := _konsolidierung_schema(_json_file(p), n)) is not None]
|
||
if len(outs) == 1: # Ersatz-Richter: EIN Timeout darf die gute Stimme nicht entwerten
|
||
ersatz = work_dir / f"sub-konsolidierung-{ns}{h}-j3.json"
|
||
await _judge(3, ersatz)
|
||
if ctx.is_cancelled():
|
||
return luecken_by_title
|
||
outs = [o for p in [*paths, ersatz]
|
||
if (o := _konsolidierung_schema(_json_file(p), n)) is not None]
|
||
# gaps need UNANIMITY (token-overlap match) — the union of both judges was uncalibrated
|
||
luecken = (_luecken_schnitt(outs[0]["luecken"], outs[1]["luecken"])
|
||
if len(outs) == _KONSOLIDIERUNG_PANEL else [])
|
||
journal = {"block": title, "richter": len(outs), "vorher": n,
|
||
"luecken_roh": [len(o["luecken"]) for o in outs],
|
||
"gruppen": [], "kataloge": [], "fremd": [], "luecken": luecken}
|
||
if len(outs) == _KONSOLIDIERUNG_PANEL:
|
||
negs = [_neg_set(s) for s in subs]
|
||
keep = list(subs)
|
||
gone: set[int] = set()
|
||
|
||
async def _fold(k: int, wf: dict | None):
|
||
lose_title = subs[k - 1]
|
||
lf = bfacts.pop(_norm_title(lose_title), None) or {}
|
||
if wf is not None:
|
||
_facts_union(wf, lf)
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(lose_title), status="variant")
|
||
keep.remove(lose_title)
|
||
gone.add(k)
|
||
|
||
# 1. Fremd (removal test): off-topic for the TOPIC → discarded, no heir.
|
||
for k in sorted(outs[0]["fremd"] & outs[1]["fremd"]):
|
||
ft = subs[k - 1]
|
||
bfacts.pop(_norm_title(ft), None)
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(ft), status="discarded")
|
||
keep.remove(ft)
|
||
gone.add(k)
|
||
journal["fremd"].append(ft)
|
||
|
||
# 2. Gruppen: winner = judge-named haupt (majority), else key_points/length heuristic.
|
||
haupt_votes: dict[int, int] = {}
|
||
for o in outs:
|
||
for g in o["gruppen"]:
|
||
if g["haupt"]:
|
||
haupt_votes[g["haupt"]] = haupt_votes.get(g["haupt"], 0) + 1
|
||
for g in _agreed_cliques([_pairs_of([x["ids"] for x in o["gruppen"]]) for o in outs], negs, n):
|
||
g = [k for k in g if k not in gone]
|
||
if len(g) < 2:
|
||
continue
|
||
win = max(g, key=lambda k: (haupt_votes.get(k, 0),
|
||
len(_kp(subs[k - 1])), len(subs[k - 1]), -k))
|
||
wf = bfacts.setdefault(_norm_title(subs[win - 1]), {})
|
||
for k in g:
|
||
if k != win:
|
||
await _fold(k, wf)
|
||
journal["gruppen"].append({"behalten": subs[win - 1],
|
||
"gefaltet": [subs[k - 1] for k in g if k != win]})
|
||
|
||
# 3. Kataloge: bundle enumeration rows into ONE new named sub (facts union).
|
||
for g in _agreed_cliques([_pairs_of([x["ids"] for x in o["kataloge"]]) for o in outs], negs, n):
|
||
g = [k for k in g if k not in gone]
|
||
if len(g) < 2:
|
||
continue
|
||
titel = next((clean_title(x["titel"]) for x in outs[0]["kataloge"] + outs[1]["kataloge"]
|
||
if set(x["ids"]) & set(g) and clean_title(x["titel"])), "")
|
||
kn = _norm_title(titel)
|
||
if not kn or kn in {_norm_title(s) for s in keep}:
|
||
continue # no usable/colliding title → members stay
|
||
kf: dict = {}
|
||
for k in g:
|
||
await _fold(k, kf)
|
||
bfacts[kn] = kf
|
||
keep.append(titel)
|
||
await db.put_subblock(topic, bnorm, kn, title, titel, status="consensus")
|
||
journal["kataloge"].append({"titel": titel, "gefaltet": [subs[k - 1] for k in g]})
|
||
|
||
if len(keep) != n:
|
||
raw[title] = keep
|
||
_log(topic, f"Sub-Konsolidierung {title}: {n} → {len(keep)}")
|
||
elif outs:
|
||
_log(topic, f"Sub-Konsolidierung {title}: nur {len(outs)}/{_KONSOLIDIERUNG_PANEL} Richter — fail-open")
|
||
if luecken:
|
||
luecken_by_title[title] = luecken
|
||
_log(topic, f"Sub-Konsolidierung {title}: mögliche Lücken: {', '.join(luecken[:5])}")
|
||
atomic_write_json(work_dir / f"sub-konsolidierung-{ns}{h}.json", journal, indent=1)
|
||
return luecken_by_title
|
||
|
||
|
||
async def _luecken_runde(ctx: GenContext, files: dict, title: str, luecken: list[str],
|
||
raw: dict, facts_map: dict, q: dict, folder, instructions: str = "",
|
||
ns: str = "", lbl: str = "", sources: list[str] | None = None) -> int:
|
||
"""ONE targeted finder round for the consolidation judges' reported gaps — no loop.
|
||
Finds are deduped against the existing subs (token containment + embedding +
|
||
negation guard, seed-guarantee pattern) and must pass the facts evidence gate
|
||
(own work subdir `nf` — the block's facts resume files must not collide) before
|
||
they join raw/facts_map as consensus rows. They then flow through levels/relevance/
|
||
questions/artefacts like any other sub. → count of adopted subs."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
have = list(raw.get(title) or [])
|
||
focus = (instructions + "\n\nFinde NUR belegbare Subbausteine zu diesen bisher fehlenden "
|
||
"Aspekten des Blocks — nichts anderes:\n" + "\n".join(f"- {l}" for l in luecken))
|
||
known = ("\n\nBEREITS ERFASST — liste diese NICHT erneut:\n"
|
||
+ "\n".join(f"- {s}" for s in have)) if have else ""
|
||
material, backing, caps = await asyncio.to_thread(
|
||
_finder_material, folder, sources, [title] + list(luecken))
|
||
paths = [work_dir / f"luecken-{ns}r1-{i}.md" for i in (1, 2, 3)]
|
||
slots = [{
|
||
"key": f"blocks-{topic}-{ns}luecken-r1-{i}",
|
||
"prompt": _prompt("Subblock-Research", topic=topic, assignment=f"- {title}", known=known, material=material, backing=backing, extra=_extra(focus)),
|
||
"role": "quick", "capabilities": caps,
|
||
"payload": (lambda result, p=p: _sink_subs(result, p)),
|
||
} for i, p in zip((1, 2, 3), paths)]
|
||
agent_texts = await _race(topic, f"{lbl}Lücken-Nachfass", slots, 2,
|
||
_timeout("subblock", 1), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
if is_cancelled() or not agent_texts:
|
||
return 0
|
||
cands: list[str] = []
|
||
seen = {_norm_title(s) for s in have}
|
||
for d in agent_texts:
|
||
for subs in d.values(): # single-block call — every marker means this block
|
||
for s in subs:
|
||
sn = _norm_title(s)
|
||
if sn and sn not in seen:
|
||
seen.add(sn)
|
||
cands.append(s)
|
||
if not cands:
|
||
return 0
|
||
emb_on = EMBEDDING_AKTIV and await asyncio.to_thread(embedding.available)
|
||
fresh: list[str] = []
|
||
for s in cands:
|
||
st = _sub_tokens(s)
|
||
base = have + fresh
|
||
if any(st <= _sub_tokens(b) or _sub_tokens(b) <= st for b in base):
|
||
continue
|
||
if emb_on and base:
|
||
sims = await asyncio.to_thread(embedding.embed_sims, [s] + base)
|
||
if sims is not None:
|
||
negs = [_neg_set(t) for t in [s] + base]
|
||
if any(float(sims[0][j]) >= SEED_COVER_COS and negs[0] == negs[j]
|
||
for j in range(1, len(base) + 1)):
|
||
continue
|
||
fresh.append(s)
|
||
if not fresh:
|
||
return 0
|
||
nf_dir = work_dir / "nf"
|
||
nf_dir.mkdir(parents=True, exist_ok=True)
|
||
res = await _facts_block(ctx, lambda *a, **k: None, {**files, "arbeit": nf_dir},
|
||
{title: list(fresh)}, q, folder, instructions,
|
||
ns=f"{ns}nf-", lbl=lbl, sources=sources, slim=True)
|
||
if is_cancelled() or res is None:
|
||
return 0
|
||
nf_facts, discarded = res
|
||
dropped = (discarded or {}).get(title) or set()
|
||
nf_map = nf_facts.get(title) or {}
|
||
|
||
def _belegt(s: str) -> bool: # HARD gate: no facts entry = no evidence = no adoption
|
||
fk = nf_map.get(_norm_title(s))
|
||
return bool(fk and (fk.get("key_points") or fk.get("cited_facts")))
|
||
|
||
kept = [s for s in fresh if _norm_title(s) not in dropped and _belegt(s)]
|
||
if not kept:
|
||
return 0
|
||
bnorm = _norm_title(title)
|
||
bfacts = facts_map.setdefault(title, {})
|
||
for s in kept:
|
||
sn = _norm_title(s)
|
||
await db.upsert_subblock(topic, bnorm, sn, title, s)
|
||
await db.set_subblock_fields(topic, bnorm, sn, status="consensus")
|
||
bfacts[sn] = nf_map[sn]
|
||
raw.setdefault(title, []).extend(kept)
|
||
_log(topic, f"Lücken-Nachfass {title}: {len(kept)}/{len(fresh)} Funde übernommen")
|
||
return len(kept)
|
||
|
||
|
||
async def _facts_nachfass(ctx: GenContext, files: dict, raw: dict, facts_map: dict, q: dict,
|
||
folder, instructions: str = "", ns: str = "", lbl: str = "",
|
||
sources: list[str] | None = None) -> int:
|
||
"""ONE slim facts round for consensus subs WITHOUT a facts entry — renames during
|
||
consolidation and catalog rows left 35 % of the subs ungrounded; the fact gate then
|
||
flagged their (correct) guide statements wholesale. The subs themselves stay either
|
||
way: they exist by consensus, only the grounding is fetched. → count of filled subs."""
|
||
fehlend = {bt: [s for s in subs if _norm_title(s) not in (facts_map.get(bt) or {})]
|
||
for bt, subs in raw.items()}
|
||
fehlend = {bt: subs for bt, subs in fehlend.items() if subs}
|
||
if not fehlend:
|
||
return 0
|
||
nf_dir = files["arbeit"] / "nf2"
|
||
nf_dir.mkdir(parents=True, exist_ok=True)
|
||
res = await _facts_block(ctx, lambda *a, **k: None, {**files, "arbeit": nf_dir},
|
||
fehlend, q, folder, instructions,
|
||
ns=f"{ns}nf2-", lbl=lbl, sources=sources, slim=True)
|
||
if ctx.is_cancelled() or res is None:
|
||
return 0
|
||
nf_facts, _discarded = res # discard verdicts ignored — consensus subs are not removed here
|
||
filled = 0
|
||
for bt, fm in nf_facts.items():
|
||
bfacts = facts_map.setdefault(bt, {})
|
||
for sn, fk in fm.items():
|
||
if sn not in bfacts and (fk.get("key_points") or fk.get("cited_facts")):
|
||
bfacts[sn] = fk
|
||
filled += 1
|
||
if filled:
|
||
_log(ctx.topic, f"Facts-Nachfass{': ' + lbl if lbl else ''}{filled} Subs nachbelegt")
|
||
return filled
|
||
|
||
|
||
def _subs_hash(sidecar_or_raw: dict) -> str:
|
||
"""Sub-set identity for the resume files of the sub-CONSUMING stages (levels/relevance/
|
||
questions/artefacts). Without it a re-run with a recut sub set adopted the stale stage
|
||
results (measured: 626 orphans — artefacts of the old 425-sub set re-imported)."""
|
||
parts: list[str] = []
|
||
for title, subs in sidecar_or_raw.items():
|
||
parts.append(str(title))
|
||
for s in subs:
|
||
parts.append(s["title"] if isinstance(s, dict) else str(s))
|
||
return hashlib.md5("\n".join(parts).encode()).hexdigest()[:8]
|
||
|
||
|
||
def _code_vote(rater: list[dict], n: int) -> tuple[dict, dict]:
|
||
"""Majority vote over rater dicts on local ids 1..n → (outcome, disputed). A clear winner
|
||
needs ≥2 votes and no tie; otherwise the id is disputed (kept with its vote list)."""
|
||
outcome: dict[int, str] = {}
|
||
disputed: dict[int, list[str]] = {}
|
||
for k in range(1, n + 1):
|
||
vote_list = [d[k] for d in rater if k in d]
|
||
counter: dict[str, int] = {}
|
||
for s in vote_list:
|
||
counter[s] = counter.get(s, 0) + 1
|
||
best = max(counter.values(), default=0)
|
||
winners = [s for s, v in counter.items() if v == best]
|
||
if len(winners) == 1 and best >= 2:
|
||
outcome[k] = winners[0]
|
||
else:
|
||
disputed[k] = vote_list
|
||
return outcome, disputed
|
||
|
||
|
||
def _disputed_lines(items, item_idxs, disputed: dict) -> str:
|
||
"""Render disputed items as `k. [block] sub — Stimmen: a, b` lines for the judge prompt."""
|
||
return "\n".join(
|
||
f"{k}. [{items[item_idxs[k - 1]][0]}] {items[item_idxs[k - 1]][1]} — Stimmen: {', '.join(vote_list) or 'none'}"
|
||
for k, vote_list in disputed.items()
|
||
)
|
||
|
||
|
||
async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None:
|
||
"""Block C: three phases with a barrier — find (classify), select (vote), clarify.
|
||
Local IDs 1..n per package, mapped to global gid afterwards.
|
||
→ {block title: [{title, level}, …]} or None."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
# key points per sub as concise context (better-grounded classification; classification needs little).
|
||
facts_map = _json_file(files["facts"])
|
||
facts_map = facts_map if isinstance(facts_map, dict) else {}
|
||
items = [(title, sub) for title, subs in raw.items() for sub in subs] # global id = index+1
|
||
if not items:
|
||
return {title: [] for title in raw}
|
||
# pack chunks from WHOLE blocks (don't split a block) → each rater sees per block
|
||
# all subs and can classify relatively. Item indices per block in raw order.
|
||
chunks, cur, i = [], [], 0
|
||
for _title_b, subs in raw.items():
|
||
g = list(range(i, i + len(subs)))
|
||
i += len(subs)
|
||
if cur and len(cur) + len(g) > LEVEL_CHUNK:
|
||
chunks.append(cur)
|
||
cur = []
|
||
cur.extend(g)
|
||
if cur:
|
||
chunks.append(cur)
|
||
n = len(chunks)
|
||
sh = _subs_hash(raw) # resume must invalidate when the sub set changed
|
||
|
||
def rater_paths(c):
|
||
return [work_dir / f"level-{sh}-c{c}-{i}.json" for i in (1, 2, 3)]
|
||
|
||
def lset(item_idxs):
|
||
return set(range(1, len(item_idxs) + 1))
|
||
|
||
# Phase "Levels find": 3 raters per package (min. 2), local IDs.
|
||
async def _rate(c, item_idxs):
|
||
local_set = lset(item_idxs)
|
||
paths = rater_paths(c)
|
||
existing = sum(1 for p in paths if _levels_schema(_json_file(p), local_set))
|
||
if existing >= 2:
|
||
return True
|
||
enum_lines, cur_b = [], None
|
||
for k, j in enumerate(item_idxs, 1):
|
||
b, sub = items[j]
|
||
if b != cur_b:
|
||
enum_lines.append(f"\nBAUSTEIN: {b}")
|
||
cur_b = b
|
||
enum_lines.append(f"{k}. {sub}")
|
||
if (kz := _core_line(facts_map.get(b, {}).get(_norm_title(sub)))):
|
||
enum_lines.append(f" {kz}")
|
||
enum = "\n".join(enum_lines).strip()
|
||
pending = [(i, p) for i, p in enumerate(paths, 1) if not _levels_schema(_json_file(p), local_set)]
|
||
slots = [{
|
||
"key": f"blocks-{topic}-{ns}level-c{c}-{i}",
|
||
"prompt": _prompt("Levels-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)),
|
||
"role": "quick", "capabilities": "none", # pure mapping, everything inline → text reply
|
||
"payload": (lambda result, p=p, ids=local_set: _sink_json(result, p, lambda d: _levels_schema(d, ids))),
|
||
} for i, p in pending]
|
||
new = await _race(topic, f"{lbl}Levels package {c}", slots, 2 - existing, _timeout("level", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
return not is_cancelled() and new is not None
|
||
|
||
oks = await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Levels find"))
|
||
if is_cancelled():
|
||
return None
|
||
if not all(ok is True for ok in oks):
|
||
_blocks_errors[topic] = "Classification failed (research)"
|
||
return None
|
||
|
||
# Phase "Levels select": code vote per package → (outcome, disputed).
|
||
set_p(f"Levels select ({n} packages)…", step=_step_idx(topic, "Levels select"))
|
||
vote_by_c = {}
|
||
for c, item_idxs in enumerate(chunks, 1):
|
||
local_set = lset(item_idxs)
|
||
rater = [d for p in rater_paths(c) if (d := _levels_schema(_json_file(p), local_set))]
|
||
vote_by_c[c] = _code_vote(rater, len(item_idxs))
|
||
|
||
# Phase "Levels clarify": one judge per package on the disputed items, all in parallel.
|
||
async def _clarify(c, item_idxs):
|
||
outcome, strittig = vote_by_c[c]
|
||
if strittig:
|
||
judge_path = work_dir / f"level-final-{sh}-c{c}.json"
|
||
decision = _levels_schema(_json_file(judge_path), set(strittig))
|
||
if decision is None:
|
||
disputed_block = _disputed_lines(items, item_idxs, strittig)
|
||
status, decision = await run_single_slot(
|
||
ctx, f"{lbl}Levels-Clarification {c}",
|
||
key=f"blocks-{topic}-{ns}level-final-c{c}",
|
||
prompt=_prompt("Levels-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)),
|
||
role="judge", capabilities="none",
|
||
payload=lambda result, p=judge_path, ids=set(strittig): _sink_json(result, p, lambda d: _levels_schema(d, ids)),
|
||
timeout=_timeout("level_check", len(strittig)),
|
||
)
|
||
if status == FAILED:
|
||
_log(topic, f"Levels clarification package {c} failed — default 'advanced'")
|
||
decision = decision if isinstance(decision, dict) else {}
|
||
# disputed without a decision → 'advanced'; vote winners stay; judge overrides.
|
||
outcome = {**{k: "advanced" for k in strittig}, **outcome, **decision}
|
||
return {item_idxs[k - 1] + 1: level for k, level in outcome.items()}
|
||
|
||
parts = await _gather_progress([_clarify(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Levels clarify"))
|
||
if is_cancelled():
|
||
return None
|
||
level_by_id: dict[int, str] = {}
|
||
for c, part in enumerate(parts, 1):
|
||
if not isinstance(part, dict):
|
||
# clarification is not fatal: vote outcome + default 'advanced' for disputed.
|
||
if isinstance(part, BaseException):
|
||
_log(topic, f"Levels clarification package {c}: {type(part).__name__}: {part}")
|
||
outcome, strittig = vote_by_c[c]
|
||
item_idxs = chunks[c - 1]
|
||
merged = {**{k: "advanced" for k in strittig}, **outcome}
|
||
part = {item_idxs[k - 1] + 1: s for k, s in merged.items()}
|
||
level_by_id.update(part)
|
||
|
||
# assemble the sidecar — same order as items → gid matches
|
||
sidecar: dict[str, list[dict]] = {}
|
||
gid = 0
|
||
for title, subs in raw.items():
|
||
lst = []
|
||
for sub in subs:
|
||
gid += 1
|
||
lst.append({"title": sub, "level": level_by_id.get(gid, "advanced")})
|
||
sidecar[title] = lst
|
||
return sidecar
|
||
|
||
|
||
_FACTS_FIELDS = ("key_points", "prerequisites", "hurdles", "cited_facts", "example_idea")
|
||
|
||
|
||
def _facts_schema(data) -> list[dict] | None:
|
||
"""{"facts": [{block, subblock, …}]} → valid list · otherwise None.
|
||
Strictly separates belegte_facts (with source) from example_idee (generative)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("facts"), list):
|
||
return None
|
||
out = []
|
||
for e in data["facts"]:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
blk = str(e.get("block", "")).strip()
|
||
sub = str(e.get("subblock", "")).strip()
|
||
if not blk or not sub:
|
||
continue
|
||
bf = [{"text": t, "source": str(f.get("source", "")).strip()}
|
||
for f in (e.get("cited_facts") or []) if isinstance(f, dict) and (t := str(f.get("text", "")).strip())]
|
||
out.append({
|
||
"block": blk, "subblock": sub,
|
||
"key_points": [k for x in (e.get("key_points") or []) if (k := str(x).strip())],
|
||
"prerequisites": str(e.get("prerequisites", "")).strip(),
|
||
"hurdles": str(e.get("hurdles", "")).strip(),
|
||
"cited_facts": bf,
|
||
"example_idea": str(e.get("example_idea", "")).strip(),
|
||
})
|
||
return out or None
|
||
|
||
|
||
def _facts_check_schema(data) -> list[tuple[str, bool]] | None:
|
||
"""Facts check → [(sub_norm, verwerfen)] per objection · {ok:true}→[] · None if invalid.
|
||
verwerfen=True: sub not supportable in substance (remove). verwerfen=False: only correct the fact."""
|
||
if not isinstance(data, dict):
|
||
return None
|
||
if data.get("ok") is True:
|
||
return []
|
||
pr = data.get("problems")
|
||
if not isinstance(pr, list):
|
||
return None
|
||
return [(sn, bool(p.get("discard")))
|
||
for p in pr if isinstance(p, dict) and (sn := _norm_title(str(p.get("subblock", ""))))]
|
||
|
||
|
||
def _core_line(fk) -> str:
|
||
"""Concise key-point line for classification (level/relevance) — less context suffices there.
|
||
Empty if no facts/key points (legacy)."""
|
||
if not isinstance(fk, dict) or not fk.get("key_points"):
|
||
return ""
|
||
return "Kern: " + " · ".join(str(k) for k in fk["key_points"])
|
||
|
||
|
||
def _facts_lines(fk: dict) -> str:
|
||
z = []
|
||
if fk.get("key_points"):
|
||
z.append("Kernpunkte: " + " · ".join(str(k) for k in fk["key_points"]))
|
||
if fk.get("prerequisites"):
|
||
z.append("Voraussetzung: " + fk["prerequisites"])
|
||
if fk.get("hurdles"):
|
||
z.append("Hürde: " + fk["hurdles"])
|
||
for bf in fk.get("cited_facts", []):
|
||
z.append(f"FAKT: {bf['text']} (Source: {bf.get('source', '?')})")
|
||
if fk.get("example_idea"):
|
||
z.append("Example: " + fk["example_idea"])
|
||
return "\n".join(z)
|
||
|
||
|
||
async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str, ns: str = "", lbl: str = "", sources: list[str] | None = None, slim: bool = False) -> tuple | None:
|
||
"""Block: per sub extract source facts (find) → verify (check) → correct/discard (fix).
|
||
Extract-once grounding: the result feeds level/relevance/questions/guide.
|
||
slim=True (gap follow-up): no supplement pass, ONE check judge — the full program cost
|
||
230 agent-minutes per run for a handful of finds; the hard adoption gate stays.
|
||
→ (facts_map, discarded_map) — facts_map {block: {sub_norm: facts}}, discarded_map
|
||
{block: {sub_norm}} (unsupportable subs to remove) — or None on cancel/error."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
caps = "files" if folder else "full"
|
||
type = q.get("type", "thema")
|
||
source = _prompt(_SOURCE_TEMPLATE[type], project=folder) if type in _SOURCE_TEMPLATE else _prompt("Blocks-Source-Thema", topic=topic)
|
||
blocks = [(title, [str(s).strip() for s in subs if str(s).strip()]) for title, subs in raw.items() if subs]
|
||
if not blocks:
|
||
return {}, {}
|
||
chunks = _lpt_chunks([len(subs) for _, subs in blocks], FACTS_CHUNK_SUBS)
|
||
sh = _subs_hash(raw) # resume must invalidate when the sub set changed
|
||
|
||
def raw_path(ci): return work_dir / f"facts-{sh}-c{ci}.json"
|
||
def supp_path(ci): return work_dir / f"facts-erg-{sh}-c{ci}.json"
|
||
def chk_path(ci, j): return work_dir / f"facts-check-{sh}-c{ci}-j{j}.json"
|
||
def fix_path(ci): return work_dir / f"facts-fix-{sh}-c{ci}.json"
|
||
def ctitle(idxs): return [blocks[i][0] for i in idxs]
|
||
def block_text(idxs):
|
||
return "\n\n".join(
|
||
f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in blocks[i][1])
|
||
for i in idxs)
|
||
|
||
# raw facts of a chunk → {block: {sub_norm: {sub, …fields}}}, matched to chunk titles.
|
||
def raw_map(ci, path):
|
||
idxs = chunks[ci]
|
||
rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
|
||
ct = ctitle(idxs)
|
||
out: dict[str, dict] = {}
|
||
for e in _facts_schema(_json_file(path)) or []:
|
||
bt = _match_sub(e["block"], ct)
|
||
if bt not in rel_by:
|
||
continue
|
||
sub = _match_sub(e["subblock"], rel_by[bt])
|
||
out.setdefault(bt, {})[_norm_title(sub)] = {"sub": sub, **{k: e[k] for k in _FACTS_FIELDS}}
|
||
return out
|
||
|
||
# union raw facts + completeness supplements (recall): only subs that exist in raw.
|
||
def _chunk_facts(ci):
|
||
raw = raw_map(ci, raw_path(ci))
|
||
erg = raw_map(ci, supp_path(ci)) if supp_path(ci).exists() else {}
|
||
if not erg:
|
||
return raw
|
||
for bt, fm in raw.items():
|
||
ebt = erg.get(bt, {})
|
||
for sn, fk in fm.items():
|
||
ek = ebt.get(sn)
|
||
if not ek:
|
||
continue
|
||
seen = {str(k).strip().casefold() for k in fk.get("key_points", [])}
|
||
for k in ek.get("key_points", []):
|
||
if str(k).strip().casefold() not in seen:
|
||
seen.add(str(k).strip().casefold())
|
||
fk["key_points"].append(k)
|
||
seent = {bf["text"].strip().casefold() for bf in fk.get("cited_facts", [])}
|
||
for bf in ek.get("cited_facts", []):
|
||
if bf["text"].strip().casefold() not in seent:
|
||
seent.add(bf["text"].strip().casefold())
|
||
fk["cited_facts"].append(bf)
|
||
for f in ("prerequisites", "hurdles", "example_idea"):
|
||
if not fk.get(f) and ek.get(f):
|
||
fk[f] = ek[f]
|
||
return raw
|
||
|
||
def _inline_source(queries: list[str]) -> tuple[str, str]:
|
||
"""Korpus-Auszüge INLINE statt Datei-Recherche → (source, capabilities). Die
|
||
Tool-Agenten (bash/read) verloren sich messbar in Reasoning-Schleifen (bis 39k
|
||
Zeichen) und endeten mit leerem Turn — Retry-Wellen à 60–90 s seriell pro Block.
|
||
No-Tool-Calls mit Inline-Material hatten 0 solcher Fälle (Muster: Facts-Check).
|
||
Fail-open: ohne Treffer bleibt die alte Selbst-Recherche."""
|
||
ev = _evidence_pack(folder, sources, queries) if folder else ""
|
||
if ev:
|
||
return _prompt("Blocks-Source-Inline", excerpts=ev), "none"
|
||
return source, caps
|
||
|
||
# Phase "Facts find": 1 generator per chunk.
|
||
async def _find(ci, idxs):
|
||
fp = raw_path(ci)
|
||
if _facts_schema(_json_file(fp)):
|
||
return True
|
||
subs_total = sum(len(blocks[i][1]) for i in idxs)
|
||
f_source, f_caps = await asyncio.to_thread(
|
||
_inline_source, [blocks[i][0] for i in idxs] + [s for i in idxs for s in blocks[i][1]])
|
||
status, _r = await run_single_slot(
|
||
ctx, f"{lbl}Facts {ci}", key=f"blocks-{topic}-{ns}facts-c{ci}",
|
||
prompt=_prompt("Facts-Research", topic=topic, source=f_source, blocks=block_text(idxs), out_path=fp, extra=_extra(instructions)),
|
||
role="quick", capabilities=f_caps,
|
||
payload=lambda result, p=fp: _sink_or_file(result, p, _facts_schema),
|
||
timeout=_timeout("content", subs_total))
|
||
return status != FAILED and _facts_schema(_json_file(fp)) is not None
|
||
|
||
oks = await _gather_progress([_find(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts find"))
|
||
if is_cancelled():
|
||
return None
|
||
if not any(ok is True for ok in oks):
|
||
_blocks_errors[topic] = "Facts extraction failed"
|
||
return None
|
||
|
||
# Phase "Facts ergänzen" (recall): a targeted gap hunt per chunk looks for source-backed facts that
|
||
# the single pass missed. Best-effort — never fails (no erg file → merge uses only raw).
|
||
async def _supplement(ci, idxs):
|
||
ep = supp_path(ci)
|
||
if _facts_schema(_json_file(ep)):
|
||
return
|
||
per = raw_map(ci, raw_path(ci))
|
||
if not per:
|
||
return
|
||
block = "\n\n".join(
|
||
f"BLOCK: {bt}\nSUBBAUSTEINE (mit bereits erfassten Facts):\n" + "\n".join(
|
||
f"- {fk['sub']}\n Erfasst: " + ("; ".join(
|
||
list(fk.get("key_points", [])) + [bf["text"] for bf in fk.get("cited_facts", [])]) or "(nichts)")
|
||
for fk in fm.values())
|
||
for bt, fm in per.items())
|
||
subs_total = sum(len(blocks[i][1]) for i in idxs)
|
||
e_source, e_caps = await asyncio.to_thread(
|
||
_inline_source, [blocks[i][0] for i in idxs] + [s for i in idxs for s in blocks[i][1]])
|
||
await run_single_slot(
|
||
ctx, f"{lbl}Facts supplement {ci}", key=f"blocks-{topic}-{ns}facts-erg-c{ci}",
|
||
prompt=_prompt("Facts-Supplement", topic=topic, source=e_source, blocks=block, out_path=ep, extra=_extra(instructions)),
|
||
role="quick", capabilities=e_caps,
|
||
payload=lambda result, p=ep: _sink_or_file(result, p, _facts_schema),
|
||
timeout=_timeout("content", subs_total))
|
||
|
||
if not slim:
|
||
set_p("Facts supplement…", step=_step_idx(topic, "Facts find"))
|
||
await _gather_progress([_supplement(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts find"))
|
||
if is_cancelled():
|
||
return None
|
||
panel = (1,) if slim else (1, 2, 3)[:FACTS_CHECK_PANEL]
|
||
min_discard = 1 if slim else 2
|
||
|
||
# Phase "Facts check": FACTS_CHECK_PANEL judges per chunk. Two majority sets:
|
||
# flagged (fact inaccurate → correct) and discard (sub not supportable → remove).
|
||
async def _check(ci, idxs):
|
||
per = _chunk_facts(ci) # raw + supplements → panel verifies the union
|
||
if not per:
|
||
return ci, set(), set()
|
||
facts_text = "\n\n".join(f"SUBBAUSTEIN: {fk['sub']}\n{_facts_lines(fk)}" for fm in per.values() for fk in fm.values())
|
||
# Inline evidence: the EXACT cited regions (falls back to keyword excerpts) go into
|
||
# the prompt; the judge answers as TEXT, the engine persists the check file.
|
||
cites = [bf.get("source", "") for fm in per.values() for fk in fm.values()
|
||
for bf in fk.get("cited_facts", [])]
|
||
fallback = [blocks[i][0] for i in idxs] + [fk["sub"] for fm in per.values() for fk in fm.values()]
|
||
ev = _cited_evidence(folder, sources, cites, fallback) if folder else ""
|
||
c_source = _prompt("Blocks-Source-Inline", excerpts=ev) if ev else source
|
||
pending = [j for j in panel if _facts_check_schema(_json_file(chk_path(ci, j))) is None]
|
||
tmap = {asyncio.create_task(
|
||
run_agent(f"blocks-{topic}-{ns}facts-check-c{ci}-j{j}",
|
||
_prompt("Facts-Check", topic=topic, source=c_source, facts=facts_text, out_path=chk_path(ci, j), extra=_extra(instructions)),
|
||
_timeout("content_check", len(per)), provider=provider, role="judge",
|
||
capabilities="none" if ev else caps,
|
||
scope=topic, label=f"{lbl}Facts check {ci}/{j}")): j
|
||
for j in pending}
|
||
await _panel_2of3(tmap, lambda j, r: _sink_json(r, chk_path(ci, j), _facts_check_schema),
|
||
lambda: [s for j in panel
|
||
if (s := _facts_check_schema(_json_file(chk_path(ci, j)))) is not None],
|
||
lambda s: {tuple(x) for x in s})
|
||
outs = [s for j in panel if (s := _facts_check_schema(_json_file(chk_path(ci, j)))) is not None]
|
||
bvotes: dict[str, int] = {}
|
||
vvotes: dict[str, int] = {}
|
||
for s in outs: # s = [(sub_norm, verwerfen)] of one judge
|
||
gb, gv = set(), set()
|
||
for sn, disc in s:
|
||
if sn not in gb:
|
||
gb.add(sn); bvotes[sn] = bvotes.get(sn, 0) + 1
|
||
if disc and sn not in gv:
|
||
gv.add(sn); vvotes[sn] = vvotes.get(sn, 0) + 1
|
||
threshold = len(outs) / 2 if outs else 99
|
||
flagged = {sn for sn, v in bvotes.items() if v > threshold}
|
||
# Discarding is irreversible → stricter than flagging: majority AND ≥2 agreeing judges
|
||
# (prevents deletion by a single vote when the panel is degraded). slim runs ONE judge
|
||
# by design — there its single vote must be allowed to discard.
|
||
to_discard = {sn for sn, v in vvotes.items() if v > threshold and v >= min_discard}
|
||
return ci, flagged, to_discard
|
||
|
||
check = await _gather_progress([_check(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts check"))
|
||
if is_cancelled():
|
||
return None
|
||
flagged: dict[int, set] = {}
|
||
to_discard: dict[int, set] = {}
|
||
for r in check:
|
||
if isinstance(r, tuple) and len(r) == 3:
|
||
ci, b, v = r
|
||
flagged[ci] = b
|
||
to_discard[ci] = v
|
||
|
||
# Phase "Facts fix": re-extract only CORRECTABLE ones (flagged without discard).
|
||
correctable = {ci: (flagged.get(ci, set()) - to_discard.get(ci, set())) for ci in flagged}
|
||
n_problem = sum(len(s) for s in correctable.values())
|
||
if n_problem:
|
||
set_p(f"Correcting facts ({n_problem})…", step=_step_idx(topic, "Facts fix"))
|
||
async def _fix(ci):
|
||
subs_norm = correctable.get(ci, set())
|
||
if not subs_norm or _facts_schema(_json_file(fix_path(ci))):
|
||
return
|
||
idxs = chunks[ci]
|
||
rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
|
||
goal = []
|
||
for bt, subs in rel_by.items():
|
||
affected_subs = [s for s in subs if _norm_title(s) in subs_norm]
|
||
if affected_subs:
|
||
goal.append(f"BLOCK: {bt}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in affected_subs))
|
||
if not goal:
|
||
return
|
||
x_source, x_caps = await asyncio.to_thread(
|
||
_inline_source, [bt for bt in rel_by] + [s for subs in rel_by.values()
|
||
for s in subs if _norm_title(s) in subs_norm])
|
||
await run_single_slot(
|
||
ctx, f"{lbl}Facts-Fix {ci}", key=f"blocks-{topic}-{ns}facts-fix-c{ci}",
|
||
prompt=_prompt("Facts-Research", topic=topic, source=x_source, blocks="\n\n".join(goal), out_path=fix_path(ci), extra=_extra(instructions)),
|
||
role="quick", capabilities=x_caps,
|
||
payload=lambda result, p=fix_path(ci): _sink_or_file(result, p, _facts_schema),
|
||
timeout=_timeout("content", len(subs_norm)))
|
||
await _gather_progress([_fix(ci) for ci in correctable], len(correctable), _report_p(set_p, topic, "Facts fix"))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# assemble: raw + fix overrides for corrected. Discarded subs out (+ report per block).
|
||
outcome: dict[str, dict] = {}
|
||
discarded_map: dict[str, set] = {}
|
||
for ci in range(len(chunks)):
|
||
per = _chunk_facts(ci) # raw + supplements (recall); fix overrides only corrected
|
||
fix = raw_map(ci, fix_path(ci)) if fix_path(ci).exists() else {}
|
||
disc = to_discard.get(ci, set())
|
||
for bt, fm in per.items():
|
||
for sn, fk in fm.items():
|
||
if sn in disc:
|
||
discarded_map.setdefault(bt, set()).add(sn)
|
||
continue
|
||
winners = fix.get(bt, {}).get(sn, fk) if sn in correctable.get(ci, set()) else fk
|
||
outcome.setdefault(bt, {})[sn] = {k: winners[k] for k in _FACTS_FIELDS}
|
||
if discarded_map:
|
||
_log(topic, f"Facts check discards {sum(len(s) for s in discarded_map.values())} unsupportable subblocks")
|
||
return outcome, discarded_map
|
||
|
||
|
||
async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None:
|
||
"""Block D: three phases with a barrier — find (relevant/peripheral), select (vote), clarify.
|
||
Items from the sidecar; local IDs 1..n per package → global gid.
|
||
→ {gid: relevance} or None on cancel/research error. Default on gap/dispute: 'relevant'."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
items = [(title, sub["title"], sub.get("facts")) for title, subs in sidecar.items() for sub in subs] # global id = index+1
|
||
if not items:
|
||
return {}
|
||
chunks = _chunk_nums(list(range(len(items))), _n_chunks(len(items), LEVEL_CHUNK))
|
||
n = len(chunks)
|
||
sh = _subs_hash(sidecar) # resume must invalidate when the sub set changed
|
||
|
||
def rater_paths(c):
|
||
return [work_dir / f"relevance-{sh}-c{c}-{i}.json" for i in (1, 2, 3)]
|
||
|
||
def lset(item_idxs):
|
||
return set(range(1, len(item_idxs) + 1))
|
||
|
||
# Phase "Relevance find": 3 raters per package (min. 2), local IDs.
|
||
async def _rate(c, item_idxs):
|
||
local_set = lset(item_idxs)
|
||
paths = rater_paths(c)
|
||
existing = sum(1 for p in paths if _relevance_schema(_json_file(p), local_set))
|
||
if existing >= 2:
|
||
return True
|
||
enum_lines = []
|
||
for k, j in enumerate(item_idxs, 1):
|
||
enum_lines.append(f"{k}. [{items[j][0]}] {items[j][1]}")
|
||
if (kz := _core_line(items[j][2])):
|
||
enum_lines.append(f" {kz}")
|
||
enum = "\n".join(enum_lines)
|
||
pending = [(i, p) for i, p in enumerate(paths, 1) if not _relevance_schema(_json_file(p), local_set)]
|
||
slots = [{
|
||
"key": f"blocks-{topic}-{ns}relevance-c{c}-{i}",
|
||
"prompt": _prompt("Relevance-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)),
|
||
"role": "quick", "capabilities": "none", # pure mapping, everything inline → text reply
|
||
"payload": (lambda result, p=p, ids=local_set: _sink_json(result, p, lambda d: _relevance_schema(d, ids))),
|
||
} for i, p in pending]
|
||
new = await _race(topic, f"{lbl}Relevance package {c}", slots, 2 - existing, _timeout("relevance", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
return not is_cancelled() and new is not None
|
||
|
||
oks = await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Relevance find"))
|
||
if is_cancelled():
|
||
return None
|
||
if not all(ok is True for ok in oks):
|
||
_blocks_errors[topic] = "Relevance failed (research)"
|
||
return None
|
||
|
||
# Phase "Relevance select": code vote per package → (outcome, disputed).
|
||
set_p(f"Relevance select ({n} packages)…", step=_step_idx(topic, "Relevance select"))
|
||
vote_by_c = {}
|
||
for c, item_idxs in enumerate(chunks, 1):
|
||
local_set = lset(item_idxs)
|
||
rater = [d for p in rater_paths(c) if (d := _relevance_schema(_json_file(p), local_set))]
|
||
vote_by_c[c] = _code_vote(rater, len(item_idxs))
|
||
|
||
# Phase "Relevance clarify": one judge per package on the disputed items, all in parallel.
|
||
async def _clarify(c, item_idxs):
|
||
outcome, strittig = vote_by_c[c]
|
||
if strittig:
|
||
judge_path = work_dir / f"relevance-final-{sh}-c{c}.json"
|
||
decision = _relevance_schema(_json_file(judge_path), set(strittig))
|
||
if decision is None:
|
||
disputed_block = _disputed_lines(items, item_idxs, strittig)
|
||
status, decision = await run_single_slot(
|
||
ctx, f"{lbl}Relevance-Clarification {c}",
|
||
key=f"blocks-{topic}-{ns}relevance-final-c{c}",
|
||
prompt=_prompt("Relevance-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)),
|
||
role="judge", capabilities="none",
|
||
payload=lambda result, p=judge_path, ids=set(strittig): _sink_json(result, p, lambda d: _relevance_schema(d, ids)),
|
||
timeout=_timeout("relevance_check", len(strittig)),
|
||
)
|
||
if status == FAILED:
|
||
_log(topic, f"Relevance clarification package {c} failed — default 'relevant'")
|
||
decision = decision if isinstance(decision, dict) else {}
|
||
# disputed without a decision → 'relevant' (never accidentally exclude).
|
||
outcome = {**{k: "relevant" for k in strittig}, **outcome, **decision}
|
||
return {item_idxs[k - 1] + 1: rel for k, rel in outcome.items()}
|
||
|
||
parts = await _gather_progress([_clarify(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Relevance clarify"))
|
||
if is_cancelled():
|
||
return None
|
||
relevance_by_id: dict[int, str] = {}
|
||
for c, part in enumerate(parts, 1):
|
||
if not isinstance(part, dict):
|
||
# clarification is not fatal: vote outcome + default 'relevant' for disputed.
|
||
if isinstance(part, BaseException):
|
||
_log(topic, f"Relevance clarification package {c}: {type(part).__name__}: {part}")
|
||
outcome, strittig = vote_by_c[c]
|
||
item_idxs = chunks[c - 1]
|
||
merged = {**{k: "relevant" for k in strittig}, **outcome}
|
||
part = {item_idxs[k - 1] + 1: s for k, s in merged.items()}
|
||
relevance_by_id.update(part)
|
||
return relevance_by_id
|
||
|
||
|
||
def _match_sub(agent_sub: str, rel: list[str]) -> str:
|
||
"""Map the agent's subblock title to the matching relevant title — exact,
|
||
then normalized, then substring (the agent drops e.g. the prefix "Question: ").
|
||
No match → keep the agent title. This way NO pattern is lost to a title mismatch."""
|
||
if agent_sub in rel:
|
||
return agent_sub
|
||
an = _norm_title(agent_sub)
|
||
for r in rel:
|
||
rn = _norm_title(r)
|
||
if an and rn and (an == rn or an in rn or rn in an):
|
||
return r
|
||
return agent_sub
|
||
|
||
|
||
async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None:
|
||
"""Block E (chunks of 10): find (1 generator per ~10 blocks, parallel), select (code:
|
||
group per block + dedup), clarify (1 critic per chunk), check (catch-up round).
|
||
Assignment per entry via the `block` field (a chunk file carries several blocks).
|
||
→ {block title: [{subblock, question}, …]} or None on cancel."""
|
||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
# ALL subblocks (including peripheral) get a pattern — peripheral is testable in the FGuide level.
|
||
# facts_by: full facts context per sub (generation benefits — better questions).
|
||
blocks = []
|
||
facts_by: dict[tuple, dict] = {}
|
||
for title, subs in sidecar.items():
|
||
all_titles = []
|
||
for s in subs:
|
||
if isinstance(s, dict) and (st := str(s.get("title", "")).strip()):
|
||
all_titles.append(st)
|
||
if isinstance(s.get("facts"), dict):
|
||
facts_by[(title, _norm_title(st))] = s["facts"]
|
||
if all_titles:
|
||
blocks.append((title, all_titles))
|
||
if not blocks:
|
||
return {}
|
||
chunks = _lpt_chunks([len(rel) for _, rel in blocks], QUESTION_CHUNK_SUBS) # load-balanced by sub count
|
||
sh = _subs_hash(sidecar) # resume must invalidate when the sub set changed
|
||
|
||
def raw_path(ci):
|
||
return work_dir / f"question-pattern-{sh}-c{ci}.json"
|
||
|
||
def final_path(ci):
|
||
return work_dir / f"question-pattern-final-{sh}-c{ci}.json"
|
||
|
||
def _chunk_title(idxs):
|
||
return [blocks[i][0] for i in idxs]
|
||
|
||
# Phase "Questions find": 1 generator per chunk, all in parallel.
|
||
async def _find(ci, idxs):
|
||
fp = raw_path(ci)
|
||
if _question_pattern_chunk_schema(_json_file(fp)):
|
||
return # Resume
|
||
def _sub_line(bi, s):
|
||
line = f"- {s}"
|
||
fk = facts_by.get((blocks[bi][0], _norm_title(s)))
|
||
if fk and (ft := _facts_lines(fk)):
|
||
line += "\n" + "\n".join(" " + l for l in ft.split("\n"))
|
||
return line
|
||
block = "\n\n".join(
|
||
f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(_sub_line(i, s) for s in blocks[i][1])
|
||
for i in idxs
|
||
)
|
||
subs_total = sum(len(blocks[i][1]) for i in idxs)
|
||
status, _ = await run_single_slot(
|
||
ctx, f"{lbl}Question-Pattern {ci}",
|
||
key=f"blocks-{topic}-{ns}question-pattern-c{ci}",
|
||
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=block,
|
||
out_path=fp, extra=_extra(instructions)),
|
||
role="quick", capabilities="files",
|
||
payload=lambda result, p=fp: _sink_or_file(result, p, _question_pattern_chunk_schema),
|
||
timeout=_timeout("question_pattern", subs_total),
|
||
)
|
||
if status == FAILED:
|
||
_log(topic, f"Question pattern chunk {ci} failed — blocks in fallback (catch-up round/live)")
|
||
|
||
async def find_all(ci_list):
|
||
ci_list = list(ci_list)
|
||
await _gather_progress([_find(ci, chunks[ci]) for ci in ci_list], len(ci_list), _report_p(set_p, topic, "Questions find"))
|
||
|
||
await find_all(range(len(chunks)))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# Phase "Questions select": code — group chunk files per block, drop duplicates,
|
||
# loosely map block/subblock titles to the targets (discard nothing for a mismatch).
|
||
def _select_chunk(ci):
|
||
idxs = chunks[ci]
|
||
ctitle = _chunk_title(idxs)
|
||
rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
|
||
out, seen_set = {}, {}
|
||
for e in _question_pattern_chunk_schema(_json_file(raw_path(ci))) or []:
|
||
title = _match_sub(e["block"], ctitle)
|
||
if title not in rel_by:
|
||
continue # not assignable → discard
|
||
sub = _match_sub(e["subblock"], rel_by[title])
|
||
seen = seen_set.setdefault(title, set())
|
||
if sub in seen:
|
||
continue # exactly one pattern per subblock
|
||
seen.add(sub)
|
||
out.setdefault(title, []).append({"subblock": sub, "question": e["question"]})
|
||
return out
|
||
|
||
def _select_all(ci_list):
|
||
raw = {}
|
||
for ci in ci_list:
|
||
for title, eintraege in _select_chunk(ci).items():
|
||
raw.setdefault(title, []).extend(eintraege)
|
||
return raw
|
||
|
||
set_p("Questions select…", step=_step_idx(topic, "Questions select"))
|
||
raw_by_title = _select_all(range(len(chunks)))
|
||
|
||
# Phase "Questions clarify": 1 critic per chunk cleans up the tables (grouped by block).
|
||
async def _clarify(ci, idxs):
|
||
fp = final_path(ci)
|
||
if _question_pattern_chunk_schema(_json_file(fp)):
|
||
return # resume
|
||
block_texts = []
|
||
for i in idxs:
|
||
t = blocks[i][0]
|
||
eintraege = raw_by_title.get(t) or []
|
||
if not eintraege:
|
||
continue
|
||
lines = "\n".join(f"- ({e['subblock']}) {e['question']}" for e in eintraege)
|
||
block_texts.append(f"BLOCK: {t}\n{lines}")
|
||
if not block_texts:
|
||
return # nothing to clarify in this chunk
|
||
subs_total = sum(len(blocks[i][1]) for i in idxs)
|
||
status, _ = await run_single_slot(
|
||
ctx, f"{lbl}Question-Pattern-Clarification {ci}",
|
||
key=f"blocks-{topic}-{ns}question-pattern-final-c{ci}",
|
||
prompt=_prompt("Question-Pattern-Critique", topic=topic, table="\n\n".join(block_texts), out_path=fp, extra=_extra(instructions)),
|
||
role="judge", capabilities="none", # pure review, everything inline → text reply
|
||
payload=lambda result, p=fp: _sink_json(result, p, _question_pattern_chunk_schema),
|
||
timeout=_timeout("question_pattern_check", subs_total),
|
||
)
|
||
if status == FAILED:
|
||
_log(topic, f"Question pattern clarification chunk {ci} failed — raw pattern adopted")
|
||
|
||
async def clarify_all(ci_list):
|
||
ci_list = list(ci_list)
|
||
await _gather_progress([_clarify(ci, chunks[ci]) for ci in ci_list], len(ci_list), _report_p(set_p, topic, "Questions clarify"))
|
||
|
||
await clarify_all(range(len(chunks)))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# Clarified chunk table per block, fallback to raw pattern. Map titles loosely.
|
||
def _final_by_title(ci_list):
|
||
out = {}
|
||
for ci in ci_list:
|
||
idxs = chunks[ci]
|
||
ctitle = _chunk_title(idxs)
|
||
rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
|
||
for e in _question_pattern_chunk_schema(_json_file(final_path(ci))) or []:
|
||
title = _match_sub(e["block"], ctitle)
|
||
if title not in rel_by:
|
||
continue
|
||
out.setdefault(title, []).append(
|
||
{"subblock": _match_sub(e["subblock"], rel_by[title]), "question": e["question"]})
|
||
return out
|
||
|
||
final_by_title = _final_by_title(range(len(chunks)))
|
||
outcome = {t: (final_by_title.get(t) or raw_by_title.get(t) or []) for t, _ in blocks}
|
||
|
||
# Phase "Questions check": per-sub completeness. Generators crash randomly (~15 %),
|
||
# 1 agent per chunk without retry → subs (whole blocks) fall through silently. Hence several
|
||
# rounds that re-request ONLY the missing subs (short packages, Question-Pattern-Research).
|
||
set_p("Questions check…", step=_step_idx(topic, "Questions check"))
|
||
|
||
def _missing_subs() -> list[tuple[str, list[str]]]:
|
||
out = []
|
||
for t, subs in blocks:
|
||
have_set = {_norm_title(e["subblock"]) for e in outcome.get(t) or []}
|
||
miss = [s for s in subs if _norm_title(s) not in have_set]
|
||
if miss:
|
||
out.append((t, miss))
|
||
return out
|
||
|
||
def _followup_block(items): # items: [(block_title, [missing sub_title])]
|
||
block_texts = []
|
||
for title, subs in items:
|
||
lines = []
|
||
for s in subs:
|
||
z = f"- {s}"
|
||
fk = facts_by.get((title, _norm_title(s)))
|
||
if fk and (ft := _facts_lines(fk)):
|
||
z += "\n" + "\n".join(" " + l for l in ft.split("\n"))
|
||
lines.append(z)
|
||
block_texts.append(f"BLOCK: {title}\nSUBBAUSTEINE:\n" + "\n".join(lines))
|
||
return "\n\n".join(block_texts)
|
||
|
||
async def _request_more(round_n, pi, items):
|
||
fp = work_dir / f"question-pattern-nach{round_n}-{sh}-c{pi}.json"
|
||
if _question_pattern_chunk_schema(_json_file(fp)):
|
||
return # resume
|
||
subs_total = sum(len(s) for _, s in items)
|
||
await run_single_slot(
|
||
ctx, f"{lbl}Question pattern catch-up R{round_n}/{pi}",
|
||
key=f"blocks-{topic}-{ns}question-pattern-nach{round_n}-c{pi}",
|
||
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=_followup_block(items),
|
||
out_path=fp, extra=_extra(instructions)),
|
||
role="quick", capabilities="files",
|
||
payload=lambda result, p=fp: _sink_or_file(result, p, _question_pattern_chunk_schema),
|
||
timeout=_timeout("question_pattern", subs_total),
|
||
)
|
||
|
||
for round_n in range(1, QUESTION_MAX_ROUNDS + 1):
|
||
missing_subs = _missing_subs()
|
||
if not missing_subs:
|
||
break
|
||
n_subs = sum(len(s) for _, s in missing_subs)
|
||
_log(topic, f"Question pattern round {round_n}: {n_subs} sub(s) in {len(missing_subs)} block(s) without a pattern — re-request")
|
||
packages = _lpt_chunks([len(s) for _, s in missing_subs], QUESTION_CHUNK_SUBS)
|
||
package_items = [[missing_subs[i] for i in idxs] for idxs in packages]
|
||
await _gather_progress(
|
||
[_request_more(round_n, pi, items) for pi, items in enumerate(package_items)],
|
||
len(package_items), _report_p(set_p, topic, "Questions check"))
|
||
if is_cancelled():
|
||
return None
|
||
# parse output per package + merge newly gained subs (don't overwrite existing ones).
|
||
for pi, items in enumerate(package_items):
|
||
title_subs = {t: subs for t, subs in items}
|
||
ctitle = list(title_subs.keys())
|
||
for e in _question_pattern_chunk_schema(_json_file(work_dir / f"question-pattern-nach{round_n}-{sh}-c{pi}.json")) or []:
|
||
title = _match_sub(e["block"], ctitle)
|
||
if title not in title_subs:
|
||
continue
|
||
sub = _match_sub(e["subblock"], title_subs[title])
|
||
have_set = {_norm_title(x["subblock"]) for x in outcome.get(title) or []}
|
||
if _norm_title(sub) in have_set:
|
||
continue
|
||
outcome.setdefault(title, []).append({"subblock": sub, "question": e["question"]})
|
||
|
||
rest = _missing_subs()
|
||
if rest:
|
||
n = sum(len(s) for _, s in rest)
|
||
_log(topic, f"Question pattern: {n} sub(s) in {len(rest)} block(s) remain empty after {QUESTION_MAX_ROUNDS} rounds: {[t for t, _ in rest][:5]}")
|
||
return outcome
|
||
|
||
|
||
# ── Inventory in the DB: research loop · consolidation · clarification ────────────
|
||
|
||
|
||
|
||
def _crawl_index(folder) -> dict[str, str]:
|
||
"""Alias (filename OR QUELLE: URL, lowercase) → canonical page key (filename)."""
|
||
idx: dict[str, str] = {}
|
||
if not folder or not Path(folder).is_dir():
|
||
return idx
|
||
for p in sorted(Path(folder).glob("*.txt")):
|
||
key = p.name
|
||
idx[key.lower()] = key
|
||
try:
|
||
first_line = p.read_text(encoding="utf-8").splitlines()[0]
|
||
except (OSError, IndexError):
|
||
first_line = ""
|
||
if first_line.startswith("QUELLE:"):
|
||
url = first_line[len("QUELLE:"):].strip()
|
||
if url:
|
||
idx[url.lower()] = key
|
||
idx[url.rstrip("/").lower()] = key
|
||
return idx
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _triage_rules(folder, pages: list[str]) -> tuple[list[str], list[str]]:
|
||
"""Deterministic content/noise filter (config.CRAWL_*). Substring match (lowercase) against
|
||
URL + filename. Order: keep > noise > min_chars > keep. → (content, noise)."""
|
||
folder = Path(folder)
|
||
content, noise = [], []
|
||
for fn in pages:
|
||
lines = _read(folder / fn).splitlines()
|
||
url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else ""
|
||
body = "\n".join(lines[1:]).strip()
|
||
hay = f"{url}\n{fn}".lower()
|
||
if any(p in hay for p in CRAWL_KEEP_PATTERNS):
|
||
content.append(fn)
|
||
elif any(p in hay for p in CRAWL_NOISE_PATTERNS):
|
||
noise.append(fn)
|
||
elif len(body) < CRAWL_MIN_CHARS:
|
||
noise.append(fn)
|
||
else:
|
||
content.append(fn) # default: keep — everything with content stays
|
||
return content, noise
|
||
|
||
|
||
def _page_snippet(folder, fn: str) -> tuple[str, str]:
|
||
"""(url, snippet) of a crawl page for the relevance gate. url from the QUELLE: line;
|
||
snippet = body excerpt (navigation boilerplate is up front — the prompt ignores it).
|
||
The URL is the primary signal (meaningful slug), the snippet only supports it."""
|
||
lines = _read(Path(folder) / fn).splitlines()
|
||
url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else ""
|
||
body = "\n".join(lines[1:]).strip()
|
||
snippet = " ".join(body.split())[:QUELLE_RELEVANZ_SNIPPET]
|
||
return (url or fn), snippet
|
||
|
||
|
||
async def _relevance_triage(ctx: GenContext, set_p, files: dict, folder, content: list[str], spec: str, instructions: str) -> tuple[list[str], list[str]]:
|
||
"""LLM topic gate after the rule filter: each content page ja/nein against the spec.
|
||
Off-topic (different field) → out. Pattern like `_relevance_block`: small packages, 3 raters
|
||
(`fast`), 2-of-3 consensus. CONSERVATIVE: drop only on a clear "nein" majority; dispute/gap/
|
||
race error → keep. SAFETY: if the gate would drop ≥80 % (or all), everything stays
|
||
(a spec mismatch/bug must not empty the source). → (kept, out) as filenames."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
pages = sorted(content)
|
||
if not pages:
|
||
return content, []
|
||
items = [_page_snippet(folder, fn) for fn in pages] # index aligns with `pages`
|
||
chunks = _chunk_nums(list(range(len(pages))), _n_chunks(len(pages), QUELLE_RELEVANZ_CHUNK))
|
||
n = len(chunks)
|
||
|
||
def rater_paths(c):
|
||
return [work_dir / f"source-relevance-c{c}-{i}.json" for i in (1, 2, 3)]
|
||
|
||
def lset(idxs):
|
||
return set(range(1, len(idxs) + 1))
|
||
|
||
async def _rate(c, idxs):
|
||
local_set = lset(idxs)
|
||
paths = rater_paths(c)
|
||
existing = sum(1 for p in paths if _yesno_schema(_json_file(p), local_set))
|
||
if existing >= 2:
|
||
return True
|
||
enum_lines = []
|
||
for k, j in enumerate(idxs, 1):
|
||
url, snip = items[j]
|
||
enum_lines.append(f"{k}. {url}")
|
||
if snip:
|
||
enum_lines.append(f" {snip}")
|
||
enum = "\n".join(enum_lines)
|
||
pending = [(i, p) for i, p in enumerate(paths, 1) if not _yesno_schema(_json_file(p), local_set)]
|
||
slots = [{
|
||
"key": f"blocks-{topic}-source-relevance-c{c}-{i}",
|
||
"prompt": _prompt("Source-Relevance", topic=topic, spec=spec, pages=enum, out_path=p, extra=_extra(instructions)),
|
||
"role": "fast", "capabilities": "files",
|
||
"payload": (lambda result, p=p, ids=local_set: _yesno_schema(_json_file(p), ids)),
|
||
} for i, p in pending]
|
||
new = await _race(topic, f"Relevance triage package {c}", slots, 2 - existing, _timeout("relevance", len(idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
return not is_cancelled() and new is not None
|
||
|
||
_qidx = _step_idx(topic, "Source prep") # gate runs in the source step (no own step)
|
||
set_p(f"Check relevance against spec ({n} packages)…", step=_qidx)
|
||
async def _report_triage(d, t):
|
||
set_p(f"Check relevance against spec {d}/{t}…", step=_qidx)
|
||
await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], n, _report_triage)
|
||
if is_cancelled():
|
||
return content, [] # cancel → drop nothing (caller aborts)
|
||
|
||
# Vote per page: only a clear "nein" majority (≥2 and more than "ja") throws it out.
|
||
dropped: list[str] = []
|
||
for c, idxs in enumerate(chunks, 1):
|
||
local_set = lset(idxs)
|
||
rater = [d for p in rater_paths(c) if (d := _yesno_schema(_json_file(p), local_set))]
|
||
for k in range(1, len(idxs) + 1):
|
||
vote_list = [d[k] for d in rater if k in d]
|
||
nein, ja = vote_list.count("nein"), vote_list.count("ja")
|
||
if nein >= 2 and nein > ja:
|
||
dropped.append(pages[idxs[k - 1]])
|
||
|
||
if dropped and len(dropped) >= max(1, int(len(pages) * 0.8)):
|
||
_log(topic, f"Relevance triage: would drop {len(dropped)}/{len(pages)} — discarded (spec mismatch?), keeping all")
|
||
return content, []
|
||
dropped_set = set(dropped)
|
||
keepers = [fn for fn in pages if fn not in dropped_set]
|
||
return keepers, dropped
|
||
|
||
|
||
async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool:
|
||
"""Step "Source prep": crawl (link) + PDF convert + content/noise triage.
|
||
Persists the triage in the coverage table (content). → True (ok) / False (cancel/error).
|
||
thema: nothing. projekt/uni: only PDFs (curated folder, no triage)."""
|
||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||
if not folder:
|
||
return True # thema → no source to prepare
|
||
if q["type"] != "link":
|
||
await asyncio.to_thread(_convert_pdfs, folder) # projekt/uni: only PDFs, no triage
|
||
return True
|
||
if await db.get_step_status(topic, "Source prep") == "done":
|
||
return True
|
||
if not _crawl_done(topic):
|
||
set_p("Loading source (crawl)…", step=_step_idx(topic, "Source prep"))
|
||
n = await asyncio.to_thread(crawl, q["location"], folder, cancelled=is_cancelled)
|
||
if is_cancelled():
|
||
return False
|
||
if not n:
|
||
_blocks_errors[topic] = "Crawl yielded no content — check link/domain"
|
||
return False
|
||
await asyncio.to_thread(_convert_pdfs, folder)
|
||
pages = sorted(set(_crawl_index(folder).values()))
|
||
if pages:
|
||
set_p("Triaging pages…", step=_step_idx(topic, "Source prep"))
|
||
await db.delete_coverage(topic)
|
||
content, noise = _triage_rules(folder, pages) # deterministic rule filter
|
||
if q.get("spec") and content: # topic gate: separates the field (rules can't)
|
||
content, dropped = await _relevance_triage(ctx, set_p, files, folder, content, q["spec"], instructions)
|
||
if is_cancelled():
|
||
return False
|
||
if dropped:
|
||
noise = sorted(set(noise) | set(dropped))
|
||
_log(topic, f"LLM relevance: {len(dropped)} pages off-topic → noise")
|
||
await db.mark_content(topic, sorted(content), sorted(noise))
|
||
_log(topic, f"Triage: {len(content)} content / {len(noise)} noise of {len(pages)} (rules + LLM gate)")
|
||
await db.set_step_status(topic, "Source prep", "done")
|
||
return True
|
||
|
||
|
||
|
||
|
||
|
||
|
||
_ASPECT_MARKER = ("∈ np", "∈np", " in np", "np-schwer", "np-vollständig", "verifizierer",
|
||
"zertifikat", "ndtm", "nicht-determ", "lower bound", "untere schranke",
|
||
"bzgl", "als sprache", "beweis", "güte", "austausch",
|
||
# generic bound/limit/runtime property stems (a "…-Grenze"/"…-Schranke"/"…-Laufzeit"
|
||
# is a property OF a concept, not the concept — MDM survivorship must never pick it as
|
||
# the representative, so it scores >0 here like the other aspect markers):
|
||
"grenze", "schranke", "laufzeit")
|
||
|
||
|
||
def _aspect_marker(title: str) -> int:
|
||
"""Number of property markers in the title (∈NP, NP-hard, verifier, lower bound …).
|
||
0 = generic main concept (the problem itself); >0 = a property of it."""
|
||
t = title.casefold()
|
||
return sum(1 for m in _ASPECT_MARKER if m in t)
|
||
|
||
|
||
_REFERENCE_RE = re.compile(r'^(Satz|Lemma|Korollar|Bemerkung|Definition)\s*[\d.]+\s*(\([a-z]\)|[a-z])?\s*$', re.I)
|
||
|
||
|
||
def _is_reference(title: str) -> bool:
|
||
"""True for pure reference/placeholder titles WITHOUT meaningful content: "Satz 7.18", "Lemma 6.2",
|
||
"Korollar 6.18" (number without a name) as well as marked spots "Bedingung (**)". NOT "Satz 6.24:
|
||
Cook/Levin" (has a name) and NOT short technical symbols like "P⊆NP"/"Σ*" (real concepts)."""
|
||
t = title.strip()
|
||
if _REFERENCE_RE.match(t):
|
||
return True
|
||
if re.search(r'\(\*+\)', t): # marked spot "(**)" / "(*)"
|
||
return True
|
||
return False
|
||
|
||
|
||
def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) -> dict:
|
||
"""Representative of a cluster = the main concept (fewest property markers — the problem
|
||
itself, not "… ∈ NP"); tie → most frequent norm title → most readers. Title globally unique
|
||
(suffix ' (2)') so it works as a key."""
|
||
by_norm: dict[str, list[int]] = {}
|
||
for k in idxs:
|
||
by_norm.setdefault(_norm_title(candidates[k]["title"]), []).append(k)
|
||
|
||
def weight(nb: str):
|
||
ms = by_norm[nb]
|
||
reader = set().union(*[set(candidates[m]["reader"]) for m in ms]) if ms else set()
|
||
# reference/placeholder titles ("Satz 7.18") last — prefer a meaningful member.
|
||
is_real = not _is_reference(candidates[ms[0]]["title"])
|
||
return (is_real, -_aspect_marker(nb), len(ms), len(reader))
|
||
|
||
best = max(by_norm, key=weight)
|
||
k = max(by_norm[best], key=lambda m: len(candidates[m]["description"]))
|
||
title = candidates[k]["title"]
|
||
n = 2
|
||
while _norm_title(title) in seen_norm:
|
||
title = f"{candidates[k]['title']} ({n})"
|
||
n += 1
|
||
seen_norm.add(_norm_title(title))
|
||
return {"title": title, "description": candidates[k]["description"]}
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _pairs_schema(data) -> dict[int, bool] | None:
|
||
"""{"pairs": {"1": "ja", "2": "nein", …}} → {pair_nr: True/False} · otherwise None."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("pairs"), dict):
|
||
return None
|
||
out: dict[int, bool] = {}
|
||
for k, v in data["pairs"].items():
|
||
try:
|
||
nr = int(k)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
out[nr] = str(v).strip().casefold() in ("ja", "yes", "true", "1")
|
||
return out or None
|
||
|
||
|
||
def _cliques(n: int, edge_list: list[tuple[int, int]]) -> list[list[int]]:
|
||
"""Complete-link: greedy maximal cliques over the confirmed duplicate edges. A group
|
||
forms only if ALL its nodes are pairwise connected → no chaining (A=B + B=C forms
|
||
NO group {A,B,C} as long as A=C is missing). Only cliques ≥2 are returned."""
|
||
adj: dict[int, set[int]] = {i: set() for i in range(n)}
|
||
for a, b in edge_list:
|
||
adj[a].add(b)
|
||
adj[b].add(a)
|
||
used: set[int] = set()
|
||
groups: list[list[int]] = []
|
||
for v in sorted(range(n), key=lambda x: -len(adj[x])):
|
||
if v in used or not adj[v]:
|
||
continue
|
||
clique = {v}
|
||
for u in sorted(adj[v], key=lambda x: -len(adj[x])):
|
||
if u not in used and clique <= adj[u] | {u}: # u connected to ALL previous ones
|
||
clique.add(u)
|
||
if len(clique) >= 2:
|
||
groups.append(sorted(clique))
|
||
used |= clique
|
||
return groups
|
||
|
||
|
||
# Canonical-name blocking (dedup recall): entity resolution's recall ceiling is set by candidate
|
||
# generation — a pair that never shares a candidate can never be merged. Normalize a title to a
|
||
# scaffolding-free, operator-class-normalized, order-independent key so surface variants of ONE entity
|
||
# collapse ("Offenes Problem P=NP?" ≡ "P vs NP" ≡ "P=NP"). Generic (no course terms): strip a small
|
||
# stoplist of catalogue/wrapper words, fold the relation operators into class tokens, sort content tokens.
|
||
_CANON_STOP = re.compile(
|
||
r'\b(?:offenes?|open|problem|frage|question|algorithmus|algorithm|verfahren|method|methode|'
|
||
r'satz|theorem|lemma|korollar|definition|def|das|der|die|the|ein|eine|einen|a|an|'
|
||
r'von|of|für|for|und|and|zum|zur|im)\b', re.I)
|
||
|
||
|
||
# Catalogue references ("Definition 6.19", "Satz 7.8") are scaffolding INCLUDING their number —
|
||
# stripped as a phrase, so the digits don't pollute the key. Variant digits ("3-SAT") have no
|
||
# scaffolding word in front and survive.
|
||
_CANON_CATALOGUE = re.compile(
|
||
r'\b(?:definition|def|satz|lemma|korollar|corollary|theorem|proposition'
|
||
r'|kapitel|chapter|abschnitt|section)\s*\d+(?:\.\d+)*\b', re.I)
|
||
|
||
|
||
_PAREN_GROUP = re.compile(r'^\s*(.*?)\s*\(([^()]{2,60})\)\s*$')
|
||
|
||
|
||
def _title_variants(title: str) -> set[str]:
|
||
"""Acronym/expansion variants of a "X (Y)" title — normalized outer part and paren
|
||
content. „Satisfiability Problem (SAT)" → {'satisfiability problem', 'sat'}: matched
|
||
against another card's norm/key this makes acronym↔expansion pairs dedup CANDIDATES
|
||
(measured: title cosine 'sat' vs the long form is 0.53, far below the floor).
|
||
Titles without exactly one trailing paren group → empty set."""
|
||
from textkit import _norm_title
|
||
m = _PAREN_GROUP.match(title or "")
|
||
if not m:
|
||
return set()
|
||
return {v for v in (_norm_title(m.group(1)), _norm_title(m.group(2))) if v}
|
||
|
||
|
||
def _canonical_key(title: str) -> str:
|
||
"""Order-independent canonical key of a title (scaffolding stripped, relation operators normalized).
|
||
Two titles with the same key denote the same entity with ~100% precision (ER blocking). Empty string
|
||
if nothing survives (never auto-merged)."""
|
||
s = unicodedata.normalize("NFKC", title)
|
||
s = re.sub(r'([a-zäöüß])([A-ZÄÖÜ])', r'\1 \2', s) # CamelCase → two tokens
|
||
s = s.casefold()
|
||
s = _CANON_CATALOGUE.sub(' ', s)
|
||
s = re.sub(r'≟|\bversus\b|\bvs\.?\b|=', ' opeq ', s) # equality / "vs" → one token
|
||
s = re.sub(r'≤|⪯|→|⇒|⟹|\breduces?\s+to\b|\breduziert\b', ' opred ', s) # reduction → one token
|
||
s = _CANON_STOP.sub(' ', s)
|
||
s = re.sub(r'[^\w ]', ' ', s) # drop punctuation/symbols
|
||
return " ".join(sorted(t for t in s.split() if t))
|
||
|
||
|
||
# Relation-triple individuation (dedup precision): a reduction/relation "A ≤ B" is identified by BOTH
|
||
# operands AND direction (RDF-triple identity / SKOS narrowMatch — a subset/restriction is NOT the same).
|
||
# So "SAT ≤ Clique" ≠ "3-SAT ≤ Clique" (source differs) and "A → B" ≠ "B → A" (direction). Two DIFFERENT
|
||
# relations must never merge, even if a judge or a high title-cosine says so.
|
||
_REL_STRIP = re.compile(r'^\s*(?:satz|lemma|korollar|corollary|theorem|proposition)\s*[\d.]*\s*:?\s*'
|
||
r'|^\s*redu[ck]tion(?:en|s)?\s*:?\s*', re.I)
|
||
# trailing scaffolding ("SetCover ≤ HittingSet Reduktion") is NOT part of the target operand —
|
||
# without this strip the guard false-alarms and blocks the correct merge with the bare relation
|
||
_REL_STRIP_TAIL = re.compile(r'[\s\-–]*(?:redu[ck]tion(?:en|s)?|transformation(?:en|s)?)\s*$', re.I)
|
||
# an attached p/m marker ("≤p", "≤m") is operator notation, not part of the right operand
|
||
_REL_OPERATOR = re.compile(r'[≤⪯≥⊆⊊→⇒⟹⇔←][pm]?|=>|<=|->', re.I)
|
||
|
||
|
||
def _relation_operands(title: str) -> tuple[str, str] | None:
|
||
"""(canonical_source, canonical_target) of a relation/reduction title, else None (not a relation).
|
||
Operands canonicalized (lowercased, non-alphanumerics stripped) so spacing/hyphenation don't matter."""
|
||
t = _REL_STRIP.sub('', title, count=1)
|
||
t = _REL_STRIP_TAIL.sub('', t, count=1)
|
||
m = _REL_OPERATOR.search(t)
|
||
if not m:
|
||
return None
|
||
left = re.sub(r'[\W_]', '', t[:m.start()].casefold()) # keep unicode letters/digits (umlauts), drop the rest
|
||
right = re.sub(r'[\W_]', '', t[m.end():].casefold())
|
||
if not left or not right:
|
||
return None
|
||
return (left, right)
|
||
|
||
|
||
def _relation_conflict(title_a: str, title_b: str) -> bool:
|
||
"""True if BOTH titles are relations/reductions but denote DIFFERENT ones (operands or direction
|
||
differ) → they must NOT be merged. False if either is not a relation, or they are the same relation."""
|
||
a, b = _relation_operands(title_a), _relation_operands(title_b)
|
||
return a is not None and b is not None and a != b
|
||
|
||
|
||
|
||
|
||
def _filter_schema(data) -> dict[int, int] | None:
|
||
"""{"fragments": {"3": 7, "12": 8}} → {block_nr: parent_nr} · None on invalid structure.
|
||
Empty dict = valid (nothing to degrade). Parent ≠ itself."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("fragments"), dict):
|
||
return None
|
||
out: dict[int, int] = {}
|
||
for k, v in data["fragments"].items():
|
||
try:
|
||
nr, parent = int(k), int(v)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if nr != parent:
|
||
out[nr] = parent
|
||
return out
|
||
|
||
|
||
# Pure notation/symbols without a standalone concept — kept narrow (FP~0, checked against aak;
|
||
# "KNF"/"MST"/"NP" do NOT match). These are discarded autonomously (need no parent).
|
||
_FILTER_NOTATION = re.compile(r'^\s*\|.{1,6}\|\s*$|^Güte\s+\d+\s*$')
|
||
# Exercise-sheet / cross-reference artefacts — the SECOND gate for a judge `drop` verdict: a hard-drop
|
||
# (parentless removal) only fires when the judge lists the number in `drop` AND `_is_artifact(title)`.
|
||
# A judge-drop without a match degrades to "keep" (logged), never deleted (FP~0 discipline like
|
||
# _FILTER_NOTATION). Shape-matching on a NORMALIZED, title-side string (see _is_artifact): four branches —
|
||
# (1) lettered/Roman/numeric sub-claims "(Aussage i)"/"Aussage (a)"/"Teil (b)"/"Fall (2)" in ANY
|
||
# parenthesization; (2) sheet refs "Blatt 10"/"Aufgabe 3"/"Übung"; (3) worked-example/table/figure refs
|
||
# "Beispiel Tab. 7.1" (a NUMBER is required — guards polysemes "Hash-Tabelle"/"Bijektive Abbildung");
|
||
# (4) parenthesized "(Variante)". Theorem words (Satz/Definition/Lemma) are deliberately NOT in the
|
||
# vocabulary, so "Satz 6.24 Cook/Levin — SAT ist NP-vollständig" is kept. Word boundaries guard prefixes
|
||
# (Aussagenlogik/Teilmenge/Blattknoten/Fallunterscheidung).
|
||
_FILTER_ARTIFACT = re.compile(r"""
|
||
\b(?:aussage|teil|behauptung|fall)\b[\s(]*(?:[ivx]{1,4}|[a-z]|\d{1,2})\)?(?![a-zäöüß])
|
||
| \b(?:blatt|aufgabe|serie|hausaufgabe)\s*\d+(?:[.,]\d+)*
|
||
| \b(?:übungsblatt|übung|uebung)\b
|
||
| \b(?:beispiel|abbildung|abb|tabelle|tab|bild|grafik|diagramm|figur|skizze)\b\.?\s*\d+(?:[.,]\d+)*
|
||
| \(\s*(?:variante|variation|spezialfall|sonderfall)\s*\)
|
||
""", re.VERBOSE)
|
||
|
||
|
||
def _is_artifact(title: str) -> bool:
|
||
"""True if the TITLE looks like exercise-sheet / cross-reference scaffolding (P1 hard-drop gate).
|
||
NFKC + casefold normalization (position/case/umlaut/Unicode-invariant: folds Ⅱ→ii, full-width, NBSP),
|
||
then match only the title side of the em-dash — a real concept whose *description* merely cites a
|
||
sheet ("… — vgl. Aufgabe 3") is never dropped. Normalize for matching only; never store the result."""
|
||
norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip()
|
||
head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0] # spaced dash only → "3-SAT"/"np-schwer" unsplit
|
||
return bool(_FILTER_ARTIFACT.search(head))
|
||
# Property/runtime suspicion — marks lines for the judge's verdict (NO auto-drop, FP too high:
|
||
# "NP-Schwere", reductions with "∈NP" are real blocks). Complements _aspekt_marker.
|
||
_FILTER_PREDICATE = re.compile(
|
||
r'ist NP-(vollständig|schwer)|NP-(Vollständigkeit|Schwere) von|ETH (Konsequenz|Lower Bound)'
|
||
r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP'
|
||
# fragment families that the aspect substrings miss (all title/desc, advisory ⚠ only):
|
||
r'|\bSatz\s+\d|\bLemma\s+\d|\bKorollar\s+\d' # bare theorem/proof references (with number)
|
||
r'|^\s*(?:Remark|Bemerkung|Anmerkung|Note|Notiz|Beobachtung|Observation)\b' # EN+DE remark labels
|
||
r'|\bSatz\s*:|\bSatz\s+[A-Z]\b' # "Satz:" (colon, no number) / "Satz D*" letter label
|
||
r'|\bGegenbeispiel\b|\bWorst[- ]?Case\b|Schärfe\s+der\b' # proof-example / sharpness facets
|
||
r'|2\s*\^\s*[{(]?\s*[Ωωoο]\s*\(' # ETH exponential bound 2^Ω(…)/2^o(…)
|
||
r'|\d\s*[−–-]\s*1\s*/\s*m' # approximation-güte ratio "2 − 1/m"
|
||
r'|Variablenungleichung|[α-ωΑ-Ω][a-z]?-?Variablen', re.I) # proof variables (αu-Variablen …)
|
||
|
||
|
||
def _filter_suspect(b: dict) -> bool:
|
||
"""Heuristic flag: could be a property/detail of another block."""
|
||
return _aspect_marker(b["title"]) > 0 or bool(_FILTER_PREDICATE.search(f"{b['title']} {b['description'] or ''}"))
|
||
|
||
|
||
def _root(nr: int, fragments: dict[int, int]) -> tuple[int, bool]:
|
||
"""Follow the fragment→parent chain to the first ancestor that is NOT itself a fragment.
|
||
Returns (root, cyclic). This walk IS the fixpoint (no separate iteration): a mid-tier
|
||
fragment whose own parent is also a fragment resolves to the top-level real block, so all
|
||
chain levels collapse in one pass (unlike the old single-level parent_set shield).
|
||
Cycle guard: on revisiting a node returns (node, True) → caller keeps both (no annihilation)."""
|
||
seen: set[int] = set()
|
||
cur = nr
|
||
while cur in fragments:
|
||
if cur in seen:
|
||
return cur, True
|
||
seen.add(cur)
|
||
cur = fragments[cur]
|
||
return cur, False
|
||
|
||
|
||
def _containment_parent(frag_norm: str, others: list[tuple[int, str]]) -> int | None:
|
||
"""Deterministic parent-by-name-containment for a ⚠-flagged survivor: a fragment whose title NAMES
|
||
another block ("Lower Bound … für VERTEX COVER" → Vertex Cover; "List Scheduling Güte …" → List
|
||
Scheduling). `others` = (nr, title_norm) of the OTHER blocks. A parent is a block whose normalized
|
||
title occurs as a WHOLE-WORD span inside `frag_norm`, is SIGNIFICANT (≥2 tokens or ≥6 chars — never
|
||
"p"/"np"/"sat") and strictly shorter than the fragment. Returns the parent nr on EXACTLY ONE match,
|
||
else None (0 or ≥2 → leave to the judge). Whole-word + significance + exactly-one → near-FP-0."""
|
||
hits = []
|
||
for nr, ptitle in others:
|
||
if not ptitle or ptitle == frag_norm or len(ptitle) >= len(frag_norm):
|
||
continue
|
||
if len(ptitle) < 6 and ptitle.count(" ") < 1: # reject short single-token names (P/NP/SAT)
|
||
continue
|
||
if re.search(r'(?<!\w)' + re.escape(ptitle) + r'(?!\w)', frag_norm):
|
||
hits.append(nr)
|
||
return hits[0] if len(hits) == 1 else None
|
||
|
||
|
||
# Two provably-noise, parent-less fragment classes safe to hard-drop (precision ~≥0.95): a bare
|
||
# label reference with an empty/thin residual ("Remark 7.28", "Satz D*", "Lemma 3.2") and a single-
|
||
# variable notation assignment ("r = n + m"). Matched on the normalized title HEAD (before the em-dash).
|
||
_PARENTLESS_NOISE = re.compile(
|
||
r'^\s*(?:remark|bemerkung|anmerkung|note|satz|lemma|korollar|proposition|folgerung)\s*[\d.]*\s*[a-z]?\*?\s*$'
|
||
# single-var arithmetic assignment ("r = n + m") — NOT a tuple/set/language/cardinality definition
|
||
# ("T = (Q,…)", "L = {…}", "n = |V|"): the RHS must not open with a bracket/pipe.
|
||
r'|^\s*[a-z](?:_?[a-z0-9])?\s*:?=\s*(?![({\[|])\S', re.I)
|
||
# KEEP-guards: never drop a named theorem WITH an author, a complexity-class (in)equality, or a Definition.
|
||
_PARENTLESS_KEEP = re.compile(
|
||
r'\b(?:cook|levin|karp|savitch|ladner|immerman|rice|håstad|hastad|christofides|dijkstra|bellman|'
|
||
r'ford|kruskal|prim|edmonds|blum|sipser|papadimitriou)\b'
|
||
r'|\b(?:p|np|conp|nl|pspace|exp|nexp|bpp|rp|zpp)\b|⇔|⇒|\bdefinition\b', re.I)
|
||
|
||
|
||
def _is_parentless_noise(title: str) -> bool:
|
||
"""True for the narrow, parent-less noise classes safe to hard-drop (subject to KEEP-guards)."""
|
||
norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip()
|
||
head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0]
|
||
return bool(_PARENTLESS_NOISE.search(head)) and not _PARENTLESS_KEEP.search(norm)
|
||
|
||
|
||
# F1 — statement-gate keep-guard (Knowledge-Component / OMDoc theory): a self-contained ASSERTION is its
|
||
# own learning unit, not a whole-part fragment. Protect two general classes from demotion:
|
||
# (a) a named reduction between two PROBLEMS ("3-SAT ≤ Clique", "Clique → Vertex Cover"), and
|
||
# (b) a LABELED or ATTRIBUTED theorem carrying its own biconditional/implication ("Satz 6.37: … ⇔ …").
|
||
# NOT protected: a bare label ("Satz 7.18", "Remark 7.28"), unary status ("X ist NP-vollständig", "X ∈ NP"),
|
||
# a proof-size step ("Strikte Reduktion |A| = O(m)"), or a güte/bound facet — those carry neither a
|
||
# two-sided reduction operator NOR a ⇔/⇒ assertion. General: attribution is structural, no author whitelist.
|
||
_STMT_LABEL = re.compile(r'^\s*(?:satz|lemma|korollar|theorem|proposition|prop|folgerung)\b', re.I)
|
||
_STMT_ATTRIB = re.compile(r'\b(?:satz|lemma|theorem|korollar)\s+von\s+[A-ZÄÖÜ]') # "Satz von Cook/Levin"
|
||
_STMT_ASSERT = re.compile(r'⇔|⇒|⟺|⟹|\bgdw\.?\b|\bgenau dann\b', re.I)
|
||
_REDUCTION_ONLY_OP = re.compile(r'[≤⪯]|→|⇒|⟹|->') # genuine reduction operators (NOT plain "=")
|
||
|
||
|
||
def _is_reduction_statement(title: str) -> bool:
|
||
"""True if the title is a reduction between two NAMED problems (both sides carry ≥3 letters and
|
||
neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP" and proof-size "|A| = O(m)"."""
|
||
t = _REL_STRIP.sub('', title, count=1)
|
||
m = _REDUCTION_ONLY_OP.search(t)
|
||
if not m:
|
||
return False
|
||
|
||
def _named(s: str) -> bool:
|
||
return len(re.findall(r'[a-zäöüß]', s, re.I)) >= 3 and not re.match(r'\s*[Oo]\s*\(', s)
|
||
|
||
return _named(t[:m.start()]) and _named(t[m.end():])
|
||
|
||
|
||
def _is_named_statement(title: str, desc: str = "") -> bool:
|
||
"""Statement-gate keep-guard: a named reduction (a) or a labeled/attributed theorem WITH its own
|
||
⇔/⇒ assertion (b). Bare labels / unary status / proof-size steps return False (stay demotable)."""
|
||
if _is_reduction_statement(title):
|
||
return True
|
||
if (_STMT_LABEL.match(title) or _STMT_ATTRIB.search(title)) and _STMT_ASSERT.search(f"{title} {desc or ''}"):
|
||
return True
|
||
return False
|
||
|
||
|
||
|
||
|
||
def _umbrella_schema(data, ids: set[int]):
|
||
"""{"umbrellas":[{"title":str,"description":str,"members":[int,…]}, …]}
|
||
→ [(title, description, [member ids])]. [] = valid (no umbrella); None ONLY on broken JSON
|
||
(so resume treats a valid-but-empty file as done, like _filter_schema's {} vs None). Each id
|
||
used at most once across all umbrellas (first wins); members filtered to `ids`; an umbrella
|
||
needs ≥2 surviving members; `description` required + non-empty — it MUST enumerate the children,
|
||
else the source-scoped subblock step can't re-derive them (no web on uni)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("umbrellas"), list):
|
||
return None
|
||
out, used = [], set()
|
||
for u in data["umbrellas"]:
|
||
if not isinstance(u, dict):
|
||
continue
|
||
title = str(u.get("title", "")).strip()
|
||
desc = str(u.get("description", "")).strip()
|
||
raw_members = u.get("members")
|
||
if not title or not desc or not isinstance(raw_members, list):
|
||
continue
|
||
members = []
|
||
for x in raw_members:
|
||
try:
|
||
m = int(x)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if m in ids and m not in used:
|
||
used.add(m)
|
||
members.append(m)
|
||
if len(members) >= 2:
|
||
out.append((title, desc, members))
|
||
return out
|
||
|
||
|
||
def _completion_schema(data, n_umbrellas: int, ids: set[int]):
|
||
"""{"additions":[{"umbrella":int,"members":[int,…]}, …]} → [(umbrella_idx, [member ids])].
|
||
[] = valid (nothing to absorb); None only on broken JSON. umbrella idx in range; members drawn
|
||
from `ids` (the still-standalone leftovers), de-duped within an addition."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("additions"), list):
|
||
return None
|
||
out = []
|
||
for a in data["additions"]:
|
||
if not isinstance(a, dict):
|
||
continue
|
||
try:
|
||
k = int(a.get("umbrella"))
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if not (0 <= k < n_umbrellas):
|
||
continue
|
||
mem = []
|
||
for x in (a.get("members") or []):
|
||
try:
|
||
m = int(x)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if m in ids and m not in mem:
|
||
mem.append(m)
|
||
if mem:
|
||
out.append((k, mem))
|
||
return out
|
||
|
||
|
||
# Deterministic backstop to the grouping judge's TEST 1 (type gate): an umbrella may bundle ONLY
|
||
# constituent sub-definitions of ONE definition. If a member title carries a standalone-unit signal
|
||
# (a named algorithm / problem / reduction / theorem), the umbrella is dissolved — those stay their own
|
||
# blocks. Kept narrow so real definition-parts (Konfiguration, Übergangsfunktion δ, Literale, Makespan,
|
||
# m Maschinen) never match; checked against the aak over-merge (member „Greedy-Algorithmus GA" hits).
|
||
# Suffix-anchored head nouns (German compounds are head-final: „Approximations+algorithmus" has NO word
|
||
# boundary before „algorithmus", so \bAlgorithmus\b misses it → the MAX-SAT over-merge). \w* absorbs the
|
||
# modifier; the head noun stays the discriminator. FP-safe: no real TM/KNF definition-part ends in these
|
||
# heads (Berechnung is deliberately NOT a head → „Akzeptierende Berechnung" stays a valid member).
|
||
_GROUP_STANDALONE = re.compile(
|
||
r'\w*algorithm(?:us|en)\b|\w*problem(?:e|s|en)?\b|\w*reduktion(?:en)?\b|\bscheduling\b|[≤⪯]'
|
||
r'|^\s*(?:Satz|Lemma|Korollar|Theorem|Bemerkung|Beobachtung)\s*\d'
|
||
# atomicity: a named COMPLEXITY CLASS / a "…-Vollständigkeit(completeness)" / a "…Transformation" is a
|
||
# self-contained concept (learning-object / atomic-KC), never a sub-definition — a bundle of ≥1 such
|
||
# member is siblings, not one model → dissolve (catches the P/NP/NP-Vollständigkeit over-merge that NO
|
||
# cosine floor separates). Head-final compounds (\w*klasse absorbs "Komplexitäts+klasse"); FP-safe —
|
||
# no real TM/KNF/TSP/Scheduling definition-part carries these heads.
|
||
r'|\w*vollständigkeit\b|\w*completeness\b|\w*transformation(?:en)?\b|\w*klasse[nr]?\b', re.I)
|
||
|
||
|
||
|
||
|
||
# --- Outline (blocks artifact: chapter structure, only read by the guide) ---
|
||
|
||
def _outline_review_schema(data, valid: set[int], n_chapters: int, n_blocks: int):
|
||
"""{"moves": {"<blocknr>": <chapter-idx>}} → {nr: idx} (may be {}) · None if broken/invalid.
|
||
A mass rewrite (more than a third of all blocks) is rejected — the reviewer's job is
|
||
spotting misplacements, not re-designing the outline."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("moves"), dict):
|
||
return None
|
||
out: dict[int, int] = {}
|
||
for k, v in data["moves"].items():
|
||
try:
|
||
nr, ch = int(k), int(v)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
if nr not in valid or not (1 <= ch <= n_chapters):
|
||
return None
|
||
out[nr] = ch
|
||
if len(out) * 3 > n_blocks:
|
||
return None
|
||
return out
|
||
|
||
|
||
def _outline_schema(data, valid: set[int]):
|
||
"""{"chapters":[{title,numbers}]} → cleaned (valid numbers, each exactly once) ·
|
||
None at <80 % coverage (agent/judge omitted too much)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("chapters"), list):
|
||
return None
|
||
out, seen = [], set()
|
||
for ch in data["chapters"]:
|
||
if not isinstance(ch, dict):
|
||
continue
|
||
title = str(ch.get("title", "")).strip() or "Chapter"
|
||
nums = []
|
||
for n in (ch.get("numbers") or []):
|
||
try:
|
||
n = int(n)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if n in valid and n not in seen:
|
||
seen.add(n)
|
||
nums.append(n)
|
||
if nums:
|
||
out.append({"title": title, "numbers": nums})
|
||
if not out or len(seen) < 0.8 * len(valid):
|
||
return None
|
||
return {"chapters": out}
|
||
|
||
|
||
def _prereq_schema(data, valid: set[int]) -> dict[int, list[int]]:
|
||
"""{"prereqs": {"3": [1, 7]}} → {num: [prereq nums]} · only numbers from `valid`, no self-edge.
|
||
Invalid/empty → {} (best-effort: then original order)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("prereqs"), dict):
|
||
return {}
|
||
out: dict[int, list[int]] = {}
|
||
for k, v in data["prereqs"].items():
|
||
try:
|
||
num = int(k)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if num not in valid or not isinstance(v, list):
|
||
continue
|
||
pres = []
|
||
for p in v:
|
||
try:
|
||
p = int(p)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if p in valid and p != num and p not in pres:
|
||
pres.append(p)
|
||
if pres:
|
||
out[num] = pres
|
||
return out
|
||
|
||
|
||
def _topo_order(nums: list[int], edges: dict[int, list[int]]) -> list[int]:
|
||
"""Kahn topo sort: prerequisites first. `edges[num]` = numbers that must come BEFORE num.
|
||
Stable tie-break (original order of `nums`); cycles are broken (never deadlock)."""
|
||
pos = {n: i for i, n in enumerate(nums)}
|
||
# remaining in-degree over valid nodes only; self/foreign edges ignored.
|
||
pre = {n: [p for p in edges.get(n, []) if p in pos and p != n] for n in nums}
|
||
done: list[int] = []
|
||
finished: set[int] = set()
|
||
rest = list(nums)
|
||
while rest:
|
||
ready_nodes = [n for n in rest if all(p in finished for p in pre[n])]
|
||
if not ready_nodes: # cycle → force the earliest remaining node in original order
|
||
ready_nodes = [min(rest, key=lambda n: pos[n])]
|
||
nxt = min(ready_nodes, key=lambda n: pos[n]) # stable: smallest original position first
|
||
done.append(nxt)
|
||
finished.add(nxt)
|
||
rest.remove(nxt)
|
||
return done
|
||
|
||
|
||
async def _learning_order(ctx: GenContext, set_p, files: dict, entries: dict, valid: set[int], instructions: str) -> dict:
|
||
"""Put entries (num→title) into learning order: the LLM extracts prereq edges from the
|
||
extracted `prerequisites`, code solves via topo sort. Best-effort → otherwise entries unchanged."""
|
||
if len(entries) < 3:
|
||
return entries
|
||
topic = ctx.topic
|
||
facts_map = _json_file(files["facts"])
|
||
facts_map = facts_map if isinstance(facts_map, dict) else {}
|
||
|
||
def _hint(title):
|
||
fm = facts_map.get(title) or {}
|
||
vs = [v for fk in fm.values() if isinstance(fk, dict) and (v := str(fk.get("prerequisites", "")).strip())]
|
||
return " · ".join(dict.fromkeys(vs))
|
||
|
||
pp = files["arbeit"] / "outline-prereqs.json"
|
||
|
||
def _payload(result, p=pp):
|
||
d = _sink_or_file(result, p, lambda x: x if isinstance(x, dict) and "prereqs" in x else None)
|
||
return d
|
||
|
||
existing = _json_file(pp)
|
||
if not (isinstance(existing, dict) and "prereqs" in existing):
|
||
lines = [f"{n}. {t}" + (f"\n braucht vorher: {h}" if (h := _hint(t)) else "") for n, t in entries.items()]
|
||
set_p("Outline — learning order…", step=_step_idx(topic, "Outline"))
|
||
await run_single_slot(
|
||
ctx, "Outline-Prerequisites", key=f"blocks-{topic}-outline-prereqs",
|
||
prompt=_prompt("Outline-Prerequisites", topic=topic, blocks="\n".join(lines), out_path=pp, extra=_extra(instructions)),
|
||
role="guide", capabilities="files", payload=_payload, timeout=_timeout("plan", len(entries)))
|
||
edges = _prereq_schema(_json_file(pp), valid)
|
||
if not edges:
|
||
return entries # no/invalid edges → original order (no regression)
|
||
ordered = _topo_order(list(entries), edges)
|
||
return {n: entries[n] for n in ordered}
|
||
|
||
|
||
async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict:
|
||
"""Format-agnostic outline over ALL blocks — 3 proposals → judge merges.
|
||
Never aborts: 0 valid → one chapter with everything; missing blocks land in "Other".
|
||
→ {"chapters":[{title,numbers}]} (also in files["outline"])."""
|
||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||
valid = set(entries)
|
||
step = _step_idx(topic, "Outline")
|
||
|
||
# Establish learning order (LLM-modulo): the LLM extracts prereq edges from the extracted
|
||
# `prerequisites`, code solves via topo sort. Best-effort → otherwise original order.
|
||
entries = await _learning_order(ctx, set_p, files, entries, valid, instructions)
|
||
liste = "\n".join(f"{n}. {t}" for n, t in entries.items())
|
||
set_p("Outline — proposals…", step=step)
|
||
|
||
async def _proposal(i, path):
|
||
if _outline_schema(_json_file(path), valid):
|
||
return True
|
||
await run_single_slot(
|
||
ctx, f"Outline {i}", key=f"blocks-{topic}-outline-{i}",
|
||
prompt=_prompt("Guide-Outline", topic=topic, blocks=liste, out_path=path, extra=_extra(instructions)),
|
||
role="guide", capabilities="files",
|
||
payload=lambda result, p=path: _sink_or_file(result, p, lambda d: _outline_schema(d, valid)),
|
||
timeout=_timeout("plan", len(entries)))
|
||
return _outline_schema(_json_file(path), valid) is not None
|
||
|
||
slots = files["outline_slots"]
|
||
await _gather_progress([_proposal(i, p) for i, p in enumerate(slots, 1)], len(slots), _report_p(set_p, topic, "Outline"))
|
||
if is_cancelled():
|
||
return {}
|
||
proposals = [v for p in slots if (v := _outline_schema(_json_file(p), valid))]
|
||
|
||
if not proposals:
|
||
plan = {"chapters": [{"title": "Contents", "numbers": list(entries)}]}
|
||
elif len(proposals) == 1:
|
||
plan = proposals[0]
|
||
else:
|
||
set_p("Outline merging…", step=step)
|
||
block_texts = "\n\n".join(
|
||
f"### Vorschlag {i}\n" + "\n".join(
|
||
f"KAPITEL: {ch['title']}\n Nummern: {', '.join(str(n) for n in ch['numbers'])}" for ch in v["chapters"])
|
||
for i, v in enumerate(proposals, 1))
|
||
await run_single_slot(
|
||
ctx, "Outline-Judge", key=f"blocks-{topic}-outline-judge",
|
||
prompt=_prompt("Guide-Outline-Judge", topic=topic, format_name="den Guide",
|
||
purpose="alle Blocks in einem roten Faden", n=len(proposals),
|
||
blocks=liste, outlines=block_texts, out_path=files["outline"], extra=_extra(instructions)),
|
||
role="judge", capabilities="files",
|
||
payload=lambda result: _sink_or_file(result, files["outline"],
|
||
lambda d: _outline_schema(d, valid)),
|
||
timeout=_timeout("plan_judge", len(entries)))
|
||
plan = _outline_schema(_json_file(files["outline"]), valid) or proposals[0]
|
||
|
||
# Placement review (best-effort): ONE judge checks every block→chapter assignment and
|
||
# reports ONLY misplacements as moves. Invalid/mass output → plan unchanged.
|
||
if proposals and len(plan["chapters"]) >= 2 and not is_cancelled():
|
||
rp = files["arbeit"] / "outline-review.json"
|
||
moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
|
||
if moves is None:
|
||
chapter_text = "\n\n".join(
|
||
f"KAPITEL {k}: {ch['title']}\n" + "\n".join(f" {n}. {_title(entries[n])}" for n in ch["numbers"])
|
||
for k, ch in enumerate(plan["chapters"], 1))
|
||
set_p("Outline review…", step=step)
|
||
await run_single_slot(
|
||
ctx, "Outline-Review", key=f"blocks-{topic}-outline-review",
|
||
prompt=_prompt("Guide-Outline-Review", topic=topic, chapters=chapter_text,
|
||
out_path=rp, extra=_extra(instructions)),
|
||
role="judge", capabilities="files",
|
||
payload=lambda result: _sink_or_file(result, rp, lambda d: _outline_review_schema(
|
||
d, valid, len(plan["chapters"]), len(entries))),
|
||
timeout=_timeout("plan_judge", len(entries)))
|
||
moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
|
||
for nr, target in (moves or {}).items():
|
||
for ch in plan["chapters"]:
|
||
if nr in ch["numbers"]:
|
||
ch["numbers"].remove(nr)
|
||
plan["chapters"][target - 1]["numbers"].append(nr)
|
||
if moves:
|
||
plan["chapters"] = [ch for ch in plan["chapters"] if ch["numbers"]]
|
||
_log(topic, f"Outline-Review: {len(moves)} Block/Blöcke umsortiert")
|
||
|
||
# Completeness: every block appears — missing in "Other" (against omitting agents/judge).
|
||
included = {n for ch in plan["chapters"] for n in ch["numbers"]}
|
||
missing = [n for n in entries if n not in included]
|
||
if missing:
|
||
plan["chapters"].append({"title": "Other", "numbers": missing})
|
||
atomic_write_json(files["outline"], plan, indent=1)
|
||
return plan
|
||
|
||
|
||
# --- Learning artefacts (flashcards/examples from the facts) ---
|
||
|
||
def _cards_schema(data):
|
||
"""{"cards":[{block,subblock,question,answer}]} → list (also empty) · None if broken."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("cards"), list):
|
||
return None
|
||
out = []
|
||
for e in data["cards"]:
|
||
if isinstance(e, dict) and (f := str(e.get("question", "")).strip()) and (a := str(e.get("answer", "")).strip()):
|
||
out.append({"block": str(e.get("block", "")).strip(), "subblock": str(e.get("subblock", "")).strip(),
|
||
"question": f, "answer": a})
|
||
return out
|
||
|
||
|
||
def _example_schema(data):
|
||
"""{"examples":[{block,subblock,problem,steps,result}]} → list (also empty) · None if broken."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("examples"), list):
|
||
return None
|
||
out = []
|
||
for e in data["examples"]:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
problem = str(e.get("problem", "")).strip()
|
||
steps = [s for x in (e.get("steps") or []) if (s := str(x).strip())]
|
||
if problem and steps:
|
||
out.append({"block": str(e.get("block", "")).strip(), "subblock": str(e.get("subblock", "")).strip(),
|
||
"problem": problem, "steps": steps, "result": str(e.get("result", "")).strip()})
|
||
return out
|
||
|
||
|
||
def _example_check_schema(data):
|
||
"""Worked-example check → {"ok": true} → set() (all correct); {"problems":[{"index":N}]} →
|
||
{N, …} (1-based flagged indices); None if broken."""
|
||
if not isinstance(data, dict):
|
||
return None
|
||
if data.get("ok") is True:
|
||
return set()
|
||
pr = data.get("problems")
|
||
if not isinstance(pr, list):
|
||
return None
|
||
out: set[int] = set()
|
||
for p in pr:
|
||
if isinstance(p, dict):
|
||
try:
|
||
out.add(int(p.get("index")))
|
||
except (ValueError, TypeError):
|
||
continue
|
||
return out
|
||
|
||
|
||
_ARTEFACT_SCHEMA = {"flashcard": _cards_schema, "example": _example_schema}
|
||
_ARTEFACT_PROMPT = {"flashcard": "Artifact-Flashcard", "example": "Artifact-Example"}
|
||
_ARTEFACT_STEP = {"flashcard": "Flashcards", "example": "Examples"}
|
||
|
||
|
||
async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None:
|
||
"""Generate learning artefacts per type from the stored facts — one generation pass
|
||
per type over chunks. Worked examples are verified against the facts (wrong ones discarded);
|
||
flashcards are low-risk and stay unchecked. → {type: [entries]} (also in files)."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
caps = "files"
|
||
sh = _subs_hash(sidecar) # resume must invalidate when the sub set changed
|
||
# Blocks with subs + facts lines as input block (extract-once from the facts).
|
||
blocks = []
|
||
for btitle, subs in sidecar.items():
|
||
if not isinstance(subs, list):
|
||
continue
|
||
lines = []
|
||
for s in subs:
|
||
if not isinstance(s, dict) or not (st := str(s.get("title", "")).strip()):
|
||
continue
|
||
fk = s.get("facts") if isinstance(s.get("facts"), dict) else {}
|
||
line = f"- {st}"
|
||
if fk and (fk_text := _facts_lines(fk)):
|
||
line += "\n" + "\n".join(" " + l for l in fk_text.split("\n"))
|
||
lines.append(line)
|
||
if lines:
|
||
blocks.append((btitle, lines))
|
||
if not blocks:
|
||
empty_map = {t: [] for t in ARTEFACT_TYPES}
|
||
atomic_write_json(files["artefakte"], empty_map, indent=1)
|
||
return empty_map
|
||
|
||
chunks = _lpt_chunks([len(z) for _, z in blocks], ARTEFACT_CHUNK_SUBS)
|
||
def block_text(idxs):
|
||
return "\n\n".join(f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(blocks[i][1]) for i in idxs)
|
||
|
||
# Check worked examples against the facts (panel majority) — discard wrong ones. CoT steps are
|
||
# error-prone; a wrong example imprints a faulty schema → no example > a wrong one.
|
||
async def _check_examples(ci, idxs, items):
|
||
if is_cancelled() or not items:
|
||
return items
|
||
def cpath(j): return work_dir / f"artifact-example-check-{sh}-c{ci}-j{j}.json"
|
||
examples_txt = "\n\n".join(
|
||
f"{k}. PROBLEM: {e['problem']}\n SCHRITTE: " + " | ".join(e.get("steps", []))
|
||
+ (f"\n ERGEBNIS: {e['result']}" if e.get("result") else "")
|
||
for k, e in enumerate(items, 1))
|
||
pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _example_check_schema(_json_file(cpath(j))) is None]
|
||
if pending:
|
||
# ground truth (facts) is fully inline → no tools, text reply, engine persists
|
||
tmap = {asyncio.create_task(
|
||
run_agent(f"blocks-{topic}-{ns}artifact-example-check-c{ci}-j{j}",
|
||
_prompt("Artifact-Example-Check", topic=topic, facts=block_text(idxs), examples=examples_txt, out_path=cpath(j), extra=_extra(instructions)),
|
||
_timeout("content_check", len(items)), provider=provider, role="judge", capabilities="none",
|
||
scope=topic, label=f"{lbl}Beispiel-Check {ci}/{j}")): j
|
||
for j in pending}
|
||
await _panel_2of3(tmap, lambda j, r: _sink_json(r, cpath(j), _example_check_schema),
|
||
lambda: [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL]
|
||
if (s := _example_check_schema(_json_file(cpath(j)))) is not None],
|
||
lambda s: frozenset(s))
|
||
outs = [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if (s := _example_check_schema(_json_file(cpath(j)))) is not None]
|
||
if not outs:
|
||
return items # no exam possible → keep (best-effort)
|
||
votes: dict[int, int] = {}
|
||
for s in outs:
|
||
for idx in s:
|
||
votes[idx] = votes.get(idx, 0) + 1
|
||
threshold = len(outs) / 2
|
||
dropped = {idx for idx, v in votes.items() if v > threshold} # majority (≥2 of 3) flagged → out
|
||
if dropped:
|
||
_log(topic, f"Worked-example check chunk {ci}: {len(dropped)}/{len(items)} discarded")
|
||
return [e for k, e in enumerate(items, 1) if k not in dropped]
|
||
|
||
async def _one_type(typ: str) -> list | None:
|
||
"""Full pipeline of ONE artefact type — the types are independent (own schemas,
|
||
own files) and run in parallel."""
|
||
schema = _ARTEFACT_SCHEMA[typ]
|
||
|
||
def apath(ci): return work_dir / f"artifact-{typ}-{sh}-c{ci}.json"
|
||
|
||
async def _gen(ci, idxs):
|
||
p = apath(ci)
|
||
if schema(_json_file(p)) is not None:
|
||
return True
|
||
await run_single_slot(
|
||
ctx, f"{lbl}{_ARTEFACT_STEP[typ]} {ci}", key=f"blocks-{topic}-{ns}artifact-{typ}-c{ci}",
|
||
prompt=_prompt(_ARTEFACT_PROMPT[typ], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)),
|
||
role="guide", capabilities="files",
|
||
payload=lambda result, p=p: _sink_or_file(result, p, schema),
|
||
timeout=_timeout("content", sum(len(blocks[i][1]) for i in idxs)))
|
||
return schema(_json_file(p)) is not None
|
||
|
||
await _gather_progress([_gen(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, _ARTEFACT_STEP[typ]))
|
||
if is_cancelled():
|
||
return None
|
||
eintraege: list = []
|
||
for ci in range(len(chunks)):
|
||
chunk_items = schema(_json_file(apath(ci))) or []
|
||
if typ == "example" and chunk_items:
|
||
chunk_items = await _check_examples(ci, chunks[ci], chunk_items)
|
||
eintraege += chunk_items
|
||
return eintraege
|
||
|
||
results = await asyncio.gather(*[_one_type(t) for t in ARTEFACT_TYPES])
|
||
if is_cancelled() or any(r is None for r in results):
|
||
return None
|
||
outcome: dict[str, list] = dict(zip(ARTEFACT_TYPES, results))
|
||
atomic_write_json(files["artefakte"], outcome, indent=1)
|
||
return outcome
|
||
|
||
|
||
async def _mirror_sidecar_db(topic: str, sidecar: dict) -> None:
|
||
"""Mirror the sidecar {block title: [{title, level, relevance}]} into the DB table subblocks."""
|
||
for btitle, subs in sidecar.items():
|
||
bnorm = _norm_title(btitle)
|
||
if not bnorm or not isinstance(subs, list):
|
||
continue
|
||
for s in subs:
|
||
if not isinstance(s, dict):
|
||
continue
|
||
st = str(s.get("title", "")).strip()
|
||
sn = _norm_title(st)
|
||
if not sn:
|
||
continue
|
||
facts = json.dumps(s["facts"], ensure_ascii=False) if isinstance(s.get("facts"), dict) else None
|
||
await db.put_subblock(topic, bnorm, sn, btitle, st,
|
||
level=s.get("level"), relevance=s.get("relevance"),
|
||
facts=facts, status="consensus")
|
||
|
||
|
||
|
||
|
||
async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER,
|
||
research: bool = True, qa_force: bool = False) -> None:
|
||
"""Kanban entry point: source prep, then both boards (inventory + artefacts) until
|
||
quiescence. research=False = Continue (drain the existing queue, no new search).
|
||
A run on a finished topic ADDS research (live extension) — full rebuild = DELETE /blocks."""
|
||
if topic in _blocks_progress:
|
||
return
|
||
_blocks_progress[topic] = "Warten…"
|
||
_blocks_errors.pop(topic, None)
|
||
|
||
files = _blocks_files(topic)
|
||
q = load_source(topic)
|
||
folder = source_folder(topic) # projekt/uni/link → folder, thema → None
|
||
instructions = q.get("spec") or instructions # prefer the persisted specification (also on resume)
|
||
|
||
def set_p(msg: str, step: int | None = None) -> None:
|
||
_blocks_progress[topic] = msg
|
||
if step is not None:
|
||
_blocks_step[topic] = step
|
||
|
||
def is_cancelled() -> bool:
|
||
return topic in _blocks_cancelled
|
||
|
||
ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled)
|
||
try:
|
||
async with _semaphore:
|
||
files["arbeit"].mkdir(parents=True, exist_ok=True)
|
||
# Step "Source prep": crawl (link) + PDFs + content/noise triage.
|
||
if not await _prepare_source(ctx, set_p, files, q, folder, instructions):
|
||
if is_cancelled():
|
||
_blocks_errors[topic] = "Cancelled — progress is preserved"
|
||
return
|
||
import board_inventory # lazy: the boards import blocks
|
||
ok = await board_inventory.run_boards(ctx, set_p, files, q, folder, instructions,
|
||
research=research, qa_force=qa_force)
|
||
if not ok and is_cancelled():
|
||
_blocks_errors[topic] = "Cancelled — progress is preserved"
|
||
except Exception as e:
|
||
log.exception("[%s] Blocks generation failed", topic)
|
||
_blocks_errors[topic] = str(e)[:2000]
|
||
finally:
|
||
# No file cleanup: intermediate files stay for resume / traceability.
|
||
_blocks_progress.pop(topic, None)
|
||
_blocks_step.pop(topic, None)
|
||
_blocks_cancelled.discard(topic)
|
||
clear_scope(f"blocks-{topic}-") # clear the scope → restart isn't blocked
|