2574 lines
122 KiB
Python
2574 lines
122 KiB
Python
"""Blocks pipeline: research consensus + clarification loop — pure inventory, unsorted.
|
||
|
||
5x research (min. 3, grace) → mapping (consensus/rest) → clarification loop (max.
|
||
CONSENSUS_MAX_ROUNDS rounds): 3 selection agents (min. 2, grace) decide
|
||
on the disputed rest, a mapping agent sorts into accept/discard/
|
||
still disputed. An empty rest ends the loop; the last round must decide
|
||
everything. Races use a grace window instead of "first N win": after the
|
||
first valid result, the remaining agents get CONSENSUS_GRACE seconds to
|
||
finish. The consensus is accumulated in code — no agent re-emits
|
||
the full list.
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import math
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import time
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
import database as db
|
||
import embedding
|
||
from agents import kill_process, cancel_scope, clear_scope, run_agent
|
||
from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_SIBLING_FLOOR, EMBEDDING_SIBLING_CAP, GROUP_RECONCILE_FLOOR, GROUP_MIN_COS_FLOOR
|
||
from fsutil import atomic_write_text, atomic_write_json
|
||
from jsonio import read_json_file as _json_file
|
||
from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder
|
||
from crawl import crawl
|
||
from pipeline import (
|
||
CANCELLED, FAILED, OK, GenContext, _extra, _gather_progress, _yesno_schema, _log, _prompt, _race,
|
||
_relevance_schema, _runde_schema, _semaphore, _str_list, _levels_schema, _timeout, run_single_slot,
|
||
)
|
||
from textkit import (
|
||
_unique_title, _load_blocks, _norm_title, _parse_selection, _parse_subblocks, _title,
|
||
_resolve_title, _title_index,
|
||
)
|
||
|
||
# Chunk the subblocks (web search per block): 1 agent per ~10 blocks, capped.
|
||
SUBBLOCK_CHUNK = 10
|
||
SUBBLOCK_MAX = 40
|
||
# Classifying is cheap (short verdict, no web search) → larger packages, fewer files/agents.
|
||
LEVEL_CHUNK = 100
|
||
|
||
# Research: fixed file batches instead of a search loop → each crawl page is assigned exactly once.
|
||
RESEARCH_BATCH = 20 # crawl pages per batch
|
||
RESEARCH_READERS = 2 # reader agents per batch (consensus ≥2 within the batch)
|
||
RESEARCH_THEMA_AGENTS = 5 # web mode (source "thema", no crawl folder)
|
||
# uni/projekt: chunk the script text into sections of ~this size (against lost-in-the-middle on
|
||
# large documents). ~12k chars ≈ 3k tokens → safely below the recall-drop threshold.
|
||
RESEARCH_SECTION_CHARS = 12000
|
||
# Triage (content/noise) is now a deterministic rule filter (config.CRAWL_*).
|
||
SUBBLOCK_CAP = 900 # subblock find loop per chunk (15 min)
|
||
CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (dedups everything); above that chunked + merge pass — fallback path only
|
||
DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair (complete-link aggregates → no chaining)
|
||
DEDUP_PAIRS_CHUNK = 40 # pairs per judge package (pairwise verification instead of a block mixer)
|
||
DEDUP_TITLE_AUTO = 0.95 # near-identical TITLE cosine ⇒ same entity → merge without the judge (recall net)
|
||
FILTER_CHUNK = 35 # blocks to assess per judge in the degrade pass (full list as context)
|
||
# Balance question-pattern chunks by sub load via LPT (makespan), not by block count.
|
||
QUESTION_CHUNK_SUBS = 50 # target sum of relevant subs per chunk
|
||
QUESTION_MAX_ROUNDS = 3 # catch-up rounds for subs without a pattern (the LLM omits ~18 % per chunk)
|
||
FACTS_CHUNK_SUBS = 25 # facts extraction: smaller chunks (facts are bulkier than patterns)
|
||
FACTS_CHECK_PANEL = 3 # judges per chunk in the facts check (majority objects)
|
||
CONSOLIDATION_PANEL = 3 # mapping judges per chunk (panel → reconcile instead of a single judge)
|
||
SUBBLOCK_PANEL = 3 # source judges in the subblock clarification (majority instead of a single judge)
|
||
FILTER_RECHECK_PANEL = 3 # judges in the survivor-recheck (rare-positive "fragment" recall; majority ≥2)
|
||
|
||
log = logging.getLogger("creator.blocks")
|
||
|
||
_blocks_progress: dict[str, str] = {}
|
||
_blocks_errors: dict[str, str] = {}
|
||
_blocks_cancelled: set[str] = set()
|
||
_blocks_step: dict[str, int] = {}
|
||
|
||
ARTEFACT_TYPES = ("flashcard", "example")
|
||
|
||
|
||
def load_source(topic: str) -> dict:
|
||
"""Read the persisted source choice. Fallback (legacy topics without source.json):
|
||
if projects/<topic> exists → projekt, otherwise thema."""
|
||
q = _json_file(source_path(topic))
|
||
if isinstance(q, dict) and q.get("type") in ("thema", "projekt", "uni", "link"):
|
||
return q
|
||
if project_dir(topic).is_dir():
|
||
return {"type": "projekt", "location": f"projects/{topic}", "spec": ""}
|
||
return {"type": "thema", "location": "", "spec": ""}
|
||
|
||
|
||
def source_folder(topic: str) -> Path | None:
|
||
"""Folder source (projekt/uni → path, link → crawl folder) — otherwise None (thema)."""
|
||
q = load_source(topic)
|
||
if q["type"] == "link":
|
||
return source_crawl_dir(topic)
|
||
if q["type"] in ("projekt", "uni"):
|
||
return safe_folder(q.get("location", ""))
|
||
return None
|
||
|
||
|
||
def _crawl_done(topic: str) -> bool:
|
||
return (source_crawl_dir(topic) / ".done").exists() # marker only on clean completion
|
||
|
||
|
||
# Learning-path levels (beginner/advanced/expert); old difficulty values are backward-compatible.
|
||
_LEVELS = ("beginner", "advanced", "expert", "easy", "medium", "hard")
|
||
|
||
|
||
async def subblocks_title(topic: str, block: str) -> list[str]:
|
||
"""Subblock titles of a block — DB-first (consensus), fallback to the sidecar file."""
|
||
rows = [s["sub_title"] for s in await db.list_subblocks(topic, _norm_title(block))
|
||
if s["status"] == "consensus" and s["sub_title"]]
|
||
if rows:
|
||
return rows
|
||
sc = _json_file(subblocks_path(topic))
|
||
if not isinstance(sc, dict):
|
||
return []
|
||
return [
|
||
t for s in (sc.get(block) or [])
|
||
if isinstance(s, dict) and (t := str(s.get("title", "")).strip())
|
||
]
|
||
|
||
|
||
async def load_question_pattern(topic: str, block: str) -> list[dict]:
|
||
"""Predefined question patterns of a block — DB-first, fallback to sidecar (empty = live)."""
|
||
rows = await db.list_question_pattern(topic, _norm_title(block))
|
||
if rows:
|
||
return [{"subblock": r["sub_title"], "question": r["question"]} for r in rows if r["question"]]
|
||
fm = _json_file(question_pattern_path(topic))
|
||
if not isinstance(fm, dict):
|
||
return []
|
||
return [
|
||
{"subblock": str(e.get("subblock", "")).strip(), "question": question}
|
||
for e in (fm.get(block) or [])
|
||
if isinstance(e, dict) and (question := str(e.get("question", "")).strip())
|
||
]
|
||
|
||
|
||
async def subblocks_frei(topic: str, block: str, max_level: int) -> list[str]:
|
||
"""Subblock titles up to the unlocked level (≤ max_level). Fallback without
|
||
level knowledge (legacy/sidecar): all subblock titles."""
|
||
rows = await db.subs_with_level(topic, block)
|
||
if not rows:
|
||
return await subblocks_title(topic, block)
|
||
return [s["title"] for s in rows if s["level"] <= max_level and s["title"]]
|
||
|
||
|
||
async def load_question_pattern_free(topic: str, block: str, max_level: int) -> list[dict]:
|
||
"""Question patterns, filtered to subblocks up to the unlocked level. Without
|
||
level knowledge (legacy/sidecar), unfiltered."""
|
||
rows = await db.subs_with_level(topic, block)
|
||
if not rows:
|
||
return await load_question_pattern(topic, block)
|
||
unlocked = {s["norm"] for s in rows if s["level"] <= max_level}
|
||
return [m for m in await load_question_pattern(topic, block) if _norm_title(m["subblock"]) in unlocked]
|
||
|
||
|
||
async def load_overview(topic: str) -> list[dict]:
|
||
"""Structured block list for the overview — DB-first (consensus + subs/levels/relevance),
|
||
fallback to blocks.md + sidecar (legacy topics)."""
|
||
bs = await db.list_blocks(topic, status="consensus")
|
||
if bs:
|
||
out = []
|
||
for num, b in enumerate(bs, 1):
|
||
subs = [s for s in await db.list_subblocks(topic, b["title_norm"]) if s["status"] == "consensus"]
|
||
out.append({
|
||
"num": num, "title": b["title"], "description": b["description"],
|
||
"subblocks": [
|
||
{"title": s["sub_title"],
|
||
"level": s["level"] if s["level"] in _LEVELS else "advanced",
|
||
"relevance": s["relevance"] if s["relevance"] in ("relevant", "peripheral") else None}
|
||
for s in subs if s["sub_title"]
|
||
],
|
||
})
|
||
return out
|
||
entries = _load_blocks(_read(blocks_path(topic)))
|
||
sidecar = _json_file(subblocks_path(topic))
|
||
sidecar = sidecar if isinstance(sidecar, dict) else {}
|
||
out = []
|
||
for num, entry in entries.items():
|
||
title = _title(entry)
|
||
split_parts = entry.split(" — ", 1)
|
||
description = split_parts[1].strip() if len(split_parts) == 2 else ""
|
||
subblocks = [
|
||
{
|
||
"title": t,
|
||
"level": s.get("level") if s.get("level") in _LEVELS else "advanced",
|
||
"relevance": s.get("relevance") if s.get("relevance") in ("relevant", "peripheral") else None,
|
||
}
|
||
for s in (sidecar.get(title) or [])
|
||
if isinstance(s, dict) and (t := str(s.get("title", "")).strip())
|
||
]
|
||
out.append({"num": num, "title": title, "description": description, "subblocks": subblocks})
|
||
return out
|
||
|
||
|
||
def _blocks_steps(topic: str) -> tuple:
|
||
"""Steps per source: link gets "Source laden" up front, projekt additionally "Supplement".
|
||
|
||
Subblocks + levels are three phases each (find, select, clarify). Per phase
|
||
all packages run in parallel; the step remains until the last package is done.
|
||
"""
|
||
q = load_source(topic)
|
||
base = ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung")
|
||
rest = (
|
||
"Subblocks find", "Subblocks select", "Subblocks clarify",
|
||
"Facts find", "Facts check", "Facts fix",
|
||
"Levels find", "Levels select", "Levels clarify",
|
||
"Relevance find", "Relevance select", "Relevance clarify",
|
||
"Outline",
|
||
"Questions find", "Questions select", "Questions clarify", "Questions check",
|
||
"Flashcards", "Examples",
|
||
)
|
||
middle = base + (("Supplement",) if q["type"] == "projekt" else ()) + rest
|
||
return (("Source prep",) if q["type"] == "link" else ()) + middle
|
||
|
||
|
||
def _step_idx(topic: str, name: str) -> int:
|
||
return _blocks_steps(topic).index(name)
|
||
|
||
|
||
def _report_p(set_p, topic: str, step: str):
|
||
"""Async report callback for _gather_progress: sets "<step> d/t…" + step index."""
|
||
idx = _step_idx(topic, step)
|
||
async def report(d, t):
|
||
set_p(f"{step} {d}/{t}…", step=idx)
|
||
return report
|
||
|
||
|
||
# Coarse display phases: bundle the fine steps (internally everything stays fine-grained).
|
||
# Special steps (Source laden, Supplement) belong to the "Inventory" phase.
|
||
PHASEN = (
|
||
("Source", ("Source prep",)),
|
||
("Inventory", ("Research", "Consolidation", "Clarification", "Dedup", "Blocks-Filter", "Blocks-Gruppierung", "Supplement")),
|
||
("Subblocks", ("Subblocks find", "Subblocks select", "Subblocks clarify")),
|
||
("Facts", ("Facts find", "Facts check", "Facts fix")),
|
||
("Levels", ("Levels find", "Levels select", "Levels clarify")),
|
||
("Relevance", ("Relevance find", "Relevance select", "Relevance clarify")),
|
||
("Outline", ("Outline",)),
|
||
("Questions", ("Questions find", "Questions select", "Questions clarify", "Questions check")),
|
||
("Artefacts", ("Flashcards", "Examples")),
|
||
)
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _blocks_files(topic: str) -> dict:
|
||
work_dir = arbeit_dir(topic)
|
||
rounds = range(1, CONSENSUS_MAX_ROUNDS + 1)
|
||
return {
|
||
"final": blocks_path(topic),
|
||
"arbeit": work_dir,
|
||
"research": [work_dir / f"research-{i}.md" for i in (1, 2, 3, 4, 5)],
|
||
"research_mapping": work_dir / "research-mapping.json",
|
||
"selection": {n: [work_dir / f"selection-r{n}-{i}.json" for i in (1, 2, 3)] for n in rounds},
|
||
"mapping": {n: work_dir / f"selection-mapping-r{n}.json" for n in rounds},
|
||
"ergaenzung": work_dir / "ergaenzung.json",
|
||
"sub_roh": work_dir / "subblocks-roh.json",
|
||
"facts": work_dir / "subblocks-facts.json",
|
||
"sidecar": subblocks_path(topic),
|
||
"question_pattern": question_pattern_path(topic),
|
||
"outline": work_dir / "outline.json",
|
||
"outline_slots": [work_dir / f"outline-{i}.json" for i in (1, 2, 3)],
|
||
"artefakte": work_dir / "artefakte.json",
|
||
}
|
||
|
||
|
||
def _all_slot_files(files: dict) -> list[Path]:
|
||
work_dir = files["arbeit"]
|
||
# Subblock/levels slots are dynamic per chunk — collect via glob.
|
||
dyn = (list(work_dir.glob("subblock-*")) + list(work_dir.glob("facts-*")) + list(work_dir.glob("level-*")) + list(work_dir.glob("relevance-*"))
|
||
+ list(work_dir.glob("question-pattern-*")) + list(work_dir.glob("outline-*")) + list(work_dir.glob("artifact-*"))
|
||
+ list(work_dir.glob("research-*")) + list(work_dir.glob("consolidation-*"))
|
||
+ list(work_dir.glob("clarification*")) + list(work_dir.glob("dedup-*"))
|
||
+ list(work_dir.glob("inventar-filter*"))
|
||
+ list(work_dir.glob("gruppierung-*")) + list(work_dir.glob("inventar-gruppierung*"))) if work_dir.is_dir() else []
|
||
return [
|
||
*files["research"], files["research_mapping"],
|
||
*(p for slots in files["selection"].values() for p in slots),
|
||
*files["mapping"].values(), files["ergaenzung"],
|
||
files["sub_roh"], files["sidecar"], files["question_pattern"],
|
||
files["facts"], files["outline"], files["artefakte"], *dyn,
|
||
]
|
||
|
||
|
||
def cancel_blocks(topic: str) -> bool:
|
||
if topic not in _blocks_progress:
|
||
return False
|
||
_blocks_cancelled.add(topic)
|
||
cancel_scope(f"blocks-{topic}-") # waiting agents bail before spawning
|
||
kill_process(f"blocks-{topic}-") # kill running subprocesses
|
||
return True
|
||
|
||
|
||
|
||
|
||
|
||
|
||
async def blocks_status(topic: str) -> dict:
|
||
"""Kanban-based status: `generating` from the run registry, progress from the card
|
||
counts. `partial` = cards sit in non-terminal columns while nothing runs (continue-able)."""
|
||
ready = blocks_path(topic).exists() # inventory written → block overview available
|
||
generating = topic in _blocks_progress
|
||
counts = await db.kanban_stage_counts(topic)
|
||
terminal = {"clustered", "done_cluster", "grouped", "rejected", "done_block", "done_artefact", "dead"}
|
||
open_cards = sum(n for stages in counts.values()
|
||
for stage, n in stages.items() if stage not in terminal)
|
||
return {
|
||
"ready": ready,
|
||
"generating": generating,
|
||
"progress": _blocks_progress.get(topic),
|
||
"error": _blocks_errors.get(topic),
|
||
"partial": not generating and open_cards > 0,
|
||
"steps": [], # legacy phase pills — replaced by the live board
|
||
"feine_steps": [],
|
||
}
|
||
|
||
|
||
def active_blocks() -> list[dict]:
|
||
return [{"topic": t, "progress": p} for t, p in _blocks_progress.items()]
|
||
|
||
|
||
def reset_blocks(topic: str) -> None:
|
||
""""Remove": deletes the ENTIRE blocks area — crawl, triage, inventory … questions.
|
||
KEEPS only the topic config `source.json` (type/link/spec). Re-generating crawls anew.
|
||
(Crawl/triage belong to the blocks; only the config is the "topic".)"""
|
||
files = _blocks_files(topic)
|
||
files["final"].unlink(missing_ok=True)
|
||
files["sidecar"].unlink(missing_ok=True)
|
||
files["question_pattern"].unlink(missing_ok=True)
|
||
shutil.rmtree(source_crawl_dir(topic), ignore_errors=True) # crawl belongs to the blocks
|
||
shutil.rmtree(files["arbeit"], ignore_errors=True)
|
||
_blocks_errors.pop(topic, None)
|
||
# source.json intentionally stays — that is the topic config.
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _supplement_schema(data):
|
||
"""{"blocks": [{"title", "description"}]} → list (empty allowed) · otherwise None."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("blocks"), list):
|
||
return None
|
||
out = []
|
||
for b in data["blocks"]:
|
||
if not isinstance(b, dict) or not isinstance(b.get("title"), str) or not isinstance(b.get("description"), str):
|
||
return None
|
||
title, description = b["title"].strip(), b["description"].strip()
|
||
if not title:
|
||
return None
|
||
out.append((title, description))
|
||
return out
|
||
|
||
|
||
def _convert_pdfs(project: Path) -> None:
|
||
"""Convert PDFs in the project to .txt (pdftotext) — agents read text instead of page images.
|
||
|
||
Called before every project generation; converts only if the
|
||
.txt is missing or older than the PDF. The original is left untouched.
|
||
If pdftotext is missing and the project contains PDFs → hard error instead of
|
||
an unreliable direct-read mode (MiniMax image limit, vision cost).
|
||
"""
|
||
pdfs = list(project.rglob("*.pdf"))
|
||
if not pdfs:
|
||
return
|
||
if shutil.which("pdftotext") is None:
|
||
raise RuntimeError("pdftotext missing (install poppler-utils) — PDFs in the project cannot be read")
|
||
for pdf in pdfs:
|
||
txt = pdf.with_suffix(".txt")
|
||
if txt.exists() and txt.stat().st_mtime >= pdf.stat().st_mtime:
|
||
continue
|
||
try:
|
||
subprocess.run(["pdftotext", "-layout", str(pdf), str(txt)], check=True, timeout=120)
|
||
_log(project.name, f"PDF converted: {pdf.name} → {txt.name}")
|
||
except Exception as e:
|
||
raise RuntimeError(f"PDF conversion failed ({pdf.name}): {e}") from e
|
||
|
||
|
||
_SOURCE_TEMPLATE = {"projekt": "Blocks-Source-Projekt", "uni": "Blocks-Source-Uni", "link": "Blocks-Source-Link"}
|
||
|
||
|
||
def _text_sections(text: str, goal: int = RESEARCH_SECTION_CHARS) -> list[str]:
|
||
"""Split text at paragraph/line boundaries into sections of ~`ziel` chars (against lost-in-the-middle
|
||
on large documents). Small text stays ONE section. Content stays complete — only
|
||
separating whitespace is dropped."""
|
||
text = text.strip()
|
||
if len(text) <= goal:
|
||
return [text] if text else []
|
||
sections: list[str] = []
|
||
buf = ""
|
||
|
||
def flush():
|
||
nonlocal buf
|
||
if buf.strip():
|
||
sections.append(buf.strip())
|
||
buf = ""
|
||
|
||
for block in re.split(r"\n\s*\n", text): # at paragraph boundaries
|
||
block = block.strip()
|
||
if not block:
|
||
continue
|
||
if len(block) > goal: # single huge paragraph → hard-cut at lines
|
||
flush()
|
||
for line in block.split("\n"):
|
||
if buf and len(buf) + len(line) + 1 > goal:
|
||
flush()
|
||
buf += line + "\n"
|
||
flush()
|
||
elif buf and len(buf) + len(block) + 2 > goal:
|
||
flush()
|
||
buf = block
|
||
else:
|
||
buf = (buf + "\n\n" + block) if buf else block
|
||
flush()
|
||
return sections
|
||
|
||
|
||
def _build_research_prompt(topic: str, out_path: Path, instructions: str, type: str, folder: Path | None, fokus: str = "", section: str = "") -> str:
|
||
if section:
|
||
# Section mode (uni/projekt): text directly in the prompt → small context, no file reading.
|
||
source = section
|
||
elif type in _SOURCE_TEMPLATE:
|
||
source = _prompt(_SOURCE_TEMPLATE[type], project=folder)
|
||
else:
|
||
source = _prompt("Blocks-Source-Thema", topic=topic)
|
||
return _prompt(
|
||
"Blocks-Research",
|
||
topic=topic, source=source, blocks_path=out_path, focus=fokus, extra=_extra(instructions),
|
||
)
|
||
|
||
|
||
def _file_payload(path: Path):
|
||
"""Valid if the slot file exists and contains numbered entries."""
|
||
if not path.exists():
|
||
return None
|
||
text = path.read_text(encoding="utf-8")
|
||
return text if _parse_selection(text) else None
|
||
|
||
|
||
def _mapping_schema(data):
|
||
"""{"blocks": [str, ≥1], "rest": [str]} → (blocks, rest) · otherwise None."""
|
||
if not isinstance(data, dict):
|
||
return None
|
||
blocks = _str_list(data.get("blocks"))
|
||
rest = _str_list(data.get("rest"))
|
||
if not blocks or rest is None:
|
||
return None
|
||
return blocks, rest
|
||
|
||
|
||
def _sub_raw_schema(data):
|
||
"""{block title: [subblock, …]} → dict · otherwise None (intermediate state of block B)."""
|
||
if not isinstance(data, dict) or not data:
|
||
return None
|
||
out: dict[str, list[str]] = {}
|
||
for k, v in data.items():
|
||
subs = _str_list(v) if isinstance(v, list) else None
|
||
if not isinstance(k, str) or not k.strip() or not subs:
|
||
return None
|
||
out[k] = subs
|
||
return out
|
||
|
||
|
||
def _sidecar_schema(data):
|
||
"""{block title: [{title, level}, …]} → dict · otherwise None (sidecar with levels)."""
|
||
if not isinstance(data, dict) or not data:
|
||
return None
|
||
for v in data.values():
|
||
if not isinstance(v, list) or not v:
|
||
return None
|
||
for s in v:
|
||
if not isinstance(s, dict) or not str(s.get("title", "")).strip() or s.get("level") not in _LEVELS:
|
||
return None
|
||
return data
|
||
|
||
|
||
def _relevance_complete(data) -> bool:
|
||
"""Does every subblock in the sidecar carry a valid relevance (relevant/peripheral)?"""
|
||
if not isinstance(data, dict) or not data:
|
||
return False
|
||
return all(
|
||
isinstance(s, dict) and s.get("relevance") in ("relevant", "peripheral")
|
||
for v in data.values() if isinstance(v, list)
|
||
for s in v
|
||
)
|
||
|
||
|
||
|
||
|
||
def _question_pattern_chunk_schema(data) -> list[dict] | None:
|
||
"""{"pattern": [{block, subblock, question}, …]} → list of valid entries · otherwise None.
|
||
|
||
One pattern per subblock (no type cross-product — the difficulty only comes at
|
||
exam time from the learner's tier). Invalid individual entries are skipped."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("pattern"), list):
|
||
return None
|
||
out = []
|
||
for e in data["pattern"]:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
blk = str(e.get("block", "")).strip()
|
||
sub = str(e.get("subblock", "")).strip()
|
||
question = str(e.get("question", "")).strip()
|
||
if not blk or not sub or not question:
|
||
continue
|
||
out.append({"block": blk, "subblock": sub, "question": question})
|
||
return out or None
|
||
|
||
|
||
def _question_pattern_complete(topic: str) -> bool:
|
||
"""Does the question-pattern sidecar exist (build ran)? Individual empty blocks
|
||
fall back to live generation at exam time — so the file is enough."""
|
||
return isinstance(_json_file(question_pattern_path(topic)), dict)
|
||
|
||
|
||
def _read(p: Path) -> str:
|
||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
||
|
||
|
||
def _chunk_nums(items: list, n: int) -> list[list]:
|
||
"""Splits a flat list into n chunks as equal in size as possible."""
|
||
n = max(1, n)
|
||
size = max(1, math.ceil(len(items) / n))
|
||
return [items[i:i + size] for i in range(0, len(items), size)]
|
||
|
||
|
||
def _n_chunks(count: int, size: int = SUBBLOCK_CHUNK) -> int:
|
||
return min(SUBBLOCK_MAX, max(1, math.ceil(count / size)))
|
||
|
||
|
||
def _lpt_chunks(weights: list[int], target: int) -> list[list[int]]:
|
||
"""Distribute indices across chunks load-balanced (LPT, makespan-minimal). Weight = cost per index.
|
||
K = ceil(total weight/target); heaviest first into the currently lightest bin. → index lists."""
|
||
if not weights:
|
||
return []
|
||
K = max(1, math.ceil(sum(weights) / max(1, target)))
|
||
bins: list[list[int]] = [[] for _ in range(K)]
|
||
last = [0] * K
|
||
for i in sorted(range(len(weights)), key=lambda x: weights[x], reverse=True):
|
||
j = min(range(K), key=lambda b: last[b])
|
||
bins[j].append(i)
|
||
last[j] += weights[i]
|
||
return [b for b in bins if b]
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
async def _subblocks_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str,
|
||
wipe: bool = True, ns: str = "") -> dict | None:
|
||
"""Block B (DB + loop): per package, find subblocks in rounds (3 finders, until 0 new/cap),
|
||
collect in the DB (≥2 mentions = consensus, 1× discarded), a judge cleans up per package.
|
||
→ {block title: [subblock, …]} (consensus) or None. Fills DB table `subblocks`.
|
||
wipe=False (kanban board: one call per block) keeps the other blocks' rows."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
folder = source_folder(topic)
|
||
caps = "files" if folder else "full"
|
||
# Source for the evidence exam in the clarify step (discards invented/unsupportable subs).
|
||
_type = load_source(topic).get("type", "thema")
|
||
source = _prompt(_SOURCE_TEMPLATE[_type], project=folder) if _type in _SOURCE_TEMPLATE else _prompt("Blocks-Source-Thema", topic=topic)
|
||
nums = list(entries)
|
||
chunks = _chunk_nums(nums, _n_chunks(len(nums)))
|
||
n = len(chunks)
|
||
title_by_num = {num: _title(entries[num]) for num in nums}
|
||
norm_by_num = {num: _norm_title(title_by_num[num]) for num in nums}
|
||
if wipe:
|
||
await db.delete_subblocks(topic) # fresh start of the block (idempotent counter)
|
||
|
||
async def _known_block(chunk):
|
||
known = []
|
||
for num in chunk:
|
||
subs = [s["sub_title"] for s in await db.list_subblocks(topic, norm_by_num[num])]
|
||
if subs:
|
||
known.append(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in subs))
|
||
if not known:
|
||
return ""
|
||
# Do NOT list known items again (otherwise re-confirmation inflates the mention count,
|
||
# self-bias/echo) — only add what's missing. This keeps the counter an honest consensus signal.
|
||
return ("\n\nBEREITS ERFASST — liste diese NICHT erneut. Finde nur, was FEHLT:\n" + "\n".join(known))
|
||
|
||
# Phase "Subblocks find": per package loop until 0 new subs / time cap.
|
||
async def _find(c, chunk):
|
||
assignment = "\n".join(f"- {entries[num]}" for num in chunk)
|
||
chunk_idx = _title_index({num: title_by_num[num] for num in chunk})
|
||
start = time.monotonic()
|
||
round_n = 0
|
||
while not is_cancelled():
|
||
round_n += 1
|
||
bekannt = await _known_block(chunk) if round_n > 1 else ""
|
||
paths = [work_dir / f"subblock-c{c}-r{round_n}-{i}.md" for i in (1, 2, 3)]
|
||
for p in paths:
|
||
p.unlink(missing_ok=True)
|
||
slots = [{
|
||
"key": f"blocks-{topic}-{ns}subblock-c{c}-r{round_n}-{i}",
|
||
"prompt": _prompt("Subblock-Research", topic=topic, assignment=assignment, known=bekannt, out_path=p, extra=_extra(instructions)),
|
||
"role": "quick", "capabilities": caps,
|
||
"payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None),
|
||
} for i, p in enumerate(paths, 1)]
|
||
agent_texts = await _race(topic, f"Subblocks package {c} R{round_n}", slots, 2, _timeout("subblock", len(chunk)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
if is_cancelled():
|
||
return False
|
||
if not agent_texts:
|
||
return round_n > 1 # round 1 without result = error; later = simply the end
|
||
existing = {num: {s["sub_norm"] for s in await db.list_subblocks(topic, norm_by_num[num])} for num in chunk}
|
||
new = 0
|
||
for d in agent_texts:
|
||
for marker, subs in d.items():
|
||
num = _resolve_title(chunk_idx, marker)
|
||
if num is None:
|
||
continue
|
||
seen_set = set()
|
||
for sub in subs:
|
||
sn = _norm_title(sub)
|
||
if not sn or sn in seen_set:
|
||
continue
|
||
seen_set.add(sn)
|
||
if sn not in existing[num]:
|
||
new += 1
|
||
existing[num].add(sn)
|
||
await db.upsert_subblock(topic, norm_by_num[num], sn, title_by_num[num], sub)
|
||
if new == 0:
|
||
break
|
||
if time.monotonic() - start > SUBBLOCK_CAP:
|
||
_log(topic, f"Subblocks package {c}: time cap reached (round {round_n})")
|
||
break
|
||
return True
|
||
|
||
oks = await _gather_progress([_find(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _report_p(set_p, topic, "Subblocks find"))
|
||
if is_cancelled():
|
||
return None
|
||
if not all(ok is True for ok in oks):
|
||
_blocks_errors[topic] = "Subblocks failed (research)"
|
||
return None
|
||
|
||
# Phase "Subblocks select": ≥2 mentions = consensus, 1× discarded (code).
|
||
set_p(f"Subblocks select ({n} packages)…", step=_step_idx(topic, "Subblocks select"))
|
||
for num in nums:
|
||
for s in await db.list_subblocks(topic, norm_by_num[num]):
|
||
await db.set_subblock_fields(topic, norm_by_num[num], s["sub_norm"],
|
||
status=("consensus" if s["mentions"] >= 2 else "discarded"))
|
||
|
||
# Phase "Subblocks clarify": source panel (SUBBAUSTEIN_PANEL judges) checks consensus + uncertain (1×)
|
||
# against the source; code majority per sub. External, multi-voice gate against single-judge bias + echo.
|
||
async def _clarify(c, chunk):
|
||
fp = work_dir / f"subblock-final-c{c}.md"
|
||
if _parse_subblocks(_read(fp)):
|
||
return
|
||
block_texts, has_any = [], False
|
||
consensus_by_num: dict[int, list[str]] = {}
|
||
for num in chunk:
|
||
rows = await db.list_subblocks(topic, norm_by_num[num])
|
||
consensus_subs = [s["sub_title"] for s in rows if s["status"] == "consensus"]
|
||
uncertain = [s["sub_title"] for s in rows if s["status"] != "consensus" and s["mentions"] == 1]
|
||
consensus_by_num[num] = consensus_subs
|
||
if not consensus_subs and not uncertain:
|
||
continue
|
||
has_any = True
|
||
k_lines = "\n".join(f"- {s}" for s in consensus_subs) if consensus_subs else "- (keiner)"
|
||
u_lines = "\n".join(f"- {s}" for s in uncertain) if uncertain else "- (keiner)"
|
||
block_texts.append(f"BLOCK: {title_by_num[num]}\nKonsens (≥2 finders):\n{k_lines}\nUnsicher (1× — streng gegen Source check):\n{u_lines}")
|
||
if not has_any:
|
||
return
|
||
|
||
chunk_idx = _title_index({num: title_by_num[num] for num in chunk})
|
||
paths = [work_dir / f"subblock-final-c{c}-j{j}.md" for j in range(1, SUBBLOCK_PANEL + 1)]
|
||
pending = [(j, p) for j, p in enumerate(paths, 1) if _parse_subblocks(_read(p)) is None]
|
||
for _, p in pending:
|
||
p.unlink(missing_ok=True)
|
||
if pending:
|
||
slots = [{
|
||
"key": f"blocks-{topic}-{ns}subblock-final-c{c}-j{j}",
|
||
"prompt": _prompt("Subblock-Mapping", topic=topic, source=source, blocks="\n\n".join(block_texts), out_path=p, extra=_extra(instructions)),
|
||
"role": "judge", "capabilities": caps,
|
||
"payload": (lambda result, p=p: _parse_subblocks(_read(p)) or None),
|
||
} for j, p in pending]
|
||
existing = SUBBLOCK_PANEL - len(pending)
|
||
await _race(topic, f"Subblock-Clarification {c}", slots, max(1, 2 - existing),
|
||
_timeout("subblock_check", len(chunk)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
if is_cancelled():
|
||
return
|
||
outs = [d for p in paths if (d := _parse_subblocks(_read(p)))]
|
||
if not outs: # panel fully failed → adopt consensus (the fallback as before)
|
||
_log(topic, f"Subblock clarification package {c} failed — consensus adopted")
|
||
text = "\n\n".join(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in consensus_by_num[num])
|
||
for num in chunk if consensus_by_num[num])
|
||
atomic_write_text(fp, text)
|
||
return
|
||
|
||
# code majority per block/sub-norm: keep if a majority of judges list it (tie → keep).
|
||
block_texts_out = []
|
||
for num in chunk:
|
||
votes: dict[str, int] = {}
|
||
form: dict[str, str] = {}
|
||
for d in outs:
|
||
seen = set()
|
||
for marker, subs in d.items():
|
||
if _resolve_title(chunk_idx, marker) != num:
|
||
continue
|
||
for sub in subs:
|
||
sn = _norm_title(sub)
|
||
if not sn or sn in seen:
|
||
continue
|
||
seen.add(sn)
|
||
form.setdefault(sn, sub)
|
||
votes[sn] = votes.get(sn, 0) + 1
|
||
kept = [form[sn] for sn in form if votes[sn] * 2 >= len(outs)]
|
||
if kept:
|
||
block_texts_out.append(f"<!-- block: {title_by_num[num]} -->\n" + "\n".join(f"- {s}" for s in kept))
|
||
atomic_write_text(fp, "\n\n".join(block_texts_out))
|
||
|
||
await _gather_progress([_clarify(c, chunk) for c, chunk in enumerate(chunks, 1)], n, _report_p(set_p, topic, "Subblocks clarify"))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# Final list per block: judge output, otherwise consensus fallback. Reconcile DB + build raw.
|
||
raw: dict[str, list[str]] = {}
|
||
for c, chunk in enumerate(chunks, 1):
|
||
final = _parse_subblocks(_read(work_dir / f"subblock-final-c{c}.md")) or {}
|
||
chunk_idx = _title_index({num: title_by_num[num] for num in chunk})
|
||
final_by_num = {_resolve_title(chunk_idx, m): subs for m, subs in final.items() if _resolve_title(chunk_idx, m) is not None}
|
||
for num in chunk:
|
||
title = title_by_num[num]
|
||
consensus = [s["sub_title"] for s in await db.list_subblocks(topic, norm_by_num[num]) if s["status"] == "consensus"]
|
||
subs = final_by_num.get(num) or consensus
|
||
if not subs:
|
||
continue
|
||
raw[title] = subs
|
||
# align DB to the final list: final = consensus, rest discarded, add new ones.
|
||
final_norms = {_norm_title(s) for s in subs}
|
||
have = {s["sub_norm"] for s in await db.list_subblocks(topic, norm_by_num[num])}
|
||
for s in await db.list_subblocks(topic, norm_by_num[num]):
|
||
await db.set_subblock_fields(topic, norm_by_num[num], s["sub_norm"],
|
||
status=("consensus" if s["sub_norm"] in final_norms else "discarded"))
|
||
for s in subs:
|
||
sn = _norm_title(s)
|
||
if sn and sn not in have:
|
||
await db.upsert_subblock(topic, norm_by_num[num], sn, title, s)
|
||
await db.set_subblock_fields(topic, norm_by_num[num], sn, status="consensus")
|
||
await _dedup_subblocks(topic, raw) # near-dup filter per block (deterministic, no LLM)
|
||
if not raw:
|
||
# Finders ran but nothing survived the consensus/evidence gates: a legitimately
|
||
# thin block (e.g. a bare named reduction). {} = done-without-subs — the guide
|
||
# writes it from description+facts. Agent FAILURES return None elsewhere (retry).
|
||
_log(topic, "Subblocks: nichts Belegbares gefunden — Block bleibt ohne Subbausteine")
|
||
return {}
|
||
return raw
|
||
|
||
|
||
async def _dedup_subblocks(topic: str, raw: dict[str, list[str]]) -> None:
|
||
"""Deterministic near-duplicate filter per block: subblocks with cosine ≥
|
||
EMBEDDING_SUB_DUP are the same statement (reliable in the narrow block context — no LLM
|
||
needed). Per duplicate group keeps the most informative (longest); rest → DB discarded + out of `roh`.
|
||
Model missing → silently skip (like the rest of the embedding fallback)."""
|
||
if not EMBEDDING_AKTIV or not await asyncio.to_thread(embedding.available):
|
||
return
|
||
for title, subs in list(raw.items()):
|
||
if len(subs) < 2:
|
||
continue
|
||
sims = await asyncio.to_thread(embedding.embed_sims, subs)
|
||
if sims is None:
|
||
return
|
||
keepers: list[int] = []
|
||
discarded: list[int] = []
|
||
for i in sorted(range(len(subs)), key=lambda x: (-len(subs[x]), x)): # most informative first
|
||
if any(float(sims[i][j]) >= EMBEDDING_SUB_DUP for j in keepers):
|
||
discarded.append(i)
|
||
else:
|
||
keepers.append(i)
|
||
if not discarded:
|
||
continue
|
||
bnorm = _norm_title(title)
|
||
for i in discarded:
|
||
await db.set_subblock_fields(topic, bnorm, _norm_title(subs[i]), status="discarded")
|
||
raw[title] = [subs[i] for i in sorted(keepers)] # original order of the kept ones
|
||
|
||
|
||
def _code_vote(rater: list[dict], n: int) -> tuple[dict, dict]:
|
||
"""Majority vote over rater dicts on local ids 1..n → (outcome, disputed). A clear winner
|
||
needs ≥2 votes and no tie; otherwise the id is disputed (kept with its vote list)."""
|
||
outcome: dict[int, str] = {}
|
||
disputed: dict[int, list[str]] = {}
|
||
for k in range(1, n + 1):
|
||
vote_list = [d[k] for d in rater if k in d]
|
||
counter: dict[str, int] = {}
|
||
for s in vote_list:
|
||
counter[s] = counter.get(s, 0) + 1
|
||
best = max(counter.values(), default=0)
|
||
winners = [s for s, v in counter.items() if v == best]
|
||
if len(winners) == 1 and best >= 2:
|
||
outcome[k] = winners[0]
|
||
else:
|
||
disputed[k] = vote_list
|
||
return outcome, disputed
|
||
|
||
|
||
def _disputed_lines(items, item_idxs, disputed: dict) -> str:
|
||
"""Render disputed items as `k. [block] sub — Stimmen: a, b` lines for the judge prompt."""
|
||
return "\n".join(
|
||
f"{k}. [{items[item_idxs[k - 1]][0]}] {items[item_idxs[k - 1]][1]} — Stimmen: {', '.join(vote_list) or 'none'}"
|
||
for k, vote_list in disputed.items()
|
||
)
|
||
|
||
|
||
async def _levels_block(ctx: GenContext, set_p, files: dict, raw: dict, instructions: str, ns: str = "") -> dict | None:
|
||
"""Block C: three phases with a barrier — find (classify), select (vote), clarify.
|
||
Local IDs 1..n per package, mapped to global gid afterwards.
|
||
→ {block title: [{title, level}, …]} or None."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
# key points per sub as concise context (better-grounded classification; classification needs little).
|
||
facts_map = _json_file(files["facts"])
|
||
facts_map = facts_map if isinstance(facts_map, dict) else {}
|
||
items = [(title, sub) for title, subs in raw.items() for sub in subs] # global id = index+1
|
||
if not items:
|
||
return {title: [] for title in raw}
|
||
# pack chunks from WHOLE blocks (don't split a block) → each rater sees per block
|
||
# all subs and can classify relatively. Item indices per block in raw order.
|
||
chunks, cur, i = [], [], 0
|
||
for _title_b, subs in raw.items():
|
||
g = list(range(i, i + len(subs)))
|
||
i += len(subs)
|
||
if cur and len(cur) + len(g) > LEVEL_CHUNK:
|
||
chunks.append(cur)
|
||
cur = []
|
||
cur.extend(g)
|
||
if cur:
|
||
chunks.append(cur)
|
||
n = len(chunks)
|
||
|
||
def rater_paths(c):
|
||
return [work_dir / f"level-c{c}-{i}.json" for i in (1, 2, 3)]
|
||
|
||
def lset(item_idxs):
|
||
return set(range(1, len(item_idxs) + 1))
|
||
|
||
# Phase "Levels find": 3 raters per package (min. 2), local IDs.
|
||
async def _rate(c, item_idxs):
|
||
local_set = lset(item_idxs)
|
||
paths = rater_paths(c)
|
||
existing = sum(1 for p in paths if _levels_schema(_json_file(p), local_set))
|
||
if existing >= 2:
|
||
return True
|
||
enum_lines, cur_b = [], None
|
||
for k, j in enumerate(item_idxs, 1):
|
||
b, sub = items[j]
|
||
if b != cur_b:
|
||
enum_lines.append(f"\nBAUSTEIN: {b}")
|
||
cur_b = b
|
||
enum_lines.append(f"{k}. {sub}")
|
||
if (kz := _core_line(facts_map.get(b, {}).get(_norm_title(sub)))):
|
||
enum_lines.append(f" {kz}")
|
||
enum = "\n".join(enum_lines).strip()
|
||
pending = [(i, p) for i, p in enumerate(paths, 1) if not _levels_schema(_json_file(p), local_set)]
|
||
slots = [{
|
||
"key": f"blocks-{topic}-{ns}level-c{c}-{i}",
|
||
"prompt": _prompt("Levels-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)),
|
||
"role": "quick", "capabilities": "files",
|
||
"payload": (lambda result, p=p, ids=local_set: _levels_schema(_json_file(p), ids)),
|
||
} for i, p in pending]
|
||
new = await _race(topic, f"Levels package {c}", slots, 2 - existing, _timeout("level", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
return not is_cancelled() and new is not None
|
||
|
||
oks = await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Levels find"))
|
||
if is_cancelled():
|
||
return None
|
||
if not all(ok is True for ok in oks):
|
||
_blocks_errors[topic] = "Classification failed (research)"
|
||
return None
|
||
|
||
# Phase "Levels select": code vote per package → (outcome, disputed).
|
||
set_p(f"Levels select ({n} packages)…", step=_step_idx(topic, "Levels select"))
|
||
vote_by_c = {}
|
||
for c, item_idxs in enumerate(chunks, 1):
|
||
local_set = lset(item_idxs)
|
||
rater = [d for p in rater_paths(c) if (d := _levels_schema(_json_file(p), local_set))]
|
||
vote_by_c[c] = _code_vote(rater, len(item_idxs))
|
||
|
||
# Phase "Levels clarify": one judge per package on the disputed items, all in parallel.
|
||
async def _clarify(c, item_idxs):
|
||
outcome, strittig = vote_by_c[c]
|
||
if strittig:
|
||
judge_path = work_dir / f"level-final-c{c}.json"
|
||
decision = _levels_schema(_json_file(judge_path), set(strittig))
|
||
if decision is None:
|
||
disputed_block = _disputed_lines(items, item_idxs, strittig)
|
||
status, decision = await run_single_slot(
|
||
ctx, f"Levels-Clarification {c}",
|
||
key=f"blocks-{topic}-{ns}level-final-c{c}",
|
||
prompt=_prompt("Levels-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)),
|
||
role="judge", capabilities="files",
|
||
payload=lambda result, p=judge_path, ids=set(strittig): _levels_schema(_json_file(p), ids),
|
||
timeout=_timeout("level_check", len(strittig)),
|
||
)
|
||
if status == FAILED:
|
||
_log(topic, f"Levels clarification package {c} failed — default 'advanced'")
|
||
decision = decision if isinstance(decision, dict) else {}
|
||
# disputed without a decision → 'advanced'; vote winners stay; judge overrides.
|
||
outcome = {**{k: "advanced" for k in strittig}, **outcome, **decision}
|
||
return {item_idxs[k - 1] + 1: level for k, level in outcome.items()}
|
||
|
||
parts = await _gather_progress([_clarify(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Levels clarify"))
|
||
if is_cancelled():
|
||
return None
|
||
level_by_id: dict[int, str] = {}
|
||
for c, part in enumerate(parts, 1):
|
||
if not isinstance(part, dict):
|
||
# clarification is not fatal: vote outcome + default 'advanced' for disputed.
|
||
if isinstance(part, BaseException):
|
||
_log(topic, f"Levels clarification package {c}: {type(part).__name__}: {part}")
|
||
outcome, strittig = vote_by_c[c]
|
||
item_idxs = chunks[c - 1]
|
||
merged = {**{k: "advanced" for k in strittig}, **outcome}
|
||
part = {item_idxs[k - 1] + 1: s for k, s in merged.items()}
|
||
level_by_id.update(part)
|
||
|
||
# assemble the sidecar — same order as items → gid matches
|
||
sidecar: dict[str, list[dict]] = {}
|
||
gid = 0
|
||
for title, subs in raw.items():
|
||
lst = []
|
||
for sub in subs:
|
||
gid += 1
|
||
lst.append({"title": sub, "level": level_by_id.get(gid, "advanced")})
|
||
sidecar[title] = lst
|
||
return sidecar
|
||
|
||
|
||
_FACTS_FIELDS = ("key_points", "prerequisites", "hurdles", "cited_facts", "example_idea")
|
||
|
||
|
||
def _facts_schema(data) -> list[dict] | None:
|
||
"""{"facts": [{block, subblock, …}]} → valid list · otherwise None.
|
||
Strictly separates belegte_facts (with source) from example_idee (generative)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("facts"), list):
|
||
return None
|
||
out = []
|
||
for e in data["facts"]:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
blk = str(e.get("block", "")).strip()
|
||
sub = str(e.get("subblock", "")).strip()
|
||
if not blk or not sub:
|
||
continue
|
||
bf = [{"text": t, "source": str(f.get("source", "")).strip()}
|
||
for f in (e.get("cited_facts") or []) if isinstance(f, dict) and (t := str(f.get("text", "")).strip())]
|
||
out.append({
|
||
"block": blk, "subblock": sub,
|
||
"key_points": [k for x in (e.get("key_points") or []) if (k := str(x).strip())],
|
||
"prerequisites": str(e.get("prerequisites", "")).strip(),
|
||
"hurdles": str(e.get("hurdles", "")).strip(),
|
||
"cited_facts": bf,
|
||
"example_idea": str(e.get("example_idea", "")).strip(),
|
||
})
|
||
return out or None
|
||
|
||
|
||
def _facts_check_schema(data) -> list[tuple[str, bool]] | None:
|
||
"""Facts check → [(sub_norm, verwerfen)] per objection · {ok:true}→[] · None if invalid.
|
||
verwerfen=True: sub not supportable in substance (remove). verwerfen=False: only correct the fact."""
|
||
if not isinstance(data, dict):
|
||
return None
|
||
if data.get("ok") is True:
|
||
return []
|
||
pr = data.get("problems")
|
||
if not isinstance(pr, list):
|
||
return None
|
||
return [(sn, bool(p.get("discard")))
|
||
for p in pr if isinstance(p, dict) and (sn := _norm_title(str(p.get("subblock", ""))))]
|
||
|
||
|
||
def _core_line(fk) -> str:
|
||
"""Concise key-point line for classification (level/relevance) — less context suffices there.
|
||
Empty if no facts/key points (legacy)."""
|
||
if not isinstance(fk, dict) or not fk.get("key_points"):
|
||
return ""
|
||
return "Kern: " + " · ".join(str(k) for k in fk["key_points"])
|
||
|
||
|
||
def _facts_lines(fk: dict) -> str:
|
||
z = []
|
||
if fk.get("key_points"):
|
||
z.append("Kernpunkte: " + " · ".join(str(k) for k in fk["key_points"]))
|
||
if fk.get("prerequisites"):
|
||
z.append("Voraussetzung: " + fk["prerequisites"])
|
||
if fk.get("hurdles"):
|
||
z.append("Hürde: " + fk["hurdles"])
|
||
for bf in fk.get("cited_facts", []):
|
||
z.append(f"FAKT: {bf['text']} (Source: {bf.get('source', '?')})")
|
||
if fk.get("example_idea"):
|
||
z.append("Example: " + fk["example_idea"])
|
||
return "\n".join(z)
|
||
|
||
|
||
def _facts_complete(files: dict) -> bool:
|
||
"""Does the facts map exist (block done)? {block: {sub_norm: {...}}}."""
|
||
d = _json_file(files["facts"])
|
||
return isinstance(d, dict) and bool(d)
|
||
|
||
|
||
async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str, ns: str = "") -> tuple | None:
|
||
"""Block: per sub extract source facts (find) → verify (check) → correct/discard (fix).
|
||
Extract-once grounding: the result feeds level/relevance/questions/guide.
|
||
→ (facts_map, discarded_map) — facts_map {block: {sub_norm: facts}}, discarded_map
|
||
{block: {sub_norm}} (unsupportable subs to remove) — or None on cancel/error."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
caps = "files" if folder else "full"
|
||
type = q.get("type", "thema")
|
||
source = _prompt(_SOURCE_TEMPLATE[type], project=folder) if type in _SOURCE_TEMPLATE else _prompt("Blocks-Source-Thema", topic=topic)
|
||
blocks = [(title, [str(s).strip() for s in subs if str(s).strip()]) for title, subs in raw.items() if subs]
|
||
if not blocks:
|
||
return {}, {}
|
||
chunks = _lpt_chunks([len(subs) for _, subs in blocks], FACTS_CHUNK_SUBS)
|
||
|
||
def raw_path(ci): return work_dir / f"facts-c{ci}.json"
|
||
def supp_path(ci): return work_dir / f"facts-erg-c{ci}.json"
|
||
def chk_path(ci, j): return work_dir / f"facts-check-c{ci}-j{j}.json"
|
||
def fix_path(ci): return work_dir / f"facts-fix-c{ci}.json"
|
||
def ctitle(idxs): return [blocks[i][0] for i in idxs]
|
||
def block_text(idxs):
|
||
return "\n\n".join(
|
||
f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in blocks[i][1])
|
||
for i in idxs)
|
||
|
||
# raw facts of a chunk → {block: {sub_norm: {sub, …fields}}}, matched to chunk titles.
|
||
def raw_map(ci, path):
|
||
idxs = chunks[ci]
|
||
rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
|
||
ct = ctitle(idxs)
|
||
out: dict[str, dict] = {}
|
||
for e in _facts_schema(_json_file(path)) or []:
|
||
bt = _match_sub(e["block"], ct)
|
||
if bt not in rel_by:
|
||
continue
|
||
sub = _match_sub(e["subblock"], rel_by[bt])
|
||
out.setdefault(bt, {})[_norm_title(sub)] = {"sub": sub, **{k: e[k] for k in _FACTS_FIELDS}}
|
||
return out
|
||
|
||
# union raw facts + completeness supplements (recall): only subs that exist in raw.
|
||
def _chunk_facts(ci):
|
||
raw = raw_map(ci, raw_path(ci))
|
||
erg = raw_map(ci, supp_path(ci)) if supp_path(ci).exists() else {}
|
||
if not erg:
|
||
return raw
|
||
for bt, fm in raw.items():
|
||
ebt = erg.get(bt, {})
|
||
for sn, fk in fm.items():
|
||
ek = ebt.get(sn)
|
||
if not ek:
|
||
continue
|
||
seen = {str(k).strip().casefold() for k in fk.get("key_points", [])}
|
||
for k in ek.get("key_points", []):
|
||
if str(k).strip().casefold() not in seen:
|
||
seen.add(str(k).strip().casefold())
|
||
fk["key_points"].append(k)
|
||
seent = {bf["text"].strip().casefold() for bf in fk.get("cited_facts", [])}
|
||
for bf in ek.get("cited_facts", []):
|
||
if bf["text"].strip().casefold() not in seent:
|
||
seent.add(bf["text"].strip().casefold())
|
||
fk["cited_facts"].append(bf)
|
||
for f in ("prerequisites", "hurdles", "example_idea"):
|
||
if not fk.get(f) and ek.get(f):
|
||
fk[f] = ek[f]
|
||
return raw
|
||
|
||
# Phase "Facts find": 1 generator per chunk.
|
||
async def _find(ci, idxs):
|
||
fp = raw_path(ci)
|
||
if _facts_schema(_json_file(fp)):
|
||
return True
|
||
subs_total = sum(len(blocks[i][1]) for i in idxs)
|
||
status, _r = await run_single_slot(
|
||
ctx, f"Facts {ci}", key=f"blocks-{topic}-{ns}facts-c{ci}",
|
||
prompt=_prompt("Facts-Research", topic=topic, source=source, blocks=block_text(idxs), out_path=fp, extra=_extra(instructions)),
|
||
role="quick", capabilities=caps,
|
||
payload=lambda result, p=fp: _facts_schema(_json_file(p)),
|
||
timeout=_timeout("content", subs_total))
|
||
return status != FAILED and _facts_schema(_json_file(fp)) is not None
|
||
|
||
oks = await _gather_progress([_find(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts find"))
|
||
if is_cancelled():
|
||
return None
|
||
if not any(ok is True for ok in oks):
|
||
_blocks_errors[topic] = "Facts extraction failed"
|
||
return None
|
||
|
||
# Phase "Facts ergänzen" (recall): a targeted gap hunt per chunk looks for source-backed facts that
|
||
# the single pass missed. Best-effort — never fails (no erg file → merge uses only raw).
|
||
async def _supplement(ci, idxs):
|
||
ep = supp_path(ci)
|
||
if _facts_schema(_json_file(ep)):
|
||
return
|
||
per = raw_map(ci, raw_path(ci))
|
||
if not per:
|
||
return
|
||
block = "\n\n".join(
|
||
f"BLOCK: {bt}\nSUBBAUSTEINE (mit bereits erfassten Facts):\n" + "\n".join(
|
||
f"- {fk['sub']}\n Erfasst: " + ("; ".join(
|
||
list(fk.get("key_points", [])) + [bf["text"] for bf in fk.get("cited_facts", [])]) or "(nichts)")
|
||
for fk in fm.values())
|
||
for bt, fm in per.items())
|
||
subs_total = sum(len(blocks[i][1]) for i in idxs)
|
||
await run_single_slot(
|
||
ctx, f"Facts supplement {ci}", key=f"blocks-{topic}-{ns}facts-erg-c{ci}",
|
||
prompt=_prompt("Facts-Supplement", topic=topic, source=source, blocks=block, out_path=ep, extra=_extra(instructions)),
|
||
role="quick", capabilities=caps,
|
||
payload=lambda result, p=ep: _facts_schema(_json_file(p)),
|
||
timeout=_timeout("content", subs_total))
|
||
|
||
set_p("Facts supplement…", step=_step_idx(topic, "Facts find"))
|
||
await _gather_progress([_supplement(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts find"))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# Phase "Facts check": FACTS_CHECK_PANEL judges per chunk. Two majority sets:
|
||
# flagged (fact inaccurate → correct) and discard (sub not supportable → remove).
|
||
async def _check(ci, idxs):
|
||
per = _chunk_facts(ci) # raw + supplements → panel verifies the union
|
||
if not per:
|
||
return ci, set(), set()
|
||
facts_text = "\n\n".join(f"SUBBAUSTEIN: {fk['sub']}\n{_facts_lines(fk)}" for fm in per.values() for fk in fm.values())
|
||
pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _facts_check_schema(_json_file(chk_path(ci, j))) is None]
|
||
await asyncio.gather(*[
|
||
run_agent(f"blocks-{topic}-{ns}facts-check-c{ci}-j{j}",
|
||
_prompt("Facts-Check", topic=topic, source=source, facts=facts_text, out_path=chk_path(ci, j), extra=_extra(instructions)),
|
||
_timeout("content_check", len(per)), provider=provider, role="judge", capabilities=caps)
|
||
for j in pending], return_exceptions=True)
|
||
outs = [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if (s := _facts_check_schema(_json_file(chk_path(ci, j)))) is not None]
|
||
bvotes: dict[str, int] = {}
|
||
vvotes: dict[str, int] = {}
|
||
for s in outs: # s = [(sub_norm, verwerfen)] of one judge
|
||
gb, gv = set(), set()
|
||
for sn, disc in s:
|
||
if sn not in gb:
|
||
gb.add(sn); bvotes[sn] = bvotes.get(sn, 0) + 1
|
||
if disc and sn not in gv:
|
||
gv.add(sn); vvotes[sn] = vvotes.get(sn, 0) + 1
|
||
threshold = len(outs) / 2 if outs else 99
|
||
flagged = {sn for sn, v in bvotes.items() if v > threshold}
|
||
# Discarding is irreversible → stricter than flagging: majority AND ≥2 agreeing judges
|
||
# (prevents deletion by a single vote when the panel is degraded).
|
||
to_discard = {sn for sn, v in vvotes.items() if v > threshold and v >= 2}
|
||
return ci, flagged, to_discard
|
||
|
||
check = await _gather_progress([_check(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, "Facts check"))
|
||
if is_cancelled():
|
||
return None
|
||
flagged: dict[int, set] = {}
|
||
to_discard: dict[int, set] = {}
|
||
for r in check:
|
||
if isinstance(r, tuple) and len(r) == 3:
|
||
ci, b, v = r
|
||
flagged[ci] = b
|
||
to_discard[ci] = v
|
||
|
||
# Phase "Facts fix": re-extract only CORRECTABLE ones (flagged without discard).
|
||
correctable = {ci: (flagged.get(ci, set()) - to_discard.get(ci, set())) for ci in flagged}
|
||
n_problem = sum(len(s) for s in correctable.values())
|
||
if n_problem:
|
||
set_p(f"Correcting facts ({n_problem})…", step=_step_idx(topic, "Facts fix"))
|
||
async def _fix(ci):
|
||
subs_norm = correctable.get(ci, set())
|
||
if not subs_norm or _facts_schema(_json_file(fix_path(ci))):
|
||
return
|
||
idxs = chunks[ci]
|
||
rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
|
||
goal = []
|
||
for bt, subs in rel_by.items():
|
||
affected_subs = [s for s in subs if _norm_title(s) in subs_norm]
|
||
if affected_subs:
|
||
goal.append(f"BLOCK: {bt}\nSUBBAUSTEINE:\n" + "\n".join(f"- {s}" for s in affected_subs))
|
||
if not goal:
|
||
return
|
||
await run_single_slot(
|
||
ctx, f"Facts-Fix {ci}", key=f"blocks-{topic}-{ns}facts-fix-c{ci}",
|
||
prompt=_prompt("Facts-Research", topic=topic, source=source, blocks="\n\n".join(goal), out_path=fix_path(ci), extra=_extra(instructions)),
|
||
role="quick", capabilities=caps,
|
||
payload=lambda result, p=fix_path(ci): _facts_schema(_json_file(p)),
|
||
timeout=_timeout("content", len(subs_norm)))
|
||
await _gather_progress([_fix(ci) for ci in correctable], len(correctable), _report_p(set_p, topic, "Facts fix"))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# assemble: raw + fix overrides for corrected. Discarded subs out (+ report per block).
|
||
outcome: dict[str, dict] = {}
|
||
discarded_map: dict[str, set] = {}
|
||
for ci in range(len(chunks)):
|
||
per = _chunk_facts(ci) # raw + supplements (recall); fix overrides only corrected
|
||
fix = raw_map(ci, fix_path(ci)) if fix_path(ci).exists() else {}
|
||
disc = to_discard.get(ci, set())
|
||
for bt, fm in per.items():
|
||
for sn, fk in fm.items():
|
||
if sn in disc:
|
||
discarded_map.setdefault(bt, set()).add(sn)
|
||
continue
|
||
winners = fix.get(bt, {}).get(sn, fk) if sn in correctable.get(ci, set()) else fk
|
||
outcome.setdefault(bt, {})[sn] = {k: winners[k] for k in _FACTS_FIELDS}
|
||
if discarded_map:
|
||
_log(topic, f"Facts check discards {sum(len(s) for s in discarded_map.values())} unsupportable subblocks")
|
||
return outcome, discarded_map
|
||
|
||
|
||
async def _relevance_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "") -> dict | None:
|
||
"""Block D: three phases with a barrier — find (relevant/peripheral), select (vote), clarify.
|
||
Items from the sidecar; local IDs 1..n per package → global gid.
|
||
→ {gid: relevance} or None on cancel/research error. Default on gap/dispute: 'relevant'."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
items = [(title, sub["title"], sub.get("facts")) for title, subs in sidecar.items() for sub in subs] # global id = index+1
|
||
if not items:
|
||
return {}
|
||
chunks = _chunk_nums(list(range(len(items))), _n_chunks(len(items), LEVEL_CHUNK))
|
||
n = len(chunks)
|
||
|
||
def rater_paths(c):
|
||
return [work_dir / f"relevance-c{c}-{i}.json" for i in (1, 2, 3)]
|
||
|
||
def lset(item_idxs):
|
||
return set(range(1, len(item_idxs) + 1))
|
||
|
||
# Phase "Relevance find": 3 raters per package (min. 2), local IDs.
|
||
async def _rate(c, item_idxs):
|
||
local_set = lset(item_idxs)
|
||
paths = rater_paths(c)
|
||
existing = sum(1 for p in paths if _relevance_schema(_json_file(p), local_set))
|
||
if existing >= 2:
|
||
return True
|
||
enum_lines = []
|
||
for k, j in enumerate(item_idxs, 1):
|
||
enum_lines.append(f"{k}. [{items[j][0]}] {items[j][1]}")
|
||
if (kz := _core_line(items[j][2])):
|
||
enum_lines.append(f" {kz}")
|
||
enum = "\n".join(enum_lines)
|
||
pending = [(i, p) for i, p in enumerate(paths, 1) if not _relevance_schema(_json_file(p), local_set)]
|
||
slots = [{
|
||
"key": f"blocks-{topic}-{ns}relevance-c{c}-{i}",
|
||
"prompt": _prompt("Relevance-Research", topic=topic, subblocks=enum, out_path=p, extra=_extra(instructions)),
|
||
"role": "quick", "capabilities": "files",
|
||
"payload": (lambda result, p=p, ids=local_set: _relevance_schema(_json_file(p), ids)),
|
||
} for i, p in pending]
|
||
new = await _race(topic, f"Relevance package {c}", slots, 2 - existing, _timeout("relevance", len(item_idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
return not is_cancelled() and new is not None
|
||
|
||
oks = await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Relevance find"))
|
||
if is_cancelled():
|
||
return None
|
||
if not all(ok is True for ok in oks):
|
||
_blocks_errors[topic] = "Relevance failed (research)"
|
||
return None
|
||
|
||
# Phase "Relevance select": code vote per package → (outcome, disputed).
|
||
set_p(f"Relevance select ({n} packages)…", step=_step_idx(topic, "Relevance select"))
|
||
vote_by_c = {}
|
||
for c, item_idxs in enumerate(chunks, 1):
|
||
local_set = lset(item_idxs)
|
||
rater = [d for p in rater_paths(c) if (d := _relevance_schema(_json_file(p), local_set))]
|
||
vote_by_c[c] = _code_vote(rater, len(item_idxs))
|
||
|
||
# Phase "Relevance clarify": one judge per package on the disputed items, all in parallel.
|
||
async def _clarify(c, item_idxs):
|
||
outcome, strittig = vote_by_c[c]
|
||
if strittig:
|
||
judge_path = work_dir / f"relevance-final-c{c}.json"
|
||
decision = _relevance_schema(_json_file(judge_path), set(strittig))
|
||
if decision is None:
|
||
disputed_block = _disputed_lines(items, item_idxs, strittig)
|
||
status, decision = await run_single_slot(
|
||
ctx, f"Relevance-Clarification {c}",
|
||
key=f"blocks-{topic}-{ns}relevance-final-c{c}",
|
||
prompt=_prompt("Relevance-Mapping", topic=topic, disputed=disputed_block, out_path=judge_path, extra=_extra(instructions)),
|
||
role="judge", capabilities="files",
|
||
payload=lambda result, p=judge_path, ids=set(strittig): _relevance_schema(_json_file(p), ids),
|
||
timeout=_timeout("relevance_check", len(strittig)),
|
||
)
|
||
if status == FAILED:
|
||
_log(topic, f"Relevance clarification package {c} failed — default 'relevant'")
|
||
decision = decision if isinstance(decision, dict) else {}
|
||
# disputed without a decision → 'relevant' (never accidentally exclude).
|
||
outcome = {**{k: "relevant" for k in strittig}, **outcome, **decision}
|
||
return {item_idxs[k - 1] + 1: rel for k, rel in outcome.items()}
|
||
|
||
parts = await _gather_progress([_clarify(c, idxs) for c, idxs in enumerate(chunks, 1)], len(chunks), _report_p(set_p, topic, "Relevance clarify"))
|
||
if is_cancelled():
|
||
return None
|
||
relevance_by_id: dict[int, str] = {}
|
||
for c, part in enumerate(parts, 1):
|
||
if not isinstance(part, dict):
|
||
# clarification is not fatal: vote outcome + default 'relevant' for disputed.
|
||
if isinstance(part, BaseException):
|
||
_log(topic, f"Relevance clarification package {c}: {type(part).__name__}: {part}")
|
||
outcome, strittig = vote_by_c[c]
|
||
item_idxs = chunks[c - 1]
|
||
merged = {**{k: "relevant" for k in strittig}, **outcome}
|
||
part = {item_idxs[k - 1] + 1: s for k, s in merged.items()}
|
||
relevance_by_id.update(part)
|
||
return relevance_by_id
|
||
|
||
|
||
def _match_sub(agent_sub: str, rel: list[str]) -> str:
|
||
"""Map the agent's subblock title to the matching relevant title — exact,
|
||
then normalized, then substring (the agent drops e.g. the prefix "Question: ").
|
||
No match → keep the agent title. This way NO pattern is lost to a title mismatch."""
|
||
if agent_sub in rel:
|
||
return agent_sub
|
||
an = _norm_title(agent_sub)
|
||
for r in rel:
|
||
rn = _norm_title(r)
|
||
if an and rn and (an == rn or an in rn or rn in an):
|
||
return r
|
||
return agent_sub
|
||
|
||
|
||
async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "") -> dict | None:
|
||
"""Block E (chunks of 10): find (1 generator per ~10 blocks, parallel), select (code:
|
||
group per block + dedup), clarify (1 critic per chunk), check (catch-up round).
|
||
Assignment per entry via the `block` field (a chunk file carries several blocks).
|
||
→ {block title: [{subblock, question}, …]} or None on cancel."""
|
||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
# ALL subblocks (including peripheral) get a pattern — peripheral is testable in the FGuide level.
|
||
# facts_by: full facts context per sub (generation benefits — better questions).
|
||
blocks = []
|
||
facts_by: dict[tuple, dict] = {}
|
||
for title, subs in sidecar.items():
|
||
all_titles = []
|
||
for s in subs:
|
||
if isinstance(s, dict) and (st := str(s.get("title", "")).strip()):
|
||
all_titles.append(st)
|
||
if isinstance(s.get("facts"), dict):
|
||
facts_by[(title, _norm_title(st))] = s["facts"]
|
||
if all_titles:
|
||
blocks.append((title, all_titles))
|
||
if not blocks:
|
||
return {}
|
||
chunks = _lpt_chunks([len(rel) for _, rel in blocks], QUESTION_CHUNK_SUBS) # load-balanced by sub count
|
||
|
||
def raw_path(ci):
|
||
return work_dir / f"question-pattern-c{ci}.json"
|
||
|
||
def final_path(ci):
|
||
return work_dir / f"question-pattern-final-c{ci}.json"
|
||
|
||
def _chunk_title(idxs):
|
||
return [blocks[i][0] for i in idxs]
|
||
|
||
# Phase "Questions find": 1 generator per chunk, all in parallel.
|
||
async def _find(ci, idxs):
|
||
fp = raw_path(ci)
|
||
if _question_pattern_chunk_schema(_json_file(fp)):
|
||
return # Resume
|
||
def _sub_line(bi, s):
|
||
line = f"- {s}"
|
||
fk = facts_by.get((blocks[bi][0], _norm_title(s)))
|
||
if fk and (ft := _facts_lines(fk)):
|
||
line += "\n" + "\n".join(" " + l for l in ft.split("\n"))
|
||
return line
|
||
block = "\n\n".join(
|
||
f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(_sub_line(i, s) for s in blocks[i][1])
|
||
for i in idxs
|
||
)
|
||
subs_total = sum(len(blocks[i][1]) for i in idxs)
|
||
status, _ = await run_single_slot(
|
||
ctx, f"Question-Pattern {ci}",
|
||
key=f"blocks-{topic}-{ns}question-pattern-c{ci}",
|
||
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=block,
|
||
out_path=fp, extra=_extra(instructions)),
|
||
role="quick", capabilities="files",
|
||
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
|
||
timeout=_timeout("question_pattern", subs_total),
|
||
)
|
||
if status == FAILED:
|
||
_log(topic, f"Question pattern chunk {ci} failed — blocks in fallback (catch-up round/live)")
|
||
|
||
async def find_all(ci_list):
|
||
ci_list = list(ci_list)
|
||
await _gather_progress([_find(ci, chunks[ci]) for ci in ci_list], len(ci_list), _report_p(set_p, topic, "Questions find"))
|
||
|
||
await find_all(range(len(chunks)))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# Phase "Questions select": code — group chunk files per block, drop duplicates,
|
||
# loosely map block/subblock titles to the targets (discard nothing for a mismatch).
|
||
def _select_chunk(ci):
|
||
idxs = chunks[ci]
|
||
ctitle = _chunk_title(idxs)
|
||
rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
|
||
out, seen_set = {}, {}
|
||
for e in _question_pattern_chunk_schema(_json_file(raw_path(ci))) or []:
|
||
title = _match_sub(e["block"], ctitle)
|
||
if title not in rel_by:
|
||
continue # not assignable → discard
|
||
sub = _match_sub(e["subblock"], rel_by[title])
|
||
seen = seen_set.setdefault(title, set())
|
||
if sub in seen:
|
||
continue # exactly one pattern per subblock
|
||
seen.add(sub)
|
||
out.setdefault(title, []).append({"subblock": sub, "question": e["question"]})
|
||
return out
|
||
|
||
def _select_all(ci_list):
|
||
raw = {}
|
||
for ci in ci_list:
|
||
for title, eintraege in _select_chunk(ci).items():
|
||
raw.setdefault(title, []).extend(eintraege)
|
||
return raw
|
||
|
||
set_p("Questions select…", step=_step_idx(topic, "Questions select"))
|
||
raw_by_title = _select_all(range(len(chunks)))
|
||
|
||
# Phase "Questions clarify": 1 critic per chunk cleans up the tables (grouped by block).
|
||
async def _clarify(ci, idxs):
|
||
fp = final_path(ci)
|
||
if _question_pattern_chunk_schema(_json_file(fp)):
|
||
return # resume
|
||
block_texts = []
|
||
for i in idxs:
|
||
t = blocks[i][0]
|
||
eintraege = raw_by_title.get(t) or []
|
||
if not eintraege:
|
||
continue
|
||
lines = "\n".join(f"- ({e['subblock']}) {e['question']}" for e in eintraege)
|
||
block_texts.append(f"BLOCK: {t}\n{lines}")
|
||
if not block_texts:
|
||
return # nothing to clarify in this chunk
|
||
subs_total = sum(len(blocks[i][1]) for i in idxs)
|
||
status, _ = await run_single_slot(
|
||
ctx, f"Question-Pattern-Clarification {ci}",
|
||
key=f"blocks-{topic}-{ns}question-pattern-final-c{ci}",
|
||
prompt=_prompt("Question-Pattern-Critique", topic=topic, table="\n\n".join(block_texts), out_path=fp, extra=_extra(instructions)),
|
||
role="judge", capabilities="files",
|
||
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
|
||
timeout=_timeout("question_pattern_check", subs_total),
|
||
)
|
||
if status == FAILED:
|
||
_log(topic, f"Question pattern clarification chunk {ci} failed — raw pattern adopted")
|
||
|
||
async def clarify_all(ci_list):
|
||
ci_list = list(ci_list)
|
||
await _gather_progress([_clarify(ci, chunks[ci]) for ci in ci_list], len(ci_list), _report_p(set_p, topic, "Questions clarify"))
|
||
|
||
await clarify_all(range(len(chunks)))
|
||
if is_cancelled():
|
||
return None
|
||
|
||
# Clarified chunk table per block, fallback to raw pattern. Map titles loosely.
|
||
def _final_by_title(ci_list):
|
||
out = {}
|
||
for ci in ci_list:
|
||
idxs = chunks[ci]
|
||
ctitle = _chunk_title(idxs)
|
||
rel_by = {blocks[i][0]: blocks[i][1] for i in idxs}
|
||
for e in _question_pattern_chunk_schema(_json_file(final_path(ci))) or []:
|
||
title = _match_sub(e["block"], ctitle)
|
||
if title not in rel_by:
|
||
continue
|
||
out.setdefault(title, []).append(
|
||
{"subblock": _match_sub(e["subblock"], rel_by[title]), "question": e["question"]})
|
||
return out
|
||
|
||
final_by_title = _final_by_title(range(len(chunks)))
|
||
outcome = {t: (final_by_title.get(t) or raw_by_title.get(t) or []) for t, _ in blocks}
|
||
|
||
# Phase "Questions check": per-sub completeness. Generators crash randomly (~15 %),
|
||
# 1 agent per chunk without retry → subs (whole blocks) fall through silently. Hence several
|
||
# rounds that re-request ONLY the missing subs (short packages, Question-Pattern-Research).
|
||
set_p("Questions check…", step=_step_idx(topic, "Questions check"))
|
||
|
||
def _missing_subs() -> list[tuple[str, list[str]]]:
|
||
out = []
|
||
for t, subs in blocks:
|
||
have_set = {_norm_title(e["subblock"]) for e in outcome.get(t) or []}
|
||
miss = [s for s in subs if _norm_title(s) not in have_set]
|
||
if miss:
|
||
out.append((t, miss))
|
||
return out
|
||
|
||
def _followup_block(items): # items: [(block_title, [missing sub_title])]
|
||
block_texts = []
|
||
for title, subs in items:
|
||
lines = []
|
||
for s in subs:
|
||
z = f"- {s}"
|
||
fk = facts_by.get((title, _norm_title(s)))
|
||
if fk and (ft := _facts_lines(fk)):
|
||
z += "\n" + "\n".join(" " + l for l in ft.split("\n"))
|
||
lines.append(z)
|
||
block_texts.append(f"BLOCK: {title}\nSUBBAUSTEINE:\n" + "\n".join(lines))
|
||
return "\n\n".join(block_texts)
|
||
|
||
async def _request_more(round_n, pi, items):
|
||
fp = work_dir / f"question-pattern-nach{round_n}-c{pi}.json"
|
||
if _question_pattern_chunk_schema(_json_file(fp)):
|
||
return # resume
|
||
subs_total = sum(len(s) for _, s in items)
|
||
await run_single_slot(
|
||
ctx, f"Question pattern catch-up R{round_n}/{pi}",
|
||
key=f"blocks-{topic}-{ns}question-pattern-nach{round_n}-c{pi}",
|
||
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=_followup_block(items),
|
||
out_path=fp, extra=_extra(instructions)),
|
||
role="quick", capabilities="files",
|
||
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
|
||
timeout=_timeout("question_pattern", subs_total),
|
||
)
|
||
|
||
for round_n in range(1, QUESTION_MAX_ROUNDS + 1):
|
||
missing_subs = _missing_subs()
|
||
if not missing_subs:
|
||
break
|
||
n_subs = sum(len(s) for _, s in missing_subs)
|
||
_log(topic, f"Question pattern round {round_n}: {n_subs} sub(s) in {len(missing_subs)} block(s) without a pattern — re-request")
|
||
packages = _lpt_chunks([len(s) for _, s in missing_subs], QUESTION_CHUNK_SUBS)
|
||
package_items = [[missing_subs[i] for i in idxs] for idxs in packages]
|
||
await _gather_progress(
|
||
[_request_more(round_n, pi, items) for pi, items in enumerate(package_items)],
|
||
len(package_items), _report_p(set_p, topic, "Questions check"))
|
||
if is_cancelled():
|
||
return None
|
||
# parse output per package + merge newly gained subs (don't overwrite existing ones).
|
||
for pi, items in enumerate(package_items):
|
||
title_subs = {t: subs for t, subs in items}
|
||
ctitle = list(title_subs.keys())
|
||
for e in _question_pattern_chunk_schema(_json_file(work_dir / f"question-pattern-nach{round_n}-c{pi}.json")) or []:
|
||
title = _match_sub(e["block"], ctitle)
|
||
if title not in title_subs:
|
||
continue
|
||
sub = _match_sub(e["subblock"], title_subs[title])
|
||
have_set = {_norm_title(x["subblock"]) for x in outcome.get(title) or []}
|
||
if _norm_title(sub) in have_set:
|
||
continue
|
||
outcome.setdefault(title, []).append({"subblock": sub, "question": e["question"]})
|
||
|
||
rest = _missing_subs()
|
||
if rest:
|
||
n = sum(len(s) for _, s in rest)
|
||
_log(topic, f"Question pattern: {n} sub(s) in {len(rest)} block(s) remain empty after {QUESTION_MAX_ROUNDS} rounds: {[t for t, _ in rest][:5]}")
|
||
return outcome
|
||
|
||
|
||
# ── Inventory in the DB: research loop · consolidation · clarification ────────────
|
||
|
||
|
||
|
||
def _crawl_index(folder) -> dict[str, str]:
|
||
"""Alias (filename OR QUELLE: URL, lowercase) → canonical page key (filename)."""
|
||
idx: dict[str, str] = {}
|
||
if not folder or not Path(folder).is_dir():
|
||
return idx
|
||
for p in sorted(Path(folder).glob("*.txt")):
|
||
key = p.name
|
||
idx[key.lower()] = key
|
||
try:
|
||
first_line = p.read_text(encoding="utf-8").splitlines()[0]
|
||
except (OSError, IndexError):
|
||
first_line = ""
|
||
if first_line.startswith("QUELLE:"):
|
||
url = first_line[len("QUELLE:"):].strip()
|
||
if url:
|
||
idx[url.lower()] = key
|
||
idx[url.rstrip("/").lower()] = key
|
||
return idx
|
||
|
||
|
||
|
||
|
||
|
||
|
||
async def _set_inventory(topic: str, record: str, status: str) -> None:
|
||
"""Write an inventory entry ('title — description') with status to the DB."""
|
||
title = _title(record)
|
||
norm = _norm_title(title)
|
||
if not norm:
|
||
return
|
||
split_parts = [t.strip() for t in record.split(" — ")]
|
||
desc = split_parts[1] if len(split_parts) >= 2 else ""
|
||
await db.upsert_block(topic, norm, title, desc)
|
||
await db.set_block_status(topic, norm, status)
|
||
|
||
|
||
def _triage_rules(folder, pages: list[str]) -> tuple[list[str], list[str]]:
|
||
"""Deterministic content/noise filter (config.CRAWL_*). Substring match (lowercase) against
|
||
URL + filename. Order: keep > noise > min_chars > keep. → (content, noise)."""
|
||
folder = Path(folder)
|
||
content, noise = [], []
|
||
for fn in pages:
|
||
lines = _read(folder / fn).splitlines()
|
||
url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else ""
|
||
body = "\n".join(lines[1:]).strip()
|
||
hay = f"{url}\n{fn}".lower()
|
||
if any(p in hay for p in CRAWL_KEEP_PATTERNS):
|
||
content.append(fn)
|
||
elif any(p in hay for p in CRAWL_NOISE_PATTERNS):
|
||
noise.append(fn)
|
||
elif len(body) < CRAWL_MIN_CHARS:
|
||
noise.append(fn)
|
||
else:
|
||
content.append(fn) # default: keep — everything with content stays
|
||
return content, noise
|
||
|
||
|
||
def _page_snippet(folder, fn: str) -> tuple[str, str]:
|
||
"""(url, snippet) of a crawl page for the relevance gate. url from the QUELLE: line;
|
||
snippet = body excerpt (navigation boilerplate is up front — the prompt ignores it).
|
||
The URL is the primary signal (meaningful slug), the snippet only supports it."""
|
||
lines = _read(Path(folder) / fn).splitlines()
|
||
url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else ""
|
||
body = "\n".join(lines[1:]).strip()
|
||
snippet = " ".join(body.split())[:QUELLE_RELEVANZ_SNIPPET]
|
||
return (url or fn), snippet
|
||
|
||
|
||
async def _relevance_triage(ctx: GenContext, set_p, files: dict, folder, content: list[str], spec: str, instructions: str) -> tuple[list[str], list[str]]:
|
||
"""LLM topic gate after the rule filter: each content page ja/nein against the spec.
|
||
Off-topic (different field) → out. Pattern like `_relevance_block`: small packages, 3 raters
|
||
(`fast`), 2-of-3 consensus. CONSERVATIVE: drop only on a clear "nein" majority; dispute/gap/
|
||
race error → keep. SAFETY: if the gate would drop ≥80 % (or all), everything stays
|
||
(a spec mismatch/bug must not empty the source). → (kept, out) as filenames."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
pages = sorted(content)
|
||
if not pages:
|
||
return content, []
|
||
items = [_page_snippet(folder, fn) for fn in pages] # index aligns with `pages`
|
||
chunks = _chunk_nums(list(range(len(pages))), _n_chunks(len(pages), QUELLE_RELEVANZ_CHUNK))
|
||
n = len(chunks)
|
||
|
||
def rater_paths(c):
|
||
return [work_dir / f"source-relevance-c{c}-{i}.json" for i in (1, 2, 3)]
|
||
|
||
def lset(idxs):
|
||
return set(range(1, len(idxs) + 1))
|
||
|
||
async def _rate(c, idxs):
|
||
local_set = lset(idxs)
|
||
paths = rater_paths(c)
|
||
existing = sum(1 for p in paths if _yesno_schema(_json_file(p), local_set))
|
||
if existing >= 2:
|
||
return True
|
||
enum_lines = []
|
||
for k, j in enumerate(idxs, 1):
|
||
url, snip = items[j]
|
||
enum_lines.append(f"{k}. {url}")
|
||
if snip:
|
||
enum_lines.append(f" {snip}")
|
||
enum = "\n".join(enum_lines)
|
||
pending = [(i, p) for i, p in enumerate(paths, 1) if not _yesno_schema(_json_file(p), local_set)]
|
||
slots = [{
|
||
"key": f"blocks-{topic}-source-relevance-c{c}-{i}",
|
||
"prompt": _prompt("Source-Relevance", topic=topic, spec=spec, pages=enum, out_path=p, extra=_extra(instructions)),
|
||
"role": "fast", "capabilities": "files",
|
||
"payload": (lambda result, p=p, ids=local_set: _yesno_schema(_json_file(p), ids)),
|
||
} for i, p in pending]
|
||
new = await _race(topic, f"Relevance triage package {c}", slots, 2 - existing, _timeout("relevance", len(idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
|
||
return not is_cancelled() and new is not None
|
||
|
||
_qidx = _step_idx(topic, "Source prep") # gate runs in the source step (no own step)
|
||
set_p(f"Check relevance against spec ({n} packages)…", step=_qidx)
|
||
async def _report_triage(d, t):
|
||
set_p(f"Check relevance against spec {d}/{t}…", step=_qidx)
|
||
await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], n, _report_triage)
|
||
if is_cancelled():
|
||
return content, [] # cancel → drop nothing (caller aborts)
|
||
|
||
# Vote per page: only a clear "nein" majority (≥2 and more than "ja") throws it out.
|
||
dropped: list[str] = []
|
||
for c, idxs in enumerate(chunks, 1):
|
||
local_set = lset(idxs)
|
||
rater = [d for p in rater_paths(c) if (d := _yesno_schema(_json_file(p), local_set))]
|
||
for k in range(1, len(idxs) + 1):
|
||
vote_list = [d[k] for d in rater if k in d]
|
||
nein, ja = vote_list.count("nein"), vote_list.count("ja")
|
||
if nein >= 2 and nein > ja:
|
||
dropped.append(pages[idxs[k - 1]])
|
||
|
||
if dropped and len(dropped) >= max(1, int(len(pages) * 0.8)):
|
||
_log(topic, f"Relevance triage: would drop {len(dropped)}/{len(pages)} — discarded (spec mismatch?), keeping all")
|
||
return content, []
|
||
dropped_set = set(dropped)
|
||
keepers = [fn for fn in pages if fn not in dropped_set]
|
||
return keepers, dropped
|
||
|
||
|
||
async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool:
|
||
"""Step "Source prep": crawl (link) + PDF convert + content/noise triage.
|
||
Persists the triage in the coverage table (content). → True (ok) / False (cancel/error).
|
||
thema: nothing. projekt/uni: only PDFs (curated folder, no triage)."""
|
||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||
if not folder:
|
||
return True # thema → no source to prepare
|
||
if q["type"] != "link":
|
||
await asyncio.to_thread(_convert_pdfs, folder) # projekt/uni: only PDFs, no triage
|
||
return True
|
||
if await db.get_step_status(topic, "Source prep") == "done":
|
||
return True
|
||
if not _crawl_done(topic):
|
||
set_p("Loading source (crawl)…", step=_step_idx(topic, "Source prep"))
|
||
n = await asyncio.to_thread(crawl, q["location"], folder, cancelled=is_cancelled)
|
||
if is_cancelled():
|
||
return False
|
||
if not n:
|
||
_blocks_errors[topic] = "Crawl yielded no content — check link/domain"
|
||
return False
|
||
await asyncio.to_thread(_convert_pdfs, folder)
|
||
pages = sorted(set(_crawl_index(folder).values()))
|
||
if pages:
|
||
set_p("Triaging pages…", step=_step_idx(topic, "Source prep"))
|
||
await db.delete_coverage(topic)
|
||
content, noise = _triage_rules(folder, pages) # deterministic rule filter
|
||
if q.get("spec") and content: # topic gate: separates the field (rules can't)
|
||
content, dropped = await _relevance_triage(ctx, set_p, files, folder, content, q["spec"], instructions)
|
||
if is_cancelled():
|
||
return False
|
||
if dropped:
|
||
noise = sorted(set(noise) | set(dropped))
|
||
_log(topic, f"LLM relevance: {len(dropped)} pages off-topic → noise")
|
||
await db.mark_content(topic, sorted(content), sorted(noise))
|
||
_log(topic, f"Triage: {len(content)} content / {len(noise)} noise of {len(pages)} (rules + LLM gate)")
|
||
await db.set_step_status(topic, "Source prep", "done")
|
||
return True
|
||
|
||
|
||
|
||
|
||
|
||
|
||
_ASPECT_MARKER = ("∈ np", "∈np", " in np", "np-schwer", "np-vollständig", "verifizierer",
|
||
"zertifikat", "ndtm", "nicht-determ", "lower bound", "untere schranke",
|
||
"bzgl", "als sprache", "beweis", "güte", "austausch",
|
||
# generic bound/limit/runtime property stems (a "…-Grenze"/"…-Schranke"/"…-Laufzeit"
|
||
# is a property OF a concept, not the concept — MDM survivorship must never pick it as
|
||
# the representative, so it scores >0 here like the other aspect markers):
|
||
"grenze", "schranke", "laufzeit")
|
||
|
||
|
||
def _aspect_marker(title: str) -> int:
|
||
"""Number of property markers in the title (∈NP, NP-hard, verifier, lower bound …).
|
||
0 = generic main concept (the problem itself); >0 = a property of it."""
|
||
t = title.casefold()
|
||
return sum(1 for m in _ASPECT_MARKER if m in t)
|
||
|
||
|
||
_REFERENCE_RE = re.compile(r'^(Satz|Lemma|Korollar|Bemerkung|Definition)\s*[\d.]+\s*(\([a-z]\)|[a-z])?\s*$', re.I)
|
||
|
||
|
||
def _is_reference(title: str) -> bool:
|
||
"""True for pure reference/placeholder titles WITHOUT meaningful content: "Satz 7.18", "Lemma 6.2",
|
||
"Korollar 6.18" (number without a name) as well as marked spots "Bedingung (**)". NOT "Satz 6.24:
|
||
Cook/Levin" (has a name) and NOT short technical symbols like "P⊆NP"/"Σ*" (real concepts)."""
|
||
t = title.strip()
|
||
if _REFERENCE_RE.match(t):
|
||
return True
|
||
if re.search(r'\(\*+\)', t): # marked spot "(**)" / "(*)"
|
||
return True
|
||
return False
|
||
|
||
|
||
def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) -> dict:
|
||
"""Representative of a cluster = the main concept (fewest property markers — the problem
|
||
itself, not "… ∈ NP"); tie → most frequent norm title → most readers. Title globally unique
|
||
(suffix ' (2)') so it works as a key."""
|
||
by_norm: dict[str, list[int]] = {}
|
||
for k in idxs:
|
||
by_norm.setdefault(_norm_title(candidates[k]["title"]), []).append(k)
|
||
|
||
def weight(nb: str):
|
||
ms = by_norm[nb]
|
||
reader = set().union(*[set(candidates[m]["reader"]) for m in ms]) if ms else set()
|
||
# reference/placeholder titles ("Satz 7.18") last — prefer a meaningful member.
|
||
is_real = not _is_reference(candidates[ms[0]]["title"])
|
||
return (is_real, -_aspect_marker(nb), len(ms), len(reader))
|
||
|
||
best = max(by_norm, key=weight)
|
||
k = max(by_norm[best], key=lambda m: len(candidates[m]["description"]))
|
||
title = candidates[k]["title"]
|
||
n = 2
|
||
while _norm_title(title) in seen_norm:
|
||
title = f"{candidates[k]['title']} ({n})"
|
||
n += 1
|
||
seen_norm.add(_norm_title(title))
|
||
return {"title": title, "description": candidates[k]["description"]}
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
def _pairs_schema(data) -> dict[int, bool] | None:
|
||
"""{"pairs": {"1": "ja", "2": "nein", …}} → {pair_nr: True/False} · otherwise None."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("pairs"), dict):
|
||
return None
|
||
out: dict[int, bool] = {}
|
||
for k, v in data["pairs"].items():
|
||
try:
|
||
nr = int(k)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
out[nr] = str(v).strip().casefold() in ("ja", "yes", "true", "1")
|
||
return out or None
|
||
|
||
|
||
def _cliques(n: int, edge_list: list[tuple[int, int]]) -> list[list[int]]:
|
||
"""Complete-link: greedy maximal cliques over the confirmed duplicate edges. A group
|
||
forms only if ALL its nodes are pairwise connected → no chaining (A=B + B=C forms
|
||
NO group {A,B,C} as long as A=C is missing). Only cliques ≥2 are returned."""
|
||
adj: dict[int, set[int]] = {i: set() for i in range(n)}
|
||
for a, b in edge_list:
|
||
adj[a].add(b)
|
||
adj[b].add(a)
|
||
used: set[int] = set()
|
||
groups: list[list[int]] = []
|
||
for v in sorted(range(n), key=lambda x: -len(adj[x])):
|
||
if v in used or not adj[v]:
|
||
continue
|
||
clique = {v}
|
||
for u in sorted(adj[v], key=lambda x: -len(adj[x])):
|
||
if u not in used and clique <= adj[u] | {u}: # u connected to ALL previous ones
|
||
clique.add(u)
|
||
if len(clique) >= 2:
|
||
groups.append(sorted(clique))
|
||
used |= clique
|
||
return groups
|
||
|
||
|
||
# Canonical-name blocking (dedup recall): entity resolution's recall ceiling is set by candidate
|
||
# generation — a pair that never shares a candidate can never be merged. Normalize a title to a
|
||
# scaffolding-free, operator-class-normalized, order-independent key so surface variants of ONE entity
|
||
# collapse ("Offenes Problem P=NP?" ≡ "P vs NP" ≡ "P=NP"). Generic (no course terms): strip a small
|
||
# stoplist of catalogue/wrapper words, fold the relation operators into class tokens, sort content tokens.
|
||
_CANON_STOP = re.compile(
|
||
r'\b(?:offenes?|open|problem|frage|question|algorithmus|algorithm|verfahren|method|methode|'
|
||
r'satz|theorem|lemma|korollar|definition|def|das|der|die|the|ein|eine|einen|a|an|'
|
||
r'von|of|für|for|und|and|zum|zur|im)\b', re.I)
|
||
|
||
|
||
def _canonical_key(title: str) -> str:
|
||
"""Order-independent canonical key of a title (scaffolding stripped, relation operators normalized).
|
||
Two titles with the same key denote the same entity with ~100% precision (ER blocking). Empty string
|
||
if nothing survives (never auto-merged)."""
|
||
s = unicodedata.normalize("NFKC", title).casefold()
|
||
s = re.sub(r'≟|\bversus\b|\bvs\.?\b|=', ' opeq ', s) # equality / "vs" → one token
|
||
s = re.sub(r'≤|⪯|→|⇒|⟹|\breduces?\s+to\b|\breduziert\b', ' opred ', s) # reduction → one token
|
||
s = _CANON_STOP.sub(' ', s)
|
||
s = re.sub(r'[^\w ]', ' ', s) # drop punctuation/symbols
|
||
return " ".join(sorted(t for t in s.split() if t))
|
||
|
||
|
||
# Relation-triple individuation (dedup precision): a reduction/relation "A ≤ B" is identified by BOTH
|
||
# operands AND direction (RDF-triple identity / SKOS narrowMatch — a subset/restriction is NOT the same).
|
||
# So "SAT ≤ Clique" ≠ "3-SAT ≤ Clique" (source differs) and "A → B" ≠ "B → A" (direction). Two DIFFERENT
|
||
# relations must never merge, even if a judge or a high title-cosine says so.
|
||
_REL_STRIP = re.compile(r'^\s*(?:satz|lemma|korollar|theorem)\s*[\d.]*\s*:?\s*|^\s*reduktion(?:en)?\s*:?\s*', re.I)
|
||
_REL_OPERATOR = re.compile(r'[≤⪯≥⊆⊊→⇒⟹⇔←]|=>|<=|->')
|
||
|
||
|
||
def _relation_operands(title: str) -> tuple[str, str] | None:
|
||
"""(canonical_source, canonical_target) of a relation/reduction title, else None (not a relation).
|
||
Operands canonicalized (lowercased, non-alphanumerics stripped) so spacing/hyphenation don't matter."""
|
||
t = _REL_STRIP.sub('', title, count=1)
|
||
m = _REL_OPERATOR.search(t)
|
||
if not m:
|
||
return None
|
||
left = re.sub(r'[\W_]', '', t[:m.start()].casefold()) # keep unicode letters/digits (umlauts), drop the rest
|
||
right = re.sub(r'[\W_]', '', t[m.end():].casefold())
|
||
if not left or not right:
|
||
return None
|
||
return (left, right)
|
||
|
||
|
||
def _relation_conflict(title_a: str, title_b: str) -> bool:
|
||
"""True if BOTH titles are relations/reductions but denote DIFFERENT ones (operands or direction
|
||
differ) → they must NOT be merged. False if either is not a relation, or they are the same relation."""
|
||
a, b = _relation_operands(title_a), _relation_operands(title_b)
|
||
return a is not None and b is not None and a != b
|
||
|
||
|
||
|
||
|
||
def _filter_schema(data) -> dict[int, int] | None:
|
||
"""{"fragments": {"3": 7, "12": 8}} → {block_nr: parent_nr} · None on invalid structure.
|
||
Empty dict = valid (nothing to degrade). Parent ≠ itself."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("fragments"), dict):
|
||
return None
|
||
out: dict[int, int] = {}
|
||
for k, v in data["fragments"].items():
|
||
try:
|
||
nr, parent = int(k), int(v)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if nr != parent:
|
||
out[nr] = parent
|
||
return out
|
||
|
||
|
||
# Pure notation/symbols without a standalone concept — kept narrow (FP~0, checked against aak;
|
||
# "KNF"/"MST"/"NP" do NOT match). These are discarded autonomously (need no parent).
|
||
_FILTER_NOTATION = re.compile(r'^\s*\|.{1,6}\|\s*$|^Güte\s+\d+\s*$')
|
||
# Exercise-sheet / cross-reference artefacts — the SECOND gate for a judge `drop` verdict: a hard-drop
|
||
# (parentless removal) only fires when the judge lists the number in `drop` AND `_is_artifact(title)`.
|
||
# A judge-drop without a match degrades to "keep" (logged), never deleted (FP~0 discipline like
|
||
# _FILTER_NOTATION). Shape-matching on a NORMALIZED, title-side string (see _is_artifact): four branches —
|
||
# (1) lettered/Roman/numeric sub-claims "(Aussage i)"/"Aussage (a)"/"Teil (b)"/"Fall (2)" in ANY
|
||
# parenthesization; (2) sheet refs "Blatt 10"/"Aufgabe 3"/"Übung"; (3) worked-example/table/figure refs
|
||
# "Beispiel Tab. 7.1" (a NUMBER is required — guards polysemes "Hash-Tabelle"/"Bijektive Abbildung");
|
||
# (4) parenthesized "(Variante)". Theorem words (Satz/Definition/Lemma) are deliberately NOT in the
|
||
# vocabulary, so "Satz 6.24 Cook/Levin — SAT ist NP-vollständig" is kept. Word boundaries guard prefixes
|
||
# (Aussagenlogik/Teilmenge/Blattknoten/Fallunterscheidung).
|
||
_FILTER_ARTIFACT = re.compile(r"""
|
||
\b(?:aussage|teil|behauptung|fall)\b[\s(]*(?:[ivx]{1,4}|[a-z]|\d{1,2})\)?(?![a-zäöüß])
|
||
| \b(?:blatt|aufgabe|serie|hausaufgabe)\s*\d+(?:[.,]\d+)*
|
||
| \b(?:übungsblatt|übung|uebung)\b
|
||
| \b(?:beispiel|abbildung|abb|tabelle|tab|bild|grafik|diagramm|figur|skizze)\b\.?\s*\d+(?:[.,]\d+)*
|
||
| \(\s*(?:variante|variation|spezialfall|sonderfall)\s*\)
|
||
""", re.VERBOSE)
|
||
|
||
|
||
def _is_artifact(title: str) -> bool:
|
||
"""True if the TITLE looks like exercise-sheet / cross-reference scaffolding (P1 hard-drop gate).
|
||
NFKC + casefold normalization (position/case/umlaut/Unicode-invariant: folds Ⅱ→ii, full-width, NBSP),
|
||
then match only the title side of the em-dash — a real concept whose *description* merely cites a
|
||
sheet ("… — vgl. Aufgabe 3") is never dropped. Normalize for matching only; never store the result."""
|
||
norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip()
|
||
head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0] # spaced dash only → "3-SAT"/"np-schwer" unsplit
|
||
return bool(_FILTER_ARTIFACT.search(head))
|
||
# Property/runtime suspicion — marks lines for the judge's verdict (NO auto-drop, FP too high:
|
||
# "NP-Schwere", reductions with "∈NP" are real blocks). Complements _aspekt_marker.
|
||
_FILTER_PREDICATE = re.compile(
|
||
r'ist NP-(vollständig|schwer)|NP-(Vollständigkeit|Schwere) von|ETH (Konsequenz|Lower Bound)'
|
||
r'|Approximationsschema nach|Laufzeit O\(|∈ ?NP'
|
||
# fragment families that the aspect substrings miss (all title/desc, advisory ⚠ only):
|
||
r'|\bSatz\s+\d|\bLemma\s+\d|\bKorollar\s+\d' # bare theorem/proof references (with number)
|
||
r'|^\s*(?:Remark|Bemerkung|Anmerkung|Note|Notiz|Beobachtung|Observation)\b' # EN+DE remark labels
|
||
r'|\bSatz\s*:|\bSatz\s+[A-Z]\b' # "Satz:" (colon, no number) / "Satz D*" letter label
|
||
r'|\bGegenbeispiel\b|\bWorst[- ]?Case\b|Schärfe\s+der\b' # proof-example / sharpness facets
|
||
r'|2\s*\^\s*[{(]?\s*[Ωωoο]\s*\(' # ETH exponential bound 2^Ω(…)/2^o(…)
|
||
r'|\d\s*[−–-]\s*1\s*/\s*m' # approximation-güte ratio "2 − 1/m"
|
||
r'|Variablenungleichung|[α-ωΑ-Ω][a-z]?-?Variablen', re.I) # proof variables (αu-Variablen …)
|
||
|
||
|
||
def _filter_suspect(b: dict) -> bool:
|
||
"""Heuristic flag: could be a property/detail of another block."""
|
||
return _aspect_marker(b["title"]) > 0 or bool(_FILTER_PREDICATE.search(f"{b['title']} {b['description'] or ''}"))
|
||
|
||
|
||
def _root(nr: int, fragments: dict[int, int]) -> tuple[int, bool]:
|
||
"""Follow the fragment→parent chain to the first ancestor that is NOT itself a fragment.
|
||
Returns (root, cyclic). This walk IS the fixpoint (no separate iteration): a mid-tier
|
||
fragment whose own parent is also a fragment resolves to the top-level real block, so all
|
||
chain levels collapse in one pass (unlike the old single-level parent_set shield).
|
||
Cycle guard: on revisiting a node returns (node, True) → caller keeps both (no annihilation)."""
|
||
seen: set[int] = set()
|
||
cur = nr
|
||
while cur in fragments:
|
||
if cur in seen:
|
||
return cur, True
|
||
seen.add(cur)
|
||
cur = fragments[cur]
|
||
return cur, False
|
||
|
||
|
||
def _containment_parent(frag_norm: str, others: list[tuple[int, str]]) -> int | None:
|
||
"""Deterministic parent-by-name-containment for a ⚠-flagged survivor: a fragment whose title NAMES
|
||
another block ("Lower Bound … für VERTEX COVER" → Vertex Cover; "List Scheduling Güte …" → List
|
||
Scheduling). `others` = (nr, title_norm) of the OTHER blocks. A parent is a block whose normalized
|
||
title occurs as a WHOLE-WORD span inside `frag_norm`, is SIGNIFICANT (≥2 tokens or ≥6 chars — never
|
||
"p"/"np"/"sat") and strictly shorter than the fragment. Returns the parent nr on EXACTLY ONE match,
|
||
else None (0 or ≥2 → leave to the judge). Whole-word + significance + exactly-one → near-FP-0."""
|
||
hits = []
|
||
for nr, ptitle in others:
|
||
if not ptitle or ptitle == frag_norm or len(ptitle) >= len(frag_norm):
|
||
continue
|
||
if len(ptitle) < 6 and ptitle.count(" ") < 1: # reject short single-token names (P/NP/SAT)
|
||
continue
|
||
if re.search(r'(?<!\w)' + re.escape(ptitle) + r'(?!\w)', frag_norm):
|
||
hits.append(nr)
|
||
return hits[0] if len(hits) == 1 else None
|
||
|
||
|
||
# Two provably-noise, parent-less fragment classes safe to hard-drop (precision ~≥0.95): a bare
|
||
# label reference with an empty/thin residual ("Remark 7.28", "Satz D*", "Lemma 3.2") and a single-
|
||
# variable notation assignment ("r = n + m"). Matched on the normalized title HEAD (before the em-dash).
|
||
_PARENTLESS_NOISE = re.compile(
|
||
r'^\s*(?:remark|bemerkung|anmerkung|note|satz|lemma|korollar|proposition|folgerung)\s*[\d.]*\s*[a-z]?\*?\s*$'
|
||
# single-var arithmetic assignment ("r = n + m") — NOT a tuple/set/language/cardinality definition
|
||
# ("T = (Q,…)", "L = {…}", "n = |V|"): the RHS must not open with a bracket/pipe.
|
||
r'|^\s*[a-z](?:_?[a-z0-9])?\s*:?=\s*(?![({\[|])\S', re.I)
|
||
# KEEP-guards: never drop a named theorem WITH an author, a complexity-class (in)equality, or a Definition.
|
||
_PARENTLESS_KEEP = re.compile(
|
||
r'\b(?:cook|levin|karp|savitch|ladner|immerman|rice|håstad|hastad|christofides|dijkstra|bellman|'
|
||
r'ford|kruskal|prim|edmonds|blum|sipser|papadimitriou)\b'
|
||
r'|\b(?:p|np|conp|nl|pspace|exp|nexp|bpp|rp|zpp)\b|⇔|⇒|\bdefinition\b', re.I)
|
||
|
||
|
||
def _is_parentless_noise(title: str) -> bool:
|
||
"""True for the narrow, parent-less noise classes safe to hard-drop (subject to KEEP-guards)."""
|
||
norm = re.sub(r"\s+", " ", unicodedata.normalize("NFKC", title).casefold()).strip()
|
||
head = re.split(r"\s[—–-]\s", norm, maxsplit=1)[0]
|
||
return bool(_PARENTLESS_NOISE.search(head)) and not _PARENTLESS_KEEP.search(norm)
|
||
|
||
|
||
# F1 — statement-gate keep-guard (Knowledge-Component / OMDoc theory): a self-contained ASSERTION is its
|
||
# own learning unit, not a whole-part fragment. Protect two general classes from demotion:
|
||
# (a) a named reduction between two PROBLEMS ("3-SAT ≤ Clique", "Clique → Vertex Cover"), and
|
||
# (b) a LABELED or ATTRIBUTED theorem carrying its own biconditional/implication ("Satz 6.37: … ⇔ …").
|
||
# NOT protected: a bare label ("Satz 7.18", "Remark 7.28"), unary status ("X ist NP-vollständig", "X ∈ NP"),
|
||
# a proof-size step ("Strikte Reduktion |A| = O(m)"), or a güte/bound facet — those carry neither a
|
||
# two-sided reduction operator NOR a ⇔/⇒ assertion. General: attribution is structural, no author whitelist.
|
||
_STMT_LABEL = re.compile(r'^\s*(?:satz|lemma|korollar|theorem|proposition|prop|folgerung)\b', re.I)
|
||
_STMT_ATTRIB = re.compile(r'\b(?:satz|lemma|theorem|korollar)\s+von\s+[A-ZÄÖÜ]') # "Satz von Cook/Levin"
|
||
_STMT_ASSERT = re.compile(r'⇔|⇒|⟺|⟹|\bgdw\.?\b|\bgenau dann\b', re.I)
|
||
_REDUCTION_ONLY_OP = re.compile(r'[≤⪯]|→|⇒|⟹|->') # genuine reduction operators (NOT plain "=")
|
||
|
||
|
||
def _is_reduction_statement(title: str) -> bool:
|
||
"""True if the title is a reduction between two NAMED problems (both sides carry ≥3 letters and
|
||
neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP" and proof-size "|A| = O(m)"."""
|
||
t = _REL_STRIP.sub('', title, count=1)
|
||
m = _REDUCTION_ONLY_OP.search(t)
|
||
if not m:
|
||
return False
|
||
|
||
def _named(s: str) -> bool:
|
||
return len(re.findall(r'[a-zäöüß]', s, re.I)) >= 3 and not re.match(r'\s*[Oo]\s*\(', s)
|
||
|
||
return _named(t[:m.start()]) and _named(t[m.end():])
|
||
|
||
|
||
def _is_named_statement(title: str, desc: str = "") -> bool:
|
||
"""Statement-gate keep-guard: a named reduction (a) or a labeled/attributed theorem WITH its own
|
||
⇔/⇒ assertion (b). Bare labels / unary status / proof-size steps return False (stay demotable)."""
|
||
if _is_reduction_statement(title):
|
||
return True
|
||
if (_STMT_LABEL.match(title) or _STMT_ATTRIB.search(title)) and _STMT_ASSERT.search(f"{title} {desc or ''}"):
|
||
return True
|
||
return False
|
||
|
||
|
||
|
||
|
||
def _umbrella_schema(data, ids: set[int]):
|
||
"""{"umbrellas":[{"title":str,"description":str,"members":[int,…]}, …]}
|
||
→ [(title, description, [member ids])]. [] = valid (no umbrella); None ONLY on broken JSON
|
||
(so resume treats a valid-but-empty file as done, like _filter_schema's {} vs None). Each id
|
||
used at most once across all umbrellas (first wins); members filtered to `ids`; an umbrella
|
||
needs ≥2 surviving members; `description` required + non-empty — it MUST enumerate the children,
|
||
else the source-scoped subblock step can't re-derive them (no web on uni)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("umbrellas"), list):
|
||
return None
|
||
out, used = [], set()
|
||
for u in data["umbrellas"]:
|
||
if not isinstance(u, dict):
|
||
continue
|
||
title = str(u.get("title", "")).strip()
|
||
desc = str(u.get("description", "")).strip()
|
||
raw_members = u.get("members")
|
||
if not title or not desc or not isinstance(raw_members, list):
|
||
continue
|
||
members = []
|
||
for x in raw_members:
|
||
try:
|
||
m = int(x)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if m in ids and m not in used:
|
||
used.add(m)
|
||
members.append(m)
|
||
if len(members) >= 2:
|
||
out.append((title, desc, members))
|
||
return out
|
||
|
||
|
||
def _completion_schema(data, n_umbrellas: int, ids: set[int]):
|
||
"""{"additions":[{"umbrella":int,"members":[int,…]}, …]} → [(umbrella_idx, [member ids])].
|
||
[] = valid (nothing to absorb); None only on broken JSON. umbrella idx in range; members drawn
|
||
from `ids` (the still-standalone leftovers), de-duped within an addition."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("additions"), list):
|
||
return None
|
||
out = []
|
||
for a in data["additions"]:
|
||
if not isinstance(a, dict):
|
||
continue
|
||
try:
|
||
k = int(a.get("umbrella"))
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if not (0 <= k < n_umbrellas):
|
||
continue
|
||
mem = []
|
||
for x in (a.get("members") or []):
|
||
try:
|
||
m = int(x)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if m in ids and m not in mem:
|
||
mem.append(m)
|
||
if mem:
|
||
out.append((k, mem))
|
||
return out
|
||
|
||
|
||
# Deterministic backstop to the grouping judge's TEST 1 (type gate): an umbrella may bundle ONLY
|
||
# constituent sub-definitions of ONE definition. If a member title carries a standalone-unit signal
|
||
# (a named algorithm / problem / reduction / theorem), the umbrella is dissolved — those stay their own
|
||
# blocks. Kept narrow so real definition-parts (Konfiguration, Übergangsfunktion δ, Literale, Makespan,
|
||
# m Maschinen) never match; checked against the aak over-merge (member „Greedy-Algorithmus GA" hits).
|
||
# Suffix-anchored head nouns (German compounds are head-final: „Approximations+algorithmus" has NO word
|
||
# boundary before „algorithmus", so \bAlgorithmus\b misses it → the MAX-SAT over-merge). \w* absorbs the
|
||
# modifier; the head noun stays the discriminator. FP-safe: no real TM/KNF definition-part ends in these
|
||
# heads (Berechnung is deliberately NOT a head → „Akzeptierende Berechnung" stays a valid member).
|
||
_GROUP_STANDALONE = re.compile(
|
||
r'\w*algorithm(?:us|en)\b|\w*problem(?:e|s|en)?\b|\w*reduktion(?:en)?\b|\bscheduling\b|[≤⪯]'
|
||
r'|^\s*(?:Satz|Lemma|Korollar|Theorem|Bemerkung|Beobachtung)\s*\d'
|
||
# atomicity: a named COMPLEXITY CLASS / a "…-Vollständigkeit(completeness)" / a "…Transformation" is a
|
||
# self-contained concept (learning-object / atomic-KC), never a sub-definition — a bundle of ≥1 such
|
||
# member is siblings, not one model → dissolve (catches the P/NP/NP-Vollständigkeit over-merge that NO
|
||
# cosine floor separates). Head-final compounds (\w*klasse absorbs "Komplexitäts+klasse"); FP-safe —
|
||
# no real TM/KNF/TSP/Scheduling definition-part carries these heads.
|
||
r'|\w*vollständigkeit\b|\w*completeness\b|\w*transformation(?:en)?\b|\w*klasse[nr]?\b', re.I)
|
||
|
||
|
||
|
||
|
||
# --- Outline (blocks artifact: chapter structure, only read by the guide) ---
|
||
|
||
def _outline_complete(files: dict) -> bool:
|
||
"""Is the outline present (chapter list exists)?"""
|
||
d = _json_file(files["outline"])
|
||
return isinstance(d, dict) and isinstance(d.get("chapters"), list) and bool(d.get("chapters"))
|
||
|
||
|
||
def _outline_schema(data, valid: set[int]):
|
||
"""{"chapters":[{title,numbers}]} → cleaned (valid numbers, each exactly once) ·
|
||
None at <80 % coverage (agent/judge omitted too much)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("chapters"), list):
|
||
return None
|
||
out, seen = [], set()
|
||
for ch in data["chapters"]:
|
||
if not isinstance(ch, dict):
|
||
continue
|
||
title = str(ch.get("title", "")).strip() or "Chapter"
|
||
nums = []
|
||
for n in (ch.get("numbers") or []):
|
||
try:
|
||
n = int(n)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if n in valid and n not in seen:
|
||
seen.add(n)
|
||
nums.append(n)
|
||
if nums:
|
||
out.append({"title": title, "numbers": nums})
|
||
if not out or len(seen) < 0.8 * len(valid):
|
||
return None
|
||
return {"chapters": out}
|
||
|
||
|
||
def _prereq_schema(data, valid: set[int]) -> dict[int, list[int]]:
|
||
"""{"prereqs": {"3": [1, 7]}} → {num: [prereq nums]} · only numbers from `valid`, no self-edge.
|
||
Invalid/empty → {} (best-effort: then original order)."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("prereqs"), dict):
|
||
return {}
|
||
out: dict[int, list[int]] = {}
|
||
for k, v in data["prereqs"].items():
|
||
try:
|
||
num = int(k)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if num not in valid or not isinstance(v, list):
|
||
continue
|
||
pres = []
|
||
for p in v:
|
||
try:
|
||
p = int(p)
|
||
except (ValueError, TypeError):
|
||
continue
|
||
if p in valid and p != num and p not in pres:
|
||
pres.append(p)
|
||
if pres:
|
||
out[num] = pres
|
||
return out
|
||
|
||
|
||
def _topo_order(nums: list[int], edges: dict[int, list[int]]) -> list[int]:
|
||
"""Kahn topo sort: prerequisites first. `edges[num]` = numbers that must come BEFORE num.
|
||
Stable tie-break (original order of `nums`); cycles are broken (never deadlock)."""
|
||
pos = {n: i for i, n in enumerate(nums)}
|
||
# remaining in-degree over valid nodes only; self/foreign edges ignored.
|
||
pre = {n: [p for p in edges.get(n, []) if p in pos and p != n] for n in nums}
|
||
done: list[int] = []
|
||
finished: set[int] = set()
|
||
rest = list(nums)
|
||
while rest:
|
||
ready_nodes = [n for n in rest if all(p in finished for p in pre[n])]
|
||
if not ready_nodes: # cycle → force the earliest remaining node in original order
|
||
ready_nodes = [min(rest, key=lambda n: pos[n])]
|
||
nxt = min(ready_nodes, key=lambda n: pos[n]) # stable: smallest original position first
|
||
done.append(nxt)
|
||
finished.add(nxt)
|
||
rest.remove(nxt)
|
||
return done
|
||
|
||
|
||
async def _learning_order(ctx: GenContext, set_p, files: dict, entries: dict, valid: set[int], instructions: str) -> dict:
|
||
"""Put entries (num→title) into learning order: the LLM extracts prereq edges from the
|
||
extracted `prerequisites`, code solves via topo sort. Best-effort → otherwise entries unchanged."""
|
||
if len(entries) < 3:
|
||
return entries
|
||
topic = ctx.topic
|
||
facts_map = _json_file(files["facts"])
|
||
facts_map = facts_map if isinstance(facts_map, dict) else {}
|
||
|
||
def _hint(title):
|
||
fm = facts_map.get(title) or {}
|
||
vs = [v for fk in fm.values() if isinstance(fk, dict) and (v := str(fk.get("prerequisites", "")).strip())]
|
||
return " · ".join(dict.fromkeys(vs))
|
||
|
||
pp = files["arbeit"] / "outline-prereqs.json"
|
||
|
||
def _payload(result, p=pp):
|
||
d = _json_file(p)
|
||
return d if isinstance(d, dict) and "prereqs" in d else None
|
||
|
||
existing = _json_file(pp)
|
||
if not (isinstance(existing, dict) and "prereqs" in existing):
|
||
lines = [f"{n}. {t}" + (f"\n braucht vorher: {h}" if (h := _hint(t)) else "") for n, t in entries.items()]
|
||
set_p("Outline — learning order…", step=_step_idx(topic, "Outline"))
|
||
await run_single_slot(
|
||
ctx, "Outline-Prerequisites", key=f"blocks-{topic}-outline-prereqs",
|
||
prompt=_prompt("Outline-Prerequisites", topic=topic, blocks="\n".join(lines), out_path=pp, extra=_extra(instructions)),
|
||
role="guide", capabilities="files", payload=_payload, timeout=_timeout("plan", len(entries)))
|
||
edges = _prereq_schema(_json_file(pp), valid)
|
||
if not edges:
|
||
return entries # no/invalid edges → original order (no regression)
|
||
ordered = _topo_order(list(entries), edges)
|
||
return {n: entries[n] for n in ordered}
|
||
|
||
|
||
async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, instructions: str) -> dict:
|
||
"""Format-agnostic outline over ALL blocks — 3 proposals → judge merges.
|
||
Never aborts: 0 valid → one chapter with everything; missing blocks land in "Other".
|
||
→ {"chapters":[{title,numbers}]} (also in files["outline"])."""
|
||
topic, is_cancelled = ctx.topic, ctx.is_cancelled
|
||
valid = set(entries)
|
||
step = _step_idx(topic, "Outline")
|
||
|
||
# Establish learning order (LLM-modulo): the LLM extracts prereq edges from the extracted
|
||
# `prerequisites`, code solves via topo sort. Best-effort → otherwise original order.
|
||
entries = await _learning_order(ctx, set_p, files, entries, valid, instructions)
|
||
liste = "\n".join(f"{n}. {t}" for n, t in entries.items())
|
||
set_p("Outline — proposals…", step=step)
|
||
|
||
async def _proposal(i, path):
|
||
if _outline_schema(_json_file(path), valid):
|
||
return True
|
||
await run_single_slot(
|
||
ctx, f"Outline {i}", key=f"blocks-{topic}-outline-{i}",
|
||
prompt=_prompt("Guide-Outline", topic=topic, blocks=liste, out_path=path, extra=_extra(instructions)),
|
||
role="guide", capabilities="files",
|
||
payload=lambda result, p=path: _outline_schema(_json_file(p), valid),
|
||
timeout=_timeout("plan", len(entries)))
|
||
return _outline_schema(_json_file(path), valid) is not None
|
||
|
||
slots = files["outline_slots"]
|
||
await _gather_progress([_proposal(i, p) for i, p in enumerate(slots, 1)], len(slots), _report_p(set_p, topic, "Outline"))
|
||
if is_cancelled():
|
||
return {}
|
||
proposals = [v for p in slots if (v := _outline_schema(_json_file(p), valid))]
|
||
|
||
if not proposals:
|
||
plan = {"chapters": [{"title": "Contents", "numbers": list(entries)}]}
|
||
elif len(proposals) == 1:
|
||
plan = proposals[0]
|
||
else:
|
||
set_p("Outline merging…", step=step)
|
||
block_texts = "\n\n".join(
|
||
f"### Vorschlag {i}\n" + "\n".join(
|
||
f"KAPITEL: {ch['title']}\n Nummern: {', '.join(str(n) for n in ch['numbers'])}" for ch in v["chapters"])
|
||
for i, v in enumerate(proposals, 1))
|
||
await run_single_slot(
|
||
ctx, "Outline-Judge", key=f"blocks-{topic}-outline-judge",
|
||
prompt=_prompt("Guide-Outline-Judge", topic=topic, format_name="den Guide",
|
||
purpose="alle Blocks in einem roten Faden", n=len(proposals),
|
||
blocks=liste, outlines=block_texts, out_path=files["outline"], extra=_extra(instructions)),
|
||
role="judge", capabilities="files",
|
||
payload=lambda result: _outline_schema(_json_file(files["outline"]), valid),
|
||
timeout=_timeout("plan_judge", len(entries)))
|
||
plan = _outline_schema(_json_file(files["outline"]), valid) or proposals[0]
|
||
|
||
# Completeness: every block appears — missing in "Other" (against omitting agents/judge).
|
||
included = {n for ch in plan["chapters"] for n in ch["numbers"]}
|
||
missing = [n for n in entries if n not in included]
|
||
if missing:
|
||
plan["chapters"].append({"title": "Other", "numbers": missing})
|
||
atomic_write_json(files["outline"], plan, indent=1)
|
||
return plan
|
||
|
||
|
||
# --- Learning artefacts (flashcards/examples from the facts) ---
|
||
|
||
def _cards_schema(data):
|
||
"""{"cards":[{block,subblock,question,answer}]} → list (also empty) · None if broken."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("cards"), list):
|
||
return None
|
||
out = []
|
||
for e in data["cards"]:
|
||
if isinstance(e, dict) and (f := str(e.get("question", "")).strip()) and (a := str(e.get("answer", "")).strip()):
|
||
out.append({"block": str(e.get("block", "")).strip(), "subblock": str(e.get("subblock", "")).strip(),
|
||
"question": f, "answer": a})
|
||
return out
|
||
|
||
|
||
def _example_schema(data):
|
||
"""{"examples":[{block,subblock,problem,steps,result}]} → list (also empty) · None if broken."""
|
||
if not isinstance(data, dict) or not isinstance(data.get("examples"), list):
|
||
return None
|
||
out = []
|
||
for e in data["examples"]:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
problem = str(e.get("problem", "")).strip()
|
||
steps = [s for x in (e.get("steps") or []) if (s := str(x).strip())]
|
||
if problem and steps:
|
||
out.append({"block": str(e.get("block", "")).strip(), "subblock": str(e.get("subblock", "")).strip(),
|
||
"problem": problem, "steps": steps, "result": str(e.get("result", "")).strip()})
|
||
return out
|
||
|
||
|
||
def _example_check_schema(data):
|
||
"""Worked-example check → {"ok": true} → set() (all correct); {"problems":[{"index":N}]} →
|
||
{N, …} (1-based flagged indices); None if broken."""
|
||
if not isinstance(data, dict):
|
||
return None
|
||
if data.get("ok") is True:
|
||
return set()
|
||
pr = data.get("problems")
|
||
if not isinstance(pr, list):
|
||
return None
|
||
out: set[int] = set()
|
||
for p in pr:
|
||
if isinstance(p, dict):
|
||
try:
|
||
out.add(int(p.get("index")))
|
||
except (ValueError, TypeError):
|
||
continue
|
||
return out
|
||
|
||
|
||
_ARTEFACT_SCHEMA = {"flashcard": _cards_schema, "example": _example_schema}
|
||
_ARTEFACT_PROMPT = {"flashcard": "Artifact-Flashcard", "example": "Artifact-Example"}
|
||
_ARTEFACT_STEP = {"flashcard": "Flashcards", "example": "Examples"}
|
||
|
||
|
||
def _artefacts_complete(files: dict) -> bool:
|
||
"""Artifact map present (all types generated)? Values may be empty (content-aware)."""
|
||
d = _json_file(files["artefakte"])
|
||
return isinstance(d, dict) and all(t in d for t in ARTEFACT_TYPES)
|
||
|
||
|
||
async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "") -> dict | None:
|
||
"""Generate learning artefacts per type from the stored facts — one generation pass
|
||
per type over chunks. Worked examples are verified against the facts (wrong ones discarded);
|
||
flashcards are low-risk and stay unchecked. → {type: [entries]} (also in files)."""
|
||
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
|
||
work_dir = files["arbeit"]
|
||
caps = "files"
|
||
# Blocks with subs + facts lines as input block (extract-once from the facts).
|
||
blocks = []
|
||
for btitle, subs in sidecar.items():
|
||
if not isinstance(subs, list):
|
||
continue
|
||
lines = []
|
||
for s in subs:
|
||
if not isinstance(s, dict) or not (st := str(s.get("title", "")).strip()):
|
||
continue
|
||
fk = s.get("facts") if isinstance(s.get("facts"), dict) else {}
|
||
line = f"- {st}"
|
||
if fk and (fk_text := _facts_lines(fk)):
|
||
line += "\n" + "\n".join(" " + l for l in fk_text.split("\n"))
|
||
lines.append(line)
|
||
if lines:
|
||
blocks.append((btitle, lines))
|
||
if not blocks:
|
||
empty_map = {t: [] for t in ARTEFACT_TYPES}
|
||
atomic_write_json(files["artefakte"], empty_map, indent=1)
|
||
return empty_map
|
||
|
||
chunks = _lpt_chunks([len(z) for _, z in blocks], FACTS_CHUNK_SUBS)
|
||
def block_text(idxs):
|
||
return "\n\n".join(f"BLOCK: {blocks[i][0]}\nSUBBAUSTEINE:\n" + "\n".join(blocks[i][1]) for i in idxs)
|
||
|
||
# Check worked examples against the facts (panel majority) — discard wrong ones. CoT steps are
|
||
# error-prone; a wrong example imprints a faulty schema → no example > a wrong one.
|
||
async def _check_examples(ci, idxs, items):
|
||
if is_cancelled() or not items:
|
||
return items
|
||
def cpath(j): return work_dir / f"artifact-example-check-c{ci}-j{j}.json"
|
||
examples_txt = "\n\n".join(
|
||
f"{k}. PROBLEM: {e['problem']}\n SCHRITTE: " + " | ".join(e.get("steps", []))
|
||
+ (f"\n ERGEBNIS: {e['result']}" if e.get("result") else "")
|
||
for k, e in enumerate(items, 1))
|
||
pending = [j for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if _example_check_schema(_json_file(cpath(j))) is None]
|
||
if pending:
|
||
await asyncio.gather(*[
|
||
run_agent(f"blocks-{topic}-{ns}artifact-example-check-c{ci}-j{j}",
|
||
_prompt("Artifact-Example-Check", topic=topic, facts=block_text(idxs), examples=examples_txt, out_path=cpath(j), extra=_extra(instructions)),
|
||
_timeout("content_check", len(items)), provider=provider, role="judge", capabilities=caps)
|
||
for j in pending], return_exceptions=True)
|
||
outs = [s for j in (1, 2, 3)[:FACTS_CHECK_PANEL] if (s := _example_check_schema(_json_file(cpath(j)))) is not None]
|
||
if not outs:
|
||
return items # no exam possible → keep (best-effort)
|
||
votes: dict[int, int] = {}
|
||
for s in outs:
|
||
for idx in s:
|
||
votes[idx] = votes.get(idx, 0) + 1
|
||
threshold = len(outs) / 2
|
||
dropped = {idx for idx, v in votes.items() if v > threshold} # majority (≥2 of 3) flagged → out
|
||
if dropped:
|
||
_log(topic, f"Worked-example check chunk {ci}: {len(dropped)}/{len(items)} discarded")
|
||
return [e for k, e in enumerate(items, 1) if k not in dropped]
|
||
|
||
async def _one_type(typ: str) -> list | None:
|
||
"""Full pipeline of ONE artefact type — the types are independent (own schemas,
|
||
own files) and run in parallel."""
|
||
schema = _ARTEFACT_SCHEMA[typ]
|
||
|
||
def apath(ci): return work_dir / f"artifact-{typ}-c{ci}.json"
|
||
|
||
async def _gen(ci, idxs):
|
||
p = apath(ci)
|
||
if schema(_json_file(p)) is not None:
|
||
return True
|
||
await run_single_slot(
|
||
ctx, f"{_ARTEFACT_STEP[typ]} {ci}", key=f"blocks-{topic}-{ns}artifact-{typ}-c{ci}",
|
||
prompt=_prompt(_ARTEFACT_PROMPT[typ], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)),
|
||
role="guide", capabilities="files",
|
||
payload=lambda result, p=p: schema(_json_file(p)),
|
||
timeout=_timeout("content", sum(len(blocks[i][1]) for i in idxs)))
|
||
return schema(_json_file(p)) is not None
|
||
|
||
await _gather_progress([_gen(ci, idxs) for ci, idxs in enumerate(chunks)], len(chunks), _report_p(set_p, topic, _ARTEFACT_STEP[typ]))
|
||
if is_cancelled():
|
||
return None
|
||
eintraege: list = []
|
||
for ci in range(len(chunks)):
|
||
chunk_items = schema(_json_file(apath(ci))) or []
|
||
if typ == "example" and chunk_items:
|
||
chunk_items = await _check_examples(ci, chunks[ci], chunk_items)
|
||
eintraege += chunk_items
|
||
return eintraege
|
||
|
||
results = await asyncio.gather(*[_one_type(t) for t in ARTEFACT_TYPES])
|
||
if is_cancelled() or any(r is None for r in results):
|
||
return None
|
||
outcome: dict[str, list] = dict(zip(ARTEFACT_TYPES, results))
|
||
atomic_write_json(files["artefakte"], outcome, indent=1)
|
||
return outcome
|
||
|
||
|
||
async def _mirror_artefacts_db(topic: str, sidecar: dict, artefacts: dict) -> None:
|
||
"""Mirror artefacts into the DB. Flashcard/example per sub (sub_norm)."""
|
||
await db.delete_sub_artefakte(topic)
|
||
btitle_list = list(sidecar.keys())
|
||
for type in ARTEFACT_TYPES:
|
||
for e in artefacts.get(type, []):
|
||
bt = _match_sub(e.get("block", ""), btitle_list)
|
||
bnorm, sn = _norm_title(bt), _norm_title(e.get("subblock", ""))
|
||
if not bnorm or not sn:
|
||
continue
|
||
data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")}, ensure_ascii=False)
|
||
await db.put_sub_artifact(topic, bnorm, sn, type, data, bt, e.get("subblock", ""))
|
||
|
||
|
||
async def _mirror_sidecar_db(topic: str, sidecar: dict) -> None:
|
||
"""Mirror the sidecar {block title: [{title, level, relevance}]} into the DB table subblocks."""
|
||
for btitle, subs in sidecar.items():
|
||
bnorm = _norm_title(btitle)
|
||
if not bnorm or not isinstance(subs, list):
|
||
continue
|
||
for s in subs:
|
||
if not isinstance(s, dict):
|
||
continue
|
||
st = str(s.get("title", "")).strip()
|
||
sn = _norm_title(st)
|
||
if not sn:
|
||
continue
|
||
facts = json.dumps(s["facts"], ensure_ascii=False) if isinstance(s.get("facts"), dict) else None
|
||
await db.put_subblock(topic, bnorm, sn, btitle, st,
|
||
level=s.get("level"), relevance=s.get("relevance"),
|
||
facts=facts, status="consensus")
|
||
|
||
|
||
async def _mirror_question_pattern_db(topic: str, pattern: dict) -> None:
|
||
"""Mirror question patterns {block title: [{subblock, question}]} into the DB table question_pattern."""
|
||
await db.delete_question_pattern(topic)
|
||
for btitle, eintraege in pattern.items():
|
||
bnorm = _norm_title(btitle)
|
||
if not bnorm or not isinstance(eintraege, list):
|
||
continue
|
||
for e in eintraege:
|
||
if not isinstance(e, dict):
|
||
continue
|
||
sub = str(e.get("subblock", "")).strip()
|
||
sn = _norm_title(sub)
|
||
question = str(e.get("question", "")).strip()
|
||
if not (sn and question):
|
||
continue
|
||
await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question)
|
||
|
||
|
||
|
||
|
||
async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER,
|
||
research: bool = True) -> None:
|
||
"""Kanban entry point: source prep, then both boards (inventory + artefacts) until
|
||
quiescence. research=False = Continue (drain the existing queue, no new search).
|
||
A run on a finished topic ADDS research (live extension) — full rebuild = DELETE /blocks."""
|
||
if topic in _blocks_progress:
|
||
return
|
||
_blocks_progress[topic] = "Warten…"
|
||
_blocks_errors.pop(topic, None)
|
||
|
||
files = _blocks_files(topic)
|
||
q = load_source(topic)
|
||
folder = source_folder(topic) # projekt/uni/link → folder, thema → None
|
||
instructions = q.get("spec") or instructions # prefer the persisted specification (also on resume)
|
||
|
||
def set_p(msg: str, step: int | None = None) -> None:
|
||
_blocks_progress[topic] = msg
|
||
if step is not None:
|
||
_blocks_step[topic] = step
|
||
|
||
def is_cancelled() -> bool:
|
||
return topic in _blocks_cancelled
|
||
|
||
ctx = GenContext(topic=topic, provider=provider, is_cancelled=is_cancelled)
|
||
try:
|
||
async with _semaphore:
|
||
files["arbeit"].mkdir(parents=True, exist_ok=True)
|
||
# Step "Source prep": crawl (link) + PDFs + content/noise triage.
|
||
if not await _prepare_source(ctx, set_p, files, q, folder, instructions):
|
||
if is_cancelled():
|
||
_blocks_errors[topic] = "Cancelled — progress is preserved"
|
||
return
|
||
import board_inventory # lazy: the boards import blocks
|
||
ok = await board_inventory.run_boards(ctx, set_p, files, q, folder, instructions,
|
||
research=research)
|
||
if not ok and is_cancelled():
|
||
_blocks_errors[topic] = "Cancelled — progress is preserved"
|
||
except Exception as e:
|
||
log.exception("[%s] Blocks generation failed", topic)
|
||
_blocks_errors[topic] = str(e)[:2000]
|
||
finally:
|
||
# No file cleanup: intermediate files stay for resume / traceability.
|
||
_blocks_progress.pop(topic, None)
|
||
_blocks_step.pop(topic, None)
|
||
_blocks_cancelled.discard(topic)
|
||
clear_scope(f"blocks-{topic}-") # clear the scope → restart isn't blocked
|