update
This commit is contained in:
team3
16
Makefile
16
Makefile
@@ -1,4 +1,4 @@
|
||||
.PHONY: install dev prod stop logs remove auth sync sync-projects sync-all sync-all-reverse projects searxng ollama qa
|
||||
.PHONY: install dev prod stop logs remove auth sync sync-projects sync-all sync-all-reverse projects searxng ollama qa test test-e2e train
|
||||
|
||||
COMPOSE = docker compose
|
||||
|
||||
@@ -114,3 +114,17 @@ qa-guide:
|
||||
cd backend && python3 guide_qa.py "$(TOPIC)" $(if $(LLM),--llm,)
|
||||
|
||||
projects: sync-projects
|
||||
|
||||
# Backend-Testsuite (Injektionstests + Fake-E2E, keine echten Agenten)
|
||||
test:
|
||||
cd backend && python3 -m pytest tests/ -q
|
||||
|
||||
# Nur die Fake-E2E-Läufe (kompletter Generierungspfad in Sekunden)
|
||||
test-e2e:
|
||||
cd backend && python3 -m pytest tests/test_e2e_fake.py -q
|
||||
|
||||
# Parameter-Training auf Mini-Themen: make train [TRIALS=40] [STUNDEN=12]
|
||||
# Achtung: jeder Trial ist ein echter Mini-Lauf (MiniMax-Tokens, Minuten).
|
||||
train:
|
||||
@set -a; [ -f .env ] && . ./.env; set +a; \
|
||||
cd backend && python3 train.py --trials $(or $(TRIALS),40) --stunden $(or $(STUNDEN),12)
|
||||
|
||||
@@ -245,6 +245,9 @@ async def run_agent(
|
||||
on_line=None,
|
||||
label: str = "",
|
||||
) -> tuple[int, str, str]:
|
||||
if os.getenv("CREATOR_FAKE_AGENTS"): # Sekunden-Smoke: deterministische Antworten statt LLM
|
||||
import fake_agents
|
||||
return await fake_agents.respond(agent_key, prompt, capabilities)
|
||||
if _scope_cancelled(agent_key): # before queueing: don't even enter the queue
|
||||
return 1, "", "cancelled"
|
||||
if provider not in PROVIDERS:
|
||||
|
||||
@@ -39,45 +39,14 @@ from textkit import (
|
||||
_resolve_title, _title_index, clean_title,
|
||||
)
|
||||
|
||||
# Chunk the subblocks (web search per block): 1 agent per ~10 blocks, capped.
|
||||
SUBBLOCK_CHUNK = 10
|
||||
SUBBLOCK_MAX = 40
|
||||
# Classifying is cheap (short verdict, no web search) → larger packages, fewer files/agents.
|
||||
LEVEL_CHUNK = 100
|
||||
|
||||
# Research: fixed file batches instead of a search loop → each crawl page is assigned exactly once.
|
||||
RESEARCH_BATCH = 20 # crawl pages per batch
|
||||
RESEARCH_READERS = 2 # reader agents per batch (consensus ≥2 within the batch)
|
||||
RESEARCH_THEMA_AGENTS = 5 # web mode (source "thema", no crawl folder)
|
||||
# uni/projekt: chunk the script text into sections of ~this size (against lost-in-the-middle on
|
||||
# large documents). ~12k chars ≈ 3k tokens → safely below the recall-drop threshold.
|
||||
RESEARCH_SECTION_CHARS = 12000
|
||||
# Triage (content/noise) is now a deterministic rule filter (config.CRAWL_*).
|
||||
SUBBLOCK_CAP = 900 # subblock find loop per chunk (15 min)
|
||||
SUBBLOCK_MIN = 5 # below this consensus count a block gets focused catch-up rounds
|
||||
SUBBLOCK_EXTRA_ROUNDS = 2 # max catch-up rounds (saturation stop still applies — thin stays thin)
|
||||
SUBBLOCK_MAX_ROUNDS = 3 # hard round cap: measured, rounds 4–5 burned 29 % of the finder agents
|
||||
# for ~zero consensus gain (fringe ideas never saturate) — thin blocks
|
||||
# are caught by the SUBBLOCK_MIN catch-up plus the gap follow-up round
|
||||
CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (dedups everything); above that chunked + merge pass — fallback path only
|
||||
DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair (complete-link aggregates → no chaining)
|
||||
DEDUP_PAIRS_CHUNK = 40 # pairs per judge package (pairwise verification instead of a block mixer)
|
||||
DEDUP_TITLE_AUTO = 0.95 # near-identical TITLE cosine ⇒ same entity → merge without the judge (recall net)
|
||||
DEDUP_GLOBAL_FLOOR = 0.65 # global post-naming dedup stage: candidate floor above the 0.5-0.65
|
||||
# same-domain noise band, below the sibling zone (~0.85) — the judge decides there
|
||||
FILTER_CHUNK = 35 # blocks to assess per judge in the degrade pass (full list as context)
|
||||
# Balance question-pattern chunks by sub load via LPT (makespan), not by block count.
|
||||
QUESTION_CHUNK_SUBS = 25 # target sum of relevant subs per chunk — at 50 the generator
|
||||
# skipped so many subs that 60 % of all question calls were catch-up
|
||||
QUESTION_MAX_ROUNDS = 3 # catch-up rounds for subs without a pattern (the LLM omits ~18 % per chunk)
|
||||
FACTS_CHUNK_SUBS = 10 # facts extraction: small chunks — the 4 phases (find/erg/check/fix) are
|
||||
# serial PER CHUNK, so chunk count = parallelism; the makespan tail of a
|
||||
# late block is bounded by ONE chunk's phase chain, not the whole block
|
||||
ARTEFACT_CHUNK_SUBS = 25 # flashcards/examples: bulk generation, phases are cheap → bigger packages
|
||||
FACTS_CHECK_PANEL = 3 # judges per chunk in the facts check (majority objects)
|
||||
CONSOLIDATION_PANEL = 3 # mapping judges per chunk (panel → reconcile instead of a single judge)
|
||||
SUBBLOCK_PANEL = 3 # source judges in the subblock clarification (majority instead of a single judge)
|
||||
FILTER_RECHECK_PANEL = 3 # judges in the survivor-recheck (rare-positive "fragment" recall; majority ≥2)
|
||||
# Pipeline-Tuning-Konstanten liegen zentral in config.py (tunebar via CREATOR_PARAMS).
|
||||
from config import ( # noqa: E402
|
||||
ARTEFACT_CHUNK_SUBS, CONSOLIDATION_CHUNK, CONSOLIDATION_PANEL, DEDUP_GLOBAL_FLOOR,
|
||||
DEDUP_PAIR_FLOOR, DEDUP_PAIRS_CHUNK, DEDUP_TITLE_AUTO, FACTS_CHECK_PANEL, FACTS_CHUNK_SUBS,
|
||||
FILTER_CHUNK, FILTER_RECHECK_PANEL, LEVEL_CHUNK, QUESTION_CHUNK_SUBS, QUESTION_MAX_ROUNDS,
|
||||
RESEARCH_BATCH, RESEARCH_READERS, RESEARCH_SECTION_CHARS, RESEARCH_THEMA_AGENTS,
|
||||
SUBBLOCK_CAP, SUBBLOCK_CHUNK, SUBBLOCK_EXTRA_ROUNDS, SUBBLOCK_MAX, SUBBLOCK_MAX_ROUNDS,
|
||||
SUBBLOCK_MIN, SUBBLOCK_PANEL)
|
||||
|
||||
log = logging.getLogger("creator.blocks")
|
||||
|
||||
@@ -608,6 +577,16 @@ def _reply_text(result) -> str:
|
||||
return (result[1] or "") if result else ""
|
||||
|
||||
|
||||
def _sink_or_file(result, path: Path, schema):
|
||||
"""Text reply preferred, file fallback. Research agents used to WRITE their big JSON —
|
||||
measured: one facts call spent 40 of 64 turns in a write/validate/repair loop (9 min).
|
||||
As text, parse_json_text repairs the escaping in one pass; tools stay for research."""
|
||||
val = _sink_json(result, path, schema)
|
||||
if val is not None:
|
||||
return val
|
||||
return schema(_json_file(path))
|
||||
|
||||
|
||||
def _sink_json(result, path: Path, schema):
|
||||
"""Payload validator for no-tool agents: the JSON comes as reply TEXT; the engine
|
||||
persists it to `path`, so resume guards and audit files keep working unchanged."""
|
||||
@@ -1502,6 +1481,38 @@ async def _luecken_runde(ctx: GenContext, files: dict, title: str, luecken: list
|
||||
return len(kept)
|
||||
|
||||
|
||||
async def _facts_nachfass(ctx: GenContext, files: dict, raw: dict, facts_map: dict, q: dict,
|
||||
folder, instructions: str = "", ns: str = "", lbl: str = "",
|
||||
sources: list[str] | None = None) -> int:
|
||||
"""ONE slim facts round for consensus subs WITHOUT a facts entry — renames during
|
||||
consolidation and catalog rows left 35 % of the subs ungrounded; the fact gate then
|
||||
flagged their (correct) guide statements wholesale. The subs themselves stay either
|
||||
way: they exist by consensus, only the grounding is fetched. → count of filled subs."""
|
||||
fehlend = {bt: [s for s in subs if _norm_title(s) not in (facts_map.get(bt) or {})]
|
||||
for bt, subs in raw.items()}
|
||||
fehlend = {bt: subs for bt, subs in fehlend.items() if subs}
|
||||
if not fehlend:
|
||||
return 0
|
||||
nf_dir = files["arbeit"] / "nf2"
|
||||
nf_dir.mkdir(parents=True, exist_ok=True)
|
||||
res = await _facts_block(ctx, lambda *a, **k: None, {**files, "arbeit": nf_dir},
|
||||
fehlend, q, folder, instructions,
|
||||
ns=f"{ns}nf2-", lbl=lbl, sources=sources, slim=True)
|
||||
if ctx.is_cancelled() or res is None:
|
||||
return 0
|
||||
nf_facts, _discarded = res # discard verdicts ignored — consensus subs are not removed here
|
||||
filled = 0
|
||||
for bt, fm in nf_facts.items():
|
||||
bfacts = facts_map.setdefault(bt, {})
|
||||
for sn, fk in fm.items():
|
||||
if sn not in bfacts and (fk.get("key_points") or fk.get("cited_facts")):
|
||||
bfacts[sn] = fk
|
||||
filled += 1
|
||||
if filled:
|
||||
_log(ctx.topic, f"Facts-Nachfass{': ' + lbl if lbl else ''}{filled} Subs nachbelegt")
|
||||
return filled
|
||||
|
||||
|
||||
def _subs_hash(sidecar_or_raw: dict) -> str:
|
||||
"""Sub-set identity for the resume files of the sub-CONSUMING stages (levels/relevance/
|
||||
questions/artefacts). Without it a re-run with a recut sub set adopted the stale stage
|
||||
@@ -1811,7 +1822,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
|
||||
ctx, f"{lbl}Facts {ci}", key=f"blocks-{topic}-{ns}facts-c{ci}",
|
||||
prompt=_prompt("Facts-Research", topic=topic, source=source, blocks=block_text(idxs), out_path=fp, extra=_extra(instructions)),
|
||||
role="quick", capabilities=caps,
|
||||
payload=lambda result, p=fp: _facts_schema(_json_file(p)),
|
||||
payload=lambda result, p=fp: _sink_or_file(result, p, _facts_schema),
|
||||
timeout=_timeout("content", subs_total))
|
||||
return status != FAILED and _facts_schema(_json_file(fp)) is not None
|
||||
|
||||
@@ -1842,7 +1853,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
|
||||
ctx, f"{lbl}Facts supplement {ci}", key=f"blocks-{topic}-{ns}facts-erg-c{ci}",
|
||||
prompt=_prompt("Facts-Supplement", topic=topic, source=source, blocks=block, out_path=ep, extra=_extra(instructions)),
|
||||
role="quick", capabilities=caps,
|
||||
payload=lambda result, p=ep: _facts_schema(_json_file(p)),
|
||||
payload=lambda result, p=ep: _sink_or_file(result, p, _facts_schema),
|
||||
timeout=_timeout("content", subs_total))
|
||||
|
||||
if not slim:
|
||||
@@ -1929,7 +1940,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
|
||||
ctx, f"{lbl}Facts-Fix {ci}", key=f"blocks-{topic}-{ns}facts-fix-c{ci}",
|
||||
prompt=_prompt("Facts-Research", topic=topic, source=source, blocks="\n\n".join(goal), out_path=fix_path(ci), extra=_extra(instructions)),
|
||||
role="quick", capabilities=caps,
|
||||
payload=lambda result, p=fix_path(ci): _facts_schema(_json_file(p)),
|
||||
payload=lambda result, p=fix_path(ci): _sink_or_file(result, p, _facts_schema),
|
||||
timeout=_timeout("content", len(subs_norm)))
|
||||
await _gather_progress([_fix(ci) for ci in correctable], len(correctable), _report_p(set_p, topic, "Facts fix"))
|
||||
if is_cancelled():
|
||||
@@ -2121,7 +2132,7 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar:
|
||||
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=block,
|
||||
out_path=fp, extra=_extra(instructions)),
|
||||
role="quick", capabilities="files",
|
||||
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
|
||||
payload=lambda result, p=fp: _sink_or_file(result, p, _question_pattern_chunk_schema),
|
||||
timeout=_timeout("question_pattern", subs_total),
|
||||
)
|
||||
if status == FAILED:
|
||||
@@ -2255,7 +2266,7 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar:
|
||||
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=_followup_block(items),
|
||||
out_path=fp, extra=_extra(instructions)),
|
||||
role="quick", capabilities="files",
|
||||
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
|
||||
payload=lambda result, p=fp: _sink_or_file(result, p, _question_pattern_chunk_schema),
|
||||
timeout=_timeout("question_pattern", subs_total),
|
||||
)
|
||||
|
||||
@@ -3013,8 +3024,8 @@ async def _learning_order(ctx: GenContext, set_p, files: dict, entries: dict, va
|
||||
pp = files["arbeit"] / "outline-prereqs.json"
|
||||
|
||||
def _payload(result, p=pp):
|
||||
d = _json_file(p)
|
||||
return d if isinstance(d, dict) and "prereqs" in d else None
|
||||
d = _sink_or_file(result, p, lambda x: x if isinstance(x, dict) and "prereqs" in x else None)
|
||||
return d
|
||||
|
||||
existing = _json_file(pp)
|
||||
if not (isinstance(existing, dict) and "prereqs" in existing):
|
||||
@@ -3052,7 +3063,7 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
|
||||
ctx, f"Outline {i}", key=f"blocks-{topic}-outline-{i}",
|
||||
prompt=_prompt("Guide-Outline", topic=topic, blocks=liste, out_path=path, extra=_extra(instructions)),
|
||||
role="guide", capabilities="files",
|
||||
payload=lambda result, p=path: _outline_schema(_json_file(p), valid),
|
||||
payload=lambda result, p=path: _sink_or_file(result, p, lambda d: _outline_schema(d, valid)),
|
||||
timeout=_timeout("plan", len(entries)))
|
||||
return _outline_schema(_json_file(path), valid) is not None
|
||||
|
||||
@@ -3078,7 +3089,8 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
|
||||
purpose="alle Blocks in einem roten Faden", n=len(proposals),
|
||||
blocks=liste, outlines=block_texts, out_path=files["outline"], extra=_extra(instructions)),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result: _outline_schema(_json_file(files["outline"]), valid),
|
||||
payload=lambda result: _sink_or_file(result, files["outline"],
|
||||
lambda d: _outline_schema(d, valid)),
|
||||
timeout=_timeout("plan_judge", len(entries)))
|
||||
plan = _outline_schema(_json_file(files["outline"]), valid) or proposals[0]
|
||||
|
||||
@@ -3097,8 +3109,8 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
|
||||
prompt=_prompt("Guide-Outline-Review", topic=topic, chapters=chapter_text,
|
||||
out_path=rp, extra=_extra(instructions)),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result: _outline_review_schema(
|
||||
_json_file(rp), valid, len(plan["chapters"]), len(entries)),
|
||||
payload=lambda result: _sink_or_file(result, rp, lambda d: _outline_review_schema(
|
||||
d, valid, len(plan["chapters"]), len(entries))),
|
||||
timeout=_timeout("plan_judge", len(entries)))
|
||||
moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
|
||||
for nr, target in (moves or {}).items():
|
||||
@@ -3257,7 +3269,7 @@ async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i
|
||||
ctx, f"{lbl}{_ARTEFACT_STEP[typ]} {ci}", key=f"blocks-{topic}-{ns}artifact-{typ}-c{ci}",
|
||||
prompt=_prompt(_ARTEFACT_PROMPT[typ], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)),
|
||||
role="guide", capabilities="files",
|
||||
payload=lambda result, p=p: schema(_json_file(p)),
|
||||
payload=lambda result, p=p: _sink_or_file(result, p, schema),
|
||||
timeout=_timeout("content", sum(len(blocks[i][1]) for i in idxs)))
|
||||
return schema(_json_file(p)) is not None
|
||||
|
||||
|
||||
@@ -19,11 +19,11 @@ import database as db
|
||||
import blocks
|
||||
import embedding
|
||||
from blocks import (
|
||||
ARTEFACT_TYPES, _artefacts_block, _facts_block, _konsolidiere_subblocks, _levels_block,
|
||||
_luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
|
||||
ARTEFACT_TYPES, _artefacts_block, _facts_block, _facts_nachfass, _konsolidiere_subblocks,
|
||||
_levels_block, _luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
|
||||
_sink_json, _subblocks_block, _outline_block,
|
||||
)
|
||||
from config import EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
|
||||
from config import CROSS_CHUNK_PAARE, EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
|
||||
from fsutil import atomic_write_json
|
||||
from jsonio import read_json_file as _json_file
|
||||
from kanban import Flow, Stage
|
||||
@@ -243,6 +243,13 @@ async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
|
||||
if ctx.is_cancelled():
|
||||
return None
|
||||
raw = {bt: subs for bt, subs in raw.items() if subs}
|
||||
# consolidation renames/catalogs can leave consensus subs without grounding — the
|
||||
# guide fact gate then flags their correct statements wholesale (measured: 74/210)
|
||||
await _facts_nachfass(ctx, _pfiles(files, norm), raw, facts_map, q, folder,
|
||||
instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
|
||||
if ctx.is_cancelled():
|
||||
return None
|
||||
p["raw"], p["facts"] = raw, facts_map
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "levels")
|
||||
@@ -323,98 +330,110 @@ async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instruc
|
||||
def _side(tag: str, ci: int, bt: str, s: str) -> str:
|
||||
return f"{tag}: [Block: {bt}] {s}" + "".join(f"\n - {p}" for p in _kp(ci, bt, s))
|
||||
|
||||
lines = "\n\n".join(
|
||||
f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
|
||||
for k, (i, j) in enumerate(pairs, 1))
|
||||
h = hashlib.md5(lines.encode()).hexdigest()[:8]
|
||||
paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
|
||||
# chunked judging: ONE call over all pairs scaled its timeout past 50 min, and a hung
|
||||
# call blocked the barrier for the full window (measured on aak: 196 pairs, 2×54 min)
|
||||
chunks = [pairs[lo:lo + CROSS_CHUNK_PAARE] for lo in range(0, len(pairs), CROSS_CHUNK_PAARE)]
|
||||
|
||||
async def _judge(j, path):
|
||||
if _cross_schema(_json_file(path)) is not None:
|
||||
return # resume
|
||||
status, _v = await run_single_slot(
|
||||
ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
|
||||
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=lines, extra=_extra(instructions)),
|
||||
role="judge", capabilities="none",
|
||||
payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
|
||||
timeout=_timeout("subblock_check", len(pairs)))
|
||||
if status == FAILED:
|
||||
_log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")
|
||||
async def _urteile_chunk(chunk: list[tuple[int, int]]) -> dict[int, str]:
|
||||
"""Two judges (+ substitute, + tie-breaker) over one pair chunk → {local_k: verdict};
|
||||
empty dict = fail-open (pairs stay)."""
|
||||
lines = "\n\n".join(
|
||||
f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
|
||||
for k, (i, j) in enumerate(chunk, 1))
|
||||
h = hashlib.md5(lines.encode()).hexdigest()[:8]
|
||||
paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
|
||||
|
||||
await asyncio.gather(*[_judge(j, p) for j, p in zip((1, 2), paths)])
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
|
||||
if len(outs) == 1: # Ersatz-Richter statt fail-open bei EINEM Ausfall
|
||||
ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
|
||||
await _judge("E", ersatz)
|
||||
async def _judge(j, path, plines, n):
|
||||
if _cross_schema(_json_file(path)) is not None:
|
||||
return # resume
|
||||
status, _v = await run_single_slot(
|
||||
ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
|
||||
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=plines, extra=_extra(instructions)),
|
||||
role="judge", capabilities="none",
|
||||
payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
|
||||
timeout=_timeout("subblock_check", n))
|
||||
if status == FAILED:
|
||||
_log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")
|
||||
|
||||
await asyncio.gather(*[_judge(j, p, lines, len(chunk)) for j, p in zip((1, 2), paths)])
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
|
||||
journal = {"paare": len(pairs), "richter": len(outs), "gefaltet": [], "verdicts": []}
|
||||
gone: set[int] = set()
|
||||
touched: set[int] = set()
|
||||
if len(outs) == 2:
|
||||
return {}
|
||||
outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
|
||||
if len(outs) == 1: # Ersatz-Richter statt fail-open bei EINEM Ausfall
|
||||
ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
|
||||
await _judge("E", ersatz, lines, len(chunk))
|
||||
if ctx.is_cancelled():
|
||||
return {}
|
||||
outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
|
||||
if len(outs) != 2:
|
||||
if outs:
|
||||
_log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
|
||||
return {}
|
||||
final = {k: (outs[0].get(k, "nein") if outs[0].get(k, "nein") == outs[1].get(k, "nein")
|
||||
else "uneinig") for k in range(1, len(pairs) + 1)}
|
||||
else "uneinig") for k in range(1, len(chunk) + 1)}
|
||||
disputed = [k for k, v in final.items() if v == "uneinig"]
|
||||
if disputed: # tie-breaker: a third judge sees ONLY the disputed pairs, majority 2/3
|
||||
d_lines = "\n\n".join(
|
||||
f"{x}.\n{_side('A', *entries[pairs[k - 1][0]])}\n{_side('B', *entries[pairs[k - 1][1]])}"
|
||||
f"{x}.\n{_side('A', *entries[chunk[k - 1][0]])}\n{_side('B', *entries[chunk[k - 1][1]])}"
|
||||
for x, k in enumerate(disputed, 1))
|
||||
p3 = work_dir / f"sub-crossblock-{h}-j3.json"
|
||||
if _cross_schema(_json_file(p3)) is None:
|
||||
status, _v = await run_single_slot(
|
||||
ctx, "Sub-Crossblock j3", key=f"blocks-{topic}-sub-crossblock-{h}-j3",
|
||||
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=d_lines, extra=_extra(instructions)),
|
||||
role="judge", capabilities="none",
|
||||
payload=lambda result, p=p3: _sink_json(result, p, _cross_schema),
|
||||
timeout=_timeout("subblock_check", len(disputed)))
|
||||
if status == FAILED:
|
||||
_log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
|
||||
await _judge(3, p3, d_lines, len(disputed))
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
return {}
|
||||
v3 = _cross_schema(_json_file(p3)) or {}
|
||||
if not v3:
|
||||
_log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
|
||||
for x, k in enumerate(disputed, 1):
|
||||
t = v3.get(x, "nein")
|
||||
if t in (outs[0].get(k, "nein"), outs[1].get(k, "nein")):
|
||||
final[k] = t # majority 2/3; anything else stays disputed → no fold
|
||||
for k, (i, j) in enumerate(pairs, 1):
|
||||
verdict = final[k]
|
||||
journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
|
||||
"b": f"{entries[j][1]} · {entries[j][2]}",
|
||||
"verdict": verdict})
|
||||
if verdict not in ("a", "b"):
|
||||
continue
|
||||
lose = j if verdict == "a" else i
|
||||
if entries[lose][0] < 0: # context never folds — the package side goes instead
|
||||
lose = i if lose == j else j
|
||||
keep = i if lose == j else j
|
||||
if lose in gone or keep in gone: # keeper already folded → don't chain away the content
|
||||
continue
|
||||
ci, bt, s = entries[lose]
|
||||
p = cards[ci]["payload"]
|
||||
if s in (p.get("raw") or {}).get(bt, []):
|
||||
p["raw"][bt].remove(s)
|
||||
(p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
|
||||
sc = (p.get("sidecar") or {}).get(bt)
|
||||
if isinstance(sc, list): # questions/artefacts consume the sidecar downstream
|
||||
p["sidecar"][bt] = [e for e in sc
|
||||
if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
|
||||
await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
|
||||
gone.add(lose)
|
||||
touched.add(ci)
|
||||
journal["gefaltet"].append({"weg": f"{bt} · {s}",
|
||||
"bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
|
||||
elif outs:
|
||||
_log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
|
||||
return final
|
||||
|
||||
chunk_finals = await asyncio.gather(*[_urteile_chunk(c) for c in chunks])
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
final_all: dict[int, str] = {} # global pair index (1-based over `pairs`) → verdict
|
||||
for cnr, fin in enumerate(chunk_finals):
|
||||
for k, v in fin.items():
|
||||
final_all[cnr * CROSS_CHUNK_PAARE + k] = v
|
||||
journal = {"paare": len(pairs), "chunks": len(chunks), "gefaltet": [], "verdicts": []}
|
||||
gone: set[int] = set()
|
||||
touched: set[int] = set()
|
||||
for k, (i, j) in enumerate(pairs, 1):
|
||||
verdict = final_all.get(k, "nein")
|
||||
journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
|
||||
"b": f"{entries[j][1]} · {entries[j][2]}",
|
||||
"verdict": verdict})
|
||||
if verdict not in ("a", "b"):
|
||||
continue
|
||||
lose = j if verdict == "a" else i
|
||||
if entries[lose][0] < 0: # context never folds — the package side goes instead
|
||||
lose = i if lose == j else j
|
||||
keep = i if lose == j else j
|
||||
if lose in gone or keep in gone: # keeper already folded → don't chain away the content
|
||||
continue
|
||||
ci, bt, s = entries[lose]
|
||||
p = cards[ci]["payload"]
|
||||
if s in (p.get("raw") or {}).get(bt, []):
|
||||
p["raw"][bt].remove(s)
|
||||
(p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
|
||||
sc = (p.get("sidecar") or {}).get(bt)
|
||||
if isinstance(sc, list): # questions/artefacts consume the sidecar downstream
|
||||
p["sidecar"][bt] = [e for e in sc
|
||||
if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
|
||||
await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
|
||||
gone.add(lose)
|
||||
touched.add(ci)
|
||||
journal["gefaltet"].append({"weg": f"{bt} · {s}",
|
||||
"bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
|
||||
for ci in touched:
|
||||
p = cards[ci]["payload"]
|
||||
p["raw"] = {bt: subs for bt, subs in (p.get("raw") or {}).items() if subs}
|
||||
await db.kanban_set_payload(topic, BOARD, cards[ci]["card_id"], p)
|
||||
if journal["gefaltet"]:
|
||||
_log(topic, f"Sub-Crossblock: {len(journal['gefaltet'])} blockübergreifende Dublette(n) gefaltet")
|
||||
atomic_write_json(work_dir / f"sub-crossblock-{h}.json", journal, indent=1)
|
||||
hg = hashlib.md5("\n".join(f"{i}:{j}" for i, j in pairs).encode()).hexdigest()[:8]
|
||||
atomic_write_json(work_dir / f"sub-crossblock-{hg}.json", journal, indent=1)
|
||||
await _advance_all()
|
||||
|
||||
|
||||
@@ -433,7 +452,10 @@ async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: s
|
||||
for btitle, subs in sidecar.items(): # merge facts into the subs (mirror + guide)
|
||||
fm = facts_map.get(btitle, {})
|
||||
for sub in subs:
|
||||
if (fk := fm.get(_norm_title(sub["title"]))):
|
||||
# level agents paraphrase titles — exact miss falls back to the unique
|
||||
# prefix/containment match, else the sub silently loses its grounding
|
||||
sn = _sub_key(set(fm), _norm_title(sub["title"]))
|
||||
if (fk := fm.get(sn)):
|
||||
sub["facts"] = fk
|
||||
p["sidecar"] = sidecar
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
@@ -535,12 +557,15 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
|
||||
# DB mirrors — per block only (no global deletes)
|
||||
await blocks._mirror_sidecar_db(topic, sidecar)
|
||||
# stale question/artefact rows of a PREVIOUS run keyed to gone subs: finalize only
|
||||
# upserts, so re-runs left orphans (measured: 28). subblocks rows stay — QA needs
|
||||
# the variant/discarded statuses, and the sidecar mirror re-writes only consensus.
|
||||
# upserts, so re-runs left orphans (measured: 28).
|
||||
await db.delete_question_pattern(topic, _norm_title(title))
|
||||
await db.delete_sub_artefakte(topic, _norm_title(title))
|
||||
# stragglers the mirror didn't level (row not in this run's sidecar): without a
|
||||
# valid level they vanish from guide/practice/level views while QA still counts them
|
||||
# consensus rows of a PREVIOUS run that this run's sidecar no longer carries would
|
||||
# linger without facts/questions/artefacts (measured: 25) — drop them per block;
|
||||
# variant/discarded stay for QA. Then default-level the mirror's own stragglers.
|
||||
for btitle, subs in sidecar.items():
|
||||
keep = {_norm_title(str(s.get("title", ""))) for s in subs if isinstance(s, dict)}
|
||||
await db.delete_stale_consensus(topic, _norm_title(btitle), keep - {""})
|
||||
await db.default_subblock_levels(topic, _norm_title(title))
|
||||
sub_keys: dict[str, set[str]] = {}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import shutil
|
||||
import unicodedata
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
@@ -63,7 +64,7 @@ from textkit import _norm_title, _parse_selection, _title, clean_title
|
||||
log = logging.getLogger("creator.board_inventory")
|
||||
|
||||
BOARD = "inventory"
|
||||
RESEARCH_RUNTIME = 900 # one research agent, one round — the tail ingests live while it writes
|
||||
from config import RESEARCH_RUNTIME # noqa: E402 — zentral tunebar
|
||||
_POLL_RESEARCH = 3 # seconds between live reads of a running research file
|
||||
|
||||
_ingest_lock = asyncio.Lock() # serializes the read-modify-write title upserts
|
||||
@@ -84,15 +85,37 @@ def _line(i: int, p: dict, mark: str = "") -> str:
|
||||
return f"{i}. {mark}{p['title']} — {d}" if d else f"{i}. {mark}{p['title']}"
|
||||
|
||||
|
||||
def _naming_schema(data, count: int) -> int | None:
|
||||
"""{"best": N} → 1-based member index in [1, count] · otherwise None."""
|
||||
def _naming_schema(data, count: int) -> tuple[int | None, str | None, bool] | None:
|
||||
"""{"ok":true} → (None,None,True) · {"best":N[,"name":…]} → (N, name|None, False).
|
||||
name über 80 Zeichen wird verworfen (Kürze ist der Zweck der Abstraktion)."""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if data.get("ok") is True:
|
||||
return None, None, True
|
||||
try:
|
||||
n = int(data.get("best"))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return n if 1 <= n <= count else None
|
||||
if not 1 <= n <= count:
|
||||
return None
|
||||
name = str(data.get("name", "")).strip() or None
|
||||
if name and len(name) > 80:
|
||||
name = None
|
||||
return n, name, False
|
||||
|
||||
|
||||
def _name_verankert(name: str, rows: list[dict], ctoks: set[str] | None) -> bool:
|
||||
"""Abstraction guard: a free-formed title must be anchored — in the corpus (uni/projekt)
|
||||
or in the members' own words (thema). Unanchored names drift to textbook canon
|
||||
(measured: 'Königsberger Brückenproblem', 0 corpus hits) → fall back to best-of-members."""
|
||||
if ctoks is not None:
|
||||
return _hat_anker(name, ctoks)
|
||||
from qa import _STOP
|
||||
basis = " ".join(f"{r.get('title', '')} {r.get('description') or ''}" for r in rows)
|
||||
fold = lambda s: unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode().casefold() # noqa: E731
|
||||
btoks = set(re.findall(r"\w{3,}", fold(basis)))
|
||||
ntoks = {t for t in re.findall(r"\w{3,}", fold(name)) if t not in _STOP}
|
||||
return bool(ntoks) and ntoks <= btoks
|
||||
|
||||
|
||||
# ── Embedding cache (per flow) ─────────────────────────────────────────────────────
|
||||
@@ -450,15 +473,25 @@ def _hat_anker(title: str, ctoks: set[str]) -> bool:
|
||||
async def _anker_beleg(ctx: GenContext, flow: Flow, kandidaten: list[tuple[str, dict]]) -> set[str]:
|
||||
"""Evidence judge for quorum titles WITHOUT any corpus anchor — two readers naming the
|
||||
same famous canon independently beat the quorum although the material never mentions it
|
||||
(measured: 'Königsberger Brückenproblem', 0 corpus hits). FAIL-OPEN: the titles carry
|
||||
2-reader backing, only an explicit 'nein' rejects. → card_ids to reject."""
|
||||
(measured: 'Königsberger Brückenproblem', 0 corpus hits). FAIL-OPEN for judge errors —
|
||||
but an EMPTY evidence pack (no distinctive token anywhere in the corpus) rejects
|
||||
deterministically: the judge approved 3 canon titles on sham excerpts. → card_ids to reject."""
|
||||
from qa import _distinctive
|
||||
topic = flow.topic
|
||||
folder = source_folder(topic)
|
||||
lines = []
|
||||
hart_nein: set[str] = set()
|
||||
for k, (cid, p) in enumerate(kandidaten, 1):
|
||||
ev = _evidence_pack(folder, None, [p.get("title", ""), p.get("description") or ""], budget=4000)
|
||||
# distinctive tokens only: title+description as raw queries pulled generic sections
|
||||
# („Beispiel", „klassisch") — the judge saw sham excerpts and waved canon through
|
||||
toks = _distinctive(p.get("title", "")) | _distinctive(p.get("description") or "")
|
||||
ev = _evidence_pack(folder, None, [" ".join(sorted(toks))], budget=4000) if toks else ""
|
||||
if not ev: # nothing to attest — deterministic reject, no judge to sweet-talk
|
||||
hart_nein.add(cid)
|
||||
lines.append(f"{k}. {p.get('title', '')} — {p.get('description') or ''}\nAUSZÜGE:\n"
|
||||
f"{ev or '(keine passenden Auszüge im Material gefunden)'}")
|
||||
if len(hart_nein) == len(kandidaten):
|
||||
return hart_nein
|
||||
h = _h(*[cid for cid, _ in kandidaten])
|
||||
path = flow.work_dir / f"anker-beleg-{h}.json"
|
||||
ids = set(range(1, len(kandidaten) + 1))
|
||||
@@ -473,7 +506,8 @@ async def _anker_beleg(ctx: GenContext, flow: Flow, kandidaten: list[tuple[str,
|
||||
timeout=_timeout("selection_mapping", len(kandidaten)))
|
||||
if status != OK or not isinstance(verdict, dict):
|
||||
verdict = {} # fail-open
|
||||
return {cid for k, (cid, _p) in enumerate(kandidaten, 1) if str(verdict.get(k, "ja")) == "nein"}
|
||||
return hart_nein | {cid for k, (cid, _p) in enumerate(kandidaten, 1)
|
||||
if str(verdict.get(k, "ja")) == "nein"}
|
||||
|
||||
|
||||
async def _proc_consensus_gate(ctx: GenContext, flow: Flow, cards):
|
||||
@@ -608,32 +642,42 @@ async def _proc_clarify(ctx: GenContext, flow: Flow, cards):
|
||||
|
||||
|
||||
async def _choose_title(ctx: GenContext, flow: Flow, cid: str, rows: list[dict],
|
||||
template: str, current: int | None = None) -> str:
|
||||
"""Judge picks the best member title (index). Parse-fail → survivorship fallback."""
|
||||
template: str, current_title: str | None = None) -> tuple[str, str | None]:
|
||||
"""Judge picks the best member title and MAY propose a short abstracted name — kept
|
||||
only when anchored (_name_verankert). → (member_norm, custom_title|None);
|
||||
("", None) on cancel; {"ok":true} in the check keeps the current title."""
|
||||
topic = flow.topic
|
||||
h = _h(*[r["norm"] for r in rows], template)
|
||||
path = flow.work_dir / f"naming-{cid}-{h}.json"
|
||||
best = _naming_schema(_json_file(path), len(rows))
|
||||
if best is None:
|
||||
verdict = _naming_schema(_json_file(path), len(rows))
|
||||
if verdict is None:
|
||||
lines = "\n".join(f"{k + 1}. {_t_text(r)}" for k, r in enumerate(rows))
|
||||
kw = dict(topic=topic, members=lines, out_path=path)
|
||||
if current is not None:
|
||||
kw["current"] = current
|
||||
status, best = await run_single_slot(
|
||||
kw = dict(topic=topic, members=lines)
|
||||
if current_title is not None:
|
||||
kw["current_title"] = current_title
|
||||
status, verdict = await run_single_slot(
|
||||
ctx, f"Naming {cid}", key=f"blocks-{topic}-naming-{cid}-{h}",
|
||||
prompt=_prompt(template, **kw), role="judge", capabilities="files",
|
||||
payload=lambda result, p=path, n=len(rows): _naming_schema(_json_file(p), n),
|
||||
prompt=_prompt(template, **kw), role="judge", capabilities="none",
|
||||
payload=lambda result, p=path, n=len(rows): _sink_json(result, p, lambda d: _naming_schema(d, n)),
|
||||
timeout=_timeout("selection_mapping", len(rows)))
|
||||
if status == CANCELLED:
|
||||
return ""
|
||||
return "", None
|
||||
if status == FAILED:
|
||||
best = None
|
||||
if best is None:
|
||||
verdict = None
|
||||
if verdict is None:
|
||||
rep = _rep(rows)
|
||||
w = _norm_title(rep["title"])
|
||||
norms = [r["norm"] for r in rows]
|
||||
return w if w in norms else norms[0]
|
||||
return rows[best - 1]["norm"]
|
||||
return (w if w in norms else norms[0]), None
|
||||
best, name, ok = verdict
|
||||
if ok: # check judge confirms the current (possibly custom) title: change nothing
|
||||
return "", (current_title or "")
|
||||
custom = None
|
||||
if name:
|
||||
ctoks = flow.state.get("korpus_tokens")
|
||||
if _name_verankert(name, rows, ctoks):
|
||||
custom = clean_title(name)
|
||||
return rows[best - 1]["norm"], custom
|
||||
|
||||
|
||||
async def _proc_naming(ctx: GenContext, flow: Flow, cards):
|
||||
@@ -650,13 +694,15 @@ async def _name_one(ctx: GenContext, flow: Flow, c):
|
||||
p = c["payload"]
|
||||
rows = await _member_rows(topic, cid)
|
||||
if len(rows) > 1 and not p.get("renamed"):
|
||||
winner = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming")
|
||||
if not winner: # cancelled
|
||||
winner, custom = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming")
|
||||
if not winner and custom is None: # cancelled
|
||||
return
|
||||
p["main_norm"] = winner
|
||||
w = next(r for r in rows if r["norm"] == winner)
|
||||
p["title"], p["description"] = w["title"], w.get("description") or p.get("description", "")
|
||||
await db.kanban_set_payload(topic, BOARD, cid, p)
|
||||
if winner:
|
||||
p["main_norm"] = winner
|
||||
w = next(r for r in rows if r["norm"] == winner)
|
||||
p["title"] = custom or w["title"]
|
||||
p["description"] = w.get("description") or p.get("description", "")
|
||||
await db.kanban_set_payload(topic, BOARD, cid, p)
|
||||
await db.kanban_advance(topic, BOARD, cid, "naming_check")
|
||||
|
||||
|
||||
@@ -675,14 +721,14 @@ async def _namecheck_one(ctx: GenContext, flow: Flow, c):
|
||||
p = c["payload"]
|
||||
rows = await _member_rows(topic, cid)
|
||||
if len(rows) > 1 and not p.get("renamed"):
|
||||
norms = [r["norm"] for r in rows]
|
||||
cur = p.get("main_norm")
|
||||
current = norms.index(cur) + 1 if cur in norms else 1
|
||||
winner = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming-Check", current=current)
|
||||
if not winner:
|
||||
winner, custom = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming-Check",
|
||||
current_title=p.get("title", ""))
|
||||
if not winner and custom is None: # cancelled
|
||||
return
|
||||
w = next(r for r in rows if r["norm"] == winner)
|
||||
p["title"], p["description"] = w["title"], w.get("description") or p.get("description", "")
|
||||
if winner: # ok-verdict ("" + custom) keeps title AND description untouched
|
||||
w = next(r for r in rows if r["norm"] == winner)
|
||||
p["title"] = custom or w["title"]
|
||||
p["description"] = w.get("description") or p.get("description", "")
|
||||
readers = sorted(set().union(*[set(r.get("readers") or []) for r in rows])) if rows else []
|
||||
sources = sorted(set().union(*[set(r.get("sources") or []) for r in rows])) if rows else []
|
||||
await db.kanban_upsert_card(topic, BOARD, f"b-{cid}", "block", "fragment_filter", {
|
||||
@@ -1872,6 +1918,13 @@ async def reset_board_from_stage(topic: str, board: str, stage: str, files: dict
|
||||
await db.delete_subblocks(topic)
|
||||
await db.delete_question_pattern(topic)
|
||||
await db.delete_sub_artefakte(topic)
|
||||
# global sidecar files and per-block resume slots: leftovers of the previous
|
||||
# derive would re-merge/resume into the fresh run — nothing here is reused
|
||||
for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"):
|
||||
files[k].unlink(missing_ok=True)
|
||||
for d in files["arbeit"].glob("ab-*"):
|
||||
if d.is_dir():
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
await db.add_event(topic, "reset", key=f"{board}:from-{stage}", status=str(moved))
|
||||
return moved
|
||||
|
||||
|
||||
@@ -75,6 +75,9 @@ SEED_COVER_COS = 0.80
|
||||
# (Markdown: 50 pairs in the band, 4 above) — below every auto-merge threshold, so an LLM
|
||||
# judge decides. Candidates only; a merge still needs judge unanimity.
|
||||
SUB_DUP_KANDIDAT_COS = 0.75
|
||||
# Cross-block judge pairs per call: ONE call over all pairs scaled its timeout to 54 min
|
||||
# and a hung call blocked the barrier that long (aak: 196 pairs) — chunks cap it at ~15 min.
|
||||
CROSS_CHUNK_PAARE = 40
|
||||
|
||||
# Umbrella grouping (block granularity level 2, step "Blocks-Gruppierung", AFTER the filter):
|
||||
# collapse sibling DEFINITIONS that are components of ONE umbrella concept (TM model:
|
||||
@@ -144,12 +147,56 @@ QUELLE_RELEVANZ_SNIPPET = 800 # body characters per page in the prompt (URL i
|
||||
QA_GATE_NOTE = 9.5 # 0 = gate off; quota-based, so the tolerated finding count scales with topic size
|
||||
QA_GATE_LLM = True # include the LLM samples (Echtheit/Dubletten) in the gate run
|
||||
|
||||
# Guide section length per relevant sub (ausführlich part) — QA detector AND the
|
||||
# deterministic readability-stage trigger share these bounds (writers overshot 2.7–4.1×).
|
||||
GUIDE_LAENGE_MIN = 150
|
||||
GUIDE_LAENGE_MAX = 1200
|
||||
|
||||
# Inline evidence for judge agents: corpus excerpts go INTO the prompt instead of letting
|
||||
# every judge re-search the source folder (measured: ~10 tool turns/judge, 82 % of the
|
||||
# run's tokens were cache reads from those loops).
|
||||
EVIDENCE_BUDGET_CHARS = 48_000 # max excerpt characters per judge prompt
|
||||
EVIDENCE_CTX_LINES = 15 # context lines around a cited source position (facts check)
|
||||
|
||||
# ── Pipeline tuning (zentral, tunebar via CREATOR_PARAMS — siehe Override-Hook am Datei-Ende;
|
||||
# Registry mit Suchraum: backend/train_params.py). QA-/Detektor-Konstanten bleiben bewusst in
|
||||
# qa.py/guide_qa.py — die Messlatte darf nie Teil des Suchraums sein. ─────────────────────────
|
||||
SUBBLOCK_CHUNK = 10 # subblock finder: 1 agent per ~10 blocks, capped
|
||||
SUBBLOCK_MAX = 40 # chunk cap
|
||||
LEVEL_CHUNK = 100 # classifying is cheap → large packages
|
||||
RESEARCH_BATCH = 20 # crawl pages per batch
|
||||
RESEARCH_READERS = 2 # reader agents per batch/section (consensus ≥2)
|
||||
RESEARCH_THEMA_AGENTS = 5 # web mode (source "thema")
|
||||
RESEARCH_SECTION_CHARS = 12000 # uni/projekt section size (lost-in-the-middle guard)
|
||||
RESEARCH_RUNTIME = 900 # one research agent, one round (tail ingests live)
|
||||
SUBBLOCK_CAP = 900 # subblock find loop per chunk (seconds)
|
||||
SUBBLOCK_MIN = 5 # below this consensus count → focused catch-up rounds
|
||||
SUBBLOCK_EXTRA_ROUNDS = 2 # max catch-up rounds
|
||||
SUBBLOCK_MAX_ROUNDS = 3 # hard round cap (rounds 4–5 burned 29 % of finders for ~0 gain)
|
||||
CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (fallback path)
|
||||
DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair
|
||||
DEDUP_PAIRS_CHUNK = 40 # pairs per judge package
|
||||
DEDUP_TITLE_AUTO = 0.95 # near-identical TITLE cosine → merge without judge
|
||||
DEDUP_GLOBAL_FLOOR = 0.65 # global post-naming dedup candidate floor
|
||||
FILTER_CHUNK = 35 # blocks per judge in the degrade pass
|
||||
QUESTION_CHUNK_SUBS = 25 # target relevant subs per question chunk (LPT)
|
||||
QUESTION_MAX_ROUNDS = 3 # catch-up rounds for subs without a pattern
|
||||
FACTS_CHUNK_SUBS = 10 # facts extraction chunk (chunk count = parallelism)
|
||||
ARTEFACT_CHUNK_SUBS = 25 # flashcards/examples bulk chunk
|
||||
FACTS_CHECK_PANEL = 3 # judges per facts-check chunk (majority)
|
||||
CONSOLIDATION_PANEL = 3 # mapping judges per chunk
|
||||
SUBBLOCK_PANEL = 3 # judges in the subblock clarification
|
||||
FILTER_RECHECK_PANEL = 3 # judges in the survivor-recheck
|
||||
MAX_WRITER_ROUNDS = 2 # guide coverage→writer loop cap
|
||||
GATE_FIX_MIN = 3 # fact-gate: unbelegt-claims below this → log only (falsch fixt immer)
|
||||
WRITER_SPLIT_SUBS = 30 # guide writer splits sections above this sub count
|
||||
KANBAN_BATCH = 5 # cards a worker pulls per micro-batch
|
||||
MAX_CARD_RETRIES = 3 # failures per card → dead-letter
|
||||
RETRY_BACKOFF = 30.0 # base seconds; backoff = base · 2^(retries-1)
|
||||
MAX_RESTARTS = 2 # agent restart cap per race slot
|
||||
JUDGE_CHUNK = 40 # repair: findings per judge call
|
||||
EVIDENCE_PER_BLOCK = 6000 # repair: excerpt chars per fremd candidate
|
||||
|
||||
# Timeouts per agent step: (base seconds, seconds per block/section).
|
||||
# Applies equally to all providers — whoever is too slow gets restarted or overtaken.
|
||||
TIMEOUTS = {
|
||||
@@ -160,17 +207,18 @@ TIMEOUTS = {
|
||||
"plan": (300, 5),
|
||||
"plan_judge": (600, 5), # judge reads up to 5 outlines, n = sections
|
||||
"content": (450, 30), # facts find/erg/fix — p95 measured 241 s (was 600+90n)
|
||||
"content_check": (300, 10), # content exam per block in the package
|
||||
# Judge caps tightened 2026-07-04: judge p50 is 6–72 s; a stalled call burns the whole
|
||||
# cap and its retry heals in seconds — the old 300 s base tripled the stall cost.
|
||||
"content_check": (150, 8), # content exam per block in the package
|
||||
"subblock": (400, 15), # finder round — p95 measured 124 s (was 900+45n)
|
||||
"subblock_check": (300, 15), # judge decides contested subblocks in the chunk
|
||||
"konsolidierung": (600, 25), # consolidation judge sees ALL subs with key points — 585 s
|
||||
# (subblock_check at n=19) produced 15 % timeouts
|
||||
"subblock_check": (150, 10), # judge decides contested subblocks in the chunk
|
||||
"konsolidierung": (300, 20), # consolidation judge sees ALL subs with key points
|
||||
"level": (300, 10), # classify subblocks per chunk
|
||||
"level_check": (300, 10), # judge decides contested levels in the chunk
|
||||
"level_check": (150, 8), # judge decides contested levels in the chunk
|
||||
"relevance": (300, 10), # subblocks relevant/peripheral per chunk
|
||||
"relevance_check": (300, 10), # judge decides contested relevance in the chunk
|
||||
"relevance_check": (150, 8), # judge decides contested relevance in the chunk
|
||||
"question_pattern": (300, 15), # question patterns per block (subblocks × types)
|
||||
"question_pattern_check": (300, 10), # critic cleans up the pattern table per block
|
||||
"question_pattern_check": (150, 8), # critic cleans up the pattern table per block
|
||||
"writer": (450, 60), # per section — split keeps sections ≤30 subs
|
||||
"lese_check": (300, 10), # per section in the package
|
||||
# guide board (per card = one block)
|
||||
@@ -209,7 +257,8 @@ PROVIDERS = {
|
||||
"cli": "opencode",
|
||||
"guide": "minimax/MiniMax-M3",
|
||||
"fast": "minimax-kalt/MiniMax-M2.7-highspeed",
|
||||
"judge": "minimax-kalt/MiniMax-M3",
|
||||
"judge": "minimax/MiniMax-M3", # native route — the kalt endpoint stalled 20 % of
|
||||
# judge calls to the timeout cap (516/2590, 2026-07-04)
|
||||
"quick": "minimax/MiniMax-M2.7-highspeed",
|
||||
"env_key": "MINIMAX_API_KEY",
|
||||
},
|
||||
@@ -246,3 +295,33 @@ def resolve_role(run_provider: str, role: str) -> tuple[str, str]:
|
||||
if not model:
|
||||
model = PROVIDERS.get(provider, {}).get(role, "")
|
||||
return provider, model
|
||||
|
||||
|
||||
# ── Trainings-Override: CREATOR_PARAMS (JSON-Dict im ENV) überschreibt gleichnamige
|
||||
# Tuning-Konstanten oben — pro Prozess-Start (der Trainer startet je Trial einen Subprozess;
|
||||
# Module binden die Werte beim Import). TIMEOUTS-Einträge via "TIMEOUT_<step>_base"/"_per".
|
||||
def _apply_param_overrides() -> None:
|
||||
raw = os.getenv("CREATOR_PARAMS")
|
||||
if not raw:
|
||||
return
|
||||
import json as _json
|
||||
try:
|
||||
overrides = _json.loads(raw)
|
||||
except ValueError:
|
||||
raise SystemExit(f"CREATOR_PARAMS ist kein gültiges JSON: {raw[:80]}")
|
||||
g = globals()
|
||||
for key, val in overrides.items():
|
||||
if key.startswith("TIMEOUT_"):
|
||||
rest = key[len("TIMEOUT_"):]
|
||||
step, _, part = rest.rpartition("_")
|
||||
if step in TIMEOUTS and part in ("base", "per"):
|
||||
base, per = TIMEOUTS[step]
|
||||
TIMEOUTS[step] = (val, per) if part == "base" else (base, val)
|
||||
continue
|
||||
raise SystemExit(f"CREATOR_PARAMS: unbekannter Timeout-Schlüssel {key}")
|
||||
if key not in g or not isinstance(g[key], (int, float)) or isinstance(g[key], bool):
|
||||
raise SystemExit(f"CREATOR_PARAMS: unbekannter/nicht-numerischer Parameter {key}")
|
||||
g[key] = type(g[key])(val)
|
||||
|
||||
|
||||
_apply_param_overrides()
|
||||
|
||||
@@ -1186,6 +1186,23 @@ async def default_subblock_levels(topic: str, block_norm: str) -> None:
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def delete_stale_consensus(topic: str, block_norm: str, keep: set[str]) -> None:
|
||||
"""Drop consensus rows of a block that are NOT in this run's sidecar (`keep`): finalize
|
||||
only upserts, so re-runs piled up orphan rows (measured: 25 subs without any board-2
|
||||
output). variant/discarded rows stay — QA reads those statuses."""
|
||||
db = await get_db()
|
||||
cursor = await db.execute(
|
||||
"SELECT sub_norm FROM subblocks WHERE topic = ? AND block_norm = ? AND status = 'consensus'",
|
||||
(topic, block_norm))
|
||||
rows = await cursor.fetchall()
|
||||
stale = [r[0] for r in rows if r[0] not in keep]
|
||||
for sn in stale:
|
||||
await db.execute("DELETE FROM subblocks WHERE topic = ? AND block_norm = ? AND sub_norm = ?",
|
||||
(topic, block_norm, sn))
|
||||
if stale:
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def set_subblock_fields(topic: str, block_norm: str, sub_norm: str, **fields) -> None:
|
||||
"""Set fields (level/relevance/status/sub_title) of a subblock row."""
|
||||
fields["updated_at"] = _now()
|
||||
|
||||
289
backend/fake_agents.py
Normal file
289
backend/fake_agents.py
Normal file
@@ -0,0 +1,289 @@
|
||||
"""Deterministischer Agenten-Ersatz für E2E-Tests: beantwortet run_agent-Aufrufe ohne LLM.
|
||||
|
||||
Eine `Welt` beschreibt das Thema (Blöcke → Subs → Facts …); `respond()` routet per
|
||||
agent_key-Muster und liefert (rc, stdout, stderr) wie ein echter Agent — files-Agenten
|
||||
schreiben die out_path-Datei aus dem Prompt, none-Agenten antworten als Text (der
|
||||
Engine-Sink parst/persistiert). Damit laufen ALLE echten Schichten (_race, Quorum,
|
||||
Retry, Panels, Producer, QA-Gate) in Sekunden.
|
||||
|
||||
Störungen sind deterministisch: `Welt.stoerungen` matcht agent_keys per Regex und
|
||||
liefert n-mal einen Fehler / kaputtes JSON / eine feste Antwort (z. B. Dissens).
|
||||
|
||||
Aktivierung: pytest-Fixture `fake_welt` (tests/conftest.py) oder ENV CREATOR_FAKE_AGENTS=1
|
||||
(echter Server, Sekunden-Smoke im Frontend).
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
_PATH_RE = re.compile(r"(/\S+\.(?:json|md))")
|
||||
_NUM_RE = re.compile(r"^\s*(\d+)[.)]\s+(.*\S)", re.MULTILINE)
|
||||
_SUBLIST_RE = re.compile(r"^- (?:\[(\w+)\] )?(.+\S)\s*$", re.MULTILINE)
|
||||
_ZIEL_RE = re.compile(r"\(([a-z]\d+)\)")
|
||||
_PAIR_RE = re.compile(r"^(\d+)\.\s*\nA: \[Block: (.*?)\] (.*?)\n", re.MULTILINE)
|
||||
|
||||
|
||||
def _norm(s: str) -> str:
|
||||
return " ".join((s or "").casefold().split())
|
||||
|
||||
|
||||
class Welt:
|
||||
"""Deterministisches Themen-Modell. bloecke: {titel: {"beschreibung": str,
|
||||
"subs": [titel]}}; optionale Regeln steuern Konsolidierung/Cross-Block."""
|
||||
|
||||
def __init__(self, bloecke: dict | None = None, *, gruppen: list | None = None,
|
||||
kataloge: list | None = None, stoerungen: list | None = None):
|
||||
self.bloecke = bloecke if bloecke is not None else standard_bloecke()
|
||||
self.gruppen = gruppen or [] # [(haupt_titel, [weitere_titel])] → In-Block-Fold
|
||||
self.kataloge = kataloge or [] # [(katalog_titel, [mitglieder_titel])]
|
||||
self.stoerungen = [dict(s, rest=int(s.get("mal", 1))) for s in (stoerungen or [])]
|
||||
self.calls: list[str] = [] # Auditspur: jeder agent_key in Reihenfolge
|
||||
|
||||
# ── Nachschlagen ────────────────────────────────────────────────────────────────
|
||||
def _alle_subs(self) -> dict[str, str]:
|
||||
"""sub_norm → sub_titel über alle Blöcke (inkl. Katalog-Titel)."""
|
||||
out = {}
|
||||
for b in self.bloecke.values():
|
||||
for s in b["subs"]:
|
||||
out[_norm(s)] = s
|
||||
for kt, _m in self.kataloge:
|
||||
out[_norm(kt)] = kt
|
||||
return out
|
||||
|
||||
def _bloecke_im_prompt(self, prompt: str) -> list[str]:
|
||||
return [t for t in self.bloecke if t in prompt]
|
||||
|
||||
def _subs_im_prompt(self, prompt: str) -> list[str]:
|
||||
gefunden = [s for s in self._alle_subs().values() if s in prompt]
|
||||
return gefunden
|
||||
|
||||
# ── Störungen ───────────────────────────────────────────────────────────────────
|
||||
def _stoerung(self, agent_key: str):
|
||||
for s in self.stoerungen:
|
||||
if s["rest"] > 0 and re.search(s["muster"], agent_key):
|
||||
s["rest"] -= 1
|
||||
return s
|
||||
return None
|
||||
|
||||
# ── Haupteinstieg ──────────────────────────────────────────────────────────────
|
||||
def respond(self, agent_key: str, prompt: str, capabilities: str) -> tuple[int, str, str]:
|
||||
self.calls.append(agent_key)
|
||||
if (s := self._stoerung(agent_key)):
|
||||
if s["modus"] == "fehler":
|
||||
return 1, "", "fake-stoerung"
|
||||
if s["modus"] == "garbage":
|
||||
return self._liefern(prompt, '{"kaputt": ')
|
||||
if s["modus"] == "antwort":
|
||||
return self._liefern(prompt, s["antwort"])
|
||||
text = self._antwort(agent_key, prompt)
|
||||
if text is None:
|
||||
return 1, "", f"fake: kein Handler für {agent_key}"
|
||||
return self._liefern(prompt, text)
|
||||
|
||||
@staticmethod
|
||||
def _liefern(prompt: str, text: str) -> tuple[int, str, str]:
|
||||
"""files-Agenten schreiben die out_path-Datei aus dem Prompt; none-Agenten
|
||||
antworten als Text. Wir tun einfach BEIDES — steht ein Pfad im Prompt, wird
|
||||
er geschrieben (dann liest das payload die Datei), und stdout trägt den Text
|
||||
(dann parst ihn der Sink). Ein Weg von beiden greift immer."""
|
||||
if (m := _PATH_RE.search(prompt)):
|
||||
p = Path(m.group(1))
|
||||
try:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(text, encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
return 0, text, ""
|
||||
|
||||
# ── Antwort-Generatoren je Key-Muster ──────────────────────────────────────────
|
||||
def _antwort(self, key: str, prompt: str) -> str | None: # noqa: C901 — bewusst ein Router
|
||||
j = json.dumps
|
||||
# Board 1 / Inventar
|
||||
if "-research-" in key:
|
||||
zeilen = []
|
||||
n = 1
|
||||
for t, b in self.bloecke.items():
|
||||
zeilen.append(f"{n}. {t} — {b['beschreibung']}")
|
||||
n += 1
|
||||
return "\n".join(zeilen)
|
||||
if "-pair-" in key:
|
||||
n = prompt.count("\nA: ") or 1
|
||||
return j({"pairs": {str(i): "ja" for i in range(1, n + 1)}})
|
||||
if "-dedup-" in key:
|
||||
n = prompt.count("\nA: ") or 1
|
||||
return j({"pairs": {str(i): "nein" for i in range(1, n + 1)}})
|
||||
if "-clarify-" in key:
|
||||
keep = [ln[2:] for ln in prompt.splitlines() if ln.startswith("- ")]
|
||||
return j({"keep": keep, "rest": []})
|
||||
if "-naming-" in key: # deckt auch naming_check (gleicher Key)
|
||||
return j({"best": 1})
|
||||
if "-filter-" in key: # auch filter-recheck
|
||||
return j({"fragments": {}, "drop": []})
|
||||
if "-gruppierung-completion-" in key:
|
||||
return j({"additions": []})
|
||||
if "-gruppierung-" in key:
|
||||
return j({"umbrellas": []})
|
||||
if "-supplement-beleg" in key or "-anker-beleg-" in key:
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevant": {k: "ja" for k in sorted(nums, key=int)} or {"1": "ja"}})
|
||||
if "-supplement" in key:
|
||||
return j({"blocks": []})
|
||||
if "-source-relevance-" in key:
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevant": {k: "ja" for k in sorted(nums, key=int)} or {"1": "ja"}})
|
||||
|
||||
# Board 2 / Artefakte
|
||||
if "-luecken-" in key:
|
||||
return "<!-- block: leer -->\n" # Lücken-Nachfass findet nichts Neues
|
||||
if "-subblock-final-" in key or "-subblock-" in key: # Finder + Judge, gleiches Format
|
||||
teile = []
|
||||
for t in self._bloecke_im_prompt(prompt):
|
||||
subs = "\n".join(f"- {s}" for s in self.bloecke[t]["subs"])
|
||||
teile.append(f"<!-- block: {t} -->\n{subs}")
|
||||
return "\n".join(teile) or "<!-- block: leer -->\n"
|
||||
if "-sub-konsolidierung-" in key:
|
||||
nummern = {_norm(m.group(2)): m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
gruppen = []
|
||||
for haupt, weitere in self.gruppen:
|
||||
h, w = nummern.get(_norm(haupt)), [nummern[_norm(x)] for x in weitere
|
||||
if _norm(x) in nummern]
|
||||
if h and w:
|
||||
gruppen.append({"haupt": int(h), "weitere": [int(x) for x in w]})
|
||||
kataloge = []
|
||||
for kt, mitglieder in self.kataloge:
|
||||
m = [int(nummern[_norm(x)]) for x in mitglieder if _norm(x) in nummern]
|
||||
if len(m) >= 2:
|
||||
kataloge.append({"titel": kt, "mitglieder": m})
|
||||
return j({"gruppen": gruppen, "kataloge": kataloge, "fremd": [], "luecken": []})
|
||||
if "-sub-crossblock-" in key:
|
||||
urteile = {}
|
||||
for m in _PAIR_RE.finditer(prompt):
|
||||
urteile[m.group(1)] = "a" # identischer Text (nur so wird gepaart) → A behält
|
||||
return j({"pairs": urteile or {"1": "nein"}})
|
||||
if "-facts-check-" in key:
|
||||
return j({"ok": True})
|
||||
if "-facts-" in key: # facts / facts-fix / facts-erg: gleiches Format
|
||||
eintraege = []
|
||||
for t in self._bloecke_im_prompt(prompt):
|
||||
for s in self.bloecke[t]["subs"]:
|
||||
if s in prompt:
|
||||
eintraege.append(self._fakt(t, s))
|
||||
for kt, _m in self.kataloge:
|
||||
if kt in prompt:
|
||||
eintraege.append(self._fakt(t, kt))
|
||||
if not eintraege: # Nachfass-Fälle: Subs ohne Blockkontext im Prompt
|
||||
eintraege = [self._fakt(bt, s) for bt, b in self.bloecke.items()
|
||||
for s in b["subs"] if s in prompt]
|
||||
return j({"facts": eintraege})
|
||||
if "-level-" in key: # Rater + final: alle geforderten Nummern
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"levels": {k: "beginner" for k in sorted(nums, key=int)} or {"1": "beginner"}})
|
||||
if "-relevance-" in key:
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevance": {k: "relevant" for k in sorted(nums, key=int)} or {"1": "relevant"}})
|
||||
if "-question-pattern-" in key:
|
||||
eintraege = []
|
||||
for t in self._bloecke_im_prompt(prompt):
|
||||
for s in self._subs_im_prompt(prompt):
|
||||
eintraege.append({"block": t, "subblock": s, "question": f"Was ist {s}?"})
|
||||
return j({"pattern": eintraege})
|
||||
if "-artifact-example-check-" in key:
|
||||
return j({"ok": True})
|
||||
if "-artifact-flashcard-" in key:
|
||||
karten = [{"block": t, "subblock": s, "question": f"F: {s}?", "answer": f"A: {s}"}
|
||||
for t in self._bloecke_im_prompt(prompt) for s in self._subs_im_prompt(prompt)]
|
||||
return j({"cards": karten})
|
||||
if "-artifact-example-" in key:
|
||||
bsp = [{"block": t, "subblock": s, "problem": f"Aufgabe zu {s}",
|
||||
"steps": ["Schritt 1", "Schritt 2"], "result": "Ergebnis"}
|
||||
for t in self._bloecke_im_prompt(prompt) for s in self._subs_im_prompt(prompt)]
|
||||
return j({"examples": bsp})
|
||||
if "-outline-prereqs" in key:
|
||||
return j({"prereqs": {}})
|
||||
if "-outline-review" in key:
|
||||
return j({"moves": {}})
|
||||
if "-outline-" in key or key.endswith("-outline-judge"):
|
||||
nums = sorted({int(m.group(1)) for m in _NUM_RE.finditer(prompt)}) or [1]
|
||||
return j({"chapters": [{"title": "Kapitel 1", "numbers": nums}]})
|
||||
|
||||
# Guide-Board
|
||||
if "-ziele-" in key:
|
||||
ziele = [{"id": f"z{i}", "text": f"Verstehen von {s}", "sub": s}
|
||||
for i, s in enumerate(self._subs_im_prompt(prompt), 1)][:8]
|
||||
return j({"ziele": ziele or [{"id": "z1", "text": "Grundlagen verstehen", "sub": ""}]})
|
||||
if "-gatefix-" in key or "-lesefix-" in key:
|
||||
return self._section_aus_prompt(prompt) or "<!-- section: X -->\nRepariert."
|
||||
if "-gate-" in key:
|
||||
return j({"ok": True})
|
||||
if "-cov-" in key:
|
||||
ids = sorted(set(_ZIEL_RE.findall(prompt)))
|
||||
return j({"ziele": {z: True for z in ids}, "luecken": [], "ballast": []})
|
||||
if "-lese-" in key:
|
||||
return j({"ok": True})
|
||||
if "-w-" in key:
|
||||
return self._writer_md(prompt)
|
||||
|
||||
# QA / Repair / Guide-QA (alle Text, _yesno_schema)
|
||||
if key.startswith(("qa-guide-",)):
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevant": {k: "nein" for k in sorted(nums, key=int)} or {"1": "nein"}})
|
||||
if key.startswith(("qa-", "repair-")):
|
||||
# Semantik je Template: Bausteine „ja" = echt; Dubletten/Lücken „nein" = kein Befund
|
||||
wert = "ja" if ("bausteine" in key or "beleg" in key or "fremd" in key) else "nein"
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevant": {k: wert for k in sorted(nums, key=int)} or {"1": wert}})
|
||||
return None
|
||||
|
||||
# ── Bausteine der Antworten ─────────────────────────────────────────────────────
|
||||
@staticmethod
|
||||
def _fakt(block: str, sub: str) -> dict:
|
||||
return {"block": block, "subblock": sub,
|
||||
"key_points": [f"Kernaussage zu {sub}", f"Zweite Aussage zu {sub}"],
|
||||
"prerequisites": "", "hurdles": "",
|
||||
"cited_facts": [{"text": f"Beleg für {sub}", "source": "Fake-Quelle"}],
|
||||
"example_idea": f"Beispiel zu {sub}"}
|
||||
|
||||
def _writer_md(self, prompt: str) -> str:
|
||||
"""Section im Marker-Format; Subs aus der SUBBLOCKS-Liste des Prompts
|
||||
(`- [label] titel`), Länge je Sub im 150–1080-Rahmen."""
|
||||
block = next(iter(self._bloecke_im_prompt(prompt)), None) or "Abschnitt"
|
||||
subs = [(lv or "beginner", t) for lv, t in _SUBLIST_RE.findall(prompt)
|
||||
if _norm(t) in self._alle_subs()]
|
||||
if not subs:
|
||||
subs = [("beginner", s) for s in self.bloecke.get(block, {}).get("subs", ["Inhalt"])]
|
||||
kompakt = "\n".join(f"<!-- sub: {lv} | {t} -->\n- Merksatz zu {t}" for lv, t in subs)
|
||||
prosa = "\n".join(f"<!-- sub: {lv} | {t} -->\n" + (f"Lehrtext über {t}. " * 12)
|
||||
for lv, t in subs)
|
||||
return (f"<!-- kapitel: Kapitel 1 -->\n<!-- section: {block} -->\n"
|
||||
f"<!-- compact -->\n{kompakt}\n<!-- ausführlich -->\n"
|
||||
f"Einstieg in {block}.\n{prosa}")
|
||||
|
||||
@staticmethod
|
||||
def _section_aus_prompt(prompt: str) -> str | None:
|
||||
"""Fix-Agenten geben die Section unverändert zurück (minimal-invasiv)."""
|
||||
m = re.search(r"(<!--\s*(?:kapitel|section):.*)", prompt, re.DOTALL)
|
||||
return m.group(1).strip() if m else None
|
||||
|
||||
|
||||
def standard_bloecke() -> dict:
|
||||
"""3 Blöcke; „Gemeinsamer Grundbegriff" liegt in Alpha UND Beta (Cross-Block-Fall)."""
|
||||
return {
|
||||
"Alpha-Konzept": {"beschreibung": "Das erste Grundkonzept",
|
||||
"subs": ["Definition Alpha", "Alpha Eigenschaften",
|
||||
"Gemeinsamer Grundbegriff"]},
|
||||
"Beta-Verfahren": {"beschreibung": "Das zentrale Verfahren",
|
||||
"subs": ["Beta Ablauf", "Beta Grenzen", "Gemeinsamer Grundbegriff"]},
|
||||
"Gamma-Anwendung": {"beschreibung": "Praktische Anwendung",
|
||||
"subs": ["Gamma Praxisfall", "Gamma Werkzeuge"]},
|
||||
}
|
||||
|
||||
|
||||
_WELT: Welt | None = None # ENV-Modus (CREATOR_FAKE_AGENTS=1): eine Welt pro Prozess
|
||||
|
||||
|
||||
async def respond(agent_key: str, prompt: str, capabilities: str) -> tuple[int, str, str]:
|
||||
global _WELT
|
||||
if _WELT is None:
|
||||
_WELT = Welt()
|
||||
return _WELT.respond(agent_key, prompt, capabilities)
|
||||
@@ -21,7 +21,8 @@ import re
|
||||
import database as db
|
||||
import readability
|
||||
from blocks import _sink_json
|
||||
from config import FORMAT_PURPOSE, READABILITY_ACTIVE, TEMPLATES_DIR, MAX_CONCURRENT_AGENTS_PER_TOPIC
|
||||
from config import (FORMAT_PURPOSE, GUIDE_LAENGE_MAX, GUIDE_LAENGE_MIN, READABILITY_ACTIVE,
|
||||
TEMPLATES_DIR, MAX_CONCURRENT_AGENTS_PER_TOPIC)
|
||||
from fsutil import atomic_write_json
|
||||
from jsonio import read_json_file as _json_file
|
||||
from pipeline import (CANCELLED, FAILED, OK, GenContext, _extra, _log, _prompt,
|
||||
@@ -34,9 +35,7 @@ GUIDE_STAGES = ("lernziele", "zuweisung", "writer", "fakten_gate", "coverage", "
|
||||
STAGE_LABELS = {"lernziele": "Lernziele", "zuweisung": "Zuweisung", "writer": "Writer",
|
||||
"fakten_gate": "Fakten-Gate", "coverage": "Coverage",
|
||||
"lesbarkeit": "Lesbarkeit", "done": "Fertig"}
|
||||
MAX_WRITER_ROUNDS = 2 # coverage → writer feedback loop cap (gains die after round 1–2)
|
||||
GATE_FIX_MIN = 3 # fact-gate claims below this: log only — a full fix rewrite for
|
||||
# 1–2 residual claims fired on 19/20 blocks (40 agent-minutes)
|
||||
from config import GATE_FIX_MIN, MAX_WRITER_ROUNDS, WRITER_SPLIT_SUBS # zentral tunebar
|
||||
# Simultaneous cards = the per-topic agent cap: every card busies exactly ONE agent at a
|
||||
# time (its stages run serially), so a lower number just idles slots (was hardcoded 10
|
||||
# from the old 10-slot era while the .env already allowed 24).
|
||||
@@ -65,7 +64,10 @@ def _ziele_schema(data):
|
||||
|
||||
|
||||
def _gate_schema(data):
|
||||
"""{"ok":true} → [] · {"claims":[{text,grund}]} → list · None invalid."""
|
||||
"""{"ok":true} → [] · {"claims":[{text,grund,urteil}]} → list · None invalid.
|
||||
urteil "falsch" (contradicts the facts/itself) vs "unbelegt" (true but underivable) —
|
||||
default unbelegt. Entries whose grund starts with "belegt" are dropped: one judge
|
||||
returned a 65-entry full inventory including SUPPORTED claims."""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if data.get("ok") is True:
|
||||
@@ -76,7 +78,12 @@ def _gate_schema(data):
|
||||
out = []
|
||||
for c in claims:
|
||||
if isinstance(c, dict) and str(c.get("text", "")).strip():
|
||||
out.append({"text": str(c["text"]).strip(), "grund": str(c.get("grund", "")).strip()})
|
||||
grund = str(c.get("grund", "")).strip()
|
||||
if grund.casefold().startswith("belegt"):
|
||||
continue
|
||||
urteil = str(c.get("urteil", "")).strip().casefold()
|
||||
out.append({"text": str(c["text"]).strip(), "grund": grund,
|
||||
"urteil": urteil if urteil == "falsch" else "unbelegt"})
|
||||
return out
|
||||
|
||||
|
||||
@@ -241,7 +248,7 @@ async def _stage_zuweisung(env: _Env, card: dict) -> bool:
|
||||
# A single section over ~45 subs measurably breaks the writer/coverage (Front Matter:
|
||||
# 4/6 objectives open after 2 rounds). First drafts of oversized cards are written in two
|
||||
# halves and merged back into ONE canonical section (all gates/assembly read one section).
|
||||
WRITER_SPLIT_SUBS = 30
|
||||
# WRITER_SPLIT_SUBS: siehe config.py
|
||||
|
||||
|
||||
def _merge_split_sections(sec_a: dict, sec_b: dict) -> str:
|
||||
@@ -397,13 +404,15 @@ async def _stage_fakten_gate(env: _Env, card: dict) -> bool:
|
||||
if status == FAILED:
|
||||
claims = [] # gate failure must not block the card — logged, text stands
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: kein Ergebnis — Text bleibt ungeprüft")
|
||||
if claims and len(claims) < GATE_FIX_MIN:
|
||||
# 1–2 Rest-Claims rechtfertigen keinen Voll-Rewrite: der Fix-Pass lief für 19/20
|
||||
# Blöcke und kostete 40 Agent-Minuten; die Guide-QA-Fachlichkeits-Stichprobe misst nach
|
||||
falsch = [c for c in claims if c.get("urteil") == "falsch"] if claims else []
|
||||
if claims and not falsch and len(claims) < GATE_FIX_MIN:
|
||||
# 1–2 merely UNSUPPORTED claims don't justify a fix pass (it ran for 19/20 blocks,
|
||||
# 40 agent-minutes) — but a WRONG claim always does: one slipped through this
|
||||
# threshold and cost the guide 1.5 QA points
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claim(s) unter Schwelle — kein Fix")
|
||||
claims = []
|
||||
if claims:
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} unbelegte Claims → Fix")
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claims ({len(falsch)} falsch) → Fix")
|
||||
fixp = env.slot(f"gatefix-{_safe(norm)}-r{card['writer_rounds']}.md")
|
||||
fixp.unlink(missing_ok=True)
|
||||
claims_text = "\n".join(f"- {c['text']}" + (f" ({c['grund']})" if c["grund"] else "")
|
||||
@@ -501,6 +510,18 @@ async def _stage_lesbarkeit(env: _Env, card: dict) -> bool:
|
||||
hints = await asyncio.to_thread(readability.rate_sections, {1: sec["md"]})
|
||||
if hints.get(1):
|
||||
problems.append(hints[1])
|
||||
# deterministic length trigger, same formula as the QA detector: prompt guidelines
|
||||
# alone left writers 2.7–4.1× over target — a measured overshoot forces the fix pass
|
||||
subs_all = env.subs_by_title.get(card["block"], [])
|
||||
n_rel = max(sum(1 for s in subs_all if s.get("relevance") != "peripheral"), 1)
|
||||
aus = re.split(r"<!--\s*ausführlich\s*-->", sec["md"], maxsplit=1)
|
||||
pro_sub = len(aus[1] if len(aus) == 2 else sec["md"]) / n_rel
|
||||
if not (GUIDE_LAENGE_MIN <= pro_sub <= GUIDE_LAENGE_MAX * 0.9):
|
||||
ziel = _writer_budget(len(subs_all))
|
||||
problems.append(
|
||||
f"Länge {round(pro_sub)} Zeichen/Sub (Rahmen {GUIDE_LAENGE_MIN}–{round(GUIDE_LAENGE_MAX * 0.9)}): "
|
||||
f"schreibe den ausführlich-Teil auf etwa {ziel} Zeichen GESAMT um — Sockel-Prosa und "
|
||||
f"Wiederholungen streichen, alle Sub-Marker und Beispiele behalten")
|
||||
if problems:
|
||||
from guide import _level_label
|
||||
subs = env.subs_by_title.get(card["block"], [])
|
||||
|
||||
@@ -20,8 +20,8 @@ import readability
|
||||
from fsutil import atomic_write_json
|
||||
from textkit import _norm_title
|
||||
|
||||
LAENGE_MIN = 150 # Zeichen je relevantem Sub im ausführlich-Teil (Untergrenze)
|
||||
LAENGE_MAX = 1200 # Obergrenze — außerhalb = Tiefen-Lotterie statt Zerlegungs-Signal
|
||||
from config import GUIDE_LAENGE_MAX as LAENGE_MAX, GUIDE_LAENGE_MIN as LAENGE_MIN
|
||||
|
||||
JACCARD_ABSATZ = 0.6 # Wort-Jaccard, ab dem zwei Absätze als Doppel gelten
|
||||
ABSATZ_MIN_CHARS = 200 # kürzere Absätze sind Übergänge — kein Dubletten-Signal
|
||||
LLM_SECTION_CHARS = 2500 # Section-Auszug je Judge-Item
|
||||
|
||||
@@ -18,18 +18,15 @@ import asyncio
|
||||
import logging
|
||||
|
||||
import database as db
|
||||
from config import MAX_CONCURRENT_AGENTS_PER_TOPIC
|
||||
from config import KANBAN_BATCH, MAX_CARD_RETRIES, MAX_CONCURRENT_AGENTS_PER_TOPIC, RETRY_BACKOFF
|
||||
|
||||
log = logging.getLogger("creator.kanban")
|
||||
|
||||
KANBAN_BATCH = 5 # cards a worker pulls per package (micro-batching)
|
||||
# How many packages ONE worker keeps in flight at once. A worker no longer blocks on a single
|
||||
# package — it keeps pulling and dispatching until this many run concurrently, so a busy column
|
||||
# fills the agent slots (the per-topic semaphore is the real cap; over-dispatch just queues cheaply).
|
||||
WORKER_INFLIGHT = MAX_CONCURRENT_AGENTS_PER_TOPIC
|
||||
_POLL = 0.3 # seconds between empty-queue polls
|
||||
MAX_CARD_RETRIES = 3 # failures per card until dead-letter
|
||||
RETRY_BACKOFF = 30.0 # base seconds; backoff = base · 2^(retries-1)
|
||||
|
||||
# Live registry of running flows (topic → Flow), so routes can attach research agents,
|
||||
# report `generating`, and cancel.
|
||||
|
||||
@@ -164,7 +164,7 @@ _relevance_schema = _enum_map_schema("relevance", _RELEVANCE) # relevance ∈
|
||||
_yesno_schema = _enum_map_schema("relevant", _YESNO) # triage gate ∈ ja/nein
|
||||
|
||||
|
||||
_MAX_RESTARTS = 2
|
||||
from config import MAX_RESTARTS as _MAX_RESTARTS # noqa: E402 — zentral tunebar
|
||||
|
||||
|
||||
async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: int, provider: str, on_update=None, cancelled=None, *, grace: int | None = None, min_runtime: int | None = None, max_runtime: int | None = None) -> list | None:
|
||||
|
||||
@@ -336,12 +336,20 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
p["llm"] = v.get(k, "?")
|
||||
unecht: list[str] | None = None
|
||||
if llm and blocks:
|
||||
unecht = []
|
||||
verdacht = []
|
||||
for lo in range(0, len(blocks), 80): # ein Call je 80 Titel
|
||||
chunk = blocks[lo:lo + 80]
|
||||
v = await _llm_verdicts("QA-Bausteine", topic, f"bausteine-{lo}",
|
||||
[f"{b['title']} — {b['description'] or '(ohne Beschreibung)'}" for b in chunk])
|
||||
unecht += [b["title"] for k, b in enumerate(chunk, 1) if v.get(k) == "nein"]
|
||||
verdacht += [b for k, b in enumerate(chunk, 1) if v.get(k) == "nein"]
|
||||
# Bestätiger-Pass nur über die Geflaggten: der Einzel-Judge flaggte pro Lauf ANDERE
|
||||
# Blöcke (gemessen aak: Note pendelte 9.3↔10.0 bei identischem Bestand) — nur
|
||||
# doppelt-„nein" zählt; Repair hat als dritte Sicherung die eigene Zweitmeinung
|
||||
unecht = []
|
||||
if verdacht:
|
||||
v2 = await _llm_verdicts("QA-Bausteine", topic, "bausteine-b2",
|
||||
[f"{b['title']} — {b['description'] or '(ohne Beschreibung)'}" for b in verdacht])
|
||||
unecht = [b["title"] for k, b in enumerate(verdacht, 1) if v2.get(k) == "nein"]
|
||||
|
||||
art_rows = [dict(r) for r in await db.get_sub_artefakte(topic)]
|
||||
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
|
||||
|
||||
@@ -23,8 +23,7 @@ from textkit import _norm_title, _title, clean_title
|
||||
log = logging.getLogger("creator.repair")
|
||||
|
||||
JUDGE_TIMEOUT = 600
|
||||
JUDGE_CHUNK = 40 # Befunde je Judge-Call
|
||||
EVIDENCE_PER_BLOCK = 6000 # Zeichen Material-Auszug je Fremd-Kandidat
|
||||
from config import EVIDENCE_PER_BLOCK, JUDGE_CHUNK # noqa: E402 — zentral tunebar
|
||||
|
||||
|
||||
async def repair_befunde(topic: str) -> dict:
|
||||
|
||||
@@ -95,6 +95,8 @@ async def remove_topic(topic: str):
|
||||
await delete_source(topic) # topic config (DB) — removed together with the topic
|
||||
await delete_guide_content(topic)
|
||||
shutil.rmtree(topic_dir(topic), ignore_errors=True)
|
||||
import qa
|
||||
shutil.rmtree(qa.QA_DIR / topic, ignore_errors=True) # QA reports belong to the topic
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
|
||||
@@ -16,3 +16,69 @@ async def testdb(tmp_path, monkeypatch):
|
||||
await database.init_db()
|
||||
yield database
|
||||
await database.close_db()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fake_welt(testdb, tmp_path, monkeypatch):
|
||||
"""E2E ohne LLM: run_agent überall durch die Fake-Welt ersetzt, Tempo-Bremsen raus.
|
||||
Alle echten Schichten (_race, Quorum, Panels, Producer, QA-Gate) laufen mit."""
|
||||
import agents
|
||||
import blocks
|
||||
import board_inventory as bi
|
||||
import guide
|
||||
import kanban
|
||||
import pipeline
|
||||
import qa
|
||||
import repair
|
||||
from fake_agents import Welt
|
||||
|
||||
welt = Welt()
|
||||
|
||||
async def fake_run_agent(agent_key, prompt, timeout, provider="claude", role="fast",
|
||||
capabilities="none", lane="batch", scope=None, on_line=None, label=""):
|
||||
return welt.respond(agent_key, prompt, capabilities)
|
||||
|
||||
for mod in (agents, pipeline, blocks, guide, repair):
|
||||
monkeypatch.setattr(mod, "run_agent", fake_run_agent)
|
||||
|
||||
# Tempo: grace/poll/backoff bremsen echte Läufe, nicht den Fake
|
||||
monkeypatch.setattr(blocks, "CONSENSUS_GRACE", 0)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
monkeypatch.setattr(kanban, "RETRY_BACKOFF", 0.05)
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
|
||||
import guide_board
|
||||
monkeypatch.setattr(guide_board, "READABILITY_ACTIVE", False) # kein Modell-Load im Test
|
||||
import asyncio as _aio
|
||||
monkeypatch.setattr(bi, "_ingest_lock", _aio.Lock()) # Modul-Lock klebt sonst am Vortest-Loop
|
||||
|
||||
class _FakeEmb: # identischer Text → cos 1.0, sonst 0.0 (deterministisch, ohne Modell)
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def embed_sims(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr @ arr.T
|
||||
|
||||
@staticmethod
|
||||
def embed(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr
|
||||
|
||||
import board_artefacts as ba
|
||||
for mod in (blocks, ba, qa):
|
||||
monkeypatch.setattr(mod, "embedding", _FakeEmb)
|
||||
|
||||
async def emb_ok(flow): # Board-1-Vektorpfade aus (wie board_env) — Judge-Wellen reichen
|
||||
return False
|
||||
monkeypatch.setattr(bi, "_emb_ok", emb_ok)
|
||||
return welt
|
||||
|
||||
96
backend/tests/invarianten.py
Normal file
96
backend/tests/invarianten.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""Invarianten nach einem (Fake-)E2E-Lauf: was IMMER gelten muss, egal welches Szenario.
|
||||
|
||||
Nutzt bewusst eigene, schlichte Prüfungen statt Pipeline-Heuristiken (Muster qa.py) —
|
||||
geteilte blinde Flecken machen den Check wertlos. Rückgabe: Liste von Verstößen,
|
||||
leer = alles konsistent.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import database as db
|
||||
from textkit import _norm_title
|
||||
|
||||
_LEVELS_OK = {"beginner", "advanced", "expert"}
|
||||
_RELEVANZ_OK = {"relevant", "peripheral"}
|
||||
|
||||
|
||||
def _json(path):
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
async def pruefe_invarianten(topic: str, files: dict | None = None,
|
||||
mit_artefakten: bool = True) -> list[str]:
|
||||
fehler: list[str] = []
|
||||
subs = [dict(r) for r in await db.list_subblocks(topic)]
|
||||
cons = [r for r in subs if r["status"] == "consensus"]
|
||||
|
||||
for r in cons:
|
||||
wo = f"{r['block']}/{r['sub_title']}"
|
||||
fk = None
|
||||
try:
|
||||
fk = json.loads(r["facts"]) if r["facts"] else None
|
||||
except ValueError:
|
||||
fehler.append(f"facts unparsebar: {wo}")
|
||||
if not (isinstance(fk, dict) and (fk.get("key_points") or fk.get("cited_facts"))):
|
||||
fehler.append(f"consensus-Sub ohne facts: {wo}")
|
||||
if r["level"] not in _LEVELS_OK:
|
||||
fehler.append(f"consensus-Sub ohne gültiges level: {wo}")
|
||||
if r["relevance"] not in _RELEVANZ_OK:
|
||||
fehler.append(f"consensus-Sub ohne relevance: {wo}")
|
||||
|
||||
if mit_artefakten:
|
||||
art = [dict(r) for r in await db.get_sub_artefakte(topic)]
|
||||
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
|
||||
versorgt = {(r["block_norm"], r["sub_norm"]) for r in art}
|
||||
versorgt |= {(r["block_norm"], r["sub_norm"]) for r in fragen}
|
||||
lebend = {(r["block_norm"], r["sub_norm"]) for r in subs if r["status"] in ("consensus", "variant")}
|
||||
for r in cons:
|
||||
if r["relevance"] == "relevant" and (r["block_norm"], r["sub_norm"]) not in versorgt:
|
||||
fehler.append(f"relevanter Sub ohne Frage/Artefakt: {r['block']}/{r['sub_title']}")
|
||||
for bn, sn in sorted({(r["block_norm"], r["sub_norm"]) for r in art} |
|
||||
{(r["block_norm"], r["sub_norm"]) for r in fragen}):
|
||||
if (bn, sn) not in lebend:
|
||||
fehler.append(f"Waise (Ziel-Sub existiert nicht): {bn}/{sn}")
|
||||
|
||||
# keine hängengebliebenen Karten
|
||||
for c in await db.kanban_cards(topic):
|
||||
if c["stage"] == "dead":
|
||||
fehler.append(f"dead-Karte: {c['board']}/{c['card_id']}")
|
||||
|
||||
if files is not None:
|
||||
if not files["final"].exists():
|
||||
fehler.append("blocks.md fehlt")
|
||||
sc = _json(files["sidecar"])
|
||||
if not isinstance(sc, dict):
|
||||
fehler.append("sidecar-Datei fehlt/unparsebar")
|
||||
else: # Sidecar und DB-consensus müssen dieselbe Sub-Menge tragen
|
||||
db_menge = {(r["block_norm"], r["sub_norm"]) for r in cons}
|
||||
sc_menge = {(_norm_title(bt), _norm_title(str(s.get("title", ""))))
|
||||
for bt, ss in sc.items() for s in ss if isinstance(s, dict)}
|
||||
for extra in sorted(sc_menge - db_menge):
|
||||
fehler.append(f"Sidecar-Sub fehlt in DB: {extra}")
|
||||
for extra in sorted(db_menge - sc_menge):
|
||||
fehler.append(f"DB-consensus fehlt im Sidecar: {extra}")
|
||||
return fehler
|
||||
|
||||
|
||||
async def pruefe_guide_invarianten(topic: str, format_name: str = "Guide") -> list[str]:
|
||||
"""Jeder relevante consensus-Sub trägt einen Sub-Marker im Guide (Muster
|
||||
guide_qa.marker_fehlend, ohne LLM)."""
|
||||
import guide_qa
|
||||
fehler: list[str] = []
|
||||
cards = [dict(r) for r in await db.list_guide_cards(topic, format_name)]
|
||||
if not cards:
|
||||
return ["keine Guide-Karten"]
|
||||
for c in cards:
|
||||
if c["status"] != "ok" or not (c.get("md") or "").strip():
|
||||
fehler.append(f"Guide-Karte nicht ok: {c['block']} ({c['status']})")
|
||||
subs_rel: dict[str, set] = {}
|
||||
for r in await db.list_subblocks(topic):
|
||||
if r["status"] == "consensus" and r["relevance"] != "peripheral":
|
||||
subs_rel.setdefault(r["block_norm"], set()).add(r["sub_norm"])
|
||||
fehler += [f"Sub-Marker fehlt: {m}" for m in guide_qa.marker_fehlend(cards, subs_rel)]
|
||||
return fehler
|
||||
@@ -1012,12 +1012,13 @@ async def test_supplement_material_mode_for_source_topics(board_env, tmp_path, m
|
||||
|
||||
# ── Anker-Gate: Quorum-Titel ohne Korpus-Beleg (Reader-Ko-Halluzination) ────────────
|
||||
|
||||
async def _anker_env(db, tmp_path, monkeypatch, titel_map):
|
||||
async def _anker_env(db, tmp_path, monkeypatch, titel_map, desc_map=None):
|
||||
(tmp_path / "korpus.txt").write_text("Der Graph ist zusammenhängend und endlich.", encoding="utf-8")
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: tmp_path)
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"title": titel_map[cid], "description": "", "readers": ["r1", "r2"], "supplement": False}]
|
||||
return [{"title": titel_map[cid], "description": (desc_map or {}).get(cid, ""),
|
||||
"readers": ["r1", "r2"], "supplement": False}]
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "_rep", lambda rows: rows[0])
|
||||
@@ -1028,11 +1029,12 @@ async def _anker_env(db, tmp_path, monkeypatch, titel_map):
|
||||
|
||||
|
||||
async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
|
||||
"""Titel ohne Korpus-Anker → Beleg-Judge; „nein" → rejected/kein-beleg.
|
||||
"""Titel ohne Korpus-Anker, aber mit Evidenz → Beleg-Judge; „nein" → rejected/kein-beleg.
|
||||
Titel MIT Anker geht ohne Judge nach naming."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch,
|
||||
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"})
|
||||
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"},
|
||||
{"c2": "Der Graph ist endlich."}) # Evidenz da → Judge entscheidet
|
||||
|
||||
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "Brückenproblem" in prompt and "Zusammenhang" not in prompt # nur der Anker-lose
|
||||
@@ -1045,11 +1047,27 @@ async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
|
||||
assert c2["stage"] == "rejected" and c2["payload"]["reason"] == "kein-beleg"
|
||||
|
||||
|
||||
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Judge-Ausfall → Titel bleibt (2-Reader-Rückhalt)."""
|
||||
async def test_anker_gate_leeres_pack_hart_nein(testdb, tmp_path, monkeypatch):
|
||||
"""KEIN distinktives Token im Korpus → deterministisch rejected, Judge läuft NICHT
|
||||
(der Judge winkte 3 Kanon-Titel auf Schein-Auszügen durch)."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"})
|
||||
|
||||
async def never_slot(*a, **kw):
|
||||
raise AssertionError("Judge darf bei leerem Evidence-Pack nicht laufen")
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", never_slot)
|
||||
await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), cards)
|
||||
c9 = await db.kanban_get_card(TOPIC, B, "c9")
|
||||
assert c9["stage"] == "rejected" and c9["payload"]["reason"] == "kein-beleg"
|
||||
|
||||
|
||||
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Judge-Ausfall bei VORHANDENER Evidenz → Titel bleibt (2-Reader-Rückhalt)."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"},
|
||||
{"c9": "Der Graph ist endlich."})
|
||||
|
||||
async def broken_slot(*a, **kw):
|
||||
return "failed", None
|
||||
|
||||
@@ -1063,3 +1081,103 @@ def test_hat_anker_ziffern_suffix():
|
||||
assert bi._hat_anker("ΔTSP1-Algorithmus", ctoks) # tsp1 → tsp
|
||||
assert not bi._hat_anker("Königsberger Brückenproblem", ctoks)
|
||||
assert not bi._hat_anker("Algorithmus Verfahren", ctoks) # nur Stopwörter → kein Anker
|
||||
|
||||
|
||||
async def test_reset_subblocks_loescht_globale_dateien(testdb, tmp_path):
|
||||
"""Reset auf Spalte subblocks: DB-Spiegel UND globale Sidecar-Dateien + ab-*-Resume-Slots
|
||||
weg — Reste des Vor-Laufs würden sonst in den frischen Lauf zurückmergen."""
|
||||
db = testdb
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "artefacts", {"title": "Alpha"})
|
||||
await db.put_subblock(TOPIC, "alpha", "s1", "Alpha", "S1", status="consensus")
|
||||
arbeit = tmp_path / "arbeit"
|
||||
(arbeit / "ab-alpha").mkdir(parents=True)
|
||||
(arbeit / "ab-alpha" / "facts.json").write_text("{}", encoding="utf-8")
|
||||
files = {"arbeit": arbeit}
|
||||
for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"):
|
||||
files[k] = tmp_path / f"{k}.json"
|
||||
files[k].write_text("{}", encoding="utf-8")
|
||||
moved = await bi.reset_board_from_stage(TOPIC, "artefacts", "subblocks", files)
|
||||
assert moved == 1
|
||||
assert not await db.list_subblocks(TOPIC)
|
||||
assert not (arbeit / "ab-alpha").exists()
|
||||
assert all(not files[k].exists() for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"))
|
||||
|
||||
|
||||
# ── Naming-Abstraktion: freier Name nur mit Anker ───────────────────────────────────
|
||||
|
||||
def test_naming_schema_varianten():
|
||||
assert bi._naming_schema({"best": 2}, 3) == (2, None, False)
|
||||
assert bi._naming_schema({"best": 1, "name": "Kurzer Titel"}, 3) == (1, "Kurzer Titel", False)
|
||||
assert bi._naming_schema({"ok": True}, 3) == (None, None, True)
|
||||
assert bi._naming_schema({"best": 9}, 3) is None
|
||||
assert bi._naming_schema({"best": 1, "name": "x" * 90}, 3) == (1, None, False) # zu lang
|
||||
|
||||
|
||||
def test_name_verankert_thema_subset():
|
||||
rows = [{"title": "Bubblesort-Schleife und Tauschoperation", "description": "innere Schleife"},
|
||||
{"title": "Bubblesort Durchläufe", "description": ""}]
|
||||
assert bi._name_verankert("Bubblesort Tauschoperation", rows, None)
|
||||
assert not bi._name_verankert("Königsberger Brückenproblem", rows, None) # fremde Begriffe
|
||||
assert not bi._name_verankert("und der", rows, None) # nur Stopwörter → kein Anker
|
||||
|
||||
|
||||
def test_name_verankert_korpus():
|
||||
ctoks = {"partition", "problem", "vollständigkeit"}
|
||||
rows = [{"title": "irrelevant", "description": ""}]
|
||||
assert bi._name_verankert("Partition-Problem", rows, ctoks)
|
||||
assert not bi._name_verankert("Rucksackproblem Optimierung", rows, ctoks)
|
||||
|
||||
|
||||
async def test_naming_vergibt_verankerten_namen(testdb, tmp_path, monkeypatch):
|
||||
"""Judge liefert best+name; verankerter Name gewinnt, unverankerter fällt auf Member zurück."""
|
||||
db = testdb
|
||||
antwort = {"val": {"best": 1, "name": "Bubblesort Grundprinzip"}}
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"norm": "bubblesort - grundprinzip und ablauf (kap. 2)",
|
||||
"title": "Bubblesort - Grundprinzip und Ablauf (Kap. 2)",
|
||||
"description": "Sortieren durch Tauschen", "readers": ["r1"], "sources": []},
|
||||
{"norm": "sortieren durch tauschen", "title": "Sortieren durch Tauschen",
|
||||
"description": "", "readers": ["r2"], "sources": []}]
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
import json as _json
|
||||
return "ok", payload((0, _json.dumps(antwort["val"]), ""))
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
await db.kanban_upsert_card(TOPIC, B, "c1", "cluster", "naming", {})
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
await bi._name_one(ctx, _mk_flow(tmp_path), {"card_id": "c1", "payload": {}})
|
||||
card = await db.kanban_get_card(TOPIC, B, "c1")
|
||||
assert card["payload"]["title"] == "Bubblesort Grundprinzip"
|
||||
|
||||
# unverankerter Name → Member-Titel gewinnt
|
||||
antwort["val"] = {"best": 2, "name": "Vergleichsbasierte Sortierverfahren"}
|
||||
await db.kanban_upsert_card(TOPIC, B, "c2", "cluster", "naming", {})
|
||||
await bi._name_one(ctx, _mk_flow(tmp_path), {"card_id": "c2", "payload": {}})
|
||||
card = await db.kanban_get_card(TOPIC, B, "c2")
|
||||
assert card["payload"]["title"] == "Sortieren durch Tauschen"
|
||||
|
||||
|
||||
async def test_namecheck_ok_behaelt_titel(testdb, tmp_path, monkeypatch):
|
||||
"""Check-Judge bestätigt mit ok:true → Titel und Beschreibung bleiben unverändert."""
|
||||
db = testdb
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"norm": "a", "title": "A", "description": "da", "readers": ["r1"], "sources": []},
|
||||
{"norm": "b", "title": "B", "description": "db", "readers": ["r2"], "sources": []}]
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "Eigener Titel" in prompt # current_title steht im Prompt
|
||||
return "ok", payload((0, '{"ok": true}', ""))
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
payload = {"title": "Eigener Titel", "description": "Eigene Beschreibung", "main_norm": "a"}
|
||||
await db.kanban_upsert_card(TOPIC, B, "c9", "cluster", "naming_check", payload)
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
await bi._namecheck_one(ctx, _mk_flow(tmp_path), {"card_id": "c9", "payload": payload})
|
||||
block = await db.kanban_get_card(TOPIC, B, "b-c9")
|
||||
assert block["payload"]["title"] == "Eigener Titel"
|
||||
assert block["payload"]["description"] == "Eigene Beschreibung"
|
||||
|
||||
197
backend/tests/test_e2e_fake.py
Normal file
197
backend/tests/test_e2e_fake.py
Normal file
@@ -0,0 +1,197 @@
|
||||
"""E2E über die ECHTE Engine mit Fake-Agenten: kompletter Generierungspfad in Sekunden.
|
||||
|
||||
Anders als test_board_inventory (dort sind die Block-Funktionen gefakt) läuft hier alles
|
||||
bis run_agent echt — _race, Quorum, Panels, Konsolidierung, Cross-Block, QA-Gate.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
import board_inventory as bi
|
||||
from pipeline import GenContext
|
||||
from tests.invarianten import pruefe_invarianten, pruefe_guide_invarianten
|
||||
|
||||
TOPIC = "t"
|
||||
|
||||
|
||||
def _files(tmp_path):
|
||||
work = tmp_path / "arbeit"
|
||||
work.mkdir(exist_ok=True)
|
||||
return {"arbeit": work, "final": tmp_path / "blocks.md",
|
||||
"sub_roh": tmp_path / "sub_roh.json", "sidecar": tmp_path / "subblocks.json",
|
||||
"facts": tmp_path / "facts.json", "question_pattern": tmp_path / "question_pattern.json",
|
||||
"artefakte": tmp_path / "artefakte.json", "outline": tmp_path / "outline.json",
|
||||
"outline_slots": [tmp_path / f"outline-{i}.json" for i in (1, 2, 3)],
|
||||
"research": [work / f"research-{i}.md" for i in (1, 2, 3, 4, 5)]}
|
||||
|
||||
|
||||
async def _lauf(tmp_path, research=True, qa_force=False, timeout=120):
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
files = _files(tmp_path)
|
||||
ok = await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "",
|
||||
research=research, qa_force=qa_force), timeout=timeout)
|
||||
return ok, files
|
||||
|
||||
|
||||
async def test_e2e_thema_vollpfad(fake_welt, testdb, tmp_path):
|
||||
"""Research → Inventar → QA-Gate → Artefakte → Finalize, alle Schichten echt."""
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = [c for c in await db.kanban_cards(TOPIC, board="inventory", stage="done_block")]
|
||||
titel = {c["payload"]["title"] for c in done}
|
||||
assert titel == {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"}
|
||||
# Cross-Block-Dublette: „Gemeinsamer Grundbegriff" überlebt in genau EINEM Block
|
||||
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
||||
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
||||
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "variant"]
|
||||
fehler = await pruefe_invarianten(TOPIC, files)
|
||||
assert fehler == []
|
||||
|
||||
|
||||
async def test_e2e_guide(fake_welt, testdb, tmp_path):
|
||||
"""Auf den Vollpfad folgt der Guide-Bau — Gate/Coverage/Lese-Stages laufen echt."""
|
||||
import guide_board
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
||||
entries = {i: f"{c['payload']['title']} — {c['payload'].get('description', '')}"
|
||||
for i, c in enumerate(done, 1)}
|
||||
chapters = await asyncio.wait_for(
|
||||
guide_board.run_guide_board("g-e2e", TOPIC, "Guide", entries, "", "claude",
|
||||
tmp_path / "guides" / "Guide.json"), timeout=120)
|
||||
assert chapters is not None
|
||||
assert await pruefe_guide_invarianten(TOPIC) == []
|
||||
|
||||
|
||||
async def test_e2e_rerun_idempotent(fake_welt, testdb, tmp_path):
|
||||
"""Zweiter Lauf (Continue, research=False) hinterlässt keine Waisen/Reste."""
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
vorher = {(r["block_norm"], r["sub_norm"], r["status"])
|
||||
for r in await db.list_subblocks(TOPIC)}
|
||||
ok2, _f = await _lauf(tmp_path, research=False)
|
||||
assert ok2
|
||||
nachher = {(r["block_norm"], r["sub_norm"], r["status"])
|
||||
for r in await db.list_subblocks(TOPIC)}
|
||||
assert nachher == vorher
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stoerung", [
|
||||
{"muster": r"-sub-crossblock-.*-j1$", "modus": "fehler", "mal": 3}, # Ersatzrichter jE
|
||||
{"muster": r"-sub-konsolidierung-.*-j1$", "modus": "garbage", "mal": 1}, # Retry heilt
|
||||
{"muster": r"-facts-c\d+$", "modus": "fehler", "mal": 1}, # Slot-Restart
|
||||
{"muster": r"-research-2$", "modus": "fehler", "mal": 3}, # 1 Producer tot
|
||||
])
|
||||
async def test_e2e_stoerungen_flow_endet(fake_welt, testdb, tmp_path, stoerung):
|
||||
"""Einzel-Ausfälle dürfen weder den Flow stoppen noch Invarianten reißen."""
|
||||
fake_welt.stoerungen.append(dict(stoerung, rest=stoerung["mal"]))
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_crossblock_dissent_failopen(fake_welt, testdb, tmp_path):
|
||||
"""j1 sagt a, j2 sagt b, j3 fällt aus → Paar bleibt (fail-open), Rest konsistent."""
|
||||
fake_welt.stoerungen += [
|
||||
{"muster": r"-sub-crossblock-.*-j2$", "modus": "antwort",
|
||||
"antwort": '{"pairs": {"1": "b"}}', "mal": 1, "rest": 1},
|
||||
{"muster": r"-sub-crossblock-.*-j3$", "modus": "fehler", "mal": 3, "rest": 3},
|
||||
]
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
||||
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
||||
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "consensus"] # kein Fold
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_inblock_gruppe_faltet(fake_welt, testdb, tmp_path):
|
||||
"""Welt-Regel: „Alpha Eigenschaften" faltet unter „Definition Alpha" — beide Judges
|
||||
liefern die Gruppe, der Verlierer wird variant, seine facts wandern zum Gewinner."""
|
||||
fake_welt.gruppen.append(("Definition Alpha", ["Alpha Eigenschaften"]))
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha-konzept")}
|
||||
assert rows.get("alpha eigenschaften") == "variant"
|
||||
assert rows.get("definition alpha") == "consensus"
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_gate_vollinventur_ohne_fix(fake_welt, testdb, tmp_path):
|
||||
"""Gate-Judge liefert eine Voll-Inventur (belegte Claims mit „Belegt…"-Grund) —
|
||||
der Schema-Filter wirft sie raus, es läuft KEIN Fakten-Fix."""
|
||||
import json
|
||||
antwort = json.dumps({"claims": [
|
||||
{"text": "Aussage 1", "grund": "Belegt durch Quelle", "urteil": "unbelegt"},
|
||||
{"text": "Aussage 2", "grund": "Belegt durch Fakten", "urteil": "unbelegt"},
|
||||
{"text": "Aussage 3", "grund": "Belegt: steht im Skript", "urteil": "unbelegt"}]})
|
||||
fake_welt.stoerungen.append({"muster": r"-gate-", "modus": "antwort",
|
||||
"antwort": antwort, "mal": 99, "rest": 99})
|
||||
import guide_board
|
||||
ok, _files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
||||
entries = {i: c["payload"]["title"] for i, c in enumerate(done, 1)}
|
||||
chapters = await asyncio.wait_for(
|
||||
guide_board.run_guide_board("g-vi", TOPIC, "Guide", entries, "", "claude",
|
||||
tmp_path / "guides" / "Guide.json"), timeout=120)
|
||||
assert chapters is not None
|
||||
assert not any("-gatefix-" in k for k in fake_welt.calls)
|
||||
|
||||
|
||||
async def test_e2e_echtheits_flattern_gestoppt(fake_welt, testdb, tmp_path):
|
||||
"""QA-Pass 1 flaggt alle Blöcke als unecht (Judge-Flattern) — der Bestätiger-Pass
|
||||
widerspricht, die Gate-Note bleibt sauber, der Flow läuft durch."""
|
||||
import json
|
||||
fake_welt.stoerungen.append({"muster": r"^qa-t-bausteine-0$", "modus": "antwort",
|
||||
"antwort": json.dumps({"relevant": {"1": "nein", "2": "nein", "3": "nein"}}),
|
||||
"mal": 1, "rest": 1})
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok # Gate hat nicht pausiert — der Zufalls-Verdacht wurde nicht bestätigt
|
||||
assert any("bausteine-b2" in k for k in fake_welt.calls)
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_uni_anker_gate(fake_welt, testdb, tmp_path, monkeypatch):
|
||||
"""uni-Modus mit Mini-Korpus: der Kanon-Titel ohne Korpus-Anker wird deterministisch
|
||||
rejected (leeres Evidence-Pack), die belegten Blöcke laufen durch; QA misst gegen
|
||||
den echten Korpus."""
|
||||
import blocks as blx
|
||||
fake_welt.bloecke["Kanon-Klassiker"] = {
|
||||
"beschreibung": "Beruehmtes Lehrbuchproblem", "subs": ["Klassiker Detail"]}
|
||||
korpus = tmp_path / "korpus"
|
||||
korpus.mkdir()
|
||||
zeilen = []
|
||||
for t, b in fake_welt.bloecke.items():
|
||||
if t == "Kanon-Klassiker":
|
||||
continue # kommt bewusst NICHT im Material vor
|
||||
zeilen.append(f"Kapitel {t}: {b['beschreibung']}. " +
|
||||
" ".join(f"Wir behandeln {s}." for s in b["subs"]))
|
||||
(korpus / "skript.txt").write_text("\n\n".join(zeilen), encoding="utf-8")
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: korpus)
|
||||
monkeypatch.setattr(blx, "source_folder", lambda t: korpus)
|
||||
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
files = _files(tmp_path)
|
||||
ok = await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "uni", "location": str(korpus)},
|
||||
korpus, "", research=True, qa_force=True), timeout=120)
|
||||
assert ok
|
||||
db = testdb
|
||||
alle = [dict(c) for c in await db.kanban_cards(TOPIC, board="inventory")]
|
||||
assert any(c["stage"] == "rejected" and c["payload"].get("title") == "Kanon-Klassiker"
|
||||
for c in alle)
|
||||
assert not any(c["kind"] == "block" and c["payload"].get("title") == "Kanon-Klassiker"
|
||||
for c in alle) # nie zum Block geworden
|
||||
done = {c["payload"].get("title") for c in alle
|
||||
if c["kind"] == "block" and c["stage"] == "done_block"}
|
||||
assert {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"} <= done
|
||||
@@ -18,8 +18,16 @@ def test_ziele_schema():
|
||||
def test_gate_schema():
|
||||
assert gb._gate_schema({"ok": True}) == []
|
||||
claims = gb._gate_schema({"claims": [{"text": "Falsch", "grund": "fehlt"}]})
|
||||
assert claims == [{"text": "Falsch", "grund": "fehlt"}]
|
||||
assert claims == [{"text": "Falsch", "grund": "fehlt", "urteil": "unbelegt"}]
|
||||
assert gb._gate_schema({}) is None
|
||||
# urteil "falsch" wird durchgereicht, alles andere defaultet auf unbelegt
|
||||
claims = gb._gate_schema({"claims": [{"text": "A", "grund": "widerspricht", "urteil": "FALSCH"},
|
||||
{"text": "B", "grund": "x", "urteil": "quatsch"}]})
|
||||
assert [c["urteil"] for c in claims] == ["falsch", "unbelegt"]
|
||||
# Voll-Inventur-Rauschen: als belegt begründete Einträge fliegen raus
|
||||
claims = gb._gate_schema({"claims": [{"text": "A", "grund": "Belegt durch Quelle X"},
|
||||
{"text": "B", "grund": "nicht ableitbar"}]})
|
||||
assert [c["text"] for c in claims] == ["B"]
|
||||
|
||||
|
||||
def test_coverage_schema():
|
||||
@@ -266,7 +274,7 @@ async def test_lese_check_text_sink(testdb, tmp_path, monkeypatch):
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-l", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
md = ("<!-- kapitel: K -->\n<!-- section: Alpha -->\n<!-- compact -->\n- x\n"
|
||||
"<!-- ausführlich -->\nText.")
|
||||
"<!-- ausführlich -->\n" + "Text im Längen-Rahmen. " * 20) # ~460 Z. — kein Längen-Trigger
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
seen = {}
|
||||
|
||||
@@ -330,3 +338,49 @@ async def test_load_subblocks_defaultet_levellose(testdb):
|
||||
subs = await _load_subblocks("t")
|
||||
by_title = {s["title"]: s["level"] for s in subs["Block"]}
|
||||
assert by_title == {"Mit Level": "beginner", "Ohne Level": "advanced"}
|
||||
|
||||
|
||||
async def test_fakten_gate_falsch_claim_erzwingt_fix(testdb, tmp_path, monkeypatch):
|
||||
"""Ein einzelner FALSCH-Claim läuft in den Fix, auch unter GATE_FIX_MIN —
|
||||
ein durchgerutschter kostete den Guide 1.5 QA-Punkte."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-gf", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
md = "<!-- section: Alpha -->\n<!-- ausführlich -->\nText."
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
keys = []
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
keys.append(key)
|
||||
if "-gate-" in key:
|
||||
return gb.OK, [{"text": "c1", "grund": "widerspricht", "urteil": "falsch"}]
|
||||
return gb.FAILED, None # Fix-Agent liefert nichts — Text bleibt, aber der Call MUSS kommen
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
assert await gb._stage_fakten_gate(env, card)
|
||||
assert any("-gatefix-" in k for k in keys)
|
||||
|
||||
|
||||
async def test_laengen_trigger_startet_lesefix(testdb, tmp_path, monkeypatch):
|
||||
"""Ausführlich-Teil über der Obergrenze → deterministisches Längen-Problem
|
||||
mit hartem Zeichenziel landet im Lese-Fix-Auftrag."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-lz", TOPIC, FMT, "", tmp_path / "Guide.json",
|
||||
{"Alpha": [{"title": "S1", "level": "beginner", "relevance": "relevant"}]},
|
||||
{}, "(q)", "spec")
|
||||
md = ("<!-- section: Alpha -->\n<!-- compact -->\n- x\n<!-- ausführlich -->\n"
|
||||
+ "Viel zu langer Sockeltext. " * 80) # ~2160 Z./Sub > 1200×0.9
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
seen = {}
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
if "-lesefix-" in key:
|
||||
seen["tasks"] = prompt
|
||||
return gb.FAILED, None
|
||||
return gb.OK, payload((0, '{"ok": true}', "")) # Lese-Check: keine Probleme
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
monkeypatch.setattr(gb, "READABILITY_ACTIVE", False)
|
||||
assert await gb._stage_lesbarkeit(env, card)
|
||||
assert "Länge" in seen["tasks"] and str(gb._writer_budget(1)) in seen["tasks"]
|
||||
|
||||
@@ -519,3 +519,90 @@ async def test_finalize_defaultet_level_nachzuegler(testdb, tmp_path):
|
||||
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
|
||||
row = next(r for r in await db.list_subblocks(TOPIC, "alpha"))
|
||||
assert row["level"] == "advanced" and row["relevance"] == "relevant"
|
||||
|
||||
|
||||
async def test_finalize_loescht_stale_consensus(testdb, tmp_path):
|
||||
"""Alt-consensus-Rows, die der Lauf-Sidecar nicht mehr trägt, fliegen raus —
|
||||
variant-Rows bleiben (QA liest die Status). Wurzel der 25 Board-2-losen Waisen."""
|
||||
db = testdb
|
||||
await db.put_subblock(TOPIC, "alpha", "alt-rest", "Alpha", "Alt-Rest", status="consensus")
|
||||
await db.put_subblock(TOPIC, "alpha", "alte-variante", "Alpha", "Alte Variante", status="variant")
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "finalize", {})
|
||||
files = {k: tmp_path / f"{k}.json" for k in
|
||||
("sub_roh", "facts", "sidecar", "question_pattern", "artefakte")}
|
||||
card = {"card_id": "alpha", "payload": {
|
||||
"title": "Alpha", "raw": {"Alpha": ["Neu"]}, "facts": {},
|
||||
"sidecar": {"Alpha": [{"title": "Neu", "level": "beginner"}]},
|
||||
"pattern": {}, "artefacts": {}}}
|
||||
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||||
assert rows == {"neu": "consensus", "alte-variante": "variant"}
|
||||
|
||||
|
||||
async def test_crossblock_chunking_faltet_global(testdb, tmp_path, monkeypatch):
|
||||
"""Paare werden gechunkt beurteilt (ein Hänger blockiert nur noch seinen Chunk);
|
||||
die Verdicts falten global über alle Chunks."""
|
||||
db = testdb
|
||||
flow = Flow(TOPIC, work_dir=tmp_path)
|
||||
cards = []
|
||||
for bnorm, subs in (("alpha", ["Gleiche Aussage", "Zweite gleiche Aussage", "Nur in Alpha"]),
|
||||
("beta", ["Gleiche Aussage", "Zweite gleiche Aussage"])):
|
||||
payload = {"title": bnorm.title(),
|
||||
"raw": {bnorm.title(): list(subs)},
|
||||
"sidecar": {bnorm.title(): [{"title": s, "level": "beginner"} for s in subs]},
|
||||
"facts": {bnorm.title(): {blocks._norm_title(s): {"key_points": []} for s in subs}}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", bnorm, "ablock", "konsolidierung", payload)
|
||||
await _seed_block(db, bnorm, subs)
|
||||
cards.append({"card_id": bnorm, "payload": payload})
|
||||
monkeypatch.setattr(ba, "embedding", _FakeEmb)
|
||||
monkeypatch.setattr(ba, "CROSS_CHUNK_PAARE", 1) # 2 Paare → 2 Chunks
|
||||
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "j2": {"pairs": {"1": "a"}}})
|
||||
monkeypatch.setattr(ba, "run_single_slot", fake)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
assert len(fake.calls) == 4 # 2 Chunks × j1/j2
|
||||
beta = await db.kanban_get_card(TOPIC, "artefacts", "beta")
|
||||
assert beta["payload"]["raw"].get("Beta", []) == [] # beide Dubletten global gefaltet
|
||||
|
||||
|
||||
async def test_facts_nachfass_holt_nur_fehlende(testdb, tmp_path, monkeypatch):
|
||||
"""Nachfass ruft den slim-Facts-Lauf NUR mit den facts-losen Subs und merged die Funde;
|
||||
Vorhandenes bleibt unberührt, Subs werden nie verworfen."""
|
||||
gesehen = {}
|
||||
|
||||
async def fake_facts_block(ctx, set_p, files, raw, q, folder, instructions, ns="", lbl="",
|
||||
sources=None, slim=False):
|
||||
gesehen["raw"] = raw
|
||||
gesehen["slim"] = slim
|
||||
return ({"Alpha": {"ohne beleg": {"key_points": ["kp neu"]},
|
||||
"mit beleg": {"key_points": ["DARF NICHT GEWINNEN"]}}},
|
||||
{"Alpha": {"ohne beleg"}}) # discard-Urteil wird ignoriert
|
||||
|
||||
monkeypatch.setattr(blocks, "_facts_block", fake_facts_block)
|
||||
raw = {"Alpha": ["Mit Beleg", "Ohne Beleg"]}
|
||||
facts_map = {"Alpha": {"mit beleg": {"key_points": ["kp alt"]}}}
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
n = await blocks._facts_nachfass(ctx, {"arbeit": tmp_path}, raw, facts_map, {}, None)
|
||||
assert n == 1
|
||||
assert gesehen["slim"] and gesehen["raw"] == {"Alpha": ["Ohne Beleg"]}
|
||||
assert facts_map["Alpha"]["ohne beleg"]["key_points"] == ["kp neu"]
|
||||
assert facts_map["Alpha"]["mit beleg"]["key_points"] == ["kp alt"]
|
||||
assert raw["Alpha"] == ["Mit Beleg", "Ohne Beleg"] # kein Verwurf
|
||||
|
||||
|
||||
async def test_levels_merge_fuzzy_match(testdb, tmp_path, monkeypatch):
|
||||
"""Levels-Agent paraphrasiert den Sub-Titel → facts hängen trotzdem am Sidecar-Eintrag
|
||||
(eindeutiger Präfix-Match statt stillem Grounding-Verlust)."""
|
||||
db = testdb
|
||||
payload = {"title": "Alpha", "raw": {"Alpha": ["Marker Regel: Details dazu"]},
|
||||
"facts": {"Alpha": {blocks._norm_title("Marker Regel: Details dazu"):
|
||||
{"key_points": ["kp"]}}}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "levels", payload)
|
||||
cards = [{"card_id": "alpha", "payload": payload}]
|
||||
|
||||
async def fake_levels_block(ctx, set_p, files, raw, instructions, ns="", lbl=""):
|
||||
return {"Alpha": [{"title": "Marker Regel", "level": "beginner"}]} # gekürzter Titel
|
||||
|
||||
monkeypatch.setattr(ba, "_levels_block", fake_levels_block)
|
||||
await ba._proc_levels(_ctx(), Flow(TOPIC, work_dir=tmp_path), {"arbeit": tmp_path}, "", cards)
|
||||
card = await db.kanban_get_card(TOPIC, "artefacts", "alpha")
|
||||
assert card["payload"]["sidecar"]["Alpha"][0]["facts"] == {"key_points": ["kp"]}
|
||||
|
||||
@@ -214,3 +214,37 @@ def test_fremd_digit_suffix_tolerant():
|
||||
b = [{"title": "ΔTSP1-Algorithmus", "description": "", "sources": []},
|
||||
{"title": "Quantencomputer", "description": "", "sources": []}]
|
||||
assert qa.fremd(b, corpus) == ["Quantencomputer"]
|
||||
|
||||
|
||||
async def test_unecht_braucht_doppelt_nein(testdb, tmp_path, monkeypatch):
|
||||
"""Echtheits-Urteil zählt nur nach Bestätiger-Pass: der Einzel-Judge flaggte pro Lauf
|
||||
andere Blöcke und pendelte die Note (aak: 9.3↔10.0 bei identischem Bestand)."""
|
||||
db = testdb
|
||||
for cid, titel in (("b1", "Wackelkandidat"), ("b2", "Zufallstreffer"), ("b3", "Solide")):
|
||||
await db.kanban_upsert_card("t", "inventory", cid, "block", "done_block",
|
||||
{"title": titel, "description": "d"})
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path)
|
||||
|
||||
async def fake_verdicts(template, topic, key, items):
|
||||
if template != "QA-Bausteine":
|
||||
return {}
|
||||
if key.startswith("bausteine-b2"): # Bestätiger sieht nur die Geflaggten
|
||||
assert len(items) == 2
|
||||
return {1: "nein", 2: "ja"} # nur der erste wird bestätigt
|
||||
return {1: "nein", 2: "nein", 3: "ja"} # Pass 1 flaggt zwei
|
||||
|
||||
monkeypatch.setattr(qa, "_llm_verdicts", fake_verdicts)
|
||||
report = await qa.qa_report("t", llm=True)
|
||||
assert report["unecht"] == ["Wackelkandidat"]
|
||||
|
||||
|
||||
async def test_topic_delete_entfernt_qa_ordner(testdb, tmp_path, monkeypatch):
|
||||
"""DELETE /topics räumt auch storage/qa/<topic>/ — Reports gehören zum Topic."""
|
||||
import routes
|
||||
monkeypatch.setattr(routes, "topic_dir", lambda t: tmp_path / "topics" / t)
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
|
||||
qdir = tmp_path / "qa" / "t"
|
||||
qdir.mkdir(parents=True)
|
||||
(qdir / "alt.json").write_text("{}", encoding="utf-8")
|
||||
await routes.remove_topic("t")
|
||||
assert not qdir.exists()
|
||||
|
||||
@@ -226,10 +226,8 @@ async def test_outline_review_moves_block(testdb, tmp_path, monkeypatch):
|
||||
out = plan_a
|
||||
elif key.endswith("outline-review"):
|
||||
out = review_out["val"]
|
||||
if out is not None:
|
||||
m = re.search(r"(/\S+\.json)", prompt)
|
||||
with open(m.group(1), "w", encoding="utf-8") as f:
|
||||
json.dump(out, f)
|
||||
if out is not None: # neue Semantik: Antwort als TEXT, der Sink persistiert
|
||||
return "ok", payload((0, json.dumps(out), ""))
|
||||
return "ok", payload(None)
|
||||
|
||||
monkeypatch.setattr(blx, "run_single_slot", fake_slot)
|
||||
|
||||
113
backend/tests/test_train.py
Normal file
113
backend/tests/test_train.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Training-Harness: Registry↔config-Konsistenz, ENV-Override, Trainer-Logik (Stub-Runner)."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import config
|
||||
import train
|
||||
import train_params
|
||||
from train import Trainer, score
|
||||
|
||||
BACKEND = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def test_registry_spiegelt_config():
|
||||
"""Jeder Registry-Parameter existiert in config mit identischem Default und
|
||||
flow-sicheren Rändern — sonst optimiert der Trainer Phantome."""
|
||||
for name, p in train_params.PARAMS.items():
|
||||
assert getattr(config, name, None) == p["default"], name
|
||||
assert p["min"] <= p["default"] <= p["max"], name
|
||||
assert p["step"] > 0, name
|
||||
|
||||
|
||||
def test_creator_params_override_wirkt_im_subprozess():
|
||||
out = subprocess.run(
|
||||
[sys.executable, "-c", "import config; print(config.FACTS_CHUNK_SUBS, config.TIMEOUTS['subblock_check'][0])"],
|
||||
capture_output=True, text=True, cwd=BACKEND,
|
||||
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"FACTS_CHUNK_SUBS": 6, "TIMEOUT_subblock_check_base": 77}'})
|
||||
assert out.stdout.split() == ["6", "77"], out.stderr
|
||||
|
||||
|
||||
def test_creator_params_unbekannter_name_bricht_ab():
|
||||
out = subprocess.run([sys.executable, "-c", "import config"],
|
||||
capture_output=True, text=True, cwd=BACKEND,
|
||||
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"GIBT_ES_NICHT": 1}'})
|
||||
assert out.returncode != 0 and "GIBT_ES_NICHT" in out.stderr
|
||||
|
||||
|
||||
def _metrics(note=8.0, dauer=10.0, tokens=1_000_000, **quoten):
|
||||
return {"note": note, "quoten": quoten, "quoten_artefakte": {},
|
||||
"dauer_min": dauer, "tokens": {"input": tokens, "output": 0}, "agents": {}}
|
||||
|
||||
|
||||
def _stub_runner(antworten):
|
||||
"""params-abhängige Metriken; zählt echte Aufrufe (Cache-Treffer zählen nicht)."""
|
||||
calls = []
|
||||
|
||||
async def runner(params, thema):
|
||||
calls.append((dict(params), thema[0]))
|
||||
for muster, m in antworten:
|
||||
if muster(params):
|
||||
return dict(m)
|
||||
return _metrics()
|
||||
|
||||
runner.calls = calls
|
||||
return runner
|
||||
|
||||
|
||||
async def test_screening_filtert_rauschen(tmp_path):
|
||||
"""Nur Parameter mit Effekt über der Rausch-Schwelle kommen in die Feinphase;
|
||||
ein bestätigter Gewinner wird übernommen."""
|
||||
wirksam = "FACTS_CHUNK_SUBS"
|
||||
runner = _stub_runner([
|
||||
(lambda p: p.get(wirksam) == 8, _metrics(note=9.5, dauer=8.0)), # klar besser
|
||||
])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
best = await t.run()
|
||||
assert best.get(wirksam) == 8
|
||||
# kein anderer Parameter übernommen (alle anderen Δ=0 < Schwelle)
|
||||
assert set(best) == {wirksam}
|
||||
|
||||
|
||||
async def test_uebernahme_braucht_bestaetigung(tmp_path):
|
||||
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird verworfen."""
|
||||
zustand = {"mal": 0}
|
||||
|
||||
async def runner(params, thema):
|
||||
if params.get("FACTS_CHUNK_SUBS") == 8:
|
||||
zustand["mal"] += 1
|
||||
return _metrics(note=9.5) if zustand["mal"] == 1 else _metrics(note=8.0)
|
||||
return _metrics()
|
||||
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
best = await t.run()
|
||||
assert best == {}
|
||||
|
||||
|
||||
async def test_cache_resume_wiederholt_keine_trials(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
await t.run()
|
||||
erste = len(runner.calls)
|
||||
t2 = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
await t2.run()
|
||||
assert len(runner.calls) == erste # alles aus trials.jsonl bedient
|
||||
|
||||
|
||||
async def test_budget_stoppt(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=3, max_stunden=1, runner=runner)
|
||||
await t.run()
|
||||
assert len(runner.calls) <= 3
|
||||
|
||||
|
||||
def test_score_richtungen():
|
||||
basis = _metrics()
|
||||
besser = _metrics(note=9.0)
|
||||
teurer = _metrics(dauer=20.0, tokens=2_000_000)
|
||||
assert score(besser, basis) > score(basis, basis)
|
||||
assert score(teurer, basis) < score(basis, basis)
|
||||
mit_befunden = _metrics(fremd=0.2, luecken=0.1)
|
||||
assert score(mit_befunden, basis) < score(basis, basis)
|
||||
198
backend/train.py
Normal file
198
backend/train.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""make train: Parameter-Optimierung auf Mini-Themen (Baseline → Screening → Koordinaten-Suche).
|
||||
|
||||
Jeder Trial ist ein Subprozess (train_lauf.py) mit CREATOR_PARAMS im ENV — so binden die
|
||||
Module die überschriebenen Werte beim Import. Metriken sind deterministisch (qa_report
|
||||
ohne LLM); gegen Judge-/Lauf-Rauschen gilt: Baseline mit Wiederholung liefert die
|
||||
Rausch-Schwelle, und eine Übernahme braucht einen BESTÄTIGUNGSLAUF (sonst Random Walk).
|
||||
|
||||
CLI: python3 train.py [--trials 40] [--stunden 8] [--sitzung NAME]
|
||||
Ergebnis: storage/train/<sitzung>/{trials.jsonl, report.md, beste_params.json}
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from config import STORAGE_DIR
|
||||
from train_params import PARAMS, schritte
|
||||
|
||||
HAUPT_THEMA = ("train-sort", "benchmarks/sortierverfahren")
|
||||
VALIDIER_THEMA = ("train-foto", "benchmarks/fotografie")
|
||||
# Score-Gewichte: Qualität + Auswahl dominieren (Entwicklungsphase), Kosten ziehen ab.
|
||||
W_NOTE, W_AUSWAHL, W_ZEIT, W_TOKEN = 4.0, 4.0, 1.0, 1.0
|
||||
|
||||
|
||||
def score(m: dict, basis: dict) -> float:
|
||||
"""Skalarer Vergleichswert eines Trials. note 0–10; auswahl aus den MECE-Quoten;
|
||||
Zeit/Tokens normiert auf die Baseline (1.0 = Baseline-Kosten)."""
|
||||
q = m.get("quoten") or {}
|
||||
qa_ = m.get("quoten_artefakte") or {}
|
||||
auswahl = 10.0 * max(0.0, 1.0 - min(1.0, (
|
||||
q.get("dubletten_verdacht", 0) + q.get("luecken", 0) + q.get("fremd", 0)
|
||||
+ qa_.get("sub_dubletten_verdacht", 0) + qa_.get("verwaiste", 0))))
|
||||
zeit = (m.get("dauer_min") or 0) / max(basis.get("dauer_min") or 1, 0.1)
|
||||
tok = _tokens(m) / max(_tokens(basis), 1)
|
||||
return round(W_NOTE * (m.get("note") or 0) + W_AUSWAHL * auswahl
|
||||
- W_ZEIT * 10 * zeit - W_TOKEN * 10 * tok, 2)
|
||||
|
||||
|
||||
def _tokens(m: dict) -> int:
|
||||
t = m.get("tokens") or {}
|
||||
return int(t.get("input") or 0) + int(t.get("output") or 0)
|
||||
|
||||
|
||||
class Trainer:
|
||||
def __init__(self, sitzung: Path, max_trials: int, max_stunden: float, runner=None):
|
||||
self.dir = sitzung
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
self.cache_pfad = self.dir / "trials.jsonl"
|
||||
self.cache: dict[str, dict] = {}
|
||||
if self.cache_pfad.exists(): # Resume: bezahlte Trials nie wiederholen
|
||||
for line in self.cache_pfad.read_text(encoding="utf-8").splitlines():
|
||||
e = json.loads(line)
|
||||
self.cache[e["key"]] = e["metrics"]
|
||||
self.max_trials = max_trials
|
||||
self.deadline = time.monotonic() + max_stunden * 3600
|
||||
self.gezahlt = 0
|
||||
self.runner = runner or self._subprozess
|
||||
self.log = []
|
||||
|
||||
# ── Trial-Ausführung ────────────────────────────────────────────────────────────
|
||||
def _key(self, params: dict, thema: tuple, tag: str = "") -> str:
|
||||
raw = json.dumps({"p": params, "t": thema[0], "tag": tag}, sort_keys=True)
|
||||
return hashlib.md5(raw.encode()).hexdigest()[:12]
|
||||
|
||||
async def trial(self, params: dict, thema: tuple = HAUPT_THEMA, tag: str = "") -> dict | None:
|
||||
"""tag unterscheidet bewusste Wiederholungen (Baseline n=2, Bestätigung)."""
|
||||
key = self._key(params, thema, tag)
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
if self.gezahlt >= self.max_trials or time.monotonic() > self.deadline:
|
||||
return None
|
||||
self.gezahlt += 1
|
||||
metrics = await self.runner(params, thema)
|
||||
if metrics is not None:
|
||||
with open(self.cache_pfad, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps({"key": key, "params": params, "thema": thema[0],
|
||||
"tag": tag, "metrics": metrics}, ensure_ascii=False) + "\n")
|
||||
self.cache[key] = metrics
|
||||
return metrics
|
||||
|
||||
async def _subprozess(self, params: dict, thema: tuple) -> dict | None:
|
||||
out = self.dir / f"metrics-{self._key(params, thema)}.json"
|
||||
env = {"CREATOR_PARAMS": json.dumps(params)}
|
||||
import os
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "train_lauf.py", thema[0], thema[1], str(out),
|
||||
env={**os.environ, **env})
|
||||
rc = await proc.wait()
|
||||
if rc != 0 or not out.exists():
|
||||
self._log(f"Trial fehlgeschlagen (rc={rc}, params={params})")
|
||||
return None
|
||||
return json.loads(out.read_text(encoding="utf-8"))
|
||||
|
||||
def _log(self, msg: str) -> None:
|
||||
line = f"{datetime.now(timezone.utc).isoformat()[11:19]} {msg}"
|
||||
print(line, flush=True)
|
||||
self.log.append(line)
|
||||
|
||||
# ── Trainings-Phasen ────────────────────────────────────────────────────────────
|
||||
async def run(self) -> dict:
|
||||
# Phase 0: Baseline zweimal → Score-Basis + Rausch-Schwelle
|
||||
self._log("Baseline (2 Läufe)…")
|
||||
b1 = await self.trial({}, tag="baseline-1")
|
||||
b2 = await self.trial({}, tag="baseline-2")
|
||||
if not b1 or not b2:
|
||||
self._log("Baseline unvollständig — Abbruch.")
|
||||
return {}
|
||||
self.basis = b1
|
||||
s1, s2 = score(b1, b1), score(b2, b1)
|
||||
self.rauschen = max(abs(s1 - s2), 0.5) # Mindest-Schwelle gegen Glücks-Übernahmen
|
||||
best_params: dict = {}
|
||||
best_score = max(s1, s2)
|
||||
self._log(f"Baseline-Score {s1}/{s2}, Rausch-Schwelle {self.rauschen}")
|
||||
|
||||
# Phase 1: Screening — je Parameter ±1 Schritt, Effekt vs. Rauschen
|
||||
effekte: list[tuple[float, str, float]] = [] # (|effekt|, name, bester_wert)
|
||||
for name in PARAMS:
|
||||
lo, hi = schritte(name)
|
||||
for wert in dict.fromkeys((lo, hi)): # lo==hi am Rand nur einmal
|
||||
if wert == PARAMS[name]["default"]:
|
||||
continue
|
||||
m = await self.trial({**best_params, name: wert})
|
||||
if m is None:
|
||||
continue
|
||||
delta = score(m, self.basis) - best_score
|
||||
self._log(f"Screening {name}={wert}: Δ{delta:+.2f}")
|
||||
if delta > self.rauschen:
|
||||
effekte.append((delta, name, wert))
|
||||
effekte.sort(reverse=True)
|
||||
self._log(f"Wirksam: {[(n, w) for _, n, w in effekte]}")
|
||||
|
||||
# Phase 2: Koordinaten-Suche über ALLE wirksamen Parameter (keine feste Obergrenze),
|
||||
# Übernahme nur nach Bestätigungslauf
|
||||
for _, name, start_wert in effekte:
|
||||
wert = start_wert
|
||||
p = PARAMS[name]
|
||||
richtung = p["step"] if wert > p["default"] else -p["step"]
|
||||
while True:
|
||||
kandidat = {**best_params, name: wert}
|
||||
m = await self.trial(kandidat)
|
||||
if m is None:
|
||||
break
|
||||
delta = score(m, self.basis) - best_score
|
||||
if delta <= self.rauschen:
|
||||
break
|
||||
m2 = await self.trial(kandidat, tag="bestaetigung")
|
||||
if m2 is None or score(m2, self.basis) - best_score <= self.rauschen:
|
||||
self._log(f"{name}={wert}: nicht bestätigt — verworfen")
|
||||
break
|
||||
best_params, best_score = kandidat, min(score(m, self.basis), score(m2, self.basis))
|
||||
self._log(f"ÜBERNOMMEN {name}={wert} → Score {best_score}")
|
||||
naechster = round(wert + richtung, 4)
|
||||
if not p["min"] <= naechster <= p["max"]:
|
||||
break
|
||||
wert = naechster
|
||||
|
||||
# Validierung auf dem zweiten Thema
|
||||
if best_params:
|
||||
v_base = await self.trial({}, thema=VALIDIER_THEMA, tag="val-base")
|
||||
v_best = await self.trial(best_params, thema=VALIDIER_THEMA, tag="val-best")
|
||||
if v_base and v_best:
|
||||
self._log(f"Validierung {VALIDIER_THEMA[0]}: Baseline {score(v_base, v_base)}"
|
||||
f" → Best {score(v_best, v_base)}")
|
||||
|
||||
self._schreibe_report(best_params, best_score)
|
||||
return best_params
|
||||
|
||||
def _schreibe_report(self, best_params: dict, best_score: float) -> None:
|
||||
from fsutil import atomic_write_json, atomic_write_text
|
||||
atomic_write_json(self.dir / "beste_params.json", best_params, indent=1)
|
||||
report = ["# Trainings-Report", "",
|
||||
f"Trials bezahlt: {self.gezahlt}/{self.max_trials}",
|
||||
f"Bester Score: {best_score} (Baseline-Rauschen {self.rauschen})",
|
||||
f"Beste Parameter: `{json.dumps(best_params, ensure_ascii=False)}`",
|
||||
"", "Nutzung: `CREATOR_PARAMS=$(cat beste_params.json) make dev` —",
|
||||
"Übernahme nach config.py bleibt eine manuelle Entscheidung.", "", "## Log", ""]
|
||||
report += [f"- {l}" for l in self.log]
|
||||
atomic_write_text(self.dir / "report.md", "\n".join(report))
|
||||
print(f"\nReport: {self.dir / 'report.md'}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--trials", type=int, default=40)
|
||||
ap.add_argument("--stunden", type=float, default=12.0)
|
||||
ap.add_argument("--sitzung", default=datetime.now(timezone.utc).strftime("%Y%m%d-%H%M"))
|
||||
args = ap.parse_args()
|
||||
trainer = Trainer(STORAGE_DIR / "train" / args.sitzung, args.trials, args.stunden)
|
||||
asyncio.run(trainer.run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
68
backend/train_lauf.py
Normal file
68
backend/train_lauf.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""EIN Trainings-Trial: frischer Prozess (CREATOR_PARAMS wirkt beim Import), ein
|
||||
kompletter Mini-Lauf, deterministische Metriken als JSON — danach ist das Topic weg.
|
||||
|
||||
CLI: python3 train_lauf.py <topic> <benchmark-location> <ausgabe.json>
|
||||
(benchmark-location repo-relativ, z. B. "benchmarks/sortierverfahren")
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import agents
|
||||
import database
|
||||
import qa
|
||||
from blocks import generate_blocks
|
||||
from fsutil import atomic_write_json
|
||||
from paths import source_path, topic_dir
|
||||
|
||||
|
||||
async def trial(topic: str, location: str, out: str) -> None:
|
||||
await database.init_db()
|
||||
agents.on_event = database.add_event # sonst keine Dauer-/Token-Events (main.py-lifespan-Pendant)
|
||||
try:
|
||||
await _aufraeumen(topic) # Reste eines abgebrochenen Trials
|
||||
await database.create_topic(topic)
|
||||
qp = source_path(topic)
|
||||
qp.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write_json(qp, {"type": "uni", "location": location, "spec": ""})
|
||||
start = datetime.now(timezone.utc)
|
||||
# qa_force=True: das Gate misst, pausiert den Trial aber nie
|
||||
await generate_blocks(topic, provider="minimax", research=True, qa_force=True)
|
||||
dauer_min = round((datetime.now(timezone.utc) - start).total_seconds() / 60, 1)
|
||||
report = await qa.qa_report(topic, llm=False) or {}
|
||||
lauf = report.get("lauf") or {}
|
||||
metrics = {
|
||||
"topic": topic,
|
||||
"note": report.get("note"),
|
||||
"note_artefakte": report.get("note_artefakte"),
|
||||
"quoten": report.get("quoten") or {},
|
||||
"quoten_artefakte": report.get("quoten_artefakte") or {},
|
||||
"bloecke": report.get("bloecke"),
|
||||
"dauer_min": lauf.get("dauer_min") or dauer_min,
|
||||
"tokens": (lauf.get("tokens") or {}),
|
||||
"agents": (lauf.get("agents") or {}),
|
||||
}
|
||||
atomic_write_json(Path(out), metrics, indent=1)
|
||||
finally:
|
||||
await _aufraeumen(topic)
|
||||
await database.close_db()
|
||||
|
||||
|
||||
async def _aufraeumen(topic: str) -> None:
|
||||
"""Topic restlos entfernen (DELETE-/topics-Sequenz aus routes.py)."""
|
||||
await database.delete_topic(topic)
|
||||
await database.delete_block_data(topic)
|
||||
await database.delete_topic_pipeline(topic)
|
||||
await database.delete_source(topic)
|
||||
await database.delete_guide_content(topic)
|
||||
shutil.rmtree(topic_dir(topic), ignore_errors=True)
|
||||
shutil.rmtree(qa.QA_DIR / topic, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 4:
|
||||
raise SystemExit("Nutzung: python3 train_lauf.py <topic> <benchmark-location> <ausgabe.json>")
|
||||
asyncio.run(trial(sys.argv[1], sys.argv[2], sys.argv[3]))
|
||||
56
backend/train_params.py
Normal file
56
backend/train_params.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""Suchraum fürs Training (make train): welche config-Parameter der Trainer bewegen darf.
|
||||
|
||||
Je Parameter: default (muss config spiegeln — Test prüft das), min/max (harte Ränder,
|
||||
flow-sicher), step (Schrittweite der Koordinaten-Suche), kategorie (welches Ziel er
|
||||
primär bewegt: qualitaet/auswahl/laufzeit/tokens). QA-/Detektor-Konstanten stehen
|
||||
bewusst NICHT hier — die Messlatte darf nie Teil des Suchraums sein.
|
||||
"""
|
||||
|
||||
PARAMS: dict[str, dict] = {
|
||||
# Recherche / Inventar
|
||||
"RESEARCH_THEMA_AGENTS": {"default": 5, "min": 2, "max": 8, "step": 1, "kategorie": "qualitaet"},
|
||||
"RESEARCH_READERS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet"},
|
||||
"RESEARCH_SECTION_CHARS": {"default": 12000, "min": 6000, "max": 24000, "step": 3000, "kategorie": "tokens"},
|
||||
"DEDUP_PAIR_FLOOR": {"default": 0.6, "min": 0.45, "max": 0.8, "step": 0.05, "kategorie": "auswahl"},
|
||||
"DEDUP_TITLE_AUTO": {"default": 0.95, "min": 0.9, "max": 0.99, "step": 0.01, "kategorie": "auswahl"},
|
||||
"DEDUP_GLOBAL_FLOOR": {"default": 0.65, "min": 0.5, "max": 0.8, "step": 0.05, "kategorie": "auswahl"},
|
||||
"DEDUP_PAIRS_CHUNK": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit"},
|
||||
"FILTER_CHUNK": {"default": 35, "min": 15, "max": 60, "step": 10, "kategorie": "laufzeit"},
|
||||
"FILTER_RECHECK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"CONSOLIDATION_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
# Subbausteine
|
||||
"SUBBLOCK_CHUNK": {"default": 10, "min": 4, "max": 20, "step": 2, "kategorie": "laufzeit"},
|
||||
"SUBBLOCK_MIN": {"default": 5, "min": 2, "max": 10, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_MAX_ROUNDS": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_EXTRA_ROUNDS": {"default": 2, "min": 0, "max": 4, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
# Facts / Artefakte / Fragen
|
||||
"FACTS_CHUNK_SUBS": {"default": 10, "min": 4, "max": 20, "step": 2, "kategorie": "laufzeit"},
|
||||
"FACTS_CHECK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"QUESTION_CHUNK_SUBS": {"default": 25, "min": 10, "max": 50, "step": 5, "kategorie": "laufzeit"},
|
||||
"QUESTION_MAX_ROUNDS": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"ARTEFACT_CHUNK_SUBS": {"default": 25, "min": 10, "max": 50, "step": 5, "kategorie": "laufzeit"},
|
||||
# Embedding-Schwellen (Auswahl-Kern)
|
||||
"SUB_VARIANT_COS": {"default": 0.90, "min": 0.85, "max": 0.96, "step": 0.01, "kategorie": "auswahl"},
|
||||
"SEED_COVER_COS": {"default": 0.80, "min": 0.7, "max": 0.9, "step": 0.02, "kategorie": "auswahl"},
|
||||
"SUB_DUP_KANDIDAT_COS": {"default": 0.75, "min": 0.65, "max": 0.85, "step": 0.02, "kategorie": "auswahl"},
|
||||
"EMBEDDING_BLOCK_FLOOR": {"default": 0.5, "min": 0.35, "max": 0.65, "step": 0.05, "kategorie": "auswahl"},
|
||||
"CROSS_CHUNK_PAARE": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit"},
|
||||
# Guide
|
||||
"MAX_WRITER_ROUNDS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet"},
|
||||
"GATE_FIX_MIN": {"default": 3, "min": 1, "max": 6, "step": 1, "kategorie": "qualitaet"},
|
||||
"WRITER_SPLIT_SUBS": {"default": 30, "min": 15, "max": 45, "step": 5, "kategorie": "qualitaet"},
|
||||
# Engine / Kosten
|
||||
"CONSENSUS_GRACE": {"default": 300, "min": 0, "max": 600, "step": 60, "kategorie": "laufzeit"},
|
||||
"MAX_RESTARTS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "laufzeit"},
|
||||
"EVIDENCE_BUDGET_CHARS": {"default": 48000, "min": 16000, "max": 64000, "step": 8000, "kategorie": "tokens"},
|
||||
"QUELLE_RELEVANZ_CHUNK": {"default": 12, "min": 6, "max": 24, "step": 3, "kategorie": "laufzeit"},
|
||||
}
|
||||
|
||||
|
||||
def schritte(name: str) -> tuple[float, float]:
|
||||
"""(wert_runter, wert_hoch) je einen step von default, an die Ränder geklemmt."""
|
||||
p = PARAMS[name]
|
||||
lo = max(p["min"], round(p["default"] - p["step"], 4))
|
||||
hi = min(p["max"], round(p["default"] + p["step"], 4))
|
||||
return lo, hi
|
||||
23
benchmarks/fotografie/skript.txt
Normal file
23
benchmarks/fotografie/skript.txt
Normal file
@@ -0,0 +1,23 @@
|
||||
Kapitel 1: Die Blende
|
||||
|
||||
Die Blende ist eine verstellbare Öffnung im Objektiv, die die einfallende Lichtmenge steuert. Ihre Größe wird als Blendenzahl angegeben, dem Verhältnis von Brennweite zu Öffnungsdurchmesser: f/2.8 bezeichnet eine große, f/16 eine kleine Öffnung. Eine ganze Blendenstufe halbiert oder verdoppelt die Lichtmenge; die Reihe ganzer Stufen lautet f/1.4, f/2, f/2.8, f/4, f/5.6, f/8, f/11, f/16, f/22. Die Blende steuert zugleich die Schärfentiefe: Eine offene Blende (kleine Blendenzahl) erzeugt geringe Schärfentiefe und Freistellung, eine geschlossene Blende große Schärfentiefe. Jenseits von etwa f/16 sinkt die Detailschärfe durch Beugung.
|
||||
|
||||
Kapitel 2: Die Belichtungszeit
|
||||
|
||||
Die Belichtungszeit ist die Dauer, während der der Sensor Licht sammelt. Sie wird in Sekundenbruchteilen angegeben; jede Halbierung oder Verdopplung entspricht einer Lichtwertstufe. Kurze Zeiten frieren Bewegung ein: 1/1000 Sekunde genügt für Sport, 1/250 Sekunde für gehende Personen. Lange Zeiten erzeugen Bewegungsunschärfe, etwa fließendes Wasser ab 1/4 Sekunde. Als Faustregel für verwacklungsfreies Fotografieren aus der Hand gilt: Belichtungszeit höchstens eins durch Brennweite (Kleinbild-äquivalent), also 1/50 Sekunde bei 50 Millimetern. Bildstabilisatoren verlängern diese Grenze um drei bis fünf Stufen.
|
||||
|
||||
Kapitel 3: Der ISO-Wert
|
||||
|
||||
Der ISO-Wert beschreibt die Signalverstärkung des Sensors. Die Basisempfindlichkeit liegt bei den meisten Kameras bei ISO 100; jede Verdopplung entspricht einer Lichtwertstufe. Höhere ISO-Werte ermöglichen kürzere Belichtungszeiten bei wenig Licht, verstärken aber das Bildrauschen und verringern den Dynamikumfang. Modernes Rauschverhalten erlaubt bei Vollformatsensoren meist saubere Bilder bis ISO 3200 bis 6400. ISO-invariante Sensoren erlauben es, die Aufhellung ins RAW-Processing zu verschieben, ohne zusätzliches Rauschen einzuhandeln.
|
||||
|
||||
Kapitel 4: Das Belichtungsdreieck
|
||||
|
||||
Blende, Belichtungszeit und ISO-Wert bilden das Belichtungsdreieck: Alle drei Größen bestimmen gemeinsam die Bildhelligkeit, und eine Stufe bei einer Größe lässt sich durch eine Stufe einer anderen ausgleichen. Ein Beispiel: f/8 bei 1/125 Sekunde und ISO 100 belichtet identisch wie f/5.6 bei 1/250 Sekunde und ISO 100 oder f/8 bei 1/250 Sekunde und ISO 200. Die Wahl innerhalb dieser äquivalenten Kombinationen ist eine gestalterische Entscheidung über Schärfentiefe, Bewegungsdarstellung und Rauschen.
|
||||
|
||||
Kapitel 5: Der Weißabgleich
|
||||
|
||||
Der Weißabgleich gleicht die Farbtemperatur der Lichtquelle aus, gemessen in Kelvin: Kerzenlicht liegt bei etwa 1800 Kelvin, Glühlampen bei 2700 Kelvin, Tageslicht bei 5500 Kelvin, bedeckter Himmel bei 6500 bis 7500 Kelvin. Ein zu niedrig eingestellter Weißabgleich macht das Bild blau, ein zu hoher macht es orange. Wer in RAW fotografiert, kann den Weißabgleich verlustfrei nachträglich setzen; bei JPEG ist die Korrektur begrenzt.
|
||||
|
||||
Kapitel 6: Autofokus-Betriebsarten
|
||||
|
||||
Der Einzel-Autofokus (AF-S) stellt einmal scharf und verriegelt die Entfernung — geeignet für statische Motive. Der kontinuierliche Autofokus (AF-C) führt die Schärfe laufend nach und ist die Wahl für bewegte Motive; die Trefferquote hängt von der Motivverfolgung ab. Beim Fokus-und-Verschwenken-Verfahren wird mit dem mittleren Feld scharfgestellt und dann der Bildausschnitt verändert; bei offener Blende und naher Distanz führt das Verschwenken zu Fokusfehlern, weil sich die Fokusebene dreht.
|
||||
13
benchmarks/fotografie/uebungen.txt
Normal file
13
benchmarks/fotografie/uebungen.txt
Normal file
@@ -0,0 +1,13 @@
|
||||
Übungsblatt Fotografie-Grundlagen
|
||||
|
||||
Aufgabe 1: Sie fotografieren mit f/8, 1/125 Sekunde, ISO 100. Das Bild ist eine Stufe zu dunkel. Nennen Sie drei Korrekturen, die je genau eine Lichtwertstufe aufhellen: Blende auf f/5.6 öffnen, Belichtungszeit auf 1/60 Sekunde verlängern oder ISO auf 200 verdoppeln.
|
||||
|
||||
Aufgabe 2: Ordnen Sie die Blendenzahlen f/4, f/11, f/2 nach Öffnungsgröße, beginnend mit der größten Öffnung. Reihenfolge: f/2, f/4, f/11.
|
||||
|
||||
Aufgabe 3: Sie fotografieren mit einem 200-Millimeter-Objektiv ohne Stabilisator aus der Hand. Welche längste Belichtungszeit empfiehlt die Faustregel? 1/200 Sekunde.
|
||||
|
||||
Aufgabe 4: Ein Porträt vor unruhigem Hintergrund soll freigestellt werden. Welche Blendenwahl unterstützt das, und welcher Nebeneffekt ist zu beachten? Offene Blende wie f/2, dabei geringe Schärfentiefe — die Fokusebene muss exakt auf den Augen liegen.
|
||||
|
||||
Aufgabe 5: Das Bild einer Kunstlicht-Szene wirkt stark orange. In welche Richtung war der Weißabgleich falsch eingestellt, und wie lautet die passende Farbtemperatur für Glühlampenlicht? Der Weißabgleich stand zu hoch; passend sind etwa 2700 Kelvin.
|
||||
|
||||
Aufgabe 6: Warum führt Fokus-und-Verschwenken bei f/1.8 und einem Meter Abstand zu unscharfen Augen? Beim Verschwenken dreht sich die Fokusebene aus dem Motiv heraus; die geringe Schärfentiefe verzeiht die Abweichung nicht.
|
||||
23
benchmarks/sortierverfahren/skript.txt
Normal file
23
benchmarks/sortierverfahren/skript.txt
Normal file
@@ -0,0 +1,23 @@
|
||||
Kapitel 1: Grundbegriffe des Sortierens
|
||||
|
||||
Ein Sortierverfahren ordnet eine Folge von n Elementen nach einem Ordnungskriterium, meist aufsteigend nach einem Schlüssel. Ein Verfahren heißt stabil, wenn Elemente mit gleichem Schlüssel ihre ursprüngliche Reihenfolge behalten. Ein Verfahren arbeitet in-place, wenn es neben der Eingabefolge nur konstant viel zusätzlichen Speicher benötigt. Die Laufzeit wird in Vergleichen und Vertauschungen gemessen; die untere Schranke für vergleichsbasierte Verfahren liegt bei n log n Vergleichen im schlechtesten Fall.
|
||||
|
||||
Kapitel 2: Bubblesort
|
||||
|
||||
Bubblesort durchläuft die Folge wiederholt von links nach rechts und vertauscht benachbarte Elemente, wenn sie in falscher Reihenfolge stehen. Nach dem ersten Durchlauf steht das größte Element sicher am rechten Ende; nach k Durchläufen stehen die k größten Elemente an ihren endgültigen Positionen. Das Verfahren endet, wenn ein Durchlauf ohne Vertauschung bleibt. Bubblesort ist stabil und arbeitet in-place. Die Laufzeit beträgt im schlechtesten und mittleren Fall Theta(n Quadrat) Vergleiche; im besten Fall (bereits sortierte Folge) genügt ein Durchlauf mit n minus 1 Vergleichen, sofern die Abbruchbedingung implementiert ist.
|
||||
|
||||
Kapitel 3: Insertionsort
|
||||
|
||||
Insertionsort baut den sortierten Bereich am linken Rand schrittweise auf: Das jeweils nächste Element wird von rechts nach links durch Vergleiche an seine Einfügeposition geschoben. Insertionsort ist stabil, arbeitet in-place und benötigt im schlechtesten Fall n mal (n minus 1) durch 2 Vergleiche. Auf fast sortierten Folgen ist Insertionsort ausgesprochen schnell: Die Laufzeit ist linear in der Zahl der Fehlstellungen (Inversionen). Deshalb wird Insertionsort in der Praxis als Basisfall in hybriden Verfahren eingesetzt, etwa für Teilfolgen unter etwa 16 Elementen.
|
||||
|
||||
Kapitel 4: Mergesort
|
||||
|
||||
Mergesort teilt die Folge in zwei Hälften, sortiert beide rekursiv und mischt die sortierten Hälften in linearer Zeit zusammen (Merge-Schritt). Der Merge-Schritt vergleicht die jeweils vordersten Elemente beider Hälften und übernimmt das kleinere. Mergesort ist stabil, benötigt aber ein Hilfsarray der Größe n und arbeitet damit nicht in-place. Die Laufzeit beträgt in allen Fällen Theta(n log n). Mergesort ist das Standardverfahren für externes Sortieren, weil es sequentiell auf Datenströmen arbeiten kann.
|
||||
|
||||
Kapitel 5: Quicksort
|
||||
|
||||
Quicksort wählt ein Pivot-Element, partitioniert die Folge in Elemente kleiner und größer als das Pivot und sortiert beide Teile rekursiv. Die Partitionierung nach Lomuto verwendet das letzte Element als Pivot und einen Lauffinger; die Partitionierung nach Hoare arbeitet mit zwei gegenläufigen Zeigern und weniger Vertauschungen. Quicksort ist nicht stabil. Die mittlere Laufzeit beträgt Theta(n log n) mit kleiner Konstante; der schlechteste Fall Theta(n Quadrat) tritt bei ungünstiger Pivot-Wahl auf, etwa beim ersten Element auf sortierter Eingabe. Randomisierte Pivot-Wahl oder Median-aus-drei machen den schlechten Fall unwahrscheinlich.
|
||||
|
||||
Kapitel 6: Heapsort
|
||||
|
||||
Heapsort baut aus der Folge einen Max-Heap: einen binären Baum in Array-Darstellung, bei dem jeder Knoten mindestens so groß ist wie seine Kinder. Der Aufbau gelingt in linearer Zeit durch absinken lassen (sift-down) von der Mitte an rückwärts. Danach wird wiederholt die Wurzel (das Maximum) mit dem letzten Heap-Element getauscht, der Heap um eins verkürzt und die neue Wurzel abgesenkt. Heapsort arbeitet in-place und garantiert Theta(n log n) im schlechtesten Fall, ist aber nicht stabil und hat schlechtere Cache-Lokalität als Quicksort.
|
||||
13
benchmarks/sortierverfahren/uebungen.txt
Normal file
13
benchmarks/sortierverfahren/uebungen.txt
Normal file
@@ -0,0 +1,13 @@
|
||||
Übungsblatt Sortierverfahren
|
||||
|
||||
Aufgabe 1: Sortieren Sie die Folge 5, 2, 8, 1, 9 mit Bubblesort. Notieren Sie nach jedem Durchlauf den Zustand der Folge und die Zahl der Vertauschungen. Nach Durchlauf 1: 2, 5, 1, 8, 9 (drei Vertauschungen). Nach Durchlauf 2: 2, 1, 5, 8, 9 (eine Vertauschung). Nach Durchlauf 3: 1, 2, 5, 8, 9 (eine Vertauschung). Durchlauf 4 bleibt ohne Vertauschung, das Verfahren endet.
|
||||
|
||||
Aufgabe 2: Zeigen Sie, dass Insertionsort auf einer Folge mit k Inversionen höchstens n minus 1 plus k Vergleiche benötigt. Hinweis: Jeder Vergleich, der zu einer Verschiebung führt, beseitigt genau eine Inversion.
|
||||
|
||||
Aufgabe 3: Führen Sie den Merge-Schritt für die sortierten Hälften 1, 4, 7 und 2, 3, 9 durch. Ergebnisfolge: 1, 2, 3, 4, 7, 9 mit fünf Vergleichen.
|
||||
|
||||
Aufgabe 4: Geben Sie für Quicksort mit Lomuto-Partitionierung und letztem Element als Pivot eine Eingabe der Länge 5 an, die den schlechtesten Fall erzeugt. Die bereits sortierte Folge 1, 2, 3, 4, 5 erzeugt Partitionen der Größen 4, 3, 2, 1 und damit quadratische Laufzeit.
|
||||
|
||||
Aufgabe 5: Bauen Sie aus der Folge 3, 7, 1, 9, 4 einen Max-Heap in Array-Darstellung. Ergebnis nach dem Heap-Aufbau: 9, 7, 1, 3, 4. Begründen Sie, warum der Aufbau von der Mitte an rückwärts in linearer Zeit gelingt.
|
||||
|
||||
Aufgabe 6: Welche der Verfahren Bubblesort, Insertionsort, Mergesort, Quicksort, Heapsort sind stabil? Stabil sind Bubblesort, Insertionsort und Mergesort; Quicksort und Heapsort sind nicht stabil.
|
||||
@@ -14,7 +14,7 @@ A good worked example:
|
||||
|
||||
Write `problem`, the `steps` and `result` in GERMAN.
|
||||
|
||||
Write all examples as ONE JSON into the file {out_path} (use your write tool), EXACTLY like this:
|
||||
Reply with ONLY the JSON (all examples) as your final message — no code fences, do NOT write a file. EXACTLY like this:
|
||||
{{"examples": [
|
||||
{{"block": "<exact block title>", "subblock": "<exact subblock title>",
|
||||
"problem": "…", "steps": ["…", "…"], "result": "…"}}
|
||||
|
||||
@@ -13,7 +13,7 @@ A good flashcard:
|
||||
|
||||
Write `question` and `answer` in GERMAN.
|
||||
|
||||
Write all cards as ONE JSON into the file {out_path} (use your write tool), EXACTLY like this:
|
||||
Reply with ONLY the JSON (all cards) as your final message — no code fences, do NOT write a file. EXACTLY like this:
|
||||
{{"cards": [
|
||||
{{"block": "<exact block title>", "subblock": "<exact subblock title>",
|
||||
"question": "…", "answer": "…"}}
|
||||
|
||||
@@ -1,15 +1,22 @@
|
||||
The numbered entries below all describe the SAME block for the topic "{topic}". Entry number {current} was chosen as the canonical title. Check whether that is the best choice — if another entry is a clearly better canonical name, pick it instead.
|
||||
The numbered entries below all describe the SAME block for the topic "{topic}".
|
||||
|
||||
CURRENT TITLE: {current_title}
|
||||
|
||||
MEMBERS:
|
||||
{members}
|
||||
|
||||
Check whether the current title is the best canonical name for this block.
|
||||
|
||||
Rules:
|
||||
- Pick an EXISTING entry number — do NOT invent a title.
|
||||
- Best = most concrete, precise, self-explanatory, established term for the shared concept.
|
||||
- If the current choice ({current}) is already the best, return it unchanged.
|
||||
- When in doubt, keep the current choice.
|
||||
- Current title good (concrete, precise, self-explanatory, no reference/placeholder like "Satz 7.18") → confirm it.
|
||||
- Otherwise pick the best member ("best" REQUIRED) and optionally propose a SHORTER name: max 8 words, ONLY terms that appear in the members, exactly as concrete as the shared content — never a broader umbrella term, never a textbook concept the members don't mention. NO catalog/reference brackets ("(Satz 6.33)"); a known acronym in brackets ("(VC)") is fine.
|
||||
- When in doubt, confirm the current title.
|
||||
|
||||
Write ONLY the JSON file to: {out_path}
|
||||
Reply with ONLY the JSON — no code fences, no other text.
|
||||
|
||||
Format (the best member number, nothing else):
|
||||
{{"best": {current}}}
|
||||
Format:
|
||||
{{"ok": true}}
|
||||
or
|
||||
{{"best": 2}}
|
||||
or
|
||||
{{"best": 2, "name": "Kurzer kanonischer Titel"}}
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
The numbered entries below all describe the SAME block (concept) for the topic "{topic}", just worded differently. Pick the ONE entry whose title is the best canonical name for this block.
|
||||
The numbered entries below all describe the SAME block (concept) for the topic "{topic}", just worded differently. Pick the ONE entry whose title is the best canonical name for this block — and optionally propose a SHORTER canonical name if none of the titles states the shared core precisely.
|
||||
|
||||
MEMBERS:
|
||||
{members}
|
||||
|
||||
Rules:
|
||||
- Pick an EXISTING entry — do NOT invent a new title or umbrella term.
|
||||
- Prefer the most CONCRETE, precise, self-explanatory title for the shared concept.
|
||||
- Prefer the established/standard term (correct spelling, full form over cryptic abbreviation) — but stay concrete, never over-general.
|
||||
- Avoid reference/placeholder titles ("Satz 7.18", "Punkt 3", "(**)") if a meaningful one exists.
|
||||
- "best" is REQUIRED: the member whose title fits the shared concept best (most concrete, precise, self-explanatory, established term; avoid reference/placeholder titles like "Satz 7.18", "Punkt 3", "(**)").
|
||||
- "name" is OPTIONAL — set it ONLY when no member title states the shared core well. It must be SHORT (max 8 words), use ONLY terms that appear in the members themselves, and stay exactly as concrete as the shared content — never a broader umbrella term, never a textbook concept the members don't mention.
|
||||
- In "name": NO catalog/reference brackets ("(Satz 6.33)", "(Kap. 4)"); a known acronym in brackets ("(VC)") is fine.
|
||||
|
||||
Write ONLY the JSON file to: {out_path}
|
||||
Reply with ONLY the JSON — no code fences, no other text.
|
||||
|
||||
Format (the chosen member number, nothing else):
|
||||
Format (chosen member number, optional short name):
|
||||
{{"best": 1}}
|
||||
or
|
||||
{{"best": 1, "name": "Kurzer kanonischer Titel"}}
|
||||
|
||||
@@ -19,7 +19,7 @@ HARD SEPARATION — important:
|
||||
- A worked example, an invented sentence, a constructed case → belongs in `example_idea`, not in `cited_facts`.
|
||||
- When in doubt: better to leave out than to claim falsely.
|
||||
|
||||
Write ALL subblocks as ONE JSON to the file {out_path} (use your write tool), EXACTLY in this format:
|
||||
Reply with ONLY the JSON as your final message — no code fences, no prose around it. Do NOT write any file; your tools are for research only. EXACTLY this format:
|
||||
{{"facts": [
|
||||
{{"block": "<exakter Block-Title>", "subblock": "<exakter Subblock-Title>",
|
||||
"key_points": ["…"], "prerequisites": "…", "hurdles": "…",
|
||||
|
||||
@@ -14,7 +14,7 @@ Rules:
|
||||
- **If you find nothing new for a subblock → leave it out.** If you find nothing at all → empty list. Better nothing than fabrication.
|
||||
- Write all field content (key_points, prerequisites, hurdles, cited_facts text, example_idea) in GERMAN, matching the source material (technical terms/code identifiers stay original).
|
||||
|
||||
Write ONLY the NEW facts as ONE JSON to the file {out_path}, EXACTLY in this format:
|
||||
Reply with ONLY the NEW facts as ONE JSON in your final message — no code fences, no file writing (tools are for research only). EXACTLY this format:
|
||||
{{"facts": [
|
||||
{{"block": "<exakter Block-Title>", "subblock": "<exakter Subblock-Title>",
|
||||
"key_points": ["…"], "prerequisites": "…", "hurdles": "…",
|
||||
|
||||
@@ -10,7 +10,7 @@ Procedure:
|
||||
1. Decompose the section text into its ATOMIC factual claims (definitions, properties, numbers, procedure steps, causal statements). Ignore pure didactics (transitions, framing, mnemonic phrasing).
|
||||
2. Worked-example passages (a concrete problem worked through in steps to a result) are DIDACTICS when they merely APPLY or ILLUSTRATE a verified fact or a provided worked example: their concretely chosen values and computed intermediates do NOT count as over-specific. Flag an example step ONLY if it contradicts the facts or smuggles in a NEW general claim not derivable from them.
|
||||
3. CONTEXT sentences are DIDACTICS too, not claims: introductions and summaries that only preview/recap the section, uncontroversial general knowledge that merely places the topic (history, origin, what an adjacent well-known technology is), and paraphrases of the verified facts. Ignore them — a guide needs connective tissue. This exemption ends the moment a sentence makes a checkable statement about THIS block's own syntax, behavior or rules: that is a claim.
|
||||
4. Check EACH claim INDIVIDUALLY and BINARY against the VERIFIED FACTS above: derivable from them → belegt. Not derivable, contradicting, or over-specific beyond the facts → nicht belegt.
|
||||
4. Check EACH claim INDIVIDUALLY and BINARY against the VERIFIED FACTS above: derivable from them → belegt. Not derivable, contradicting, or over-specific beyond the facts → nicht belegt. For unsupported claims set "urteil": **"falsch"** when the claim CONTRADICTS the verified facts or contradicts the section itself (e.g. a rule its own example violates); **"unbelegt"** when it is merely not derivable from the facts.
|
||||
5. Do NOT search the web, do NOT use outside knowledge as EVIDENCE — a claim about the block that is true in the world but absent from the facts is still "nicht belegt".
|
||||
6. When a sentence is genuinely ambiguous between context and claim → treat it as a claim (safety before cost).
|
||||
|
||||
@@ -18,6 +18,6 @@ Write ONLY the JSON file to: {out_path}
|
||||
|
||||
Format — everything supported:
|
||||
{{"ok": true}}
|
||||
Otherwise (each unsupported claim VERBATIM as it appears in the text):
|
||||
{{"claims": [{{"text": "verbatim claim from the section", "grund": "why unsupported (German, short)"}}]}}
|
||||
Otherwise (ONLY unsupported claims, VERBATIM as they appear in the text — NEVER list supported ones):
|
||||
{{"claims": [{{"text": "verbatim claim from the section", "grund": "why unsupported (German, short)", "urteil": "falsch|unbelegt"}}]}}
|
||||
{extra}
|
||||
|
||||
@@ -19,7 +19,7 @@ Rules:
|
||||
|
||||
Write the chapter titles in GERMAN (the guide is for German-speaking learners), even though these instructions are in English.
|
||||
|
||||
Write ONLY the JSON file to: {out_path}
|
||||
Reply with ONLY the JSON as your final message — no code fences, do NOT write a file.
|
||||
|
||||
Format (numbers = the block numbers in the desired order):
|
||||
{{"chapters": [{{"title": "Grundlagen", "numbers": [3, 1, 7]}}]}}
|
||||
|
||||
@@ -10,7 +10,7 @@ OUTLINE (chapters are numbered, blocks carry their block number):
|
||||
- If everything fits, report no moves.
|
||||
{extra}
|
||||
|
||||
Write ONLY the JSON file to: {out_path}
|
||||
Reply with ONLY the JSON as your final message — no code fences, do NOT write a file.
|
||||
|
||||
Format — `moves` maps a misplaced BLOCK number to the target CHAPTER number (may be empty):
|
||||
{{"moves": {{"7": 2, "15": 4}}}}
|
||||
|
||||
@@ -18,7 +18,7 @@ Rules:
|
||||
|
||||
Write the chapter titles in GERMAN (the guide is for German-speaking learners), even though these instructions are in English.
|
||||
|
||||
Write ONLY the JSON file to: {out_path}
|
||||
Reply with ONLY the JSON as your final message — no code fences, do NOT write a file.
|
||||
|
||||
Format (numbers = the block numbers in the desired order):
|
||||
{{"chapters": [{{"title": "Grundlagen", "numbers": [3, 1, 7]}}]}}
|
||||
|
||||
@@ -10,7 +10,7 @@ TASK:
|
||||
- Numbers **only from the list**. **No self-references.** When in doubt, leave it **empty** — better no edge than an invented one.
|
||||
- Keep it sparse: most blocks have 0-2 direct prerequisites, not more.
|
||||
|
||||
Write ONLY the JSON file to: {out_path}, EXACTLY in this format (key = block number, value = list of prerequisite numbers):
|
||||
Reply with ONLY the JSON as your final message — no code fences, do NOT write a file. EXACTLY this format (key = block number, value = list of prerequisite numbers):
|
||||
{{"prereqs": {{"3": [1, 7], "5": []}}}}
|
||||
|
||||
Output no other text.
|
||||
|
||||
@@ -20,7 +20,7 @@ HARD STYLE RULES PER QUESTION:
|
||||
Write every question in GERMAN (the questions are for German-speaking learners), even though these instructions are in English.
|
||||
{extra}
|
||||
|
||||
Write ALL patterns of all blocks as ONE JSON to the file {out_path} (use your write tool), EXACTLY in this format:
|
||||
Reply with ONLY the JSON (all patterns of all blocks) as your final message — no code fences, do NOT write a file. EXACTLY this format:
|
||||
{{"pattern": [
|
||||
{{"block": "<exact block title>", "subblock": "<exact subblock title>", "question": "<one concrete question>"}}
|
||||
]}}
|
||||
|
||||
Reference in New Issue
Block a user