update
This commit is contained in:
team3
@@ -245,6 +245,9 @@ async def run_agent(
|
||||
on_line=None,
|
||||
label: str = "",
|
||||
) -> tuple[int, str, str]:
|
||||
if os.getenv("CREATOR_FAKE_AGENTS"): # Sekunden-Smoke: deterministische Antworten statt LLM
|
||||
import fake_agents
|
||||
return await fake_agents.respond(agent_key, prompt, capabilities)
|
||||
if _scope_cancelled(agent_key): # before queueing: don't even enter the queue
|
||||
return 1, "", "cancelled"
|
||||
if provider not in PROVIDERS:
|
||||
|
||||
@@ -39,45 +39,14 @@ from textkit import (
|
||||
_resolve_title, _title_index, clean_title,
|
||||
)
|
||||
|
||||
# Chunk the subblocks (web search per block): 1 agent per ~10 blocks, capped.
|
||||
SUBBLOCK_CHUNK = 10
|
||||
SUBBLOCK_MAX = 40
|
||||
# Classifying is cheap (short verdict, no web search) → larger packages, fewer files/agents.
|
||||
LEVEL_CHUNK = 100
|
||||
|
||||
# Research: fixed file batches instead of a search loop → each crawl page is assigned exactly once.
|
||||
RESEARCH_BATCH = 20 # crawl pages per batch
|
||||
RESEARCH_READERS = 2 # reader agents per batch (consensus ≥2 within the batch)
|
||||
RESEARCH_THEMA_AGENTS = 5 # web mode (source "thema", no crawl folder)
|
||||
# uni/projekt: chunk the script text into sections of ~this size (against lost-in-the-middle on
|
||||
# large documents). ~12k chars ≈ 3k tokens → safely below the recall-drop threshold.
|
||||
RESEARCH_SECTION_CHARS = 12000
|
||||
# Triage (content/noise) is now a deterministic rule filter (config.CRAWL_*).
|
||||
SUBBLOCK_CAP = 900 # subblock find loop per chunk (15 min)
|
||||
SUBBLOCK_MIN = 5 # below this consensus count a block gets focused catch-up rounds
|
||||
SUBBLOCK_EXTRA_ROUNDS = 2 # max catch-up rounds (saturation stop still applies — thin stays thin)
|
||||
SUBBLOCK_MAX_ROUNDS = 3 # hard round cap: measured, rounds 4–5 burned 29 % of the finder agents
|
||||
# for ~zero consensus gain (fringe ideas never saturate) — thin blocks
|
||||
# are caught by the SUBBLOCK_MIN catch-up plus the gap follow-up round
|
||||
CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (dedups everything); above that chunked + merge pass — fallback path only
|
||||
DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair (complete-link aggregates → no chaining)
|
||||
DEDUP_PAIRS_CHUNK = 40 # pairs per judge package (pairwise verification instead of a block mixer)
|
||||
DEDUP_TITLE_AUTO = 0.95 # near-identical TITLE cosine ⇒ same entity → merge without the judge (recall net)
|
||||
DEDUP_GLOBAL_FLOOR = 0.65 # global post-naming dedup stage: candidate floor above the 0.5-0.65
|
||||
# same-domain noise band, below the sibling zone (~0.85) — the judge decides there
|
||||
FILTER_CHUNK = 35 # blocks to assess per judge in the degrade pass (full list as context)
|
||||
# Balance question-pattern chunks by sub load via LPT (makespan), not by block count.
|
||||
QUESTION_CHUNK_SUBS = 25 # target sum of relevant subs per chunk — at 50 the generator
|
||||
# skipped so many subs that 60 % of all question calls were catch-up
|
||||
QUESTION_MAX_ROUNDS = 3 # catch-up rounds for subs without a pattern (the LLM omits ~18 % per chunk)
|
||||
FACTS_CHUNK_SUBS = 10 # facts extraction: small chunks — the 4 phases (find/erg/check/fix) are
|
||||
# serial PER CHUNK, so chunk count = parallelism; the makespan tail of a
|
||||
# late block is bounded by ONE chunk's phase chain, not the whole block
|
||||
ARTEFACT_CHUNK_SUBS = 25 # flashcards/examples: bulk generation, phases are cheap → bigger packages
|
||||
FACTS_CHECK_PANEL = 3 # judges per chunk in the facts check (majority objects)
|
||||
CONSOLIDATION_PANEL = 3 # mapping judges per chunk (panel → reconcile instead of a single judge)
|
||||
SUBBLOCK_PANEL = 3 # source judges in the subblock clarification (majority instead of a single judge)
|
||||
FILTER_RECHECK_PANEL = 3 # judges in the survivor-recheck (rare-positive "fragment" recall; majority ≥2)
|
||||
# Pipeline-Tuning-Konstanten liegen zentral in config.py (tunebar via CREATOR_PARAMS).
|
||||
from config import ( # noqa: E402
|
||||
ARTEFACT_CHUNK_SUBS, CONSOLIDATION_CHUNK, CONSOLIDATION_PANEL, DEDUP_GLOBAL_FLOOR,
|
||||
DEDUP_PAIR_FLOOR, DEDUP_PAIRS_CHUNK, DEDUP_TITLE_AUTO, FACTS_CHECK_PANEL, FACTS_CHUNK_SUBS,
|
||||
FILTER_CHUNK, FILTER_RECHECK_PANEL, LEVEL_CHUNK, QUESTION_CHUNK_SUBS, QUESTION_MAX_ROUNDS,
|
||||
RESEARCH_BATCH, RESEARCH_READERS, RESEARCH_SECTION_CHARS, RESEARCH_THEMA_AGENTS,
|
||||
SUBBLOCK_CAP, SUBBLOCK_CHUNK, SUBBLOCK_EXTRA_ROUNDS, SUBBLOCK_MAX, SUBBLOCK_MAX_ROUNDS,
|
||||
SUBBLOCK_MIN, SUBBLOCK_PANEL)
|
||||
|
||||
log = logging.getLogger("creator.blocks")
|
||||
|
||||
@@ -608,6 +577,16 @@ def _reply_text(result) -> str:
|
||||
return (result[1] or "") if result else ""
|
||||
|
||||
|
||||
def _sink_or_file(result, path: Path, schema):
|
||||
"""Text reply preferred, file fallback. Research agents used to WRITE their big JSON —
|
||||
measured: one facts call spent 40 of 64 turns in a write/validate/repair loop (9 min).
|
||||
As text, parse_json_text repairs the escaping in one pass; tools stay for research."""
|
||||
val = _sink_json(result, path, schema)
|
||||
if val is not None:
|
||||
return val
|
||||
return schema(_json_file(path))
|
||||
|
||||
|
||||
def _sink_json(result, path: Path, schema):
|
||||
"""Payload validator for no-tool agents: the JSON comes as reply TEXT; the engine
|
||||
persists it to `path`, so resume guards and audit files keep working unchanged."""
|
||||
@@ -1502,6 +1481,38 @@ async def _luecken_runde(ctx: GenContext, files: dict, title: str, luecken: list
|
||||
return len(kept)
|
||||
|
||||
|
||||
async def _facts_nachfass(ctx: GenContext, files: dict, raw: dict, facts_map: dict, q: dict,
|
||||
folder, instructions: str = "", ns: str = "", lbl: str = "",
|
||||
sources: list[str] | None = None) -> int:
|
||||
"""ONE slim facts round for consensus subs WITHOUT a facts entry — renames during
|
||||
consolidation and catalog rows left 35 % of the subs ungrounded; the fact gate then
|
||||
flagged their (correct) guide statements wholesale. The subs themselves stay either
|
||||
way: they exist by consensus, only the grounding is fetched. → count of filled subs."""
|
||||
fehlend = {bt: [s for s in subs if _norm_title(s) not in (facts_map.get(bt) or {})]
|
||||
for bt, subs in raw.items()}
|
||||
fehlend = {bt: subs for bt, subs in fehlend.items() if subs}
|
||||
if not fehlend:
|
||||
return 0
|
||||
nf_dir = files["arbeit"] / "nf2"
|
||||
nf_dir.mkdir(parents=True, exist_ok=True)
|
||||
res = await _facts_block(ctx, lambda *a, **k: None, {**files, "arbeit": nf_dir},
|
||||
fehlend, q, folder, instructions,
|
||||
ns=f"{ns}nf2-", lbl=lbl, sources=sources, slim=True)
|
||||
if ctx.is_cancelled() or res is None:
|
||||
return 0
|
||||
nf_facts, _discarded = res # discard verdicts ignored — consensus subs are not removed here
|
||||
filled = 0
|
||||
for bt, fm in nf_facts.items():
|
||||
bfacts = facts_map.setdefault(bt, {})
|
||||
for sn, fk in fm.items():
|
||||
if sn not in bfacts and (fk.get("key_points") or fk.get("cited_facts")):
|
||||
bfacts[sn] = fk
|
||||
filled += 1
|
||||
if filled:
|
||||
_log(ctx.topic, f"Facts-Nachfass{': ' + lbl if lbl else ''}{filled} Subs nachbelegt")
|
||||
return filled
|
||||
|
||||
|
||||
def _subs_hash(sidecar_or_raw: dict) -> str:
|
||||
"""Sub-set identity for the resume files of the sub-CONSUMING stages (levels/relevance/
|
||||
questions/artefacts). Without it a re-run with a recut sub set adopted the stale stage
|
||||
@@ -1811,7 +1822,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
|
||||
ctx, f"{lbl}Facts {ci}", key=f"blocks-{topic}-{ns}facts-c{ci}",
|
||||
prompt=_prompt("Facts-Research", topic=topic, source=source, blocks=block_text(idxs), out_path=fp, extra=_extra(instructions)),
|
||||
role="quick", capabilities=caps,
|
||||
payload=lambda result, p=fp: _facts_schema(_json_file(p)),
|
||||
payload=lambda result, p=fp: _sink_or_file(result, p, _facts_schema),
|
||||
timeout=_timeout("content", subs_total))
|
||||
return status != FAILED and _facts_schema(_json_file(fp)) is not None
|
||||
|
||||
@@ -1842,7 +1853,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
|
||||
ctx, f"{lbl}Facts supplement {ci}", key=f"blocks-{topic}-{ns}facts-erg-c{ci}",
|
||||
prompt=_prompt("Facts-Supplement", topic=topic, source=source, blocks=block, out_path=ep, extra=_extra(instructions)),
|
||||
role="quick", capabilities=caps,
|
||||
payload=lambda result, p=ep: _facts_schema(_json_file(p)),
|
||||
payload=lambda result, p=ep: _sink_or_file(result, p, _facts_schema),
|
||||
timeout=_timeout("content", subs_total))
|
||||
|
||||
if not slim:
|
||||
@@ -1929,7 +1940,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
|
||||
ctx, f"{lbl}Facts-Fix {ci}", key=f"blocks-{topic}-{ns}facts-fix-c{ci}",
|
||||
prompt=_prompt("Facts-Research", topic=topic, source=source, blocks="\n\n".join(goal), out_path=fix_path(ci), extra=_extra(instructions)),
|
||||
role="quick", capabilities=caps,
|
||||
payload=lambda result, p=fix_path(ci): _facts_schema(_json_file(p)),
|
||||
payload=lambda result, p=fix_path(ci): _sink_or_file(result, p, _facts_schema),
|
||||
timeout=_timeout("content", len(subs_norm)))
|
||||
await _gather_progress([_fix(ci) for ci in correctable], len(correctable), _report_p(set_p, topic, "Facts fix"))
|
||||
if is_cancelled():
|
||||
@@ -2121,7 +2132,7 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar:
|
||||
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=block,
|
||||
out_path=fp, extra=_extra(instructions)),
|
||||
role="quick", capabilities="files",
|
||||
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
|
||||
payload=lambda result, p=fp: _sink_or_file(result, p, _question_pattern_chunk_schema),
|
||||
timeout=_timeout("question_pattern", subs_total),
|
||||
)
|
||||
if status == FAILED:
|
||||
@@ -2255,7 +2266,7 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar:
|
||||
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=_followup_block(items),
|
||||
out_path=fp, extra=_extra(instructions)),
|
||||
role="quick", capabilities="files",
|
||||
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
|
||||
payload=lambda result, p=fp: _sink_or_file(result, p, _question_pattern_chunk_schema),
|
||||
timeout=_timeout("question_pattern", subs_total),
|
||||
)
|
||||
|
||||
@@ -3013,8 +3024,8 @@ async def _learning_order(ctx: GenContext, set_p, files: dict, entries: dict, va
|
||||
pp = files["arbeit"] / "outline-prereqs.json"
|
||||
|
||||
def _payload(result, p=pp):
|
||||
d = _json_file(p)
|
||||
return d if isinstance(d, dict) and "prereqs" in d else None
|
||||
d = _sink_or_file(result, p, lambda x: x if isinstance(x, dict) and "prereqs" in x else None)
|
||||
return d
|
||||
|
||||
existing = _json_file(pp)
|
||||
if not (isinstance(existing, dict) and "prereqs" in existing):
|
||||
@@ -3052,7 +3063,7 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
|
||||
ctx, f"Outline {i}", key=f"blocks-{topic}-outline-{i}",
|
||||
prompt=_prompt("Guide-Outline", topic=topic, blocks=liste, out_path=path, extra=_extra(instructions)),
|
||||
role="guide", capabilities="files",
|
||||
payload=lambda result, p=path: _outline_schema(_json_file(p), valid),
|
||||
payload=lambda result, p=path: _sink_or_file(result, p, lambda d: _outline_schema(d, valid)),
|
||||
timeout=_timeout("plan", len(entries)))
|
||||
return _outline_schema(_json_file(path), valid) is not None
|
||||
|
||||
@@ -3078,7 +3089,8 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
|
||||
purpose="alle Blocks in einem roten Faden", n=len(proposals),
|
||||
blocks=liste, outlines=block_texts, out_path=files["outline"], extra=_extra(instructions)),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result: _outline_schema(_json_file(files["outline"]), valid),
|
||||
payload=lambda result: _sink_or_file(result, files["outline"],
|
||||
lambda d: _outline_schema(d, valid)),
|
||||
timeout=_timeout("plan_judge", len(entries)))
|
||||
plan = _outline_schema(_json_file(files["outline"]), valid) or proposals[0]
|
||||
|
||||
@@ -3097,8 +3109,8 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
|
||||
prompt=_prompt("Guide-Outline-Review", topic=topic, chapters=chapter_text,
|
||||
out_path=rp, extra=_extra(instructions)),
|
||||
role="judge", capabilities="files",
|
||||
payload=lambda result: _outline_review_schema(
|
||||
_json_file(rp), valid, len(plan["chapters"]), len(entries)),
|
||||
payload=lambda result: _sink_or_file(result, rp, lambda d: _outline_review_schema(
|
||||
d, valid, len(plan["chapters"]), len(entries))),
|
||||
timeout=_timeout("plan_judge", len(entries)))
|
||||
moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
|
||||
for nr, target in (moves or {}).items():
|
||||
@@ -3257,7 +3269,7 @@ async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i
|
||||
ctx, f"{lbl}{_ARTEFACT_STEP[typ]} {ci}", key=f"blocks-{topic}-{ns}artifact-{typ}-c{ci}",
|
||||
prompt=_prompt(_ARTEFACT_PROMPT[typ], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)),
|
||||
role="guide", capabilities="files",
|
||||
payload=lambda result, p=p: schema(_json_file(p)),
|
||||
payload=lambda result, p=p: _sink_or_file(result, p, schema),
|
||||
timeout=_timeout("content", sum(len(blocks[i][1]) for i in idxs)))
|
||||
return schema(_json_file(p)) is not None
|
||||
|
||||
|
||||
@@ -19,11 +19,11 @@ import database as db
|
||||
import blocks
|
||||
import embedding
|
||||
from blocks import (
|
||||
ARTEFACT_TYPES, _artefacts_block, _facts_block, _konsolidiere_subblocks, _levels_block,
|
||||
_luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
|
||||
ARTEFACT_TYPES, _artefacts_block, _facts_block, _facts_nachfass, _konsolidiere_subblocks,
|
||||
_levels_block, _luecken_runde, _match_sub, _neg_set, _question_pattern_block, _relevance_block,
|
||||
_sink_json, _subblocks_block, _outline_block,
|
||||
)
|
||||
from config import EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
|
||||
from config import CROSS_CHUNK_PAARE, EMBEDDING_AKTIV, SUB_DUP_KANDIDAT_COS
|
||||
from fsutil import atomic_write_json
|
||||
from jsonio import read_json_file as _json_file
|
||||
from kanban import Flow, Stage
|
||||
@@ -243,6 +243,13 @@ async def _proc_facts(ctx: GenContext, flow: Flow, files: dict, q: dict, folder,
|
||||
if ctx.is_cancelled():
|
||||
return None
|
||||
raw = {bt: subs for bt, subs in raw.items() if subs}
|
||||
# consolidation renames/catalogs can leave consensus subs without grounding — the
|
||||
# guide fact gate then flags their correct statements wholesale (measured: 74/210)
|
||||
await _facts_nachfass(ctx, _pfiles(files, norm), raw, facts_map, q, folder,
|
||||
instructions, ns=f"{_safe(norm)}-",
|
||||
lbl=f"{p.get('title', norm)} · ", sources=p.get("sources"))
|
||||
if ctx.is_cancelled():
|
||||
return None
|
||||
p["raw"], p["facts"] = raw, facts_map
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
await db.kanban_advance(topic, BOARD, norm, "levels")
|
||||
@@ -323,98 +330,110 @@ async def _proc_konsolidierung(ctx: GenContext, flow: Flow, files: dict, instruc
|
||||
def _side(tag: str, ci: int, bt: str, s: str) -> str:
|
||||
return f"{tag}: [Block: {bt}] {s}" + "".join(f"\n - {p}" for p in _kp(ci, bt, s))
|
||||
|
||||
lines = "\n\n".join(
|
||||
f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
|
||||
for k, (i, j) in enumerate(pairs, 1))
|
||||
h = hashlib.md5(lines.encode()).hexdigest()[:8]
|
||||
paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
|
||||
# chunked judging: ONE call over all pairs scaled its timeout past 50 min, and a hung
|
||||
# call blocked the barrier for the full window (measured on aak: 196 pairs, 2×54 min)
|
||||
chunks = [pairs[lo:lo + CROSS_CHUNK_PAARE] for lo in range(0, len(pairs), CROSS_CHUNK_PAARE)]
|
||||
|
||||
async def _judge(j, path):
|
||||
if _cross_schema(_json_file(path)) is not None:
|
||||
return # resume
|
||||
status, _v = await run_single_slot(
|
||||
ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
|
||||
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=lines, extra=_extra(instructions)),
|
||||
role="judge", capabilities="none",
|
||||
payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
|
||||
timeout=_timeout("subblock_check", len(pairs)))
|
||||
if status == FAILED:
|
||||
_log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")
|
||||
async def _urteile_chunk(chunk: list[tuple[int, int]]) -> dict[int, str]:
|
||||
"""Two judges (+ substitute, + tie-breaker) over one pair chunk → {local_k: verdict};
|
||||
empty dict = fail-open (pairs stay)."""
|
||||
lines = "\n\n".join(
|
||||
f"{k}.\n{_side('A', *entries[i])}\n{_side('B', *entries[j])}"
|
||||
for k, (i, j) in enumerate(chunk, 1))
|
||||
h = hashlib.md5(lines.encode()).hexdigest()[:8]
|
||||
paths = [work_dir / f"sub-crossblock-{h}-j{j}.json" for j in (1, 2)]
|
||||
|
||||
await asyncio.gather(*[_judge(j, p) for j, p in zip((1, 2), paths)])
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
|
||||
if len(outs) == 1: # Ersatz-Richter statt fail-open bei EINEM Ausfall
|
||||
ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
|
||||
await _judge("E", ersatz)
|
||||
async def _judge(j, path, plines, n):
|
||||
if _cross_schema(_json_file(path)) is not None:
|
||||
return # resume
|
||||
status, _v = await run_single_slot(
|
||||
ctx, f"Sub-Crossblock j{j}", key=f"blocks-{topic}-sub-crossblock-{h}-j{j}",
|
||||
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=plines, extra=_extra(instructions)),
|
||||
role="judge", capabilities="none",
|
||||
payload=lambda result, p=path: _sink_json(result, p, _cross_schema),
|
||||
timeout=_timeout("subblock_check", n))
|
||||
if status == FAILED:
|
||||
_log(topic, f"Sub-Crossblock j{j} ohne Ergebnis — fail-open")
|
||||
|
||||
await asyncio.gather(*[_judge(j, p, lines, len(chunk)) for j, p in zip((1, 2), paths)])
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
|
||||
journal = {"paare": len(pairs), "richter": len(outs), "gefaltet": [], "verdicts": []}
|
||||
gone: set[int] = set()
|
||||
touched: set[int] = set()
|
||||
if len(outs) == 2:
|
||||
return {}
|
||||
outs = [o for p in paths if (o := _cross_schema(_json_file(p))) is not None]
|
||||
if len(outs) == 1: # Ersatz-Richter statt fail-open bei EINEM Ausfall
|
||||
ersatz = work_dir / f"sub-crossblock-{h}-jE.json"
|
||||
await _judge("E", ersatz, lines, len(chunk))
|
||||
if ctx.is_cancelled():
|
||||
return {}
|
||||
outs = [o for p in [*paths, ersatz] if (o := _cross_schema(_json_file(p))) is not None]
|
||||
if len(outs) != 2:
|
||||
if outs:
|
||||
_log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
|
||||
return {}
|
||||
final = {k: (outs[0].get(k, "nein") if outs[0].get(k, "nein") == outs[1].get(k, "nein")
|
||||
else "uneinig") for k in range(1, len(pairs) + 1)}
|
||||
else "uneinig") for k in range(1, len(chunk) + 1)}
|
||||
disputed = [k for k, v in final.items() if v == "uneinig"]
|
||||
if disputed: # tie-breaker: a third judge sees ONLY the disputed pairs, majority 2/3
|
||||
d_lines = "\n\n".join(
|
||||
f"{x}.\n{_side('A', *entries[pairs[k - 1][0]])}\n{_side('B', *entries[pairs[k - 1][1]])}"
|
||||
f"{x}.\n{_side('A', *entries[chunk[k - 1][0]])}\n{_side('B', *entries[chunk[k - 1][1]])}"
|
||||
for x, k in enumerate(disputed, 1))
|
||||
p3 = work_dir / f"sub-crossblock-{h}-j3.json"
|
||||
if _cross_schema(_json_file(p3)) is None:
|
||||
status, _v = await run_single_slot(
|
||||
ctx, "Sub-Crossblock j3", key=f"blocks-{topic}-sub-crossblock-{h}-j3",
|
||||
prompt=_prompt("Subblock-Crossblock", topic=topic, pairs=d_lines, extra=_extra(instructions)),
|
||||
role="judge", capabilities="none",
|
||||
payload=lambda result, p=p3: _sink_json(result, p, _cross_schema),
|
||||
timeout=_timeout("subblock_check", len(disputed)))
|
||||
if status == FAILED:
|
||||
_log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
|
||||
await _judge(3, p3, d_lines, len(disputed))
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
return {}
|
||||
v3 = _cross_schema(_json_file(p3)) or {}
|
||||
if not v3:
|
||||
_log(topic, "Sub-Crossblock j3 ohne Ergebnis — strittige Paare bleiben")
|
||||
for x, k in enumerate(disputed, 1):
|
||||
t = v3.get(x, "nein")
|
||||
if t in (outs[0].get(k, "nein"), outs[1].get(k, "nein")):
|
||||
final[k] = t # majority 2/3; anything else stays disputed → no fold
|
||||
for k, (i, j) in enumerate(pairs, 1):
|
||||
verdict = final[k]
|
||||
journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
|
||||
"b": f"{entries[j][1]} · {entries[j][2]}",
|
||||
"verdict": verdict})
|
||||
if verdict not in ("a", "b"):
|
||||
continue
|
||||
lose = j if verdict == "a" else i
|
||||
if entries[lose][0] < 0: # context never folds — the package side goes instead
|
||||
lose = i if lose == j else j
|
||||
keep = i if lose == j else j
|
||||
if lose in gone or keep in gone: # keeper already folded → don't chain away the content
|
||||
continue
|
||||
ci, bt, s = entries[lose]
|
||||
p = cards[ci]["payload"]
|
||||
if s in (p.get("raw") or {}).get(bt, []):
|
||||
p["raw"][bt].remove(s)
|
||||
(p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
|
||||
sc = (p.get("sidecar") or {}).get(bt)
|
||||
if isinstance(sc, list): # questions/artefacts consume the sidecar downstream
|
||||
p["sidecar"][bt] = [e for e in sc
|
||||
if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
|
||||
await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
|
||||
gone.add(lose)
|
||||
touched.add(ci)
|
||||
journal["gefaltet"].append({"weg": f"{bt} · {s}",
|
||||
"bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
|
||||
elif outs:
|
||||
_log(topic, "Sub-Crossblock: nur 1/2 Richter — fail-open")
|
||||
return final
|
||||
|
||||
chunk_finals = await asyncio.gather(*[_urteile_chunk(c) for c in chunks])
|
||||
if ctx.is_cancelled():
|
||||
return
|
||||
final_all: dict[int, str] = {} # global pair index (1-based over `pairs`) → verdict
|
||||
for cnr, fin in enumerate(chunk_finals):
|
||||
for k, v in fin.items():
|
||||
final_all[cnr * CROSS_CHUNK_PAARE + k] = v
|
||||
journal = {"paare": len(pairs), "chunks": len(chunks), "gefaltet": [], "verdicts": []}
|
||||
gone: set[int] = set()
|
||||
touched: set[int] = set()
|
||||
for k, (i, j) in enumerate(pairs, 1):
|
||||
verdict = final_all.get(k, "nein")
|
||||
journal["verdicts"].append({"a": f"{entries[i][1]} · {entries[i][2]}",
|
||||
"b": f"{entries[j][1]} · {entries[j][2]}",
|
||||
"verdict": verdict})
|
||||
if verdict not in ("a", "b"):
|
||||
continue
|
||||
lose = j if verdict == "a" else i
|
||||
if entries[lose][0] < 0: # context never folds — the package side goes instead
|
||||
lose = i if lose == j else j
|
||||
keep = i if lose == j else j
|
||||
if lose in gone or keep in gone: # keeper already folded → don't chain away the content
|
||||
continue
|
||||
ci, bt, s = entries[lose]
|
||||
p = cards[ci]["payload"]
|
||||
if s in (p.get("raw") or {}).get(bt, []):
|
||||
p["raw"][bt].remove(s)
|
||||
(p.get("facts") or {}).get(bt, {}).pop(_norm_title(s), None)
|
||||
sc = (p.get("sidecar") or {}).get(bt)
|
||||
if isinstance(sc, list): # questions/artefacts consume the sidecar downstream
|
||||
p["sidecar"][bt] = [e for e in sc
|
||||
if _norm_title(str((e or {}).get("title", ""))) != _norm_title(s)]
|
||||
await db.set_subblock_fields(topic, _norm_title(bt), _norm_title(s), status="variant")
|
||||
gone.add(lose)
|
||||
touched.add(ci)
|
||||
journal["gefaltet"].append({"weg": f"{bt} · {s}",
|
||||
"bleibt": f"{entries[keep][1]} · {entries[keep][2]}"})
|
||||
for ci in touched:
|
||||
p = cards[ci]["payload"]
|
||||
p["raw"] = {bt: subs for bt, subs in (p.get("raw") or {}).items() if subs}
|
||||
await db.kanban_set_payload(topic, BOARD, cards[ci]["card_id"], p)
|
||||
if journal["gefaltet"]:
|
||||
_log(topic, f"Sub-Crossblock: {len(journal['gefaltet'])} blockübergreifende Dublette(n) gefaltet")
|
||||
atomic_write_json(work_dir / f"sub-crossblock-{h}.json", journal, indent=1)
|
||||
hg = hashlib.md5("\n".join(f"{i}:{j}" for i, j in pairs).encode()).hexdigest()[:8]
|
||||
atomic_write_json(work_dir / f"sub-crossblock-{hg}.json", journal, indent=1)
|
||||
await _advance_all()
|
||||
|
||||
|
||||
@@ -433,7 +452,10 @@ async def _proc_levels(ctx: GenContext, flow: Flow, files: dict, instructions: s
|
||||
for btitle, subs in sidecar.items(): # merge facts into the subs (mirror + guide)
|
||||
fm = facts_map.get(btitle, {})
|
||||
for sub in subs:
|
||||
if (fk := fm.get(_norm_title(sub["title"]))):
|
||||
# level agents paraphrase titles — exact miss falls back to the unique
|
||||
# prefix/containment match, else the sub silently loses its grounding
|
||||
sn = _sub_key(set(fm), _norm_title(sub["title"]))
|
||||
if (fk := fm.get(sn)):
|
||||
sub["facts"] = fk
|
||||
p["sidecar"] = sidecar
|
||||
await db.kanban_set_payload(topic, BOARD, norm, p)
|
||||
@@ -535,12 +557,15 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
|
||||
# DB mirrors — per block only (no global deletes)
|
||||
await blocks._mirror_sidecar_db(topic, sidecar)
|
||||
# stale question/artefact rows of a PREVIOUS run keyed to gone subs: finalize only
|
||||
# upserts, so re-runs left orphans (measured: 28). subblocks rows stay — QA needs
|
||||
# the variant/discarded statuses, and the sidecar mirror re-writes only consensus.
|
||||
# upserts, so re-runs left orphans (measured: 28).
|
||||
await db.delete_question_pattern(topic, _norm_title(title))
|
||||
await db.delete_sub_artefakte(topic, _norm_title(title))
|
||||
# stragglers the mirror didn't level (row not in this run's sidecar): without a
|
||||
# valid level they vanish from guide/practice/level views while QA still counts them
|
||||
# consensus rows of a PREVIOUS run that this run's sidecar no longer carries would
|
||||
# linger without facts/questions/artefacts (measured: 25) — drop them per block;
|
||||
# variant/discarded stay for QA. Then default-level the mirror's own stragglers.
|
||||
for btitle, subs in sidecar.items():
|
||||
keep = {_norm_title(str(s.get("title", ""))) for s in subs if isinstance(s, dict)}
|
||||
await db.delete_stale_consensus(topic, _norm_title(btitle), keep - {""})
|
||||
await db.default_subblock_levels(topic, _norm_title(title))
|
||||
sub_keys: dict[str, set[str]] = {}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import shutil
|
||||
import unicodedata
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
@@ -63,7 +64,7 @@ from textkit import _norm_title, _parse_selection, _title, clean_title
|
||||
log = logging.getLogger("creator.board_inventory")
|
||||
|
||||
BOARD = "inventory"
|
||||
RESEARCH_RUNTIME = 900 # one research agent, one round — the tail ingests live while it writes
|
||||
from config import RESEARCH_RUNTIME # noqa: E402 — zentral tunebar
|
||||
_POLL_RESEARCH = 3 # seconds between live reads of a running research file
|
||||
|
||||
_ingest_lock = asyncio.Lock() # serializes the read-modify-write title upserts
|
||||
@@ -84,15 +85,37 @@ def _line(i: int, p: dict, mark: str = "") -> str:
|
||||
return f"{i}. {mark}{p['title']} — {d}" if d else f"{i}. {mark}{p['title']}"
|
||||
|
||||
|
||||
def _naming_schema(data, count: int) -> int | None:
|
||||
"""{"best": N} → 1-based member index in [1, count] · otherwise None."""
|
||||
def _naming_schema(data, count: int) -> tuple[int | None, str | None, bool] | None:
|
||||
"""{"ok":true} → (None,None,True) · {"best":N[,"name":…]} → (N, name|None, False).
|
||||
name über 80 Zeichen wird verworfen (Kürze ist der Zweck der Abstraktion)."""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if data.get("ok") is True:
|
||||
return None, None, True
|
||||
try:
|
||||
n = int(data.get("best"))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return n if 1 <= n <= count else None
|
||||
if not 1 <= n <= count:
|
||||
return None
|
||||
name = str(data.get("name", "")).strip() or None
|
||||
if name and len(name) > 80:
|
||||
name = None
|
||||
return n, name, False
|
||||
|
||||
|
||||
def _name_verankert(name: str, rows: list[dict], ctoks: set[str] | None) -> bool:
|
||||
"""Abstraction guard: a free-formed title must be anchored — in the corpus (uni/projekt)
|
||||
or in the members' own words (thema). Unanchored names drift to textbook canon
|
||||
(measured: 'Königsberger Brückenproblem', 0 corpus hits) → fall back to best-of-members."""
|
||||
if ctoks is not None:
|
||||
return _hat_anker(name, ctoks)
|
||||
from qa import _STOP
|
||||
basis = " ".join(f"{r.get('title', '')} {r.get('description') or ''}" for r in rows)
|
||||
fold = lambda s: unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode().casefold() # noqa: E731
|
||||
btoks = set(re.findall(r"\w{3,}", fold(basis)))
|
||||
ntoks = {t for t in re.findall(r"\w{3,}", fold(name)) if t not in _STOP}
|
||||
return bool(ntoks) and ntoks <= btoks
|
||||
|
||||
|
||||
# ── Embedding cache (per flow) ─────────────────────────────────────────────────────
|
||||
@@ -450,15 +473,25 @@ def _hat_anker(title: str, ctoks: set[str]) -> bool:
|
||||
async def _anker_beleg(ctx: GenContext, flow: Flow, kandidaten: list[tuple[str, dict]]) -> set[str]:
|
||||
"""Evidence judge for quorum titles WITHOUT any corpus anchor — two readers naming the
|
||||
same famous canon independently beat the quorum although the material never mentions it
|
||||
(measured: 'Königsberger Brückenproblem', 0 corpus hits). FAIL-OPEN: the titles carry
|
||||
2-reader backing, only an explicit 'nein' rejects. → card_ids to reject."""
|
||||
(measured: 'Königsberger Brückenproblem', 0 corpus hits). FAIL-OPEN for judge errors —
|
||||
but an EMPTY evidence pack (no distinctive token anywhere in the corpus) rejects
|
||||
deterministically: the judge approved 3 canon titles on sham excerpts. → card_ids to reject."""
|
||||
from qa import _distinctive
|
||||
topic = flow.topic
|
||||
folder = source_folder(topic)
|
||||
lines = []
|
||||
hart_nein: set[str] = set()
|
||||
for k, (cid, p) in enumerate(kandidaten, 1):
|
||||
ev = _evidence_pack(folder, None, [p.get("title", ""), p.get("description") or ""], budget=4000)
|
||||
# distinctive tokens only: title+description as raw queries pulled generic sections
|
||||
# („Beispiel", „klassisch") — the judge saw sham excerpts and waved canon through
|
||||
toks = _distinctive(p.get("title", "")) | _distinctive(p.get("description") or "")
|
||||
ev = _evidence_pack(folder, None, [" ".join(sorted(toks))], budget=4000) if toks else ""
|
||||
if not ev: # nothing to attest — deterministic reject, no judge to sweet-talk
|
||||
hart_nein.add(cid)
|
||||
lines.append(f"{k}. {p.get('title', '')} — {p.get('description') or ''}\nAUSZÜGE:\n"
|
||||
f"{ev or '(keine passenden Auszüge im Material gefunden)'}")
|
||||
if len(hart_nein) == len(kandidaten):
|
||||
return hart_nein
|
||||
h = _h(*[cid for cid, _ in kandidaten])
|
||||
path = flow.work_dir / f"anker-beleg-{h}.json"
|
||||
ids = set(range(1, len(kandidaten) + 1))
|
||||
@@ -473,7 +506,8 @@ async def _anker_beleg(ctx: GenContext, flow: Flow, kandidaten: list[tuple[str,
|
||||
timeout=_timeout("selection_mapping", len(kandidaten)))
|
||||
if status != OK or not isinstance(verdict, dict):
|
||||
verdict = {} # fail-open
|
||||
return {cid for k, (cid, _p) in enumerate(kandidaten, 1) if str(verdict.get(k, "ja")) == "nein"}
|
||||
return hart_nein | {cid for k, (cid, _p) in enumerate(kandidaten, 1)
|
||||
if str(verdict.get(k, "ja")) == "nein"}
|
||||
|
||||
|
||||
async def _proc_consensus_gate(ctx: GenContext, flow: Flow, cards):
|
||||
@@ -608,32 +642,42 @@ async def _proc_clarify(ctx: GenContext, flow: Flow, cards):
|
||||
|
||||
|
||||
async def _choose_title(ctx: GenContext, flow: Flow, cid: str, rows: list[dict],
|
||||
template: str, current: int | None = None) -> str:
|
||||
"""Judge picks the best member title (index). Parse-fail → survivorship fallback."""
|
||||
template: str, current_title: str | None = None) -> tuple[str, str | None]:
|
||||
"""Judge picks the best member title and MAY propose a short abstracted name — kept
|
||||
only when anchored (_name_verankert). → (member_norm, custom_title|None);
|
||||
("", None) on cancel; {"ok":true} in the check keeps the current title."""
|
||||
topic = flow.topic
|
||||
h = _h(*[r["norm"] for r in rows], template)
|
||||
path = flow.work_dir / f"naming-{cid}-{h}.json"
|
||||
best = _naming_schema(_json_file(path), len(rows))
|
||||
if best is None:
|
||||
verdict = _naming_schema(_json_file(path), len(rows))
|
||||
if verdict is None:
|
||||
lines = "\n".join(f"{k + 1}. {_t_text(r)}" for k, r in enumerate(rows))
|
||||
kw = dict(topic=topic, members=lines, out_path=path)
|
||||
if current is not None:
|
||||
kw["current"] = current
|
||||
status, best = await run_single_slot(
|
||||
kw = dict(topic=topic, members=lines)
|
||||
if current_title is not None:
|
||||
kw["current_title"] = current_title
|
||||
status, verdict = await run_single_slot(
|
||||
ctx, f"Naming {cid}", key=f"blocks-{topic}-naming-{cid}-{h}",
|
||||
prompt=_prompt(template, **kw), role="judge", capabilities="files",
|
||||
payload=lambda result, p=path, n=len(rows): _naming_schema(_json_file(p), n),
|
||||
prompt=_prompt(template, **kw), role="judge", capabilities="none",
|
||||
payload=lambda result, p=path, n=len(rows): _sink_json(result, p, lambda d: _naming_schema(d, n)),
|
||||
timeout=_timeout("selection_mapping", len(rows)))
|
||||
if status == CANCELLED:
|
||||
return ""
|
||||
return "", None
|
||||
if status == FAILED:
|
||||
best = None
|
||||
if best is None:
|
||||
verdict = None
|
||||
if verdict is None:
|
||||
rep = _rep(rows)
|
||||
w = _norm_title(rep["title"])
|
||||
norms = [r["norm"] for r in rows]
|
||||
return w if w in norms else norms[0]
|
||||
return rows[best - 1]["norm"]
|
||||
return (w if w in norms else norms[0]), None
|
||||
best, name, ok = verdict
|
||||
if ok: # check judge confirms the current (possibly custom) title: change nothing
|
||||
return "", (current_title or "")
|
||||
custom = None
|
||||
if name:
|
||||
ctoks = flow.state.get("korpus_tokens")
|
||||
if _name_verankert(name, rows, ctoks):
|
||||
custom = clean_title(name)
|
||||
return rows[best - 1]["norm"], custom
|
||||
|
||||
|
||||
async def _proc_naming(ctx: GenContext, flow: Flow, cards):
|
||||
@@ -650,13 +694,15 @@ async def _name_one(ctx: GenContext, flow: Flow, c):
|
||||
p = c["payload"]
|
||||
rows = await _member_rows(topic, cid)
|
||||
if len(rows) > 1 and not p.get("renamed"):
|
||||
winner = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming")
|
||||
if not winner: # cancelled
|
||||
winner, custom = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming")
|
||||
if not winner and custom is None: # cancelled
|
||||
return
|
||||
p["main_norm"] = winner
|
||||
w = next(r for r in rows if r["norm"] == winner)
|
||||
p["title"], p["description"] = w["title"], w.get("description") or p.get("description", "")
|
||||
await db.kanban_set_payload(topic, BOARD, cid, p)
|
||||
if winner:
|
||||
p["main_norm"] = winner
|
||||
w = next(r for r in rows if r["norm"] == winner)
|
||||
p["title"] = custom or w["title"]
|
||||
p["description"] = w.get("description") or p.get("description", "")
|
||||
await db.kanban_set_payload(topic, BOARD, cid, p)
|
||||
await db.kanban_advance(topic, BOARD, cid, "naming_check")
|
||||
|
||||
|
||||
@@ -675,14 +721,14 @@ async def _namecheck_one(ctx: GenContext, flow: Flow, c):
|
||||
p = c["payload"]
|
||||
rows = await _member_rows(topic, cid)
|
||||
if len(rows) > 1 and not p.get("renamed"):
|
||||
norms = [r["norm"] for r in rows]
|
||||
cur = p.get("main_norm")
|
||||
current = norms.index(cur) + 1 if cur in norms else 1
|
||||
winner = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming-Check", current=current)
|
||||
if not winner:
|
||||
winner, custom = await _choose_title(ctx, flow, cid, rows, "Blocks-Naming-Check",
|
||||
current_title=p.get("title", ""))
|
||||
if not winner and custom is None: # cancelled
|
||||
return
|
||||
w = next(r for r in rows if r["norm"] == winner)
|
||||
p["title"], p["description"] = w["title"], w.get("description") or p.get("description", "")
|
||||
if winner: # ok-verdict ("" + custom) keeps title AND description untouched
|
||||
w = next(r for r in rows if r["norm"] == winner)
|
||||
p["title"] = custom or w["title"]
|
||||
p["description"] = w.get("description") or p.get("description", "")
|
||||
readers = sorted(set().union(*[set(r.get("readers") or []) for r in rows])) if rows else []
|
||||
sources = sorted(set().union(*[set(r.get("sources") or []) for r in rows])) if rows else []
|
||||
await db.kanban_upsert_card(topic, BOARD, f"b-{cid}", "block", "fragment_filter", {
|
||||
@@ -1872,6 +1918,13 @@ async def reset_board_from_stage(topic: str, board: str, stage: str, files: dict
|
||||
await db.delete_subblocks(topic)
|
||||
await db.delete_question_pattern(topic)
|
||||
await db.delete_sub_artefakte(topic)
|
||||
# global sidecar files and per-block resume slots: leftovers of the previous
|
||||
# derive would re-merge/resume into the fresh run — nothing here is reused
|
||||
for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"):
|
||||
files[k].unlink(missing_ok=True)
|
||||
for d in files["arbeit"].glob("ab-*"):
|
||||
if d.is_dir():
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
await db.add_event(topic, "reset", key=f"{board}:from-{stage}", status=str(moved))
|
||||
return moved
|
||||
|
||||
|
||||
@@ -75,6 +75,9 @@ SEED_COVER_COS = 0.80
|
||||
# (Markdown: 50 pairs in the band, 4 above) — below every auto-merge threshold, so an LLM
|
||||
# judge decides. Candidates only; a merge still needs judge unanimity.
|
||||
SUB_DUP_KANDIDAT_COS = 0.75
|
||||
# Cross-block judge pairs per call: ONE call over all pairs scaled its timeout to 54 min
|
||||
# and a hung call blocked the barrier that long (aak: 196 pairs) — chunks cap it at ~15 min.
|
||||
CROSS_CHUNK_PAARE = 40
|
||||
|
||||
# Umbrella grouping (block granularity level 2, step "Blocks-Gruppierung", AFTER the filter):
|
||||
# collapse sibling DEFINITIONS that are components of ONE umbrella concept (TM model:
|
||||
@@ -144,12 +147,56 @@ QUELLE_RELEVANZ_SNIPPET = 800 # body characters per page in the prompt (URL i
|
||||
QA_GATE_NOTE = 9.5 # 0 = gate off; quota-based, so the tolerated finding count scales with topic size
|
||||
QA_GATE_LLM = True # include the LLM samples (Echtheit/Dubletten) in the gate run
|
||||
|
||||
# Guide section length per relevant sub (ausführlich part) — QA detector AND the
|
||||
# deterministic readability-stage trigger share these bounds (writers overshot 2.7–4.1×).
|
||||
GUIDE_LAENGE_MIN = 150
|
||||
GUIDE_LAENGE_MAX = 1200
|
||||
|
||||
# Inline evidence for judge agents: corpus excerpts go INTO the prompt instead of letting
|
||||
# every judge re-search the source folder (measured: ~10 tool turns/judge, 82 % of the
|
||||
# run's tokens were cache reads from those loops).
|
||||
EVIDENCE_BUDGET_CHARS = 48_000 # max excerpt characters per judge prompt
|
||||
EVIDENCE_CTX_LINES = 15 # context lines around a cited source position (facts check)
|
||||
|
||||
# ── Pipeline tuning (zentral, tunebar via CREATOR_PARAMS — siehe Override-Hook am Datei-Ende;
|
||||
# Registry mit Suchraum: backend/train_params.py). QA-/Detektor-Konstanten bleiben bewusst in
|
||||
# qa.py/guide_qa.py — die Messlatte darf nie Teil des Suchraums sein. ─────────────────────────
|
||||
SUBBLOCK_CHUNK = 10 # subblock finder: 1 agent per ~10 blocks, capped
|
||||
SUBBLOCK_MAX = 40 # chunk cap
|
||||
LEVEL_CHUNK = 100 # classifying is cheap → large packages
|
||||
RESEARCH_BATCH = 20 # crawl pages per batch
|
||||
RESEARCH_READERS = 2 # reader agents per batch/section (consensus ≥2)
|
||||
RESEARCH_THEMA_AGENTS = 5 # web mode (source "thema")
|
||||
RESEARCH_SECTION_CHARS = 12000 # uni/projekt section size (lost-in-the-middle guard)
|
||||
RESEARCH_RUNTIME = 900 # one research agent, one round (tail ingests live)
|
||||
SUBBLOCK_CAP = 900 # subblock find loop per chunk (seconds)
|
||||
SUBBLOCK_MIN = 5 # below this consensus count → focused catch-up rounds
|
||||
SUBBLOCK_EXTRA_ROUNDS = 2 # max catch-up rounds
|
||||
SUBBLOCK_MAX_ROUNDS = 3 # hard round cap (rounds 4–5 burned 29 % of finders for ~0 gain)
|
||||
CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (fallback path)
|
||||
DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair
|
||||
DEDUP_PAIRS_CHUNK = 40 # pairs per judge package
|
||||
DEDUP_TITLE_AUTO = 0.95 # near-identical TITLE cosine → merge without judge
|
||||
DEDUP_GLOBAL_FLOOR = 0.65 # global post-naming dedup candidate floor
|
||||
FILTER_CHUNK = 35 # blocks per judge in the degrade pass
|
||||
QUESTION_CHUNK_SUBS = 25 # target relevant subs per question chunk (LPT)
|
||||
QUESTION_MAX_ROUNDS = 3 # catch-up rounds for subs without a pattern
|
||||
FACTS_CHUNK_SUBS = 10 # facts extraction chunk (chunk count = parallelism)
|
||||
ARTEFACT_CHUNK_SUBS = 25 # flashcards/examples bulk chunk
|
||||
FACTS_CHECK_PANEL = 3 # judges per facts-check chunk (majority)
|
||||
CONSOLIDATION_PANEL = 3 # mapping judges per chunk
|
||||
SUBBLOCK_PANEL = 3 # judges in the subblock clarification
|
||||
FILTER_RECHECK_PANEL = 3 # judges in the survivor-recheck
|
||||
MAX_WRITER_ROUNDS = 2 # guide coverage→writer loop cap
|
||||
GATE_FIX_MIN = 3 # fact-gate: unbelegt-claims below this → log only (falsch fixt immer)
|
||||
WRITER_SPLIT_SUBS = 30 # guide writer splits sections above this sub count
|
||||
KANBAN_BATCH = 5 # cards a worker pulls per micro-batch
|
||||
MAX_CARD_RETRIES = 3 # failures per card → dead-letter
|
||||
RETRY_BACKOFF = 30.0 # base seconds; backoff = base · 2^(retries-1)
|
||||
MAX_RESTARTS = 2 # agent restart cap per race slot
|
||||
JUDGE_CHUNK = 40 # repair: findings per judge call
|
||||
EVIDENCE_PER_BLOCK = 6000 # repair: excerpt chars per fremd candidate
|
||||
|
||||
# Timeouts per agent step: (base seconds, seconds per block/section).
|
||||
# Applies equally to all providers — whoever is too slow gets restarted or overtaken.
|
||||
TIMEOUTS = {
|
||||
@@ -160,17 +207,18 @@ TIMEOUTS = {
|
||||
"plan": (300, 5),
|
||||
"plan_judge": (600, 5), # judge reads up to 5 outlines, n = sections
|
||||
"content": (450, 30), # facts find/erg/fix — p95 measured 241 s (was 600+90n)
|
||||
"content_check": (300, 10), # content exam per block in the package
|
||||
# Judge caps tightened 2026-07-04: judge p50 is 6–72 s; a stalled call burns the whole
|
||||
# cap and its retry heals in seconds — the old 300 s base tripled the stall cost.
|
||||
"content_check": (150, 8), # content exam per block in the package
|
||||
"subblock": (400, 15), # finder round — p95 measured 124 s (was 900+45n)
|
||||
"subblock_check": (300, 15), # judge decides contested subblocks in the chunk
|
||||
"konsolidierung": (600, 25), # consolidation judge sees ALL subs with key points — 585 s
|
||||
# (subblock_check at n=19) produced 15 % timeouts
|
||||
"subblock_check": (150, 10), # judge decides contested subblocks in the chunk
|
||||
"konsolidierung": (300, 20), # consolidation judge sees ALL subs with key points
|
||||
"level": (300, 10), # classify subblocks per chunk
|
||||
"level_check": (300, 10), # judge decides contested levels in the chunk
|
||||
"level_check": (150, 8), # judge decides contested levels in the chunk
|
||||
"relevance": (300, 10), # subblocks relevant/peripheral per chunk
|
||||
"relevance_check": (300, 10), # judge decides contested relevance in the chunk
|
||||
"relevance_check": (150, 8), # judge decides contested relevance in the chunk
|
||||
"question_pattern": (300, 15), # question patterns per block (subblocks × types)
|
||||
"question_pattern_check": (300, 10), # critic cleans up the pattern table per block
|
||||
"question_pattern_check": (150, 8), # critic cleans up the pattern table per block
|
||||
"writer": (450, 60), # per section — split keeps sections ≤30 subs
|
||||
"lese_check": (300, 10), # per section in the package
|
||||
# guide board (per card = one block)
|
||||
@@ -209,7 +257,8 @@ PROVIDERS = {
|
||||
"cli": "opencode",
|
||||
"guide": "minimax/MiniMax-M3",
|
||||
"fast": "minimax-kalt/MiniMax-M2.7-highspeed",
|
||||
"judge": "minimax-kalt/MiniMax-M3",
|
||||
"judge": "minimax/MiniMax-M3", # native route — the kalt endpoint stalled 20 % of
|
||||
# judge calls to the timeout cap (516/2590, 2026-07-04)
|
||||
"quick": "minimax/MiniMax-M2.7-highspeed",
|
||||
"env_key": "MINIMAX_API_KEY",
|
||||
},
|
||||
@@ -246,3 +295,33 @@ def resolve_role(run_provider: str, role: str) -> tuple[str, str]:
|
||||
if not model:
|
||||
model = PROVIDERS.get(provider, {}).get(role, "")
|
||||
return provider, model
|
||||
|
||||
|
||||
# ── Trainings-Override: CREATOR_PARAMS (JSON-Dict im ENV) überschreibt gleichnamige
|
||||
# Tuning-Konstanten oben — pro Prozess-Start (der Trainer startet je Trial einen Subprozess;
|
||||
# Module binden die Werte beim Import). TIMEOUTS-Einträge via "TIMEOUT_<step>_base"/"_per".
|
||||
def _apply_param_overrides() -> None:
|
||||
raw = os.getenv("CREATOR_PARAMS")
|
||||
if not raw:
|
||||
return
|
||||
import json as _json
|
||||
try:
|
||||
overrides = _json.loads(raw)
|
||||
except ValueError:
|
||||
raise SystemExit(f"CREATOR_PARAMS ist kein gültiges JSON: {raw[:80]}")
|
||||
g = globals()
|
||||
for key, val in overrides.items():
|
||||
if key.startswith("TIMEOUT_"):
|
||||
rest = key[len("TIMEOUT_"):]
|
||||
step, _, part = rest.rpartition("_")
|
||||
if step in TIMEOUTS and part in ("base", "per"):
|
||||
base, per = TIMEOUTS[step]
|
||||
TIMEOUTS[step] = (val, per) if part == "base" else (base, val)
|
||||
continue
|
||||
raise SystemExit(f"CREATOR_PARAMS: unbekannter Timeout-Schlüssel {key}")
|
||||
if key not in g or not isinstance(g[key], (int, float)) or isinstance(g[key], bool):
|
||||
raise SystemExit(f"CREATOR_PARAMS: unbekannter/nicht-numerischer Parameter {key}")
|
||||
g[key] = type(g[key])(val)
|
||||
|
||||
|
||||
_apply_param_overrides()
|
||||
|
||||
@@ -1186,6 +1186,23 @@ async def default_subblock_levels(topic: str, block_norm: str) -> None:
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def delete_stale_consensus(topic: str, block_norm: str, keep: set[str]) -> None:
|
||||
"""Drop consensus rows of a block that are NOT in this run's sidecar (`keep`): finalize
|
||||
only upserts, so re-runs piled up orphan rows (measured: 25 subs without any board-2
|
||||
output). variant/discarded rows stay — QA reads those statuses."""
|
||||
db = await get_db()
|
||||
cursor = await db.execute(
|
||||
"SELECT sub_norm FROM subblocks WHERE topic = ? AND block_norm = ? AND status = 'consensus'",
|
||||
(topic, block_norm))
|
||||
rows = await cursor.fetchall()
|
||||
stale = [r[0] for r in rows if r[0] not in keep]
|
||||
for sn in stale:
|
||||
await db.execute("DELETE FROM subblocks WHERE topic = ? AND block_norm = ? AND sub_norm = ?",
|
||||
(topic, block_norm, sn))
|
||||
if stale:
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def set_subblock_fields(topic: str, block_norm: str, sub_norm: str, **fields) -> None:
|
||||
"""Set fields (level/relevance/status/sub_title) of a subblock row."""
|
||||
fields["updated_at"] = _now()
|
||||
|
||||
289
backend/fake_agents.py
Normal file
289
backend/fake_agents.py
Normal file
@@ -0,0 +1,289 @@
|
||||
"""Deterministischer Agenten-Ersatz für E2E-Tests: beantwortet run_agent-Aufrufe ohne LLM.
|
||||
|
||||
Eine `Welt` beschreibt das Thema (Blöcke → Subs → Facts …); `respond()` routet per
|
||||
agent_key-Muster und liefert (rc, stdout, stderr) wie ein echter Agent — files-Agenten
|
||||
schreiben die out_path-Datei aus dem Prompt, none-Agenten antworten als Text (der
|
||||
Engine-Sink parst/persistiert). Damit laufen ALLE echten Schichten (_race, Quorum,
|
||||
Retry, Panels, Producer, QA-Gate) in Sekunden.
|
||||
|
||||
Störungen sind deterministisch: `Welt.stoerungen` matcht agent_keys per Regex und
|
||||
liefert n-mal einen Fehler / kaputtes JSON / eine feste Antwort (z. B. Dissens).
|
||||
|
||||
Aktivierung: pytest-Fixture `fake_welt` (tests/conftest.py) oder ENV CREATOR_FAKE_AGENTS=1
|
||||
(echter Server, Sekunden-Smoke im Frontend).
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
_PATH_RE = re.compile(r"(/\S+\.(?:json|md))")
|
||||
_NUM_RE = re.compile(r"^\s*(\d+)[.)]\s+(.*\S)", re.MULTILINE)
|
||||
_SUBLIST_RE = re.compile(r"^- (?:\[(\w+)\] )?(.+\S)\s*$", re.MULTILINE)
|
||||
_ZIEL_RE = re.compile(r"\(([a-z]\d+)\)")
|
||||
_PAIR_RE = re.compile(r"^(\d+)\.\s*\nA: \[Block: (.*?)\] (.*?)\n", re.MULTILINE)
|
||||
|
||||
|
||||
def _norm(s: str) -> str:
|
||||
return " ".join((s or "").casefold().split())
|
||||
|
||||
|
||||
class Welt:
|
||||
"""Deterministisches Themen-Modell. bloecke: {titel: {"beschreibung": str,
|
||||
"subs": [titel]}}; optionale Regeln steuern Konsolidierung/Cross-Block."""
|
||||
|
||||
def __init__(self, bloecke: dict | None = None, *, gruppen: list | None = None,
|
||||
kataloge: list | None = None, stoerungen: list | None = None):
|
||||
self.bloecke = bloecke if bloecke is not None else standard_bloecke()
|
||||
self.gruppen = gruppen or [] # [(haupt_titel, [weitere_titel])] → In-Block-Fold
|
||||
self.kataloge = kataloge or [] # [(katalog_titel, [mitglieder_titel])]
|
||||
self.stoerungen = [dict(s, rest=int(s.get("mal", 1))) for s in (stoerungen or [])]
|
||||
self.calls: list[str] = [] # Auditspur: jeder agent_key in Reihenfolge
|
||||
|
||||
# ── Nachschlagen ────────────────────────────────────────────────────────────────
|
||||
def _alle_subs(self) -> dict[str, str]:
|
||||
"""sub_norm → sub_titel über alle Blöcke (inkl. Katalog-Titel)."""
|
||||
out = {}
|
||||
for b in self.bloecke.values():
|
||||
for s in b["subs"]:
|
||||
out[_norm(s)] = s
|
||||
for kt, _m in self.kataloge:
|
||||
out[_norm(kt)] = kt
|
||||
return out
|
||||
|
||||
def _bloecke_im_prompt(self, prompt: str) -> list[str]:
|
||||
return [t for t in self.bloecke if t in prompt]
|
||||
|
||||
def _subs_im_prompt(self, prompt: str) -> list[str]:
|
||||
gefunden = [s for s in self._alle_subs().values() if s in prompt]
|
||||
return gefunden
|
||||
|
||||
# ── Störungen ───────────────────────────────────────────────────────────────────
|
||||
def _stoerung(self, agent_key: str):
|
||||
for s in self.stoerungen:
|
||||
if s["rest"] > 0 and re.search(s["muster"], agent_key):
|
||||
s["rest"] -= 1
|
||||
return s
|
||||
return None
|
||||
|
||||
# ── Haupteinstieg ──────────────────────────────────────────────────────────────
|
||||
def respond(self, agent_key: str, prompt: str, capabilities: str) -> tuple[int, str, str]:
|
||||
self.calls.append(agent_key)
|
||||
if (s := self._stoerung(agent_key)):
|
||||
if s["modus"] == "fehler":
|
||||
return 1, "", "fake-stoerung"
|
||||
if s["modus"] == "garbage":
|
||||
return self._liefern(prompt, '{"kaputt": ')
|
||||
if s["modus"] == "antwort":
|
||||
return self._liefern(prompt, s["antwort"])
|
||||
text = self._antwort(agent_key, prompt)
|
||||
if text is None:
|
||||
return 1, "", f"fake: kein Handler für {agent_key}"
|
||||
return self._liefern(prompt, text)
|
||||
|
||||
@staticmethod
|
||||
def _liefern(prompt: str, text: str) -> tuple[int, str, str]:
|
||||
"""files-Agenten schreiben die out_path-Datei aus dem Prompt; none-Agenten
|
||||
antworten als Text. Wir tun einfach BEIDES — steht ein Pfad im Prompt, wird
|
||||
er geschrieben (dann liest das payload die Datei), und stdout trägt den Text
|
||||
(dann parst ihn der Sink). Ein Weg von beiden greift immer."""
|
||||
if (m := _PATH_RE.search(prompt)):
|
||||
p = Path(m.group(1))
|
||||
try:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(text, encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
return 0, text, ""
|
||||
|
||||
# ── Antwort-Generatoren je Key-Muster ──────────────────────────────────────────
|
||||
def _antwort(self, key: str, prompt: str) -> str | None: # noqa: C901 — bewusst ein Router
|
||||
j = json.dumps
|
||||
# Board 1 / Inventar
|
||||
if "-research-" in key:
|
||||
zeilen = []
|
||||
n = 1
|
||||
for t, b in self.bloecke.items():
|
||||
zeilen.append(f"{n}. {t} — {b['beschreibung']}")
|
||||
n += 1
|
||||
return "\n".join(zeilen)
|
||||
if "-pair-" in key:
|
||||
n = prompt.count("\nA: ") or 1
|
||||
return j({"pairs": {str(i): "ja" for i in range(1, n + 1)}})
|
||||
if "-dedup-" in key:
|
||||
n = prompt.count("\nA: ") or 1
|
||||
return j({"pairs": {str(i): "nein" for i in range(1, n + 1)}})
|
||||
if "-clarify-" in key:
|
||||
keep = [ln[2:] for ln in prompt.splitlines() if ln.startswith("- ")]
|
||||
return j({"keep": keep, "rest": []})
|
||||
if "-naming-" in key: # deckt auch naming_check (gleicher Key)
|
||||
return j({"best": 1})
|
||||
if "-filter-" in key: # auch filter-recheck
|
||||
return j({"fragments": {}, "drop": []})
|
||||
if "-gruppierung-completion-" in key:
|
||||
return j({"additions": []})
|
||||
if "-gruppierung-" in key:
|
||||
return j({"umbrellas": []})
|
||||
if "-supplement-beleg" in key or "-anker-beleg-" in key:
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevant": {k: "ja" for k in sorted(nums, key=int)} or {"1": "ja"}})
|
||||
if "-supplement" in key:
|
||||
return j({"blocks": []})
|
||||
if "-source-relevance-" in key:
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevant": {k: "ja" for k in sorted(nums, key=int)} or {"1": "ja"}})
|
||||
|
||||
# Board 2 / Artefakte
|
||||
if "-luecken-" in key:
|
||||
return "<!-- block: leer -->\n" # Lücken-Nachfass findet nichts Neues
|
||||
if "-subblock-final-" in key or "-subblock-" in key: # Finder + Judge, gleiches Format
|
||||
teile = []
|
||||
for t in self._bloecke_im_prompt(prompt):
|
||||
subs = "\n".join(f"- {s}" for s in self.bloecke[t]["subs"])
|
||||
teile.append(f"<!-- block: {t} -->\n{subs}")
|
||||
return "\n".join(teile) or "<!-- block: leer -->\n"
|
||||
if "-sub-konsolidierung-" in key:
|
||||
nummern = {_norm(m.group(2)): m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
gruppen = []
|
||||
for haupt, weitere in self.gruppen:
|
||||
h, w = nummern.get(_norm(haupt)), [nummern[_norm(x)] for x in weitere
|
||||
if _norm(x) in nummern]
|
||||
if h and w:
|
||||
gruppen.append({"haupt": int(h), "weitere": [int(x) for x in w]})
|
||||
kataloge = []
|
||||
for kt, mitglieder in self.kataloge:
|
||||
m = [int(nummern[_norm(x)]) for x in mitglieder if _norm(x) in nummern]
|
||||
if len(m) >= 2:
|
||||
kataloge.append({"titel": kt, "mitglieder": m})
|
||||
return j({"gruppen": gruppen, "kataloge": kataloge, "fremd": [], "luecken": []})
|
||||
if "-sub-crossblock-" in key:
|
||||
urteile = {}
|
||||
for m in _PAIR_RE.finditer(prompt):
|
||||
urteile[m.group(1)] = "a" # identischer Text (nur so wird gepaart) → A behält
|
||||
return j({"pairs": urteile or {"1": "nein"}})
|
||||
if "-facts-check-" in key:
|
||||
return j({"ok": True})
|
||||
if "-facts-" in key: # facts / facts-fix / facts-erg: gleiches Format
|
||||
eintraege = []
|
||||
for t in self._bloecke_im_prompt(prompt):
|
||||
for s in self.bloecke[t]["subs"]:
|
||||
if s in prompt:
|
||||
eintraege.append(self._fakt(t, s))
|
||||
for kt, _m in self.kataloge:
|
||||
if kt in prompt:
|
||||
eintraege.append(self._fakt(t, kt))
|
||||
if not eintraege: # Nachfass-Fälle: Subs ohne Blockkontext im Prompt
|
||||
eintraege = [self._fakt(bt, s) for bt, b in self.bloecke.items()
|
||||
for s in b["subs"] if s in prompt]
|
||||
return j({"facts": eintraege})
|
||||
if "-level-" in key: # Rater + final: alle geforderten Nummern
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"levels": {k: "beginner" for k in sorted(nums, key=int)} or {"1": "beginner"}})
|
||||
if "-relevance-" in key:
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevance": {k: "relevant" for k in sorted(nums, key=int)} or {"1": "relevant"}})
|
||||
if "-question-pattern-" in key:
|
||||
eintraege = []
|
||||
for t in self._bloecke_im_prompt(prompt):
|
||||
for s in self._subs_im_prompt(prompt):
|
||||
eintraege.append({"block": t, "subblock": s, "question": f"Was ist {s}?"})
|
||||
return j({"pattern": eintraege})
|
||||
if "-artifact-example-check-" in key:
|
||||
return j({"ok": True})
|
||||
if "-artifact-flashcard-" in key:
|
||||
karten = [{"block": t, "subblock": s, "question": f"F: {s}?", "answer": f"A: {s}"}
|
||||
for t in self._bloecke_im_prompt(prompt) for s in self._subs_im_prompt(prompt)]
|
||||
return j({"cards": karten})
|
||||
if "-artifact-example-" in key:
|
||||
bsp = [{"block": t, "subblock": s, "problem": f"Aufgabe zu {s}",
|
||||
"steps": ["Schritt 1", "Schritt 2"], "result": "Ergebnis"}
|
||||
for t in self._bloecke_im_prompt(prompt) for s in self._subs_im_prompt(prompt)]
|
||||
return j({"examples": bsp})
|
||||
if "-outline-prereqs" in key:
|
||||
return j({"prereqs": {}})
|
||||
if "-outline-review" in key:
|
||||
return j({"moves": {}})
|
||||
if "-outline-" in key or key.endswith("-outline-judge"):
|
||||
nums = sorted({int(m.group(1)) for m in _NUM_RE.finditer(prompt)}) or [1]
|
||||
return j({"chapters": [{"title": "Kapitel 1", "numbers": nums}]})
|
||||
|
||||
# Guide-Board
|
||||
if "-ziele-" in key:
|
||||
ziele = [{"id": f"z{i}", "text": f"Verstehen von {s}", "sub": s}
|
||||
for i, s in enumerate(self._subs_im_prompt(prompt), 1)][:8]
|
||||
return j({"ziele": ziele or [{"id": "z1", "text": "Grundlagen verstehen", "sub": ""}]})
|
||||
if "-gatefix-" in key or "-lesefix-" in key:
|
||||
return self._section_aus_prompt(prompt) or "<!-- section: X -->\nRepariert."
|
||||
if "-gate-" in key:
|
||||
return j({"ok": True})
|
||||
if "-cov-" in key:
|
||||
ids = sorted(set(_ZIEL_RE.findall(prompt)))
|
||||
return j({"ziele": {z: True for z in ids}, "luecken": [], "ballast": []})
|
||||
if "-lese-" in key:
|
||||
return j({"ok": True})
|
||||
if "-w-" in key:
|
||||
return self._writer_md(prompt)
|
||||
|
||||
# QA / Repair / Guide-QA (alle Text, _yesno_schema)
|
||||
if key.startswith(("qa-guide-",)):
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevant": {k: "nein" for k in sorted(nums, key=int)} or {"1": "nein"}})
|
||||
if key.startswith(("qa-", "repair-")):
|
||||
# Semantik je Template: Bausteine „ja" = echt; Dubletten/Lücken „nein" = kein Befund
|
||||
wert = "ja" if ("bausteine" in key or "beleg" in key or "fremd" in key) else "nein"
|
||||
nums = {m.group(1) for m in _NUM_RE.finditer(prompt)}
|
||||
return j({"relevant": {k: wert for k in sorted(nums, key=int)} or {"1": wert}})
|
||||
return None
|
||||
|
||||
# ── Bausteine der Antworten ─────────────────────────────────────────────────────
|
||||
@staticmethod
|
||||
def _fakt(block: str, sub: str) -> dict:
|
||||
return {"block": block, "subblock": sub,
|
||||
"key_points": [f"Kernaussage zu {sub}", f"Zweite Aussage zu {sub}"],
|
||||
"prerequisites": "", "hurdles": "",
|
||||
"cited_facts": [{"text": f"Beleg für {sub}", "source": "Fake-Quelle"}],
|
||||
"example_idea": f"Beispiel zu {sub}"}
|
||||
|
||||
def _writer_md(self, prompt: str) -> str:
|
||||
"""Section im Marker-Format; Subs aus der SUBBLOCKS-Liste des Prompts
|
||||
(`- [label] titel`), Länge je Sub im 150–1080-Rahmen."""
|
||||
block = next(iter(self._bloecke_im_prompt(prompt)), None) or "Abschnitt"
|
||||
subs = [(lv or "beginner", t) for lv, t in _SUBLIST_RE.findall(prompt)
|
||||
if _norm(t) in self._alle_subs()]
|
||||
if not subs:
|
||||
subs = [("beginner", s) for s in self.bloecke.get(block, {}).get("subs", ["Inhalt"])]
|
||||
kompakt = "\n".join(f"<!-- sub: {lv} | {t} -->\n- Merksatz zu {t}" for lv, t in subs)
|
||||
prosa = "\n".join(f"<!-- sub: {lv} | {t} -->\n" + (f"Lehrtext über {t}. " * 12)
|
||||
for lv, t in subs)
|
||||
return (f"<!-- kapitel: Kapitel 1 -->\n<!-- section: {block} -->\n"
|
||||
f"<!-- compact -->\n{kompakt}\n<!-- ausführlich -->\n"
|
||||
f"Einstieg in {block}.\n{prosa}")
|
||||
|
||||
@staticmethod
|
||||
def _section_aus_prompt(prompt: str) -> str | None:
|
||||
"""Fix-Agenten geben die Section unverändert zurück (minimal-invasiv)."""
|
||||
m = re.search(r"(<!--\s*(?:kapitel|section):.*)", prompt, re.DOTALL)
|
||||
return m.group(1).strip() if m else None
|
||||
|
||||
|
||||
def standard_bloecke() -> dict:
|
||||
"""3 Blöcke; „Gemeinsamer Grundbegriff" liegt in Alpha UND Beta (Cross-Block-Fall)."""
|
||||
return {
|
||||
"Alpha-Konzept": {"beschreibung": "Das erste Grundkonzept",
|
||||
"subs": ["Definition Alpha", "Alpha Eigenschaften",
|
||||
"Gemeinsamer Grundbegriff"]},
|
||||
"Beta-Verfahren": {"beschreibung": "Das zentrale Verfahren",
|
||||
"subs": ["Beta Ablauf", "Beta Grenzen", "Gemeinsamer Grundbegriff"]},
|
||||
"Gamma-Anwendung": {"beschreibung": "Praktische Anwendung",
|
||||
"subs": ["Gamma Praxisfall", "Gamma Werkzeuge"]},
|
||||
}
|
||||
|
||||
|
||||
_WELT: Welt | None = None # ENV-Modus (CREATOR_FAKE_AGENTS=1): eine Welt pro Prozess
|
||||
|
||||
|
||||
async def respond(agent_key: str, prompt: str, capabilities: str) -> tuple[int, str, str]:
|
||||
global _WELT
|
||||
if _WELT is None:
|
||||
_WELT = Welt()
|
||||
return _WELT.respond(agent_key, prompt, capabilities)
|
||||
@@ -21,7 +21,8 @@ import re
|
||||
import database as db
|
||||
import readability
|
||||
from blocks import _sink_json
|
||||
from config import FORMAT_PURPOSE, READABILITY_ACTIVE, TEMPLATES_DIR, MAX_CONCURRENT_AGENTS_PER_TOPIC
|
||||
from config import (FORMAT_PURPOSE, GUIDE_LAENGE_MAX, GUIDE_LAENGE_MIN, READABILITY_ACTIVE,
|
||||
TEMPLATES_DIR, MAX_CONCURRENT_AGENTS_PER_TOPIC)
|
||||
from fsutil import atomic_write_json
|
||||
from jsonio import read_json_file as _json_file
|
||||
from pipeline import (CANCELLED, FAILED, OK, GenContext, _extra, _log, _prompt,
|
||||
@@ -34,9 +35,7 @@ GUIDE_STAGES = ("lernziele", "zuweisung", "writer", "fakten_gate", "coverage", "
|
||||
STAGE_LABELS = {"lernziele": "Lernziele", "zuweisung": "Zuweisung", "writer": "Writer",
|
||||
"fakten_gate": "Fakten-Gate", "coverage": "Coverage",
|
||||
"lesbarkeit": "Lesbarkeit", "done": "Fertig"}
|
||||
MAX_WRITER_ROUNDS = 2 # coverage → writer feedback loop cap (gains die after round 1–2)
|
||||
GATE_FIX_MIN = 3 # fact-gate claims below this: log only — a full fix rewrite for
|
||||
# 1–2 residual claims fired on 19/20 blocks (40 agent-minutes)
|
||||
from config import GATE_FIX_MIN, MAX_WRITER_ROUNDS, WRITER_SPLIT_SUBS # zentral tunebar
|
||||
# Simultaneous cards = the per-topic agent cap: every card busies exactly ONE agent at a
|
||||
# time (its stages run serially), so a lower number just idles slots (was hardcoded 10
|
||||
# from the old 10-slot era while the .env already allowed 24).
|
||||
@@ -65,7 +64,10 @@ def _ziele_schema(data):
|
||||
|
||||
|
||||
def _gate_schema(data):
|
||||
"""{"ok":true} → [] · {"claims":[{text,grund}]} → list · None invalid."""
|
||||
"""{"ok":true} → [] · {"claims":[{text,grund,urteil}]} → list · None invalid.
|
||||
urteil "falsch" (contradicts the facts/itself) vs "unbelegt" (true but underivable) —
|
||||
default unbelegt. Entries whose grund starts with "belegt" are dropped: one judge
|
||||
returned a 65-entry full inventory including SUPPORTED claims."""
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
if data.get("ok") is True:
|
||||
@@ -76,7 +78,12 @@ def _gate_schema(data):
|
||||
out = []
|
||||
for c in claims:
|
||||
if isinstance(c, dict) and str(c.get("text", "")).strip():
|
||||
out.append({"text": str(c["text"]).strip(), "grund": str(c.get("grund", "")).strip()})
|
||||
grund = str(c.get("grund", "")).strip()
|
||||
if grund.casefold().startswith("belegt"):
|
||||
continue
|
||||
urteil = str(c.get("urteil", "")).strip().casefold()
|
||||
out.append({"text": str(c["text"]).strip(), "grund": grund,
|
||||
"urteil": urteil if urteil == "falsch" else "unbelegt"})
|
||||
return out
|
||||
|
||||
|
||||
@@ -241,7 +248,7 @@ async def _stage_zuweisung(env: _Env, card: dict) -> bool:
|
||||
# A single section over ~45 subs measurably breaks the writer/coverage (Front Matter:
|
||||
# 4/6 objectives open after 2 rounds). First drafts of oversized cards are written in two
|
||||
# halves and merged back into ONE canonical section (all gates/assembly read one section).
|
||||
WRITER_SPLIT_SUBS = 30
|
||||
# WRITER_SPLIT_SUBS: siehe config.py
|
||||
|
||||
|
||||
def _merge_split_sections(sec_a: dict, sec_b: dict) -> str:
|
||||
@@ -397,13 +404,15 @@ async def _stage_fakten_gate(env: _Env, card: dict) -> bool:
|
||||
if status == FAILED:
|
||||
claims = [] # gate failure must not block the card — logged, text stands
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: kein Ergebnis — Text bleibt ungeprüft")
|
||||
if claims and len(claims) < GATE_FIX_MIN:
|
||||
# 1–2 Rest-Claims rechtfertigen keinen Voll-Rewrite: der Fix-Pass lief für 19/20
|
||||
# Blöcke und kostete 40 Agent-Minuten; die Guide-QA-Fachlichkeits-Stichprobe misst nach
|
||||
falsch = [c for c in claims if c.get("urteil") == "falsch"] if claims else []
|
||||
if claims and not falsch and len(claims) < GATE_FIX_MIN:
|
||||
# 1–2 merely UNSUPPORTED claims don't justify a fix pass (it ran for 19/20 blocks,
|
||||
# 40 agent-minutes) — but a WRONG claim always does: one slipped through this
|
||||
# threshold and cost the guide 1.5 QA points
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claim(s) unter Schwelle — kein Fix")
|
||||
claims = []
|
||||
if claims:
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} unbelegte Claims → Fix")
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claims ({len(falsch)} falsch) → Fix")
|
||||
fixp = env.slot(f"gatefix-{_safe(norm)}-r{card['writer_rounds']}.md")
|
||||
fixp.unlink(missing_ok=True)
|
||||
claims_text = "\n".join(f"- {c['text']}" + (f" ({c['grund']})" if c["grund"] else "")
|
||||
@@ -501,6 +510,18 @@ async def _stage_lesbarkeit(env: _Env, card: dict) -> bool:
|
||||
hints = await asyncio.to_thread(readability.rate_sections, {1: sec["md"]})
|
||||
if hints.get(1):
|
||||
problems.append(hints[1])
|
||||
# deterministic length trigger, same formula as the QA detector: prompt guidelines
|
||||
# alone left writers 2.7–4.1× over target — a measured overshoot forces the fix pass
|
||||
subs_all = env.subs_by_title.get(card["block"], [])
|
||||
n_rel = max(sum(1 for s in subs_all if s.get("relevance") != "peripheral"), 1)
|
||||
aus = re.split(r"<!--\s*ausführlich\s*-->", sec["md"], maxsplit=1)
|
||||
pro_sub = len(aus[1] if len(aus) == 2 else sec["md"]) / n_rel
|
||||
if not (GUIDE_LAENGE_MIN <= pro_sub <= GUIDE_LAENGE_MAX * 0.9):
|
||||
ziel = _writer_budget(len(subs_all))
|
||||
problems.append(
|
||||
f"Länge {round(pro_sub)} Zeichen/Sub (Rahmen {GUIDE_LAENGE_MIN}–{round(GUIDE_LAENGE_MAX * 0.9)}): "
|
||||
f"schreibe den ausführlich-Teil auf etwa {ziel} Zeichen GESAMT um — Sockel-Prosa und "
|
||||
f"Wiederholungen streichen, alle Sub-Marker und Beispiele behalten")
|
||||
if problems:
|
||||
from guide import _level_label
|
||||
subs = env.subs_by_title.get(card["block"], [])
|
||||
|
||||
@@ -20,8 +20,8 @@ import readability
|
||||
from fsutil import atomic_write_json
|
||||
from textkit import _norm_title
|
||||
|
||||
LAENGE_MIN = 150 # Zeichen je relevantem Sub im ausführlich-Teil (Untergrenze)
|
||||
LAENGE_MAX = 1200 # Obergrenze — außerhalb = Tiefen-Lotterie statt Zerlegungs-Signal
|
||||
from config import GUIDE_LAENGE_MAX as LAENGE_MAX, GUIDE_LAENGE_MIN as LAENGE_MIN
|
||||
|
||||
JACCARD_ABSATZ = 0.6 # Wort-Jaccard, ab dem zwei Absätze als Doppel gelten
|
||||
ABSATZ_MIN_CHARS = 200 # kürzere Absätze sind Übergänge — kein Dubletten-Signal
|
||||
LLM_SECTION_CHARS = 2500 # Section-Auszug je Judge-Item
|
||||
|
||||
@@ -18,18 +18,15 @@ import asyncio
|
||||
import logging
|
||||
|
||||
import database as db
|
||||
from config import MAX_CONCURRENT_AGENTS_PER_TOPIC
|
||||
from config import KANBAN_BATCH, MAX_CARD_RETRIES, MAX_CONCURRENT_AGENTS_PER_TOPIC, RETRY_BACKOFF
|
||||
|
||||
log = logging.getLogger("creator.kanban")
|
||||
|
||||
KANBAN_BATCH = 5 # cards a worker pulls per package (micro-batching)
|
||||
# How many packages ONE worker keeps in flight at once. A worker no longer blocks on a single
|
||||
# package — it keeps pulling and dispatching until this many run concurrently, so a busy column
|
||||
# fills the agent slots (the per-topic semaphore is the real cap; over-dispatch just queues cheaply).
|
||||
WORKER_INFLIGHT = MAX_CONCURRENT_AGENTS_PER_TOPIC
|
||||
_POLL = 0.3 # seconds between empty-queue polls
|
||||
MAX_CARD_RETRIES = 3 # failures per card until dead-letter
|
||||
RETRY_BACKOFF = 30.0 # base seconds; backoff = base · 2^(retries-1)
|
||||
|
||||
# Live registry of running flows (topic → Flow), so routes can attach research agents,
|
||||
# report `generating`, and cancel.
|
||||
|
||||
@@ -164,7 +164,7 @@ _relevance_schema = _enum_map_schema("relevance", _RELEVANCE) # relevance ∈
|
||||
_yesno_schema = _enum_map_schema("relevant", _YESNO) # triage gate ∈ ja/nein
|
||||
|
||||
|
||||
_MAX_RESTARTS = 2
|
||||
from config import MAX_RESTARTS as _MAX_RESTARTS # noqa: E402 — zentral tunebar
|
||||
|
||||
|
||||
async def _race(topic: str, label: str, slots: list[dict], quorum: int, timeout: int, provider: str, on_update=None, cancelled=None, *, grace: int | None = None, min_runtime: int | None = None, max_runtime: int | None = None) -> list | None:
|
||||
|
||||
@@ -336,12 +336,20 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
p["llm"] = v.get(k, "?")
|
||||
unecht: list[str] | None = None
|
||||
if llm and blocks:
|
||||
unecht = []
|
||||
verdacht = []
|
||||
for lo in range(0, len(blocks), 80): # ein Call je 80 Titel
|
||||
chunk = blocks[lo:lo + 80]
|
||||
v = await _llm_verdicts("QA-Bausteine", topic, f"bausteine-{lo}",
|
||||
[f"{b['title']} — {b['description'] or '(ohne Beschreibung)'}" for b in chunk])
|
||||
unecht += [b["title"] for k, b in enumerate(chunk, 1) if v.get(k) == "nein"]
|
||||
verdacht += [b for k, b in enumerate(chunk, 1) if v.get(k) == "nein"]
|
||||
# Bestätiger-Pass nur über die Geflaggten: der Einzel-Judge flaggte pro Lauf ANDERE
|
||||
# Blöcke (gemessen aak: Note pendelte 9.3↔10.0 bei identischem Bestand) — nur
|
||||
# doppelt-„nein" zählt; Repair hat als dritte Sicherung die eigene Zweitmeinung
|
||||
unecht = []
|
||||
if verdacht:
|
||||
v2 = await _llm_verdicts("QA-Bausteine", topic, "bausteine-b2",
|
||||
[f"{b['title']} — {b['description'] or '(ohne Beschreibung)'}" for b in verdacht])
|
||||
unecht = [b["title"] for k, b in enumerate(verdacht, 1) if v2.get(k) == "nein"]
|
||||
|
||||
art_rows = [dict(r) for r in await db.get_sub_artefakte(topic)]
|
||||
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
|
||||
|
||||
@@ -23,8 +23,7 @@ from textkit import _norm_title, _title, clean_title
|
||||
log = logging.getLogger("creator.repair")
|
||||
|
||||
JUDGE_TIMEOUT = 600
|
||||
JUDGE_CHUNK = 40 # Befunde je Judge-Call
|
||||
EVIDENCE_PER_BLOCK = 6000 # Zeichen Material-Auszug je Fremd-Kandidat
|
||||
from config import EVIDENCE_PER_BLOCK, JUDGE_CHUNK # noqa: E402 — zentral tunebar
|
||||
|
||||
|
||||
async def repair_befunde(topic: str) -> dict:
|
||||
|
||||
@@ -95,6 +95,8 @@ async def remove_topic(topic: str):
|
||||
await delete_source(topic) # topic config (DB) — removed together with the topic
|
||||
await delete_guide_content(topic)
|
||||
shutil.rmtree(topic_dir(topic), ignore_errors=True)
|
||||
import qa
|
||||
shutil.rmtree(qa.QA_DIR / topic, ignore_errors=True) # QA reports belong to the topic
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
|
||||
@@ -16,3 +16,69 @@ async def testdb(tmp_path, monkeypatch):
|
||||
await database.init_db()
|
||||
yield database
|
||||
await database.close_db()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fake_welt(testdb, tmp_path, monkeypatch):
|
||||
"""E2E ohne LLM: run_agent überall durch die Fake-Welt ersetzt, Tempo-Bremsen raus.
|
||||
Alle echten Schichten (_race, Quorum, Panels, Producer, QA-Gate) laufen mit."""
|
||||
import agents
|
||||
import blocks
|
||||
import board_inventory as bi
|
||||
import guide
|
||||
import kanban
|
||||
import pipeline
|
||||
import qa
|
||||
import repair
|
||||
from fake_agents import Welt
|
||||
|
||||
welt = Welt()
|
||||
|
||||
async def fake_run_agent(agent_key, prompt, timeout, provider="claude", role="fast",
|
||||
capabilities="none", lane="batch", scope=None, on_line=None, label=""):
|
||||
return welt.respond(agent_key, prompt, capabilities)
|
||||
|
||||
for mod in (agents, pipeline, blocks, guide, repair):
|
||||
monkeypatch.setattr(mod, "run_agent", fake_run_agent)
|
||||
|
||||
# Tempo: grace/poll/backoff bremsen echte Läufe, nicht den Fake
|
||||
monkeypatch.setattr(blocks, "CONSENSUS_GRACE", 0)
|
||||
monkeypatch.setattr(bi, "_QA_GATE_POLL", 0.05)
|
||||
monkeypatch.setattr(kanban, "RETRY_BACKOFF", 0.05)
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
|
||||
import guide_board
|
||||
monkeypatch.setattr(guide_board, "READABILITY_ACTIVE", False) # kein Modell-Load im Test
|
||||
import asyncio as _aio
|
||||
monkeypatch.setattr(bi, "_ingest_lock", _aio.Lock()) # Modul-Lock klebt sonst am Vortest-Loop
|
||||
|
||||
class _FakeEmb: # identischer Text → cos 1.0, sonst 0.0 (deterministisch, ohne Modell)
|
||||
@staticmethod
|
||||
def available():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def embed_sims(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr @ arr.T
|
||||
|
||||
@staticmethod
|
||||
def embed(texts):
|
||||
import numpy as np
|
||||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||||
for r, t in enumerate(texts):
|
||||
arr[r, uniq[t]] = 1.0
|
||||
return arr
|
||||
|
||||
import board_artefacts as ba
|
||||
for mod in (blocks, ba, qa):
|
||||
monkeypatch.setattr(mod, "embedding", _FakeEmb)
|
||||
|
||||
async def emb_ok(flow): # Board-1-Vektorpfade aus (wie board_env) — Judge-Wellen reichen
|
||||
return False
|
||||
monkeypatch.setattr(bi, "_emb_ok", emb_ok)
|
||||
return welt
|
||||
|
||||
96
backend/tests/invarianten.py
Normal file
96
backend/tests/invarianten.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""Invarianten nach einem (Fake-)E2E-Lauf: was IMMER gelten muss, egal welches Szenario.
|
||||
|
||||
Nutzt bewusst eigene, schlichte Prüfungen statt Pipeline-Heuristiken (Muster qa.py) —
|
||||
geteilte blinde Flecken machen den Check wertlos. Rückgabe: Liste von Verstößen,
|
||||
leer = alles konsistent.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import database as db
|
||||
from textkit import _norm_title
|
||||
|
||||
_LEVELS_OK = {"beginner", "advanced", "expert"}
|
||||
_RELEVANZ_OK = {"relevant", "peripheral"}
|
||||
|
||||
|
||||
def _json(path):
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
async def pruefe_invarianten(topic: str, files: dict | None = None,
|
||||
mit_artefakten: bool = True) -> list[str]:
|
||||
fehler: list[str] = []
|
||||
subs = [dict(r) for r in await db.list_subblocks(topic)]
|
||||
cons = [r for r in subs if r["status"] == "consensus"]
|
||||
|
||||
for r in cons:
|
||||
wo = f"{r['block']}/{r['sub_title']}"
|
||||
fk = None
|
||||
try:
|
||||
fk = json.loads(r["facts"]) if r["facts"] else None
|
||||
except ValueError:
|
||||
fehler.append(f"facts unparsebar: {wo}")
|
||||
if not (isinstance(fk, dict) and (fk.get("key_points") or fk.get("cited_facts"))):
|
||||
fehler.append(f"consensus-Sub ohne facts: {wo}")
|
||||
if r["level"] not in _LEVELS_OK:
|
||||
fehler.append(f"consensus-Sub ohne gültiges level: {wo}")
|
||||
if r["relevance"] not in _RELEVANZ_OK:
|
||||
fehler.append(f"consensus-Sub ohne relevance: {wo}")
|
||||
|
||||
if mit_artefakten:
|
||||
art = [dict(r) for r in await db.get_sub_artefakte(topic)]
|
||||
fragen = [dict(r) for r in await db.list_question_pattern(topic)]
|
||||
versorgt = {(r["block_norm"], r["sub_norm"]) for r in art}
|
||||
versorgt |= {(r["block_norm"], r["sub_norm"]) for r in fragen}
|
||||
lebend = {(r["block_norm"], r["sub_norm"]) for r in subs if r["status"] in ("consensus", "variant")}
|
||||
for r in cons:
|
||||
if r["relevance"] == "relevant" and (r["block_norm"], r["sub_norm"]) not in versorgt:
|
||||
fehler.append(f"relevanter Sub ohne Frage/Artefakt: {r['block']}/{r['sub_title']}")
|
||||
for bn, sn in sorted({(r["block_norm"], r["sub_norm"]) for r in art} |
|
||||
{(r["block_norm"], r["sub_norm"]) for r in fragen}):
|
||||
if (bn, sn) not in lebend:
|
||||
fehler.append(f"Waise (Ziel-Sub existiert nicht): {bn}/{sn}")
|
||||
|
||||
# keine hängengebliebenen Karten
|
||||
for c in await db.kanban_cards(topic):
|
||||
if c["stage"] == "dead":
|
||||
fehler.append(f"dead-Karte: {c['board']}/{c['card_id']}")
|
||||
|
||||
if files is not None:
|
||||
if not files["final"].exists():
|
||||
fehler.append("blocks.md fehlt")
|
||||
sc = _json(files["sidecar"])
|
||||
if not isinstance(sc, dict):
|
||||
fehler.append("sidecar-Datei fehlt/unparsebar")
|
||||
else: # Sidecar und DB-consensus müssen dieselbe Sub-Menge tragen
|
||||
db_menge = {(r["block_norm"], r["sub_norm"]) for r in cons}
|
||||
sc_menge = {(_norm_title(bt), _norm_title(str(s.get("title", ""))))
|
||||
for bt, ss in sc.items() for s in ss if isinstance(s, dict)}
|
||||
for extra in sorted(sc_menge - db_menge):
|
||||
fehler.append(f"Sidecar-Sub fehlt in DB: {extra}")
|
||||
for extra in sorted(db_menge - sc_menge):
|
||||
fehler.append(f"DB-consensus fehlt im Sidecar: {extra}")
|
||||
return fehler
|
||||
|
||||
|
||||
async def pruefe_guide_invarianten(topic: str, format_name: str = "Guide") -> list[str]:
|
||||
"""Jeder relevante consensus-Sub trägt einen Sub-Marker im Guide (Muster
|
||||
guide_qa.marker_fehlend, ohne LLM)."""
|
||||
import guide_qa
|
||||
fehler: list[str] = []
|
||||
cards = [dict(r) for r in await db.list_guide_cards(topic, format_name)]
|
||||
if not cards:
|
||||
return ["keine Guide-Karten"]
|
||||
for c in cards:
|
||||
if c["status"] != "ok" or not (c.get("md") or "").strip():
|
||||
fehler.append(f"Guide-Karte nicht ok: {c['block']} ({c['status']})")
|
||||
subs_rel: dict[str, set] = {}
|
||||
for r in await db.list_subblocks(topic):
|
||||
if r["status"] == "consensus" and r["relevance"] != "peripheral":
|
||||
subs_rel.setdefault(r["block_norm"], set()).add(r["sub_norm"])
|
||||
fehler += [f"Sub-Marker fehlt: {m}" for m in guide_qa.marker_fehlend(cards, subs_rel)]
|
||||
return fehler
|
||||
@@ -1012,12 +1012,13 @@ async def test_supplement_material_mode_for_source_topics(board_env, tmp_path, m
|
||||
|
||||
# ── Anker-Gate: Quorum-Titel ohne Korpus-Beleg (Reader-Ko-Halluzination) ────────────
|
||||
|
||||
async def _anker_env(db, tmp_path, monkeypatch, titel_map):
|
||||
async def _anker_env(db, tmp_path, monkeypatch, titel_map, desc_map=None):
|
||||
(tmp_path / "korpus.txt").write_text("Der Graph ist zusammenhängend und endlich.", encoding="utf-8")
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: tmp_path)
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"title": titel_map[cid], "description": "", "readers": ["r1", "r2"], "supplement": False}]
|
||||
return [{"title": titel_map[cid], "description": (desc_map or {}).get(cid, ""),
|
||||
"readers": ["r1", "r2"], "supplement": False}]
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "_rep", lambda rows: rows[0])
|
||||
@@ -1028,11 +1029,12 @@ async def _anker_env(db, tmp_path, monkeypatch, titel_map):
|
||||
|
||||
|
||||
async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
|
||||
"""Titel ohne Korpus-Anker → Beleg-Judge; „nein" → rejected/kein-beleg.
|
||||
"""Titel ohne Korpus-Anker, aber mit Evidenz → Beleg-Judge; „nein" → rejected/kein-beleg.
|
||||
Titel MIT Anker geht ohne Judge nach naming."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch,
|
||||
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"})
|
||||
{"c1": "Graph Zusammenhang", "c2": "Königsberger Brückenproblem"},
|
||||
{"c2": "Der Graph ist endlich."}) # Evidenz da → Judge entscheidet
|
||||
|
||||
async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "Brückenproblem" in prompt and "Zusammenhang" not in prompt # nur der Anker-lose
|
||||
@@ -1045,11 +1047,27 @@ async def test_anker_gate_rejects_unbelegtes(testdb, tmp_path, monkeypatch):
|
||||
assert c2["stage"] == "rejected" and c2["payload"]["reason"] == "kein-beleg"
|
||||
|
||||
|
||||
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Judge-Ausfall → Titel bleibt (2-Reader-Rückhalt)."""
|
||||
async def test_anker_gate_leeres_pack_hart_nein(testdb, tmp_path, monkeypatch):
|
||||
"""KEIN distinktives Token im Korpus → deterministisch rejected, Judge läuft NICHT
|
||||
(der Judge winkte 3 Kanon-Titel auf Schein-Auszügen durch)."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"})
|
||||
|
||||
async def never_slot(*a, **kw):
|
||||
raise AssertionError("Judge darf bei leerem Evidence-Pack nicht laufen")
|
||||
|
||||
monkeypatch.setattr(bi, "run_single_slot", never_slot)
|
||||
await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), cards)
|
||||
c9 = await db.kanban_get_card(TOPIC, B, "c9")
|
||||
assert c9["stage"] == "rejected" and c9["payload"]["reason"] == "kein-beleg"
|
||||
|
||||
|
||||
async def test_anker_gate_fail_open(testdb, tmp_path, monkeypatch):
|
||||
"""Judge-Ausfall bei VORHANDENER Evidenz → Titel bleibt (2-Reader-Rückhalt)."""
|
||||
db = testdb
|
||||
ctx, cards = await _anker_env(db, tmp_path, monkeypatch, {"c9": "Königsberger Brückenproblem"},
|
||||
{"c9": "Der Graph ist endlich."})
|
||||
|
||||
async def broken_slot(*a, **kw):
|
||||
return "failed", None
|
||||
|
||||
@@ -1063,3 +1081,103 @@ def test_hat_anker_ziffern_suffix():
|
||||
assert bi._hat_anker("ΔTSP1-Algorithmus", ctoks) # tsp1 → tsp
|
||||
assert not bi._hat_anker("Königsberger Brückenproblem", ctoks)
|
||||
assert not bi._hat_anker("Algorithmus Verfahren", ctoks) # nur Stopwörter → kein Anker
|
||||
|
||||
|
||||
async def test_reset_subblocks_loescht_globale_dateien(testdb, tmp_path):
|
||||
"""Reset auf Spalte subblocks: DB-Spiegel UND globale Sidecar-Dateien + ab-*-Resume-Slots
|
||||
weg — Reste des Vor-Laufs würden sonst in den frischen Lauf zurückmergen."""
|
||||
db = testdb
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "artefacts", {"title": "Alpha"})
|
||||
await db.put_subblock(TOPIC, "alpha", "s1", "Alpha", "S1", status="consensus")
|
||||
arbeit = tmp_path / "arbeit"
|
||||
(arbeit / "ab-alpha").mkdir(parents=True)
|
||||
(arbeit / "ab-alpha" / "facts.json").write_text("{}", encoding="utf-8")
|
||||
files = {"arbeit": arbeit}
|
||||
for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"):
|
||||
files[k] = tmp_path / f"{k}.json"
|
||||
files[k].write_text("{}", encoding="utf-8")
|
||||
moved = await bi.reset_board_from_stage(TOPIC, "artefacts", "subblocks", files)
|
||||
assert moved == 1
|
||||
assert not await db.list_subblocks(TOPIC)
|
||||
assert not (arbeit / "ab-alpha").exists()
|
||||
assert all(not files[k].exists() for k in ("sidecar", "facts", "sub_roh", "question_pattern", "artefakte"))
|
||||
|
||||
|
||||
# ── Naming-Abstraktion: freier Name nur mit Anker ───────────────────────────────────
|
||||
|
||||
def test_naming_schema_varianten():
|
||||
assert bi._naming_schema({"best": 2}, 3) == (2, None, False)
|
||||
assert bi._naming_schema({"best": 1, "name": "Kurzer Titel"}, 3) == (1, "Kurzer Titel", False)
|
||||
assert bi._naming_schema({"ok": True}, 3) == (None, None, True)
|
||||
assert bi._naming_schema({"best": 9}, 3) is None
|
||||
assert bi._naming_schema({"best": 1, "name": "x" * 90}, 3) == (1, None, False) # zu lang
|
||||
|
||||
|
||||
def test_name_verankert_thema_subset():
|
||||
rows = [{"title": "Bubblesort-Schleife und Tauschoperation", "description": "innere Schleife"},
|
||||
{"title": "Bubblesort Durchläufe", "description": ""}]
|
||||
assert bi._name_verankert("Bubblesort Tauschoperation", rows, None)
|
||||
assert not bi._name_verankert("Königsberger Brückenproblem", rows, None) # fremde Begriffe
|
||||
assert not bi._name_verankert("und der", rows, None) # nur Stopwörter → kein Anker
|
||||
|
||||
|
||||
def test_name_verankert_korpus():
|
||||
ctoks = {"partition", "problem", "vollständigkeit"}
|
||||
rows = [{"title": "irrelevant", "description": ""}]
|
||||
assert bi._name_verankert("Partition-Problem", rows, ctoks)
|
||||
assert not bi._name_verankert("Rucksackproblem Optimierung", rows, ctoks)
|
||||
|
||||
|
||||
async def test_naming_vergibt_verankerten_namen(testdb, tmp_path, monkeypatch):
|
||||
"""Judge liefert best+name; verankerter Name gewinnt, unverankerter fällt auf Member zurück."""
|
||||
db = testdb
|
||||
antwort = {"val": {"best": 1, "name": "Bubblesort Grundprinzip"}}
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"norm": "bubblesort - grundprinzip und ablauf (kap. 2)",
|
||||
"title": "Bubblesort - Grundprinzip und Ablauf (Kap. 2)",
|
||||
"description": "Sortieren durch Tauschen", "readers": ["r1"], "sources": []},
|
||||
{"norm": "sortieren durch tauschen", "title": "Sortieren durch Tauschen",
|
||||
"description": "", "readers": ["r2"], "sources": []}]
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
import json as _json
|
||||
return "ok", payload((0, _json.dumps(antwort["val"]), ""))
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
await db.kanban_upsert_card(TOPIC, B, "c1", "cluster", "naming", {})
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
await bi._name_one(ctx, _mk_flow(tmp_path), {"card_id": "c1", "payload": {}})
|
||||
card = await db.kanban_get_card(TOPIC, B, "c1")
|
||||
assert card["payload"]["title"] == "Bubblesort Grundprinzip"
|
||||
|
||||
# unverankerter Name → Member-Titel gewinnt
|
||||
antwort["val"] = {"best": 2, "name": "Vergleichsbasierte Sortierverfahren"}
|
||||
await db.kanban_upsert_card(TOPIC, B, "c2", "cluster", "naming", {})
|
||||
await bi._name_one(ctx, _mk_flow(tmp_path), {"card_id": "c2", "payload": {}})
|
||||
card = await db.kanban_get_card(TOPIC, B, "c2")
|
||||
assert card["payload"]["title"] == "Sortieren durch Tauschen"
|
||||
|
||||
|
||||
async def test_namecheck_ok_behaelt_titel(testdb, tmp_path, monkeypatch):
|
||||
"""Check-Judge bestätigt mit ok:true → Titel und Beschreibung bleiben unverändert."""
|
||||
db = testdb
|
||||
|
||||
async def fake_members(topic, cid):
|
||||
return [{"norm": "a", "title": "A", "description": "da", "readers": ["r1"], "sources": []},
|
||||
{"norm": "b", "title": "B", "description": "db", "readers": ["r2"], "sources": []}]
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
assert "Eigener Titel" in prompt # current_title steht im Prompt
|
||||
return "ok", payload((0, '{"ok": true}', ""))
|
||||
|
||||
monkeypatch.setattr(bi, "_member_rows", fake_members)
|
||||
monkeypatch.setattr(bi, "run_single_slot", fake_slot)
|
||||
payload = {"title": "Eigener Titel", "description": "Eigene Beschreibung", "main_norm": "a"}
|
||||
await db.kanban_upsert_card(TOPIC, B, "c9", "cluster", "naming_check", payload)
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
await bi._namecheck_one(ctx, _mk_flow(tmp_path), {"card_id": "c9", "payload": payload})
|
||||
block = await db.kanban_get_card(TOPIC, B, "b-c9")
|
||||
assert block["payload"]["title"] == "Eigener Titel"
|
||||
assert block["payload"]["description"] == "Eigene Beschreibung"
|
||||
|
||||
197
backend/tests/test_e2e_fake.py
Normal file
197
backend/tests/test_e2e_fake.py
Normal file
@@ -0,0 +1,197 @@
|
||||
"""E2E über die ECHTE Engine mit Fake-Agenten: kompletter Generierungspfad in Sekunden.
|
||||
|
||||
Anders als test_board_inventory (dort sind die Block-Funktionen gefakt) läuft hier alles
|
||||
bis run_agent echt — _race, Quorum, Panels, Konsolidierung, Cross-Block, QA-Gate.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
import board_inventory as bi
|
||||
from pipeline import GenContext
|
||||
from tests.invarianten import pruefe_invarianten, pruefe_guide_invarianten
|
||||
|
||||
TOPIC = "t"
|
||||
|
||||
|
||||
def _files(tmp_path):
|
||||
work = tmp_path / "arbeit"
|
||||
work.mkdir(exist_ok=True)
|
||||
return {"arbeit": work, "final": tmp_path / "blocks.md",
|
||||
"sub_roh": tmp_path / "sub_roh.json", "sidecar": tmp_path / "subblocks.json",
|
||||
"facts": tmp_path / "facts.json", "question_pattern": tmp_path / "question_pattern.json",
|
||||
"artefakte": tmp_path / "artefakte.json", "outline": tmp_path / "outline.json",
|
||||
"outline_slots": [tmp_path / f"outline-{i}.json" for i in (1, 2, 3)],
|
||||
"research": [work / f"research-{i}.md" for i in (1, 2, 3, 4, 5)]}
|
||||
|
||||
|
||||
async def _lauf(tmp_path, research=True, qa_force=False, timeout=120):
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
files = _files(tmp_path)
|
||||
ok = await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "thema"}, None, "",
|
||||
research=research, qa_force=qa_force), timeout=timeout)
|
||||
return ok, files
|
||||
|
||||
|
||||
async def test_e2e_thema_vollpfad(fake_welt, testdb, tmp_path):
|
||||
"""Research → Inventar → QA-Gate → Artefakte → Finalize, alle Schichten echt."""
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = [c for c in await db.kanban_cards(TOPIC, board="inventory", stage="done_block")]
|
||||
titel = {c["payload"]["title"] for c in done}
|
||||
assert titel == {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"}
|
||||
# Cross-Block-Dublette: „Gemeinsamer Grundbegriff" überlebt in genau EINEM Block
|
||||
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
||||
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
||||
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "variant"]
|
||||
fehler = await pruefe_invarianten(TOPIC, files)
|
||||
assert fehler == []
|
||||
|
||||
|
||||
async def test_e2e_guide(fake_welt, testdb, tmp_path):
|
||||
"""Auf den Vollpfad folgt der Guide-Bau — Gate/Coverage/Lese-Stages laufen echt."""
|
||||
import guide_board
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
||||
entries = {i: f"{c['payload']['title']} — {c['payload'].get('description', '')}"
|
||||
for i, c in enumerate(done, 1)}
|
||||
chapters = await asyncio.wait_for(
|
||||
guide_board.run_guide_board("g-e2e", TOPIC, "Guide", entries, "", "claude",
|
||||
tmp_path / "guides" / "Guide.json"), timeout=120)
|
||||
assert chapters is not None
|
||||
assert await pruefe_guide_invarianten(TOPIC) == []
|
||||
|
||||
|
||||
async def test_e2e_rerun_idempotent(fake_welt, testdb, tmp_path):
|
||||
"""Zweiter Lauf (Continue, research=False) hinterlässt keine Waisen/Reste."""
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
vorher = {(r["block_norm"], r["sub_norm"], r["status"])
|
||||
for r in await db.list_subblocks(TOPIC)}
|
||||
ok2, _f = await _lauf(tmp_path, research=False)
|
||||
assert ok2
|
||||
nachher = {(r["block_norm"], r["sub_norm"], r["status"])
|
||||
for r in await db.list_subblocks(TOPIC)}
|
||||
assert nachher == vorher
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stoerung", [
|
||||
{"muster": r"-sub-crossblock-.*-j1$", "modus": "fehler", "mal": 3}, # Ersatzrichter jE
|
||||
{"muster": r"-sub-konsolidierung-.*-j1$", "modus": "garbage", "mal": 1}, # Retry heilt
|
||||
{"muster": r"-facts-c\d+$", "modus": "fehler", "mal": 1}, # Slot-Restart
|
||||
{"muster": r"-research-2$", "modus": "fehler", "mal": 3}, # 1 Producer tot
|
||||
])
|
||||
async def test_e2e_stoerungen_flow_endet(fake_welt, testdb, tmp_path, stoerung):
|
||||
"""Einzel-Ausfälle dürfen weder den Flow stoppen noch Invarianten reißen."""
|
||||
fake_welt.stoerungen.append(dict(stoerung, rest=stoerung["mal"]))
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_crossblock_dissent_failopen(fake_welt, testdb, tmp_path):
|
||||
"""j1 sagt a, j2 sagt b, j3 fällt aus → Paar bleibt (fail-open), Rest konsistent."""
|
||||
fake_welt.stoerungen += [
|
||||
{"muster": r"-sub-crossblock-.*-j2$", "modus": "antwort",
|
||||
"antwort": '{"pairs": {"1": "b"}}', "mal": 1, "rest": 1},
|
||||
{"muster": r"-sub-crossblock-.*-j3$", "modus": "fehler", "mal": 3, "rest": 3},
|
||||
]
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
subs = [dict(r) for r in await db.list_subblocks(TOPIC)]
|
||||
gemeinsam = [r for r in subs if r["sub_norm"] == "gemeinsamer grundbegriff"]
|
||||
assert sorted(r["status"] for r in gemeinsam) == ["consensus", "consensus"] # kein Fold
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_inblock_gruppe_faltet(fake_welt, testdb, tmp_path):
|
||||
"""Welt-Regel: „Alpha Eigenschaften" faltet unter „Definition Alpha" — beide Judges
|
||||
liefern die Gruppe, der Verlierer wird variant, seine facts wandern zum Gewinner."""
|
||||
fake_welt.gruppen.append(("Definition Alpha", ["Alpha Eigenschaften"]))
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha-konzept")}
|
||||
assert rows.get("alpha eigenschaften") == "variant"
|
||||
assert rows.get("definition alpha") == "consensus"
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_gate_vollinventur_ohne_fix(fake_welt, testdb, tmp_path):
|
||||
"""Gate-Judge liefert eine Voll-Inventur (belegte Claims mit „Belegt…"-Grund) —
|
||||
der Schema-Filter wirft sie raus, es läuft KEIN Fakten-Fix."""
|
||||
import json
|
||||
antwort = json.dumps({"claims": [
|
||||
{"text": "Aussage 1", "grund": "Belegt durch Quelle", "urteil": "unbelegt"},
|
||||
{"text": "Aussage 2", "grund": "Belegt durch Fakten", "urteil": "unbelegt"},
|
||||
{"text": "Aussage 3", "grund": "Belegt: steht im Skript", "urteil": "unbelegt"}]})
|
||||
fake_welt.stoerungen.append({"muster": r"-gate-", "modus": "antwort",
|
||||
"antwort": antwort, "mal": 99, "rest": 99})
|
||||
import guide_board
|
||||
ok, _files = await _lauf(tmp_path)
|
||||
assert ok
|
||||
db = testdb
|
||||
done = await db.kanban_cards(TOPIC, board="inventory", stage="done_block")
|
||||
entries = {i: c["payload"]["title"] for i, c in enumerate(done, 1)}
|
||||
chapters = await asyncio.wait_for(
|
||||
guide_board.run_guide_board("g-vi", TOPIC, "Guide", entries, "", "claude",
|
||||
tmp_path / "guides" / "Guide.json"), timeout=120)
|
||||
assert chapters is not None
|
||||
assert not any("-gatefix-" in k for k in fake_welt.calls)
|
||||
|
||||
|
||||
async def test_e2e_echtheits_flattern_gestoppt(fake_welt, testdb, tmp_path):
|
||||
"""QA-Pass 1 flaggt alle Blöcke als unecht (Judge-Flattern) — der Bestätiger-Pass
|
||||
widerspricht, die Gate-Note bleibt sauber, der Flow läuft durch."""
|
||||
import json
|
||||
fake_welt.stoerungen.append({"muster": r"^qa-t-bausteine-0$", "modus": "antwort",
|
||||
"antwort": json.dumps({"relevant": {"1": "nein", "2": "nein", "3": "nein"}}),
|
||||
"mal": 1, "rest": 1})
|
||||
ok, files = await _lauf(tmp_path)
|
||||
assert ok # Gate hat nicht pausiert — der Zufalls-Verdacht wurde nicht bestätigt
|
||||
assert any("bausteine-b2" in k for k in fake_welt.calls)
|
||||
assert await pruefe_invarianten(TOPIC, files) == []
|
||||
|
||||
|
||||
async def test_e2e_uni_anker_gate(fake_welt, testdb, tmp_path, monkeypatch):
|
||||
"""uni-Modus mit Mini-Korpus: der Kanon-Titel ohne Korpus-Anker wird deterministisch
|
||||
rejected (leeres Evidence-Pack), die belegten Blöcke laufen durch; QA misst gegen
|
||||
den echten Korpus."""
|
||||
import blocks as blx
|
||||
fake_welt.bloecke["Kanon-Klassiker"] = {
|
||||
"beschreibung": "Beruehmtes Lehrbuchproblem", "subs": ["Klassiker Detail"]}
|
||||
korpus = tmp_path / "korpus"
|
||||
korpus.mkdir()
|
||||
zeilen = []
|
||||
for t, b in fake_welt.bloecke.items():
|
||||
if t == "Kanon-Klassiker":
|
||||
continue # kommt bewusst NICHT im Material vor
|
||||
zeilen.append(f"Kapitel {t}: {b['beschreibung']}. " +
|
||||
" ".join(f"Wir behandeln {s}." for s in b["subs"]))
|
||||
(korpus / "skript.txt").write_text("\n\n".join(zeilen), encoding="utf-8")
|
||||
monkeypatch.setattr(bi, "source_folder", lambda t: korpus)
|
||||
monkeypatch.setattr(blx, "source_folder", lambda t: korpus)
|
||||
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
files = _files(tmp_path)
|
||||
ok = await asyncio.wait_for(
|
||||
bi.run_boards(ctx, lambda *a, **k: None, files, {"type": "uni", "location": str(korpus)},
|
||||
korpus, "", research=True, qa_force=True), timeout=120)
|
||||
assert ok
|
||||
db = testdb
|
||||
alle = [dict(c) for c in await db.kanban_cards(TOPIC, board="inventory")]
|
||||
assert any(c["stage"] == "rejected" and c["payload"].get("title") == "Kanon-Klassiker"
|
||||
for c in alle)
|
||||
assert not any(c["kind"] == "block" and c["payload"].get("title") == "Kanon-Klassiker"
|
||||
for c in alle) # nie zum Block geworden
|
||||
done = {c["payload"].get("title") for c in alle
|
||||
if c["kind"] == "block" and c["stage"] == "done_block"}
|
||||
assert {"Alpha-Konzept", "Beta-Verfahren", "Gamma-Anwendung"} <= done
|
||||
@@ -18,8 +18,16 @@ def test_ziele_schema():
|
||||
def test_gate_schema():
|
||||
assert gb._gate_schema({"ok": True}) == []
|
||||
claims = gb._gate_schema({"claims": [{"text": "Falsch", "grund": "fehlt"}]})
|
||||
assert claims == [{"text": "Falsch", "grund": "fehlt"}]
|
||||
assert claims == [{"text": "Falsch", "grund": "fehlt", "urteil": "unbelegt"}]
|
||||
assert gb._gate_schema({}) is None
|
||||
# urteil "falsch" wird durchgereicht, alles andere defaultet auf unbelegt
|
||||
claims = gb._gate_schema({"claims": [{"text": "A", "grund": "widerspricht", "urteil": "FALSCH"},
|
||||
{"text": "B", "grund": "x", "urteil": "quatsch"}]})
|
||||
assert [c["urteil"] for c in claims] == ["falsch", "unbelegt"]
|
||||
# Voll-Inventur-Rauschen: als belegt begründete Einträge fliegen raus
|
||||
claims = gb._gate_schema({"claims": [{"text": "A", "grund": "Belegt durch Quelle X"},
|
||||
{"text": "B", "grund": "nicht ableitbar"}]})
|
||||
assert [c["text"] for c in claims] == ["B"]
|
||||
|
||||
|
||||
def test_coverage_schema():
|
||||
@@ -266,7 +274,7 @@ async def test_lese_check_text_sink(testdb, tmp_path, monkeypatch):
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-l", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
md = ("<!-- kapitel: K -->\n<!-- section: Alpha -->\n<!-- compact -->\n- x\n"
|
||||
"<!-- ausführlich -->\nText.")
|
||||
"<!-- ausführlich -->\n" + "Text im Längen-Rahmen. " * 20) # ~460 Z. — kein Längen-Trigger
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
seen = {}
|
||||
|
||||
@@ -330,3 +338,49 @@ async def test_load_subblocks_defaultet_levellose(testdb):
|
||||
subs = await _load_subblocks("t")
|
||||
by_title = {s["title"]: s["level"] for s in subs["Block"]}
|
||||
assert by_title == {"Mit Level": "beginner", "Ohne Level": "advanced"}
|
||||
|
||||
|
||||
async def test_fakten_gate_falsch_claim_erzwingt_fix(testdb, tmp_path, monkeypatch):
|
||||
"""Ein einzelner FALSCH-Claim läuft in den Fix, auch unter GATE_FIX_MIN —
|
||||
ein durchgerutschter kostete den Guide 1.5 QA-Punkte."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-gf", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
md = "<!-- section: Alpha -->\n<!-- ausführlich -->\nText."
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
keys = []
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
keys.append(key)
|
||||
if "-gate-" in key:
|
||||
return gb.OK, [{"text": "c1", "grund": "widerspricht", "urteil": "falsch"}]
|
||||
return gb.FAILED, None # Fix-Agent liefert nichts — Text bleibt, aber der Call MUSS kommen
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
assert await gb._stage_fakten_gate(env, card)
|
||||
assert any("-gatefix-" in k for k in keys)
|
||||
|
||||
|
||||
async def test_laengen_trigger_startet_lesefix(testdb, tmp_path, monkeypatch):
|
||||
"""Ausführlich-Teil über der Obergrenze → deterministisches Längen-Problem
|
||||
mit hartem Zeichenziel landet im Lese-Fix-Auftrag."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-lz", TOPIC, FMT, "", tmp_path / "Guide.json",
|
||||
{"Alpha": [{"title": "S1", "level": "beginner", "relevance": "relevant"}]},
|
||||
{}, "(q)", "spec")
|
||||
md = ("<!-- section: Alpha -->\n<!-- compact -->\n- x\n<!-- ausführlich -->\n"
|
||||
+ "Viel zu langer Sockeltext. " * 80) # ~2160 Z./Sub > 1200×0.9
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
seen = {}
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
if "-lesefix-" in key:
|
||||
seen["tasks"] = prompt
|
||||
return gb.FAILED, None
|
||||
return gb.OK, payload((0, '{"ok": true}', "")) # Lese-Check: keine Probleme
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
monkeypatch.setattr(gb, "READABILITY_ACTIVE", False)
|
||||
assert await gb._stage_lesbarkeit(env, card)
|
||||
assert "Länge" in seen["tasks"] and str(gb._writer_budget(1)) in seen["tasks"]
|
||||
|
||||
@@ -519,3 +519,90 @@ async def test_finalize_defaultet_level_nachzuegler(testdb, tmp_path):
|
||||
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
|
||||
row = next(r for r in await db.list_subblocks(TOPIC, "alpha"))
|
||||
assert row["level"] == "advanced" and row["relevance"] == "relevant"
|
||||
|
||||
|
||||
async def test_finalize_loescht_stale_consensus(testdb, tmp_path):
|
||||
"""Alt-consensus-Rows, die der Lauf-Sidecar nicht mehr trägt, fliegen raus —
|
||||
variant-Rows bleiben (QA liest die Status). Wurzel der 25 Board-2-losen Waisen."""
|
||||
db = testdb
|
||||
await db.put_subblock(TOPIC, "alpha", "alt-rest", "Alpha", "Alt-Rest", status="consensus")
|
||||
await db.put_subblock(TOPIC, "alpha", "alte-variante", "Alpha", "Alte Variante", status="variant")
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "finalize", {})
|
||||
files = {k: tmp_path / f"{k}.json" for k in
|
||||
("sub_roh", "facts", "sidecar", "question_pattern", "artefakte")}
|
||||
card = {"card_id": "alpha", "payload": {
|
||||
"title": "Alpha", "raw": {"Alpha": ["Neu"]}, "facts": {},
|
||||
"sidecar": {"Alpha": [{"title": "Neu", "level": "beginner"}]},
|
||||
"pattern": {}, "artefacts": {}}}
|
||||
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
|
||||
rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "alpha")}
|
||||
assert rows == {"neu": "consensus", "alte-variante": "variant"}
|
||||
|
||||
|
||||
async def test_crossblock_chunking_faltet_global(testdb, tmp_path, monkeypatch):
|
||||
"""Paare werden gechunkt beurteilt (ein Hänger blockiert nur noch seinen Chunk);
|
||||
die Verdicts falten global über alle Chunks."""
|
||||
db = testdb
|
||||
flow = Flow(TOPIC, work_dir=tmp_path)
|
||||
cards = []
|
||||
for bnorm, subs in (("alpha", ["Gleiche Aussage", "Zweite gleiche Aussage", "Nur in Alpha"]),
|
||||
("beta", ["Gleiche Aussage", "Zweite gleiche Aussage"])):
|
||||
payload = {"title": bnorm.title(),
|
||||
"raw": {bnorm.title(): list(subs)},
|
||||
"sidecar": {bnorm.title(): [{"title": s, "level": "beginner"} for s in subs]},
|
||||
"facts": {bnorm.title(): {blocks._norm_title(s): {"key_points": []} for s in subs}}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", bnorm, "ablock", "konsolidierung", payload)
|
||||
await _seed_block(db, bnorm, subs)
|
||||
cards.append({"card_id": bnorm, "payload": payload})
|
||||
monkeypatch.setattr(ba, "embedding", _FakeEmb)
|
||||
monkeypatch.setattr(ba, "CROSS_CHUNK_PAARE", 1) # 2 Paare → 2 Chunks
|
||||
fake = _fake_slot({"j1": {"pairs": {"1": "a"}}, "j2": {"pairs": {"1": "a"}}})
|
||||
monkeypatch.setattr(ba, "run_single_slot", fake)
|
||||
await ba._proc_konsolidierung(_ctx(), flow, {}, "", cards)
|
||||
assert len(fake.calls) == 4 # 2 Chunks × j1/j2
|
||||
beta = await db.kanban_get_card(TOPIC, "artefacts", "beta")
|
||||
assert beta["payload"]["raw"].get("Beta", []) == [] # beide Dubletten global gefaltet
|
||||
|
||||
|
||||
async def test_facts_nachfass_holt_nur_fehlende(testdb, tmp_path, monkeypatch):
|
||||
"""Nachfass ruft den slim-Facts-Lauf NUR mit den facts-losen Subs und merged die Funde;
|
||||
Vorhandenes bleibt unberührt, Subs werden nie verworfen."""
|
||||
gesehen = {}
|
||||
|
||||
async def fake_facts_block(ctx, set_p, files, raw, q, folder, instructions, ns="", lbl="",
|
||||
sources=None, slim=False):
|
||||
gesehen["raw"] = raw
|
||||
gesehen["slim"] = slim
|
||||
return ({"Alpha": {"ohne beleg": {"key_points": ["kp neu"]},
|
||||
"mit beleg": {"key_points": ["DARF NICHT GEWINNEN"]}}},
|
||||
{"Alpha": {"ohne beleg"}}) # discard-Urteil wird ignoriert
|
||||
|
||||
monkeypatch.setattr(blocks, "_facts_block", fake_facts_block)
|
||||
raw = {"Alpha": ["Mit Beleg", "Ohne Beleg"]}
|
||||
facts_map = {"Alpha": {"mit beleg": {"key_points": ["kp alt"]}}}
|
||||
ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False)
|
||||
n = await blocks._facts_nachfass(ctx, {"arbeit": tmp_path}, raw, facts_map, {}, None)
|
||||
assert n == 1
|
||||
assert gesehen["slim"] and gesehen["raw"] == {"Alpha": ["Ohne Beleg"]}
|
||||
assert facts_map["Alpha"]["ohne beleg"]["key_points"] == ["kp neu"]
|
||||
assert facts_map["Alpha"]["mit beleg"]["key_points"] == ["kp alt"]
|
||||
assert raw["Alpha"] == ["Mit Beleg", "Ohne Beleg"] # kein Verwurf
|
||||
|
||||
|
||||
async def test_levels_merge_fuzzy_match(testdb, tmp_path, monkeypatch):
|
||||
"""Levels-Agent paraphrasiert den Sub-Titel → facts hängen trotzdem am Sidecar-Eintrag
|
||||
(eindeutiger Präfix-Match statt stillem Grounding-Verlust)."""
|
||||
db = testdb
|
||||
payload = {"title": "Alpha", "raw": {"Alpha": ["Marker Regel: Details dazu"]},
|
||||
"facts": {"Alpha": {blocks._norm_title("Marker Regel: Details dazu"):
|
||||
{"key_points": ["kp"]}}}}
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "levels", payload)
|
||||
cards = [{"card_id": "alpha", "payload": payload}]
|
||||
|
||||
async def fake_levels_block(ctx, set_p, files, raw, instructions, ns="", lbl=""):
|
||||
return {"Alpha": [{"title": "Marker Regel", "level": "beginner"}]} # gekürzter Titel
|
||||
|
||||
monkeypatch.setattr(ba, "_levels_block", fake_levels_block)
|
||||
await ba._proc_levels(_ctx(), Flow(TOPIC, work_dir=tmp_path), {"arbeit": tmp_path}, "", cards)
|
||||
card = await db.kanban_get_card(TOPIC, "artefacts", "alpha")
|
||||
assert card["payload"]["sidecar"]["Alpha"][0]["facts"] == {"key_points": ["kp"]}
|
||||
|
||||
@@ -214,3 +214,37 @@ def test_fremd_digit_suffix_tolerant():
|
||||
b = [{"title": "ΔTSP1-Algorithmus", "description": "", "sources": []},
|
||||
{"title": "Quantencomputer", "description": "", "sources": []}]
|
||||
assert qa.fremd(b, corpus) == ["Quantencomputer"]
|
||||
|
||||
|
||||
async def test_unecht_braucht_doppelt_nein(testdb, tmp_path, monkeypatch):
|
||||
"""Echtheits-Urteil zählt nur nach Bestätiger-Pass: der Einzel-Judge flaggte pro Lauf
|
||||
andere Blöcke und pendelte die Note (aak: 9.3↔10.0 bei identischem Bestand)."""
|
||||
db = testdb
|
||||
for cid, titel in (("b1", "Wackelkandidat"), ("b2", "Zufallstreffer"), ("b3", "Solide")):
|
||||
await db.kanban_upsert_card("t", "inventory", cid, "block", "done_block",
|
||||
{"title": titel, "description": "d"})
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path)
|
||||
|
||||
async def fake_verdicts(template, topic, key, items):
|
||||
if template != "QA-Bausteine":
|
||||
return {}
|
||||
if key.startswith("bausteine-b2"): # Bestätiger sieht nur die Geflaggten
|
||||
assert len(items) == 2
|
||||
return {1: "nein", 2: "ja"} # nur der erste wird bestätigt
|
||||
return {1: "nein", 2: "nein", 3: "ja"} # Pass 1 flaggt zwei
|
||||
|
||||
monkeypatch.setattr(qa, "_llm_verdicts", fake_verdicts)
|
||||
report = await qa.qa_report("t", llm=True)
|
||||
assert report["unecht"] == ["Wackelkandidat"]
|
||||
|
||||
|
||||
async def test_topic_delete_entfernt_qa_ordner(testdb, tmp_path, monkeypatch):
|
||||
"""DELETE /topics räumt auch storage/qa/<topic>/ — Reports gehören zum Topic."""
|
||||
import routes
|
||||
monkeypatch.setattr(routes, "topic_dir", lambda t: tmp_path / "topics" / t)
|
||||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
|
||||
qdir = tmp_path / "qa" / "t"
|
||||
qdir.mkdir(parents=True)
|
||||
(qdir / "alt.json").write_text("{}", encoding="utf-8")
|
||||
await routes.remove_topic("t")
|
||||
assert not qdir.exists()
|
||||
|
||||
@@ -226,10 +226,8 @@ async def test_outline_review_moves_block(testdb, tmp_path, monkeypatch):
|
||||
out = plan_a
|
||||
elif key.endswith("outline-review"):
|
||||
out = review_out["val"]
|
||||
if out is not None:
|
||||
m = re.search(r"(/\S+\.json)", prompt)
|
||||
with open(m.group(1), "w", encoding="utf-8") as f:
|
||||
json.dump(out, f)
|
||||
if out is not None: # neue Semantik: Antwort als TEXT, der Sink persistiert
|
||||
return "ok", payload((0, json.dumps(out), ""))
|
||||
return "ok", payload(None)
|
||||
|
||||
monkeypatch.setattr(blx, "run_single_slot", fake_slot)
|
||||
|
||||
113
backend/tests/test_train.py
Normal file
113
backend/tests/test_train.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Training-Harness: Registry↔config-Konsistenz, ENV-Override, Trainer-Logik (Stub-Runner)."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import config
|
||||
import train
|
||||
import train_params
|
||||
from train import Trainer, score
|
||||
|
||||
BACKEND = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def test_registry_spiegelt_config():
|
||||
"""Jeder Registry-Parameter existiert in config mit identischem Default und
|
||||
flow-sicheren Rändern — sonst optimiert der Trainer Phantome."""
|
||||
for name, p in train_params.PARAMS.items():
|
||||
assert getattr(config, name, None) == p["default"], name
|
||||
assert p["min"] <= p["default"] <= p["max"], name
|
||||
assert p["step"] > 0, name
|
||||
|
||||
|
||||
def test_creator_params_override_wirkt_im_subprozess():
|
||||
out = subprocess.run(
|
||||
[sys.executable, "-c", "import config; print(config.FACTS_CHUNK_SUBS, config.TIMEOUTS['subblock_check'][0])"],
|
||||
capture_output=True, text=True, cwd=BACKEND,
|
||||
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"FACTS_CHUNK_SUBS": 6, "TIMEOUT_subblock_check_base": 77}'})
|
||||
assert out.stdout.split() == ["6", "77"], out.stderr
|
||||
|
||||
|
||||
def test_creator_params_unbekannter_name_bricht_ab():
|
||||
out = subprocess.run([sys.executable, "-c", "import config"],
|
||||
capture_output=True, text=True, cwd=BACKEND,
|
||||
env={"PATH": "/usr/bin:/bin", "CREATOR_PARAMS": '{"GIBT_ES_NICHT": 1}'})
|
||||
assert out.returncode != 0 and "GIBT_ES_NICHT" in out.stderr
|
||||
|
||||
|
||||
def _metrics(note=8.0, dauer=10.0, tokens=1_000_000, **quoten):
|
||||
return {"note": note, "quoten": quoten, "quoten_artefakte": {},
|
||||
"dauer_min": dauer, "tokens": {"input": tokens, "output": 0}, "agents": {}}
|
||||
|
||||
|
||||
def _stub_runner(antworten):
|
||||
"""params-abhängige Metriken; zählt echte Aufrufe (Cache-Treffer zählen nicht)."""
|
||||
calls = []
|
||||
|
||||
async def runner(params, thema):
|
||||
calls.append((dict(params), thema[0]))
|
||||
for muster, m in antworten:
|
||||
if muster(params):
|
||||
return dict(m)
|
||||
return _metrics()
|
||||
|
||||
runner.calls = calls
|
||||
return runner
|
||||
|
||||
|
||||
async def test_screening_filtert_rauschen(tmp_path):
|
||||
"""Nur Parameter mit Effekt über der Rausch-Schwelle kommen in die Feinphase;
|
||||
ein bestätigter Gewinner wird übernommen."""
|
||||
wirksam = "FACTS_CHUNK_SUBS"
|
||||
runner = _stub_runner([
|
||||
(lambda p: p.get(wirksam) == 8, _metrics(note=9.5, dauer=8.0)), # klar besser
|
||||
])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
best = await t.run()
|
||||
assert best.get(wirksam) == 8
|
||||
# kein anderer Parameter übernommen (alle anderen Δ=0 < Schwelle)
|
||||
assert set(best) == {wirksam}
|
||||
|
||||
|
||||
async def test_uebernahme_braucht_bestaetigung(tmp_path):
|
||||
"""Einmaliger Glückstreffer ohne bestätigten Zweitlauf wird verworfen."""
|
||||
zustand = {"mal": 0}
|
||||
|
||||
async def runner(params, thema):
|
||||
if params.get("FACTS_CHUNK_SUBS") == 8:
|
||||
zustand["mal"] += 1
|
||||
return _metrics(note=9.5) if zustand["mal"] == 1 else _metrics(note=8.0)
|
||||
return _metrics()
|
||||
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
best = await t.run()
|
||||
assert best == {}
|
||||
|
||||
|
||||
async def test_cache_resume_wiederholt_keine_trials(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
await t.run()
|
||||
erste = len(runner.calls)
|
||||
t2 = Trainer(tmp_path / "s", max_trials=999, max_stunden=1, runner=runner)
|
||||
await t2.run()
|
||||
assert len(runner.calls) == erste # alles aus trials.jsonl bedient
|
||||
|
||||
|
||||
async def test_budget_stoppt(tmp_path):
|
||||
runner = _stub_runner([])
|
||||
t = Trainer(tmp_path / "s", max_trials=3, max_stunden=1, runner=runner)
|
||||
await t.run()
|
||||
assert len(runner.calls) <= 3
|
||||
|
||||
|
||||
def test_score_richtungen():
|
||||
basis = _metrics()
|
||||
besser = _metrics(note=9.0)
|
||||
teurer = _metrics(dauer=20.0, tokens=2_000_000)
|
||||
assert score(besser, basis) > score(basis, basis)
|
||||
assert score(teurer, basis) < score(basis, basis)
|
||||
mit_befunden = _metrics(fremd=0.2, luecken=0.1)
|
||||
assert score(mit_befunden, basis) < score(basis, basis)
|
||||
198
backend/train.py
Normal file
198
backend/train.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""make train: Parameter-Optimierung auf Mini-Themen (Baseline → Screening → Koordinaten-Suche).
|
||||
|
||||
Jeder Trial ist ein Subprozess (train_lauf.py) mit CREATOR_PARAMS im ENV — so binden die
|
||||
Module die überschriebenen Werte beim Import. Metriken sind deterministisch (qa_report
|
||||
ohne LLM); gegen Judge-/Lauf-Rauschen gilt: Baseline mit Wiederholung liefert die
|
||||
Rausch-Schwelle, und eine Übernahme braucht einen BESTÄTIGUNGSLAUF (sonst Random Walk).
|
||||
|
||||
CLI: python3 train.py [--trials 40] [--stunden 8] [--sitzung NAME]
|
||||
Ergebnis: storage/train/<sitzung>/{trials.jsonl, report.md, beste_params.json}
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from config import STORAGE_DIR
|
||||
from train_params import PARAMS, schritte
|
||||
|
||||
HAUPT_THEMA = ("train-sort", "benchmarks/sortierverfahren")
|
||||
VALIDIER_THEMA = ("train-foto", "benchmarks/fotografie")
|
||||
# Score-Gewichte: Qualität + Auswahl dominieren (Entwicklungsphase), Kosten ziehen ab.
|
||||
W_NOTE, W_AUSWAHL, W_ZEIT, W_TOKEN = 4.0, 4.0, 1.0, 1.0
|
||||
|
||||
|
||||
def score(m: dict, basis: dict) -> float:
|
||||
"""Skalarer Vergleichswert eines Trials. note 0–10; auswahl aus den MECE-Quoten;
|
||||
Zeit/Tokens normiert auf die Baseline (1.0 = Baseline-Kosten)."""
|
||||
q = m.get("quoten") or {}
|
||||
qa_ = m.get("quoten_artefakte") or {}
|
||||
auswahl = 10.0 * max(0.0, 1.0 - min(1.0, (
|
||||
q.get("dubletten_verdacht", 0) + q.get("luecken", 0) + q.get("fremd", 0)
|
||||
+ qa_.get("sub_dubletten_verdacht", 0) + qa_.get("verwaiste", 0))))
|
||||
zeit = (m.get("dauer_min") or 0) / max(basis.get("dauer_min") or 1, 0.1)
|
||||
tok = _tokens(m) / max(_tokens(basis), 1)
|
||||
return round(W_NOTE * (m.get("note") or 0) + W_AUSWAHL * auswahl
|
||||
- W_ZEIT * 10 * zeit - W_TOKEN * 10 * tok, 2)
|
||||
|
||||
|
||||
def _tokens(m: dict) -> int:
|
||||
t = m.get("tokens") or {}
|
||||
return int(t.get("input") or 0) + int(t.get("output") or 0)
|
||||
|
||||
|
||||
class Trainer:
|
||||
def __init__(self, sitzung: Path, max_trials: int, max_stunden: float, runner=None):
|
||||
self.dir = sitzung
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
self.cache_pfad = self.dir / "trials.jsonl"
|
||||
self.cache: dict[str, dict] = {}
|
||||
if self.cache_pfad.exists(): # Resume: bezahlte Trials nie wiederholen
|
||||
for line in self.cache_pfad.read_text(encoding="utf-8").splitlines():
|
||||
e = json.loads(line)
|
||||
self.cache[e["key"]] = e["metrics"]
|
||||
self.max_trials = max_trials
|
||||
self.deadline = time.monotonic() + max_stunden * 3600
|
||||
self.gezahlt = 0
|
||||
self.runner = runner or self._subprozess
|
||||
self.log = []
|
||||
|
||||
# ── Trial-Ausführung ────────────────────────────────────────────────────────────
|
||||
def _key(self, params: dict, thema: tuple, tag: str = "") -> str:
|
||||
raw = json.dumps({"p": params, "t": thema[0], "tag": tag}, sort_keys=True)
|
||||
return hashlib.md5(raw.encode()).hexdigest()[:12]
|
||||
|
||||
async def trial(self, params: dict, thema: tuple = HAUPT_THEMA, tag: str = "") -> dict | None:
|
||||
"""tag unterscheidet bewusste Wiederholungen (Baseline n=2, Bestätigung)."""
|
||||
key = self._key(params, thema, tag)
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
if self.gezahlt >= self.max_trials or time.monotonic() > self.deadline:
|
||||
return None
|
||||
self.gezahlt += 1
|
||||
metrics = await self.runner(params, thema)
|
||||
if metrics is not None:
|
||||
with open(self.cache_pfad, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps({"key": key, "params": params, "thema": thema[0],
|
||||
"tag": tag, "metrics": metrics}, ensure_ascii=False) + "\n")
|
||||
self.cache[key] = metrics
|
||||
return metrics
|
||||
|
||||
async def _subprozess(self, params: dict, thema: tuple) -> dict | None:
|
||||
out = self.dir / f"metrics-{self._key(params, thema)}.json"
|
||||
env = {"CREATOR_PARAMS": json.dumps(params)}
|
||||
import os
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "train_lauf.py", thema[0], thema[1], str(out),
|
||||
env={**os.environ, **env})
|
||||
rc = await proc.wait()
|
||||
if rc != 0 or not out.exists():
|
||||
self._log(f"Trial fehlgeschlagen (rc={rc}, params={params})")
|
||||
return None
|
||||
return json.loads(out.read_text(encoding="utf-8"))
|
||||
|
||||
def _log(self, msg: str) -> None:
|
||||
line = f"{datetime.now(timezone.utc).isoformat()[11:19]} {msg}"
|
||||
print(line, flush=True)
|
||||
self.log.append(line)
|
||||
|
||||
# ── Trainings-Phasen ────────────────────────────────────────────────────────────
|
||||
async def run(self) -> dict:
|
||||
# Phase 0: Baseline zweimal → Score-Basis + Rausch-Schwelle
|
||||
self._log("Baseline (2 Läufe)…")
|
||||
b1 = await self.trial({}, tag="baseline-1")
|
||||
b2 = await self.trial({}, tag="baseline-2")
|
||||
if not b1 or not b2:
|
||||
self._log("Baseline unvollständig — Abbruch.")
|
||||
return {}
|
||||
self.basis = b1
|
||||
s1, s2 = score(b1, b1), score(b2, b1)
|
||||
self.rauschen = max(abs(s1 - s2), 0.5) # Mindest-Schwelle gegen Glücks-Übernahmen
|
||||
best_params: dict = {}
|
||||
best_score = max(s1, s2)
|
||||
self._log(f"Baseline-Score {s1}/{s2}, Rausch-Schwelle {self.rauschen}")
|
||||
|
||||
# Phase 1: Screening — je Parameter ±1 Schritt, Effekt vs. Rauschen
|
||||
effekte: list[tuple[float, str, float]] = [] # (|effekt|, name, bester_wert)
|
||||
for name in PARAMS:
|
||||
lo, hi = schritte(name)
|
||||
for wert in dict.fromkeys((lo, hi)): # lo==hi am Rand nur einmal
|
||||
if wert == PARAMS[name]["default"]:
|
||||
continue
|
||||
m = await self.trial({**best_params, name: wert})
|
||||
if m is None:
|
||||
continue
|
||||
delta = score(m, self.basis) - best_score
|
||||
self._log(f"Screening {name}={wert}: Δ{delta:+.2f}")
|
||||
if delta > self.rauschen:
|
||||
effekte.append((delta, name, wert))
|
||||
effekte.sort(reverse=True)
|
||||
self._log(f"Wirksam: {[(n, w) for _, n, w in effekte]}")
|
||||
|
||||
# Phase 2: Koordinaten-Suche über ALLE wirksamen Parameter (keine feste Obergrenze),
|
||||
# Übernahme nur nach Bestätigungslauf
|
||||
for _, name, start_wert in effekte:
|
||||
wert = start_wert
|
||||
p = PARAMS[name]
|
||||
richtung = p["step"] if wert > p["default"] else -p["step"]
|
||||
while True:
|
||||
kandidat = {**best_params, name: wert}
|
||||
m = await self.trial(kandidat)
|
||||
if m is None:
|
||||
break
|
||||
delta = score(m, self.basis) - best_score
|
||||
if delta <= self.rauschen:
|
||||
break
|
||||
m2 = await self.trial(kandidat, tag="bestaetigung")
|
||||
if m2 is None or score(m2, self.basis) - best_score <= self.rauschen:
|
||||
self._log(f"{name}={wert}: nicht bestätigt — verworfen")
|
||||
break
|
||||
best_params, best_score = kandidat, min(score(m, self.basis), score(m2, self.basis))
|
||||
self._log(f"ÜBERNOMMEN {name}={wert} → Score {best_score}")
|
||||
naechster = round(wert + richtung, 4)
|
||||
if not p["min"] <= naechster <= p["max"]:
|
||||
break
|
||||
wert = naechster
|
||||
|
||||
# Validierung auf dem zweiten Thema
|
||||
if best_params:
|
||||
v_base = await self.trial({}, thema=VALIDIER_THEMA, tag="val-base")
|
||||
v_best = await self.trial(best_params, thema=VALIDIER_THEMA, tag="val-best")
|
||||
if v_base and v_best:
|
||||
self._log(f"Validierung {VALIDIER_THEMA[0]}: Baseline {score(v_base, v_base)}"
|
||||
f" → Best {score(v_best, v_base)}")
|
||||
|
||||
self._schreibe_report(best_params, best_score)
|
||||
return best_params
|
||||
|
||||
def _schreibe_report(self, best_params: dict, best_score: float) -> None:
|
||||
from fsutil import atomic_write_json, atomic_write_text
|
||||
atomic_write_json(self.dir / "beste_params.json", best_params, indent=1)
|
||||
report = ["# Trainings-Report", "",
|
||||
f"Trials bezahlt: {self.gezahlt}/{self.max_trials}",
|
||||
f"Bester Score: {best_score} (Baseline-Rauschen {self.rauschen})",
|
||||
f"Beste Parameter: `{json.dumps(best_params, ensure_ascii=False)}`",
|
||||
"", "Nutzung: `CREATOR_PARAMS=$(cat beste_params.json) make dev` —",
|
||||
"Übernahme nach config.py bleibt eine manuelle Entscheidung.", "", "## Log", ""]
|
||||
report += [f"- {l}" for l in self.log]
|
||||
atomic_write_text(self.dir / "report.md", "\n".join(report))
|
||||
print(f"\nReport: {self.dir / 'report.md'}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--trials", type=int, default=40)
|
||||
ap.add_argument("--stunden", type=float, default=12.0)
|
||||
ap.add_argument("--sitzung", default=datetime.now(timezone.utc).strftime("%Y%m%d-%H%M"))
|
||||
args = ap.parse_args()
|
||||
trainer = Trainer(STORAGE_DIR / "train" / args.sitzung, args.trials, args.stunden)
|
||||
asyncio.run(trainer.run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
68
backend/train_lauf.py
Normal file
68
backend/train_lauf.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""EIN Trainings-Trial: frischer Prozess (CREATOR_PARAMS wirkt beim Import), ein
|
||||
kompletter Mini-Lauf, deterministische Metriken als JSON — danach ist das Topic weg.
|
||||
|
||||
CLI: python3 train_lauf.py <topic> <benchmark-location> <ausgabe.json>
|
||||
(benchmark-location repo-relativ, z. B. "benchmarks/sortierverfahren")
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import agents
|
||||
import database
|
||||
import qa
|
||||
from blocks import generate_blocks
|
||||
from fsutil import atomic_write_json
|
||||
from paths import source_path, topic_dir
|
||||
|
||||
|
||||
async def trial(topic: str, location: str, out: str) -> None:
|
||||
await database.init_db()
|
||||
agents.on_event = database.add_event # sonst keine Dauer-/Token-Events (main.py-lifespan-Pendant)
|
||||
try:
|
||||
await _aufraeumen(topic) # Reste eines abgebrochenen Trials
|
||||
await database.create_topic(topic)
|
||||
qp = source_path(topic)
|
||||
qp.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic_write_json(qp, {"type": "uni", "location": location, "spec": ""})
|
||||
start = datetime.now(timezone.utc)
|
||||
# qa_force=True: das Gate misst, pausiert den Trial aber nie
|
||||
await generate_blocks(topic, provider="minimax", research=True, qa_force=True)
|
||||
dauer_min = round((datetime.now(timezone.utc) - start).total_seconds() / 60, 1)
|
||||
report = await qa.qa_report(topic, llm=False) or {}
|
||||
lauf = report.get("lauf") or {}
|
||||
metrics = {
|
||||
"topic": topic,
|
||||
"note": report.get("note"),
|
||||
"note_artefakte": report.get("note_artefakte"),
|
||||
"quoten": report.get("quoten") or {},
|
||||
"quoten_artefakte": report.get("quoten_artefakte") or {},
|
||||
"bloecke": report.get("bloecke"),
|
||||
"dauer_min": lauf.get("dauer_min") or dauer_min,
|
||||
"tokens": (lauf.get("tokens") or {}),
|
||||
"agents": (lauf.get("agents") or {}),
|
||||
}
|
||||
atomic_write_json(Path(out), metrics, indent=1)
|
||||
finally:
|
||||
await _aufraeumen(topic)
|
||||
await database.close_db()
|
||||
|
||||
|
||||
async def _aufraeumen(topic: str) -> None:
|
||||
"""Topic restlos entfernen (DELETE-/topics-Sequenz aus routes.py)."""
|
||||
await database.delete_topic(topic)
|
||||
await database.delete_block_data(topic)
|
||||
await database.delete_topic_pipeline(topic)
|
||||
await database.delete_source(topic)
|
||||
await database.delete_guide_content(topic)
|
||||
shutil.rmtree(topic_dir(topic), ignore_errors=True)
|
||||
shutil.rmtree(qa.QA_DIR / topic, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 4:
|
||||
raise SystemExit("Nutzung: python3 train_lauf.py <topic> <benchmark-location> <ausgabe.json>")
|
||||
asyncio.run(trial(sys.argv[1], sys.argv[2], sys.argv[3]))
|
||||
56
backend/train_params.py
Normal file
56
backend/train_params.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""Suchraum fürs Training (make train): welche config-Parameter der Trainer bewegen darf.
|
||||
|
||||
Je Parameter: default (muss config spiegeln — Test prüft das), min/max (harte Ränder,
|
||||
flow-sicher), step (Schrittweite der Koordinaten-Suche), kategorie (welches Ziel er
|
||||
primär bewegt: qualitaet/auswahl/laufzeit/tokens). QA-/Detektor-Konstanten stehen
|
||||
bewusst NICHT hier — die Messlatte darf nie Teil des Suchraums sein.
|
||||
"""
|
||||
|
||||
PARAMS: dict[str, dict] = {
|
||||
# Recherche / Inventar
|
||||
"RESEARCH_THEMA_AGENTS": {"default": 5, "min": 2, "max": 8, "step": 1, "kategorie": "qualitaet"},
|
||||
"RESEARCH_READERS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet"},
|
||||
"RESEARCH_SECTION_CHARS": {"default": 12000, "min": 6000, "max": 24000, "step": 3000, "kategorie": "tokens"},
|
||||
"DEDUP_PAIR_FLOOR": {"default": 0.6, "min": 0.45, "max": 0.8, "step": 0.05, "kategorie": "auswahl"},
|
||||
"DEDUP_TITLE_AUTO": {"default": 0.95, "min": 0.9, "max": 0.99, "step": 0.01, "kategorie": "auswahl"},
|
||||
"DEDUP_GLOBAL_FLOOR": {"default": 0.65, "min": 0.5, "max": 0.8, "step": 0.05, "kategorie": "auswahl"},
|
||||
"DEDUP_PAIRS_CHUNK": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit"},
|
||||
"FILTER_CHUNK": {"default": 35, "min": 15, "max": 60, "step": 10, "kategorie": "laufzeit"},
|
||||
"FILTER_RECHECK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"CONSOLIDATION_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
# Subbausteine
|
||||
"SUBBLOCK_CHUNK": {"default": 10, "min": 4, "max": 20, "step": 2, "kategorie": "laufzeit"},
|
||||
"SUBBLOCK_MIN": {"default": 5, "min": 2, "max": 10, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_MAX_ROUNDS": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_EXTRA_ROUNDS": {"default": 2, "min": 0, "max": 4, "step": 1, "kategorie": "auswahl"},
|
||||
"SUBBLOCK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
# Facts / Artefakte / Fragen
|
||||
"FACTS_CHUNK_SUBS": {"default": 10, "min": 4, "max": 20, "step": 2, "kategorie": "laufzeit"},
|
||||
"FACTS_CHECK_PANEL": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"QUESTION_CHUNK_SUBS": {"default": 25, "min": 10, "max": 50, "step": 5, "kategorie": "laufzeit"},
|
||||
"QUESTION_MAX_ROUNDS": {"default": 3, "min": 1, "max": 5, "step": 1, "kategorie": "qualitaet"},
|
||||
"ARTEFACT_CHUNK_SUBS": {"default": 25, "min": 10, "max": 50, "step": 5, "kategorie": "laufzeit"},
|
||||
# Embedding-Schwellen (Auswahl-Kern)
|
||||
"SUB_VARIANT_COS": {"default": 0.90, "min": 0.85, "max": 0.96, "step": 0.01, "kategorie": "auswahl"},
|
||||
"SEED_COVER_COS": {"default": 0.80, "min": 0.7, "max": 0.9, "step": 0.02, "kategorie": "auswahl"},
|
||||
"SUB_DUP_KANDIDAT_COS": {"default": 0.75, "min": 0.65, "max": 0.85, "step": 0.02, "kategorie": "auswahl"},
|
||||
"EMBEDDING_BLOCK_FLOOR": {"default": 0.5, "min": 0.35, "max": 0.65, "step": 0.05, "kategorie": "auswahl"},
|
||||
"CROSS_CHUNK_PAARE": {"default": 40, "min": 15, "max": 80, "step": 10, "kategorie": "laufzeit"},
|
||||
# Guide
|
||||
"MAX_WRITER_ROUNDS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "qualitaet"},
|
||||
"GATE_FIX_MIN": {"default": 3, "min": 1, "max": 6, "step": 1, "kategorie": "qualitaet"},
|
||||
"WRITER_SPLIT_SUBS": {"default": 30, "min": 15, "max": 45, "step": 5, "kategorie": "qualitaet"},
|
||||
# Engine / Kosten
|
||||
"CONSENSUS_GRACE": {"default": 300, "min": 0, "max": 600, "step": 60, "kategorie": "laufzeit"},
|
||||
"MAX_RESTARTS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "laufzeit"},
|
||||
"EVIDENCE_BUDGET_CHARS": {"default": 48000, "min": 16000, "max": 64000, "step": 8000, "kategorie": "tokens"},
|
||||
"QUELLE_RELEVANZ_CHUNK": {"default": 12, "min": 6, "max": 24, "step": 3, "kategorie": "laufzeit"},
|
||||
}
|
||||
|
||||
|
||||
def schritte(name: str) -> tuple[float, float]:
|
||||
"""(wert_runter, wert_hoch) je einen step von default, an die Ränder geklemmt."""
|
||||
p = PARAMS[name]
|
||||
lo = max(p["min"], round(p["default"] - p["step"], 4))
|
||||
hi = min(p["max"], round(p["default"] + p["step"], 4))
|
||||
return lo, hi
|
||||
Reference in New Issue
Block a user