marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-04 12:21:45 +02:00
parent 2f5d5b9ca1
commit 8d8f6c8e51
43 changed files with 1920 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
backend/blocks.py
View File

@@ -39,45 +39,14 @@ from textkit import (
_resolve_title, _title_index, clean_title,
)
# Chunk the subblocks (web search per block): 1 agent per ~10 blocks, capped.
SUBBLOCK_CHUNK = 10
SUBBLOCK_MAX = 40
# Classifying is cheap (short verdict, no web search) → larger packages, fewer files/agents.
LEVEL_CHUNK = 100
# Research: fixed file batches instead of a search loop → each crawl page is assigned exactly once.
RESEARCH_BATCH = 20 # crawl pages per batch
RESEARCH_READERS = 2 # reader agents per batch (consensus ≥2 within the batch)
RESEARCH_THEMA_AGENTS = 5 # web mode (source "thema", no crawl folder)
# uni/projekt: chunk the script text into sections of ~this size (against lost-in-the-middle on
# large documents). ~12k chars ≈ 3k tokens → safely below the recall-drop threshold.
RESEARCH_SECTION_CHARS = 12000
# Triage (content/noise) is now a deterministic rule filter (config.CRAWL_*).
SUBBLOCK_CAP = 900 # subblock find loop per chunk (15 min)
SUBBLOCK_MIN = 5 # below this consensus count a block gets focused catch-up rounds
SUBBLOCK_EXTRA_ROUNDS = 2 # max catch-up rounds (saturation stop still applies — thin stays thin)
SUBBLOCK_MAX_ROUNDS = 3 # hard round cap: measured, rounds 4–5 burned 29 % of the finder agents
# for ~zero consensus gain (fringe ideas never saturate) — thin blocks
# are caught by the SUBBLOCK_MIN catch-up plus the gap follow-up round
CONSOLIDATION_CHUNK = 600 # up to here ONE global judge (dedups everything); above that chunked + merge pass — fallback path only
DEDUP_PAIR_FLOOR = 0.6 # min cosine for a candidate pair (complete-link aggregates → no chaining)
DEDUP_PAIRS_CHUNK = 40 # pairs per judge package (pairwise verification instead of a block mixer)
DEDUP_TITLE_AUTO = 0.95 # near-identical TITLE cosine ⇒ same entity → merge without the judge (recall net)
DEDUP_GLOBAL_FLOOR = 0.65 # global post-naming dedup stage: candidate floor above the 0.5-0.65
# same-domain noise band, below the sibling zone (~0.85) — the judge decides there
FILTER_CHUNK = 35 # blocks to assess per judge in the degrade pass (full list as context)
# Balance question-pattern chunks by sub load via LPT (makespan), not by block count.
QUESTION_CHUNK_SUBS = 25 # target sum of relevant subs per chunk — at 50 the generator
# skipped so many subs that 60 % of all question calls were catch-up
QUESTION_MAX_ROUNDS = 3 # catch-up rounds for subs without a pattern (the LLM omits ~18 % per chunk)
FACTS_CHUNK_SUBS = 10 # facts extraction: small chunks — the 4 phases (find/erg/check/fix) are
# serial PER CHUNK, so chunk count = parallelism; the makespan tail of a
# late block is bounded by ONE chunk's phase chain, not the whole block
ARTEFACT_CHUNK_SUBS = 25 # flashcards/examples: bulk generation, phases are cheap → bigger packages
FACTS_CHECK_PANEL = 3 # judges per chunk in the facts check (majority objects)
CONSOLIDATION_PANEL = 3 # mapping judges per chunk (panel → reconcile instead of a single judge)
SUBBLOCK_PANEL = 3 # source judges in the subblock clarification (majority instead of a single judge)
FILTER_RECHECK_PANEL = 3 # judges in the survivor-recheck (rare-positive "fragment" recall; majority ≥2)
# Pipeline-Tuning-Konstanten liegen zentral in config.py (tunebar via CREATOR_PARAMS).
from config import ( # noqa: E402
ARTEFACT_CHUNK_SUBS, CONSOLIDATION_CHUNK, CONSOLIDATION_PANEL, DEDUP_GLOBAL_FLOOR,
DEDUP_PAIR_FLOOR, DEDUP_PAIRS_CHUNK, DEDUP_TITLE_AUTO, FACTS_CHECK_PANEL, FACTS_CHUNK_SUBS,
FILTER_CHUNK, FILTER_RECHECK_PANEL, LEVEL_CHUNK, QUESTION_CHUNK_SUBS, QUESTION_MAX_ROUNDS,
RESEARCH_BATCH, RESEARCH_READERS, RESEARCH_SECTION_CHARS, RESEARCH_THEMA_AGENTS,
SUBBLOCK_CAP, SUBBLOCK_CHUNK, SUBBLOCK_EXTRA_ROUNDS, SUBBLOCK_MAX, SUBBLOCK_MAX_ROUNDS,
SUBBLOCK_MIN, SUBBLOCK_PANEL)
log = logging.getLogger("creator.blocks")
@@ -608,6 +577,16 @@ def _reply_text(result) -> str:
return (result[1] or "") if result else ""
def _sink_or_file(result, path: Path, schema):
"""Text reply preferred, file fallback. Research agents used to WRITE their big JSON —
measured: one facts call spent 40 of 64 turns in a write/validate/repair loop (9 min).
As text, parse_json_text repairs the escaping in one pass; tools stay for research."""
val = _sink_json(result, path, schema)
if val is not None:
return val
return schema(_json_file(path))
def _sink_json(result, path: Path, schema):
"""Payload validator for no-tool agents: the JSON comes as reply TEXT; the engine
persists it to `path`, so resume guards and audit files keep working unchanged."""
@@ -1502,6 +1481,38 @@ async def _luecken_runde(ctx: GenContext, files: dict, title: str, luecken: list
return len(kept)
async def _facts_nachfass(ctx: GenContext, files: dict, raw: dict, facts_map: dict, q: dict,
folder, instructions: str = "", ns: str = "", lbl: str = "",
sources: list[str] | None = None) -> int:
"""ONE slim facts round for consensus subs WITHOUT a facts entry — renames during
consolidation and catalog rows left 35 % of the subs ungrounded; the fact gate then
flagged their (correct) guide statements wholesale. The subs themselves stay either
way: they exist by consensus, only the grounding is fetched. → count of filled subs."""
fehlend = {bt: [s for s in subs if _norm_title(s) not in (facts_map.get(bt) or {})]
for bt, subs in raw.items()}
fehlend = {bt: subs for bt, subs in fehlend.items() if subs}
if not fehlend:
return 0
nf_dir = files["arbeit"] / "nf2"
nf_dir.mkdir(parents=True, exist_ok=True)
res = await _facts_block(ctx, lambda *a, **k: None, {**files, "arbeit": nf_dir},
fehlend, q, folder, instructions,
ns=f"{ns}nf2-", lbl=lbl, sources=sources, slim=True)
if ctx.is_cancelled() or res is None:
return 0
nf_facts, _discarded = res # discard verdicts ignored — consensus subs are not removed here
filled = 0
for bt, fm in nf_facts.items():
bfacts = facts_map.setdefault(bt, {})
for sn, fk in fm.items():
if sn not in bfacts and (fk.get("key_points") or fk.get("cited_facts")):
bfacts[sn] = fk
filled += 1
if filled:
_log(ctx.topic, f"Facts-Nachfass{': ' + lbl if lbl else ''}{filled} Subs nachbelegt")
return filled
def _subs_hash(sidecar_or_raw: dict) -> str:
"""Sub-set identity for the resume files of the sub-CONSUMING stages (levels/relevance/
questions/artefacts). Without it a re-run with a recut sub set adopted the stale stage
@@ -1811,7 +1822,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
ctx, f"{lbl}Facts {ci}", key=f"blocks-{topic}-{ns}facts-c{ci}",
prompt=_prompt("Facts-Research", topic=topic, source=source, blocks=block_text(idxs), out_path=fp, extra=_extra(instructions)),
role="quick", capabilities=caps,
payload=lambda result, p=fp: _facts_schema(_json_file(p)),
payload=lambda result, p=fp: _sink_or_file(result, p, _facts_schema),
timeout=_timeout("content", subs_total))
return status != FAILED and _facts_schema(_json_file(fp)) is not None
@@ -1842,7 +1853,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
ctx, f"{lbl}Facts supplement {ci}", key=f"blocks-{topic}-{ns}facts-erg-c{ci}",
prompt=_prompt("Facts-Supplement", topic=topic, source=source, blocks=block, out_path=ep, extra=_extra(instructions)),
role="quick", capabilities=caps,
payload=lambda result, p=ep: _facts_schema(_json_file(p)),
payload=lambda result, p=ep: _sink_or_file(result, p, _facts_schema),
timeout=_timeout("content", subs_total))
if not slim:
@@ -1929,7 +1940,7 @@ async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, inst
ctx, f"{lbl}Facts-Fix {ci}", key=f"blocks-{topic}-{ns}facts-fix-c{ci}",
prompt=_prompt("Facts-Research", topic=topic, source=source, blocks="\n\n".join(goal), out_path=fix_path(ci), extra=_extra(instructions)),
role="quick", capabilities=caps,
payload=lambda result, p=fix_path(ci): _facts_schema(_json_file(p)),
payload=lambda result, p=fix_path(ci): _sink_or_file(result, p, _facts_schema),
timeout=_timeout("content", len(subs_norm)))
await _gather_progress([_fix(ci) for ci in correctable], len(correctable), _report_p(set_p, topic, "Facts fix"))
if is_cancelled():
@@ -2121,7 +2132,7 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar:
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=block,
out_path=fp, extra=_extra(instructions)),
role="quick", capabilities="files",
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
payload=lambda result, p=fp: _sink_or_file(result, p, _question_pattern_chunk_schema),
timeout=_timeout("question_pattern", subs_total),
)
if status == FAILED:
@@ -2255,7 +2266,7 @@ async def _question_pattern_block(ctx: GenContext, set_p, files: dict, sidecar:
prompt=_prompt("Question-Pattern-Research", topic=topic, blocks=_followup_block(items),
out_path=fp, extra=_extra(instructions)),
role="quick", capabilities="files",
payload=lambda result, p=fp: _question_pattern_chunk_schema(_json_file(p)),
payload=lambda result, p=fp: _sink_or_file(result, p, _question_pattern_chunk_schema),
timeout=_timeout("question_pattern", subs_total),
)
@@ -3013,8 +3024,8 @@ async def _learning_order(ctx: GenContext, set_p, files: dict, entries: dict, va
pp = files["arbeit"] / "outline-prereqs.json"
def _payload(result, p=pp):
d = _json_file(p)
return d if isinstance(d, dict) and "prereqs" in d else None
d = _sink_or_file(result, p, lambda x: x if isinstance(x, dict) and "prereqs" in x else None)
return d
existing = _json_file(pp)
if not (isinstance(existing, dict) and "prereqs" in existing):
@@ -3052,7 +3063,7 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
ctx, f"Outline {i}", key=f"blocks-{topic}-outline-{i}",
prompt=_prompt("Guide-Outline", topic=topic, blocks=liste, out_path=path, extra=_extra(instructions)),
role="guide", capabilities="files",
payload=lambda result, p=path: _outline_schema(_json_file(p), valid),
payload=lambda result, p=path: _sink_or_file(result, p, lambda d: _outline_schema(d, valid)),
timeout=_timeout("plan", len(entries)))
return _outline_schema(_json_file(path), valid) is not None
@@ -3078,7 +3089,8 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
purpose="alle Blocks in einem roten Faden", n=len(proposals),
blocks=liste, outlines=block_texts, out_path=files["outline"], extra=_extra(instructions)),
role="judge", capabilities="files",
payload=lambda result: _outline_schema(_json_file(files["outline"]), valid),
payload=lambda result: _sink_or_file(result, files["outline"],
lambda d: _outline_schema(d, valid)),
timeout=_timeout("plan_judge", len(entries)))
plan = _outline_schema(_json_file(files["outline"]), valid) or proposals[0]
@@ -3097,8 +3109,8 @@ async def _outline_block(ctx: GenContext, set_p, files: dict, entries: dict, ins
prompt=_prompt("Guide-Outline-Review", topic=topic, chapters=chapter_text,
out_path=rp, extra=_extra(instructions)),
role="judge", capabilities="files",
payload=lambda result: _outline_review_schema(
_json_file(rp), valid, len(plan["chapters"]), len(entries)),
payload=lambda result: _sink_or_file(result, rp, lambda d: _outline_review_schema(
d, valid, len(plan["chapters"]), len(entries))),
timeout=_timeout("plan_judge", len(entries)))
moves = _outline_review_schema(_json_file(rp), valid, len(plan["chapters"]), len(entries))
for nr, target in (moves or {}).items():
@@ -3257,7 +3269,7 @@ async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i
ctx, f"{lbl}{_ARTEFACT_STEP[typ]} {ci}", key=f"blocks-{topic}-{ns}artifact-{typ}-c{ci}",
prompt=_prompt(_ARTEFACT_PROMPT[typ], topic=topic, blocks=block_text(idxs), out_path=p, extra=_extra(instructions)),
role="guide", capabilities="files",
payload=lambda result, p=p: schema(_json_file(p)),
payload=lambda result, p=p: _sink_or_file(result, p, schema),
timeout=_timeout("content", sum(len(blocks[i][1]) for i in idxs)))
return schema(_json_file(p)) is not None