242 lines
14 KiB
Python
242 lines
14 KiB
Python
import os
|
||
from pathlib import Path
|
||
|
||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||
TEMPLATES_DIR = PROJECT_ROOT / "templates"
|
||
STORAGE_DIR = PROJECT_ROOT / "storage"
|
||
FRONTEND_DIST = PROJECT_ROOT / "frontend" / "dist"
|
||
DB_PATH = STORAGE_DIR / "creator.db"
|
||
PROJECTS_DIR = PROJECT_ROOT / "projects"
|
||
UNI_DIR = PROJECT_ROOT / "uni"
|
||
|
||
|
||
def _load_env(path: Path) -> None:
|
||
"""Mini .env loader (no dependency): KEY=VALUE lines. The FILE wins over inherited
|
||
env: a --reload master keeps its startup environment forever, so "existing env wins"
|
||
silently pinned stale values across .env edits (measured: file said 24, workers
|
||
inherited 15 for hours). Trade-off: ad-hoc shell overrides lose against the file."""
|
||
try:
|
||
text = path.read_text(encoding="utf-8")
|
||
except OSError:
|
||
return
|
||
for line in text.splitlines():
|
||
line = line.strip()
|
||
if not line or line.startswith("#") or "=" not in line:
|
||
continue
|
||
key, _, value = line.partition("=")
|
||
key, value = key.strip(), value.strip().strip('"').strip("'")
|
||
if key:
|
||
os.environ[key] = value
|
||
|
||
|
||
_load_env(PROJECT_ROOT / ".env")
|
||
|
||
MAX_CONCURRENT_GENERATIONS = 10
|
||
|
||
# Readability gate: deterministic checker (small German complexity model,
|
||
# scale 1–7). Sections that are too hard go into the read-exam revision.
|
||
# If transformers/torch or the model are missing → gate silently off.
|
||
READABILITY_ACTIVE = True
|
||
READABILITY_MODEL = "MiriUll/distilbert-german-text-complexity"
|
||
# Anchors on the 1–7 scale (TextComplexityDE): plain language ~1.2; Wikipedia average
|
||
# ~3.22; from MOS > 4 a sentence counts as "truly complex" (the paper's simplification cutoff).
|
||
READABILITY_MAX = 3.5 # section too hard when the sentence average is above this
|
||
READABILITY_HARD = 4.0 # an individual sentence is "hard" from here on
|
||
READABILITY_HARD_SHARE = 0.30 # … OR when this share of sentences is hard
|
||
|
||
# Kanban clustering: semantic embeddings drive the online title clustering and the
|
||
# candidate pairs of the pair check. If transformers/torch are missing or the model
|
||
# won't load → embedding silently off (all pairs go to the judge, clusters stay singletons).
|
||
EMBEDDING_AKTIV = True
|
||
EMBEDDING_MODELL = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" # CPU, multilingual, ~470 MB
|
||
# Stronger (larger) CPU alternative if needed: "BAAI/bge-m3".
|
||
# Consolidation = two-stage: (1) the embedding builds COARSE similarity blocks (high recall),
|
||
# (2) an LLM judge groups EACH block into the real blocks (merge paraphrases, split
|
||
# over-merges). Pure threshold blocking creates a giant component (everything chained) →
|
||
# hence "capped blocking": greedily merge by cosine, but cap the block size. This keeps the
|
||
# LLM lists short and stable (evidenced: embedding block + LLM judge ≈ 95% precision).
|
||
EMBEDDING_BLOCK_FLOOR = 0.5 # minimum cosine for two candidates to share ONE block
|
||
EMBEDDING_BLOCK_CAP = 25 # max. titles per block (keep the LLM list short/stable)
|
||
# Subblock dedup: purely deterministic (no LLM). Subblocks are short statements IN THE SAME
|
||
# block context — from this cosine on two are the same statement (checked on aak: ≥0.88 are
|
||
# without exception true duplicates). Conservative 0.90 so different aspects (∈NP ≠ NP-hard) stay separate.
|
||
EMBEDDING_SUB_DUP = 0.90
|
||
# Variant folding BEFORE the subblock consensus count: finders rephrase the same concept each
|
||
# round, so exact-norm counting starves real concepts (measured Markdown run: 623/965 mentions
|
||
# discarded, „Zeichenkodierung" 73/74). 0.90 folds true paraphrases at ~0 false folds (0.85/0.88
|
||
# fold distinct aspects like ** vs ***). Antonym pairs measure 0.91–0.95 → negation guard required.
|
||
SUB_VARIANT_COS = 0.90
|
||
# Seed coverage check is LEXICAL first (token containment) — seeds are short fragment NAMES,
|
||
# subs are statements: true covers measure 0.27–0.38 while a wrong hit measured 0.76. The
|
||
# embedding stage only backs up the lexical one (catches „Line Breaks (Soft)" 0.888).
|
||
SEED_COVER_COS = 0.80
|
||
|
||
# Umbrella grouping (block granularity level 2, step "Blocks-Gruppierung", AFTER the filter):
|
||
# collapse sibling DEFINITIONS that are components of ONE umbrella concept (TM model:
|
||
# Konfiguration/Start-/Folge-/Stopkonfiguration/Berechnung/Alphabet/δ; KNF: Literale/Klauseln/
|
||
# Variablen) into ONE block whose description ENUMERATES the children — so the later subblock step
|
||
# re-derives them from the source under the umbrella scope (demotion is re-derivation, not transfer).
|
||
# If the flag is off or no embedding model → step is silently skipped (like Dedup).
|
||
BLOCKS_GRUPPIERUNG_AKTIV = True
|
||
# Lower floor than consolidation (0.5, paraphrase-tuned) for higher sibling recall; the LLM judge is
|
||
# the precision gate. Smaller cap since a lower floor pulls in more nodes → keep the judge lists short.
|
||
EMBEDDING_SIBLING_FLOOR = 0.35 # heterogeneous facets of one model co-cluster weakly → low floor (recall)
|
||
EMBEDDING_SIBLING_CAP = 18 # a rich model (TM) can have many constituent parts
|
||
# Reconcile pass: two independently-judged clusters can emit the SAME parent concept under different
|
||
# titles (e.g. two „Turingmaschine"-umbrellas). Merge umbrella pairs whose title+description cosine is
|
||
# ≥ this (conservative → only true same-parent duplicates, never two distinct umbrellas).
|
||
GROUP_RECONCILE_FLOOR = 0.75
|
||
# Over-merge backstop ONLY (no-structure floor). Research (meronymy ≠ similarity): parts of ONE model are
|
||
# legitimately DISSIMILAR (TM: Alphabet/Konfiguration/δ ~0.22), while distinct same-type concepts (P/NP/…)
|
||
# are SIMILAR (~0.85) — so member-vs-member cosine is the WRONG instrument for over-merge (empirically
|
||
# inverted: TM 0.218 < the P/NP bundle 0.227). The real precision floor is the ATOMICITY type-guard
|
||
# (_GROUP_STANDALONE: a member that is a named algorithm/problem/theorem/complexity-class dissolves the
|
||
# umbrella). This floor is demoted to a near-zero backstop that only rejects a literally structureless
|
||
# chain (random-pair baseline), set BELOW the legitimate heterogeneous minimum so it never kills a real model.
|
||
GROUP_MIN_COS_FLOOR = 0.15
|
||
# Fragment-demote backstop, same logic as GROUP_MIN_COS_FLOOR: fragment↔parent cosine is a BAD
|
||
# fragment detector (measured, Markdown run: wrong demotes Blockzitate→Codeblöcke 0.353 and
|
||
# Zeichenkodierung→Überschriften 0.640 sit ABOVE any usable floor, while true NP proof-gadget
|
||
# demotes αu-Variablen→Cook/Levin 0.172 sit low). So this only vetoes judge/panel demotes with
|
||
# NO containment match whose pair is literally structureless (Emoji→Tabelle 0.136).
|
||
FRAGMENT_MIN_COS = 0.15
|
||
|
||
# Caps for concurrent CLI agent processes (env-overridable). Two nested limits, both always active:
|
||
# a per-topic cap and a global cap across all topics. Defaults 10/10 = previous behavior (global
|
||
# dominates). Locally raise the global cap to actually parallelize across topics (per-topic stays 10).
|
||
# Own lane for interactive calls (chat, elements) so they don't hang behind running writers.
|
||
MAX_CONCURRENT_AGENTS = int(os.getenv("MAX_CONCURRENT_AGENTS", "16")) # global, all topics
|
||
MAX_CONCURRENT_AGENTS_PER_TOPIC = int(os.getenv("MAX_CONCURRENT_AGENTS_PER_TOPIC", "12")) # per topic
|
||
MAX_CONCURRENT_INTERACTIVE = 8
|
||
|
||
# Grace window of the consensus races (blocks, guide, OnePager): after the first
|
||
# valid result the remaining agents may still become done for this many seconds
|
||
# (kill only once the minimum is already in).
|
||
CONSENSUS_GRACE = 300
|
||
|
||
# Research race: longer grace window. Research drives the whole block count;
|
||
# with slow providers (e.g. MiniMax) ALL 5 agents should become done, not just
|
||
# the quorum of 3. The per-agent timeout (TIMEOUTS["research"]=1800s) caps real hangs.
|
||
RESEARCH_GRACE = 900
|
||
|
||
# Cap of the clarification and check loops: maximum rounds until everything must be
|
||
# decided. In the last round the mapping agent MUST decide every entry;
|
||
# check loops leave any remaining objections standing after that.
|
||
CONSENSUS_MAX_ROUNDS = 3
|
||
|
||
# Crawler triage (content/noise) — deterministic rule filter instead of an LLM.
|
||
# Match: substring (lowercase) against URL AND file name. Order: keep > noise > min_chars > keep.
|
||
# Just add special rules here.
|
||
CRAWL_KEEP_PATTERNS = ["learn-unit", "learn-course"] # always content
|
||
CRAWL_NOISE_PATTERNS = [ # clearly off-topic → out
|
||
"clubs", "events", "podcasts", "resources", "-u-",
|
||
"academy", "pricing", "/plans", "career", "newsletter", "impressum", "login",
|
||
]
|
||
CRAWL_MIN_CHARS = 400 # too little text → out
|
||
|
||
# LLM topic relevance gate (after the rule filter): per content page yes/no against the spec.
|
||
# Separates the subject area (e.g. backend vs frontend), which the global CRAWL_* rules can't.
|
||
QUELLE_RELEVANZ_CHUNK = 12 # pages per rater package (small, since a snippet ships per page)
|
||
QUELLE_RELEVANZ_SNIPPET = 800 # body characters per page in the prompt (URL is the primary signal)
|
||
|
||
# Inline evidence for judge agents: corpus excerpts go INTO the prompt instead of letting
|
||
# every judge re-search the source folder (measured: ~10 tool turns/judge, 82 % of the
|
||
# run's tokens were cache reads from those loops).
|
||
EVIDENCE_BUDGET_CHARS = 48_000 # max excerpt characters per judge prompt
|
||
EVIDENCE_CTX_LINES = 15 # context lines around a cited source position (facts check)
|
||
|
||
# Timeouts per agent step: (base seconds, seconds per block/section).
|
||
# Applies equally to all providers — whoever is too slow gets restarted or overtaken.
|
||
TIMEOUTS = {
|
||
"research": (900, 0), # p95 measured 125 s (web mode); uni/link sections need headroom
|
||
"research_mapping": (600, 3), # n = pre-merged entries
|
||
"selection_mapping": (600, 2), # n = remaining entries (block inventory)
|
||
"ergaenzung": (600, 0), # subject-field extension for projects (web research)
|
||
"plan": (300, 5),
|
||
"plan_judge": (600, 5), # judge reads up to 5 outlines, n = sections
|
||
"content": (450, 30), # facts find/erg/fix — p95 measured 241 s (was 600+90n)
|
||
"content_check": (300, 10), # content exam per block in the package
|
||
"subblock": (400, 15), # finder round — p95 measured 124 s (was 900+45n)
|
||
"subblock_check": (300, 15), # judge decides contested subblocks in the chunk
|
||
"level": (300, 10), # classify subblocks per chunk
|
||
"level_check": (300, 10), # judge decides contested levels in the chunk
|
||
"relevance": (300, 10), # subblocks relevant/peripheral per chunk
|
||
"relevance_check": (300, 10), # judge decides contested relevance in the chunk
|
||
"question_pattern": (300, 15), # question patterns per block (subblocks × types)
|
||
"question_pattern_check": (300, 10), # critic cleans up the pattern table per block
|
||
"writer": (450, 60), # per section — split keeps sections ≤30 subs
|
||
"lese_check": (300, 10), # per section in the package
|
||
# guide board (per card = one block)
|
||
"lernziele": (300, 5), # backward-design objectives per block
|
||
"fakten_gate": (600, 5), # CoVe claim check per block
|
||
"coverage": (300, 5), # objective↔section mapping per block
|
||
}
|
||
|
||
# Purpose per format — flows into the outline judge (what the guide should achieve).
|
||
# German strings: these are inserted verbatim into the judge prompt → kept German on purpose.
|
||
FORMAT_PURPOSE = {
|
||
"Guide": "einen fokussierten Guide — alles Relevante ohne Randthemen",
|
||
"FullGuide": "einen Komplett-Guide — das ganze Thema inkl. Randthemen",
|
||
"Rest": "einen Ergänzungs-Guide — nur die Randthemen",
|
||
}
|
||
|
||
# Provider stacks: completely independent, any one can be removed at any time.
|
||
# Roles: "quick" = bulk work (research, classification),
|
||
# "fast" = interaction + voting (chat, exam, clarification, elements),
|
||
# "judge" = mapping/judge/check agents — cold (low temperature,
|
||
# no thinking) for stable verdicts; Claude/local map to "fast",
|
||
# "guide" = large generation (proposals, writer).
|
||
DEFAULT_PROVIDER = "claude"
|
||
PROVIDERS = {
|
||
"claude": {
|
||
"cli": "claude",
|
||
"guide": "claude-opus-4-8[1m]",
|
||
"fast": "claude-sonnet-4-6",
|
||
"judge": "claude-sonnet-4-6", # the CLI has no temperature setting
|
||
"quick": "claude-sonnet-4-6",
|
||
"env_key": None, # auth via CLAUDE_CODE_OAUTH_TOKEN or ~/.claude
|
||
},
|
||
# "minimax-kalt/…" is NOT its own stack, just an opencode provider entry
|
||
# (dev-ops/opencode.json) with low temperature; M3 there without thinking.
|
||
"minimax": {
|
||
"cli": "opencode",
|
||
"guide": "minimax/MiniMax-M3",
|
||
"fast": "minimax-kalt/MiniMax-M2.7-highspeed",
|
||
"judge": "minimax-kalt/MiniMax-M3",
|
||
"quick": "minimax/MiniMax-M2.7-highspeed",
|
||
"env_key": "MINIMAX_API_KEY",
|
||
},
|
||
"lokal": {
|
||
"cli": "opencode",
|
||
"guide": "ollama/qwen3.6:27b",
|
||
"fast": "ollama/qwen3.5:9b",
|
||
"judge": "ollama/qwen3.5:9b",
|
||
"quick": "ollama/qwen3.5:9b",
|
||
"env_key": None,
|
||
"check_url": "http://localhost:11434/api/tags", # Ollama reachable?
|
||
},
|
||
}
|
||
|
||
# Role routing: by DEFAULT the run's provider (the UI choice) handles ALL roles —
|
||
# the role only picks the model WITHIN that stack (PROVIDERS[stack][role]).
|
||
# Opt-in cross-provider mixing via env: ROLE_JUDGE=claude routes every judge call
|
||
# to the claude stack regardless of the UI choice ("provider:model" pins a model).
|
||
ROLE_ROUTING = {
|
||
"quick": os.getenv("ROLE_QUICK", ""),
|
||
"judge": os.getenv("ROLE_JUDGE", ""),
|
||
"guide": os.getenv("ROLE_GUIDE", ""),
|
||
"fast": os.getenv("ROLE_FAST", ""),
|
||
}
|
||
|
||
|
||
def resolve_role(run_provider: str, role: str) -> tuple[str, str]:
|
||
"""→ (provider, model) for one agent call. Pure routing, no availability check —
|
||
the caller (agents.run_agent) falls back to run_provider if the target is unavailable."""
|
||
target = ROLE_ROUTING.get(role, "") or run_provider
|
||
provider, _, model = target.partition(":")
|
||
if provider not in PROVIDERS:
|
||
provider, model = run_provider, ""
|
||
if not model:
|
||
model = PROVIDERS.get(provider, {}).get(role, "")
|
||
return provider, model
|