marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-06 14:44:20 +02:00
parent f4c5116abb
commit cc27e53b9e
20 changed files with 504 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
backend/blocks.py
View File

@@ -24,11 +24,11 @@ from pathlib import Path
import database as db
import embedding
from agents import kill_process, cancel_scope, clear_scope
from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, SUB_VARIANT_COS, SUBBLOCK_MAX, EVIDENCE_BUDGET_CHARS, EVIDENCE_CTX_LINES
from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, SUB_VARIANT_COS, SUBBLOCK_MAX, EVIDENCE_BUDGET_CHARS, EVIDENCE_CTX_LINES
from fsutil import atomic_write_json
from jsonio import parse_json_text, read_json_file as _json_file
from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder
from crawl import crawl
from crawl import load_pages
from pipeline import (
GenContext, _extra, _gather_progress, _yesno_schema, _log, _prompt, _race,
_semaphore, _timeout, run_single_slot,
@@ -853,117 +853,10 @@ def _crawl_index(folder) -> dict[str, str]:
return idx
def _triage_rules(folder, pages: list[str]) -> tuple[list[str], list[str]]:
"""Deterministic content/noise filter (config.CRAWL_*). Substring match (lowercase) against
URL + filename. Order: keep > noise > min_chars > keep. → (content, noise)."""
folder = Path(folder)
content, noise = [], []
for fn in pages:
lines = _read(folder / fn).splitlines()
url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else ""
body = "\n".join(lines[1:]).strip()
hay = f"{url}\n{fn}".lower()
if any(p in hay for p in CRAWL_KEEP_PATTERNS):
content.append(fn)
elif any(p in hay for p in CRAWL_NOISE_PATTERNS):
noise.append(fn)
elif len(body) < CRAWL_MIN_CHARS:
noise.append(fn)
else:
content.append(fn) # default: keep — everything with content stays
return content, noise
def _page_snippet(folder, fn: str) -> tuple[str, str]:
"""(url, snippet) of a crawl page for the relevance gate. url from the QUELLE: line;
snippet = body excerpt (navigation boilerplate is up front — the prompt ignores it).
The URL is the primary signal (meaningful slug), the snippet only supports it."""
lines = _read(Path(folder) / fn).splitlines()
url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else ""
body = "\n".join(lines[1:]).strip()
snippet = " ".join(body.split())[:QUELLE_RELEVANZ_SNIPPET]
return (url or fn), snippet
async def _relevance_triage(ctx: GenContext, set_p, files: dict, folder, content: list[str], spec: str, instructions: str) -> tuple[list[str], list[str]]:
"""LLM topic gate after the rule filter: each content page ja/nein against the spec.
Off-topic (different field) → out. Pattern like `_relevance_block`: small packages, 3 raters
(`fast`), 2-of-3 consensus. CONSERVATIVE: drop only on a clear "nein" majority; dispute/gap/
race error → keep. SAFETY: if the gate would drop ≥80 % (or all), everything stays
(a spec mismatch/bug must not empty the source). → (kept, out) as filenames."""
topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled
work_dir = files["arbeit"]
pages = sorted(content)
if not pages:
return content, []
items = [_page_snippet(folder, fn) for fn in pages] # index aligns with `pages`
chunks = _chunk_nums(list(range(len(pages))), _n_chunks(len(pages), QUELLE_RELEVANZ_CHUNK))
n = len(chunks)
def rater_paths(c):
return [work_dir / f"source-relevance-c{c}-{i}.json" for i in (1, 2, 3)]
def lset(idxs):
return set(range(1, len(idxs) + 1))
async def _rate(c, idxs):
local_set = lset(idxs)
paths = rater_paths(c)
existing = sum(1 for p in paths if _yesno_schema(_json_file(p), local_set))
if existing >= 2:
return True
enum_lines = []
for k, j in enumerate(idxs, 1):
url, snip = items[j]
enum_lines.append(f"{k}. {url}")
if snip:
enum_lines.append(f" {snip}")
enum = "\n".join(enum_lines)
pending = [(i, p) for i, p in enumerate(paths, 1) if not _yesno_schema(_json_file(p), local_set)]
slots = [{
"key": f"blocks-{topic}-source-relevance-c{c}-{i}",
"prompt": _prompt("Source-Relevance", topic=topic, spec=spec, pages=enum, out_path=p, extra=_extra(instructions)),
"role": "fast", "capabilities": "files",
"payload": (lambda result, p=p, ids=local_set: _yesno_schema(_json_file(p), ids)),
} for i, p in pending]
new = await _race(topic, f"Relevance triage package {c}", slots, 2 - existing, _timeout("relevance", len(idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE)
return not is_cancelled() and new is not None
_qidx = _step_idx(topic, "Source prep") # gate runs in the source step (no own step)
set_p(f"Check relevance against spec ({n} packages)…", step=_qidx)
async def _report_triage(d, t):
set_p(f"Check relevance against spec {d}/{t}…", step=_qidx)
await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], n, _report_triage)
if is_cancelled():
return content, [] # cancel → drop nothing (caller aborts)
# Vote per page: only a clear "nein" majority (≥2 and more than "ja") throws it out.
dropped: list[str] = []
for c, idxs in enumerate(chunks, 1):
local_set = lset(idxs)
rater = [d for p in rater_paths(c) if (d := _yesno_schema(_json_file(p), local_set))]
for k in range(1, len(idxs) + 1):
vote_list = [d[k] for d in rater if k in d]
nein, ja = vote_list.count("nein"), vote_list.count("ja")
if nein >= 2 and nein > ja:
dropped.append(pages[idxs[k - 1]])
if dropped and len(dropped) >= max(1, int(len(pages) * 0.8)):
_log(topic, f"Relevance triage: would drop {len(dropped)}/{len(pages)} — discarded (spec mismatch?), keeping all")
return content, []
dropped_set = set(dropped)
keepers = [fn for fn in pages if fn not in dropped_set]
return keepers, dropped
async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool:
"""Step "Source prep": crawl (link) + PDF convert + content/noise triage.
Persists the triage in the coverage table (content). → True (ok) / False (cancel/error).
thema: nothing. projekt/uni: only PDFs (curated folder, no triage)."""
"""Step "Source prep": load link pages + PDF convert. → True (ok) / False (cancel/error).
link: load each user-supplied URL (one page, no following); ALL pages count as content
(curated — no triage). thema: nothing. projekt/uni: only PDFs (curated folder)."""
topic, is_cancelled = ctx.topic, ctx.is_cancelled
if not folder:
return True # thema → no source to prepare
@@ -973,28 +866,21 @@ async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder,
if await db.get_step_status(topic, "Source prep") == "done":
return True
if not _crawl_done(topic):
set_p("Loading source (crawl)…", step=_step_idx(topic, "Source prep"))
n = await asyncio.to_thread(crawl, q["location"], folder, cancelled=is_cancelled)
urls = [ln.strip() for ln in q["location"].splitlines() if ln.strip()]
set_p("Loading pages…", step=_step_idx(topic, "Source prep"))
n = await asyncio.to_thread(load_pages, urls, folder, cancelled=is_cancelled)
if is_cancelled():
return False
if not n:
_blocks_errors[topic] = "Crawl yielded no content — check link/domain"
_blocks_errors[topic] = "Links yielded no content — check the URLs"
return False
await asyncio.to_thread(_convert_pdfs, folder)
pages = sorted(set(_crawl_index(folder).values()))
if pages:
set_p("Triaging pages…", step=_step_idx(topic, "Source prep"))
# Curated links: the user picked them — keep every page, no rule/LLM triage.
await db.delete_coverage(topic)
content, noise = _triage_rules(folder, pages) # deterministic rule filter
if q.get("spec") and content: # topic gate: separates the field (rules can't)
content, dropped = await _relevance_triage(ctx, set_p, files, folder, content, q["spec"], instructions)
if is_cancelled():
return False
if dropped:
noise = sorted(set(noise) | set(dropped))
_log(topic, f"LLM relevance: {len(dropped)} pages off-topic → noise")
await db.mark_content(topic, sorted(content), sorted(noise))
_log(topic, f"Triage: {len(content)} content / {len(noise)} noise of {len(pages)} (rules + LLM gate)")
await db.mark_content(topic, pages, [])
_log(topic, f"Links: {len(pages)} pages as content (curated, no triage)")
await db.set_step_status(topic, "Source prep", "done")
return True
@@ -1021,17 +907,41 @@ def _aspect_marker(title: str) -> int:
_REFERENCE_RE = re.compile(r'^(Satz|Lemma|Korollar|Bemerkung|Definition)\s*[\d.]+\s*(\([a-z]\)|[a-z])?\s*$', re.I)
# Catalogue scaffolding around a concept name: a leading label+number ("Satz 7.13", "Satz 7.6:")
# or a trailing "(Definition 6.19)". _reference_strip peels it off and returns the concept remainder;
# _is_reference is then just "nothing survives the strip". Generic — no author/domain whitelist. The
# remainder feeds the rename step (a card must be titled by its concept, not its catalogue number).
_REF_WORDS = (r'(?:Satz|Lemma|Korollar|Corollary|Bemerkung|Beobachtung|Definition|Def'
r'|Theorem|Proposition|Folgerung|Kapitel|Abschnitt)')
_REF_PREFIX_RE = re.compile(rf'^\s*{_REF_WORDS}\s*\d+(?:\.\d+)*[a-z]?\s*[:.\-–—]?\s*', re.I)
_REF_SUFFIX_RE = re.compile(rf'\s*[\(\[]\s*{_REF_WORDS}\s*\d+(?:\.\d+)*[a-z]?\s*[\)\]]\s*$', re.I)
_WHOLE_PAREN_RE = re.compile(r'^\s*[\(\[]\s*(.{2,}?)\s*[\)\]]\s*$')
def _reference_strip(title: str) -> str:
"""Concept remainder of a catalogue-referenced title, scaffolding removed. '' for a pure reference
without a concept ('Bemerkung 7.22' → ''); the concept otherwise ('Satz 7.13 (Christofides)' →
'Christofides', 'Satz 7.6: Kriterium für X' → 'Kriterium für X', 'N P … (Definition 6.19)' →
'N P …'). A title without any catalogue reference comes back unchanged."""
t = (title or "").strip()
t = _REF_SUFFIX_RE.sub("", t).strip()
m = _REF_PREFIX_RE.match(t)
if m:
t = t[m.end():].strip()
if (w := _WHOLE_PAREN_RE.match(t)): # leading label left a lone '(Christofides)'
t = w.group(1).strip()
return t
def _is_reference(title: str) -> bool:
"""True for pure reference/placeholder titles WITHOUT meaningful content: "Satz 7.18", "Lemma 6.2",
"Korollar 6.18" (number without a name) as well as marked spots "Bedingung (**)". NOT "Satz 6.24:
Cook/Levin" (has a name) and NOT short technical symbols like "P⊆NP"/"Σ*" (real concepts)."""
"""True for a pure reference/placeholder title WITHOUT concept content: "Satz 7.18", "Lemma 6.2",
"Korollar 6.18" (number, no name) and marked spots "Bedingung (**)". False once a concept survives
the catalogue strip ("Satz 7.13 (Christofides)", "Satz 6.24: Cook/Levin") and for short technical
symbols like "P⊆NP"/"Σ*" (real concepts, no reference scaffolding)."""
t = title.strip()
if _REFERENCE_RE.match(t):
return True
if re.search(r'\(\*+\)', t): # marked spot "(**)" / "(*)"
return True
return False
return bool(_REFERENCE_RE.match(t)) or not _reference_strip(t)
def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) -> dict:
@@ -1118,6 +1028,15 @@ _CANON_STOP = re.compile(
r'von|of|für|for|und|and|zum|zur|im)\b', re.I)
# German compounds are head-final: "Cliquenproblem" = stem + optional Fuge + head noun "problem".
# The standalone wrapper "problem" is already in _CANON_STOP, but a GLUED head hides the stem from ER
# blocking ("Cliquenproblem" would never share a key with "Clique"). Strip the glued "problem" head so
# the stem surfaces — a generic suffix rule, not a term list. The Fuge is only a consonant n/s
# ("Cliquen-", "Entscheidungs-"): matching a vowel "e" would eat the stem's own "e" ("clique" → "cliqu").
# The ≥3-char stem lookbehind keeps a card literally titled "Problem"/"Probleme" (→ _CANON_STOP) intact.
_CANON_GLUED_HEAD = re.compile(r'(?<=\w{3})[ns]?problem(?:en|e|s)?\b', re.I)
# Catalogue references ("Definition 6.19", "Satz 7.8") are scaffolding INCLUDING their number —
# stripped as a phrase, so the digits don't pollute the key. Variant digits ("3-SAT") have no
# scaffolding word in front and survive.
@@ -1152,6 +1071,7 @@ def _canonical_key(title: str) -> str:
s = _CANON_CATALOGUE.sub(' ', s)
s = re.sub(r'≟|\bversus\b|\bvs\.?\b|=', ' opeq ', s) # equality / "vs" → one token
s = re.sub(r'≤|⪯|→|⇒|⟹|\breduces?\s+to\b|\breduziert\b', ' opred ', s) # reduction → one token
s = _CANON_GLUED_HEAD.sub(' ', s) # glued "…problem" head → stem
s = _CANON_STOP.sub(' ', s)
s = re.sub(r'[^\w ]', ' ', s) # drop punctuation/symbols
return " ".join(sorted(t for t in s.split() if t))
@@ -1340,16 +1260,21 @@ _REDUCTION_ONLY_OP = re.compile(r'[≤⪯]|→|⇒|⟹|->') # genuine reduction
def _is_reduction_statement(title: str) -> bool:
"""True if the title is a reduction between two NAMED problems (both sides carry ≥3 letters and
neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP" and proof-size "|A| = O(m)"."""
neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP", proof-size "|A| = O(m)", and a
construction/assignment suffix after the target ("3-SAT ≤ K-COLOR: G=(V,E) Konstruktion" is a
construction fragment, not a standalone statement — a bare target carries no ':' or '=')."""
t = _REL_STRIP.sub('', title, count=1)
m = _REDUCTION_ONLY_OP.search(t)
if not m:
return False
right = t[m.end():]
if ':' in right or '=' in right: # construction/assignment detail after the target → fragment
return False
def _named(s: str) -> bool:
return len(re.findall(r'[a-zäöüß]', s, re.I)) >= 3 and not re.match(r'\s*[Oo]\s*\(', s)
return _named(t[:m.start()]) and _named(t[m.end():])
return _named(t[:m.start()]) and _named(right)
def _is_named_statement(title: str, desc: str = "") -> bool: