From cc27e53b9eeb32d62539bb7a1f28f3155f59687d Mon Sep 17 00:00:00 2001 From: team3 Date: Mon, 6 Jul 2026 14:44:20 +0200 Subject: [PATCH] update --- backend/blocks.py | 193 ++++++++---------------- backend/board_inventory.py | 44 ++++-- backend/config.py | 15 -- backend/crawl.py | 74 +++------ backend/models.py | 4 +- backend/qa.py | 57 ++++++- backend/routes.py | 17 +-- backend/tests/test_board_inventory.py | 109 ++++++++++++- backend/tests/test_qa.py | 50 +++++- backend/tests/test_source_links.py | 55 +++++++ backend/tests/test_textkit.py | 57 +++++++ backend/textkit.py | 51 ++++++- backend/train_params.py | 1 - frontend/src/components/SourceForm.vue | 9 +- templates/Prompt/Blocks-Klaerung.md | 2 +- templates/Prompt/Blocks-Naming-Check.md | 1 + templates/Prompt/Blocks-Naming.md | 1 + templates/Prompt/Blocks-Research.md | 6 +- templates/Prompt/Blocks-Sanierung.md | 2 +- templates/Prompt/Source-Relevance.md | 23 --- 20 files changed, 504 insertions(+), 267 deletions(-) create mode 100644 backend/tests/test_source_links.py create mode 100644 backend/tests/test_textkit.py delete mode 100644 templates/Prompt/Source-Relevance.md diff --git a/backend/blocks.py b/backend/blocks.py index 0205536..b35963c 100644 --- a/backend/blocks.py +++ b/backend/blocks.py @@ -24,11 +24,11 @@ from pathlib import Path import database as db import embedding from agents import kill_process, cancel_scope, clear_scope -from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, CRAWL_KEEP_PATTERNS, CRAWL_NOISE_PATTERNS, CRAWL_MIN_CHARS, QUELLE_RELEVANZ_CHUNK, QUELLE_RELEVANZ_SNIPPET, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, SUB_VARIANT_COS, SUBBLOCK_MAX, EVIDENCE_BUDGET_CHARS, EVIDENCE_CTX_LINES +from config import CONSENSUS_GRACE, CONSENSUS_MAX_ROUNDS, DEFAULT_PROVIDER, EMBEDDING_AKTIV, EMBEDDING_SUB_DUP, SUB_VARIANT_COS, SUBBLOCK_MAX, EVIDENCE_BUDGET_CHARS, EVIDENCE_CTX_LINES from fsutil import atomic_write_json from jsonio import parse_json_text, read_json_file as _json_file from paths import arbeit_dir, blocks_path, question_pattern_path, project_dir, subblocks_path, source_path, source_crawl_dir, safe_folder -from crawl import crawl +from crawl import load_pages from pipeline import ( GenContext, _extra, _gather_progress, _yesno_schema, _log, _prompt, _race, _semaphore, _timeout, run_single_slot, @@ -853,117 +853,10 @@ def _crawl_index(folder) -> dict[str, str]: return idx - - - - -def _triage_rules(folder, pages: list[str]) -> tuple[list[str], list[str]]: - """Deterministic content/noise filter (config.CRAWL_*). Substring match (lowercase) against - URL + filename. Order: keep > noise > min_chars > keep. → (content, noise).""" - folder = Path(folder) - content, noise = [], [] - for fn in pages: - lines = _read(folder / fn).splitlines() - url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else "" - body = "\n".join(lines[1:]).strip() - hay = f"{url}\n{fn}".lower() - if any(p in hay for p in CRAWL_KEEP_PATTERNS): - content.append(fn) - elif any(p in hay for p in CRAWL_NOISE_PATTERNS): - noise.append(fn) - elif len(body) < CRAWL_MIN_CHARS: - noise.append(fn) - else: - content.append(fn) # default: keep — everything with content stays - return content, noise - - -def _page_snippet(folder, fn: str) -> tuple[str, str]: - """(url, snippet) of a crawl page for the relevance gate. url from the QUELLE: line; - snippet = body excerpt (navigation boilerplate is up front — the prompt ignores it). - The URL is the primary signal (meaningful slug), the snippet only supports it.""" - lines = _read(Path(folder) / fn).splitlines() - url = lines[0][len("QUELLE:"):].strip() if lines and lines[0].startswith("QUELLE:") else "" - body = "\n".join(lines[1:]).strip() - snippet = " ".join(body.split())[:QUELLE_RELEVANZ_SNIPPET] - return (url or fn), snippet - - -async def _relevance_triage(ctx: GenContext, set_p, files: dict, folder, content: list[str], spec: str, instructions: str) -> tuple[list[str], list[str]]: - """LLM topic gate after the rule filter: each content page ja/nein against the spec. - Off-topic (different field) → out. Pattern like `_relevance_block`: small packages, 3 raters - (`fast`), 2-of-3 consensus. CONSERVATIVE: drop only on a clear "nein" majority; dispute/gap/ - race error → keep. SAFETY: if the gate would drop ≥80 % (or all), everything stays - (a spec mismatch/bug must not empty the source). → (kept, out) as filenames.""" - topic, provider, is_cancelled = ctx.topic, ctx.provider, ctx.is_cancelled - work_dir = files["arbeit"] - pages = sorted(content) - if not pages: - return content, [] - items = [_page_snippet(folder, fn) for fn in pages] # index aligns with `pages` - chunks = _chunk_nums(list(range(len(pages))), _n_chunks(len(pages), QUELLE_RELEVANZ_CHUNK)) - n = len(chunks) - - def rater_paths(c): - return [work_dir / f"source-relevance-c{c}-{i}.json" for i in (1, 2, 3)] - - def lset(idxs): - return set(range(1, len(idxs) + 1)) - - async def _rate(c, idxs): - local_set = lset(idxs) - paths = rater_paths(c) - existing = sum(1 for p in paths if _yesno_schema(_json_file(p), local_set)) - if existing >= 2: - return True - enum_lines = [] - for k, j in enumerate(idxs, 1): - url, snip = items[j] - enum_lines.append(f"{k}. {url}") - if snip: - enum_lines.append(f" {snip}") - enum = "\n".join(enum_lines) - pending = [(i, p) for i, p in enumerate(paths, 1) if not _yesno_schema(_json_file(p), local_set)] - slots = [{ - "key": f"blocks-{topic}-source-relevance-c{c}-{i}", - "prompt": _prompt("Source-Relevance", topic=topic, spec=spec, pages=enum, out_path=p, extra=_extra(instructions)), - "role": "fast", "capabilities": "files", - "payload": (lambda result, p=p, ids=local_set: _yesno_schema(_json_file(p), ids)), - } for i, p in pending] - new = await _race(topic, f"Relevance triage package {c}", slots, 2 - existing, _timeout("relevance", len(idxs)), provider, cancelled=is_cancelled, grace=CONSENSUS_GRACE) - return not is_cancelled() and new is not None - - _qidx = _step_idx(topic, "Source prep") # gate runs in the source step (no own step) - set_p(f"Check relevance against spec ({n} packages)…", step=_qidx) - async def _report_triage(d, t): - set_p(f"Check relevance against spec {d}/{t}…", step=_qidx) - await _gather_progress([_rate(c, idxs) for c, idxs in enumerate(chunks, 1)], n, _report_triage) - if is_cancelled(): - return content, [] # cancel → drop nothing (caller aborts) - - # Vote per page: only a clear "nein" majority (≥2 and more than "ja") throws it out. - dropped: list[str] = [] - for c, idxs in enumerate(chunks, 1): - local_set = lset(idxs) - rater = [d for p in rater_paths(c) if (d := _yesno_schema(_json_file(p), local_set))] - for k in range(1, len(idxs) + 1): - vote_list = [d[k] for d in rater if k in d] - nein, ja = vote_list.count("nein"), vote_list.count("ja") - if nein >= 2 and nein > ja: - dropped.append(pages[idxs[k - 1]]) - - if dropped and len(dropped) >= max(1, int(len(pages) * 0.8)): - _log(topic, f"Relevance triage: would drop {len(dropped)}/{len(pages)} — discarded (spec mismatch?), keeping all") - return content, [] - dropped_set = set(dropped) - keepers = [fn for fn in pages if fn not in dropped_set] - return keepers, dropped - - async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, instructions: str) -> bool: - """Step "Source prep": crawl (link) + PDF convert + content/noise triage. - Persists the triage in the coverage table (content). → True (ok) / False (cancel/error). - thema: nothing. projekt/uni: only PDFs (curated folder, no triage).""" + """Step "Source prep": load link pages + PDF convert. → True (ok) / False (cancel/error). + link: load each user-supplied URL (one page, no following); ALL pages count as content + (curated — no triage). thema: nothing. projekt/uni: only PDFs (curated folder).""" topic, is_cancelled = ctx.topic, ctx.is_cancelled if not folder: return True # thema → no source to prepare @@ -973,28 +866,21 @@ async def _prepare_source(ctx: GenContext, set_p, files: dict, q: dict, folder, if await db.get_step_status(topic, "Source prep") == "done": return True if not _crawl_done(topic): - set_p("Loading source (crawl)…", step=_step_idx(topic, "Source prep")) - n = await asyncio.to_thread(crawl, q["location"], folder, cancelled=is_cancelled) + urls = [ln.strip() for ln in q["location"].splitlines() if ln.strip()] + set_p("Loading pages…", step=_step_idx(topic, "Source prep")) + n = await asyncio.to_thread(load_pages, urls, folder, cancelled=is_cancelled) if is_cancelled(): return False if not n: - _blocks_errors[topic] = "Crawl yielded no content — check link/domain" + _blocks_errors[topic] = "Links yielded no content — check the URLs" return False await asyncio.to_thread(_convert_pdfs, folder) pages = sorted(set(_crawl_index(folder).values())) if pages: - set_p("Triaging pages…", step=_step_idx(topic, "Source prep")) + # Curated links: the user picked them — keep every page, no rule/LLM triage. await db.delete_coverage(topic) - content, noise = _triage_rules(folder, pages) # deterministic rule filter - if q.get("spec") and content: # topic gate: separates the field (rules can't) - content, dropped = await _relevance_triage(ctx, set_p, files, folder, content, q["spec"], instructions) - if is_cancelled(): - return False - if dropped: - noise = sorted(set(noise) | set(dropped)) - _log(topic, f"LLM relevance: {len(dropped)} pages off-topic → noise") - await db.mark_content(topic, sorted(content), sorted(noise)) - _log(topic, f"Triage: {len(content)} content / {len(noise)} noise of {len(pages)} (rules + LLM gate)") + await db.mark_content(topic, pages, []) + _log(topic, f"Links: {len(pages)} pages as content (curated, no triage)") await db.set_step_status(topic, "Source prep", "done") return True @@ -1021,17 +907,41 @@ def _aspect_marker(title: str) -> int: _REFERENCE_RE = re.compile(r'^(Satz|Lemma|Korollar|Bemerkung|Definition)\s*[\d.]+\s*(\([a-z]\)|[a-z])?\s*$', re.I) +# Catalogue scaffolding around a concept name: a leading label+number ("Satz 7.13", "Satz 7.6:") +# or a trailing "(Definition 6.19)". _reference_strip peels it off and returns the concept remainder; +# _is_reference is then just "nothing survives the strip". Generic — no author/domain whitelist. The +# remainder feeds the rename step (a card must be titled by its concept, not its catalogue number). +_REF_WORDS = (r'(?:Satz|Lemma|Korollar|Corollary|Bemerkung|Beobachtung|Definition|Def' + r'|Theorem|Proposition|Folgerung|Kapitel|Abschnitt)') +_REF_PREFIX_RE = re.compile(rf'^\s*{_REF_WORDS}\s*\d+(?:\.\d+)*[a-z]?\s*[:.\-–—]?\s*', re.I) +_REF_SUFFIX_RE = re.compile(rf'\s*[\(\[]\s*{_REF_WORDS}\s*\d+(?:\.\d+)*[a-z]?\s*[\)\]]\s*$', re.I) +_WHOLE_PAREN_RE = re.compile(r'^\s*[\(\[]\s*(.{2,}?)\s*[\)\]]\s*$') + + +def _reference_strip(title: str) -> str: + """Concept remainder of a catalogue-referenced title, scaffolding removed. '' for a pure reference + without a concept ('Bemerkung 7.22' → ''); the concept otherwise ('Satz 7.13 (Christofides)' → + 'Christofides', 'Satz 7.6: Kriterium für X' → 'Kriterium für X', 'N P … (Definition 6.19)' → + 'N P …'). A title without any catalogue reference comes back unchanged.""" + t = (title or "").strip() + t = _REF_SUFFIX_RE.sub("", t).strip() + m = _REF_PREFIX_RE.match(t) + if m: + t = t[m.end():].strip() + if (w := _WHOLE_PAREN_RE.match(t)): # leading label left a lone '(Christofides)' + t = w.group(1).strip() + return t + def _is_reference(title: str) -> bool: - """True for pure reference/placeholder titles WITHOUT meaningful content: "Satz 7.18", "Lemma 6.2", - "Korollar 6.18" (number without a name) as well as marked spots "Bedingung (**)". NOT "Satz 6.24: - Cook/Levin" (has a name) and NOT short technical symbols like "P⊆NP"/"Σ*" (real concepts).""" + """True for a pure reference/placeholder title WITHOUT concept content: "Satz 7.18", "Lemma 6.2", + "Korollar 6.18" (number, no name) and marked spots "Bedingung (**)". False once a concept survives + the catalogue strip ("Satz 7.13 (Christofides)", "Satz 6.24: Cook/Levin") and for short technical + symbols like "P⊆NP"/"Σ*" (real concepts, no reference scaffolding).""" t = title.strip() - if _REFERENCE_RE.match(t): - return True if re.search(r'\(\*+\)', t): # marked spot "(**)" / "(*)" return True - return False + return bool(_REFERENCE_RE.match(t)) or not _reference_strip(t) def _canonical(candidates: list[dict], idxs: list[int], seen_norm: set[str]) -> dict: @@ -1118,6 +1028,15 @@ _CANON_STOP = re.compile( r'von|of|für|for|und|and|zum|zur|im)\b', re.I) +# German compounds are head-final: "Cliquenproblem" = stem + optional Fuge + head noun "problem". +# The standalone wrapper "problem" is already in _CANON_STOP, but a GLUED head hides the stem from ER +# blocking ("Cliquenproblem" would never share a key with "Clique"). Strip the glued "problem" head so +# the stem surfaces — a generic suffix rule, not a term list. The Fuge is only a consonant n/s +# ("Cliquen-", "Entscheidungs-"): matching a vowel "e" would eat the stem's own "e" ("clique" → "cliqu"). +# The ≥3-char stem lookbehind keeps a card literally titled "Problem"/"Probleme" (→ _CANON_STOP) intact. +_CANON_GLUED_HEAD = re.compile(r'(?<=\w{3})[ns]?problem(?:en|e|s)?\b', re.I) + + # Catalogue references ("Definition 6.19", "Satz 7.8") are scaffolding INCLUDING their number — # stripped as a phrase, so the digits don't pollute the key. Variant digits ("3-SAT") have no # scaffolding word in front and survive. @@ -1152,6 +1071,7 @@ def _canonical_key(title: str) -> str: s = _CANON_CATALOGUE.sub(' ', s) s = re.sub(r'≟|\bversus\b|\bvs\.?\b|=', ' opeq ', s) # equality / "vs" → one token s = re.sub(r'≤|⪯|→|⇒|⟹|\breduces?\s+to\b|\breduziert\b', ' opred ', s) # reduction → one token + s = _CANON_GLUED_HEAD.sub(' ', s) # glued "…problem" head → stem s = _CANON_STOP.sub(' ', s) s = re.sub(r'[^\w ]', ' ', s) # drop punctuation/symbols return " ".join(sorted(t for t in s.split() if t)) @@ -1340,16 +1260,21 @@ _REDUCTION_ONLY_OP = re.compile(r'[≤⪯]|→|⇒|⟹|->') # genuine reduction def _is_reduction_statement(title: str) -> bool: """True if the title is a reduction between two NAMED problems (both sides carry ≥3 letters and - neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP" and proof-size "|A| = O(m)".""" + neither is a pure bound like "O(m)"). Rejects unary "X ∈ NP", proof-size "|A| = O(m)", and a + construction/assignment suffix after the target ("3-SAT ≤ K-COLOR: G=(V,E) Konstruktion" is a + construction fragment, not a standalone statement — a bare target carries no ':' or '=').""" t = _REL_STRIP.sub('', title, count=1) m = _REDUCTION_ONLY_OP.search(t) if not m: return False + right = t[m.end():] + if ':' in right or '=' in right: # construction/assignment detail after the target → fragment + return False def _named(s: str) -> bool: return len(re.findall(r'[a-zäöüß]', s, re.I)) >= 3 and not re.match(r'\s*[Oo]\s*\(', s) - return _named(t[:m.start()]) and _named(t[m.end():]) + return _named(t[:m.start()]) and _named(right) def _is_named_statement(title: str, desc: str = "") -> bool: diff --git a/backend/board_inventory.py b/backend/board_inventory.py index 8e375f9..8824bac 100644 --- a/backend/board_inventory.py +++ b/backend/board_inventory.py @@ -43,7 +43,7 @@ from blocks import ( _build_research_prompt, _canonical, _canonical_key, _chunk_nums, _cliques, _completion_schema, _containment_parent, _crawl_index, _file_payload, _filter_schema, _filter_suspect, _is_artifact, _is_named_statement, - _is_parentless_noise, _is_reference, _pairs_schema, _read, + _is_parentless_noise, _is_reference, _reference_strip, _pairs_schema, _read, _direction_conflict, _relation_conflict, _root, _supplement_schema, _text_sections, _umbrella_schema, _aspect_marker, _title_variants, _corpus_files, _evidence_pack, _sink_json, source_folder, ) @@ -59,7 +59,7 @@ from fsutil import atomic_write_json, atomic_write_text from jsonio import read_json_file as _json_file from pipeline import (CANCELLED, FAILED, OK, GenContext, _extra, _log, _prompt, _runde_schema, _timeout, _yesno_schema, run_single_slot) -from textkit import _norm_title, _parse_selection, _title, clean_title +from textkit import _norm_title, _parse_selection, _title, _unclosed, clean_title log = logging.getLogger("creator.board_inventory") @@ -491,12 +491,16 @@ def _hat_anker(title: str, ctoks: set[str]) -> bool: def _sanierung_noetig(p: dict, ctoks: set[str] | None) -> bool: """QA-messbare Befund-Formen am Entstehungsort: Titel ohne Korpus-Anker (QA: fremd — - misst Token-Anker, nicht Semantik) oder leere Beschreibung (QA: hygiene). Gilt auch für - Singleton-Cluster, die das Naming sonst überspringen — Reader-Rohtitel gingen wörtlich - bis done_block durch (gemessen: 'k-Coloring' statt Korpus-Form 'k-Color').""" + misst Token-Anker, nicht Semantik) oder leere Beschreibung (QA: hygiene). Zusätzlich ein + Referenz-Titel (reine Katalog-Nummer), eine unbalancierte Klammer oder ein überlanger Titel + (>80). Gilt auch für Singleton-Cluster, die das Naming sonst überspringen — Reader-Rohtitel + gingen wörtlich bis done_block durch (gemessen: 'k-Coloring' statt Korpus-Form 'k-Color').""" + title = p.get("title", "") if not (p.get("description") or "").strip(): return True - return bool(ctoks) and not _hat_anker(p.get("title", ""), ctoks) + if _is_reference(title) or _unclosed(title) or len(title) > 80: + return True + return bool(ctoks) and not _hat_anker(title, ctoks) async def _anker_beleg(ctx: GenContext, flow: Flow, kandidaten: list[tuple[str, dict]]) -> set[str]: @@ -574,6 +578,9 @@ async def _proc_consensus_gate(ctx: GenContext, flow: Flow, cards): else: moves.append((cid, "naming")) else: + # single-reader find: majority of the Klärung panel suffices (was unanimity — a single + # "nein" discarded a real once-mentioned concept as failed-quorum, e.g. Cook-Levin) + p["quorum"] = "majority" moves.append((cid, "clarify")) await db.kanban_set_payload(topic, BOARD, cid, p) if anker_kandidaten: @@ -593,8 +600,9 @@ async def _proc_consensus_gate(ctx: GenContext, flow: Flow, cards): async def _proc_clarify(ctx: GenContext, flow: Flow, cards): """Single-reader finds: deterministic pre-reject, then a 3-judge panel (Blocks-Klaerung). - Quorum: UNANIMITY for single-reader clusters (origin-split C1), majority for reference-titled - consensus clusters. Kept reference titles get the panel's rename.""" + Quorum: MAJORITY of the panel for single-reader clusters and for reference-titled consensus + clusters (unanimity dropped real once-mentioned concepts on a single dissent). Kept reference + titles get the panel's rename.""" topic = flow.topic moves: list[tuple[str, str]] = [] pending = [] @@ -810,6 +818,12 @@ async def _namecheck_one(ctx: GenContext, flow: Flow, c): p["description"] = w.get("description") or p.get("description", "") readers = sorted(set().union(*[set(r.get("readers") or []) for r in rows])) if rows else [] sources = sorted(set().union(*[set(r.get("sources") or []) for r in rows])) if rows else [] + # deterministic catalogue-number strip (final title): a source numbering is never canonical — + # "Satz 7.13 (Christofides)" → "Christofides", "Satz 7.6: Kriterium …" → "Kriterium …". Runs AFTER + # naming_check so a re-picked member title cannot re-introduce the number. + stripped = _reference_strip(p.get("title", "")) + if stripped and _norm_title(stripped) != _norm_title(p.get("title", "")) and not _is_reference(stripped): + p["title"] = clean_title(stripped) await db.kanban_upsert_card(topic, BOARD, f"b-{cid}", "block", "fragment_filter", { "title": p.get("title", ""), "description": p.get("description", ""), "readers": readers, "sources": sources, "n_size": len(readers), @@ -1025,13 +1039,21 @@ async def _proc_fragment_filter(ctx: GenContext, flow: Flow, cards): for nr in nums: if nr in fragments or nr in honored: continue + votes = len(dem.get(nr, [])) if drp.get(nr, 0) >= 2 and (_is_artifact(allrows[nr - 1]["title"]) or _is_parentless_noise(allrows[nr - 1]["title"])): honored.add(nr) - elif len(dem.get(nr, [])) >= 2: - fragments[nr] = max(set(dem[nr]), key=dem[nr].count) elif nr in proposals: - overruled.append(nr) + # a pass-1 judge already proposed this demote — overturn it ONLY on a + # UNANIMOUS recheck keep (zero demote votes). 15/17 majority-rescued cards + # were later QA duplicates, so a single recheck demote reconfirms the fragment. + if votes >= 1: + fragments[nr] = max(set(dem[nr]), key=dem[nr].count) + else: + overruled.append(nr) + elif votes >= 2: + # pure ⚠ suspect without a pass-1 proposal: still needs a panel majority + fragments[nr] = max(set(dem[nr]), key=dem[nr].count) # embedding backstop: veto confirmed non-containment demotes whose direct title pair is # literally structureless (see FRAGMENT_MIN_COS) — applied BEFORE _root resolution. floor_veto: list[int] = [] diff --git a/backend/config.py b/backend/config.py index c18346f..94357e0 100644 --- a/backend/config.py +++ b/backend/config.py @@ -135,21 +135,6 @@ CONSENSUS_GRACE = 300 # check loops leave any remaining objections standing after that. CONSENSUS_MAX_ROUNDS = 3 -# Crawler triage (content/noise) — deterministic rule filter instead of an LLM. -# Match: substring (lowercase) against URL AND file name. Order: keep > noise > min_chars > keep. -# Just add special rules here. -CRAWL_KEEP_PATTERNS = ["learn-unit", "learn-course"] # always content -CRAWL_NOISE_PATTERNS = [ # clearly off-topic → out - "clubs", "events", "podcasts", "resources", "-u-", - "academy", "pricing", "/plans", "career", "newsletter", "impressum", "login", -] -CRAWL_MIN_CHARS = 400 # too little text → out - -# LLM topic relevance gate (after the rule filter): per content page yes/no against the spec. -# Separates the subject area (e.g. backend vs frontend), which the global CRAWL_* rules can't. -QUELLE_RELEVANZ_CHUNK = 12 # pages per rater package (small, since a snippet ships per page) -QUELLE_RELEVANZ_SNIPPET = 800 # body characters per page in the prompt (URL is the primary signal) - # QA gate: after the inventory phase an automatic QA run scores the blocks; below the # threshold the flow PAUSES before board 2 burns tokens (frontend offers force-continue). QA_GATE_NOTE = 9.5 # 0 = gate off; quota-based, so the tolerated finding count scales with topic size diff --git a/backend/crawl.py b/backend/crawl.py index 7b0e743..d7a45fc 100644 --- a/backend/crawl.py +++ b/backend/crawl.py @@ -1,9 +1,9 @@ -"""Bounded domain crawler for link sources — renders JS via Playwright (Chromium). +"""Page loader for link sources — renders JS via Playwright (Chromium). -Loads pages + PDFs starting from a start URL — ONLY the same domain, limited depth -and page count. HTML pages are rendered in a headless browser (needed for SPAs), then -links + text are pulled from the finished DOM. PDFs are loaded directly as bytes. -Deterministic, bounded; runs via asyncio.to_thread (sync API, no event loop). +Loads a FIXED list of URLs the user supplied — one page each, NO link following. +HTML pages are rendered in a headless browser (needed for SPAs), then the main text +is pulled from the finished DOM. PDFs are loaded directly as bytes. Deterministic, +bounded; runs via asyncio.to_thread (sync API, no event loop). """ import hashlib @@ -17,8 +17,6 @@ from fsutil import atomic_write_text log = logging.getLogger("creator.crawl") -MAX_DEPTH = 3 -MAX_PAGES = 500 PAGE_TIMEOUT = 30 # seconds per page (render or PDF download) CRAWL_SETTLE_MS = 3000 # capped settle after domcontentloaded (SPA render); no 30s networkidle hang MAX_BYTES = 10_000_000 # 10 MB cap per PDF @@ -33,7 +31,7 @@ def _fetch_bytes(url: str) -> bytes | None: data = resp.read(MAX_BYTES + 1) return None if len(data) > MAX_BYTES else data except Exception as e: - log.debug("crawl: PDF fetch failed %s: %s", url, e) + log.debug("load: PDF fetch failed %s: %s", url, e) return None @@ -47,21 +45,6 @@ def _is_pdf(url: str) -> bool: return url.lower().split("?")[0].rstrip("/").endswith(".pdf") -def _scope_prefix(start_url: str) -> str: - """First non-empty path segment of the start URL as the crawl scope, e.g. - `/learn/path/x` → `/learn`. No path segment → `""` (whole domain, no narrowing).""" - seg = [s for s in urlparse(start_url).path.split("/") if s] - return f"/{seg[0]}" if seg else "" - - -def _in_scope(url: str, prefix: str) -> bool: - """Segment-exact prefix match (no `/learn` ⊃ `/learning-x`). Empty prefix → everything allowed.""" - if not prefix: - return True - p = urlparse(url).path - return p == prefix or p.startswith(prefix + "/") - - def _page_text(page) -> str: """Main text of the rendered page — nav/footer/boilerplate removed via trafilatura. Falls back to the raw body text when extraction is empty/too short (non-article pages).""" @@ -78,34 +61,34 @@ def _page_text(page) -> str: return text.strip() -def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages: int = MAX_PAGES, cancelled=None) -> int: - """Crawl from start_url (same domain only), render JS and store pages/PDFs in `target`. +def load_pages(urls: list[str], target: Path, *, cancelled=None) -> int: + """Load each URL in `urls` (one page each, NO link following), render JS and store + pages/PDFs in `target`. - BFS up to `max_depth` / `max_pages`. Errors on individual pages are skipped. - Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it, - so a restart crawls again. Returns the number of saved sources. + Duplicate/fragment-only URLs collapse to one. Errors on individual pages are skipped. + Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it, so a + restart loads again. Returns the number of saved sources. """ - # Lazy: this way the backend starts even without Playwright installed; only crawling then fails. + # Lazy: this way the backend starts even without Playwright installed; only loading then fails. from playwright.sync_api import sync_playwright target.mkdir(parents=True, exist_ok=True) - domain = urlparse(start_url).netloc - prefix = _scope_prefix(start_url) # only follow links under this path segment seen: set[str] = set() - queue: list[tuple[str, int]] = [(urldefrag(start_url)[0], 0)] + todo: list[str] = [] + for u in urls: + nu = urldefrag(u)[0].strip() + if nu and nu not in seen: + seen.add(nu) + todo.append(nu) saved = 0 with sync_playwright() as pw: browser = pw.chromium.launch(args=["--no-sandbox"]) # non-root (Docker user app) page = browser.new_page(user_agent=_UA) try: - while queue and saved < max_pages: + for url in todo: if cancelled and cancelled(): - return saved # abort → NO .done marker → restart crawls again - url, depth = queue.pop(0) - if url in seen: - continue - seen.add(url) + return saved # abort → NO .done marker → restart loads again # PDFs need no rendering — load directly. if _is_pdf(url): @@ -120,7 +103,7 @@ def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages try: page.goto(url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT * 1000) except Exception as e: - log.debug("crawl: goto incomplete %s: %s", url, e) # still try to read the content + log.debug("load: goto incomplete %s: %s", url, e) # still try to read the content try: page.wait_for_load_state("networkidle", timeout=CRAWL_SETTLE_MS) except Exception: @@ -129,20 +112,9 @@ def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages if text: atomic_write_text(target / _name(url, ".txt"), f"QUELLE: {url}\n\n{text}") saved += 1 - if depth < max_depth: - try: - hrefs = page.eval_on_selector_all("a[href]", "els => els.map(e => e.href)") - except Exception: - hrefs = [] - for href in hrefs: - nxt = urldefrag(href)[0] - if (nxt.startswith(("http://", "https://")) - and urlparse(nxt).netloc == domain and _in_scope(nxt, prefix) - and nxt not in seen): - queue.append((nxt, depth + 1)) finally: browser.close() (target / ".done").write_text("ok", encoding="utf-8") # ran through cleanly - log.info("crawl %s → %d sources in %s", start_url, saved, target) + log.info("load_pages → %d sources in %s", saved, target) return saved diff --git a/backend/models.py b/backend/models.py index 5f8e9ce..26f5cca 100644 --- a/backend/models.py +++ b/backend/models.py @@ -46,7 +46,7 @@ class BlocksCreateRequest(BaseModel): instructions: str = Field(default="", max_length=2000) provider: ProviderType = DEFAULT_PROVIDER source_type: SourceType = "thema" - source_location: str = Field(default="", max_length=2000) + source_location: str = Field(default="", max_length=20000) # link mode: one URL per line research: bool = True # False = Continue: drain the existing kanban queue, no new search qa_force: bool = False # True = übersteuert ein pausierendes QA-Gate („Trotzdem fortsetzen") @@ -97,7 +97,7 @@ class FolderResponse(BaseModel): class BlocksSourceUpdate(BaseModel): topic: str = Field(min_length=1, max_length=100) type: SourceType = "thema" - location: str = Field(default="", max_length=2000) + location: str = Field(default="", max_length=20000) # link mode: one URL per line spec: str = Field(default="", max_length=2000) diff --git a/backend/qa.py b/backend/qa.py index 99c2050..2d05010 100644 --- a/backend/qa.py +++ b/backend/qa.py @@ -33,11 +33,14 @@ EMB_FLOOR = 0.82 # casefolded title cosine (own threshold, NOT the pipel SECTION_CHARS = 4000 # own paragraph splitter — independent of _text_sections COVER_MIN_TOKENS = 2 # distinctive block tokens a section must share to count as covered FREMD_MIN_TOKENS = 1 # distinctive title tokens that must appear in the corpus +CONCEPT_COVER_RATIO = 0.6 # fraction of a named result's stems a block must share to count as covered LLM_SAMPLE = 12 # pairs/sections per judge call with --llm # Note 0-10, deterministisch aus den Quoten (transparent, diffbar — keine LLM-"Gefühlsnote"). -# Lücken/Fremd wiegen am schwersten (fehlender/falscher Stoff); Dubletten-VERDACHT enthält -# bewusst Rauschen und wiegt daher wenig. -NOTE_GEWICHTE = {"luecken": 3.0, "fremd": 2.5, "unechte_bloecke": 2.5, "hygiene": 0.5} +# Lücken/Fremd wiegen am schwersten (fehlender/falscher Stoff). konzept_luecken (benannte Kernresultate +# ohne Baustein) ist heuristisch → mittleres Gewicht. Dubletten-VERDACHT enthält bewusst Rauschen, +# bildet aber das Nutzerproblem (Dopplungen) ab → leichtestes Gewicht, aber nicht null. +NOTE_GEWICHTE = {"luecken": 3.0, "fremd": 2.5, "unechte_bloecke": 2.5, "konzept_luecken": 1.5, + "dubletten_verdacht": 1.0, "hygiene": 0.5} # subs/artefacts only exist after board 2 — at gate time these quotas would always be 0 # and water down the inventory score, hence a separate score. # sub_dubletten counts only with --llm (confirmed pairs); the bare candidate list is @@ -181,6 +184,48 @@ def luecken(blocks: list[dict], subs_by_norm: dict[str, list[str]], corpus: dict return out +# A NAMED result carries a concept name, not just a number: "Satz 7.13 (Christofides)", +# "Satz 6.24: Cook-Levin". These are the core statements the token-based `luecken` misses — 108 +# over-granular blocks cover every SECTION, yet the named core result may have no block at all. +# Generic + domain-safe: the NUMBER is mandatory (a numbered labeled unit is a formal statement in +# a structured document, any field), so prose like "ein Satz von Goethe" (no number) never matches. +_NAMED_RESULT_RE = re.compile( + r'\b(?:Satz|Lemma|Korollar|Theorem|Proposition|Folgerung|Definition|Algorithmus)\s+\d+(?:\.\d+)*\s*' + r'(?:\(\s*([^()\n]{3,60}?)\s*\)|:\s*([^\n.;·]{3,60}?)\s*(?:[.\n;·]|$))', re.M) + + +def _stem(t: str) -> str: + """Declension-tolerant token stem: drop trailing digits, keep the 5-char prefix + ('Eulerschen'/'Eulerscher' → 'euler', 'Kreise'/'Kreis' → 'kreis').""" + return t.rstrip("0123456789")[:5] + + +def _named_results(corpus: dict[str, str]) -> dict[str, set[str]]: + """Named/attributed corpus results → {concept name: distinctive stems}. A bare 'Satz 7.18' + (number, no name) yields nothing to match. Same catalogue vocabulary as the title strip.""" + out: dict[str, set[str]] = {} + for text in corpus.values(): + for m in _NAMED_RESULT_RE.finditer(text): + name = (m.group(1) or m.group(2) or "").strip() + toks = _distinctive(name) + if len(name) >= 3 and toks: + out.setdefault(name, set()).update(_stem(t) for t in toks) + return out + + +def konzept_luecken(blocks: list[dict], named: dict[str, set[str]]) -> list[str]: + """Named corpus results that NO block covers — concept gaps the token-based `luecken` cannot see. + Covered = a block whose title+description share ≥ CONCEPT_COVER_RATIO of the result's distinctive + stems (declension-tolerant). Errs toward 'covered' so the heuristic never invents a false gap.""" + anchors = [{_stem(t) for t in _distinctive(b["title"]) | _distinctive(b.get("description") or "")} + for b in blocks] + out = [] + for name, stems in named.items(): + if not any(len(stems & a) >= CONCEPT_COVER_RATIO * len(stems) for a in anchors): + out.append(name) + return sorted(out) + + def fremd(blocks: list[dict], corpus: dict[str, str]) -> list[str]: """Blocks whose distinctive title tokens never appear in the corpus (scope creep). Token/stem match, NOT raw substring — 'bergang' ⊂ 'Übergang' had whitewashed the @@ -384,6 +429,8 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None: sd = sub_dubletten(sub_rows) lk = luecken(blocks, subs_by_norm, corpus) if corpus else [] fr = fremd(blocks, corpus) if corpus else [] + named = _named_results(corpus) if corpus else {} + kl = konzept_luecken(blocks, named) if corpus else [] bl = beleg(blocks, sub_rows) hy = hygiene(blocks) n_sections = sum(len(_sections(t)) for t in corpus.values()) or 1 @@ -452,13 +499,15 @@ async def qa_report(topic: str, llm: bool = False) -> dict | None: "dubletten_verdacht": round(len(d) / max(len(blocks), 1), 3), "luecken": round(len(_zaehlbare_luecken(lk, llm)) / n_sections, 3), "fremd": round(len(fr) / max(len(blocks), 1), 3), + **({"konzept_luecken": round(len(kl) / max(len(named), 1), 3)} if corpus else {}), "hygiene": round(len(hy) / max(len(blocks), 1), 3), **({"unechte_bloecke": round(len(unecht) / max(len(blocks), 1), 3)} if unecht is not None else {}), }, "quoten_artefakte": quoten_art, **({"unecht": unecht} if unecht is not None else {}), **({"fremd_freigesprochen": fremd_frei} if fremd_frei else {}), - "dubletten": d, "sub_dubletten": sd, "luecken": lk, "fremd": fr, "beleg": bl, "hygiene": hy, + "dubletten": d, "sub_dubletten": sd, "luecken": lk, "konzept_luecken": kl, + "fremd": fr, "beleg": bl, "hygiene": hy, "artefakte": art, "lauf": summary, } diff --git a/backend/routes.py b/backend/routes.py index 58acf39..830ed2c 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -162,13 +162,7 @@ async def create_blocks(req: BlocksCreateRequest): # Persist the source only the FIRST time; ▶/Resume keeps the existing choice. if not qp.exists(): type, location = req.source_type, req.source_location.strip() - if type in ("projekt", "uni"): - folder = safe_folder(location) - if folder is None or not folder.is_dir(): - raise HTTPException(400, "Folder invalid or not found (path relative to the project root, no ../).") - elif type == "link": - if not location.lower().startswith(("http://", "https://")): - raise HTTPException(400, "Link must start with http:// or https://.") + _validate_source(type, location) qp.parent.mkdir(parents=True, exist_ok=True) atomic_write_json(qp, {"type": type, "location": location, "spec": req.instructions.strip()}) asyncio.create_task(generate_blocks(topic, req.instructions.strip(), req.provider, @@ -315,14 +309,17 @@ async def reset_block_progress(topic: str, block: str): def _validate_source(type: str, location: str) -> None: - """Check source input (same rules as on creation).""" + """Check source input (shared by create + edit).""" if type in ("projekt", "uni"): folder = safe_folder(location) if folder is None or not folder.is_dir(): raise HTTPException(400, "Folder invalid or not found (path relative to the project root, no ../).") elif type == "link": - if not location.lower().startswith(("http://", "https://")): - raise HTTPException(400, "Link must start with http:// or https://.") + urls = [ln.strip() for ln in location.splitlines() if ln.strip()] + if not urls: + raise HTTPException(400, "Enter at least one link.") + if not all(u.lower().startswith(("http://", "https://")) for u in urls): + raise HTTPException(400, "Each link must start with http:// or https:// (one per line).") @router.get("/blocks/source", response_model=BlocksSourceResponse) diff --git a/backend/tests/test_board_inventory.py b/backend/tests/test_board_inventory.py index 236c79b..3a2ed49 100644 --- a/backend/tests/test_board_inventory.py +++ b/backend/tests/test_board_inventory.py @@ -324,8 +324,9 @@ async def test_panel_confirms_demote(board_env, tmp_path, monkeypatch): assert (await db.kanban_get_card(TOPIC, B, "b-2"))["stage"] == "dedup" -async def test_panel_overrules_single_vote(board_env, tmp_path, monkeypatch): - """Nur 1 von 3 Panel-Stimmen bestätigt den Judge-Demote → Karte überlebt (Journal).""" +async def test_single_recheck_vote_confirms_proposal(board_env, tmp_path, monkeypatch): + """M3.2: ein pass-1-Vorschlag wird schon von EINER Recheck-Stimme bestätigt → rejected. + (15/17 mehrheitlich geretteten Karten waren später QA-Dubletten — Overturn nur einstimmig.)""" db, ctx, files = board_env monkeypatch.setattr(bi, "run_single_slot", _slot_router([ ("-filter-recheck-", _confirm_votes({"1"}, {"fragments": {"1": 2}, "drop": []})), @@ -335,6 +336,23 @@ async def test_panel_overrules_single_vote(board_env, tmp_path, monkeypatch): ("b-1", {"title": "Blockzitat", "description": "Zitat mit >"}), ("b-2", {"title": "Codeblock", "description": "Code mit Einrückung"}), ]) + c1 = await db.kanban_get_card(TOPIC, B, "b-1") + assert c1["stage"] == "rejected" + assert c1["payload"]["reason"] == "fragment" + assert c1["payload"]["parent_norm"] == "codeblock" + + +async def test_unanimous_recheck_keep_overrules_proposal(board_env, tmp_path, monkeypatch): + """M3.2: nur ein EINSTIMMIGES Recheck-Panel (0 Demote-Stimmen) hebt den pass-1-Vorschlag auf.""" + db, ctx, files = board_env + monkeypatch.setattr(bi, "run_single_slot", _slot_router([ + ("-filter-recheck-", _confirm_votes(set(), {"fragments": {"1": 2}, "drop": []})), + ("-filter-", {"fragments": {"1": 2}, "drop": []}), + ])) + await _run_filter(db, ctx, tmp_path, [ + ("b-1", {"title": "Blockzitat", "description": "Zitat mit >"}), + ("b-2", {"title": "Codeblock", "description": "Code mit Einrückung"}), + ]) assert (await db.kanban_get_card(TOPIC, B, "b-1"))["stage"] == "dedup" journal = json.loads(next(tmp_path.glob("inventar-filter-*.json")).read_text(encoding="utf-8")) assert journal["ueberstimmt"] == ["Blockzitat"] @@ -587,6 +605,45 @@ def test_canonical_key_camel_and_catalogue(): assert k("Section 2.1 Matching") == k("Matching") +def test_canonical_key_glued_problem_suffix(): + """Verklebtes Kompositum-Suffix „…problem" wird abgetrennt, damit „Cliquenproblem" und + „Clique" denselben Blocking-Key teilen (aak: sonst nie Dedup-Kandidat). Fugen-n/-s + Plural + inklusive; die generische Suffix-Regel darf keine echten Varianten über-mergen.""" + from blocks import _canonical_key as k + assert k("Clique") == k("Cliquenproblem") == k("Cliquenprobleme") != "" + assert k("Set Cover") == k("SetCover-Problem") # bestehender Hyphen-Pfad bleibt + assert k("SAT") != k("3-SAT") # Varianten-Ziffer bleibt Signal + assert k("Problem") != k("Clique") # bloßes „Problem" wird nicht zum Stamm + + +def test_reference_strip_and_is_reference(): + """Katalog-Nummern werden gestrippt, der Konzeptname bleibt; reine Nummern → leer + Referenz. + Reale aak-Schadensfälle (Satz-/Bemerkungs-Titel liefen wörtlich bis done).""" + from blocks import _reference_strip as strip, _is_reference as isref + assert strip("Satz 7.13 (Christofides)") == "Christofides" + assert strip("Satz 7.6: Kriterium für Eulerschen Kreis") == "Kriterium für Eulerschen Kreis" + assert strip("N P via nicht-deterministische Turingmaschine (Definition 6.19)") \ + == "N P via nicht-deterministische Turingmaschine" + assert strip("Bemerkung 7.22") == "" + assert strip("Vertex Cover") == "Vertex Cover" # kein Katalog-Gerüst → unverändert + assert isref("Bemerkung 7.22") and isref("Satz 7.18") and isref("Korollar 6.18") + assert isref("Bedingung (**)") + assert not isref("Satz 7.13 (Christofides)") # hat Konzept → keine Referenz + assert not isref("Satz 7.6: Kriterium für Eulerschen Kreis") + assert not isref("P⊆NP") and not isref("Σ*") # kurze Symbole bleiben echt + + +def test_is_named_statement_construction_suffix(): + """Reduktion mit Konstruktions-Suffix (: / = nach dem Ziel) ist ein Fragment, kein Statement — + darf nicht mehr vor Demotion geschützt sein (aak: „3-SAT ≤ K-COLOR: G=(V,E) Konstruktion").""" + from blocks import _is_named_statement as named + assert not named("3-SAT ≤ K-COLOR: G=(V,E) Konstruktion") + assert named("3-SAT ≤ Clique") # saubere Reduktion bleibt geschützt + assert named("Clique → Vertex Cover") + assert not named("X ist NP-vollständig") # unäre Aussage bleibt demotable + assert named("Satz von Cook/Levin: SAT ist NP-vollständig ⇔ …") # benanntes Ergebnis bleibt + + def test_relation_guard_ignores_trailing_scaffolding(): """Trailing „Reduktion/Transformation" ist kein Operand — sonst blockt der Guard den korrekten Merge; Richtungs-Konflikte bleiben erkannt.""" @@ -1213,6 +1270,49 @@ async def test_namecheck_ok_behaelt_titel(testdb, tmp_path, monkeypatch): assert block["payload"]["description"] == "Eigene Beschreibung" +async def test_namecheck_strips_source_numbering(testdb, tmp_path, monkeypatch): + """M1.2: wählt der Check-Judge einen numerierten Titel, wird die Katalog-Nummer deterministisch + zum Konzept gestrippt ('Satz 7.13 (Christofides)' → 'Christofides').""" + db = testdb + + async def fake_members(topic, cid): + return [{"norm": "satz 7.13 (christofides)", "title": "Satz 7.13 (Christofides)", + "description": "3/2-Approximation für metrisches TSP", "readers": ["r1"], "sources": []}, + {"norm": "b", "title": "B", "description": "db", "readers": ["r2"], "sources": []}] + + async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout): + return "ok", payload((0, '{"best": 1}', "")) # Judge wählt den numerierten Member-Titel + + monkeypatch.setattr(bi, "_member_rows", fake_members) + monkeypatch.setattr(bi, "run_single_slot", fake_slot) + payload = {"title": "Satz 7.13 (Christofides)", "description": "3/2-Approximation", + "main_norm": "satz 7.13 (christofides)"} + await db.kanban_upsert_card(TOPIC, B, "c7", "cluster", "naming_check", payload) + ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False) + await bi._namecheck_one(ctx, _mk_flow(tmp_path), {"card_id": "c7", "payload": payload}) + block = await db.kanban_get_card(TOPIC, B, "b-c7") + assert block["payload"]["title"] == "Christofides" + + +async def test_consensus_gate_single_reader_majority_quorum(testdb, tmp_path, monkeypatch): + """M5.1: Einzel-Reader-Fund → clarify mit Mehrheits-Quorum statt Einstimmigkeit.""" + db = testdb + monkeypatch.setattr(bi, "source_folder", lambda t: None) # thema: kein Korpus + + async def fake_members(topic, cid): + return [{"title": "Seltenes Konzept", "description": "einmal erwähnt", + "readers": ["r1"], "supplement": False}] + + monkeypatch.setattr(bi, "_member_rows", fake_members) + monkeypatch.setattr(bi, "_rep", lambda rows: rows[0]) + await db.kanban_upsert_card(TOPIC, B, "cs", "cluster", "consensus_gate", {}) + ctx = GenContext(topic=TOPIC, provider="claude", is_cancelled=lambda: False) + await bi._proc_consensus_gate(ctx, _mk_flow(tmp_path), [{"card_id": "cs", "payload": {}}]) + card = await db.kanban_get_card(TOPIC, B, "cs") + assert card["stage"] == "clarify" + assert card["payload"]["quorum"] == "majority" + + # ── Sanierung: Titel auf Korpus-Form, Beschreibungspflicht (QA: fremd/hygiene) ────── def test_sanierung_schema_varianten(): @@ -1229,6 +1329,11 @@ def test_sanierung_noetig(): assert bi._sanierung_noetig({"title": "k-Coloring", "description": "d"}, ctoks) # kein Korpus-Anker assert not bi._sanierung_noetig({"title": "k-Color", "description": "d"}, ctoks) assert not bi._sanierung_noetig({"title": "k-Coloring", "description": "d"}, None) # thema: kein Korpus + # M1.3: Referenz-Titel / unbalancierte Klammer / >80 Zeichen triggern auch ohne Korpus (Singletons) + assert bi._sanierung_noetig({"title": "Bemerkung 7.22", "description": "d"}, None) + assert bi._sanierung_noetig({"title": "N P (Definition 6.19", "description": "d"}, None) + assert bi._sanierung_noetig({"title": "A" * 81, "description": "d"}, None) + assert not bi._sanierung_noetig({"title": "Clique", "description": "d"}, None) def _sanierung_env(tmp_path, monkeypatch, antwort): diff --git a/backend/tests/test_qa.py b/backend/tests/test_qa.py index 63a21b9..dd936ed 100644 --- a/backend/tests/test_qa.py +++ b/backend/tests/test_qa.py @@ -93,9 +93,53 @@ def test_note_kalibrierung(): assert qa.note({"fremd": 1.0}) == 0.0 # komplett fremdes Inventar = 0, nicht 7.7 -def test_note_verdacht_zaehlt_nicht(): - """dubletten_verdacht ist Verdachtsliste, kein Urteil — beeinflusst die Note nicht.""" - assert qa.note({"dubletten_verdacht": 1.0}) == 10.0 +def test_note_dubletten_verdacht_zaehlt(): + """dubletten_verdacht bildet das Dopplungs-Nutzerproblem ab → zählt jetzt (Gewicht 1.0), + bleibt aber das leichteste Gewicht. 36 % Verdacht → −3.6 → 6.4; die Liste kann >1 sein → geklemmt.""" + assert qa.note({"dubletten_verdacht": 0.0}) == 10.0 + assert qa.note({"dubletten_verdacht": 0.36}) == 6.4 + assert qa.note({"dubletten_verdacht": 2.0}) == 0.0 # >1 wird auf 1.0 geklemmt + + # sub_dubletten_verdacht bleibt gewichtslos (Artefakt-Verdacht, kein Urteil) + assert qa.note({"sub_dubletten_verdacht": 1.0}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 10.0 + + +def test_note_konzept_luecken_zaehlt(): + """Fehlende benannte Kernresultate drücken die Note (Gewicht 1.5): 20 % → −3.0 → 7.0.""" + assert qa.note({"konzept_luecken": 0.0}) == 10.0 + assert qa.note({"konzept_luecken": 0.2}) == 7.0 + + +def test_named_results_only_numbered(monkeypatch): + """Benannte Ergebnisse: nur NUMMERIERTE Katalog-Referenzen mit Namen zählen — Prosa ohne + Nummer ('Satz von Goethe') und nackte Nummern ('Satz 7.18') liefern nichts (generisch).""" + corpus = {"skript.txt": + "Satz 7.13 (Christofides) liefert eine 3/2-Approximation.\n" + "Satz 6.24: Satz von Cook und Levin.\n" + "Nach Satz 7.18 folgt daraus die Schranke.\n" + "Ein Satz von Goethe steht hier."} + named = qa._named_results(corpus) + assert "Christofides" in named + assert any("Cook" in n and "Levin" in n for n in named) + assert not any("Goethe" in n for n in named) # keine Nummer → kein False Positive + assert all(n.strip() for n in named) + + +def test_konzept_luecken_flags_missing(): + """Benanntes Kernresultat ohne Baustein = Lücke; ein gedecktes Resultat nicht.""" + corpus = {"s.txt": "Satz 7.13 (Christofides). Satz 6.16 (Kriterium für P=NP)."} + named = qa._named_results(corpus) + blocks = [{"title": "Christofides-Algorithmus", "description": "3/2-Approximation für TSP"}] + gaps = qa.konzept_luecken(blocks, named) + assert "Christofides" not in gaps + assert any("P=NP" in g or "Kriterium" in g for g in gaps) + + +def test_konzept_luecken_declension_tolerant(): + """Andere Flexion im Baustein deckt das Resultat trotzdem — kein falscher Lücken-Alarm.""" + named = qa._named_results({"s.txt": "Satz 7.6: Kriterium für Eulerschen Kreis."}) + blocks = [{"title": "Kriterium für Eulerscher Kreis", "description": "Grad aller Knoten gerade"}] + assert qa.konzept_luecken(blocks, named) == [] def test_note_artefakte_getrennt(): diff --git a/backend/tests/test_source_links.py b/backend/tests/test_source_links.py new file mode 100644 index 0000000..5d639f9 --- /dev/null +++ b/backend/tests/test_source_links.py @@ -0,0 +1,55 @@ +"""Link source = curated URL list (one page per line, no crawl following).""" +from pathlib import Path + +import pytest +from fastapi import HTTPException + +import blocks +from pipeline import GenContext +from routes import _validate_source + + +# --- _validate_source: per-line validation --------------------------------- + +def test_validate_source_link_multiline_ok(): + _validate_source("link", "https://a.com/x\nhttp://b.com/y") # no raise + + +def test_validate_source_link_rejects_bad_line(): + with pytest.raises(HTTPException): + _validate_source("link", "https://a.com/x\nftp://bad") + + +def test_validate_source_link_rejects_empty(): + with pytest.raises(HTTPException): + _validate_source("link", " \n ") + + +# --- _prepare_source: load every line, all pages as content, no triage ----- + +async def test_prepare_source_link_loads_all_urls_as_content(testdb, tmp_path, monkeypatch): + topic = "T" + folder = tmp_path / "source" + folder.mkdir() + + captured = {} + + def fake_load(urls, target, *, cancelled=None): + captured["urls"] = list(urls) + for i, u in enumerate(urls): + (Path(target) / f"p{i}.txt").write_text(f"QUELLE: {u}\n\nInhalt {i}", encoding="utf-8") + return len(urls) + + monkeypatch.setattr(blocks, "load_pages", fake_load) + monkeypatch.setattr(blocks, "_convert_pdfs", lambda f: None) + monkeypatch.setattr(blocks, "_crawl_done", lambda t: False) + monkeypatch.setattr(blocks, "_step_idx", lambda t, n: 0) # step list needs source.json — irrelevant here + + ctx = GenContext(topic=topic, provider="claude", is_cancelled=lambda: False) + q = {"type": "link", "location": " https://a.com/x \n\nhttps://b.com/y\n", "spec": ""} + ok = await blocks._prepare_source(ctx, lambda *a, **k: None, {"arbeit": tmp_path}, q, folder, "") + + assert ok is True + assert captured["urls"] == ["https://a.com/x", "https://b.com/y"] # split + strip + blanks dropped + content = await testdb.list_content(topic) + assert len(content) == 2 # every page kept, no relevance triage diff --git a/backend/tests/test_textkit.py b/backend/tests/test_textkit.py new file mode 100644 index 0000000..06b0db1 --- /dev/null +++ b/backend/tests/test_textkit.py @@ -0,0 +1,57 @@ +"""Titel-Hygiene: klammer-bewusster Split + Dash-Normalisierung (M1.1). + +Reale aak-Schadensfälle: ein en-Dash mit einseitigem Space INNERHALB eines Titels +wurde zum Separator → Split mitten im Titel („Aε-Algorithmus (Güte 1+ε, Laufzeit O(n3"). +""" + +from textkit import _norm_dash, _title, _split_top, _unclosed, _parse_selection + + +def test_norm_dash_top_level_only(): + # einseitiger Space am Dash → Separator (Modelle liefern das so) + assert _norm_dash("Titel –Beschreibung") == "Titel — Beschreibung" + assert _norm_dash("Titel— Beschreibung") == "Titel — Beschreibung" + # Dash INNERHALB offener Klammer bleibt unangetastet (kein Separator) + assert _norm_dash("f (a – b) — Rest") == "f (a – b) — Rest" + # ASCII-Hyphen und geklebte/rangebasierte Dashes sind nie Separatoren + assert _norm_dash("3-SAT") == "3-SAT" + assert _norm_dash("12–15") == "12–15" + + +def test_split_top_bracket_aware(): + assert _split_top("a (b — c) — d", " — ") == "a (b — c)" + assert _split_top("a — b", " — ") == "a" + assert _split_top("kein Separator hier", " — ") == "kein Separator hier" + + +def test_unclosed_bracket_detection(): + assert _unclosed("a (b") is True + assert _unclosed("a [b {c") is True + assert _unclosed("a (b) c") is False + assert _unclosed("a) b") is False # überzähliger Schließer ist kein Abschnitt + + +def test_title_keeps_bracketed_dash(): + """Der Separator innerhalb der Klammer zerschneidet den Titel nicht mehr.""" + t = _title("Aε-Algorithmus (Güte 1+ε – Laufzeit O(n³)) — der Approx-Algorithmus") + assert t == "Aε-Algorithmus (Güte 1+ε – Laufzeit O(n³))" + + +def test_title_plain_split_unchanged(): + assert _title("Vertex Cover — minimale Knotenüberdeckung") == "Vertex Cover" + assert _title("Ohne Separator bleibt ganz") == "Ohne Separator bleibt ganz" + + +def test_title_unclosed_bracket_keeps_entry(): + """Malformed: offene Klammer nie geschlossen → keinen abgeschnittenen Titel ausgeben, + stattdessen den ganzen Eintrag behalten (Repair = Split rückgängig).""" + e = "N P via Turingmaschine (Definition 6.19 — Rest" + assert _title(e) == e + + +def test_parse_selection_then_title_regression(): + """End-to-end der Parse-Kette (wie in der Pipeline): _norm_dash + _title lassen den + Klammer-Inhalt intakt statt mitten im Titel zu splitten (der reale aak-Bug).""" + line = "1. Aε-Algorithmus (Güte 1+ε – Laufzeit O(n³)) — der Approximations-Algorithmus" + entries = _parse_selection(line) + assert _title(entries[1]) == "Aε-Algorithmus (Güte 1+ε – Laufzeit O(n³))" diff --git a/backend/textkit.py b/backend/textkit.py index e69d04b..603e709 100644 --- a/backend/textkit.py +++ b/backend/textkit.py @@ -32,8 +32,46 @@ def _norm_title(s: str) -> str: return s.casefold() +# Bracket-aware separator handling: a ' — ' (or a dash that _norm_dash would normalize) INSIDE +# ()[]{} is part of the title, not a title/description separator. Without this a spaced dash within +# a math title truncates it: 'Aε-Algorithmus (Güte 1+ε, Laufzeit O(n³)) — …' → 'Aε-Algorithmus (Güte…'. +_BRACKETS = {"(": ")", "[": "]", "{": "}"} +_CLOSERS = {v: k for k, v in _BRACKETS.items()} + + +def _unclosed(s: str) -> bool: + """True if s has a dangling opening bracket ()[]{} — the symptom of a mid-bracket cut. Extra + closers (more ')' than '(') do NOT count; only an unclosed opener signals a truncated title.""" + depth = 0 + for c in s: + if c in _BRACKETS: + depth += 1 + elif c in _CLOSERS and depth > 0: + depth -= 1 + return depth > 0 + + +def _split_top(s: str, sep: str) -> str: + """First bracket-depth-0 segment of s split on sep; whole s if sep never occurs at depth 0.""" + depth = 0 + for i, c in enumerate(s): + if c in _BRACKETS: + depth += 1 + elif c in _CLOSERS and depth > 0: + depth -= 1 + elif depth == 0 and s.startswith(sep, i): + return s[:i] + return s + + def _title(entry: str) -> str: - return entry.split(" — ")[0].strip() or entry + """Title = text before the first TOP-LEVEL ' — '. A separator inside ()[]{} does not split + (keeps math titles intact); if the extracted title has an unclosed bracket (malformed source), + keep the whole entry rather than emit a truncated title.""" + title = _split_top(entry, " — ") + if _unclosed(title): + return entry.strip() or entry + return title.strip() or entry def clean_title(s: str) -> str: @@ -74,6 +112,9 @@ def _resolve_title(idx: dict[str, int], t: str) -> int | None: return idx.get(_norm_title(t)) or idx.get(_norm_title(_title(t))) +_DASH_SEP_RE = re.compile(r"\s*[‒–—―]\s+|\s+[‒–—―]\s*") + + def _norm_dash(s: str) -> str: """Dash variants (en/em/figure/bar) with whitespace on AT LEAST ONE side → uniform separator ' — '. Some models (especially non-western ones) use an en-dash "–" instead of the em-dash; without @@ -81,8 +122,12 @@ def _norm_dash(s: str) -> str: space ("Titel —Beschreibung" / "Titel— Beschreibung") also breaks the split and leaks the source filename into the description — so a dash with a space on either side is repaired too. The ASCII hyphen "-" is deliberately NOT in the class (would split "n - 1"/"3-SAT"); requiring ≥1 surrounding - space keeps glued compounds like "Backtracking—Verfahren" and number ranges like "12–15" untouched.""" - return re.sub(r"\s*[‒–—―]\s+|\s+[‒–—―]\s*", " — ", s) + space keeps glued compounds like "Backtracking—Verfahren" and number ranges like "12–15" untouched. + A qualifying dash INSIDE an open bracket ()[]{} is left untouched — it is part of the title, not a + separator (else 'Algorithmus (Güte 1+ε – O(n³)) — …' would split mid-parenthetical).""" + def _repl(m): + return m.group(0) if _unclosed(s[:m.start()]) else " — " + return _DASH_SEP_RE.sub(_repl, s) def _parse_selection(text: str) -> dict[int, str]: diff --git a/backend/train_params.py b/backend/train_params.py index 7eca560..ea163e3 100644 --- a/backend/train_params.py +++ b/backend/train_params.py @@ -36,7 +36,6 @@ PARAMS: dict[str, dict] = { "MAX_RESTARTS": {"default": 2, "min": 1, "max": 3, "step": 1, "kategorie": "laufzeit", "fidelity": "board2"}, "HEDGE_NACH_S": {"default": 90, "min": 30, "max": 240, "step": 30, "kategorie": "laufzeit", "fidelity": "board2"}, "EVIDENCE_BUDGET_CHARS": {"default": 48000, "min": 16000, "max": 64000, "step": 8000, "kategorie": "tokens", "fidelity": "board2"}, - "QUELLE_RELEVANZ_CHUNK": {"default": 12, "min": 6, "max": 24, "step": 3, "kategorie": "laufzeit", "fidelity": "voll"}, } diff --git a/frontend/src/components/SourceForm.vue b/frontend/src/components/SourceForm.vue index a4e26e3..b6c146a 100644 --- a/frontend/src/components/SourceForm.vue +++ b/frontend/src/components/SourceForm.vue @@ -27,12 +27,12 @@ function setOrt(ort) { -