diff --git a/Makefile b/Makefile index b60dd31..3ecb122 100644 --- a/Makefile +++ b/Makefile @@ -65,12 +65,11 @@ ollama: # Remote wird für die DB-Kopie gestoppt (sonst reißt der Snapshot mitten im Schreibvorgang) # und danach wieder gestartet. sync: stop - @mkdir -p storage/themen storage/topics uni + @mkdir -p storage/topics uni @rm -f storage/creator.db-shm storage/creator.db-wal ssh root@178.104.67.87 'cd /var/www/creator && docker compose down' rsync -avz --progress root@178.104.67.87:/var/www/creator/storage/creator.db storage/ -rsync -avz --progress root@178.104.67.87:/var/www/creator/storage/creator.db-wal storage/ - rsync -avz --progress --delete root@178.104.67.87:/var/www/creator/storage/themen/ storage/themen/ rsync -avz --progress --delete root@178.104.67.87:/var/www/creator/storage/topics/ storage/topics/ rsync -avz --progress --delete root@178.104.67.87:/var/www/creator/uni/ uni/ ssh root@178.104.67.87 'cd /var/www/creator && docker compose up -d' @@ -90,13 +89,12 @@ sync-all: sync sync-projects # Stoppt den Remote-Server (DB-Sicherheit), pusht DB + Themen + Projekte, startet ihn neu. # --delete = exakter Spiegel: Remote-Dateien ohne lokales Pendant werden gelöscht. sync-all-reverse: stop - @echo "ACHTUNG: überschreibt den Remote-Stand (DB, themen/, topics/, uni/, projects/) mit dem lokalen." + @echo "ACHTUNG: überschreibt den Remote-Stand (DB, topics/, uni/, projects/) mit dem lokalen." @[ -f storage/creator.db ] || { echo "Keine lokale DB — abgebrochen."; exit 1; } ssh root@178.104.67.87 'cd /var/www/creator && docker compose down' ssh root@178.104.67.87 'rm -f /var/www/creator/storage/creator.db-shm /var/www/creator/storage/creator.db-wal' rsync -avz --progress storage/creator.db root@178.104.67.87:/var/www/creator/storage/ @[ -f storage/creator.db-wal ] && rsync -avz --progress storage/creator.db-wal root@178.104.67.87:/var/www/creator/storage/ || true - rsync -avz --progress --delete storage/themen/ root@178.104.67.87:/var/www/creator/storage/themen/ rsync -avz --progress --delete storage/topics/ root@178.104.67.87:/var/www/creator/storage/topics/ rsync -avz --progress --delete uni/ root@178.104.67.87:/var/www/creator/uni/ rsync -avz --progress --delete projects/ root@178.104.67.87:/var/www/creator/projects/ diff --git a/backend/blocks.py b/backend/blocks.py index 0e48753..015a781 100644 --- a/backend/blocks.py +++ b/backend/blocks.py @@ -276,24 +276,6 @@ def _blocks_files(topic: str) -> dict: } -def _all_slot_files(files: dict) -> list[Path]: - work_dir = files["arbeit"] - # Subblock/levels slots are dynamic per chunk — collect via glob. - dyn = (list(work_dir.glob("subblock-*")) + list(work_dir.glob("facts-*")) + list(work_dir.glob("level-*")) + list(work_dir.glob("relevance-*")) - + list(work_dir.glob("question-pattern-*")) + list(work_dir.glob("outline-*")) + list(work_dir.glob("artifact-*")) - + list(work_dir.glob("research-*")) + list(work_dir.glob("consolidation-*")) - + list(work_dir.glob("clarification*")) + list(work_dir.glob("dedup-*")) - + list(work_dir.glob("inventar-filter*")) - + list(work_dir.glob("gruppierung-*")) + list(work_dir.glob("inventar-gruppierung*"))) if work_dir.is_dir() else [] - return [ - *files["research"], files["research_mapping"], - *(p for slots in files["selection"].values() for p in slots), - *files["mapping"].values(), files["ergaenzung"], - files["sub_roh"], files["sidecar"], files["question_pattern"], - files["facts"], files["outline"], files["artefakte"], *dyn, - ] - - def cancel_blocks(topic: str) -> bool: if topic not in _blocks_progress: return False @@ -655,54 +637,6 @@ def _file_payload(path: Path): return text if _parse_selection(text) else None -def _mapping_schema(data): - """{"blocks": [str, ≥1], "rest": [str]} → (blocks, rest) · otherwise None.""" - if not isinstance(data, dict): - return None - blocks = _str_list(data.get("blocks")) - rest = _str_list(data.get("rest")) - if not blocks or rest is None: - return None - return blocks, rest - - -def _sub_raw_schema(data): - """{block title: [subblock, …]} → dict · otherwise None (intermediate state of block B).""" - if not isinstance(data, dict) or not data: - return None - out: dict[str, list[str]] = {} - for k, v in data.items(): - subs = _str_list(v) if isinstance(v, list) else None - if not isinstance(k, str) or not k.strip() or not subs: - return None - out[k] = subs - return out - - -def _sidecar_schema(data): - """{block title: [{title, level}, …]} → dict · otherwise None (sidecar with levels).""" - if not isinstance(data, dict) or not data: - return None - for v in data.values(): - if not isinstance(v, list) or not v: - return None - for s in v: - if not isinstance(s, dict) or not str(s.get("title", "")).strip() or s.get("level") not in _LEVELS: - return None - return data - - -def _relevance_complete(data) -> bool: - """Does every subblock in the sidecar carry a valid relevance (relevant/peripheral)?""" - if not isinstance(data, dict) or not data: - return False - return all( - isinstance(s, dict) and s.get("relevance") in ("relevant", "peripheral") - for v in data.values() if isinstance(v, list) - for s in v - ) - - def _question_pattern_chunk_schema(data) -> list[dict] | None: @@ -725,12 +659,6 @@ def _question_pattern_chunk_schema(data) -> list[dict] | None: return out or None -def _question_pattern_complete(topic: str) -> bool: - """Does the question-pattern sidecar exist (build ran)? Individual empty blocks - fall back to live generation at exam time — so the file is enough.""" - return isinstance(_json_file(question_pattern_path(topic)), dict) - - def _read(p: Path) -> str: return p.read_text(encoding="utf-8") if p.exists() else "" @@ -1457,12 +1385,6 @@ def _facts_lines(fk: dict) -> str: return "\n".join(z) -def _facts_complete(files: dict) -> bool: - """Does the facts map exist (block done)? {block: {sub_norm: {...}}}.""" - d = _json_file(files["facts"]) - return isinstance(d, dict) and bool(d) - - async def _facts_block(ctx, set_p, files: dict, raw: dict, q: dict, folder, instructions: str, ns: str = "", lbl: str = "", sources: list[str] | None = None) -> tuple | None: """Block: per sub extract source facts (find) → verify (check) → correct/discard (fix). Extract-once grounding: the result feeds level/relevance/questions/guide. @@ -2043,18 +1965,6 @@ def _crawl_index(folder) -> dict[str, str]: -async def _set_inventory(topic: str, record: str, status: str) -> None: - """Write an inventory entry ('title — description') with status to the DB.""" - title = _title(record) - norm = _norm_title(title) - if not norm: - return - split_parts = [t.strip() for t in record.split(" — ")] - desc = split_parts[1] if len(split_parts) >= 2 else "" - await db.upsert_block(topic, norm, title, desc) - await db.set_block_status(topic, norm, status) - - def _triage_rules(folder, pages: list[str]) -> tuple[list[str], list[str]]: """Deterministic content/noise filter (config.CRAWL_*). Substring match (lowercase) against URL + filename. Order: keep > noise > min_chars > keep. → (content, noise).""" @@ -2324,6 +2234,22 @@ _CANON_CATALOGUE = re.compile( r'|kapitel|chapter|abschnitt|section)\s*\d+(?:\.\d+)*\b', re.I) +_PAREN_GROUP = re.compile(r'^\s*(.*?)\s*\(([^()]{2,60})\)\s*$') + + +def _title_variants(title: str) -> set[str]: + """Acronym/expansion variants of a "X (Y)" title — normalized outer part and paren + content. „Satisfiability Problem (SAT)" → {'satisfiability problem', 'sat'}: matched + against another card's norm/key this makes acronym↔expansion pairs dedup CANDIDATES + (measured: title cosine 'sat' vs the long form is 0.53, far below the floor). + Titles without exactly one trailing paren group → empty set.""" + from textkit import _norm_title + m = _PAREN_GROUP.match(title or "") + if not m: + return set() + return {v for v in (_norm_title(m.group(1)), _norm_title(m.group(2))) if v} + + def _canonical_key(title: str) -> str: """Order-independent canonical key of a title (scaffolding stripped, relation operators normalized). Two titles with the same key denote the same entity with ~100% precision (ER blocking). Empty string @@ -2622,12 +2548,6 @@ _GROUP_STANDALONE = re.compile( # --- Outline (blocks artifact: chapter structure, only read by the guide) --- -def _outline_complete(files: dict) -> bool: - """Is the outline present (chapter list exists)?""" - d = _json_file(files["outline"]) - return isinstance(d, dict) and isinstance(d.get("chapters"), list) and bool(d.get("chapters")) - - def _outline_review_schema(data, valid: set[int], n_chapters: int, n_blocks: int): """{"moves": {"": }} → {nr: idx} (may be {}) · None if broken/invalid. A mass rewrite (more than a third of all blocks) is rejected — the reviewer's job is @@ -2898,12 +2818,6 @@ _ARTEFACT_PROMPT = {"flashcard": "Artifact-Flashcard", "example": "Artifact-Exam _ARTEFACT_STEP = {"flashcard": "Flashcards", "example": "Examples"} -def _artefacts_complete(files: dict) -> bool: - """Artifact map present (all types generated)? Values may be empty (content-aware).""" - d = _json_file(files["artefakte"]) - return isinstance(d, dict) and all(t in d for t in ARTEFACT_TYPES) - - async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, instructions: str, ns: str = "", lbl: str = "") -> dict | None: """Generate learning artefacts per type from the stored facts — one generation pass per type over chunks. Worked examples are verified against the facts (wrong ones discarded); @@ -3009,20 +2923,6 @@ async def _artefacts_block(ctx: GenContext, set_p, files: dict, sidecar: dict, i return outcome -async def _mirror_artefacts_db(topic: str, sidecar: dict, artefacts: dict) -> None: - """Mirror artefacts into the DB. Flashcard/example per sub (sub_norm).""" - await db.delete_sub_artefakte(topic) - btitle_list = list(sidecar.keys()) - for type in ARTEFACT_TYPES: - for e in artefacts.get(type, []): - bt = _match_sub(e.get("block", ""), btitle_list) - bnorm, sn = _norm_title(bt), _norm_title(e.get("subblock", "")) - if not bnorm or not sn: - continue - data = json.dumps({k: v for k, v in e.items() if k not in ("block", "subblock")}, ensure_ascii=False) - await db.put_sub_artifact(topic, bnorm, sn, type, data, bt, e.get("subblock", "")) - - async def _mirror_sidecar_db(topic: str, sidecar: dict) -> None: """Mirror the sidecar {block title: [{title, level, relevance}]} into the DB table subblocks.""" for btitle, subs in sidecar.items(): @@ -3042,24 +2942,6 @@ async def _mirror_sidecar_db(topic: str, sidecar: dict) -> None: facts=facts, status="consensus") -async def _mirror_question_pattern_db(topic: str, pattern: dict) -> None: - """Mirror question patterns {block title: [{subblock, question}]} into the DB table question_pattern.""" - await db.delete_question_pattern(topic) - for btitle, eintraege in pattern.items(): - bnorm = _norm_title(btitle) - if not bnorm or not isinstance(eintraege, list): - continue - for e in eintraege: - if not isinstance(e, dict): - continue - sub = str(e.get("subblock", "")).strip() - sn = _norm_title(sub) - question = str(e.get("question", "")).strip() - if not (sn and question): - continue - await db.upsert_question_pattern(topic, bnorm, sn, btitle, sub, question) - - async def generate_blocks(topic: str, instructions: str = "", provider: str = DEFAULT_PROVIDER, diff --git a/backend/board_inventory.py b/backend/board_inventory.py index 351bf39..c621f44 100644 --- a/backend/board_inventory.py +++ b/backend/board_inventory.py @@ -43,7 +43,7 @@ from blocks import ( _filter_schema, _filter_suspect, _is_artifact, _is_named_statement, _is_parentless_noise, _is_reference, _pairs_schema, _read, _relation_conflict, _root, _supplement_schema, _text_sections, _umbrella_schema, - _aspect_marker, + _aspect_marker, _title_variants, _evidence_pack, _sink_json, source_folder, ) from config import ( BLOCKS_GRUPPIERUNG_AKTIV, EMBEDDING_AKTIV, EMBEDDING_BLOCK_CAP, @@ -53,8 +53,8 @@ from config import ( from fsutil import atomic_write_json, atomic_write_text from jsonio import read_json_file as _json_file from pipeline import (CANCELLED, FAILED, OK, GenContext, _extra, _log, _prompt, - _runde_schema, _timeout, run_single_slot) -from textkit import _norm_title, _parse_selection, _title + _runde_schema, _timeout, _yesno_schema, run_single_slot) +from textkit import _norm_title, _parse_selection, _title, clean_title log = logging.getLogger("creator.board_inventory") @@ -144,13 +144,13 @@ async def _ingest_titles(flow: Flow, text: str, reader: str, source: str = "") - reader union. Repeated drains over a growing buffer are idempotent. → new count.""" n, seen = 0, set() for record in _parse_selection(text).values(): - title = _title(record) + title = clean_title(_title(record)) norm = _norm_title(title) if not norm or norm in seen: continue seen.add(norm) # one reader = one vote per concept parts = [t.strip() for t in record.split(" — ")] - desc = parts[1] if len(parts) >= 2 else "" + desc = clean_title(parts[1]) if len(parts) >= 2 else "" src = source or (parts[2] if len(parts) >= 3 else "") async with _ingest_lock: if await db.kanban_add_title(flow.topic, BOARD, norm, title, desc, src, reader): @@ -924,6 +924,15 @@ async def _proc_dedup(ctx: GenContext, flow: Flow, cards): for y in range(x + 1, len(grp)): if _demotable_pair(grp[x], grp[y]): pairs.add((grp[x], grp[y])) + # acronym↔expansion pairs ("SAT" vs "Satisfiability Problem (SAT)"): the title cosine + # of short vs long form sits far below the floor — variant match makes them candidates, + # the judge panel decides as usual (no auto-merge) + idents = [{_norm_title(r["title"]), _canonical_key(r["title"])} - {""} for r in allrows] + variants = [_title_variants(r["title"]) for r in allrows] + for i in range(n_all): + for j in range(i + 1, n_all): + if _demotable_pair(i, j) and (variants[i] & idents[j] or variants[j] & idents[i]): + pairs.add((i, j)) ordered = sorted(pairs) h = _h(*[r["card_id"] for r in allrows]) if not ordered: @@ -1176,7 +1185,7 @@ async def _proc_grouping(ctx: GenContext, flow: Flow, cards): readers = sorted(set().union(*[set(r["payload"].get("readers") or []) for r in mrows])) srcs = sorted(set().union(*[set(r["payload"].get("sources") or []) for r in mrows])) await db.kanban_upsert_card(topic, BOARD, f"b-u-{uuid.uuid4().hex[:8]}", "block", "gap_check", { - "title": c["umbrella"], "description": c["description"], + "title": clean_title(c["umbrella"]), "description": c["description"], "readers": readers, "sources": srcs, "umbrella": True, "children": [r["title"] for r in mrows], }) @@ -1221,6 +1230,37 @@ async def _proc_gap_check(ctx: GenContext, flow: Flow, cards): flow.wake.set() +async def _supplement_beleg(ctx: GenContext, flow: Flow, supplements: list) -> list: + """Evidence gate for supplement proposals: keyword excerpts per proposal, ONE no-tool + judge marks material coverage (ja/nein). Proposals without any matching excerpt drop + immediately; a failed gate keeps nothing (creep is costlier than a lost bonus round).""" + topic = flow.topic + folder = source_folder(topic) + packs = [(t, d, _evidence_pack(folder, None, [t], budget=6000)) for t, d in supplements] + cands = [(t, d, ev) for t, d, ev in packs if ev] + kept: list = [] + if cands: + path = flow.work_dir / "supplement-beleg.json" + ids = set(range(1, len(cands) + 1)) + verdict = _yesno_schema(_json_file(path), ids) + if verdict is None: + lines = "\n\n".join(f"{k}. {t} — {d}\nAUSZÜGE:\n{ev}" + for k, (t, d, ev) in enumerate(cands, 1)) + status, verdict = await run_single_slot( + ctx, "Supplement-Beleg", key=f"blocks-{topic}-supplement-beleg", + prompt=_prompt("Blocks-Supplement-Beleg", topic=topic, proposals=lines, + extra=_extra(flow.state.get("instructions", ""))), + role="judge", capabilities="none", + payload=lambda result, p=path, i=ids: _sink_json(result, p, lambda d2: _yesno_schema(d2, i)), + timeout=_timeout("selection_mapping", len(cands))) + if status != OK or not isinstance(verdict, dict): + verdict = {} + kept = [(t, d) for k, (t, d, ev) in enumerate(cands, 1) if str(verdict.get(k, "nein")) == "ja"] + if len(kept) < len(supplements): + _log(topic, f"Supplement: {len(supplements) - len(kept)}/{len(supplements)} ohne Materialbeleg verworfen") + return kept + + async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]): """One web agent proposes canonically missing blocks → new title cards (reader 'supplement' skips only the ≥2 consensus bar, every other gate applies).""" @@ -1242,6 +1282,11 @@ async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]): if status != OK: _log(topic, "Supplement fehlgeschlagen — übersprungen (optional)") supplements = [] + # Material anchoring (uni/projekt): the web agent proposes canonical knowledge BLIND + # to the source (measured: 22/107 aak blocks were textbook standard the script never + # treats). Only proposals the material itself covers may enter the inventory. + if supplements and source_folder(topic): + supplements = await _supplement_beleg(ctx, flow, supplements) # Dead lineage: blocks demoted by the fragment filter (and their cluster + title cards) # must NOT dedup a supplement proposal — their content is gone. A hit on a dead title # REOPENS the lineage instead: the title card rejoins its cluster (live re-cluster) and @@ -1281,6 +1326,7 @@ async def _supplement_producer(ctx: GenContext, flow: Flow, titles: list[str]): known_keys.add(k) new = reopened = 0 for t, d in (supplements or []): + t, d = clean_title(t), clean_title(d) norm = _norm_title(t) key = _canonical_key(t) if not norm or norm in known_norms or (key and key in known_keys): diff --git a/backend/config.py b/backend/config.py index 62ce590..8d5819f 100644 --- a/backend/config.py +++ b/backend/config.py @@ -114,11 +114,6 @@ MAX_CONCURRENT_INTERACTIVE = 8 # (kill only once the minimum is already in). CONSENSUS_GRACE = 300 -# Research race: longer grace window. Research drives the whole block count; -# with slow providers (e.g. MiniMax) ALL 5 agents should become done, not just -# the quorum of 3. The per-agent timeout (TIMEOUTS["research"]=1800s) caps real hangs. -RESEARCH_GRACE = 900 - # Cap of the clarification and check loops: maximum rounds until everything must be # decided. In the last round the mapping agent MUST decide every entry; # check loops leave any remaining objections standing after that. diff --git a/backend/database.py b/backend/database.py index 761e8c4..ac82c9b 100644 --- a/backend/database.py +++ b/backend/database.py @@ -18,15 +18,6 @@ CREATE TABLE IF NOT EXISTS guides ( ) """ -CREATE_PROGRESS = """ -CREATE TABLE IF NOT EXISTS guide_progress ( - guide_id TEXT NOT NULL, - chapter TEXT NOT NULL, - created_at TEXT NOT NULL, - PRIMARY KEY (guide_id, chapter) -) -""" - CREATE_TOPICS = """ CREATE TABLE IF NOT EXISTS topics ( name TEXT PRIMARY KEY, @@ -34,18 +25,6 @@ CREATE TABLE IF NOT EXISTS topics ( ) """ -CREATE_BLOCK_TEXTE = """ -CREATE TABLE IF NOT EXISTS block_texte ( - topic TEXT NOT NULL, - block TEXT NOT NULL, - kind TEXT NOT NULL, - md TEXT NOT NULL, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - PRIMARY KEY (topic, block, kind) -) -""" - CREATE_BLOCK_PROGRESS = """ CREATE TABLE IF NOT EXISTS block_progress ( topic TEXT NOT NULL, @@ -308,9 +287,7 @@ async def init_db(): await db.execute("PRAGMA synchronous=NORMAL") await db.execute("PRAGMA busy_timeout=5000") await db.execute(CREATE_GUIDES) - await db.execute(CREATE_PROGRESS) await db.execute(CREATE_TOPICS) - await db.execute(CREATE_BLOCK_TEXTE) await db.execute(CREATE_BLOCK_PROGRESS) await db.execute(CREATE_BLOCKS) await db.execute(CREATE_SUBBLOCKS) @@ -370,16 +347,13 @@ async def init_db(): await db.execute("ALTER TABLE blocks ADD COLUMN reader TEXT NOT NULL DEFAULT '[]'") except aiosqlite.OperationalError: pass - # Migration: old vertiefungen table → block_texte (existing = long form, kind 'deepdive') - cursor = await db.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'vertiefungen'") - if await cursor.fetchone(): - await db.execute( - "INSERT OR IGNORE INTO block_texte (topic, block, kind, md, created_at, updated_at) " - "SELECT topic, block, 'deepdive', md, created_at, updated_at FROM vertiefungen" - ) - await db.execute("DROP TABLE vertiefungen") - # Migration: the elements feature was removed entirely — drop its orphaned table. + # Migration: removed features leave orphaned tables behind — drop them. + # elements (feature removed), vertiefungen/block_texte (never read), guide_progress + # (chapter progress had no frontend and its only reader ignored the value). await db.execute("DROP TABLE IF EXISTS elements") + await db.execute("DROP TABLE IF EXISTS vertiefungen") + await db.execute("DROP TABLE IF EXISTS block_texte") + await db.execute("DROP TABLE IF EXISTS guide_progress") await db.execute( "UPDATE guides SET status = 'error', progress = NULL, error_msg = 'Server restart' " "WHERE status IN ('queued', 'generating')" @@ -473,49 +447,6 @@ async def delete_topic(name: str) -> None: await db.commit() -# --- Chapter progress --- - -async def list_progress_all() -> dict[str, set[str]]: - """Complete chapter progress in one query: guide_id → chapter title.""" - db = await get_db() - cursor = await db.execute("SELECT guide_id, chapter FROM guide_progress") - rows = await cursor.fetchall() - out: dict[str, set[str]] = {} - for guide_id, chapter in rows: - out.setdefault(guide_id, set()).add(chapter) - return out - - -async def list_progress(guide_id: str) -> list[str]: - db = await get_db() - cursor = await db.execute( - "SELECT chapter FROM guide_progress WHERE guide_id = ?", (guide_id,) - ) - rows = await cursor.fetchall() - return [row[0] for row in rows] - - -async def set_progress(guide_id: str, chapter: str, done: bool) -> None: - from datetime import datetime, timezone - db = await get_db() - if done: - await db.execute( - "INSERT OR IGNORE INTO guide_progress (guide_id, chapter, created_at) VALUES (?, ?, ?)", - (guide_id, chapter, datetime.now(timezone.utc).isoformat()), - ) - else: - await db.execute( - "DELETE FROM guide_progress WHERE guide_id = ? AND chapter = ?", (guide_id, chapter) - ) - await db.commit() - - -async def delete_progress(guide_id: str) -> None: - db = await get_db() - await db.execute("DELETE FROM guide_progress WHERE guide_id = ?", (guide_id,)) - await db.commit() - - # --- Block learning: deep-dives + exam progress --- def _now() -> str: @@ -667,7 +598,6 @@ async def list_block_scores_all() -> list[tuple[str, str, int]]: async def delete_block_data(topic: str) -> None: db = await get_db() - await db.execute("DELETE FROM block_texte WHERE topic = ?", (topic,)) await db.execute("DELETE FROM block_progress WHERE topic = ?", (topic,)) await db.commit() @@ -984,14 +914,6 @@ async def kanban_members_of(topic: str, group_id: str) -> list[str]: return [r[0] for r in await cursor.fetchall()] -async def kanban_member_group(topic: str, member_id: str) -> str | None: - db = await get_db() - cursor = await db.execute( - "SELECT group_id FROM kanban_members WHERE topic = ? AND member_id = ?", (topic, member_id)) - row = await cursor.fetchone() - return row[0] if row else None - - # ── Guide board (one card per block, linear stages) ────────────────────────────── async def upsert_guide_card(topic: str, format: str, block_norm: str, block: str, stage: str = "lernziele") -> None: @@ -1277,13 +1199,6 @@ async def mark_sources_read_done(topic: str, sources: list[str]) -> None: await db.commit() -async def list_coverage(topic: str) -> dict[str, int]: - db = await get_db() - cursor = await db.execute("SELECT source, read_done FROM research_coverage WHERE topic = ?", (topic,)) - rows = await cursor.fetchall() - return {q: g for q, g in rows} - - async def mark_content(topic: str, content: list[str], noise: list[str]) -> None: """Store the triage result per crawl page: content=1 (content) or 0 (noise).""" db = await get_db() @@ -1335,16 +1250,6 @@ async def get_step_status(topic: str, step: str) -> str: -async def delete_pipeline_state(topic: str, steps: list[str] | None = None) -> None: - db = await get_db() - if steps is None: - await db.execute("DELETE FROM pipeline_state WHERE topic = ?", (topic,)) - elif steps: - marks = ",".join("?" for _ in steps) - await db.execute(f"DELETE FROM pipeline_state WHERE topic = ? AND step IN ({marks})", (topic, *steps)) - await db.commit() - - diff --git a/backend/models.py b/backend/models.py index 3e530bc..1ffeb93 100644 --- a/backend/models.py +++ b/backend/models.py @@ -90,10 +90,6 @@ class BlocksStatusResponse(BaseModel): feine_steps: list[BlocksFineStep] = [] -class ProjectResponse(BaseModel): - name: str - - class FolderResponse(BaseModel): name: str location: str # path relative to the repo root (e.g. "projects/foo") @@ -158,15 +154,6 @@ class GuideChatResponse(BaseModel): reply: str -class ProgressUpdate(BaseModel): - chapter: str = Field(min_length=1, max_length=100) - done: bool - - -class ProgressResponse(BaseModel): - chapters: list[str] - - # --- Block learning --- class BlockChatRequest(BaseModel): diff --git a/backend/routes.py b/backend/routes.py index 2bf7695..a3a0430 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -12,7 +12,6 @@ from config import PROJECTS_DIR, UNI_DIR, PROVIDERS from database import ( create_guide, delete_guide, get_guide, list_guides, create_topic, list_topics as db_list_topics, delete_topic, - list_progress, set_progress, delete_progress, list_block_progress, get_block_progress, set_open_question, set_block_score_and_streak, delete_block_data, delete_block_progress, subs_per_level, subs_per_level_raw, @@ -33,7 +32,7 @@ from models import ( BlocksCreateRequest, BlocksResetStageRequest, BlocksCardRestartRequest, BlocksStatusResponse, GuideCardResetRequest, GuideFormatRequest, GuideBoardResetRequest, GuideChatRequest, GuideChatResponse, - ProgressUpdate, ProgressResponse, ProjectResponse, ProviderInfo, + ProviderInfo, FolderResponse, BlocksSourceUpdate, BlocksSourceResponse, BlockOverview, BlockChatRequest, BlockChatResponse, BlockExamRequest, BlockExamResponse, BlockLearnStateResponse, @@ -65,19 +64,19 @@ async def get_topics(): @router.get("/stats") async def get_stats(): """Tracker: number of topics + per format created/completed.""" - guides, progress, levels = await load_learnstate() + guides, levels = await load_learnstate() topics = set(await db_list_topics()) | {g["topic"] for g in guides} | set(blocks_topics()) if PROJECTS_DIR.is_dir(): topics |= {e.name for e in PROJECTS_DIR.iterdir() if e.is_dir()} - return {"topics": len(topics), "formats": formats_stats(guides, progress, levels)} + return {"topics": len(topics), "formats": formats_stats(guides, levels)} @router.get("/topics/progress") async def topic_progress(topic: str): """Completion status per format + topic completion — for unlocking the next expansion stage.""" - guides, progress, levels = await load_learnstate() - status = {fmt: ist_completed(topic, fmt, guides, progress, levels) for fmt in FORMATE} - status["completed"] = topic_completed(topic, guides, progress, levels) + guides, levels = await load_learnstate() + status = {fmt: ist_completed(topic, fmt, guides, levels) for fmt in FORMATE} + status["completed"] = topic_completed(topic, guides, levels) return status @@ -98,29 +97,6 @@ async def remove_topic(topic: str): return {"ok": True} -def _safe_project_name(name: str) -> str: - if not name or "/" in name or "\\" in name or ".." in name or "\x00" in name: - raise HTTPException(400, "Invalid project name") - return name - - -@router.get("/projects", response_model=list[ProjectResponse]) -async def list_projects(): - if not PROJECTS_DIR.is_dir(): - return [] - return [{"name": entry.name} for entry in sorted(PROJECTS_DIR.iterdir()) if entry.is_dir()] - - -@router.delete("/projects/{name}") -async def remove_project(name: str): - _safe_project_name(name) - pdir = project_dir(name) - if not pdir.is_dir(): - raise HTTPException(404, "Project not found") - shutil.rmtree(pdir) - return {"ok": True} - - @router.get("/folders", response_model=list[FolderResponse]) async def list_folders(kind: str): """Folders for the source selection: kind=projekt → projects/, kind=uni → uni/.""" @@ -181,12 +157,6 @@ async def get_blocks_board(topic: str): return snap -@router.get("/blocks/agents") -async def get_blocks_agents(topic: str): - return [{"key": a["key"], "label": a["label"] or a["key"].removeprefix(f"blocks-{topic}-"), "runtime": a["runtime"]} - for a in active_agents(f"blocks-{topic}-")] - - @router.post("/blocks/research") async def add_blocks_research(topic: str, provider: str = "claude"): """Attach one more research agent — to the live flow, or attach-or-start.""" @@ -335,24 +305,6 @@ async def get_question_pattern(topic: str, block: str): return {"pattern": await load_question_pattern_free(topic, block, fe)} -@router.get("/blocks/artefakte") -async def get_artefakte(topic: str, type: str | None = None): - """Learning artifacts (flashcards/examples) per topic, grouped by block norm — per subblock.""" - rows = await get_sub_artefakte(topic, type) - out: dict[str, dict] = {} - for r in rows: - b = out.setdefault(r["block_norm"], {"block": r["block"], "flashcard": [], "example": []}) - if r["block"] and not b["block"]: - b["block"] = r["block"] - try: - data = json.loads(r["data"]) - except (ValueError, TypeError): - continue - if r["type"] in ("flashcard", "example"): - b[r["type"]].append({"subblock": r["sub_title"], **data}) - return {"artefakte": out} - - # --- Practice deck: Leitner flashcard pool per topic --- async def build_practice_deck(topic: str) -> dict: @@ -627,8 +579,8 @@ async def block_exam_route(req: BlockExamRequest): @router.post("/guides", response_model=GuideResponse) async def create(req: GuideCreateRequest): - guides, progress, levels = await load_learnstate() - reason = guide_lock(req.topic.strip(), req.format, guides, progress, levels) + guides, levels = await load_learnstate() + reason = guide_lock(req.topic.strip(), req.format, guides, levels) if reason: raise HTTPException(400 if reason == "Erst Blocks erstellen" else 409, reason) # string matches rules.py contract await create_topic(req.topic.strip()) @@ -653,27 +605,6 @@ async def list_all(): return await list_guides() -@router.get("/guides/locks") -async def guide_locks(topic: str): - """Lock reasons per format for the ▶ button — None = creatable.""" - guides, progress, levels = await load_learnstate() - return {fmt: guide_lock(topic, fmt, guides, progress, levels) for fmt in ("FullGuide", "Rest", *FORMATE)} - - -@router.get("/guides/steps") -async def guide_steps(topic: str): - """Highest fully completed stage index per format (card-based, -1 = none). - Content file present (legacy without cards) → everything done.""" - import guide_board - out = {} - for fmt in ("Guide", "FullGuide", "Rest"): - step = await guide_board.done_step(topic, fmt) - if step < 0 and guide_content_path(topic, fmt).exists(): - step = len(guide_board.GUIDE_STAGES) - out[fmt] = step - return out - - @router.get("/guides/board") async def get_guide_board(topic: str, format: str = "Guide"): """Live guide board: columns with counts + cards (rounds, covered objectives), agents.""" @@ -704,14 +635,6 @@ async def reset_guide_board(req: GuideBoardResetRequest): return {"ok": True, "moved": moved} -@router.get("/guides/{guide_id}", response_model=GuideResponse) -async def get_one(guide_id: str): - guide = await get_guide(guide_id) - if guide is None: - raise HTTPException(404, "Guide not found") - return guide - - @router.get("/guides/{guide_id}/content") async def guide_content(guide_id: str, level: int = 4): """Guide content. `level` (1=A · 2=F · 3=E · 4=V) filters to subblocks up to this @@ -789,7 +712,6 @@ async def remove_guide_format(req: GuideFormatRequest): if any(g["status"] in ("queued", "generating") for g in doomed): return {"ok": True, "status": "generating"} for g in doomed: - await delete_progress(g["id"]) await delete_guide(g["id"]) await delete_guide_content(req.topic, req.format) await delete_guide_board(req.topic, req.format) @@ -805,7 +727,6 @@ async def remove(guide_id: str, slots: bool = False): guide = await get_guide(guide_id) if guide is None: raise HTTPException(404, "Guide not found") - await delete_progress(guide_id) await delete_guide(guide_id) # Content/step files are shared by all runs of a topic+format — only delete them # once no entry needs them anymore. Partial progress (step files without finished @@ -820,20 +741,3 @@ async def remove(guide_id: str, slots: bool = False): p.unlink(missing_ok=True) content.unlink(missing_ok=True) return {"ok": True} - - -@router.get("/guides/{guide_id}/progress", response_model=ProgressResponse) -async def get_progress(guide_id: str): - guide = await get_guide(guide_id) - if guide is None: - raise HTTPException(404, "Guide not found") - return {"chapters": await list_progress(guide_id)} - - -@router.post("/guides/{guide_id}/progress", response_model=ProgressResponse) -async def update_progress(guide_id: str, req: ProgressUpdate): - guide = await get_guide(guide_id) - if guide is None: - raise HTTPException(404, "Guide not found") - await set_progress(guide_id, req.chapter, req.done) - return {"chapters": await list_progress(guide_id)} diff --git a/backend/rules.py b/backend/rules.py index c669039..f145838 100644 --- a/backend/rules.py +++ b/backend/rules.py @@ -11,7 +11,7 @@ query loops per guide. import json -from database import list_block_scores_all, subs_per_level_all, list_guides, list_progress_all +from database import list_block_scores_all, subs_per_level_all, list_guides from guide import guide_slot_files from learning import cap_final, LEVELS, _threshold from paths import blocks_path, guide_content_path @@ -33,8 +33,8 @@ _LEVEL_WORT = { } -async def load_learnstate() -> tuple[list[dict], dict[str, set[str]], dict[str, dict[str, set[str]]]]: - """Guides + chapter progress + blocks per level. +async def load_learnstate() -> tuple[list[dict], dict[str, dict[str, set[str]]]]: + """Guides + blocks per level. levels: {"beginner"/"advanced"/"expert"/"master": {topic → normalized title}}. The level per block is derived from score + cap (4×relevant subs). @@ -47,7 +47,7 @@ async def load_learnstate() -> tuple[list[dict], dict[str, set[str]], dict[str, for key, p in LEVELS: if cf and score >= _threshold(p, cf): levels[key].setdefault(topic, set()).add(_norm_title(block)) - return await list_guides(), await list_progress_all(), levels + return await list_guides(), levels def _content_json(topic: str, fmt: str) -> dict | None: @@ -84,39 +84,39 @@ def _latest_done(guides: list[dict], fmt: str) -> dict[str, dict]: return latest -def _guide_all(g: dict, progress: dict[str, set[str]], levelset: dict[str, set[str]]) -> bool: +def _guide_all(g: dict, levelset: dict[str, set[str]]) -> bool: """Are ALL blocks of the guide at the required level?""" sections = _section_title(g["topic"], g["format"]) return bool(sections) and sections <= levelset.get(g["topic"], set()) -def is_level(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levelset: dict[str, set[str]]) -> bool: +def is_level(topic: str, fmt: str, guides: list[dict], levelset: dict[str, set[str]]) -> bool: """Latest finished guide (topic+format): all blocks at the level of levelset?""" g = _latest_done(guides, fmt).get(topic) - return g is not None and _guide_all(g, progress, levelset) + return g is not None and _guide_all(g, levelset) -def ist_completed(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> bool: +def ist_completed(topic: str, fmt: str, guides: list[dict], levels: dict[str, dict[str, set[str]]]) -> bool: """All blocks of the latest finished guide at least beginner (≥20%)?""" - return is_level(topic, fmt, guides, progress, levels["beginner"]) + return is_level(topic, fmt, guides, levels["beginner"]) -def topic_completed(topic: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> bool: +def topic_completed(topic: str, guides: list[dict], levels: dict[str, dict[str, set[str]]]) -> bool: """Topic done: latest finished guide, all blocks at master (100%)?""" - return is_level(topic, "Guide", guides, progress, levels["master"]) + return is_level(topic, "Guide", guides, levels["master"]) -def formats_stats(guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> dict: +def formats_stats(guides: list[dict], levels: dict[str, dict[str, set[str]]]) -> dict: """Per format created/completed — per topic only the latest finished guide counts.""" formats = {} for fmt in FORMATE: latest = _latest_done(guides, fmt) - completed = sum(1 for g in latest.values() if _guide_all(g, progress, levels["beginner"])) + completed = sum(1 for g in latest.values() if _guide_all(g, levels["beginner"])) formats[fmt] = {"created": len(latest), "completed": completed} return formats -def guide_lock(topic: str, fmt: str, guides: list[dict], progress: dict[str, set[str]], levels: dict[str, dict[str, set[str]]]) -> str | None: +def guide_lock(topic: str, fmt: str, guides: list[dict], levels: dict[str, dict[str, set[str]]]) -> str | None: """Reason why a fresh start for topic+format is locked — None = allowed. Exactly the rules from POST /guides: blocks required, no duplicate start, @@ -132,9 +132,9 @@ def guide_lock(topic: str, fmt: str, guides: list[dict], progress: dict[str, set prereq = PRESTAGE.get(fmt) if prereq: level = FREISCHALT_LEVEL[fmt] # completed=10 · understood=20 · mastered=30 - if not is_level(topic, prereq, guides, progress, levels[level]): + if not is_level(topic, prereq, guides, levels[level]): return f"First take the {prereq} of this topic {_LEVEL_WORT[level]}" - stat = formats_stats(guides, progress, levels).get(fmt, {"created": 0, "completed": 0}) + stat = formats_stats(guides, levels).get(fmt, {"created": 0, "completed": 0}) open_count = stat["created"] - stat["completed"] if open_count >= MAX_OFFENE_GUIDES: return f"Complete {fmt}s first — at most {MAX_OFFENE_GUIDES} open allowed ({open_count} open)" diff --git a/backend/tests/test_board_inventory.py b/backend/tests/test_board_inventory.py index b6411e7..60fa9ae 100644 --- a/backend/tests/test_board_inventory.py +++ b/backend/tests/test_board_inventory.py @@ -764,3 +764,78 @@ def test_per_block_functions_accept_wrapper_kwargs(): assert "seeds" in inspect.signature(blx._subblocks_block).parameters for fn in ("_subblocks_block", "_facts_block"): # Board 2 reicht die Block-Quellen durch assert "sources" in inspect.signature(getattr(blx, fn)).parameters, fn + + +# ── Inventar-Härtung: Sanitizer, Akronym-Regel, Supplement-Beleg ───────────────────── + +def test_clean_title_strips_markdown_only(): + """`**` und Backticks fliegen; Math-Zeichen (|, _, einzelnes *) bleiben.""" + from textkit import clean_title + assert clean_title("**ListScheduling**") == "ListScheduling" + assert clean_title("**LPT** - Algo") == "LPT - Algo" + assert clean_title("`code` doppelt") == "code doppelt" + assert clean_title("2|prec, pi∈{1,2}|Cmax") == "2|prec, pi∈{1,2}|Cmax" + assert clean_title("x_i und P*") == "x_i und P*" + + +def test_title_variants_acronym_expansion(): + from blocks import _title_variants + assert _title_variants("Satisfiability Problem (SAT)") == {"satisfiability problem", "sat"} + assert _title_variants("DEA (Deterministischer Endlicher Automat)") == { + "dea", "deterministischer endlicher automat"} + assert _title_variants("SAT") == set() + assert _title_variants("2|prec, pi∈{1,2}|Cmax") == set() + + +async def test_dedup_acronym_pair_judged(board_env, tmp_path, monkeypatch, emb_on): + """Kurzform vs. Langform liegt unterm Embedding-Floor (real: Cos 0.53) — die + Akronym-Regel macht das Paar trotzdem zum Kandidaten, das Panel merged bei 2× ja.""" + monkeypatch.setattr(bi, "_vec_rows", _angle_vecs({})) # alles orthogonal + counter = {} + monkeypatch.setattr(bi, "run_single_slot", _slot_router([ + ("-dedup-", {"pairs": {"1": "ja"}}), + ], counter)) + await _run_dedup(board_env[0], board_env[1], tmp_path, [ + ("b-1", {"title": "SAT", "description": "kurz"}), + ("b-2", {"title": "Satisfiability Problem (SAT)", "description": "lang"}), + ]) + stages = sorted([(await board_env[0].kanban_get_card(TOPIC, B, c))["stage"] + for c in ("b-1", "b-2")]) + assert stages == ["grouped", "grouping"] + assert counter["-dedup-"] == 2 # Panel lief — kein Auto-Merge + + +async def test_supplement_beleg_gate(board_env, tmp_path, monkeypatch): + """Ohne Auszugs-Treffer fällt ein Vorschlag sofort; der Judge verwirft „nein"; + nur belegte Vorschläge überleben. Antwort ist Text (no-tool), Engine persistiert.""" + db, ctx, files = board_env + korpus = tmp_path / "korpus" + korpus.mkdir() + (korpus / "Skript.txt").write_text( + "Kapitel 1\nVertex Cover Definition und Übung dazu.\nMatching Grundlagen kurz.\n", + encoding="utf-8") + monkeypatch.setattr(bi, "source_folder", lambda t: korpus) + calls = {} + + async def fake_slot(ctx2, label, *, key, prompt, role, capabilities, payload, timeout, on_line=None): + calls["key"], calls["caps"], calls["prompt"] = key, capabilities, prompt + return bi.OK, payload((0, '{"relevant": {"1": "ja", "2": "nein"}}', "")) + + monkeypatch.setattr(bi, "run_single_slot", fake_slot) + kept = await bi._supplement_beleg(ctx, _mk_flow(tmp_path), [ + ("Vertex Cover", "Knotenüberdeckung"), + ("Matching", "Paarung"), + ("Quantencomputer", "nicht im Material"), + ]) + assert kept == [("Vertex Cover", "Knotenüberdeckung")] + assert calls["caps"] == "none" and "-supplement-beleg" in calls["key"] + assert "Quantencomputer" not in calls["prompt"] # fiel schon am Auszugs-Filter + assert (tmp_path / "supplement-beleg.json").exists() # Resume-Guard + + +async def test_ingest_strips_markdown_title(testdb, tmp_path): + flow = _mk_flow(tmp_path) + n = await bi._ingest_titles(flow, "1. **ListScheduling** — Greedy-Verfahren", "r1") + assert n == 1 + card = await testdb.kanban_get_card(TOPIC, B, "listscheduling") + assert card["payload"]["title"] == "ListScheduling" diff --git a/backend/tests/test_events.py b/backend/tests/test_events.py index 7ef9da8..8c8410a 100644 --- a/backend/tests/test_events.py +++ b/backend/tests/test_events.py @@ -117,7 +117,7 @@ async def test_learnstate_smoke(testdb): """Regression: P5-Ausbau hatte die _LEVEL_CASE-Konstante mitgerissen — load_learnstate (Guide-Start-Pfad) muss ohne NameError laufen.""" from rules import load_learnstate - guides, progress, levels = await load_learnstate() + guides, levels = await load_learnstate() assert isinstance(levels, dict) diff --git a/backend/textkit.py b/backend/textkit.py index af9bb09..5d45ea0 100644 --- a/backend/textkit.py +++ b/backend/textkit.py @@ -26,6 +26,15 @@ def _title(entry: str) -> str: return entry.split(" — ")[0].strip() or entry +def clean_title(s: str) -> str: + """Strip markdown noise from a DISPLAY title (norm keys use _norm_title). + Only clearly-markdown characters go: `**` pairs and backticks. Single `*`, + underscores and pipes stay — they are legitimate in math titles + („2|prec, pi∈{1,2}|Cmax", „x_i", „P*").""" + s = (s or "").replace("**", "").replace("`", "") + return re.sub(r"\s+", " ", s).strip() + + def _unique_title(entries: dict[int, str]) -> dict[int, str]: """Make titles unique (suffix " (2)", " (3)" …) so they work as keys.""" seen: dict[str, int] = {} diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 4673de5..3c1bfbc 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -1,6 +1,6 @@