From 2f5d5b9ca121cc0b07cf2d5c46712d854c1347b0 Mon Sep 17 00:00:00 2001 From: team3 Date: Sat, 4 Jul 2026 03:25:02 +0200 Subject: [PATCH] update --- backend/board_artefacts.py | 3 ++ backend/database.py | 15 ++++++ backend/guide.py | 9 ++-- backend/guide_board.py | 7 +++ backend/guide_qa.py | 51 ++++++++++++++----- backend/routes.py | 16 +++++- backend/tests/test_guide_board.py | 33 ++++++++++++ backend/tests/test_guide_qa.py | 41 +++++++++++++++ backend/tests/test_konsolidierung.py | 14 +++++ frontend/src/components/GenerationView.vue | 3 ++ frontend/src/components/GuideBoardSection.vue | 2 + 11 files changed, 176 insertions(+), 18 deletions(-) diff --git a/backend/board_artefacts.py b/backend/board_artefacts.py index 8eebc1a..5631c33 100644 --- a/backend/board_artefacts.py +++ b/backend/board_artefacts.py @@ -539,6 +539,9 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards): # the variant/discarded statuses, and the sidecar mirror re-writes only consensus. await db.delete_question_pattern(topic, _norm_title(title)) await db.delete_sub_artefakte(topic, _norm_title(title)) + # stragglers the mirror didn't level (row not in this run's sidecar): without a + # valid level they vanish from guide/practice/level views while QA still counts them + await db.default_subblock_levels(topic, _norm_title(title)) sub_keys: dict[str, set[str]] = {} async def _keys(bnorm: str) -> set[str]: diff --git a/backend/database.py b/backend/database.py index ac49a9e..857a11c 100644 --- a/backend/database.py +++ b/backend/database.py @@ -1171,6 +1171,21 @@ async def list_subblocks(topic: str, block_norm: str | None = None) -> list[dict return [_row_to_dict(row, cursor) for row in rows] +async def default_subblock_levels(topic: str, block_norm: str) -> None: + """Classify stragglers after finalize: consensus rows without a valid level fall out of + the guide/practice/level queries (re-run resume left 25 such rows — invisible content).""" + db = await get_db() + await db.execute( + """UPDATE subblocks SET level = 'advanced' WHERE topic = ? AND block_norm = ? + AND status = 'consensus' AND (level IS NULL OR level NOT IN ('beginner', 'advanced', 'expert'))""", + (topic, block_norm)) + await db.execute( + """UPDATE subblocks SET relevance = 'relevant' WHERE topic = ? AND block_norm = ? + AND status = 'consensus' AND relevance IS NULL""", + (topic, block_norm)) + await db.commit() + + async def set_subblock_fields(topic: str, block_norm: str, sub_norm: str, **fields) -> None: """Set fields (level/relevance/status/sub_title) of a subblock row.""" fields["updated_at"] = _now() diff --git a/backend/guide.py b/backend/guide.py index 9a3ac56..6304bc3 100644 --- a/backend/guide.py +++ b/backend/guide.py @@ -60,16 +60,19 @@ _LEVELS_OK = ("beginner", "advanced", "expert", "easy", "medium", "hard") async def _load_subblocks(topic: str) -> dict[str, list[dict]]: """Subblocks per block — DB-first ({title, level, relevance}), fallback sidecar file. - Both missing → {} (guide takes everything).""" + Both missing → {} (guide takes everything). A missing/invalid level defaults to + 'advanced' instead of dropping the row: re-run resume left 25 consensus subs + level-less, the writer silently lost them while the guide QA still counted them.""" out: dict[str, list[dict]] = {} for r in await list_subblocks(topic): - if r["status"] == "consensus" and r["sub_title"] and r["level"] in _LEVELS_OK: + if r["status"] == "consensus" and r["sub_title"]: try: facts = json.loads(r["facts"]) if r.get("facts") else {} except (ValueError, TypeError): facts = {} + level = r["level"] if r["level"] in _LEVELS_OK else "advanced" out.setdefault(r["block"], []).append( - {"title": r["sub_title"], "level": r["level"], "relevance": r["relevance"], "facts": facts}) + {"title": r["sub_title"], "level": level, "relevance": r["relevance"], "facts": facts}) if out: return out data = _json_file(subblocks_path(topic)) diff --git a/backend/guide_board.py b/backend/guide_board.py index fd7ebb2..be271d5 100644 --- a/backend/guide_board.py +++ b/backend/guide_board.py @@ -35,6 +35,8 @@ STAGE_LABELS = {"lernziele": "Lernziele", "zuweisung": "Zuweisung", "writer": "W "fakten_gate": "Fakten-Gate", "coverage": "Coverage", "lesbarkeit": "Lesbarkeit", "done": "Fertig"} MAX_WRITER_ROUNDS = 2 # coverage → writer feedback loop cap (gains die after round 1–2) +GATE_FIX_MIN = 3 # fact-gate claims below this: log only — a full fix rewrite for + # 1–2 residual claims fired on 19/20 blocks (40 agent-minutes) # Simultaneous cards = the per-topic agent cap: every card busies exactly ONE agent at a # time (its stages run serially), so a lower number just idles slots (was hardcoded 10 # from the old 10-slot era while the .env already allowed 24). @@ -395,6 +397,11 @@ async def _stage_fakten_gate(env: _Env, card: dict) -> bool: if status == FAILED: claims = [] # gate failure must not block the card — logged, text stands _log(env.topic, f"Fakten-Gate {card['block']}: kein Ergebnis — Text bleibt ungeprüft") + if claims and len(claims) < GATE_FIX_MIN: + # 1–2 Rest-Claims rechtfertigen keinen Voll-Rewrite: der Fix-Pass lief für 19/20 + # Blöcke und kostete 40 Agent-Minuten; die Guide-QA-Fachlichkeits-Stichprobe misst nach + _log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claim(s) unter Schwelle — kein Fix") + claims = [] if claims: _log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} unbelegte Claims → Fix") fixp = env.slot(f"gatefix-{_safe(norm)}-r{card['writer_rounds']}.md") diff --git a/backend/guide_qa.py b/backend/guide_qa.py index aadb1c4..caa85bf 100644 --- a/backend/guide_qa.py +++ b/backend/guide_qa.py @@ -32,6 +32,12 @@ NOTE_GEWICHTE_GUIDE = {"fachlich_falsch": 3.0, "ziel_ohne_anker": 2.0, "marker_f _MARKER = re.compile(r"") +def _mnorm(s: str) -> str: + """Marker-/Sub-Norm ohne Backslashes — escapte Titel (`h\\~2\\~o`) erzeugten + falsch-positive „Marker fehlt"-Befunde, weil Writer und DB verschieden escapen.""" + return _norm_title(s.replace("\\", "")) + + def _ausfuehrlich(md: str) -> str: """Der Lern-Fließtext einer Karte (hinter dem ausführlich-Marker, sonst alles).""" teile = re.split(r"", md or "", maxsplit=1) @@ -42,9 +48,10 @@ def marker_fehlend(cards: list[dict], subs_rel: dict[str, set]) -> list[str]: """Relevante Subs ohne Sub-Marker in der Section — der Level-Filter verliert sie.""" out = [] for c in cards: - marker = {_norm_title(m) for m in _MARKER.findall(c["md"] or "")} + marker = {_mnorm(m) for m in _MARKER.findall(c["md"] or "")} for sn in sorted(subs_rel.get(c["block_norm"], set())): - if sn not in marker and not any(m.startswith(sn) or sn.startswith(m) for m in marker): + mn = _mnorm(sn) + if mn not in marker and not any(m.startswith(mn) or mn.startswith(m) for m in marker): out.append(f"{c['block']} · {sn}") return out @@ -101,21 +108,37 @@ def lesbarkeit(cards: list[dict]) -> list[str]: async def _fachlich_falsch(topic: str, cards: list[dict]) -> list[str]: - """LLM-Stichprobe: Section enthält eine fachlich falsche Aussage? (eigenes Template).""" + """LLM-Stichprobe: Section enthält eine fachlich falsche Aussage? Zwei unabhängige + Durchgänge, nur DOPPELT bestätigte zählen — ein Einzel-Judge schwankte zwischen + 0 und 5 Befunden am selben Guide und kippte die Note (Gewicht 3.0) auf 0.""" from agents import run_agent from jsonio import parse_json_text from pipeline import _yesno_schema - out = [] - for lo in range(0, len(cards), 5): - chunk = cards[lo:lo + 5] - listing = "\n\n".join(f"{k}. SECTION {c['block']}:\n{_ausfuehrlich(c['md'])[:LLM_SECTION_CHARS]}" - for k, c in enumerate(chunk, 1)) - rc, txt, _err = await run_agent( - f"qa-guide-{topic}-fakten-{lo}", qa._qa_prompt("QA-Guide-Fakten", topic=topic, extra="", sections=listing), - 600, role="judge", capabilities="none", scope=topic, label=f"Guide-QA Fakten {lo}") - v = (_yesno_schema(parse_json_text(txt)) or {}) if rc == 0 else {} - out += [c["block"] for k, c in enumerate(chunk, 1) if v.get(k) == "ja"] - return out + + async def _pass(kandidaten: list[dict], tag: str) -> list[str]: + out = [] + for lo in range(0, len(kandidaten), 5): + chunk = kandidaten[lo:lo + 5] + listing = "\n\n".join(f"{k}. SECTION {c['block']}:\n{_ausfuehrlich(c['md'])[:LLM_SECTION_CHARS]}" + for k, c in enumerate(chunk, 1)) + rc, txt, _err = await run_agent( + f"qa-guide-{topic}-fakten{tag}-{lo}", qa._qa_prompt("QA-Guide-Fakten", topic=topic, extra="", sections=listing), + 600, role="judge", capabilities="none", scope=topic, label=f"Guide-QA Fakten{tag} {lo}") + v = (_yesno_schema(parse_json_text(txt)) or {}) if rc == 0 else {} + out += [c["block"] for k, c in enumerate(chunk, 1) if v.get(k) == "ja"] + return out + + verdacht = await _pass(cards, "") + if not verdacht: + return [] + # ZWEI unabhängige Bestätiger, beide müssen zustimmen — mit nur einem sprang die + # Note desselben Guides weiter zwischen 2.0 und 6.6 (ein Zufalls-ja kostet 1.5 Punkte) + kandidaten = [c for c in cards if c["block"] in set(verdacht)] + b1 = set(await _pass(kandidaten, "-2")) + if not b1: + return [] + b2 = set(await _pass([c for c in kandidaten if c["block"] in b1], "-3")) + return [b for b in verdacht if b in b1 and b in b2] async def guide_qa_report(topic: str, llm: bool = False) -> dict | None: diff --git a/backend/routes.py b/backend/routes.py index aaec934..9d8941c 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import shutil import uuid from datetime import datetime, timedelta, timezone @@ -173,7 +174,20 @@ async def run_qa_route(req: QaRunRequest): if report is None: raise HTTPException(status_code=404, detail="keine fertigen Bausteine") await asyncio.to_thread(qa._write_report, report) - return {"note": report["note"], "note_artefakte": report["note_artefakte"]} + note_guide = None + try: # fertiger Guide vorhanden → mitmessen (ein Klick, drei Noten); best-effort + import guide_qa + from database import list_guide_cards + if any(c["stage"] == "done" and (c.get("md") or "").strip() + for c in await list_guide_cards(req.topic)): + grep = await guide_qa.guide_qa_report(req.topic, llm=req.llm) + if grep: + await asyncio.to_thread(guide_qa._write_report, grep) + note_guide = grep["note_guide"] + except Exception: + logging.getLogger("creator.routes").exception("[%s] Guide-QA im QA-Button fehlgeschlagen", req.topic) + return {"note": report["note"], "note_artefakte": report["note_artefakte"], + "note_guide": note_guide} finally: _qa_laeuft.discard(req.topic) diff --git a/backend/tests/test_guide_board.py b/backend/tests/test_guide_board.py index 81c72f8..ce5a289 100644 --- a/backend/tests/test_guide_board.py +++ b/backend/tests/test_guide_board.py @@ -297,3 +297,36 @@ async def test_writer_prompt_traegt_budget(testdb, tmp_path, monkeypatch): monkeypatch.setattr(gb, "run_single_slot", fake_slot) await gb._stage_writer(env, card) assert str(gb._writer_budget(2)) in seen["prompt"] # 800 + 2×400 + + +async def test_fakten_gate_schwelle(testdb, tmp_path, monkeypatch): + """1–2 Claims → kein Fix-Rewrite (nur Log); ab GATE_FIX_MIN läuft der Fix.""" + db = testdb + await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha") + env = gb._Env(None, "g-g", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec") + md = "\n\nText." + card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md} + keys = [] + + async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout): + keys.append(key) + if "-gate-" in key: + return gb.OK, [{"text": "c1", "grund": ""}, {"text": "c2", "grund": ""}] + raise AssertionError("Fix darf unter der Schwelle nicht laufen") + + monkeypatch.setattr(gb, "run_single_slot", fake_slot) + assert await gb._stage_fakten_gate(env, card) + assert all("-gate-" in k for k in keys) + + +async def test_load_subblocks_defaultet_levellose(testdb): + """Consensus-Row ohne level fällt NICHT mehr raus — Default 'advanced'. + (Re-Run-Resume hinterließ 25 solcher Rows; der Writer verlor sie stumm.)""" + from guide import _load_subblocks + db = testdb + await db.put_subblock("t", "block", "mit level", "Block", "Mit Level", + level="beginner", status="consensus") + await db.put_subblock("t", "block", "ohne level", "Block", "Ohne Level", status="consensus") + subs = await _load_subblocks("t") + by_title = {s["title"]: s["level"] for s in subs["Block"]} + assert by_title == {"Mit Level": "beginner", "Ohne Level": "advanced"} diff --git a/backend/tests/test_guide_qa.py b/backend/tests/test_guide_qa.py index a72eeb9..8d89774 100644 --- a/backend/tests/test_guide_qa.py +++ b/backend/tests/test_guide_qa.py @@ -63,3 +63,44 @@ def test_note_guide_kalibrierung(): assert qa.note({"fachlich_falsch": 0.1}, gq.NOTE_GEWICHTE_GUIDE) == 7.0 ohne = {"marker_fehlend": 0.0, "ziel_ohne_anker": 0.0} assert qa.note(ohne, gq.NOTE_GEWICHTE_GUIDE) == 10.0 + + +def test_marker_escaping_tolerant(): + """Escapte Titel (h\\~2\\~o) sind kein „Marker fehlt" — Writer und DB escapen verschieden.""" + md = "\n\nText." + cards = [{"block": "Hoch", "block_norm": "hoch", "md": md}] + rel = {"hoch": {gq._norm_title("^ und ~ müssen escaped werden (h~2~o)")}} + assert gq.marker_fehlend(cards, rel) == [] + + +async def test_fachlich_falsch_braucht_beide_bestaetiger(monkeypatch): + """Befund zählt nur, wenn BEIDE Bestätiger zustimmen — Einzel-/Zweifach-Urteile + ließen die Note desselben Guides zwischen 2.0 und 6.6 springen.""" + import agents + calls = {"n": 0} + + async def fake_agent(key, prompt, timeout, **kw): + calls["n"] += 1 + if "-fakten-3-" in key: # Bestätiger 2: nur #1 bleibt + return 0, '{"relevant": {"1": "ja"}}', "" + if "-fakten-2-" in key: # Bestätiger 1: #1 und #2 + return 0, '{"relevant": {"1": "ja", "2": "ja"}}', "" + return 0, '{"relevant": {"1": "ja", "2": "ja", "3": "nein"}}', "" # Pass 1 + + monkeypatch.setattr(agents, "run_agent", fake_agent) + cards = [_card("Alpha", AUSF), _card("Beta", AUSF), _card("Gamma", AUSF)] + out = await gq._fachlich_falsch("t", cards) + assert out == ["Alpha"] and calls["n"] == 3 + + +async def test_fachlich_falsch_ohne_verdacht_kein_zweiter_pass(monkeypatch): + import agents + calls = {"n": 0} + + async def fake_agent(key, prompt, timeout, **kw): + calls["n"] += 1 + return 0, '{"relevant": {"1": "nein", "2": "nein"}}', "" + + monkeypatch.setattr(agents, "run_agent", fake_agent) + out = await gq._fachlich_falsch("t", [_card("Alpha", AUSF), _card("Beta", AUSF)]) + assert out == [] and calls["n"] == 1 diff --git a/backend/tests/test_konsolidierung.py b/backend/tests/test_konsolidierung.py index 5a67ad5..0a6b1a9 100644 --- a/backend/tests/test_konsolidierung.py +++ b/backend/tests/test_konsolidierung.py @@ -505,3 +505,17 @@ async def test_crossblock_context_wins(testdb, tmp_path, monkeypatch): assert alpha["payload"]["raw"].get("Alpha", []) == [] # Paket-Seite gefaltet gamma_rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "gamma")} assert gamma_rows[blocks._norm_title("Gleiche Aussage")] == "consensus" # Kontext unberührt + + +async def test_finalize_defaultet_level_nachzuegler(testdb, tmp_path): + """Finalize klassifiziert level-/relevance-lose consensus-Rows (Default advanced/relevant).""" + db = testdb + await db.put_subblock(TOPIC, "alpha", "nachzuegler", "Alpha", "Nachzügler", status="consensus") + await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "finalize", {}) + files = {k: tmp_path / f"{k}.json" for k in + ("sub_roh", "facts", "sidecar", "question_pattern", "artefakte")} + card = {"card_id": "alpha", "payload": {"title": "Alpha", "raw": {}, "facts": {}, + "sidecar": {}, "pattern": {}, "artefacts": {}}} + await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card]) + row = next(r for r in await db.list_subblocks(TOPIC, "alpha")) + assert row["level"] == "advanced" and row["relevance"] == "relevant" diff --git a/frontend/src/components/GenerationView.vue b/frontend/src/components/GenerationView.vue index b10d13e..c4741a7 100644 --- a/frontend/src/components/GenerationView.vue +++ b/frontend/src/components/GenerationView.vue @@ -84,6 +84,7 @@ function resetHere(restart) { } const qaBusy = ref(false) +const guideRefresh = ref(0) // QA/Repair schreiben auch den Guide-Report → Badge neu laden async function runQaClick() { if (qaBusy.value) return qaBusy.value = true @@ -92,6 +93,7 @@ async function runQaClick() { } finally { qaBusy.value = false pollBoard() + guideRefresh.value++ } } @@ -209,6 +211,7 @@ async function repairClick() { props.topic, () => { board.value = null; poll() }, { immediate: true }) +watch(() => props.refresh, () => poll()) watch(() => board.value?.generating, (g) => { if (g) startPoll(); else stopPoll() }) onUnmounted(stopPoll)