This commit is contained in:
team3
2026-07-04 03:25:02 +02:00
parent c4caf31ed0
commit 2f5d5b9ca1
11 changed files with 176 additions and 18 deletions

View File

@@ -539,6 +539,9 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
# the variant/discarded statuses, and the sidecar mirror re-writes only consensus.
await db.delete_question_pattern(topic, _norm_title(title))
await db.delete_sub_artefakte(topic, _norm_title(title))
# stragglers the mirror didn't level (row not in this run's sidecar): without a
# valid level they vanish from guide/practice/level views while QA still counts them
await db.default_subblock_levels(topic, _norm_title(title))
sub_keys: dict[str, set[str]] = {}
async def _keys(bnorm: str) -> set[str]:

View File

@@ -1171,6 +1171,21 @@ async def list_subblocks(topic: str, block_norm: str | None = None) -> list[dict
return [_row_to_dict(row, cursor) for row in rows]
async def default_subblock_levels(topic: str, block_norm: str) -> None:
"""Classify stragglers after finalize: consensus rows without a valid level fall out of
the guide/practice/level queries (re-run resume left 25 such rows — invisible content)."""
db = await get_db()
await db.execute(
"""UPDATE subblocks SET level = 'advanced' WHERE topic = ? AND block_norm = ?
AND status = 'consensus' AND (level IS NULL OR level NOT IN ('beginner', 'advanced', 'expert'))""",
(topic, block_norm))
await db.execute(
"""UPDATE subblocks SET relevance = 'relevant' WHERE topic = ? AND block_norm = ?
AND status = 'consensus' AND relevance IS NULL""",
(topic, block_norm))
await db.commit()
async def set_subblock_fields(topic: str, block_norm: str, sub_norm: str, **fields) -> None:
"""Set fields (level/relevance/status/sub_title) of a subblock row."""
fields["updated_at"] = _now()

View File

@@ -60,16 +60,19 @@ _LEVELS_OK = ("beginner", "advanced", "expert", "easy", "medium", "hard")
async def _load_subblocks(topic: str) -> dict[str, list[dict]]:
"""Subblocks per block — DB-first ({title, level, relevance}), fallback sidecar file.
Both missing → {} (guide takes everything)."""
Both missing → {} (guide takes everything). A missing/invalid level defaults to
'advanced' instead of dropping the row: re-run resume left 25 consensus subs
level-less, the writer silently lost them while the guide QA still counted them."""
out: dict[str, list[dict]] = {}
for r in await list_subblocks(topic):
if r["status"] == "consensus" and r["sub_title"] and r["level"] in _LEVELS_OK:
if r["status"] == "consensus" and r["sub_title"]:
try:
facts = json.loads(r["facts"]) if r.get("facts") else {}
except (ValueError, TypeError):
facts = {}
level = r["level"] if r["level"] in _LEVELS_OK else "advanced"
out.setdefault(r["block"], []).append(
{"title": r["sub_title"], "level": r["level"], "relevance": r["relevance"], "facts": facts})
{"title": r["sub_title"], "level": level, "relevance": r["relevance"], "facts": facts})
if out:
return out
data = _json_file(subblocks_path(topic))

View File

@@ -35,6 +35,8 @@ STAGE_LABELS = {"lernziele": "Lernziele", "zuweisung": "Zuweisung", "writer": "W
"fakten_gate": "Fakten-Gate", "coverage": "Coverage",
"lesbarkeit": "Lesbarkeit", "done": "Fertig"}
MAX_WRITER_ROUNDS = 2 # coverage → writer feedback loop cap (gains die after round 12)
GATE_FIX_MIN = 3 # fact-gate claims below this: log only — a full fix rewrite for
# 12 residual claims fired on 19/20 blocks (40 agent-minutes)
# Simultaneous cards = the per-topic agent cap: every card busies exactly ONE agent at a
# time (its stages run serially), so a lower number just idles slots (was hardcoded 10
# from the old 10-slot era while the .env already allowed 24).
@@ -395,6 +397,11 @@ async def _stage_fakten_gate(env: _Env, card: dict) -> bool:
if status == FAILED:
claims = [] # gate failure must not block the card — logged, text stands
_log(env.topic, f"Fakten-Gate {card['block']}: kein Ergebnis — Text bleibt ungeprüft")
if claims and len(claims) < GATE_FIX_MIN:
# 12 Rest-Claims rechtfertigen keinen Voll-Rewrite: der Fix-Pass lief für 19/20
# Blöcke und kostete 40 Agent-Minuten; die Guide-QA-Fachlichkeits-Stichprobe misst nach
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claim(s) unter Schwelle — kein Fix")
claims = []
if claims:
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} unbelegte Claims → Fix")
fixp = env.slot(f"gatefix-{_safe(norm)}-r{card['writer_rounds']}.md")

View File

@@ -32,6 +32,12 @@ NOTE_GEWICHTE_GUIDE = {"fachlich_falsch": 3.0, "ziel_ohne_anker": 2.0, "marker_f
_MARKER = re.compile(r"<!--\s*sub:\s*\w+\s*\|\s*(.*?)\s*-->")
def _mnorm(s: str) -> str:
"""Marker-/Sub-Norm ohne Backslashes — escapte Titel (`h\\~2\\~o`) erzeugten
falsch-positive „Marker fehlt"-Befunde, weil Writer und DB verschieden escapen."""
return _norm_title(s.replace("\\", ""))
def _ausfuehrlich(md: str) -> str:
"""Der Lern-Fließtext einer Karte (hinter dem ausführlich-Marker, sonst alles)."""
teile = re.split(r"<!--\s*ausführlich\s*-->", md or "", maxsplit=1)
@@ -42,9 +48,10 @@ def marker_fehlend(cards: list[dict], subs_rel: dict[str, set]) -> list[str]:
"""Relevante Subs ohne Sub-Marker in der Section — der Level-Filter verliert sie."""
out = []
for c in cards:
marker = {_norm_title(m) for m in _MARKER.findall(c["md"] or "")}
marker = {_mnorm(m) for m in _MARKER.findall(c["md"] or "")}
for sn in sorted(subs_rel.get(c["block_norm"], set())):
if sn not in marker and not any(m.startswith(sn) or sn.startswith(m) for m in marker):
mn = _mnorm(sn)
if mn not in marker and not any(m.startswith(mn) or mn.startswith(m) for m in marker):
out.append(f"{c['block']} · {sn}")
return out
@@ -101,22 +108,38 @@ def lesbarkeit(cards: list[dict]) -> list[str]:
async def _fachlich_falsch(topic: str, cards: list[dict]) -> list[str]:
"""LLM-Stichprobe: Section enthält eine fachlich falsche Aussage? (eigenes Template)."""
"""LLM-Stichprobe: Section enthält eine fachlich falsche Aussage? Zwei unabhängige
Durchgänge, nur DOPPELT bestätigte zählen — ein Einzel-Judge schwankte zwischen
0 und 5 Befunden am selben Guide und kippte die Note (Gewicht 3.0) auf 0."""
from agents import run_agent
from jsonio import parse_json_text
from pipeline import _yesno_schema
async def _pass(kandidaten: list[dict], tag: str) -> list[str]:
out = []
for lo in range(0, len(cards), 5):
chunk = cards[lo:lo + 5]
for lo in range(0, len(kandidaten), 5):
chunk = kandidaten[lo:lo + 5]
listing = "\n\n".join(f"{k}. SECTION {c['block']}:\n{_ausfuehrlich(c['md'])[:LLM_SECTION_CHARS]}"
for k, c in enumerate(chunk, 1))
rc, txt, _err = await run_agent(
f"qa-guide-{topic}-fakten-{lo}", qa._qa_prompt("QA-Guide-Fakten", topic=topic, extra="", sections=listing),
600, role="judge", capabilities="none", scope=topic, label=f"Guide-QA Fakten {lo}")
f"qa-guide-{topic}-fakten{tag}-{lo}", qa._qa_prompt("QA-Guide-Fakten", topic=topic, extra="", sections=listing),
600, role="judge", capabilities="none", scope=topic, label=f"Guide-QA Fakten{tag} {lo}")
v = (_yesno_schema(parse_json_text(txt)) or {}) if rc == 0 else {}
out += [c["block"] for k, c in enumerate(chunk, 1) if v.get(k) == "ja"]
return out
verdacht = await _pass(cards, "")
if not verdacht:
return []
# ZWEI unabhängige Bestätiger, beide müssen zustimmen — mit nur einem sprang die
# Note desselben Guides weiter zwischen 2.0 und 6.6 (ein Zufalls-ja kostet 1.5 Punkte)
kandidaten = [c for c in cards if c["block"] in set(verdacht)]
b1 = set(await _pass(kandidaten, "-2"))
if not b1:
return []
b2 = set(await _pass([c for c in kandidaten if c["block"] in b1], "-3"))
return [b for b in verdacht if b in b1 and b in b2]
async def guide_qa_report(topic: str, llm: bool = False) -> dict | None:
cards = [dict(r) for r in await db.list_guide_cards(topic)]

View File

@@ -1,5 +1,6 @@
import asyncio
import json
import logging
import shutil
import uuid
from datetime import datetime, timedelta, timezone
@@ -173,7 +174,20 @@ async def run_qa_route(req: QaRunRequest):
if report is None:
raise HTTPException(status_code=404, detail="keine fertigen Bausteine")
await asyncio.to_thread(qa._write_report, report)
return {"note": report["note"], "note_artefakte": report["note_artefakte"]}
note_guide = None
try: # fertiger Guide vorhanden → mitmessen (ein Klick, drei Noten); best-effort
import guide_qa
from database import list_guide_cards
if any(c["stage"] == "done" and (c.get("md") or "").strip()
for c in await list_guide_cards(req.topic)):
grep = await guide_qa.guide_qa_report(req.topic, llm=req.llm)
if grep:
await asyncio.to_thread(guide_qa._write_report, grep)
note_guide = grep["note_guide"]
except Exception:
logging.getLogger("creator.routes").exception("[%s] Guide-QA im QA-Button fehlgeschlagen", req.topic)
return {"note": report["note"], "note_artefakte": report["note_artefakte"],
"note_guide": note_guide}
finally:
_qa_laeuft.discard(req.topic)

View File

@@ -297,3 +297,36 @@ async def test_writer_prompt_traegt_budget(testdb, tmp_path, monkeypatch):
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
await gb._stage_writer(env, card)
assert str(gb._writer_budget(2)) in seen["prompt"] # 800 + 2×400
async def test_fakten_gate_schwelle(testdb, tmp_path, monkeypatch):
"""12 Claims → kein Fix-Rewrite (nur Log); ab GATE_FIX_MIN läuft der Fix."""
db = testdb
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
env = gb._Env(None, "g-g", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
md = "<!-- section: Alpha -->\n<!-- ausführlich -->\nText."
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
keys = []
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
keys.append(key)
if "-gate-" in key:
return gb.OK, [{"text": "c1", "grund": ""}, {"text": "c2", "grund": ""}]
raise AssertionError("Fix darf unter der Schwelle nicht laufen")
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
assert await gb._stage_fakten_gate(env, card)
assert all("-gate-" in k for k in keys)
async def test_load_subblocks_defaultet_levellose(testdb):
"""Consensus-Row ohne level fällt NICHT mehr raus — Default 'advanced'.
(Re-Run-Resume hinterließ 25 solcher Rows; der Writer verlor sie stumm.)"""
from guide import _load_subblocks
db = testdb
await db.put_subblock("t", "block", "mit level", "Block", "Mit Level",
level="beginner", status="consensus")
await db.put_subblock("t", "block", "ohne level", "Block", "Ohne Level", status="consensus")
subs = await _load_subblocks("t")
by_title = {s["title"]: s["level"] for s in subs["Block"]}
assert by_title == {"Mit Level": "beginner", "Ohne Level": "advanced"}

View File

@@ -63,3 +63,44 @@ def test_note_guide_kalibrierung():
assert qa.note({"fachlich_falsch": 0.1}, gq.NOTE_GEWICHTE_GUIDE) == 7.0
ohne = {"marker_fehlend": 0.0, "ziel_ohne_anker": 0.0}
assert qa.note(ohne, gq.NOTE_GEWICHTE_GUIDE) == 10.0
def test_marker_escaping_tolerant():
"""Escapte Titel (h\\~2\\~o) sind kein „Marker fehlt" — Writer und DB escapen verschieden."""
md = "<!-- ausführlich -->\n<!-- sub: beginner | ^ und ~ müssen escaped werden (h\\\\~2\\\\~o) -->\nText."
cards = [{"block": "Hoch", "block_norm": "hoch", "md": md}]
rel = {"hoch": {gq._norm_title("^ und ~ müssen escaped werden (h~2~o)")}}
assert gq.marker_fehlend(cards, rel) == []
async def test_fachlich_falsch_braucht_beide_bestaetiger(monkeypatch):
"""Befund zählt nur, wenn BEIDE Bestätiger zustimmen — Einzel-/Zweifach-Urteile
ließen die Note desselben Guides zwischen 2.0 und 6.6 springen."""
import agents
calls = {"n": 0}
async def fake_agent(key, prompt, timeout, **kw):
calls["n"] += 1
if "-fakten-3-" in key: # Bestätiger 2: nur #1 bleibt
return 0, '{"relevant": {"1": "ja"}}', ""
if "-fakten-2-" in key: # Bestätiger 1: #1 und #2
return 0, '{"relevant": {"1": "ja", "2": "ja"}}', ""
return 0, '{"relevant": {"1": "ja", "2": "ja", "3": "nein"}}', "" # Pass 1
monkeypatch.setattr(agents, "run_agent", fake_agent)
cards = [_card("Alpha", AUSF), _card("Beta", AUSF), _card("Gamma", AUSF)]
out = await gq._fachlich_falsch("t", cards)
assert out == ["Alpha"] and calls["n"] == 3
async def test_fachlich_falsch_ohne_verdacht_kein_zweiter_pass(monkeypatch):
import agents
calls = {"n": 0}
async def fake_agent(key, prompt, timeout, **kw):
calls["n"] += 1
return 0, '{"relevant": {"1": "nein", "2": "nein"}}', ""
monkeypatch.setattr(agents, "run_agent", fake_agent)
out = await gq._fachlich_falsch("t", [_card("Alpha", AUSF), _card("Beta", AUSF)])
assert out == [] and calls["n"] == 1

View File

@@ -505,3 +505,17 @@ async def test_crossblock_context_wins(testdb, tmp_path, monkeypatch):
assert alpha["payload"]["raw"].get("Alpha", []) == [] # Paket-Seite gefaltet
gamma_rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "gamma")}
assert gamma_rows[blocks._norm_title("Gleiche Aussage")] == "consensus" # Kontext unberührt
async def test_finalize_defaultet_level_nachzuegler(testdb, tmp_path):
"""Finalize klassifiziert level-/relevance-lose consensus-Rows (Default advanced/relevant)."""
db = testdb
await db.put_subblock(TOPIC, "alpha", "nachzuegler", "Alpha", "Nachzügler", status="consensus")
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "finalize", {})
files = {k: tmp_path / f"{k}.json" for k in
("sub_roh", "facts", "sidecar", "question_pattern", "artefakte")}
card = {"card_id": "alpha", "payload": {"title": "Alpha", "raw": {}, "facts": {},
"sidecar": {}, "pattern": {}, "artefacts": {}}}
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
row = next(r for r in await db.list_subblocks(TOPIC, "alpha"))
assert row["level"] == "advanced" and row["relevance"] == "relevant"

View File

@@ -84,6 +84,7 @@ function resetHere(restart) {
}
const qaBusy = ref(false)
const guideRefresh = ref(0) // QA/Repair schreiben auch den Guide-Report → Badge neu laden
async function runQaClick() {
if (qaBusy.value) return
qaBusy.value = true
@@ -92,6 +93,7 @@ async function runQaClick() {
} finally {
qaBusy.value = false
pollBoard()
guideRefresh.value++
}
}
@@ -209,6 +211,7 @@ async function repairClick() {
<GuideBoardSection
:topic="topic"
:format="guideFormat"
:refresh="guideRefresh"
@cancelGuide="(id) => emit('cancelGuide', id)"
@startGuide="(p) => emit('startGuide', p)"
@resetStage="(p) => emit('resetGuideStage', p)"

View File

@@ -6,6 +6,7 @@ import KanbanBoard from './KanbanBoard.vue'
const props = defineProps({
topic: { type: String, required: true },
format: { type: String, default: 'Guide' },
refresh: { type: Number, default: 0 }, // Eltern-Signal: QA/Repair schrieben einen Guide-Report
})
const emit = defineEmits(['cancelGuide', 'startGuide', 'resetStage', 'preview', 'removeFormat', 'resetCard'])
@@ -21,6 +22,7 @@ function startPoll() { stopPoll(); timer = setInterval(poll, 1200) }
function stopPoll() { if (timer) { clearInterval(timer); timer = null } }
watch(() => props.topic, () => { board.value = null; poll() }, { immediate: true })
watch(() => props.refresh, () => poll())
watch(() => board.value?.generating, (g) => { if (g) startPoll(); else stopPoll() })
onUnmounted(stopPoll)