update
This commit is contained in:
@@ -539,6 +539,9 @@ async def _proc_finalize(ctx: GenContext, flow: Flow, files: dict, cards):
|
||||
# the variant/discarded statuses, and the sidecar mirror re-writes only consensus.
|
||||
await db.delete_question_pattern(topic, _norm_title(title))
|
||||
await db.delete_sub_artefakte(topic, _norm_title(title))
|
||||
# stragglers the mirror didn't level (row not in this run's sidecar): without a
|
||||
# valid level they vanish from guide/practice/level views while QA still counts them
|
||||
await db.default_subblock_levels(topic, _norm_title(title))
|
||||
sub_keys: dict[str, set[str]] = {}
|
||||
|
||||
async def _keys(bnorm: str) -> set[str]:
|
||||
|
||||
@@ -1171,6 +1171,21 @@ async def list_subblocks(topic: str, block_norm: str | None = None) -> list[dict
|
||||
return [_row_to_dict(row, cursor) for row in rows]
|
||||
|
||||
|
||||
async def default_subblock_levels(topic: str, block_norm: str) -> None:
|
||||
"""Classify stragglers after finalize: consensus rows without a valid level fall out of
|
||||
the guide/practice/level queries (re-run resume left 25 such rows — invisible content)."""
|
||||
db = await get_db()
|
||||
await db.execute(
|
||||
"""UPDATE subblocks SET level = 'advanced' WHERE topic = ? AND block_norm = ?
|
||||
AND status = 'consensus' AND (level IS NULL OR level NOT IN ('beginner', 'advanced', 'expert'))""",
|
||||
(topic, block_norm))
|
||||
await db.execute(
|
||||
"""UPDATE subblocks SET relevance = 'relevant' WHERE topic = ? AND block_norm = ?
|
||||
AND status = 'consensus' AND relevance IS NULL""",
|
||||
(topic, block_norm))
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def set_subblock_fields(topic: str, block_norm: str, sub_norm: str, **fields) -> None:
|
||||
"""Set fields (level/relevance/status/sub_title) of a subblock row."""
|
||||
fields["updated_at"] = _now()
|
||||
|
||||
@@ -60,16 +60,19 @@ _LEVELS_OK = ("beginner", "advanced", "expert", "easy", "medium", "hard")
|
||||
|
||||
async def _load_subblocks(topic: str) -> dict[str, list[dict]]:
|
||||
"""Subblocks per block — DB-first ({title, level, relevance}), fallback sidecar file.
|
||||
Both missing → {} (guide takes everything)."""
|
||||
Both missing → {} (guide takes everything). A missing/invalid level defaults to
|
||||
'advanced' instead of dropping the row: re-run resume left 25 consensus subs
|
||||
level-less, the writer silently lost them while the guide QA still counted them."""
|
||||
out: dict[str, list[dict]] = {}
|
||||
for r in await list_subblocks(topic):
|
||||
if r["status"] == "consensus" and r["sub_title"] and r["level"] in _LEVELS_OK:
|
||||
if r["status"] == "consensus" and r["sub_title"]:
|
||||
try:
|
||||
facts = json.loads(r["facts"]) if r.get("facts") else {}
|
||||
except (ValueError, TypeError):
|
||||
facts = {}
|
||||
level = r["level"] if r["level"] in _LEVELS_OK else "advanced"
|
||||
out.setdefault(r["block"], []).append(
|
||||
{"title": r["sub_title"], "level": r["level"], "relevance": r["relevance"], "facts": facts})
|
||||
{"title": r["sub_title"], "level": level, "relevance": r["relevance"], "facts": facts})
|
||||
if out:
|
||||
return out
|
||||
data = _json_file(subblocks_path(topic))
|
||||
|
||||
@@ -35,6 +35,8 @@ STAGE_LABELS = {"lernziele": "Lernziele", "zuweisung": "Zuweisung", "writer": "W
|
||||
"fakten_gate": "Fakten-Gate", "coverage": "Coverage",
|
||||
"lesbarkeit": "Lesbarkeit", "done": "Fertig"}
|
||||
MAX_WRITER_ROUNDS = 2 # coverage → writer feedback loop cap (gains die after round 1–2)
|
||||
GATE_FIX_MIN = 3 # fact-gate claims below this: log only — a full fix rewrite for
|
||||
# 1–2 residual claims fired on 19/20 blocks (40 agent-minutes)
|
||||
# Simultaneous cards = the per-topic agent cap: every card busies exactly ONE agent at a
|
||||
# time (its stages run serially), so a lower number just idles slots (was hardcoded 10
|
||||
# from the old 10-slot era while the .env already allowed 24).
|
||||
@@ -395,6 +397,11 @@ async def _stage_fakten_gate(env: _Env, card: dict) -> bool:
|
||||
if status == FAILED:
|
||||
claims = [] # gate failure must not block the card — logged, text stands
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: kein Ergebnis — Text bleibt ungeprüft")
|
||||
if claims and len(claims) < GATE_FIX_MIN:
|
||||
# 1–2 Rest-Claims rechtfertigen keinen Voll-Rewrite: der Fix-Pass lief für 19/20
|
||||
# Blöcke und kostete 40 Agent-Minuten; die Guide-QA-Fachlichkeits-Stichprobe misst nach
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} Claim(s) unter Schwelle — kein Fix")
|
||||
claims = []
|
||||
if claims:
|
||||
_log(env.topic, f"Fakten-Gate {card['block']}: {len(claims)} unbelegte Claims → Fix")
|
||||
fixp = env.slot(f"gatefix-{_safe(norm)}-r{card['writer_rounds']}.md")
|
||||
|
||||
@@ -32,6 +32,12 @@ NOTE_GEWICHTE_GUIDE = {"fachlich_falsch": 3.0, "ziel_ohne_anker": 2.0, "marker_f
|
||||
_MARKER = re.compile(r"<!--\s*sub:\s*\w+\s*\|\s*(.*?)\s*-->")
|
||||
|
||||
|
||||
def _mnorm(s: str) -> str:
|
||||
"""Marker-/Sub-Norm ohne Backslashes — escapte Titel (`h\\~2\\~o`) erzeugten
|
||||
falsch-positive „Marker fehlt"-Befunde, weil Writer und DB verschieden escapen."""
|
||||
return _norm_title(s.replace("\\", ""))
|
||||
|
||||
|
||||
def _ausfuehrlich(md: str) -> str:
|
||||
"""Der Lern-Fließtext einer Karte (hinter dem ausführlich-Marker, sonst alles)."""
|
||||
teile = re.split(r"<!--\s*ausführlich\s*-->", md or "", maxsplit=1)
|
||||
@@ -42,9 +48,10 @@ def marker_fehlend(cards: list[dict], subs_rel: dict[str, set]) -> list[str]:
|
||||
"""Relevante Subs ohne Sub-Marker in der Section — der Level-Filter verliert sie."""
|
||||
out = []
|
||||
for c in cards:
|
||||
marker = {_norm_title(m) for m in _MARKER.findall(c["md"] or "")}
|
||||
marker = {_mnorm(m) for m in _MARKER.findall(c["md"] or "")}
|
||||
for sn in sorted(subs_rel.get(c["block_norm"], set())):
|
||||
if sn not in marker and not any(m.startswith(sn) or sn.startswith(m) for m in marker):
|
||||
mn = _mnorm(sn)
|
||||
if mn not in marker and not any(m.startswith(mn) or mn.startswith(m) for m in marker):
|
||||
out.append(f"{c['block']} · {sn}")
|
||||
return out
|
||||
|
||||
@@ -101,21 +108,37 @@ def lesbarkeit(cards: list[dict]) -> list[str]:
|
||||
|
||||
|
||||
async def _fachlich_falsch(topic: str, cards: list[dict]) -> list[str]:
|
||||
"""LLM-Stichprobe: Section enthält eine fachlich falsche Aussage? (eigenes Template)."""
|
||||
"""LLM-Stichprobe: Section enthält eine fachlich falsche Aussage? Zwei unabhängige
|
||||
Durchgänge, nur DOPPELT bestätigte zählen — ein Einzel-Judge schwankte zwischen
|
||||
0 und 5 Befunden am selben Guide und kippte die Note (Gewicht 3.0) auf 0."""
|
||||
from agents import run_agent
|
||||
from jsonio import parse_json_text
|
||||
from pipeline import _yesno_schema
|
||||
out = []
|
||||
for lo in range(0, len(cards), 5):
|
||||
chunk = cards[lo:lo + 5]
|
||||
listing = "\n\n".join(f"{k}. SECTION {c['block']}:\n{_ausfuehrlich(c['md'])[:LLM_SECTION_CHARS]}"
|
||||
for k, c in enumerate(chunk, 1))
|
||||
rc, txt, _err = await run_agent(
|
||||
f"qa-guide-{topic}-fakten-{lo}", qa._qa_prompt("QA-Guide-Fakten", topic=topic, extra="", sections=listing),
|
||||
600, role="judge", capabilities="none", scope=topic, label=f"Guide-QA Fakten {lo}")
|
||||
v = (_yesno_schema(parse_json_text(txt)) or {}) if rc == 0 else {}
|
||||
out += [c["block"] for k, c in enumerate(chunk, 1) if v.get(k) == "ja"]
|
||||
return out
|
||||
|
||||
async def _pass(kandidaten: list[dict], tag: str) -> list[str]:
|
||||
out = []
|
||||
for lo in range(0, len(kandidaten), 5):
|
||||
chunk = kandidaten[lo:lo + 5]
|
||||
listing = "\n\n".join(f"{k}. SECTION {c['block']}:\n{_ausfuehrlich(c['md'])[:LLM_SECTION_CHARS]}"
|
||||
for k, c in enumerate(chunk, 1))
|
||||
rc, txt, _err = await run_agent(
|
||||
f"qa-guide-{topic}-fakten{tag}-{lo}", qa._qa_prompt("QA-Guide-Fakten", topic=topic, extra="", sections=listing),
|
||||
600, role="judge", capabilities="none", scope=topic, label=f"Guide-QA Fakten{tag} {lo}")
|
||||
v = (_yesno_schema(parse_json_text(txt)) or {}) if rc == 0 else {}
|
||||
out += [c["block"] for k, c in enumerate(chunk, 1) if v.get(k) == "ja"]
|
||||
return out
|
||||
|
||||
verdacht = await _pass(cards, "")
|
||||
if not verdacht:
|
||||
return []
|
||||
# ZWEI unabhängige Bestätiger, beide müssen zustimmen — mit nur einem sprang die
|
||||
# Note desselben Guides weiter zwischen 2.0 und 6.6 (ein Zufalls-ja kostet 1.5 Punkte)
|
||||
kandidaten = [c for c in cards if c["block"] in set(verdacht)]
|
||||
b1 = set(await _pass(kandidaten, "-2"))
|
||||
if not b1:
|
||||
return []
|
||||
b2 = set(await _pass([c for c in kandidaten if c["block"] in b1], "-3"))
|
||||
return [b for b in verdacht if b in b1 and b in b2]
|
||||
|
||||
|
||||
async def guide_qa_report(topic: str, llm: bool = False) -> dict | None:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -173,7 +174,20 @@ async def run_qa_route(req: QaRunRequest):
|
||||
if report is None:
|
||||
raise HTTPException(status_code=404, detail="keine fertigen Bausteine")
|
||||
await asyncio.to_thread(qa._write_report, report)
|
||||
return {"note": report["note"], "note_artefakte": report["note_artefakte"]}
|
||||
note_guide = None
|
||||
try: # fertiger Guide vorhanden → mitmessen (ein Klick, drei Noten); best-effort
|
||||
import guide_qa
|
||||
from database import list_guide_cards
|
||||
if any(c["stage"] == "done" and (c.get("md") or "").strip()
|
||||
for c in await list_guide_cards(req.topic)):
|
||||
grep = await guide_qa.guide_qa_report(req.topic, llm=req.llm)
|
||||
if grep:
|
||||
await asyncio.to_thread(guide_qa._write_report, grep)
|
||||
note_guide = grep["note_guide"]
|
||||
except Exception:
|
||||
logging.getLogger("creator.routes").exception("[%s] Guide-QA im QA-Button fehlgeschlagen", req.topic)
|
||||
return {"note": report["note"], "note_artefakte": report["note_artefakte"],
|
||||
"note_guide": note_guide}
|
||||
finally:
|
||||
_qa_laeuft.discard(req.topic)
|
||||
|
||||
|
||||
@@ -297,3 +297,36 @@ async def test_writer_prompt_traegt_budget(testdb, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
await gb._stage_writer(env, card)
|
||||
assert str(gb._writer_budget(2)) in seen["prompt"] # 800 + 2×400
|
||||
|
||||
|
||||
async def test_fakten_gate_schwelle(testdb, tmp_path, monkeypatch):
|
||||
"""1–2 Claims → kein Fix-Rewrite (nur Log); ab GATE_FIX_MIN läuft der Fix."""
|
||||
db = testdb
|
||||
await db.upsert_guide_card(TOPIC, FMT, "alpha", "Alpha")
|
||||
env = gb._Env(None, "g-g", TOPIC, FMT, "", tmp_path / "Guide.json", {"Alpha": []}, {}, "(q)", "spec")
|
||||
md = "<!-- section: Alpha -->\n<!-- ausführlich -->\nText."
|
||||
card = {"block_norm": "alpha", "block": "Alpha", "writer_rounds": 0, "md": md}
|
||||
keys = []
|
||||
|
||||
async def fake_slot(ctx, label, *, key, prompt, role, capabilities, payload, timeout):
|
||||
keys.append(key)
|
||||
if "-gate-" in key:
|
||||
return gb.OK, [{"text": "c1", "grund": ""}, {"text": "c2", "grund": ""}]
|
||||
raise AssertionError("Fix darf unter der Schwelle nicht laufen")
|
||||
|
||||
monkeypatch.setattr(gb, "run_single_slot", fake_slot)
|
||||
assert await gb._stage_fakten_gate(env, card)
|
||||
assert all("-gate-" in k for k in keys)
|
||||
|
||||
|
||||
async def test_load_subblocks_defaultet_levellose(testdb):
|
||||
"""Consensus-Row ohne level fällt NICHT mehr raus — Default 'advanced'.
|
||||
(Re-Run-Resume hinterließ 25 solcher Rows; der Writer verlor sie stumm.)"""
|
||||
from guide import _load_subblocks
|
||||
db = testdb
|
||||
await db.put_subblock("t", "block", "mit level", "Block", "Mit Level",
|
||||
level="beginner", status="consensus")
|
||||
await db.put_subblock("t", "block", "ohne level", "Block", "Ohne Level", status="consensus")
|
||||
subs = await _load_subblocks("t")
|
||||
by_title = {s["title"]: s["level"] for s in subs["Block"]}
|
||||
assert by_title == {"Mit Level": "beginner", "Ohne Level": "advanced"}
|
||||
|
||||
@@ -63,3 +63,44 @@ def test_note_guide_kalibrierung():
|
||||
assert qa.note({"fachlich_falsch": 0.1}, gq.NOTE_GEWICHTE_GUIDE) == 7.0
|
||||
ohne = {"marker_fehlend": 0.0, "ziel_ohne_anker": 0.0}
|
||||
assert qa.note(ohne, gq.NOTE_GEWICHTE_GUIDE) == 10.0
|
||||
|
||||
|
||||
def test_marker_escaping_tolerant():
|
||||
"""Escapte Titel (h\\~2\\~o) sind kein „Marker fehlt" — Writer und DB escapen verschieden."""
|
||||
md = "<!-- ausführlich -->\n<!-- sub: beginner | ^ und ~ müssen escaped werden (h\\\\~2\\\\~o) -->\nText."
|
||||
cards = [{"block": "Hoch", "block_norm": "hoch", "md": md}]
|
||||
rel = {"hoch": {gq._norm_title("^ und ~ müssen escaped werden (h~2~o)")}}
|
||||
assert gq.marker_fehlend(cards, rel) == []
|
||||
|
||||
|
||||
async def test_fachlich_falsch_braucht_beide_bestaetiger(monkeypatch):
|
||||
"""Befund zählt nur, wenn BEIDE Bestätiger zustimmen — Einzel-/Zweifach-Urteile
|
||||
ließen die Note desselben Guides zwischen 2.0 und 6.6 springen."""
|
||||
import agents
|
||||
calls = {"n": 0}
|
||||
|
||||
async def fake_agent(key, prompt, timeout, **kw):
|
||||
calls["n"] += 1
|
||||
if "-fakten-3-" in key: # Bestätiger 2: nur #1 bleibt
|
||||
return 0, '{"relevant": {"1": "ja"}}', ""
|
||||
if "-fakten-2-" in key: # Bestätiger 1: #1 und #2
|
||||
return 0, '{"relevant": {"1": "ja", "2": "ja"}}', ""
|
||||
return 0, '{"relevant": {"1": "ja", "2": "ja", "3": "nein"}}', "" # Pass 1
|
||||
|
||||
monkeypatch.setattr(agents, "run_agent", fake_agent)
|
||||
cards = [_card("Alpha", AUSF), _card("Beta", AUSF), _card("Gamma", AUSF)]
|
||||
out = await gq._fachlich_falsch("t", cards)
|
||||
assert out == ["Alpha"] and calls["n"] == 3
|
||||
|
||||
|
||||
async def test_fachlich_falsch_ohne_verdacht_kein_zweiter_pass(monkeypatch):
|
||||
import agents
|
||||
calls = {"n": 0}
|
||||
|
||||
async def fake_agent(key, prompt, timeout, **kw):
|
||||
calls["n"] += 1
|
||||
return 0, '{"relevant": {"1": "nein", "2": "nein"}}', ""
|
||||
|
||||
monkeypatch.setattr(agents, "run_agent", fake_agent)
|
||||
out = await gq._fachlich_falsch("t", [_card("Alpha", AUSF), _card("Beta", AUSF)])
|
||||
assert out == [] and calls["n"] == 1
|
||||
|
||||
@@ -505,3 +505,17 @@ async def test_crossblock_context_wins(testdb, tmp_path, monkeypatch):
|
||||
assert alpha["payload"]["raw"].get("Alpha", []) == [] # Paket-Seite gefaltet
|
||||
gamma_rows = {r["sub_norm"]: r["status"] for r in await db.list_subblocks(TOPIC, "gamma")}
|
||||
assert gamma_rows[blocks._norm_title("Gleiche Aussage")] == "consensus" # Kontext unberührt
|
||||
|
||||
|
||||
async def test_finalize_defaultet_level_nachzuegler(testdb, tmp_path):
|
||||
"""Finalize klassifiziert level-/relevance-lose consensus-Rows (Default advanced/relevant)."""
|
||||
db = testdb
|
||||
await db.put_subblock(TOPIC, "alpha", "nachzuegler", "Alpha", "Nachzügler", status="consensus")
|
||||
await db.kanban_upsert_card(TOPIC, "artefacts", "alpha", "ablock", "finalize", {})
|
||||
files = {k: tmp_path / f"{k}.json" for k in
|
||||
("sub_roh", "facts", "sidecar", "question_pattern", "artefakte")}
|
||||
card = {"card_id": "alpha", "payload": {"title": "Alpha", "raw": {}, "facts": {},
|
||||
"sidecar": {}, "pattern": {}, "artefacts": {}}}
|
||||
await ba._proc_finalize(_ctx(), Flow(TOPIC, work_dir=tmp_path), files, [card])
|
||||
row = next(r for r in await db.list_subblocks(TOPIC, "alpha"))
|
||||
assert row["level"] == "advanced" and row["relevance"] == "relevant"
|
||||
|
||||
Reference in New Issue
Block a user