251 lines
12 KiB
Python
251 lines
12 KiB
Python
"""QA-Detektoren: Fehler-Injektion auf Mini-Korpus — deterministisch, ohne LLM/Embedding."""
|
||
|
||
import qa
|
||
|
||
|
||
CORPUS = {"Skript.txt": (
|
||
"Kapitel 1: Vertex Cover — Definition, Approximation und Beweis der Guete.\n\n"
|
||
"Kapitel 2: Matching in Graphen — perfektes Matching und Augmentationswege.")}
|
||
BLOCKS = [
|
||
{"title": "Vertex Cover", "description": "Knotenüberdeckung", "sources": ["Skript.txt"]},
|
||
{"title": "Matching", "description": "Paarung in Graphen", "sources": ["Skript.txt"]},
|
||
]
|
||
SUBS = {"vertex cover": ["Approximation der Guete"], "matching": ["Matching in Graphen", "Augmentationswege"]}
|
||
|
||
|
||
def test_baseline_clean(monkeypatch):
|
||
"""Sauberes Soll-Inventar → alle Detektoren still (jeder Absatz ein Abschnitt)."""
|
||
monkeypatch.setattr(qa, "SECTION_CHARS", 20)
|
||
assert qa.dubletten(BLOCKS, emb_on=False) == []
|
||
assert qa.luecken(BLOCKS, SUBS, CORPUS) == []
|
||
assert qa.fremd(BLOCKS, CORPUS) == []
|
||
assert qa.hygiene(BLOCKS) == []
|
||
|
||
|
||
def test_injected_duplicate_found():
|
||
b = BLOCKS + [{"title": "Vertex-Cover-Problem", "description": "", "sources": []}]
|
||
pairs = qa.dubletten(b, emb_on=False)
|
||
assert any({p["a"], p["b"]} == {"Vertex Cover", "Vertex-Cover-Problem"} for p in pairs)
|
||
|
||
|
||
def test_acronym_signal():
|
||
b = BLOCKS + [{"title": "VC (Vertex Cover)", "description": "", "sources": []}]
|
||
pairs = qa.dubletten(b, emb_on=False)
|
||
hit = next(p for p in pairs if "VC (Vertex Cover)" in (p["a"], p["b"]) and "Vertex Cover" in (p["a"], p["b"]))
|
||
assert hit["signale"].get("akronym") is True
|
||
|
||
|
||
def test_relation_operand_not_suspicious():
|
||
"""Relation vs. Operand ist per Design getrennt — kein Verdachtspaar."""
|
||
b = BLOCKS + [{"title": "3-SAT ≤ Vertex Cover", "description": "", "sources": []}]
|
||
pairs = qa.dubletten(b, emb_on=False)
|
||
assert not any("≤" in p["a"] + p["b"] for p in pairs)
|
||
|
||
|
||
def test_removed_block_creates_gap(monkeypatch):
|
||
monkeypatch.setattr(qa, "SECTION_CHARS", 20)
|
||
only_vc = [BLOCKS[0]]
|
||
gaps = qa.luecken(only_vc, {"vertex cover": SUBS["vertex cover"]}, CORPUS)
|
||
assert len(gaps) == 1 and "Matching" in gaps[0]["vorschau"]
|
||
|
||
|
||
def test_foreign_block_flagged():
|
||
b = BLOCKS + [{"title": "Quantencomputer Grundlagen", "description": "", "sources": []}]
|
||
assert qa.fremd(b, CORPUS) == ["Quantencomputer Grundlagen"]
|
||
|
||
|
||
def test_beleg_flags_unbacked_sub():
|
||
rows = [{"block": "Matching", "sub_title": "Erfunden", "mentions": 0, "status": "consensus"},
|
||
{"block": "Matching", "sub_title": "Belegt", "mentions": 3, "status": "consensus"}]
|
||
r = qa.beleg([{"title": "Matching", "sources": []}], rows)
|
||
assert r["subs_ohne_beleg"] == ["Matching · Erfunden"]
|
||
assert r["bloecke_ohne_quelle"] == ["Matching"]
|
||
|
||
|
||
def test_hygiene_flags():
|
||
b = [{"title": "**Fett**", "description": "", "sources": []},
|
||
{"title": "Block (2)", "description": "ok", "sources": []}]
|
||
h = {x["titel"]: x["probleme"] for x in qa.hygiene(b)}
|
||
assert "markdown" in h["**Fett**"] and "leere-beschreibung" in h["**Fett**"]
|
||
assert h["Block (2)"] == ["kollisions-suffix"]
|
||
|
||
|
||
def test_sections_split_on_paragraphs():
|
||
secs = qa._sections("a\n\nb\n\nc", goal=3)
|
||
assert len(secs) >= 2 and "".join(secs).replace("\n", "") == "abc"
|
||
|
||
|
||
def test_note_deterministic_and_monotonic():
|
||
"""Saubere Quoten → 10; jede zusätzliche Quote drückt die Note."""
|
||
sauber = {k: 0 for k in qa.NOTE_GEWICHTE}
|
||
assert qa.note(sauber) == 10.0
|
||
schlechter = dict(sauber, luecken=0.05)
|
||
noch_schlechter = dict(schlechter, fremd=0.05)
|
||
assert 10.0 > qa.note(schlechter) > qa.note(noch_schlechter) >= 0.0
|
||
assert qa.note({k: 1 for k in qa.NOTE_GEWICHTE}) == 0.0
|
||
|
||
|
||
def test_note_kalibrierung():
|
||
"""Gewicht = Punktabzug bei 100 %: 5 % Fremd × 2.5 → −1.25 → 8.8 gerundet."""
|
||
assert qa.note({"fremd": 0.05}) == 8.8
|
||
assert qa.note({"fremd": 1.0}) == 0.0 # komplett fremdes Inventar = 0, nicht 7.7
|
||
|
||
|
||
def test_note_verdacht_zaehlt_nicht():
|
||
"""dubletten_verdacht ist Verdachtsliste, kein Urteil — beeinflusst die Note nicht."""
|
||
assert qa.note({"dubletten_verdacht": 1.0}) == 10.0
|
||
|
||
|
||
def test_note_artefakte_getrennt():
|
||
"""Subs/Artefakte haben eigene Gewichte — zur Gate-Zeit existieren sie noch nicht
|
||
und dürfen die Inventar-Note weder schönen noch drücken."""
|
||
assert "subs_ohne_beleg" not in qa.NOTE_GEWICHTE
|
||
assert qa.note({"subs_ohne_beleg": 0.0, "verwaiste": 0.1}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 9.0
|
||
assert qa.note({"subs_ohne_beleg": 1.0}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 0.0
|
||
|
||
|
||
def test_artefakte_coverage_and_orphans():
|
||
subs = [{"block_norm": "b", "sub_norm": "s1: lange beschreibung", "status": "consensus"},
|
||
{"block_norm": "b", "sub_norm": "s2", "status": "consensus"},
|
||
{"block_norm": "b", "sub_norm": "alt", "status": "variant"},
|
||
{"block_norm": "b", "sub_norm": "weg", "status": "discarded"}]
|
||
arts = [{"block_norm": "b", "sub_norm": "s1", "type": "flashcard"}, # Präfix-Treffer
|
||
{"block_norm": "b", "sub_norm": "alt", "type": "flashcard"}, # variant → lebt, keine Waise
|
||
{"block_norm": "b", "sub_norm": "weg", "type": "flashcard"}, # verworfen → Waise
|
||
{"block_norm": "b", "sub_norm": "tot", "type": "flashcard"}] # fehlt → Waise
|
||
fragen = [{"block_norm": "b", "sub_norm": "s1: lange beschreibung"},
|
||
{"block_norm": "b", "sub_norm": "s2"}]
|
||
r = qa.artefakte(subs, arts, fragen)
|
||
assert r["frage_abdeckung"] == 1.0
|
||
assert r["flashcard_abdeckung"] == 0.5 # nur s1 der beiden consensus-Subs
|
||
assert r["verwaiste"] == ["flashcard: b · tot", "flashcard: b · weg"]
|
||
|
||
|
||
def test_artefakte_prefix_family_resolves_to_consensus():
|
||
"""Kurz-Key trifft consensus-Sub PLUS gefaltete Varianten mit gleichem Präfix —
|
||
das ist keine Waise, das Ziel ist der consensus-Sub."""
|
||
subs = [{"block_norm": "b", "sub_norm": "auto: echte fassung", "status": "consensus"},
|
||
{"block_norm": "b", "sub_norm": "auto: variante eins", "status": "variant"},
|
||
{"block_norm": "b", "sub_norm": "auto: variante zwei", "status": "variant"}]
|
||
arts = [{"block_norm": "b", "sub_norm": "auto", "type": "example"}]
|
||
r = qa.artefakte(subs, arts, [])
|
||
assert r["verwaiste"] == []
|
||
assert r["beispiel_abdeckung"] == 1.0
|
||
|
||
|
||
def test_artefakte_not_generated():
|
||
assert qa.artefakte([{"block_norm": "b", "sub_norm": "s", "status": "consensus"}], [], []) == {"status": "nicht generiert"}
|
||
|
||
|
||
def test_fremd_glued_prefix_not_whitewashed():
|
||
"""'αÜbergang' darf nicht über den Substring 'bergang'⊂'Übergang' als belegt gelten;
|
||
Symbol-Varianten (Δ/∆) bleiben über die ASCII-Form gedeckt."""
|
||
corpus = {"S.txt": "Der Übergang ist wichtig.\n\nDer ∆TSP1 Algorithmus folgt."}
|
||
b = [{"title": "αÜbergang", "description": "", "sources": []},
|
||
{"title": "ΔTSP1-Algorithmus", "description": "", "sources": []}]
|
||
assert qa.fremd(b, corpus) == ["αÜbergang"]
|
||
|
||
|
||
def test_note_ignores_unmeasured_quotes():
|
||
"""unechte_bloecke zählt nur, wenn gemessen (--llm) — sonst weder Schaden noch Schönung."""
|
||
ohne = {k: 0.02 for k in qa.NOTE_GEWICHTE if k != "unechte_bloecke"}
|
||
mit_null = dict(ohne, unechte_bloecke=0.0)
|
||
mit_schaden = dict(ohne, unechte_bloecke=0.5)
|
||
assert qa.note(mit_null) == qa.note(ohne)
|
||
assert qa.note(mit_schaden) < qa.note(ohne)
|
||
|
||
|
||
class _FakeEmb:
|
||
"""Gleicher Text → gleicher Einheitsvektor, sonst orthogonal (cos 1.0 / 0.0)."""
|
||
|
||
@staticmethod
|
||
def available():
|
||
return True
|
||
|
||
@staticmethod
|
||
def embed(texts):
|
||
import numpy as np
|
||
uniq = {t: k for k, t in enumerate(dict.fromkeys(texts))}
|
||
arr = np.zeros((len(texts), max(len(uniq), 1)))
|
||
for r, t in enumerate(texts):
|
||
arr[r, uniq[t]] = 1.0
|
||
return arr
|
||
|
||
|
||
def test_sub_dubletten_detector(monkeypatch):
|
||
"""Kandidaten in-block UND cross-block; nur consensus-Subs zählen."""
|
||
monkeypatch.setattr(qa, "embedding", _FakeEmb)
|
||
rows = [{"block": "Alpha", "block_norm": "alpha", "sub_title": "Gleiche Aussage", "status": "consensus"},
|
||
{"block": "Beta", "block_norm": "beta", "sub_title": "Gleiche Aussage", "status": "consensus"},
|
||
{"block": "Beta", "block_norm": "beta", "sub_title": "Andere Aussage", "status": "consensus"},
|
||
{"block": "Beta", "block_norm": "beta", "sub_title": "Gleiche Aussage", "status": "variant"}]
|
||
pairs = qa.sub_dubletten(rows)
|
||
assert len(pairs) == 1
|
||
assert pairs[0]["cross"] is True and pairs[0]["cos"] == 1.0
|
||
assert qa.sub_dubletten(rows, emb_on=False) == []
|
||
|
||
|
||
def test_note_sub_dubletten():
|
||
"""Bestätigte Sub-Dubletten drücken die Artefakt-Note; der bloße Verdacht nicht."""
|
||
assert qa.note({"sub_dubletten": 0.1}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 9.0
|
||
assert qa.note({"sub_dubletten_verdacht": 1.0}, qa.NOTE_GEWICHTE_ARTEFAKTE) == 10.0
|
||
|
||
|
||
def test_zaehlbare_luecken():
|
||
"""Mit LLM zählen widerlegte Lücken nicht; unbeurteilte ('?'/ohne Key) konservativ schon."""
|
||
lk = [{"llm": "ja"}, {"llm": "nein"}, {"llm": "?"}, {}]
|
||
assert len(qa._zaehlbare_luecken(lk, llm=True)) == 3
|
||
assert len(qa._zaehlbare_luecken(lk, llm=False)) == 4
|
||
|
||
|
||
def test_description_anchors_cover(monkeypatch):
|
||
"""Am Gate existieren keine Subs — Beschreibungs-Tokens müssen Abschnitte decken."""
|
||
monkeypatch.setattr(qa, "SECTION_CHARS", 20)
|
||
corpus = {"S.txt": "Kapitel 9: Augmentationswege und perfektes Matching."}
|
||
block = [{"title": "Paarungen", "description": "perfektes Matching mit Augmentationswege", "sources": []}]
|
||
assert qa.luecken(block, {}, corpus) == []
|
||
ohne = [{"title": "Paarungen", "description": "", "sources": []}]
|
||
assert len(qa.luecken(ohne, {}, corpus)) == 1
|
||
|
||
|
||
def test_fremd_digit_suffix_tolerant():
|
||
"""'ΔTSP1' matcht Korpus-'∆TSP' (tokenisiert zu 'tsp') via Ziffern-Suffix-Fallback."""
|
||
corpus = {"S.txt": "Der ∆TSP Algorithmus verdoppelt Kanten im Graphen."}
|
||
b = [{"title": "ΔTSP1-Algorithmus", "description": "", "sources": []},
|
||
{"title": "Quantencomputer", "description": "", "sources": []}]
|
||
assert qa.fremd(b, corpus) == ["Quantencomputer"]
|
||
|
||
|
||
async def test_unecht_braucht_doppelt_nein(testdb, tmp_path, monkeypatch):
|
||
"""Echtheits-Urteil zählt nur nach Bestätiger-Pass: der Einzel-Judge flaggte pro Lauf
|
||
andere Blöcke und pendelte die Note (aak: 9.3↔10.0 bei identischem Bestand)."""
|
||
db = testdb
|
||
for cid, titel in (("b1", "Wackelkandidat"), ("b2", "Zufallstreffer"), ("b3", "Solide")):
|
||
await db.kanban_upsert_card("t", "inventory", cid, "block", "done_block",
|
||
{"title": titel, "description": "d"})
|
||
monkeypatch.setattr(qa, "QA_DIR", tmp_path)
|
||
|
||
async def fake_verdicts(template, topic, key, items):
|
||
if template != "QA-Bausteine":
|
||
return {}
|
||
if key.startswith("bausteine-b2"): # Bestätiger sieht nur die Geflaggten
|
||
assert len(items) == 2
|
||
return {1: "nein", 2: "ja"} # nur der erste wird bestätigt
|
||
return {1: "nein", 2: "nein", 3: "ja"} # Pass 1 flaggt zwei
|
||
|
||
monkeypatch.setattr(qa, "_llm_verdicts", fake_verdicts)
|
||
report = await qa.qa_report("t", llm=True)
|
||
assert report["unecht"] == ["Wackelkandidat"]
|
||
|
||
|
||
async def test_topic_delete_entfernt_qa_ordner(testdb, tmp_path, monkeypatch):
|
||
"""DELETE /topics räumt auch storage/qa/<topic>/ — Reports gehören zum Topic."""
|
||
import routes
|
||
monkeypatch.setattr(routes, "topic_dir", lambda t: tmp_path / "topics" / t)
|
||
monkeypatch.setattr(qa, "QA_DIR", tmp_path / "qa")
|
||
qdir = tmp_path / "qa" / "t"
|
||
qdir.mkdir(parents=True)
|
||
(qdir / "alt.json").write_text("{}", encoding="utf-8")
|
||
await routes.remove_topic("t")
|
||
assert not qdir.exists()
|