168 lines
7.1 KiB
Python
168 lines
7.1 KiB
Python
"""Suche: Fallback-Kette, Circuit Breaker, URL-Dedupe. Laden: Snapshot stabil."""
|
||
import pytest
|
||
|
||
from backend import config, db, laden, suche
|
||
from .conftest import topic_anlegen
|
||
|
||
|
||
def test_url_normieren():
|
||
assert suche.url_normieren("HTTPS://Beispiel.de/Pfad/?utm_source=x&a=1") == \
|
||
"https://beispiel.de/Pfad?a=1"
|
||
assert suche.url_normieren("https://beispiel.de/seite/") == \
|
||
suche.url_normieren("https://beispiel.de/seite")
|
||
|
||
|
||
async def test_fallback_kette(monkeypatch):
|
||
monkeypatch.setattr(config, "FAKE", False)
|
||
suche._ausfaelle.clear()
|
||
|
||
async def leer_wiki(q):
|
||
return []
|
||
|
||
async def kaputt(q):
|
||
raise RuntimeError("down")
|
||
|
||
async def liefert(q):
|
||
return [{"titel": "T", "url": "https://x.de/a", "backend": "b2"}]
|
||
|
||
monkeypatch.setattr(suche, "_wikipedia", leer_wiki)
|
||
monkeypatch.setattr(suche, "_KETTE", (("b1", kaputt), ("b2", liefert)))
|
||
monkeypatch.setattr(config, "SUCHE_RETRIES", 0)
|
||
treffer, fehler = await suche.web_suchen("query")
|
||
assert [t["backend"] for t in treffer] == ["b2"]
|
||
assert fehler == ["b1: RuntimeError"]
|
||
|
||
|
||
async def test_circuit_breaker(monkeypatch):
|
||
monkeypatch.setattr(config, "FAKE", False)
|
||
monkeypatch.setattr(config, "SUCHE_RETRIES", 0)
|
||
suche._ausfaelle.clear()
|
||
aufrufe = {"n": 0}
|
||
|
||
async def leer_wiki(q):
|
||
return []
|
||
|
||
async def kaputt(q):
|
||
aufrufe["n"] += 1
|
||
raise RuntimeError("down")
|
||
|
||
monkeypatch.setattr(suche, "_wikipedia", leer_wiki)
|
||
monkeypatch.setattr(suche, "_KETTE", (("b1", kaputt),))
|
||
for _ in range(config.CIRCUIT_BREAKER_N + 3):
|
||
await suche.web_suchen("q")
|
||
assert aufrufe["n"] == config.CIRCUIT_BREAKER_N # danach übersprungen
|
||
|
||
|
||
async def test_suche_dedupliziert_urls():
|
||
topic = topic_anlegen()
|
||
run_id = db.insert("runs", topic=topic, status="running")
|
||
t = {"run_id": run_id, "topic": topic, "runde": 1,
|
||
"payload": db.j({"query": "thema ueberblick r1"})}
|
||
erg1 = await suche.suchen({**t})
|
||
erg2 = await suche.suchen({**t})
|
||
assert erg1.daten["neu"] == 1
|
||
assert erg2.daten["neu"] == 0 # gleiche URL → kein zweiter Laden-Task
|
||
|
||
|
||
async def test_relevanz_filter_wirft_listen_raus(monkeypatch):
|
||
"""Läuft OHNE Fake-Kurzschluss — genau der Pfad, der im Echtlauf brach."""
|
||
monkeypatch.setattr(config, "FAKE", False)
|
||
from backend import llm as llm_mod
|
||
|
||
async def urteil(**kw):
|
||
assert "quellen-relevanz" in kw["skill_namen"]
|
||
assert "Pasta" in kw["werte"]["treffer"] # Wiki läuft MIT durchs Urteil
|
||
return "===RELEVANT===\nIDS: 0,1,3"
|
||
monkeypatch.setattr(llm_mod, "call", urteil)
|
||
treffer = [
|
||
{"titel": "Pomodoro-Technik – Wikipedia",
|
||
"url": "https://de.wikipedia.org/wiki/X", "backend": "wikipedia_de"},
|
||
{"titel": "Guter Artikel", "url": "https://a.de/1", "backend": "ddgs"},
|
||
{"titel": "Pasta", "url": "https://en.wikipedia.org/wiki/Pasta",
|
||
"backend": "wikipedia_en"},
|
||
{"titel": "Anleitung", "url": "https://a.de/3", "backend": "ddgs"},
|
||
]
|
||
topic = topic_anlegen()
|
||
run_id = db.insert("runs", topic=topic, status="running")
|
||
task = {"run_id": run_id, "topic": topic, "item": "t", "runde": 1}
|
||
ergebnis = await suche._relevanz_filter(task, "Thema", treffer)
|
||
urls = [t["url"] for t in ergebnis]
|
||
assert "https://de.wikipedia.org/wiki/X" in urls
|
||
assert "https://a.de/1" in urls and "https://a.de/3" in urls
|
||
assert "https://en.wikipedia.org/wiki/Pasta" not in urls # Off-Topic-Wiki raus
|
||
|
||
|
||
async def test_wikipedia_seed_umgeht_deckel(monkeypatch):
|
||
monkeypatch.setattr(config, "QUELLEN_MAX", 1)
|
||
topic = topic_anlegen(name="seedtest", titel="Pomodoro Technik")
|
||
run_id = db.insert("runs", topic=topic, status="running")
|
||
erg = await suche.suchen({"run_id": run_id, "topic": topic, "runde": 1,
|
||
"item": "r1:wikiseed",
|
||
"payload": db.j({"seed": True})})
|
||
assert erg.daten["neu"] == 2 # de+en trotz Deckel 1
|
||
urls = [q["url"] for q in db.query(
|
||
"SELECT url FROM quellen WHERE topic=?", topic)]
|
||
assert any("de.wikipedia.org/wiki/Pomodoro_Technik" in u for u in urls)
|
||
assert any("en.wikipedia.org" in u for u in urls)
|
||
# idempotent: zweiter Lauf legt nichts Neues an
|
||
erg2 = await suche.suchen({"run_id": run_id, "topic": topic, "runde": 1,
|
||
"item": "r1:wikiseed",
|
||
"payload": db.j({"seed": True})})
|
||
assert erg2.daten["neu"] == 0
|
||
|
||
|
||
async def test_quellen_deckel(monkeypatch):
|
||
monkeypatch.setattr(config, "QUELLEN_MAX", 1)
|
||
topic = topic_anlegen()
|
||
run_id = db.insert("runs", topic=topic, status="running")
|
||
basis = {"run_id": run_id, "topic": topic, "runde": 1, "item": "t"}
|
||
erg1 = await suche.suchen({**basis,
|
||
"payload": db.j({"query": "thema ueberblick r1"})})
|
||
erg2 = await suche.suchen({**basis,
|
||
"payload": db.j({"query": "thema tutorial r1"})})
|
||
assert erg1.daten["neu"] == 1
|
||
assert erg2.daten["neu"] == 0 # Deckel erreicht — keine neue Quelle
|
||
|
||
|
||
async def test_quellen_deckel_pro_topic(monkeypatch):
|
||
monkeypatch.setattr(config, "QUELLEN_MAX", 1)
|
||
topic = topic_anlegen()
|
||
db.update("topics", "name=?", (topic,), quellen_max=0) # 0 = unbegrenzt
|
||
run_id = db.insert("runs", topic=topic, status="running")
|
||
basis = {"run_id": run_id, "topic": topic, "runde": 1, "item": "t"}
|
||
erg1 = await suche.suchen({**basis,
|
||
"payload": db.j({"query": "thema ueberblick r1"})})
|
||
erg2 = await suche.suchen({**basis,
|
||
"payload": db.j({"query": "thema tutorial r1"})})
|
||
assert erg1.daten["neu"] == 1
|
||
assert erg2.daten["neu"] == 1 # Topic-Einstellung schlägt globalen Deckel
|
||
|
||
|
||
async def test_laden_snapshot_stabil():
|
||
topic = topic_anlegen()
|
||
run_id = db.insert("runs", topic=topic, status="running")
|
||
qid = db.insert("quellen", topic=topic, url="https://beispiel.de/grundlagen",
|
||
url_norm="https://beispiel.de/grundlagen", zweck="korpus")
|
||
t = {"run_id": run_id, "topic": topic, "runde": 1,
|
||
"payload": db.j({"quelle_id": qid})}
|
||
erg1 = await laden.laden({**t})
|
||
q = db.one("SELECT * FROM quellen WHERE id=?", qid)
|
||
assert q["status"] == "geladen" and q["snapshot"]
|
||
inhalt1 = laden.snapshot_lesen(q)
|
||
erg2 = await laden.laden({**t}) # Resume: kein Neuschreiben
|
||
assert erg2.daten.get("skip")
|
||
assert laden.snapshot_lesen(q) == inhalt1
|
||
assert len(erg1.neue_tasks) >= 1 # Soll-Extraktions-Chunks
|
||
|
||
|
||
async def test_laden_leere_quelle_befund():
|
||
topic = topic_anlegen()
|
||
run_id = db.insert("runs", topic=topic, status="running")
|
||
qid = db.insert("quellen", topic=topic, url="https://unbekannt.example/x",
|
||
url_norm="https://unbekannt.example/x")
|
||
erg = await laden.laden({"run_id": run_id, "topic": topic, "runde": 1,
|
||
"payload": db.j({"quelle_id": qid})})
|
||
assert erg.neue_tasks == []
|
||
assert db.one("SELECT status FROM quellen WHERE id=?", qid)["status"] == "leer"
|
||
assert db.one("SELECT id FROM befunde WHERE run_id=? AND art='leer'", run_id)
|