init

tests/test_suche_laden.py

@@ -0,0 +1,167 @@
+"""Suche: Fallback-Kette, Circuit Breaker, URL-Dedupe. Laden: Snapshot stabil."""
+import pytest
+
+from backend import config, db, laden, suche
+from .conftest import topic_anlegen
+
+
+def test_url_normieren():
+    assert suche.url_normieren("HTTPS://Beispiel.de/Pfad/?utm_source=x&a=1") == \
+        "https://beispiel.de/Pfad?a=1"
+    assert suche.url_normieren("https://beispiel.de/seite/") == \
+        suche.url_normieren("https://beispiel.de/seite")
+
+
+async def test_fallback_kette(monkeypatch):
+    monkeypatch.setattr(config, "FAKE", False)
+    suche._ausfaelle.clear()
+
+    async def leer_wiki(q):
+        return []
+
+    async def kaputt(q):
+        raise RuntimeError("down")
+
+    async def liefert(q):
+        return [{"titel": "T", "url": "https://x.de/a", "backend": "b2"}]
+
+    monkeypatch.setattr(suche, "_wikipedia", leer_wiki)
+    monkeypatch.setattr(suche, "_KETTE", (("b1", kaputt), ("b2", liefert)))
+    monkeypatch.setattr(config, "SUCHE_RETRIES", 0)
+    treffer, fehler = await suche.web_suchen("query")
+    assert [t["backend"] for t in treffer] == ["b2"]
+    assert fehler == ["b1: RuntimeError"]
+
+
+async def test_circuit_breaker(monkeypatch):
+    monkeypatch.setattr(config, "FAKE", False)
+    monkeypatch.setattr(config, "SUCHE_RETRIES", 0)
+    suche._ausfaelle.clear()
+    aufrufe = {"n": 0}
+
+    async def leer_wiki(q):
+        return []
+
+    async def kaputt(q):
+        aufrufe["n"] += 1
+        raise RuntimeError("down")
+
+    monkeypatch.setattr(suche, "_wikipedia", leer_wiki)
+    monkeypatch.setattr(suche, "_KETTE", (("b1", kaputt),))
+    for _ in range(config.CIRCUIT_BREAKER_N + 3):
+        await suche.web_suchen("q")
+    assert aufrufe["n"] == config.CIRCUIT_BREAKER_N  # danach übersprungen
+
+
+async def test_suche_dedupliziert_urls():
+    topic = topic_anlegen()
+    run_id = db.insert("runs", topic=topic, status="running")
+    t = {"run_id": run_id, "topic": topic, "runde": 1,
+         "payload": db.j({"query": "thema ueberblick r1"})}
+    erg1 = await suche.suchen({**t})
+    erg2 = await suche.suchen({**t})
+    assert erg1.daten["neu"] == 1
+    assert erg2.daten["neu"] == 0  # gleiche URL → kein zweiter Laden-Task
+
+
+async def test_relevanz_filter_wirft_listen_raus(monkeypatch):
+    """Läuft OHNE Fake-Kurzschluss — genau der Pfad, der im Echtlauf brach."""
+    monkeypatch.setattr(config, "FAKE", False)
+    from backend import llm as llm_mod
+
+    async def urteil(**kw):
+        assert "quellen-relevanz" in kw["skill_namen"]
+        assert "Pasta" in kw["werte"]["treffer"]  # Wiki läuft MIT durchs Urteil
+        return "===RELEVANT===\nIDS: 0,1,3"
+    monkeypatch.setattr(llm_mod, "call", urteil)
+    treffer = [
+        {"titel": "Pomodoro-Technik – Wikipedia",
+         "url": "https://de.wikipedia.org/wiki/X", "backend": "wikipedia_de"},
+        {"titel": "Guter Artikel", "url": "https://a.de/1", "backend": "ddgs"},
+        {"titel": "Pasta", "url": "https://en.wikipedia.org/wiki/Pasta",
+         "backend": "wikipedia_en"},
+        {"titel": "Anleitung", "url": "https://a.de/3", "backend": "ddgs"},
+    ]
+    topic = topic_anlegen()
+    run_id = db.insert("runs", topic=topic, status="running")
+    task = {"run_id": run_id, "topic": topic, "item": "t", "runde": 1}
+    ergebnis = await suche._relevanz_filter(task, "Thema", treffer)
+    urls = [t["url"] for t in ergebnis]
+    assert "https://de.wikipedia.org/wiki/X" in urls
+    assert "https://a.de/1" in urls and "https://a.de/3" in urls
+    assert "https://en.wikipedia.org/wiki/Pasta" not in urls  # Off-Topic-Wiki raus
+
+
+async def test_wikipedia_seed_umgeht_deckel(monkeypatch):
+    monkeypatch.setattr(config, "QUELLEN_MAX", 1)
+    topic = topic_anlegen(name="seedtest", titel="Pomodoro Technik")
+    run_id = db.insert("runs", topic=topic, status="running")
+    erg = await suche.suchen({"run_id": run_id, "topic": topic, "runde": 1,
+                              "item": "r1:wikiseed",
+                              "payload": db.j({"seed": True})})
+    assert erg.daten["neu"] == 2  # de+en trotz Deckel 1
+    urls = [q["url"] for q in db.query(
+        "SELECT url FROM quellen WHERE topic=?", topic)]
+    assert any("de.wikipedia.org/wiki/Pomodoro_Technik" in u for u in urls)
+    assert any("en.wikipedia.org" in u for u in urls)
+    # idempotent: zweiter Lauf legt nichts Neues an
+    erg2 = await suche.suchen({"run_id": run_id, "topic": topic, "runde": 1,
+                               "item": "r1:wikiseed",
+                               "payload": db.j({"seed": True})})
+    assert erg2.daten["neu"] == 0
+
+
+async def test_quellen_deckel(monkeypatch):
+    monkeypatch.setattr(config, "QUELLEN_MAX", 1)
+    topic = topic_anlegen()
+    run_id = db.insert("runs", topic=topic, status="running")
+    basis = {"run_id": run_id, "topic": topic, "runde": 1, "item": "t"}
+    erg1 = await suche.suchen({**basis,
+                               "payload": db.j({"query": "thema ueberblick r1"})})
+    erg2 = await suche.suchen({**basis,
+                               "payload": db.j({"query": "thema tutorial r1"})})
+    assert erg1.daten["neu"] == 1
+    assert erg2.daten["neu"] == 0  # Deckel erreicht — keine neue Quelle
+
+
+async def test_quellen_deckel_pro_topic(monkeypatch):
+    monkeypatch.setattr(config, "QUELLEN_MAX", 1)
+    topic = topic_anlegen()
+    db.update("topics", "name=?", (topic,), quellen_max=0)  # 0 = unbegrenzt
+    run_id = db.insert("runs", topic=topic, status="running")
+    basis = {"run_id": run_id, "topic": topic, "runde": 1, "item": "t"}
+    erg1 = await suche.suchen({**basis,
+                               "payload": db.j({"query": "thema ueberblick r1"})})
+    erg2 = await suche.suchen({**basis,
+                               "payload": db.j({"query": "thema tutorial r1"})})
+    assert erg1.daten["neu"] == 1
+    assert erg2.daten["neu"] == 1  # Topic-Einstellung schlägt globalen Deckel
+
+
+async def test_laden_snapshot_stabil():
+    topic = topic_anlegen()
+    run_id = db.insert("runs", topic=topic, status="running")
+    qid = db.insert("quellen", topic=topic, url="https://beispiel.de/grundlagen",
+                    url_norm="https://beispiel.de/grundlagen", zweck="korpus")
+    t = {"run_id": run_id, "topic": topic, "runde": 1,
+         "payload": db.j({"quelle_id": qid})}
+    erg1 = await laden.laden({**t})
+    q = db.one("SELECT * FROM quellen WHERE id=?", qid)
+    assert q["status"] == "geladen" and q["snapshot"]
+    inhalt1 = laden.snapshot_lesen(q)
+    erg2 = await laden.laden({**t})  # Resume: kein Neuschreiben
+    assert erg2.daten.get("skip")
+    assert laden.snapshot_lesen(q) == inhalt1
+    assert len(erg1.neue_tasks) >= 1  # Soll-Extraktions-Chunks
+
+
+async def test_laden_leere_quelle_befund():
+    topic = topic_anlegen()
+    run_id = db.insert("runs", topic=topic, status="running")
+    qid = db.insert("quellen", topic=topic, url="https://unbekannt.example/x",
+                    url_norm="https://unbekannt.example/x")
+    erg = await laden.laden({"run_id": run_id, "topic": topic, "runde": 1,
+                             "payload": db.j({"quelle_id": qid})})
+    assert erg.neue_tasks == []
+    assert db.one("SELECT status FROM quellen WHERE id=?", qid)["status"] == "leer"
+    assert db.one("SELECT id FROM befunde WHERE run_id=? AND art='leer'", run_id)