init

backend/suche.py

@@ -0,0 +1,200 @@
+"""Such-Schicht (R2): Wikipedia de+en immer, dazu Kette Tavily→Brave→ddgs.
+Circuit Breaker je Backend; Teilausfall = degraded-Befund, nie Lauf-Crash."""
+import asyncio
+import os
+import re
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import httpx
+
+from . import config, db, engine, llm, textkit
+
+_TRACKING = {"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
+             "fbclid", "gclid", "ref"}
+_ausfaelle: dict[str, int] = {}
+_wiki_sem = asyncio.Semaphore(config.WIKI_MAX_PARALLEL)
+
+
+def url_normieren(url: str) -> str:
+    s = urlsplit(url)
+    query = urlencode([(k, v) for k, v in parse_qsl(s.query)
+                       if k.lower() not in _TRACKING])
+    return urlunsplit((s.scheme.lower(), s.netloc.lower(),
+                       s.path.rstrip("/") or "/", query, ""))
+
+
+def _tot(backend: str) -> bool:
+    return _ausfaelle.get(backend, 0) >= config.CIRCUIT_BREAKER_N
+
+
+def _ausfall(backend: str) -> None:
+    _ausfaelle[backend] = _ausfaelle.get(backend, 0) + 1
+
+
+async def _http_json(url: str, **kw) -> dict | list | None:
+    async with httpx.AsyncClient(
+            timeout=config.SUCHE_TIMEOUT_S,
+            headers={"User-Agent": config.user_agent()}) as client:
+        r = await client.get(url, **kw)
+        r.raise_for_status()
+        return r.json()
+
+
+async def _wikipedia(query: str) -> list[dict]:
+    treffer = []
+    for sprache in ("de", "en"):
+        backend = f"wikipedia_{sprache}"
+        if _tot(backend):
+            continue
+        try:
+            async with _wiki_sem:
+                daten = await _http_json(
+                    f"https://{sprache}.wikipedia.org/w/rest.php/v1/search/page",
+                    params={"q": query, "limit": 4})
+            for s in (daten or {}).get("pages", []):
+                treffer.append({
+                    "titel": s.get("title", ""), "backend": backend,
+                    "url": f"https://{sprache}.wikipedia.org/wiki/"
+                           f"{s.get('key', s.get('title', ''))}"})
+            _ausfaelle[backend] = 0
+        except Exception:
+            _ausfall(backend)
+    return treffer
+
+
+async def _tavily(query: str) -> list[dict]:
+    key = os.getenv("TAVILY_API_KEY")
+    if not key:
+        return []
+    async with httpx.AsyncClient(timeout=config.SUCHE_TIMEOUT_S) as client:
+        r = await client.post("https://api.tavily.com/search",
+                              json={"api_key": key, "query": query,
+                                    "max_results": config.SUCHE_MAX_TREFFER})
+        r.raise_for_status()
+        return [{"titel": t.get("title", ""), "url": t["url"], "backend": "tavily"}
+                for t in r.json().get("results", [])]
+
+
+async def _brave(query: str) -> list[dict]:
+    key = os.getenv("BRAVE_API_KEY")
+    if not key:
+        return []
+    daten = await _http_json("https://api.search.brave.com/res/v1/web/search",
+                             params={"q": query, "count": config.SUCHE_MAX_TREFFER,
+                                     "country": "DE", "search_lang": "de"},
+                             headers={"X-Subscription-Token": key})
+    return [{"titel": t.get("title", ""), "url": t["url"], "backend": "brave"}
+            for t in (daten or {}).get("web", {}).get("results", [])]
+
+
+async def _ddgs(query: str) -> list[dict]:
+    def _sync():
+        from ddgs import DDGS
+        return DDGS().text(query, region="de-de",
+                           max_results=config.SUCHE_MAX_TREFFER)
+    await asyncio.sleep(config.DDGS_PAUSE_S)  # Drosselung (ToS-Grauzone)
+    rows = await asyncio.to_thread(_sync)
+    return [{"titel": r.get("title", ""), "url": r.get("href") or r.get("link", ""),
+             "backend": "ddgs"} for r in rows or [] if r.get("href") or r.get("link")]
+
+
+_KETTE = (("tavily", _tavily), ("brave", _brave), ("ddgs", _ddgs))
+
+
+async def web_suchen(query: str) -> tuple[list[dict], list[str]]:
+    """→ (treffer, fehler). Wikipedia immer; Kette bis zum ersten Backend mit
+    Ergebnissen (Fallback statt Fan-out, R2)."""
+    treffer = await _wikipedia(query)
+    fehler = []
+    for name, fn in _KETTE:
+        if _tot(name):
+            continue
+        for versuch in range(config.SUCHE_RETRIES + 1):
+            try:
+                web = await asyncio.wait_for(fn(query), config.SUCHE_TIMEOUT_S * 2)
+                if web:
+                    _ausfaelle[name] = 0
+                    return treffer + web, fehler
+                break  # Backend ohne Key/Treffer → nächstes
+            except Exception as e:
+                if versuch == config.SUCHE_RETRIES:
+                    _ausfall(name)
+                    fehler.append(f"{name}: {type(e).__name__}")
+    return treffer, fehler
+
+
+async def _relevanz_filter(task: dict, thema: str, treffer: list) -> list:
+    """Kurzurteil je Trefferliste: Werbe-/App-Listen und Off-Topic raus
+    (Lektion 91). AUCH Wikipedia-Treffer — Run 7 lud „Pasta" und „Xinjiang"."""
+    web = treffer
+    if len(web) <= 2:
+        return treffer
+    liste = "\n".join(f"[{i}] {t['titel']} — {t['url']}" for i, t in enumerate(web))
+    antwort = await llm.call(
+        run_id=task["run_id"], stufe="korpus", knoten="suche",
+        item=f"{task['item']}:relevanz", role="judge",
+        skill_namen=["richter", "deutsch-praezise", "quellen-relevanz"],
+        werte={"thema": thema, "treffer": liste})
+    if antwort is None:
+        return treffer  # Urteil ausgefallen → lieber behalten als verlieren
+    bloecke = textkit.bloecke(antwort, "RELEVANT")
+    ids = {int(i) for i in (bloecke[0].get("ids", "") if bloecke else "")
+           .replace(" ", "").split(",") if i.isdigit()}
+    return [t for i, t in enumerate(web) if i in ids]
+
+
+@engine.worker("suche.suchen")
+async def suchen(task: dict) -> engine.Ergebnis:
+    payload = db.uj(task["payload"], {})
+    query = payload.get("query", "")
+    zweck = payload.get("zweck", "korpus")
+    seed = bool(payload.get("seed"))
+    if seed:
+        # Wikipedia-Seed: deterministisch de+en zur Topic-Bezeichnung —
+        # umgeht Suche, Relevanz-Judge und Quellen-Deckel (2 Pflichtquellen)
+        from urllib.parse import quote
+        titel = db.one("SELECT COALESCE(NULLIF(titel,''), name) t FROM topics "
+                       "WHERE name=?", task["topic"])["t"]
+        treffer = [{"titel": f"Wikipedia ({s}): {titel}",
+                    "url": f"https://{s}.wikipedia.org/wiki/"
+                           f"{quote(titel.replace(' ', '_'))}",
+                    "backend": f"wikipedia_{s}"} for s in ("de", "en")]
+        fehler = []
+    elif config.FAKE:
+        from . import fakes
+        treffer, fehler = fakes.suche(query), []
+    else:
+        treffer, fehler = await web_suchen(query)
+    # Wikipedia zuerst (kuratierter Grundstock schlägt Blog-Treffer)
+    treffer.sort(key=lambda t: not t["backend"].startswith("wikipedia"))
+    if zweck == "korpus" and treffer and not config.FAKE and not seed:
+        thema = db.one("SELECT COALESCE(NULLIF(titel,''), name) t FROM topics "
+                       "WHERE name=?", task["topic"])["t"]
+        treffer = await _relevanz_filter(task, thema, treffer)
+    # Quellen-Deckel je Topic (0 = unbegrenzt, dann stoppt nur die Sättigung)
+    einstellung = db.one("SELECT quellen_max FROM topics WHERE name=?",
+                         task["topic"]) or {}
+    deckel = einstellung.get("quellen_max")
+    if deckel is None:
+        deckel = config.QUELLEN_MAX
+    neue = []
+    for t in treffer:
+        if zweck == "korpus" and deckel > 0 and not seed:
+            vorhanden = db.one("SELECT COUNT(*) c FROM quellen WHERE topic=? AND "
+                               "zweck='korpus' AND status NOT IN "
+                               "('fehler','leer','robots','braucht_js')",
+                               task["topic"])["c"]
+            if vorhanden >= deckel:
+                break
+        url_norm = url_normieren(t["url"])
+        qid = db.insert("quellen", ignore=True, topic=task["topic"],
+                        titel=t["titel"][:200], url=t["url"], url_norm=url_norm,
+                        backend=t["backend"], runde=task["runde"], zweck=zweck)
+        if qid:
+            neue.append({"knoten": "laden", "item": f"quelle:{qid}",
+                         "payload": {"quelle_id": qid}})
+    if fehler and not treffer:
+        db.insert("befunde", run_id=task["run_id"], stufe="korpus", knoten="suche",
+                  art="degraded", item=query[:80], detail="; ".join(fehler)[:300])
+    return engine.Ergebnis(daten={"treffer": len(treffer), "neu": len(neue)},
+                           neue_tasks=neue)