204 lines
8.1 KiB
Python
204 lines
8.1 KiB
Python
"""Ingestion (R6): Bytes → trafilatura(markdown) → ftfy NFC → Snapshot.
|
|
Normalisierung passiert GENAU EINMAL, danach ist der Snapshot unveränderlich —
|
|
alle Verbatim-Zitate stammen ausschließlich aus dem Snapshot."""
|
|
import asyncio
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import time
|
|
import urllib.robotparser
|
|
from urllib.parse import urlsplit
|
|
|
|
import httpx
|
|
|
|
from . import config, db, engine
|
|
|
|
_robots: dict[str, urllib.robotparser.RobotFileParser | None] = {}
|
|
_letzter_fetch: dict[str, float] = {}
|
|
|
|
|
|
def _text_reparieren(text: str) -> str:
|
|
text = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", " ", text) # VOR ftfy
|
|
try:
|
|
import ftfy
|
|
return ftfy.fix_text(text, normalization="NFC")
|
|
except ImportError:
|
|
import unicodedata
|
|
return unicodedata.normalize("NFC", text)
|
|
|
|
|
|
def _robots_erlaubt(url: str) -> bool:
|
|
domain = urlsplit(url).netloc
|
|
if domain not in _robots:
|
|
rp = urllib.robotparser.RobotFileParser()
|
|
try:
|
|
rp.set_url(f"https://{domain}/robots.txt")
|
|
rp.read()
|
|
_robots[domain] = rp
|
|
except Exception:
|
|
_robots[domain] = None # nicht erreichbar → fail-open
|
|
rp = _robots[domain]
|
|
return rp is None or rp.can_fetch(config.user_agent(), url)
|
|
|
|
|
|
async def _domain_bremse(url: str) -> None:
|
|
domain = urlsplit(url).netloc
|
|
delta = time.monotonic() - _letzter_fetch.get(domain, 0)
|
|
if delta < config.DOMAIN_RATE_S:
|
|
await asyncio.sleep(config.DOMAIN_RATE_S - delta)
|
|
_letzter_fetch[domain] = time.monotonic()
|
|
|
|
|
|
_WIKI_RE = re.compile(r"https?://([a-z]{2})\.wikipedia\.org/wiki/(.+)$")
|
|
|
|
|
|
async def _wikipedia_api(url: str) -> str:
|
|
"""Wikipedia über die Action-API laden (bot-freundlich; die HTML-Seiten
|
|
blockierten per robots.txt die Kernquelle des ersten Echtlaufs)."""
|
|
m = _WIKI_RE.match(url)
|
|
if not m:
|
|
return ""
|
|
sprache, titel = m.group(1), m.group(2).split("#")[0]
|
|
async with httpx.AsyncClient(
|
|
timeout=config.FETCH_READ_S,
|
|
headers={"User-Agent": config.user_agent()}) as client:
|
|
r = await client.get(
|
|
f"https://{sprache}.wikipedia.org/w/api.php",
|
|
params={"action": "query", "prop": "extracts", "explaintext": 1,
|
|
"format": "json", "redirects": 1, "titles": titel})
|
|
r.raise_for_status()
|
|
seiten = r.json().get("query", {}).get("pages", {})
|
|
return "\n\n".join(p.get("extract", "") for p in seiten.values())
|
|
|
|
|
|
async def _fetch(url: str) -> tuple[bytes, str]:
|
|
async with httpx.AsyncClient(
|
|
follow_redirects=True,
|
|
timeout=httpx.Timeout(config.FETCH_READ_S, connect=config.FETCH_CONNECT_S),
|
|
headers={"User-Agent": config.user_agent()},
|
|
transport=httpx.AsyncHTTPTransport(retries=2)) as client:
|
|
r = await client.get(url)
|
|
r.raise_for_status()
|
|
return r.content, r.headers.get("content-type", "")
|
|
|
|
|
|
def _extrahieren(inhalt: bytes) -> str:
|
|
import trafilatura
|
|
return trafilatura.extract(inhalt, output_format="markdown",
|
|
include_tables=True, include_links=False,
|
|
include_images=False) or ""
|
|
|
|
|
|
def snapshot_pfad(topic: str, hash_: str) -> str:
|
|
return str(config.KORPUS_DIR / topic / f"q{hash_}.md")
|
|
|
|
|
|
def snapshot_lesen(quelle: dict) -> str:
|
|
from pathlib import Path
|
|
return Path(quelle["snapshot"]).read_text(encoding="utf-8")
|
|
|
|
|
|
def _atomar_schreiben(pfad: str, daten: bytes) -> None:
|
|
os.makedirs(os.path.dirname(pfad), exist_ok=True)
|
|
tmp = pfad + ".tmp"
|
|
with open(tmp, "wb") as f:
|
|
f.write(daten)
|
|
os.replace(tmp, pfad)
|
|
|
|
|
|
@engine.worker("laden.laden")
|
|
async def laden(task: dict) -> engine.Ergebnis:
|
|
quelle = db.one("SELECT * FROM quellen WHERE id=?",
|
|
db.uj(task["payload"], {}).get("quelle_id"))
|
|
if quelle is None:
|
|
raise RuntimeError("Quelle fehlt")
|
|
if quelle["status"] == "geladen": # Resume
|
|
return engine.Ergebnis(daten={"skip": True},
|
|
neue_tasks=_folge_tasks(task, quelle))
|
|
|
|
if config.FAKE:
|
|
from . import fakes
|
|
text = fakes.laden(quelle["url"])
|
|
roh = b""
|
|
else:
|
|
if quelle["url"].lower().endswith(".pdf"):
|
|
db.update("quellen", "id=?", (quelle["id"],), status="fehler",
|
|
grund="pdf_uebersprungen")
|
|
db.insert("befunde", run_id=task["run_id"], stufe="korpus",
|
|
knoten="laden", art="pdf", item=quelle["url"][:120],
|
|
detail="PDF in v1 übersprungen")
|
|
return engine.Ergebnis(daten={"pdf": True})
|
|
if _WIKI_RE.match(quelle["url"]):
|
|
try:
|
|
text = await _wikipedia_api(quelle["url"])
|
|
except Exception:
|
|
text = ""
|
|
if text:
|
|
text = _text_reparieren(text)
|
|
return _snapshot_ablegen(task, quelle, text, b"")
|
|
# API-Fehler → normaler Weg (inkl. robots-Check) als Fallback
|
|
if not _robots_erlaubt(quelle["url"]):
|
|
db.update("quellen", "id=?", (quelle["id"],), status="robots")
|
|
db.insert("befunde", run_id=task["run_id"], stufe="korpus",
|
|
knoten="laden", art="robots", item=quelle["url"][:120])
|
|
return engine.Ergebnis(daten={"robots": True})
|
|
await _domain_bremse(quelle["url"])
|
|
try:
|
|
roh, _ = await _fetch(quelle["url"])
|
|
except Exception as e:
|
|
db.update("quellen", "id=?", (quelle["id"],), status="fehler",
|
|
grund=f"{type(e).__name__}"[:80])
|
|
return engine.Ergebnis(daten={"fetch_fehler": str(e)[:120]})
|
|
text = _extrahieren(roh)
|
|
|
|
text = _text_reparieren(text)
|
|
return _snapshot_ablegen(task, quelle, text, roh)
|
|
|
|
|
|
def _snapshot_ablegen(task: dict, quelle: dict, text: str,
|
|
roh: bytes) -> engine.Ergebnis:
|
|
if len(text) < config.SNAPSHOT_MIN_ZEICHEN or "enable javascript" in text.lower():
|
|
status = "braucht_js" if text and "javascript" in text.lower() else "leer"
|
|
db.update("quellen", "id=?", (quelle["id"],), status=status)
|
|
db.insert("befunde", run_id=task["run_id"], stufe="korpus", knoten="laden",
|
|
art=status, item=quelle["url"][:120])
|
|
return engine.Ergebnis(daten={status: True})
|
|
|
|
hash_ = hashlib.sha256(text.encode()).hexdigest()[:16]
|
|
dublette = db.one("SELECT id FROM quellen WHERE topic=? AND hash=? AND id!=?",
|
|
task["topic"], hash_, quelle["id"])
|
|
if dublette:
|
|
db.update("quellen", "id=?", (quelle["id"],), status="fehler",
|
|
grund="inhalt_dublette")
|
|
return engine.Ergebnis(daten={"dublette": True})
|
|
pfad = snapshot_pfad(task["topic"], hash_)
|
|
_atomar_schreiben(pfad, text.encode("utf-8"))
|
|
roh_pfad = ""
|
|
if roh:
|
|
roh_pfad = pfad.replace(".md", ".html")
|
|
_atomar_schreiben(roh_pfad, roh) # Re-Extraktion ohne Re-Fetch (R6)
|
|
db.update("quellen", "id=?", (quelle["id"],), status="geladen", hash=hash_,
|
|
snapshot=pfad, roh=roh_pfad)
|
|
quelle = {**quelle, "snapshot": pfad, "status": "geladen"}
|
|
return engine.Ergebnis(daten={"zeichen": len(text)},
|
|
neue_tasks=_folge_tasks(task, quelle))
|
|
|
|
|
|
def _folge_tasks(task: dict, quelle: dict) -> list:
|
|
"""korpus-Quellen → Soll-Extraktion je 40k-Chunk; Lücken-Quellen →
|
|
direkte Atom-Extraktion (Soll steht schon fest)."""
|
|
from . import textkit
|
|
if quelle["zweck"].startswith("luecke:"):
|
|
soll_id = int(quelle["zweck"].split(":")[1])
|
|
text = snapshot_lesen(quelle)
|
|
return [{"knoten": "atom_extraktion",
|
|
"item": f"quelle:{quelle['id']}:a{i}:r1",
|
|
"payload": {"quelle_id": quelle["id"], "offset": o,
|
|
"chars": len(c), "soll_id": soll_id}}
|
|
for i, (o, c) in enumerate(textkit.abschnitte(text))]
|
|
text = snapshot_lesen(quelle)
|
|
chunks = textkit.abschnitte(text, config.SOLL_CHUNK_CHARS)
|
|
return [{"knoten": "soll_extraktion", "item": f"quelle:{quelle['id']}:c{i}",
|
|
"payload": {"quelle_id": quelle["id"], "offset": o, "chars": len(c)}}
|
|
for i, (o, c) in enumerate(chunks)]
|