121 lines
4.7 KiB
Python
121 lines
4.7 KiB
Python
"""Page loader for link sources — renders JS via Playwright (Chromium).
|
|
|
|
Loads a FIXED list of URLs the user supplied — one page each, NO link following.
|
|
HTML pages are rendered in a headless browser (needed for SPAs), then the main text
|
|
is pulled from the finished DOM. PDFs are loaded directly as bytes. Deterministic,
|
|
bounded; runs via asyncio.to_thread (sync API, no event loop).
|
|
"""
|
|
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
from pathlib import Path
|
|
from urllib.parse import urldefrag, urlparse
|
|
from urllib.request import Request, urlopen
|
|
|
|
from fsutil import atomic_write_text
|
|
|
|
log = logging.getLogger("creator.crawl")
|
|
|
|
PAGE_TIMEOUT = 30 # seconds per page (render or PDF download)
|
|
CRAWL_SETTLE_MS = 3000 # capped settle after domcontentloaded (SPA render); no 30s networkidle hang
|
|
MAX_BYTES = 10_000_000 # 10 MB cap per PDF
|
|
_UA = "Mozilla/5.0 (creator-lernbot)"
|
|
|
|
|
|
def _fetch_bytes(url: str) -> bytes | None:
|
|
"""Load PDF bytes via urllib (no rendering needed). None on error/too large."""
|
|
try:
|
|
req = Request(url, headers={"User-Agent": _UA})
|
|
with urlopen(req, timeout=PAGE_TIMEOUT) as resp:
|
|
data = resp.read(MAX_BYTES + 1)
|
|
return None if len(data) > MAX_BYTES else data
|
|
except Exception as e:
|
|
log.debug("load: PDF fetch failed %s: %s", url, e)
|
|
return None
|
|
|
|
|
|
def _name(url: str, ext: str) -> str:
|
|
h = hashlib.md5(url.encode("utf-8")).hexdigest()[:8]
|
|
slug = re.sub(r"[^a-zA-Z0-9]+", "-", urlparse(url).path).strip("-")[:60] or "index"
|
|
return f"{h}-{slug}{ext}"
|
|
|
|
|
|
def _is_pdf(url: str) -> bool:
|
|
return url.lower().split("?")[0].rstrip("/").endswith(".pdf")
|
|
|
|
|
|
def _page_text(page) -> str:
|
|
"""Main text of the rendered page — nav/footer/boilerplate removed via trafilatura.
|
|
Falls back to the raw body text when extraction is empty/too short (non-article pages)."""
|
|
try:
|
|
from trafilatura import extract # lazy: the backend starts even without the package
|
|
text = extract(page.content(), include_comments=False, include_tables=True) or ""
|
|
except Exception:
|
|
text = ""
|
|
if len(text.strip()) >= 200:
|
|
return text.strip()
|
|
try:
|
|
return page.inner_text("body").strip()
|
|
except Exception:
|
|
return text.strip()
|
|
|
|
|
|
def load_pages(urls: list[str], target: Path, *, cancelled=None) -> int:
|
|
"""Load each URL in `urls` (one page each, NO link following), render JS and store
|
|
pages/PDFs in `target`.
|
|
|
|
Duplicate/fragment-only URLs collapse to one. Errors on individual pages are skipped.
|
|
Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it, so a
|
|
restart loads again. Returns the number of saved sources.
|
|
"""
|
|
# Lazy: this way the backend starts even without Playwright installed; only loading then fails.
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
seen: set[str] = set()
|
|
todo: list[str] = []
|
|
for u in urls:
|
|
nu = urldefrag(u)[0].strip()
|
|
if nu and nu not in seen:
|
|
seen.add(nu)
|
|
todo.append(nu)
|
|
saved = 0
|
|
|
|
with sync_playwright() as pw:
|
|
browser = pw.chromium.launch(args=["--no-sandbox"]) # non-root (Docker user app)
|
|
page = browser.new_page(user_agent=_UA)
|
|
try:
|
|
for url in todo:
|
|
if cancelled and cancelled():
|
|
return saved # abort → NO .done marker → restart loads again
|
|
|
|
# PDFs need no rendering — load directly.
|
|
if _is_pdf(url):
|
|
data = _fetch_bytes(url)
|
|
if data:
|
|
p = target / _name(url, ".pdf")
|
|
if not p.exists():
|
|
p.write_bytes(data)
|
|
saved += 1
|
|
continue
|
|
|
|
try:
|
|
page.goto(url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT * 1000)
|
|
except Exception as e:
|
|
log.debug("load: goto incomplete %s: %s", url, e) # still try to read the content
|
|
try:
|
|
page.wait_for_load_state("networkidle", timeout=CRAWL_SETTLE_MS)
|
|
except Exception:
|
|
pass # an SPA with constant traffic never reaches idle → continue after settle, no 30s hang
|
|
text = _page_text(page) # main text, nav/footer removed (fallback: raw body)
|
|
if text:
|
|
atomic_write_text(target / _name(url, ".txt"), f"QUELLE: {url}\n\n{text}")
|
|
saved += 1
|
|
finally:
|
|
browser.close()
|
|
|
|
(target / ".done").write_text("ok", encoding="utf-8") # ran through cleanly
|
|
log.info("load_pages → %d sources in %s", saved, target)
|
|
return saved
|