149 lines
6.1 KiB
Python
149 lines
6.1 KiB
Python
"""Bounded domain crawler for link sources — renders JS via Playwright (Chromium).
|
|
|
|
Loads pages + PDFs starting from a start URL — ONLY the same domain, limited depth
|
|
and page count. HTML pages are rendered in a headless browser (needed for SPAs), then
|
|
links + text are pulled from the finished DOM. PDFs are loaded directly as bytes.
|
|
Deterministic, bounded; runs via asyncio.to_thread (sync API, no event loop).
|
|
"""
|
|
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
from pathlib import Path
|
|
from urllib.parse import urldefrag, urlparse
|
|
from urllib.request import Request, urlopen
|
|
|
|
from fsutil import atomic_write_text
|
|
|
|
log = logging.getLogger("creator.crawl")
|
|
|
|
MAX_DEPTH = 3
|
|
MAX_PAGES = 500
|
|
PAGE_TIMEOUT = 30 # seconds per page (render or PDF download)
|
|
CRAWL_SETTLE_MS = 3000 # capped settle after domcontentloaded (SPA render); no 30s networkidle hang
|
|
MAX_BYTES = 10_000_000 # 10 MB cap per PDF
|
|
_UA = "Mozilla/5.0 (creator-lernbot)"
|
|
|
|
|
|
def _fetch_bytes(url: str) -> bytes | None:
|
|
"""Load PDF bytes via urllib (no rendering needed). None on error/too large."""
|
|
try:
|
|
req = Request(url, headers={"User-Agent": _UA})
|
|
with urlopen(req, timeout=PAGE_TIMEOUT) as resp:
|
|
data = resp.read(MAX_BYTES + 1)
|
|
return None if len(data) > MAX_BYTES else data
|
|
except Exception as e:
|
|
log.debug("crawl: PDF fetch failed %s: %s", url, e)
|
|
return None
|
|
|
|
|
|
def _name(url: str, ext: str) -> str:
|
|
h = hashlib.md5(url.encode("utf-8")).hexdigest()[:8]
|
|
slug = re.sub(r"[^a-zA-Z0-9]+", "-", urlparse(url).path).strip("-")[:60] or "index"
|
|
return f"{h}-{slug}{ext}"
|
|
|
|
|
|
def _is_pdf(url: str) -> bool:
|
|
return url.lower().split("?")[0].rstrip("/").endswith(".pdf")
|
|
|
|
|
|
def _scope_prefix(start_url: str) -> str:
|
|
"""First non-empty path segment of the start URL as the crawl scope, e.g.
|
|
`/learn/path/x` → `/learn`. No path segment → `""` (whole domain, no narrowing)."""
|
|
seg = [s for s in urlparse(start_url).path.split("/") if s]
|
|
return f"/{seg[0]}" if seg else ""
|
|
|
|
|
|
def _in_scope(url: str, prefix: str) -> bool:
|
|
"""Segment-exact prefix match (no `/learn` ⊃ `/learning-x`). Empty prefix → everything allowed."""
|
|
if not prefix:
|
|
return True
|
|
p = urlparse(url).path
|
|
return p == prefix or p.startswith(prefix + "/")
|
|
|
|
|
|
def _page_text(page) -> str:
|
|
"""Main text of the rendered page — nav/footer/boilerplate removed via trafilatura.
|
|
Falls back to the raw body text when extraction is empty/too short (non-article pages)."""
|
|
try:
|
|
from trafilatura import extract # lazy: the backend starts even without the package
|
|
text = extract(page.content(), include_comments=False, include_tables=True) or ""
|
|
except Exception:
|
|
text = ""
|
|
if len(text.strip()) >= 200:
|
|
return text.strip()
|
|
try:
|
|
return page.inner_text("body").strip()
|
|
except Exception:
|
|
return text.strip()
|
|
|
|
|
|
def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages: int = MAX_PAGES, cancelled=None) -> int:
|
|
"""Crawl from start_url (same domain only), render JS and store pages/PDFs in `target`.
|
|
|
|
BFS up to `max_depth` / `max_pages`. Errors on individual pages are skipped.
|
|
Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it,
|
|
so a restart crawls again. Returns the number of saved sources.
|
|
"""
|
|
# Lazy: this way the backend starts even without Playwright installed; only crawling then fails.
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
domain = urlparse(start_url).netloc
|
|
prefix = _scope_prefix(start_url) # only follow links under this path segment
|
|
seen: set[str] = set()
|
|
queue: list[tuple[str, int]] = [(urldefrag(start_url)[0], 0)]
|
|
saved = 0
|
|
|
|
with sync_playwright() as pw:
|
|
browser = pw.chromium.launch(args=["--no-sandbox"]) # non-root (Docker user app)
|
|
page = browser.new_page(user_agent=_UA)
|
|
try:
|
|
while queue and saved < max_pages:
|
|
if cancelled and cancelled():
|
|
return saved # abort → NO .done marker → restart crawls again
|
|
url, depth = queue.pop(0)
|
|
if url in seen:
|
|
continue
|
|
seen.add(url)
|
|
|
|
# PDFs need no rendering — load directly.
|
|
if _is_pdf(url):
|
|
data = _fetch_bytes(url)
|
|
if data:
|
|
p = target / _name(url, ".pdf")
|
|
if not p.exists():
|
|
p.write_bytes(data)
|
|
saved += 1
|
|
continue
|
|
|
|
try:
|
|
page.goto(url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT * 1000)
|
|
except Exception as e:
|
|
log.debug("crawl: goto incomplete %s: %s", url, e) # still try to read the content
|
|
try:
|
|
page.wait_for_load_state("networkidle", timeout=CRAWL_SETTLE_MS)
|
|
except Exception:
|
|
pass # an SPA with constant traffic never reaches idle → continue after settle, no 30s hang
|
|
text = _page_text(page) # main text, nav/footer removed (fallback: raw body)
|
|
if text:
|
|
atomic_write_text(target / _name(url, ".txt"), f"QUELLE: {url}\n\n{text}")
|
|
saved += 1
|
|
if depth < max_depth:
|
|
try:
|
|
hrefs = page.eval_on_selector_all("a[href]", "els => els.map(e => e.href)")
|
|
except Exception:
|
|
hrefs = []
|
|
for href in hrefs:
|
|
nxt = urldefrag(href)[0]
|
|
if (nxt.startswith(("http://", "https://"))
|
|
and urlparse(nxt).netloc == domain and _in_scope(nxt, prefix)
|
|
and nxt not in seen):
|
|
queue.append((nxt, depth + 1))
|
|
finally:
|
|
browser.close()
|
|
|
|
(target / ".done").write_text("ok", encoding="utf-8") # ran through cleanly
|
|
log.info("crawl %s → %d sources in %s", start_url, saved, target)
|
|
return saved
|