marek/creator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
team3
2026-07-06 14:44:20 +02:00
parent f4c5116abb
commit cc27e53b9e
20 changed files with 504 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
backend/crawl.py
View File

@@ -1,9 +1,9 @@
"""Bounded domain crawler for link sources — renders JS via Playwright (Chromium).
"""Page loader for link sources — renders JS via Playwright (Chromium).
Loads pages + PDFs starting from a start URL — ONLY the same domain, limited depth
and page count. HTML pages are rendered in a headless browser (needed for SPAs), then
links + text are pulled from the finished DOM. PDFs are loaded directly as bytes.
Deterministic, bounded; runs via asyncio.to_thread (sync API, no event loop).
Loads a FIXED list of URLs the user supplied — one page each, NO link following.
HTML pages are rendered in a headless browser (needed for SPAs), then the main text
is pulled from the finished DOM. PDFs are loaded directly as bytes. Deterministic,
bounded; runs via asyncio.to_thread (sync API, no event loop).
"""
import hashlib
@@ -17,8 +17,6 @@ from fsutil import atomic_write_text
log = logging.getLogger("creator.crawl")
MAX_DEPTH = 3
MAX_PAGES = 500
PAGE_TIMEOUT = 30 # seconds per page (render or PDF download)
CRAWL_SETTLE_MS = 3000 # capped settle after domcontentloaded (SPA render); no 30s networkidle hang
MAX_BYTES = 10_000_000 # 10 MB cap per PDF
@@ -33,7 +31,7 @@ def _fetch_bytes(url: str) -> bytes | None:
data = resp.read(MAX_BYTES + 1)
return None if len(data) > MAX_BYTES else data
except Exception as e:
log.debug("crawl: PDF fetch failed %s: %s", url, e)
log.debug("load: PDF fetch failed %s: %s", url, e)
return None
@@ -47,21 +45,6 @@ def _is_pdf(url: str) -> bool:
return url.lower().split("?")[0].rstrip("/").endswith(".pdf")
def _scope_prefix(start_url: str) -> str:
"""First non-empty path segment of the start URL as the crawl scope, e.g.
`/learn/path/x` → `/learn`. No path segment → `""` (whole domain, no narrowing)."""
seg = [s for s in urlparse(start_url).path.split("/") if s]
return f"/{seg[0]}" if seg else ""
def _in_scope(url: str, prefix: str) -> bool:
"""Segment-exact prefix match (no `/learn` ⊃ `/learning-x`). Empty prefix → everything allowed."""
if not prefix:
return True
p = urlparse(url).path
return p == prefix or p.startswith(prefix + "/")
def _page_text(page) -> str:
"""Main text of the rendered page — nav/footer/boilerplate removed via trafilatura.
Falls back to the raw body text when extraction is empty/too short (non-article pages)."""
@@ -78,34 +61,34 @@ def _page_text(page) -> str:
return text.strip()
def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages: int = MAX_PAGES, cancelled=None) -> int:
"""Crawl from start_url (same domain only), render JS and store pages/PDFs in `target`.
def load_pages(urls: list[str], target: Path, *, cancelled=None) -> int:
"""Load each URL in `urls` (one page each, NO link following), render JS and store
pages/PDFs in `target`.
BFS up to `max_depth` / `max_pages`. Errors on individual pages are skipped.
Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it,
so a restart crawls again. Returns the number of saved sources.
Duplicate/fragment-only URLs collapse to one. Errors on individual pages are skipped.
Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it, so a
restart loads again. Returns the number of saved sources.
"""
# Lazy: this way the backend starts even without Playwright installed; only crawling then fails.
# Lazy: this way the backend starts even without Playwright installed; only loading then fails.
from playwright.sync_api import sync_playwright
target.mkdir(parents=True, exist_ok=True)
domain = urlparse(start_url).netloc
prefix = _scope_prefix(start_url) # only follow links under this path segment
seen: set[str] = set()
queue: list[tuple[str, int]] = [(urldefrag(start_url)[0], 0)]
todo: list[str] = []
for u in urls:
nu = urldefrag(u)[0].strip()
if nu and nu not in seen:
seen.add(nu)
todo.append(nu)
saved = 0
with sync_playwright() as pw:
browser = pw.chromium.launch(args=["--no-sandbox"]) # non-root (Docker user app)
page = browser.new_page(user_agent=_UA)
try:
while queue and saved < max_pages:
for url in todo:
if cancelled and cancelled():
return saved # abort → NO .done marker → restart crawls again
url, depth = queue.pop(0)
if url in seen:
continue
seen.add(url)
return saved # abort → NO .done marker → restart loads again
# PDFs need no rendering — load directly.
if _is_pdf(url):
@@ -120,7 +103,7 @@ def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages
try:
page.goto(url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT * 1000)
except Exception as e:
log.debug("crawl: goto incomplete %s: %s", url, e) # still try to read the content
log.debug("load: goto incomplete %s: %s", url, e) # still try to read the content
try:
page.wait_for_load_state("networkidle", timeout=CRAWL_SETTLE_MS)
except Exception:
@@ -129,20 +112,9 @@ def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages
if text:
atomic_write_text(target / _name(url, ".txt"), f"QUELLE: {url}\n\n{text}")
saved += 1
if depth < max_depth:
try:
hrefs = page.eval_on_selector_all("a[href]", "els => els.map(e => e.href)")
except Exception:
hrefs = []
for href in hrefs:
nxt = urldefrag(href)[0]
if (nxt.startswith(("http://", "https://"))
and urlparse(nxt).netloc == domain and _in_scope(nxt, prefix)
and nxt not in seen):
queue.append((nxt, depth + 1))
finally:
browser.close()
(target / ".done").write_text("ok", encoding="utf-8") # ran through cleanly
log.info("crawl %s → %d sources in %s", start_url, saved, target)
log.info("load_pages → %d sources in %s", saved, target)
return saved