"""Bounded domain crawler for link sources — renders JS via Playwright (Chromium).

Loads pages + PDFs starting from a start URL — ONLY the same domain, limited depth
and page count. HTML pages are rendered in a headless browser (needed for SPAs), then
links + text are pulled from the finished DOM. PDFs are loaded directly as bytes.
Deterministic, bounded; runs via asyncio.to_thread (sync API, no event loop).
"""

import hashlib
import logging
import re
from pathlib import Path
from urllib.parse import urldefrag, urlparse
from urllib.request import Request, urlopen

from fsutil import atomic_write_text

log = logging.getLogger("creator.crawl")

MAX_DEPTH = 3
MAX_PAGES = 500
PAGE_TIMEOUT = 30         # seconds per page (render or PDF download)
CRAWL_SETTLE_MS = 3000    # capped settle after domcontentloaded (SPA render); no 30s networkidle hang
MAX_BYTES = 10_000_000    # 10 MB cap per PDF
_UA = "Mozilla/5.0 (creator-lernbot)"


def _fetch_bytes(url: str) -> bytes | None:
    """Load PDF bytes via urllib (no rendering needed). None on error/too large."""
    try:
        req = Request(url, headers={"User-Agent": _UA})
        with urlopen(req, timeout=PAGE_TIMEOUT) as resp:
            data = resp.read(MAX_BYTES + 1)
        return None if len(data) > MAX_BYTES else data
    except Exception as e:
        log.debug("crawl: PDF fetch failed %s: %s", url, e)
        return None


def _name(url: str, ext: str) -> str:
    h = hashlib.md5(url.encode("utf-8")).hexdigest()[:8]
    slug = re.sub(r"[^a-zA-Z0-9]+", "-", urlparse(url).path).strip("-")[:60] or "index"
    return f"{h}-{slug}{ext}"


def _is_pdf(url: str) -> bool:
    return url.lower().split("?")[0].rstrip("/").endswith(".pdf")


def _scope_prefix(start_url: str) -> str:
    """First non-empty path segment of the start URL as the crawl scope, e.g.
    `/learn/path/x` → `/learn`. No path segment → `""` (whole domain, no narrowing)."""
    seg = [s for s in urlparse(start_url).path.split("/") if s]
    return f"/{seg[0]}" if seg else ""


def _in_scope(url: str, prefix: str) -> bool:
    """Segment-exact prefix match (no `/learn` ⊃ `/learning-x`). Empty prefix → everything allowed."""
    if not prefix:
        return True
    p = urlparse(url).path
    return p == prefix or p.startswith(prefix + "/")


def _page_text(page) -> str:
    """Main text of the rendered page — nav/footer/boilerplate removed via trafilatura.
    Falls back to the raw body text when extraction is empty/too short (non-article pages)."""
    try:
        from trafilatura import extract  # lazy: the backend starts even without the package
        text = extract(page.content(), include_comments=False, include_tables=True) or ""
    except Exception:
        text = ""
    if len(text.strip()) >= 200:
        return text.strip()
    try:
        return page.inner_text("body").strip()
    except Exception:
        return text.strip()


def crawl(start_url: str, target: Path, *, max_depth: int = MAX_DEPTH, max_pages: int = MAX_PAGES, cancelled=None) -> int:
    """Crawl from start_url (same domain only), render JS and store pages/PDFs in `target`.

    BFS up to `max_depth` / `max_pages`. Errors on individual pages are skipped.
    Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it,
    so a restart crawls again. Returns the number of saved sources.
    """
    # Lazy: this way the backend starts even without Playwright installed; only crawling then fails.
    from playwright.sync_api import sync_playwright

    target.mkdir(parents=True, exist_ok=True)
    domain = urlparse(start_url).netloc
    prefix = _scope_prefix(start_url)  # only follow links under this path segment
    seen: set[str] = set()
    queue: list[tuple[str, int]] = [(urldefrag(start_url)[0], 0)]
    saved = 0

    with sync_playwright() as pw:
        browser = pw.chromium.launch(args=["--no-sandbox"])  # non-root (Docker user app)
        page = browser.new_page(user_agent=_UA)
        try:
            while queue and saved < max_pages:
                if cancelled and cancelled():
                    return saved  # abort → NO .done marker → restart crawls again
                url, depth = queue.pop(0)
                if url in seen:
                    continue
                seen.add(url)

                # PDFs need no rendering — load directly.
                if _is_pdf(url):
                    data = _fetch_bytes(url)
                    if data:
                        p = target / _name(url, ".pdf")
                        if not p.exists():
                            p.write_bytes(data)
                        saved += 1
                    continue

                try:
                    page.goto(url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT * 1000)
                except Exception as e:
                    log.debug("crawl: goto incomplete %s: %s", url, e)  # still try to read the content
                try:
                    page.wait_for_load_state("networkidle", timeout=CRAWL_SETTLE_MS)
                except Exception:
                    pass  # an SPA with constant traffic never reaches idle → continue after settle, no 30s hang
                text = _page_text(page)  # main text, nav/footer removed (fallback: raw body)
                if text:
                    atomic_write_text(target / _name(url, ".txt"), f"QUELLE: {url}\n\n{text}")
                    saved += 1
                if depth < max_depth:
                    try:
                        hrefs = page.eval_on_selector_all("a[href]", "els => els.map(e => e.href)")
                    except Exception:
                        hrefs = []
                    for href in hrefs:
                        nxt = urldefrag(href)[0]
                        if (nxt.startswith(("http://", "https://"))
                                and urlparse(nxt).netloc == domain and _in_scope(nxt, prefix)
                                and nxt not in seen):
                            queue.append((nxt, depth + 1))
        finally:
            browser.close()

    (target / ".done").write_text("ok", encoding="utf-8")  # ran through cleanly
    log.info("crawl %s → %d sources in %s", start_url, saved, target)
    return saved