"""Page loader for link sources — renders JS via Playwright (Chromium).

Loads a FIXED list of URLs the user supplied — one page each, NO link following.
HTML pages are rendered in a headless browser (needed for SPAs), then the main text
is pulled from the finished DOM. PDFs are loaded directly as bytes. Deterministic,
bounded; runs via asyncio.to_thread (sync API, no event loop).
"""

import hashlib
import logging
import re
from pathlib import Path
from urllib.parse import urldefrag, urlparse
from urllib.request import Request, urlopen

from fsutil import atomic_write_text

log = logging.getLogger("creator.crawl")

PAGE_TIMEOUT = 30         # seconds per page (render or PDF download)
CRAWL_SETTLE_MS = 3000    # capped settle after domcontentloaded (SPA render); no 30s networkidle hang
MAX_BYTES = 10_000_000    # 10 MB cap per PDF
_UA = "Mozilla/5.0 (creator-lernbot)"


def _fetch_bytes(url: str) -> bytes | None:
    """Load PDF bytes via urllib (no rendering needed). None on error/too large."""
    try:
        req = Request(url, headers={"User-Agent": _UA})
        with urlopen(req, timeout=PAGE_TIMEOUT) as resp:
            data = resp.read(MAX_BYTES + 1)
        return None if len(data) > MAX_BYTES else data
    except Exception as e:
        log.debug("load: PDF fetch failed %s: %s", url, e)
        return None


def _name(url: str, ext: str) -> str:
    h = hashlib.md5(url.encode("utf-8")).hexdigest()[:8]
    slug = re.sub(r"[^a-zA-Z0-9]+", "-", urlparse(url).path).strip("-")[:60] or "index"
    return f"{h}-{slug}{ext}"


def _is_pdf(url: str) -> bool:
    return url.lower().split("?")[0].rstrip("/").endswith(".pdf")


def _page_text(page) -> str:
    """Main text of the rendered page — nav/footer/boilerplate removed via trafilatura.
    Falls back to the raw body text when extraction is empty/too short (non-article pages)."""
    try:
        from trafilatura import extract  # lazy: the backend starts even without the package
        text = extract(page.content(), include_comments=False, include_tables=True) or ""
    except Exception:
        text = ""
    if len(text.strip()) >= 200:
        return text.strip()
    try:
        return page.inner_text("body").strip()
    except Exception:
        return text.strip()


def load_pages(urls: list[str], target: Path, *, cancelled=None) -> int:
    """Load each URL in `urls` (one page each, NO link following), render JS and store
    pages/PDFs in `target`.

    Duplicate/fragment-only URLs collapse to one. Errors on individual pages are skipped.
    Writes a `.done` marker at the END; an abort (`cancelled()` → True) omits it, so a
    restart loads again. Returns the number of saved sources.
    """
    # Lazy: this way the backend starts even without Playwright installed; only loading then fails.
    from playwright.sync_api import sync_playwright

    target.mkdir(parents=True, exist_ok=True)
    seen: set[str] = set()
    todo: list[str] = []
    for u in urls:
        nu = urldefrag(u)[0].strip()
        if nu and nu not in seen:
            seen.add(nu)
            todo.append(nu)
    saved = 0

    with sync_playwright() as pw:
        browser = pw.chromium.launch(args=["--no-sandbox"])  # non-root (Docker user app)
        page = browser.new_page(user_agent=_UA)
        try:
            for url in todo:
                if cancelled and cancelled():
                    return saved  # abort → NO .done marker → restart loads again

                # PDFs need no rendering — load directly.
                if _is_pdf(url):
                    data = _fetch_bytes(url)
                    if data:
                        p = target / _name(url, ".pdf")
                        if not p.exists():
                            p.write_bytes(data)
                        saved += 1
                    continue

                try:
                    page.goto(url, wait_until="domcontentloaded", timeout=PAGE_TIMEOUT * 1000)
                except Exception as e:
                    log.debug("load: goto incomplete %s: %s", url, e)  # still try to read the content
                try:
                    page.wait_for_load_state("networkidle", timeout=CRAWL_SETTLE_MS)
                except Exception:
                    pass  # an SPA with constant traffic never reaches idle → continue after settle, no 30s hang
                text = _page_text(page)  # main text, nav/footer removed (fallback: raw body)
                if text:
                    atomic_write_text(target / _name(url, ".txt"), f"QUELLE: {url}\n\n{text}")
                    saved += 1
        finally:
            browser.close()

    (target / ".done").write_text("ok", encoding="utf-8")  # ran through cleanly
    log.info("load_pages → %d sources in %s", saved, target)
    return saved