#!/usr/bin/env python3
"""Maven free-lesson metadata crawler.

Reads URLs from /root/aria/state/maven/lessons-urls.txt and fetches the
embedded __NEXT_DATA__ JSON. Writes one JSON blob per lesson into
/root/aria/state/maven/lessons-raw/<slug>.json and updates an index file.

Idempotent: skips lessons already on disk. Polite: 1.5s sleep between
requests, configurable concurrency (default 1).

Usage:
    python3 maven-crawl.py [--limit N] [--start N] [--end N] [--workers N]
"""
from __future__ import annotations

import argparse
import json
import re
import sys
import time
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed

import urllib.request

URLS_PATH = Path("/root/aria/state/maven/lessons-urls.txt")
RAW_DIR = Path("/root/aria/state/maven/lessons-raw")
INDEX_PATH = Path("/root/aria/state/maven/lessons-index.jsonl")
ERRORS_PATH = Path("/root/aria/state/maven/errors.jsonl")
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
NEXT_DATA_RE = re.compile(r'__NEXT_DATA__"\s*type="application/json">(\{.*?\})</script>', re.DOTALL)


def slug_from_url(url: str) -> str:
    # https://maven.com/p/<code>/<slug>
    parts = url.rstrip("/").split("/")
    return f"{parts[-2]}--{parts[-1]}"


def extract_core(data: dict) -> dict:
    """Boil down a NEXT_DATA blob into the fields we care about for the index."""
    try:
        pp = data["props"]["pageProps"]
    except (KeyError, TypeError):
        return {"error": "no pageProps"}
    cp = pp.get("contentPage") or {}
    sevt = (cp.get("school_event") or {})
    sections = cp.get("sections") or []
    section_main = sections[0] if sections else {}
    instructor = (section_main.get("instructor_infos") or [{}])[0]
    course = pp.get("course") or {}
    school = pp.get("school") or {}

    return {
        "title": sevt.get("title") or section_main.get("title"),
        "slug": cp.get("slug"),
        "id": cp.get("id"),
        "page_type": pp.get("pageType"),
        "description": section_main.get("topic_desc"),
        "start_datetime": sevt.get("start_datetime"),
        "duration_min": sevt.get("duration_min"),
        "is_recording_public": sevt.get("is_recording_public"),
        "has_internal_recording": sevt.get("has_internal_recording"),
        "instructor_name": instructor.get("name"),
        "instructor_title": instructor.get("title"),
        "instructor_headline": instructor.get("headline"),
        "course_name": course.get("name"),
        "course_slug": course.get("slug"),
        "school_name": school.get("name"),
        "school_slug": school.get("slug"),
        "chapter_count": len(pp.get("videoChapters") or []),
    }


def load_cookies() -> dict:
    """Load Maven cookies from /root/aria/maven-cookies.txt if present.

    Supports Netscape format (tab-separated, 7 fields per line).
    """
    cookies_path = Path("/root/aria/maven-cookies.txt")
    if not cookies_path.exists():
        return {}
    cookies = {}
    for line in cookies_path.read_text().splitlines():
        if line.startswith("#") or not line.strip():
            continue
        parts = line.split("\t")
        if len(parts) >= 7:
            name = parts[5]
            value = parts[6]
            cookies[name] = value
    return cookies


def fetch(url: str, timeout: int = 30) -> tuple[bool, str]:
    headers = {"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"}
    cookies = load_cookies()
    if cookies:
        cookie_header = "; ".join(f"{k}={v}" for k, v in cookies.items())
        headers["Cookie"] = cookie_header
    req = urllib.request.Request(url, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            html = resp.read().decode("utf-8", errors="replace")
            return True, html
    except Exception as e:
        return False, str(e)


def process_one(url: str, sleep_s: float = 1.5) -> dict:
    slug = slug_from_url(url)
    raw_path = RAW_DIR / f"{slug}.json"
    if raw_path.exists():
        return {"url": url, "skipped": True, "slug": slug}

    ok, body = fetch(url)
    if not ok:
        with ERRORS_PATH.open("a") as f:
            f.write(json.dumps({"url": url, "error": body, "ts": time.time()}) + "\n")
        return {"url": url, "error": body}

    m = NEXT_DATA_RE.search(body)
    if not m:
        with ERRORS_PATH.open("a") as f:
            f.write(json.dumps({"url": url, "error": "no NEXT_DATA", "ts": time.time()}) + "\n")
        return {"url": url, "error": "no NEXT_DATA"}

    try:
        data = json.loads(m.group(1))
    except json.JSONDecodeError as e:
        with ERRORS_PATH.open("a") as f:
            f.write(json.dumps({"url": url, "error": f"json: {e}", "ts": time.time()}) + "\n")
        return {"url": url, "error": str(e)}

    core = extract_core(data)
    raw_path.write_text(json.dumps({"url": url, "core": core, "next_data": data}, ensure_ascii=False))
    # Append to index (just the core for fast scanning)
    with INDEX_PATH.open("a") as f:
        f.write(json.dumps({"url": url, "slug": slug, **core}, ensure_ascii=False) + "\n")

    time.sleep(sleep_s)
    return {"url": url, "ok": True, "slug": slug, "title": core.get("title")}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--limit", type=int, default=None)
    ap.add_argument("--start", type=int, default=0)
    ap.add_argument("--end", type=int, default=None)
    ap.add_argument("--workers", type=int, default=1)
    ap.add_argument("--sleep", type=float, default=1.0)
    args = ap.parse_args()

    RAW_DIR.mkdir(parents=True, exist_ok=True)
    urls = URLS_PATH.read_text().splitlines()
    urls = [u for u in urls if u.startswith("https://")]
    end = args.end if args.end is not None else len(urls)
    target = urls[args.start:end]
    if args.limit:
        target = target[: args.limit]
    print(f"# target: {len(target)} urls (slice {args.start}..{end}, limit {args.limit})", file=sys.stderr)

    done = 0
    failed = 0
    skipped = 0
    t0 = time.time()
    if args.workers == 1:
        for i, u in enumerate(target):
            res = process_one(u, sleep_s=args.sleep)
            if res.get("ok"):
                done += 1
            elif res.get("skipped"):
                skipped += 1
            else:
                failed += 1
            if (i + 1) % 20 == 0:
                elapsed = time.time() - t0
                rate = (i + 1) / elapsed
                eta_min = (len(target) - i - 1) / rate / 60
                print(f"# {i+1}/{len(target)} · ok={done} skip={skipped} fail={failed} · {rate:.2f}/s · ETA {eta_min:.1f}m", file=sys.stderr)
    else:
        with ThreadPoolExecutor(max_workers=args.workers) as exe:
            futures = {exe.submit(process_one, u, args.sleep / max(1, args.workers // 2)): u for u in target}
            for fut in as_completed(futures):
                res = fut.result()
                if res.get("ok"):
                    done += 1
                elif res.get("skipped"):
                    skipped += 1
                else:
                    failed += 1
                total = done + failed + skipped
                if total % 20 == 0:
                    elapsed = time.time() - t0
                    rate = total / elapsed
                    eta_min = (len(target) - total) / rate / 60
                    print(f"# {total}/{len(target)} · ok={done} skip={skipped} fail={failed} · {rate:.2f}/s · ETA {eta_min:.1f}m", file=sys.stderr)

    print(f"DONE · ok={done} skipped={skipped} fail={failed} · total elapsed {(time.time()-t0)/60:.1f}m", file=sys.stderr)


if __name__ == "__main__":
    main()