#!/usr/bin/env python3
"""
Aria Knowledge Pipeline (AKP) — RSS-Connector Stage 1 (KAR-83)

Liest RSS-Feed-Liste aus brain/05-Referenzen/rss-feed-subscriptions.md,
holt neue Entries via feedparser, filtert per Cheap-Filter, schreibt
JSON nach brain/akp/raw/rss/<feed-slug>/<entry-id>.json, trackt State
in shared brain/youtube/.state.sqlite (source='rss').

Reuse: Stage 2 Triage + Stage 3 Deep + Stage 4 Briefing — alles
source-agnostic via raw_path-Lookup.
"""
from __future__ import annotations
import sys as _sys
_sys.path.insert(0, "/root/aria/lib")
from aria_logging import get_logger as _get_logger
_log = _get_logger("aria-akp-rss-ingest")

import hashlib, json, os, re, sqlite3, sys, time
from datetime import datetime, timezone, timedelta
from pathlib import Path

import feedparser
import yaml

CONFIG_PATH = Path("/root/aria/brain/youtube/.config.yaml")
FEED_LIST_PATH = Path("/root/aria/brain/05-Referenzen/rss-feed-subscriptions.md")
RSS_RAW_DIR = Path("/root/aria/brain/akp/raw/rss")


def load_config() -> dict:
    with CONFIG_PATH.open() as f:
        return yaml.safe_load(f)


def db_conn(db_path: str) -> sqlite3.Connection:
    con = sqlite3.connect(db_path)
    con.row_factory = sqlite3.Row
    return con


def parse_feeds(memory_path: Path) -> list[str]:
    """Read RSS-feed URLs from markdown memory file. Returns list of URLs."""
    out: list[str] = []
    url_re = re.compile(r"^-\s+(https?://\S+)")
    if not memory_path.exists():
        return []
    for line in memory_path.read_text().splitlines():
        m = url_re.match(line.strip())
        if m:
            out.append(m.group(1).rstrip(",;)"))
    # dedupe preserving order
    seen = set()
    deduped = []
    for u in out:
        if u in seen:
            continue
        seen.add(u)
        deduped.append(u)
    return deduped


def feed_slug(url: str) -> str:
    """Stable slug from URL: hostname + path-tail."""
    s = re.sub(r"https?://", "", url)
    s = re.sub(r"[^\w]+", "-", s).strip("-").lower()
    return s[:60]


def entry_id(entry) -> str:
    """Stable ID per entry: prefer guid/id, fallback to hash of link+title."""
    for key in ("id", "guid"):
        if hasattr(entry, key) and getattr(entry, key):
            return hashlib.sha1(getattr(entry, key).encode()).hexdigest()[:16]
    link = getattr(entry, "link", "") or ""
    title = getattr(entry, "title", "") or ""
    return hashlib.sha1((link + title).encode()).hexdigest()[:16]


def entry_published_at(entry) -> datetime | None:
    """Parse published_parsed or updated_parsed into datetime."""
    for key in ("published_parsed", "updated_parsed"):
        if hasattr(entry, key) and getattr(entry, key):
            try:
                return datetime(*getattr(entry, key)[:6], tzinfo=timezone.utc)
            except (TypeError, ValueError):
                pass
    return None


def cheap_filter(entry, cfg: dict, content: str) -> str | None:
    """Return skip-reason or None if pass."""
    # Age: skip wenn > 14 Tage alt (RSS-Feeds enthalten oft alte Items)
    pub = entry_published_at(entry)
    if pub and (datetime.now(timezone.utc) - pub) > timedelta(days=14):
        return "too_old"
    # Content-length: skip wenn < 500 chars (zu kurz für Wissenswert)
    if len(content.strip()) < 500:
        return "too_short_content"
    # Title-Blocklist: gleiche Regex wie YouTube reuse
    title = getattr(entry, "title", "") or ""
    blocklist = cfg["ingest"]["title_blocklist_regex"]
    if blocklist and re.search(blocklist, title, re.IGNORECASE):
        return "title_blocklist"
    return None


def already_ingested(con: sqlite3.Connection, eid: str) -> bool:
    cur = con.execute("SELECT 1 FROM ingested WHERE video_id=?", (eid,))
    return cur.fetchone() is not None


def store_skipped(con: sqlite3.Connection, eid: str, feed_slug_str: str, title: str, reason: str, url: str) -> None:
    con.execute(
        "INSERT OR IGNORE INTO ingested (video_id, channel, title, url, skipped_reason, source) VALUES (?,?,?,?,?,'rss')",
        (eid, feed_slug_str, title[:200], url, reason),
    )
    con.commit()


def store_ingested(con: sqlite3.Connection, eid: str, feed_slug_str: str, title: str,
                   url: str, raw_path: str, content_chars: int, published_at: str) -> None:
    con.execute(
        "INSERT OR REPLACE INTO ingested (video_id, channel, channel_handle, title, "
        "duration_seconds, published_at, url, transcript_chars, raw_path, source) "
        "VALUES (?,?,?,?,?,?,?,?,?,'rss')",
        (eid, feed_slug_str, feed_slug_str, title[:300], None, published_at, url,
         content_chars, raw_path),
    )
    con.commit()


def write_raw(eid: str, feed_slug_str: str, entry, content: str) -> str:
    out_dir = RSS_RAW_DIR / feed_slug_str
    out_dir.mkdir(parents=True, exist_ok=True)
    out_path = out_dir / f"{eid}.json"
    pub = entry_published_at(entry)
    payload = {
        "video_id": eid,
        "channel_slug": feed_slug_str,
        "channel_handle": feed_slug_str,
        "title": getattr(entry, "title", "") or "",
        "url": getattr(entry, "link", "") or "",
        "upload_date": pub.strftime("%Y%m%d") if pub else "",
        "published_at": pub.isoformat() if pub else "",
        "transcript": content,  # Stage 2/3 erwarten "transcript" key — reuse same shape
        "transcript_chars": len(content),
        "ingested_at": datetime.now(timezone.utc).isoformat(),
        "source": "rss",
        "yt_meta": {
            "author": getattr(entry, "author", None) if hasattr(entry, "author") else None,
            "tags": [t.get("term", "") for t in getattr(entry, "tags", [])][:20] if hasattr(entry, "tags") else None,
        },
    }
    out_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False))
    return str(out_path)


def extract_content(entry) -> str:
    """Get the longest content field: content[0].value > summary > description > title.

    With ARIA_USE_DEFUDDLE=1 set, html is routed through `aria_lib.defuddle_clean`
    which returns clean Markdown instead of stripped plain text. Saves
    ~30-50% tokens downstream when the content is article-shaped.
    """
    if hasattr(entry, "content") and entry.content:
        c = entry.content[0]
        if isinstance(c, dict) and "value" in c:
            return clean_content(c["value"])
    for key in ("summary", "description", "summary_detail"):
        v = getattr(entry, key, None)
        if isinstance(v, dict) and "value" in v:
            return clean_content(v["value"])
        if isinstance(v, str) and v:
            return clean_content(v)
    return getattr(entry, "title", "") or ""


def clean_content(html: str) -> str:
    """Route through defuddle if opted in (ARIA_USE_DEFUDDLE=1), else naive strip.

    Threshold: only defuddle entries >2KB — short feed snippets gain
    nothing from the heavier path.
    """
    if os.environ.get("ARIA_USE_DEFUDDLE") == "1" and len(html) > 2048:
        try:
            from aria_lib.defuddle_clean import clean_html as _clean
            out = _clean(html)
            if out.get("ok") and out.get("markdown"):
                return out["markdown"]
        except Exception:
            pass
    return strip_html(html)


def strip_html(html: str) -> str:
    """Naive HTML-strip fallback — fast, no external dependency."""
    text = re.sub(r"<script\b[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r"<style\b[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"&\w+;", " ", text)
    text = re.sub(r"\s+", " ", text)
    return text.strip()


def log_run_start(con: sqlite3.Connection) -> int:
    cur = con.execute("INSERT INTO run_log (stage, status) VALUES ('ingest-rss', 'running')")
    con.commit()
    return cur.lastrowid


def log_run_finish(con: sqlite3.Connection, run_id: int, processed: int, skipped: int, failed: int, notes: str = "") -> None:
    con.execute(
        "UPDATE run_log SET finished_at=datetime('now'), status='completed', "
        "items_processed=?, items_skipped=?, items_failed=?, notes=? WHERE id=?",
        (processed, skipped, failed, notes, run_id),
    )
    con.commit()


def main(argv: list[str]) -> int:
    cfg = load_config()
    con = db_conn(cfg["paths"]["state_db"])
    feeds = parse_feeds(FEED_LIST_PATH)
    if not feeds:
        print("[akp-rss] no feeds in", FEED_LIST_PATH)
        return 0

    only = None
    if "--only" in argv:
        idx = argv.index("--only")
        if idx + 1 < len(argv):
            only = argv[idx + 1]

    max_entries_per_feed = 20
    if "--backfill" in argv:
        max_entries_per_feed = 50

    run_id = log_run_start(con)
    processed = 0
    skipped = 0
    failed = 0
    notes_lines = []

    for url in feeds:
        slug = feed_slug(url)
        if only and only not in slug:
            continue
        try:
            parsed = feedparser.parse(url)
        except Exception as e:
            notes_lines.append(f"{slug}: parse_error: {str(e)[:80]}")
            failed += 1
            continue
        if parsed.bozo and not parsed.entries:
            notes_lines.append(f"{slug}: bozo+no-entries")
            failed += 1
            continue
        for entry in parsed.entries[:max_entries_per_feed]:
            eid = entry_id(entry)
            if already_ingested(con, eid):
                continue
            content = extract_content(entry)
            reason = cheap_filter(entry, cfg, content)
            if reason:
                store_skipped(con, eid, slug, getattr(entry, "title", ""), reason, getattr(entry, "link", ""))
                skipped += 1
                continue
            try:
                raw_path = write_raw(eid, slug, entry, content)
            except Exception as e:
                failed += 1
                notes_lines.append(f"{slug}/{eid}: write_error: {str(e)[:80]}")
                continue
            pub = entry_published_at(entry)
            store_ingested(con, eid, slug, getattr(entry, "title", ""),
                           getattr(entry, "link", ""), raw_path, len(content),
                           pub.isoformat() if pub else "")
            processed += 1
            time.sleep(0.05)

    log_run_finish(con, run_id, processed, skipped, failed, "; ".join(notes_lines)[:500])
    print(f"[akp-rss] processed={processed} skipped={skipped} failed={failed} feeds={len(feeds)}")
    if notes_lines:
        for n in notes_lines[:5]:
            print(f"  NOTE: {n}")
    con.close()
    return 0


if __name__ == "__main__":
    _log.event("script_start")
    sys.exit(main(sys.argv[1:]))