#!/usr/bin/env python3
"""KAR-758 — Knowledge-Gap-Scanner (Demand-Side-Audit).

Misst die DEMAND-Seite: was wurde im Brain gesucht, aber NICHT (gut) gefunden?
Statt supply-getrieben alles in den Context zu stopfen (IKEA/Raj-Navakoti-Pattern,
KAR-478), zieht das hier ein echtes Pull-Signal aus realen Queries.

Query-Quellen (in Reihenfolge):
  1. /root/aria/logs/brain-queries.log  (dediziert, von aria-active-memory.sh, KAR-758)
  2. Fallback: `[active-memory query: ...]`-Zeilen aus den JSONL-Transkripten.

Pro Query: aria-brain-search.py laufen lassen, Top-Treffer-Score bewerten.
Schwacher/kein Treffer -> Gap (critical/high/medium). Output: knowledge-gaps.md.

BM25-Scores sind laengenabhaengig -> Thresholds sind HEURISTISCH + per Flag tunebar.
Pure stdlib. Read-only ausser Report-Write.
"""
from __future__ import annotations
import argparse
import glob
import json
import os
import re
import subprocess
from collections import OrderedDict
from datetime import datetime, timezone
from pathlib import Path

QUERY_LOG = Path("/root/aria/logs/brain-queries.log")
TRANSCRIPT_GLOB = "/root/.claude/projects/-/*.jsonl"
SEARCH = "/root/aria/scripts/aria-brain-search.py"

# Heuristische BM25-Top-Score-Schwellen (tunebar via --crit/--high)
CRIT = 2.0   # darunter (oder 0 Treffer): Brain hat quasi nichts
HIGH = 5.0   # darunter: nur schwache Treffer
MED = 8.0    # darunter: mittelmaessige Abdeckung

CHANNEL_RE = re.compile(r'<channel[^>]*>(.*?)</channel>', re.S)
AM_RE = re.compile(r'\[active-memory query:\s*(.+?)\]')
TOPSCORE_RE = re.compile(r'^\s*1\.\s*\[([\d.]+)\]', re.M)


def _clean(q: str) -> str:
    q = CHANNEL_RE.sub(r'\1', q)          # Telegram-Wrapper entfernen
    q = re.sub(r'<[^>]+>', ' ', q)        # Rest-Tags
    q = re.sub(r'\s+', ' ', q).strip()
    return q


def queries_from_log(limit: int) -> list[str]:
    if not QUERY_LOG.exists():
        return []
    lines = QUERY_LOG.read_text(encoding="utf-8", errors="ignore").splitlines()
    out = []
    for ln in lines[-limit * 3:]:
        # Format: "<ts>\t<query>"
        parts = ln.split("\t", 1)
        out.append(parts[1] if len(parts) == 2 else ln)
    return out


def queries_from_transcripts(limit: int) -> list[str]:
    out = []
    files = sorted(glob.glob(TRANSCRIPT_GLOB), key=os.path.getmtime, reverse=True)
    for f in files[:25]:
        try:
            for line in open(f, encoding="utf-8", errors="ignore"):
                for m in AM_RE.finditer(line):
                    out.append(m.group(1))
        except OSError:
            continue
    return out


def dedup_clean(raw: list[str], limit: int) -> list[str]:
    seen = OrderedDict()
    for q in raw:
        c = _clean(q)
        if len(c) < 8 or len(c) > 300:
            continue
        if c.lower() in ("ok", "los", "weiter", "ja", "ja weiter"):
            continue
        # natuerliche Sprache verlangen: >=2 Wort-Tokens len>=3
        words = re.findall(r"[A-Za-zÄÖÜäöüß]{3,}", c)
        if len(words) < 2:
            continue
        # Regex-/Code-Fragmente raus: zu viele Nicht-Alnum-Zeichen
        nonword = sum(1 for ch in c if not (ch.isalnum() or ch.isspace()))
        if nonword > len(c) * 0.3:
            continue
        seen.setdefault(c.lower(), c)
    return list(seen.values())[-limit:]


def top_score(query: str) -> tuple[float, str]:
    try:
        r = subprocess.run(["python3", SEARCH, query, "--top", "1"],
                           capture_output=True, text=True, timeout=30)
        out = r.stdout
        m = TOPSCORE_RE.search(out)
        score = float(m.group(1)) if m else 0.0
        # erste Treffer-Pfadzeile
        pm = re.search(r'^\s*1\.\s*\[[\d.]+\]\s*(.+)$', out, re.M)
        path = pm.group(1).strip() if pm else "(kein Treffer)"
        return score, path
    except (subprocess.SubprocessError, ValueError):
        return -1.0, "(scan-fehler)"


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--limit", type=int, default=60, help="max Queries")
    ap.add_argument("--out", default=None)
    ap.add_argument("--crit", type=float, default=CRIT)
    ap.add_argument("--high", type=float, default=HIGH)
    ap.add_argument("--med", type=float, default=MED)
    args = ap.parse_args()

    raw = queries_from_log(args.limit)
    source = "brain-queries.log"
    if not raw:
        raw = queries_from_transcripts(args.limit)
        source = "JSONL-Transkripte (active-memory query)"
    queries = dedup_clean(raw, args.limit)

    # Roh-BM25 skaliert mit Query-Laenge -> normalisieren auf Score-pro-Wort,
    # dann RELATIV ranken (schwaechst-abgedeckte Queries = Gaps). Absolut-Floor
    # zusaetzlich: 0 Treffer / sehr niedriger Norm-Score immer critical.
    recs = []
    for q in queries:
        s, path = top_score(q)
        nwords = max(1, len(re.findall(r"[A-Za-zÄÖÜäöüß]{3,}", q)))
        norm = round(s / nwords, 3)
        recs.append({"q": q, "score": round(s, 2), "norm": norm, "top": path})
    ranked = sorted(recs, key=lambda r: r["norm"])
    n = len(ranked)
    crit, high, med = [], [], []
    for i, r in enumerate(ranked):
        pctile = i / n if n else 1.0
        if r["score"] <= 0 or pctile < 0.10:
            crit.append(r)
        elif pctile < 0.25:
            high.append(r)
        elif pctile < 0.45:
            med.append(r)

    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
    L = [f"# Knowledge-Gaps (Demand-Side-Audit, KAR-758)\n",
         f"> Generiert {now}. Quelle: {source}. {len(queries)} unique Queries gescannt.",
         f"> Signal: BM25-Top-Score / Query-Wort (laengen-normalisiert), dann RELATIV gerankt — "
         f"critical = 0 Treffer oder schwächste 10%, high = bis 25%, medium = bis 45%.",
         f"> Niedriger Norm-Score = Brain deckt diese Query relativ schlecht ab → Kandidat für neue Note/Skill.\n"]

    def block(title, items):
        L.append(f"## {title} ({len(items)})\n")
        if not items:
            L.append("_keine_\n")
            return
        L.append("| Query | Norm | Top-Score | Bester Treffer |")
        L.append("|---|---|---|---|")
        for r in sorted(items, key=lambda x: x["norm"]):
            q = r["q"].replace("|", "\\|")[:80]
            top = r["top"].replace("|", "\\|")[:60]
            L.append(f"| {q} | {r['norm']} | {r['score']} | {top} |")
        L.append("")

    block("🔴 Critical — kaum/keine Abdeckung", crit)
    block("🟠 High — nur schwache Treffer", high)
    block("🟡 Medium — mittelmäßige Abdeckung", med)
    L.append("## Nutzung\n")
    L.append("Critical/High → Input für AKP-Triage: gezielt Brain-Notes/Skills für diese Themen anlegen. "
             "Re-Scan nach Ingestion zeigt, ob die Lücke geschlossen ist.")
    report = "\n".join(L) + "\n"

    if args.out:
        Path(args.out).write_text(report, encoding="utf-8")
        print(f"Report: {args.out} — {len(queries)} Queries, {len(crit)} crit / {len(high)} high / {len(med)} med")
    else:
        print(report)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())