#!/usr/bin/env python3
"""
aria-eval-ci — KAR-745: Automated Eval CI Runner

Findet alle /root/aria/evals/*/golden-dataset.yaml, faehrt aria-eval-judge.py
gegen jedes Dataset mit passendem judge-prompt.md, berechnet P/R/F1 + Delta
zum letzten Lauf, schreibt Markdown-Report, und exitiert mit != 0 bei Regression.

Run-History: /root/aria/state/eval-ci-history.json
Report:      /root/aria/state/eval-ci-report-<TIMESTAMP>.md

Usage:
  python3 aria-eval-ci.py [--dry-run] [--model MODEL] [--max-cases N]
                          [--regression-threshold FLOAT] [--evals-dir DIR]
"""
from __future__ import annotations

import argparse
import json
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path

ARIA_ROOT = Path("/root/aria")
EVALS_DIR = ARIA_ROOT / "evals"
JUDGE_SCRIPT = ARIA_ROOT / "scripts" / "aria-eval-judge.py"
HISTORY_FILE = ARIA_ROOT / "state" / "eval-ci-history.json"
REPORT_DIR = ARIA_ROOT / "state"

DEFAULT_MODEL = "claude-haiku-4-5-20251001"
DEFAULT_REGRESSION_THRESHOLD = 0.05  # 5 percentage points drop triggers ALERT


# ---------------------------------------------------------------------------
# History helpers
# ---------------------------------------------------------------------------

def load_history() -> dict:
    if HISTORY_FILE.exists():
        try:
            return json.loads(HISTORY_FILE.read_text())
        except (json.JSONDecodeError, OSError):
            pass
    return {"runs": []}


def save_history(history: dict) -> None:
    HISTORY_FILE.parent.mkdir(parents=True, exist_ok=True)
    HISTORY_FILE.write_text(json.dumps(history, indent=2, ensure_ascii=False))


def last_metrics_for_dataset(history: dict, dataset_key: str) -> dict | None:
    """Return the most recent metric snapshot for this dataset key, or None."""
    for run in reversed(history.get("runs", [])):
        for ds in run.get("datasets", []):
            if ds.get("key") == dataset_key:
                return ds.get("metrics")
    return None


# ---------------------------------------------------------------------------
# Dataset discovery
# ---------------------------------------------------------------------------

def find_datasets(evals_dir: Path) -> list[dict]:
    """Return list of {key, dataset_path, judge_prompt_path} dicts."""
    datasets = []
    for ds_path in sorted(evals_dir.glob("*/golden-dataset.yaml")):
        feature_dir = ds_path.parent
        judge_prompt = feature_dir / "judge-prompt.md"
        if not judge_prompt.exists():
            # Skip datasets without a judge-prompt (e.g. hook-memory-injection
            # uses a custom runner script, not aria-eval-judge)
            continue
        datasets.append({
            "key": feature_dir.name,
            "dataset_path": ds_path,
            "judge_prompt_path": judge_prompt,
        })
    return datasets


# ---------------------------------------------------------------------------
# Run judge for one dataset
# ---------------------------------------------------------------------------

def run_judge(ds: dict, model: str, max_cases: int, dry_run: bool) -> dict:
    """
    Invoke aria-eval-judge.py as subprocess.
    Returns a result dict with keys: key, success, metrics, error, runs_path.
    """
    cmd = [
        sys.executable,
        str(JUDGE_SCRIPT),
        "--dataset", str(ds["dataset_path"]),
        "--judge-prompt", str(ds["judge_prompt_path"]),
        "--model", model,
        "--max-cases", str(max_cases),
    ]
    if dry_run:
        cmd.append("--dry-run")

    result = subprocess.run(
        cmd,
        capture_output=True,
        text=True,
    )

    stdout = result.stdout.strip()
    stderr = result.stderr.strip()
    success = result.returncode in (0, 1)  # judge exits 1 for fp+fn>0, 0 for clean

    metrics: dict = {}
    runs_path: str | None = None

    if not dry_run and success:
        # Parse summary line: "[aria-eval-judge] run ID: TP=N TN=N FP=N FN=N ..."
        for line in stdout.splitlines():
            if line.startswith("[aria-eval-judge]"):
                parts = line.split()
                kv = {}
                for token in parts:
                    if "=" in token:
                        k, v = token.split("=", 1)
                        kv[k] = v
                tp = int(kv.get("TP", 0))
                tn = int(kv.get("TN", 0))
                fp = int(kv.get("FP", 0))
                fn = int(kv.get("FN", 0))
                precision_str = kv.get("precision", "None")
                recall_str = kv.get("recall", "None")
                precision = float(precision_str) if precision_str != "None" else None
                recall = float(recall_str) if recall_str != "None" else None
                f1: float | None = None
                if precision is not None and recall is not None:
                    denom = (precision + recall)
                    f1 = (2 * precision * recall / denom) if denom > 0 else 0.0
                metrics = {
                    "tp": tp, "tn": tn, "fp": fp, "fn": fn,
                    "precision": precision,
                    "recall": recall,
                    "f1": f1,
                    "pass_rate": (tp + tn) / (tp + tn + fp + fn) if (tp + tn + fp + fn) > 0 else None,
                }
            if "saved to" in line:
                runs_path = line.split("saved to", 1)[-1].strip()

    return {
        "key": ds["key"],
        "success": success,
        "returncode": result.returncode,
        "metrics": metrics,
        "stdout": stdout,
        "stderr": stderr,
        "runs_path": runs_path,
        "dry_run": dry_run,
    }


# ---------------------------------------------------------------------------
# Delta computation
# ---------------------------------------------------------------------------

def compute_delta(current: dict, previous: dict | None) -> dict:
    """Compute metric deltas vs previous run. Returns empty dict if no previous."""
    if not previous:
        return {}
    delta = {}
    for key in ("precision", "recall", "f1", "pass_rate"):
        cur = current.get(key)
        prev = previous.get(key)
        if cur is not None and prev is not None:
            delta[key] = round(cur - prev, 4)
    return delta


# ---------------------------------------------------------------------------
# Regression check
# ---------------------------------------------------------------------------

def check_regression(
    results: list[dict],
    history: dict,
    threshold: float,
) -> list[str]:
    """Return list of ALERT strings for any metric drops exceeding threshold."""
    alerts = []
    for r in results:
        if r["dry_run"] or not r["success"] or not r["metrics"]:
            continue
        prev = last_metrics_for_dataset(history, r["key"])
        delta = compute_delta(r["metrics"], prev)
        for metric, drop in delta.items():
            if drop < -threshold:
                alerts.append(
                    f"ALERT: [{r['key']}] {metric} dropped by {abs(drop):.1%} "
                    f"(threshold {threshold:.0%}) — prev={r.get('_prev_metrics', {}).get(metric)} "
                    f"curr={r['metrics'].get(metric)}"
                )
    return alerts


# ---------------------------------------------------------------------------
# Markdown report
# ---------------------------------------------------------------------------

def build_report(
    run_id: str,
    results: list[dict],
    history: dict,
    alerts: list[str],
    model: str,
    dry_run: bool,
) -> str:
    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
    lines = [
        f"# Aria Eval-CI Report — {run_id}",
        f"",
        f"**Run:** {now}  **Model:** {model}  **Mode:** {'dry-run' if dry_run else 'live'}",
        f"",
    ]

    if alerts:
        lines.append("## Regressions")
        for a in alerts:
            lines.append(f"- {a}")
        lines.append("")

    lines.append("## Dataset Results")
    lines.append("")
    lines.append("| Dataset | Pass% | Precision | Recall | F1 | Delta P | Delta R | Delta F1 |")
    lines.append("|---------|-------|-----------|--------|----|---------|---------|----------|")

    for r in results:
        key = r["key"]
        if r["dry_run"]:
            lines.append(f"| {key} | dry-run | — | — | — | — | — | — |")
            continue
        if not r["success"]:
            lines.append(f"| {key} | ERROR | — | — | — | — | — | — |")
            continue
        m = r["metrics"]
        prev = last_metrics_for_dataset(history, key)
        delta = compute_delta(m, prev)

        def fmt(v: float | None) -> str:
            return f"{v:.1%}" if v is not None else "—"

        def dfmt(v: float | None) -> str:
            if v is None:
                return "—"
            sign = "+" if v >= 0 else ""
            return f"{sign}{v:.1%}"

        lines.append(
            f"| {key} "
            f"| {fmt(m.get('pass_rate'))} "
            f"| {fmt(m.get('precision'))} "
            f"| {fmt(m.get('recall'))} "
            f"| {fmt(m.get('f1'))} "
            f"| {dfmt(delta.get('precision'))} "
            f"| {dfmt(delta.get('recall'))} "
            f"| {dfmt(delta.get('f1'))} |"
        )

    lines.append("")

    if alerts:
        lines.append("## Alert Details")
        for a in alerts:
            lines.append(f"```")
            lines.append(a)
            lines.append(f"```")
        lines.append("")

    lines.append("## Run Details")
    for r in results:
        lines.append(f"### {r['key']}")
        if r["dry_run"]:
            lines.append("Mode: dry-run (no LLM calls)")
        elif r["success"]:
            m = r["metrics"]
            lines.append(
                f"TP={m.get('tp')} TN={m.get('tn')} FP={m.get('fp')} FN={m.get('fn')}"
            )
            if r.get("runs_path"):
                lines.append(f"Run JSON: `{r['runs_path']}`")
        else:
            lines.append(f"ERROR (exit {r['returncode']})")
            if r.get("stderr"):
                lines.append(f"```\n{r['stderr'][:500]}\n```")
        lines.append("")

    return "\n".join(lines)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main(argv: list[str]) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="List datasets and preview prompts WITHOUT making LLM calls.",
    )
    parser.add_argument("--model", default=DEFAULT_MODEL)
    parser.add_argument("--max-cases", type=int, default=100)
    parser.add_argument(
        "--regression-threshold",
        type=float,
        default=DEFAULT_REGRESSION_THRESHOLD,
        help="Metric drop fraction that triggers ALERT + non-zero exit (default 0.05 = 5pp).",
    )
    parser.add_argument(
        "--evals-dir",
        default=str(EVALS_DIR),
        help="Root directory to scan for golden-dataset.yaml files.",
    )
    args = parser.parse_args(argv)

    evals_dir = Path(args.evals_dir)
    datasets = find_datasets(evals_dir)

    if not datasets:
        print(f"[eval-ci] No datasets with judge-prompt.md found under {evals_dir}", file=sys.stderr)
        return 2

    if args.dry_run:
        print("[eval-ci] DRY RUN — listing datasets (no LLM calls):")
        for ds in datasets:
            print(f"  {ds['key']:40s}  dataset={ds['dataset_path']}")
            print(f"  {'':40s}  judge  ={ds['judge_prompt_path']}")
        print(f"\n[eval-ci] {len(datasets)} dataset(s) found. Pass --dry-run to skip LLM calls.")
        return 0

    # Load history before running (so deltas compare to LAST run, not this one)
    history = load_history()

    run_id = datetime.now().strftime("%Y-%m-%d-%H%M%S")
    print(f"[eval-ci] Starting run {run_id} — {len(datasets)} dataset(s)")

    results: list[dict] = []
    for ds in datasets:
        print(f"  [eval-ci] Running: {ds['key']} ...", end="", flush=True)
        r = run_judge(ds, args.model, args.max_cases, dry_run=False)
        results.append(r)
        if r["success"]:
            m = r["metrics"]
            print(
                f" pass_rate={m.get('pass_rate', 0):.1%} "
                f"P={m.get('precision')} R={m.get('recall')} F1={m.get('f1')}"
            )
        else:
            print(f" ERROR (exit {r['returncode']})")

    # Attach previous metrics for alert messages
    for r in results:
        prev = last_metrics_for_dataset(history, r["key"])
        r["_prev_metrics"] = prev or {}

    # Regression check
    alerts = check_regression(results, history, args.regression_threshold)

    # Build report
    report_md = build_report(run_id, results, history, alerts, args.model, dry_run=False)
    report_path = REPORT_DIR / f"eval-ci-report-{run_id}.md"
    REPORT_DIR.mkdir(parents=True, exist_ok=True)
    report_path.write_text(report_md, encoding="utf-8")
    print(f"\n[eval-ci] Report: {report_path}")

    # Persist this run to history
    run_entry = {
        "run_id": run_id,
        "timestamp": datetime.now(timezone.utc).isoformat(),
        "model": args.model,
        "datasets": [
            {
                "key": r["key"],
                "metrics": r["metrics"],
                "success": r["success"],
            }
            for r in results
        ],
    }
    history.setdefault("runs", []).append(run_entry)
    # Keep last 50 runs in history to avoid unbounded growth
    history["runs"] = history["runs"][-50:]
    save_history(history)

    # Summary print
    passed = sum(1 for r in results if r["success"] and not r.get("dry_run"))
    failed = sum(1 for r in results if not r["success"])
    print(f"[eval-ci] Summary: {passed} dataset(s) OK, {failed} error(s), {len(alerts)} alert(s)")

    if alerts:
        for a in alerts:
            print(a)
        return 1

    return 0 if failed == 0 else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))