#!/bin/bash
# KAR-80 Spike: Eval-Run für auto-memory-inject Hook
#
# Liest golden-dataset.yaml, simuliert UserPromptSubmit-Hook-Input pro Case,
# prüft ob expected_brain_hits / expected_memory_hits in Hook-Output enthalten sind.
#
# Output: JSON nach runs/<timestamp>.json

set -u

DATASET="/root/aria/evals/hook-memory-injection/golden-dataset.yaml"
HOOK="/root/.claude/hooks/auto-memory-inject.sh"
RUN_ID=$(date +%Y-%m-%d-%H%M%S)
OUT="/root/aria/evals/hook-memory-injection/runs/${RUN_ID}.json"

TP=0; TN=0; FP=0; FN=0

python3 - <<EOF > "$OUT"
import yaml, json, subprocess, time
from pathlib import Path

with open("$DATASET") as f:
    ds = yaml.safe_load(f)

cases = ds["cases"]
results = []
tp = tn = fp = fn = 0
total_latency_ms = 0

for case in cases:
    query = case["input"]["query"]
    expected_brain = case["input"].get("expected_brain_hits", [])
    expected_mem = case["input"].get("expected_memory_hits", [])
    expected_verdict = case["expected"]["verdict"]

    payload = json.dumps({"prompt": query})
    t0 = time.time()
    r = subprocess.run(
        ["bash", "$HOOK"],
        input=payload, capture_output=True, text=True, timeout=10,
    )
    latency_ms = int((time.time() - t0) * 1000)
    total_latency_ms += latency_ms

    output = r.stdout
    short_query = (case["id"] == "case-005-short-trigger")

    if short_query:
        # Hook MUSS skip → output empty
        if not output.strip():
            actual_verdict = "pass"
        else:
            actual_verdict = "fail"
    else:
        # Hook MUSS injizieren mit erwarteten Hits
        all_hits = expected_brain + expected_mem
        missing = [h for h in all_hits if h not in output]
        actual_verdict = "pass" if not missing else "fail"

    match = (actual_verdict == expected_verdict)

    if expected_verdict == "fail":
        if actual_verdict == "fail": tp += 1
        else: fn += 1
    else:
        if actual_verdict == "pass": tn += 1
        else: fp += 1

    results.append({
        "case_id": case["id"],
        "query": query[:60],
        "expected_verdict": expected_verdict,
        "actual_verdict": actual_verdict,
        "match": match,
        "latency_ms": latency_ms,
        "output_bytes": len(output),
        "expected_hits": expected_brain + expected_mem,
        "missing_hits": [h for h in (expected_brain + expected_mem) if h not in output] if not short_query else [],
    })

precision = tp / (tp + fp) if (tp + fp) else None
recall = tp / (tp + fn) if (tp + fn) else None
all_pass = all(r["match"] for r in results)

summary = {
    "run_id": "${RUN_ID}",
    "cases_total": len(cases),
    "tp": tp, "tn": tn, "fp": fp, "fn": fn,
    "precision": precision,
    "recall": recall,
    "all_match": all_pass,
    "avg_latency_ms": int(total_latency_ms / len(cases)),
    "results": results,
}
print(json.dumps(summary, indent=2, ensure_ascii=False))
EOF

echo "Result saved: $OUT"
echo ""
python3 -c "
import json
with open('$OUT') as f: r = json.load(f)
for c in r['results']:
    icon = '✓' if c['match'] else '✗'
    print(f'  {icon} {c[\"case_id\"]:30s} expected={c[\"expected_verdict\"]:5s} got={c[\"actual_verdict\"]:5s} latency={c[\"latency_ms\"]}ms output={c[\"output_bytes\"]}B')
    if c.get('missing_hits'):
        for h in c['missing_hits']: print(f'      MISSING: {h}')
print()
print(f'Summary: TP={r[\"tp\"]} TN={r[\"tn\"]} FP={r[\"fp\"]} FN={r[\"fn\"]}')
print(f'  Precision: {r[\"precision\"]}')
print(f'  Recall:    {r[\"recall\"]}')
print(f'  Avg-Latency: {r[\"avg_latency_ms\"]}ms')
print(f'  All-Match: {r[\"all_match\"]}')
"