#!/usr/bin/env python3 """Aria Brain Search Eval — gegen aria-queries.jsonl. Misst P@5 und R@5 für BM25 und Hybrid (BM25+Vector RRF). Schreibt Markdown-Report nach results/{ISO-date}-eval.md. KAR-187 Phase 2 — First Live-Run. """ from __future__ import annotations import json import re import subprocess import sys from datetime import datetime, timezone, timedelta from pathlib import Path ROOT = Path("/root/aria/eval-suite") QUERIES_FILE = ROOT / "aria-queries.jsonl" SEARCH_CLI = "/root/aria/scripts/aria-brain-search.py" TOP_K = 5 BERLIN = timezone(timedelta(hours=2)) # CEST def search(query: str, hybrid: bool = False, top_k: int = TOP_K) -> list[dict]: args = ["python3", SEARCH_CLI, query, "--top", str(top_k), "--json"] if hybrid: args.append("--hybrid") r = subprocess.run(args, capture_output=True, text=True, timeout=30) if r.returncode != 0: return [] try: return json.loads(r.stdout) except json.JSONDecodeError: return [] def slug_match(hit_path: str, expected_slugs: list[str]) -> bool: """Fuzzy slug-match — Expected slugs are file-stems oder substrings.""" hit_lower = hit_path.lower() for slug in expected_slugs: slug_lower = slug.lower() if slug_lower in hit_lower: return True # File-stem ohne Date-Suffix stem = re.sub(r"-\d{4}-\d{2}-\d{2}", "", Path(hit_path).stem.lower()) if slug_lower in stem or stem in slug_lower: return True return False def precision_recall(hits: list[dict], expected: list[str]) -> tuple[float, float, int]: matched_paths = [h for h in hits if slug_match(h["path"], expected)] matched_distinct_expected = sum( 1 for slug in expected if any(slug_match(h["path"], [slug]) for h in hits) ) p_at_k = len(matched_paths) / max(1, len(hits)) r_at_k = matched_distinct_expected / max(1, len(expected)) return p_at_k, r_at_k, len(matched_paths) def main() -> int: queries = [json.loads(l) for l in QUERIES_FILE.read_text().splitlines() if l.strip()] print(f"# Aria Brain Eval — {len(queries)} Queries") print() bm25_p, bm25_r = [], [] hybrid_p, hybrid_r = [], [] rows = [] for q in queries: text = q["query"] expected = q["expected_slugs"] bm25_hits = search(text, hybrid=False) hybrid_hits = search(text, hybrid=True) bm25_pk, bm25_rk, bm25_match = precision_recall(bm25_hits, expected) hyb_pk, hyb_rk, hyb_match = precision_recall(hybrid_hits, expected) bm25_p.append(bm25_pk); bm25_r.append(bm25_rk) hybrid_p.append(hyb_pk); hybrid_r.append(hyb_rk) rows.append({ "query": text, "category": q.get("category", "general"), "expected": expected, "bm25_top5": [h["path"] for h in bm25_hits], "hybrid_top5": [h["path"] for h in hybrid_hits], "bm25_pk": bm25_pk, "bm25_rk": bm25_rk, "bm25_match": bm25_match, "hyb_pk": hyb_pk, "hyb_rk": hyb_rk, "hyb_match": hyb_match, }) avg = lambda xs: sum(xs) / len(xs) if xs else 0.0 # Markdown Report now = datetime.now(BERLIN) out_path = ROOT / "results" / f"{now.strftime('%Y-%m-%d-%H%M')}-eval.md" out_path.parent.mkdir(parents=True, exist_ok=True) lines = [] lines.append(f"# Aria Brain Eval — {now.strftime('%Y-%m-%d %H:%M %Z')}") lines.append("") lines.append(f"**Queries:** {len(queries)} ") lines.append(f"**Corpus:** Aria-Brain (238 MD) ") lines.append(f"**Backend:** Supabase pgvector + BM25 (KAR-120) ") lines.append(f"**Top-K:** {TOP_K}") lines.append("") lines.append("## Summary") lines.append("") lines.append("| Mode | P@5 | R@5 | Hit-Sum |") lines.append("|---|---|---|---|") lines.append(f"| BM25 | {avg(bm25_p):.1%} | {avg(bm25_r):.1%} | {sum(r['bm25_match'] for r in rows)} |") lines.append(f"| Hybrid (BM25+Vec RRF) | {avg(hybrid_p):.1%} | {avg(hybrid_r):.1%} | {sum(r['hyb_match'] for r in rows)} |") lines.append("") lines.append("## gbrain Baseline (Vergleich)") lines.append("") lines.append("- gbrain BrainBench P@5 ~49.1%, R@5 ~97.9% (anderer Corpus, andere Queries — Hausnummer)") lines.append("") lines.append("## Detail pro Query") lines.append("") for r in rows: lines.append(f"### {r['category']} — `{r['query']}`") lines.append("") lines.append(f"**Expected:** `{', '.join(r['expected'])}`") lines.append("") lines.append(f"**BM25** (P@5={r['bm25_pk']:.0%}, R@5={r['bm25_rk']:.0%}):") for p in r["bm25_top5"]: mark = "✓" if slug_match(p, r["expected"]) else " " lines.append(f"- [{mark}] `{p}`") lines.append("") lines.append(f"**Hybrid** (P@5={r['hyb_pk']:.0%}, R@5={r['hyb_rk']:.0%}):") for p in r["hybrid_top5"]: mark = "✓" if slug_match(p, r["expected"]) else " " lines.append(f"- [{mark}] `{p}`") lines.append("") lines.append("## Interpretation") lines.append("") if avg(hybrid_p) > avg(bm25_p): delta = (avg(hybrid_p) - avg(bm25_p)) * 100 lines.append(f"- Hybrid schlägt BM25 in P@5 um {delta:.1f} pp → Embeddings adden Wert") elif avg(hybrid_p) < avg(bm25_p): delta = (avg(bm25_p) - avg(hybrid_p)) * 100 lines.append(f"- BM25 schlägt Hybrid in P@5 um {delta:.1f} pp → Tuning der RRF-Weights nötig") else: lines.append("- BM25 und Hybrid gleichauf → Embeddings adden wenig (Query-Set könnte zu Keyword-lastig sein)") lines.append("") lines.append("## Was als nächstes") lines.append("") lines.append("- Query-Set auf 50+ erweitern für statistische Robustheit") lines.append("- Bei Hybrid