#!/usr/bin/env python3
"""qmd vs aria-brain Hybrid-Search-Benchmark (KAR-594).

Misst hit@5 (taucht eine erwartete Note in Top-5 auf?) + Latenz pro Engine.
Engines: aria-brain --hybrid (BM25+OpenAI-vec+RRF) vs qmd query (BM25+local-vec+rerank)
         + qmd vsearch (vektor-only) als Referenz.
"""
import json, re, subprocess, time, os, sys
from pathlib import Path

TRIAL = Path("/root/aria/tmp/qmd-trial")
QMD = str(TRIAL / "node_modules/.bin/qmd")
ARIA_SEARCH = "/root/aria/scripts/aria-brain-search.py"
TESTSET = json.loads((TRIAL / "testset.json").read_text())

env = dict(os.environ)
for ln in Path("/root/aria/.env").read_text().splitlines():
    if "=" in ln and not ln.strip().startswith("#"):
        k, v = ln.split("=", 1)
        env[k.strip()] = v.strip().strip('"').strip("'")


def hit(paths, expect):
    low = [p.lower() for p in paths]
    return any(any(e.lower() in p for p in low) for e in expect)


def run_aria(q):
    t = time.time()
    r = subprocess.run(["python3", ARIA_SEARCH, q, "--hybrid", "--top", "5", "--json"],
                       capture_output=True, text=True, env=env, timeout=90)
    dt = time.time() - t
    try:
        rows = json.loads(r.stdout)
        return [row["path"] for row in rows][:5], dt
    except Exception:
        return [], dt


def parse_qmd(out):
    paths = []
    for m in re.finditer(r"qmd://([^\s:]+)", out):
        p = m.group(1)
        if p not in paths:
            paths.append(p)
    return paths[:5]


def run_qmd(q, mode):
    t = time.time()
    r = subprocess.run([QMD, mode, q], capture_output=True, text=True, env=env, timeout=120, cwd=str(TRIAL))
    dt = time.time() - t
    return parse_qmd(r.stdout), dt


def main():
    # qmd query (hybrid+rerank) ist auf CPU unbrauchbar (>240s/Query, Timeout) → ausgeschlossen.
    # Verglichen: aria-brain hybrid (BM25+OpenAI-vec+RRF) vs qmd vsearch (vektor-only) vs qmd search (BM25).
    results = {"aria_hybrid": [], "qmd_vsearch": [], "qmd_search": []}
    lat = {k: [] for k in results}
    print(f"{'Query':<46} {'kind':<9} {'aria':<5} {'qmd-v':<6} {'qmd-bm':<6}")
    print("-" * 80)
    for case in TESTSET:
        q, kind, expect = case["q"], case["kind"], case["expect"]
        ap, adt = run_aria(q)
        vp, vdt = run_qmd(q, "vsearch")
        sp, sdt = run_qmd(q, "search")
        ah, vh, sh = hit(ap, expect), hit(vp, expect), hit(sp, expect)
        results["aria_hybrid"].append((kind, ah))
        results["qmd_vsearch"].append((kind, vh))
        results["qmd_search"].append((kind, sh))
        lat["aria_hybrid"].append(adt); lat["qmd_vsearch"].append(vdt); lat["qmd_search"].append(sdt)
        print(f"{q[:44]:<46} {kind:<9} {'✓' if ah else '✗':<5} {'✓' if vh else '✗':<6} {'✓' if sh else '✗':<6}")

    print("\n=== hit@5 ===")
    for eng, rows in results.items():
        n = len(rows); h = sum(1 for _, ok in rows if ok)
        sem = [ok for k, ok in rows if k == "semantic"]; lex = [ok for k, ok in rows if k == "lexical"]
        print(f"{eng:<14} overall {h}/{n} ({100*h/n:.0f}%)  | semantic {sum(sem)}/{len(sem)}  lexical {sum(lex)}/{len(lex)}")
    print("\n=== Latenz (mean / max, s) ===")
    for eng, ds in lat.items():
        print(f"{eng:<14} mean {sum(ds)/len(ds):.2f}s  max {max(ds):.2f}s")


if __name__ == "__main__":
    main()