#!/usr/bin/env python3
"""Phase 0 — Korpus-Verifikation (Master-Prompt §5, deterministisch).
Erzeugt corpus-manifest.csv/.json + corpus.sha256 unter manifests/.
Read-only auf incoming/. Format-Sniffing über Magic Bytes, Macro-/External-Link-
Erkennung über ZIP-Inhalt (xlsx/xlsm), Duplikate über SHA-256.
"""
import csv, hashlib, json, os, zipfile, datetime, collections

BASE = "/home/aria/work/qaf-corpus/incoming"
OUT = "/home/aria/work/qaf-corpus/manifests"
EXPECTED = 374

SUPPORTED = {".xlsx", ".xlsm", ".xlsb", ".xls"}

def sniff_format(path):
    with open(path, "rb") as f:
        head = f.read(8)
    if head[:4] == b"PK\x03\x04":
        return "ooxml_zip"
    if head == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1":
        return "ole2_biff"  # legacy .xls Container (oder verschlüsseltes OOXML)
    if not head:
        return "empty"
    return "unknown"

def zip_facts(path):
    """Macros, externe Links, echte Excel-Struktur — nur für OOXML-Zips."""
    facts = {"is_excel": False, "has_macros": False, "external_links": 0, "zip_error": ""}
    try:
        with zipfile.ZipFile(path) as z:
            names = set(z.namelist())
            facts["is_excel"] = "xl/workbook.xml" in names or "[Content_Types].xml" in names and any(n.startswith("xl/") for n in names)
            facts["has_macros"] = "xl/vbaProject.bin" in names
            facts["external_links"] = sum(1 for n in names if n.startswith("xl/externalLinks/"))
    except Exception as e:
        facts["zip_error"] = str(e)[:120]
    return facts

rows, hashes = [], collections.defaultdict(list)
names_seen = collections.defaultdict(list)
for root, dirs, files in os.walk(BASE):
    for fn in sorted(files):
        p = os.path.join(root, fn)
        rel = os.path.relpath(p, BASE)
        ext = os.path.splitext(fn)[1].lower()
        size = os.path.getsize(p)
        row = {
            "relative_path": rel, "filename": fn, "extension": ext, "size_bytes": size,
            "mtime": datetime.datetime.fromtimestamp(os.path.getmtime(p)).isoformat(timespec="seconds"),
            "sha256": "", "detected_format": "", "macro_status": "", "external_links": "",
            "processing_status": "", "error": "",
        }
        if fn.startswith("~$"):
            row["processing_status"] = "excluded_lock_file"
            rows.append(row); continue
        h = hashlib.sha256(open(p, "rb").read()).hexdigest()
        row["sha256"] = h
        hashes[h].append(rel); names_seen[fn].append(rel)
        if size == 0:
            row["processing_status"] = "zero_byte"; rows.append(row); continue
        if ext not in SUPPORTED:
            row["processing_status"] = "unsupported_extension"
            row["detected_format"] = sniff_format(p)
            rows.append(row); continue
        fmt = sniff_format(p)
        row["detected_format"] = fmt
        if fmt == "ooxml_zip":
            zf = zip_facts(p)
            row["macro_status"] = "macros" if zf["has_macros"] else "none"
            row["external_links"] = zf["external_links"]
            if zf["zip_error"]:
                row["processing_status"] = "corrupt_zip"; row["error"] = zf["zip_error"]
            elif not zf["is_excel"]:
                row["processing_status"] = "not_excel_workbook"
            elif ext == ".xlsx" and zf["has_macros"]:
                row["processing_status"] = "ok_ext_mismatch_macros_in_xlsx"
            else:
                row["processing_status"] = "ok"
        elif fmt == "ole2_biff":
            row["macro_status"] = "unknown_biff"
            row["processing_status"] = "legacy_biff" if ext == ".xls" else "ext_mismatch_biff_container"
        else:
            row["processing_status"] = "unreadable_format"
        rows.append(row)

# Duplikate markieren
dupe_hashes = {h: ps for h, ps in hashes.items() if len(ps) > 1}
dupe_names = {n: ps for n, ps in names_seen.items() if len(ps) > 1}
for r in rows:
    r["duplicate_content_of"] = ";".join(x for x in dupe_hashes.get(r["sha256"], []) if x != r["relative_path"])

supported_ok = [r for r in rows if r["processing_status"].startswith("ok") or r["processing_status"] == "legacy_biff"]
summary = {
    "expected_workbook_count": EXPECTED,
    "actual_supported_workbook_count": len([r for r in rows if r["extension"] in SUPPORTED and not r["processing_status"].startswith("excluded")]),
    "ok": len([r for r in rows if r["processing_status"].startswith("ok")]),
    "status_counts": dict(collections.Counter(r["processing_status"] for r in rows)),
    "macro_files": len([r for r in rows if r["macro_status"] == "macros"]),
    "files_with_external_links": len([r for r in rows if isinstance(r["external_links"], int) and r["external_links"] > 0]),
    "duplicate_content_groups": len(dupe_hashes),
    "duplicate_filenames": len(dupe_names),
    "generated": datetime.datetime.now().isoformat(timespec="seconds"),
}

os.makedirs(OUT, exist_ok=True)
with open(f"{OUT}/corpus-manifest.csv", "w", newline="") as f:
    w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
    w.writeheader(); w.writerows(rows)
json.dump({"summary": summary, "files": rows}, open(f"{OUT}/corpus-manifest.json", "w"), indent=1, ensure_ascii=False)
with open(f"{OUT}/corpus.sha256", "w") as f:
    for r in rows:
        if r["sha256"]:
            f.write(f"{r['sha256']}  {r['relative_path']}\n")
print(json.dumps(summary, indent=1, ensure_ascii=False))