#!/usr/bin/env python3
"""Orlando Capital — vollständiger Site-Crawl (Phase 1a).
BFS über interne Links, DE+EN. Speichert pro URL Metadaten (JSON) + HTML-Kopie.
Read-only, höflich gedrosselt (0.5s Delay), identifiziert sich als Audit-Bot.
"""
import json, re, time, hashlib, sys
from collections import deque
from urllib.parse import urljoin, urlparse, urldefrag
import requests
from bs4 import BeautifulSoup

BASE = "https://www.orlandofund.com"
HOST = "www.orlandofund.com"
OUT = "/home/aria/projekte/orlando-redesign/evidence/crawl"
import os
os.makedirs(f"{OUT}/html", exist_ok=True)

session = requests.Session()
session.headers.update({"User-Agent": "Mozilla/5.0 (compatible; OrlandoRedesignAudit/1.0; site-owner-authorized)"})

seen = {}          # url -> record
queue = deque(["https://www.orlandofund.com/", "https://www.orlandofund.com/en/"])
enqueued = set(queue)
assets = {}        # pdfs, docs
external = {}      # external links

def norm(u):
    u, _ = urldefrag(u)
    return u

def classify(path):
    if path.startswith("/en/"): lang = "en"; p = path[3:]
    else: lang = "de"; p = path
    if p in ("/", ""): return lang, "home"
    if "/team" in p: return lang, "team"
    if "/beteiligung" in p or "/portfolio" in p or "/investment" in p: return lang, "portfolio"
    if "/news" in p or "/aktuell" in p or "/insight" in p: return lang, "news"
    if "/karriere" in p or "/career" in p or "/job" in p: return lang, "career"
    if "/kontakt" in p or "/contact" in p: return lang, "contact"
    if "/impressum" in p or "/imprint" in p or "/datenschutz" in p or "/privacy" in p or "/agb" in p or "/disclaimer" in p: return lang, "legal"
    if "/verantwortung" in p or "/responsib" in p or "/esg" in p or "/nachhaltig" in p: return lang, "esg"
    if "/profil" in p or "/profile" in p or "/ueber" in p or "/about" in p or "/unternehmen" in p: return lang, "profile"
    if "/404" in p: return lang, "error"
    return lang, "other"

while queue and len(seen) < 500:
    url = queue.popleft()
    try:
        r = session.get(url, timeout=20, allow_redirects=True)
    except Exception as e:
        seen[url] = {"url": url, "status": "ERR", "error": str(e)[:200]}
        continue
    final = norm(r.url)
    rec = {
        "url": url, "final_url": final, "status": r.status_code,
        "redirected": final != norm(url),
        "content_type": r.headers.get("content-type", ""),
        "size": len(r.content),
    }
    if "text/html" in rec["content_type"] and r.status_code == 200:
        soup = BeautifulSoup(r.text, "lxml")
        t = soup.find("title"); rec["title"] = t.get_text(strip=True) if t else ""
        md = soup.find("meta", attrs={"name": "description"})
        rec["meta_description"] = md.get("content", "") if md else ""
        mr = soup.find("meta", attrs={"name": "robots"})
        rec["meta_robots"] = mr.get("content", "") if mr else ""
        can = soup.find("link", rel="canonical")
        rec["canonical"] = can.get("href", "") if can else ""
        h1s = [h.get_text(" ", strip=True) for h in soup.find_all("h1")]
        rec["h1"] = h1s
        rec["hreflang"] = {l.get("hreflang"): l.get("href") for l in soup.find_all("link", rel="alternate") if l.get("hreflang")}
        rec["lang_attr"] = soup.html.get("lang", "") if soup.html else ""
        lang, ptype = classify(urlparse(final).path)
        rec["lang"] = lang; rec["page_type"] = ptype
        rec["headings"] = [{"tag": h.name, "text": h.get_text(" ", strip=True)[:120]} for h in soup.find_all(re.compile("^h[1-3]$"))][:40]
        rec["images"] = len(soup.find_all("img"))
        rec["structured_data"] = len(soup.find_all("script", type="application/ld+json"))
        rec["forms"] = len(soup.find_all("form"))
        # save html copy
        fn = hashlib.md5(final.encode()).hexdigest()[:12]
        with open(f"{OUT}/html/{fn}.html", "w") as f:
            f.write(r.text)
        rec["html_file"] = f"{fn}.html"
        # link discovery
        internal_links = set()
        for a in soup.find_all("a", href=True):
            href = norm(urljoin(final, a["href"]))
            pu = urlparse(href)
            if pu.scheme not in ("http", "https"): continue
            if pu.netloc == HOST:
                if re.search(r"\.(pdf|docx?|xlsx?|pptx?|zip|jpg|jpeg|png|svg|webp)$", pu.path, re.I):
                    assets.setdefault(href, []).append(final)
                elif href not in enqueued:
                    enqueued.add(href); queue.append(href)
                internal_links.add(href)
            else:
                external.setdefault(href, []).append(final)
        rec["internal_link_count"] = len(internal_links)
    seen[norm(url)] = rec
    time.sleep(0.5)

# check assets with HEAD
asset_recs = {}
for a, sources in list(assets.items())[:200]:
    try:
        h = session.head(a, timeout=15, allow_redirects=True)
        asset_recs[a] = {"status": h.status_code, "content_type": h.headers.get("content-type",""), "size": h.headers.get("content-length",""), "linked_from": sources[:5]}
    except Exception as e:
        asset_recs[a] = {"status": "ERR", "error": str(e)[:100], "linked_from": sources[:5]}
    time.sleep(0.3)

with open(f"{OUT}/pages.json", "w") as f:
    json.dump(seen, f, indent=1, ensure_ascii=False)
with open(f"{OUT}/assets.json", "w") as f:
    json.dump(asset_recs, f, indent=1, ensure_ascii=False)
with open(f"{OUT}/external_links.json", "w") as f:
    json.dump(external, f, indent=1, ensure_ascii=False)
print(f"DONE pages={len(seen)} assets={len(asset_recs)} external={len(external)}")