"""
Scan every URL that appears in the ingested content and report which are dead or
redirected — so you can decide what to add to link-corrections.csv.

    python3 check_links.py --limit 50     # quick test on 50 URLs first
    python3 check_links.py                 # scan everything (can take a while)
    python3 check_links.py --all           # include the healthy links in the report too

Writes link-report.csv (status, code, url, final-url-or-error). Runs on your
machine; it makes a lot of outbound requests, so start with --limit.
"""

import re
import csv
import json
import os
import argparse

URL_RE = re.compile(r'https?://[^\s"\'<>()\[\]{}]+')
STORE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "store", "chunks.json")


def gather_urls():
    urls = set()
    chunks = json.load(open(STORE, encoding="utf-8"))
    for c in chunks:
        for field in (c.get("source", ""), c.get("book_link", "")):
            if field.startswith("http"):
                urls.add(field.rstrip(".,);"))
        for u in URL_RE.findall(c.get("text", "")):
            urls.add(u.rstrip(".,);"))
    return sorted(urls)


def check(url, timeout):
    import requests
    try:
        r = requests.get(url, timeout=timeout, allow_redirects=True, stream=True,
                         headers={"User-Agent": "WayneTrailGuide-linkcheck/1.0"})
        code, final = r.status_code, r.url
        r.close()
        return code, final
    except Exception as e:
        return None, str(e)[:100]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--limit", type=int, default=0, help="0 = all; use a small number to test")
    ap.add_argument("--timeout", type=float, default=10)
    ap.add_argument("--all", action="store_true", help="include healthy links in the report")
    args = ap.parse_args()

    urls = gather_urls()
    if args.limit:
        urls = urls[:args.limit]
    print(f"Checking {len(urls)} unique URLs from the corpus...")

    rows, problems = [], 0
    for i, u in enumerate(urls, 1):
        code, final = check(u, args.timeout)
        dead = code is None or (isinstance(code, int) and code >= 400)
        redirected = isinstance(code, int) and not dead and final.rstrip("/") != u.rstrip("/")
        status = "DEAD" if dead else ("REDIRECT" if redirected else "ok")
        if status != "ok":
            problems += 1
        if args.all or status != "ok":
            rows.append({"status": status, "code": code, "url": u, "final_or_error": final})
        if i % 25 == 0:
            print(f"  {i}/{len(urls)}  problems so far: {problems}")

    out = os.path.join(os.path.dirname(STORE), "..", "link-report.csv")
    out = os.path.abspath(out)
    with open(out, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=["status", "code", "url", "final_or_error"])
        w.writeheader()
        w.writerows(rows)
    print(f"\nDone. {problems} problem link(s) out of {len(urls)}.")
    print(f"Report: {out}")
    print("Review it, then add any dead/moved ones to link-corrections.csv.")


if __name__ == "__main__":
    main()
