"""
Crawl Wayne's content from lifestream.org and save it as clean text for ingest.

Two modes:

  python3 crawl.py                 # articles: Posts AND Pages -> content/lifestream/
  python3 crawl.py --limit 20      # small test batch first (recommended)
  python3 crawl.py --book-pages    # book product-page descriptions -> content/book-pages/

Articles are saved one .txt per page with the URL as the first line, so the Trail
Guide can cite and link back. Utility pages (cart, checkout, contact, etc.) and
pages too thin to be real content are skipped. Both modes resume safely: already
saved files are skipped, so you can stop and restart anytime.

When finished, run:  python3 ingest.py
"""

import os
import re
import csv
import time
import argparse
import xml.etree.ElementTree as ET

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
SITEMAP_INDEX = "https://www.lifestream.org/sitemap.xml"
INCLUDE_DEFAULT = r"(post|page)-sitemap"           # Posts AND Pages
EXCLUDE_DEFAULT = r"/(cart|checkout|my-account|account|wishlist|contact|privacy|terms|refund|return|shipping|sample-page)(/|$)"
OUT_ARTICLES = os.path.join(BASE_DIR, "content", "lifestream")
OUT_BOOKS = os.path.join(BASE_DIR, "content", "book-pages")
OUT_PAGES = os.path.join(BASE_DIR, "content", "pages")
CATALOG_PATH = os.path.join(BASE_DIR, "book-catalog.csv")
HEADERS = {"User-Agent": "WayneTrailGuide/1.0 (content ingest for lifestream.org)"}


def _get(url):
    import requests
    r = requests.get(url, headers=HEADERS, timeout=30)
    r.raise_for_status()
    return r.text


def _locs(xml_text):
    root = ET.fromstring(xml_text)
    return [el.text.strip() for el in root.iter()
            if el.tag.endswith("loc") and el.text and el.text.strip()]


def _slug(url):
    s = re.sub(r"^https?://", "", url.lower())
    s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")
    return s[-120:] or "page"


def _extract(html_text):
    import trafilatura
    return (trafilatura.extract(html_text, include_comments=False,
                                include_tables=False) or "").strip()


def _extract_full(html_text):
    """Full extraction for index/list pages (archives). Article extractors drop
    long link/list blocks as 'navigation', so grab the main content container's
    entire text with BeautifulSoup instead."""
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html_text, "html.parser")
    for tag in soup(["script", "style", "nav", "header", "footer", "form", "aside", "noscript"]):
        tag.decompose()
    node = (soup.select_one(".entry-content") or soup.select_one("article")
            or soup.select_one("main") or soup.body or soup)
    # Mark headings with "## " so ingest can keep each section's heading attached
    # to its content (so a FAQ answer travels with its question, a guest's
    # episodes with the guest's name, etc.). Covers real headings AND common
    # accordion/toggle title elements (e.g. SiteOrigin's sow-accordion-title).
    title_class = re.compile(r"accordion-title|accordion-button|toggle-title|panel-title|faq-(?:title|question)", re.I)
    heads = node.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]) + node.find_all(class_=title_class)
    for h in heads:
        try:
            h.replace_with(f"\n## {h.get_text(' ', strip=True)}\n")
        except Exception:
            pass
    text = node.get_text("\n", strip=True)
    return re.sub(r"\n{3,}", "\n\n", text).strip()


def do_pages(args):
    """Fetch arbitrary pages (e.g. the God Journey guest/topic archives) into
    content/pages/. Always refreshes so living index pages stay current."""
    os.makedirs(OUT_PAGES, exist_ok=True)
    with open(args.pages) as f:
        urls = [ln.strip() for ln in f if ln.strip() and not ln.strip().startswith("#")]
    print(f"Fetching {len(urls)} page(s) into {OUT_PAGES} ...")
    counts = {"saved": 0, "thin": 0, "fail": 0}
    for u in urls:
        try:
            text = _extract_full(_get(u))
            if len(text) < 200:
                counts["thin"] += 1
                print(f"  ! thin/empty: {u}")
            else:
                with open(os.path.join(OUT_PAGES, _slug(u) + ".txt"), "w", encoding="utf-8") as fo:
                    fo.write(u + "\n\n" + text)
                counts["saved"] += 1
                print(f"  saved {u} ({len(text):,} chars)")
        except Exception as e:
            counts["fail"] += 1
            print(f"  ! {u}: {e}")
        time.sleep(args.delay)
    print(f"\nDone. {counts}\nNext: python3 ingest.py")


def _fetch_save(url, out_dir, delay):
    """Fetch one URL, extract, save with URL as first line. Returns status."""
    out = os.path.join(out_dir, _slug(url) + ".txt")
    if os.path.exists(out):
        return "skip"
    try:
        text = _extract(_get(url))
        if len(text) < 200:
            return "thin"
        with open(out, "w", encoding="utf-8") as f:
            f.write(url + "\n\n" + text)
        return "saved"
    except Exception as e:
        print(f"  ! {url}: {e}")
        return "fail"
    finally:
        time.sleep(delay)


def _run(urls, out_dir, delay, label):
    os.makedirs(out_dir, exist_ok=True)
    counts = {"saved": 0, "skip": 0, "thin": 0, "fail": 0}
    for i, url in enumerate(urls, 1):
        counts[_fetch_save(url, out_dir, delay)] += 1
        if i % 25 == 0:
            print(f"  {i}/{len(urls)}  {counts}")
    print(f"\nDone ({label}). {counts}")
    print("Next: python3 ingest.py")


def do_articles(args):
    subs = [u for u in _locs(_get(args.sitemap)) if re.search(args.include, u)]
    print(f"Matching sub-sitemaps: {subs}")
    urls = []
    for s in subs:
        found = _locs(_get(s))
        print(f"  {s}: {len(found)} urls")
        urls.extend(found)
    urls = [u for u in urls if not re.search(EXCLUDE_DEFAULT, u.lower())]
    seen = set()
    urls = [u for u in urls if not (u in seen or seen.add(u))]
    if args.limit:
        urls = urls[:args.limit]
    print(f"Crawling {len(urls)} article/page(s) into {OUT_ARTICLES} ...")
    _run(urls, OUT_ARTICLES, args.delay, "articles + pages")


def do_book_pages(args):
    if not os.path.exists(CATALOG_PATH):
        print("No book-catalog.csv found.")
        return
    urls = []
    with open(CATALOG_PATH, encoding="utf-8-sig", newline="") as f:
        for r in csv.DictReader(f):
            link = (r.get("Link") or "").strip()
            status = (r.get("Status") or "").strip().lower()
            if "lifestream.org/product/" in link and status != "hold":
                urls.append(link)
    seen = set()
    urls = [u for u in urls if not (u in seen or seen.add(u))]
    print(f"Fetching {len(urls)} book description page(s) into {OUT_BOOKS} ...")
    _run(urls, OUT_BOOKS, args.delay, "book pages")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--sitemap", default=SITEMAP_INDEX)
    ap.add_argument("--include", default=INCLUDE_DEFAULT,
                    help="regex for which sub-sitemaps to crawl")
    ap.add_argument("--limit", type=int, default=0, help="0 = all")
    ap.add_argument("--delay", type=float, default=1.0, help="seconds between requests")
    ap.add_argument("--book-pages", action="store_true",
                    help="crawl book product-page descriptions instead of articles")
    ap.add_argument("--pages", help="text file of arbitrary page URLs (e.g. archive index pages) "
                                    "to fetch into content/pages/")
    args = ap.parse_args()

    if args.pages:
        do_pages(args)
    elif args.book_pages:
        do_book_pages(args)
    else:
        do_articles(args)


if __name__ == "__main__":
    main()
