"""
Ingest: read every .txt / .md / .pdf / .epub under ./content (including
subfolders like content/books/ and content/lifestream/), chunk it, embed it,
and save the local store. Run this once, and again whenever you add content.

    python ingest.py

- PDF and EPUB books are converted to text automatically. (.mobi isn't supported
  — convert it to EPUB first, e.g. with Calibre.)
- If a file's FIRST LINE is a URL (as the crawler writes), that URL becomes the
  source the Trail Guide cites. Otherwise the filename is the source.
- book-catalog.csv (in this folder) is the master list: it says which books are
  free vs. for-sale and gives each a purchase/read link. Edit it in any
  spreadsheet app. The "Match" column is a keyword found in the file's name (or
  an article URL) that ties content to that catalog row.
- Files under content/for-sale/ are also treated as for-sale as a safe default,
  even before you add them to the catalog.
"""

import os
import re
import csv
import glob
import numpy as np
import rag

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
CONTENT_DIR = os.path.join(BASE_DIR, "content")
CATALOG_PATHS = [os.path.join(BASE_DIR, "book-catalog.csv"),
                 os.path.join(BASE_DIR, "media-catalog.csv"),
                 os.path.join(BASE_DIR, "video-catalog.csv")]


def _paths(ext):
    return sorted(glob.glob(os.path.join(CONTENT_DIR, "**", f"*.{ext}"), recursive=True))


def _norm(s):
    return re.sub(r"[^a-z0-9]", "", s.lower())


def _is_for_sale_folder(path):
    return "for-sale" in os.path.normpath(path).split(os.sep)


def _in_book_folder(path):
    parts = os.path.normpath(path).split(os.sep)
    return "books" in parts or "for-sale" in parts


IR_FALLBACK_LINK = "https://www.lifestream.org/audio-video-library/"


def load_catalog():
    rows = []
    for path in CATALOG_PATHS:
        if os.path.exists(path):
            is_media = os.path.basename(path) == "media-catalog.csv"
            with open(path, encoding="utf-8-sig", newline="") as f:
                for r in csv.DictReader(f):
                    row = {(k or "").strip().lower(): (v or "").strip() for k, v in r.items()}
                    row["_media"] = is_media
                    rows.append(row)
    return rows


CATALOG = load_catalog()


def _individual_meta(source):
    """Title + link for an Individual Recording, from media-catalog rows only
    (so a recording named like a book doesn't inherit the book's identity)."""
    ns, best, title, link = _norm(source), 0, "", ""
    for r in CATALOG:
        if not r.get("_media"):
            continue
        for kw in re.split(r"[|,]", r.get("match", "")):
            nm = _norm(kw)
            if nm and nm in ns and len(nm) > best:
                best, title, link = len(nm), r.get("title", ""), r.get("link", "")
    return title, link


def classify(source, in_forsale_folder):
    """Return (for_sale, book_link, book_title, attribution, hold) for a source,
    using the catalog. Matches only on the explicit 'Match' keyword to stay
    predictable. A 'hold' status means: keep this content out of the corpus."""
    for_sale, link, title, attribution, hold = in_forsale_folder, "", "", "", False
    ns, best, best_row = _norm(source), 0, None
    for r in CATALOG:
        for kw in re.split(r"[|,]", r.get("match", "")):
            nm = _norm(kw)
            if nm and nm in ns and len(nm) > best:
                best, best_row = len(nm), r
    if best_row is not None:
        title = best_row.get("title", "")
        link = best_row.get("link", "")
        attribution = best_row.get("attribution", "")
        status = best_row.get("status", "").lower()
        if status:
            hold = status in ("hold", "exclude", "pending")
            for_sale = status in ("for-sale", "forsale", "paid")
    return for_sale, link, title, attribution, hold


def read_pdf(path):
    from pdfminer.high_level import extract_text
    return extract_text(path) or ""


def read_epub(path):
    import ebooklib
    from ebooklib import epub
    from bs4 import BeautifulSoup
    book = epub.read_epub(path)
    parts = []
    for item in book.get_items():
        if item.get_type() == ebooklib.ITEM_DOCUMENT:
            soup = BeautifulSoup(item.get_content(), "html.parser")
            parts.append(soup.get_text(" ", strip=True))
    return "\n\n".join(parts)


def read_book(path):
    return read_epub(path) if path.lower().endswith(".epub") else read_pdf(path)


def load_text_file(path):
    """Return (source_label, text, link).

    First-line forms handled:
      "Title — https://url"  -> clean title is the source, URL is the link
                                (God Journey episodes carry the real feed title
                                here, since URL slugs aren't reliable)
      "https://url"          -> the URL is the source (and its own link)
      "folder/title" label   -> readable transcript label as the source
      (anything else)        -> the filename is the source
    """
    with open(path, encoding="utf-8", errors="ignore") as f:
        raw = f.read()
    first, _, rest = raw.partition("\n")
    first = first.strip()
    m = re.match(r"^(?P<title>.+)\s+—\s+(?P<url>https?://\S+)\s*$", first)
    if m:
        return m.group("title").strip(), rest.strip(), m.group("url").strip()
    if first.lower().startswith(("http://", "https://")):
        return first, rest.strip(), ""
    # Transcripts carry a readable "folder/title" label as their first line.
    if "transcripts" in os.path.normpath(path).split(os.sep) and first:
        return first, rest.strip(), ""
    return os.path.basename(path), raw.strip(), ""


def _make_doc(path, source, text, parsed_link=""):
    # Individual Recordings are standalone teachings — cite each by its own title
    # (from the transcript's source label), not the generic series category.
    if "individual recordings" in source.lower():
        title, link = _individual_meta(source)
        return {"source": source, "text": text, "for_sale": False,
                "book_link": link or parsed_link or IR_FALLBACK_LINK, "book_title": title, "attribution": ""}

    for_sale, link, title, attribution, hold = classify(source, _is_for_sale_folder(path))
    book_file = _in_book_folder(path)
    if hold:
        if book_file:
            # Withhold the held book's own text.
            print(f"  (on hold — not ingesting the book itself: {source})")
            return None
        # An article/episode that merely references a held title still belongs;
        # keep it, but don't attach the held book's link/attribution.
        link = title = attribution = ""
        for_sale = False
    # The for-sale restriction (summarize, don't reproduce) applies ONLY to an
    # actual book file. Articles, episodes, and book-description pages that merely
    # reference a for-sale book stay freely usable — they just carry its link.
    if not book_file:
        for_sale = False
    # parsed_link is an episode/article's own URL taken from a "Title — URL"
    # first line; use it when the catalog didn't supply one.
    return {"source": source, "text": text, "for_sale": for_sale,
            "book_link": link or parsed_link, "book_title": title, "attribution": attribution}


def read_files():
    text_paths = _paths("txt") + _paths("md")
    book_paths = _paths("pdf") + _paths("epub")
    text_stems = {os.path.splitext(os.path.basename(p))[0].lower() for p in text_paths}

    docs = []
    for p in text_paths:
        base = os.path.basename(p)
        if base.lower() == "readme.txt" or base.startswith("_"):
            continue
        source, text, link = load_text_file(p)
        if text:
            d = _make_doc(p, source, text, link)
            if d:
                docs.append(d)

    for p in book_paths:
        stem = os.path.splitext(os.path.basename(p))[0].lower()
        if stem in text_stems:
            print(f"  (skipping {os.path.basename(p)} — already have a text version)")
            continue
        text = read_book(p).strip()
        if text:
            d = _make_doc(p, os.path.basename(p), text)
            if d:
                docs.append(d)

    return docs


def chunk_doc(source, text):
    """Chunk a document. If it has '## ' section headings (from crawled index/FAQ
    pages), split by section and keep each heading attached to its chunks — so a
    query for a heading (a guest's name, a FAQ topic) retrieves that section."""
    if "\n## " not in text and not text.startswith("## "):
        return rag.chunk_text(text, source=source)

    chunks = []
    for section in re.split(r"\n(?=## )", text):
        section = section.strip()
        if not section:
            continue
        m = re.match(r"##\s*(.+)", section)
        heading = m.group(1).strip() if m else ""
        body = section[m.end():].strip() if m else section
        for c in rag.chunk_text(body or heading, source=source):
            if heading:
                c["text"] = f"{heading} — {c['text']}"
            chunks.append(c)
    return chunks


def main():
    docs = read_files()
    if not docs:
        print("No content found. Add .txt/.md/.pdf/.epub files under ./content and rerun.")
        return

    all_chunks = []
    for d in docs:
        for c in chunk_doc(d["source"], d["text"]):
            c["for_sale"] = d["for_sale"]
            c["book_link"] = d["book_link"]
            c["book_title"] = d["book_title"]
            c["attribution"] = d["attribution"]
            all_chunks.append(c)

    n_paid = sum(1 for d in docs if d["for_sale"])
    print(f"Read {len(docs)} document(s) ({n_paid} for-sale), {len(all_chunks)} chunks.")

    print(f"Embedding with backend '{rag.EMBED_BACKEND}'... (this can take a while for large corpora)")
    vectors = rag.embed([c["text"] for c in all_chunks])
    rag.save_store(all_chunks, np.asarray(vectors))
    print(f"Done. Store saved with {len(all_chunks)} chunks from {len(docs)} document(s).")


if __name__ == "__main__":
    main()
