"""
Core RAG logic for the Wayne concierge pilot.

Two responsibilities:
  1. Turn text into embeddings (mock backend for offline testing, or a real
     local model via sentence-transformers).
  2. Retrieve the most relevant chunks for a question and ask Claude to answer
     using ONLY those chunks, with citations.

Nothing here talks to a paid service except the answer step (Claude), and even
that is skipped by the offline smoke test.
"""

import os
import re
import csv
import glob
import json
import hashlib
import numpy as np

# ---------------------------------------------------------------------------
# Configuration (env-overridable; sensible defaults for a pilot)
# ---------------------------------------------------------------------------
EMBED_BACKEND = os.environ.get("EMBED_BACKEND", "local")   # "local" | "mock"
# Multilingual embedder (50+ languages incl. Spanish, French, German, Portuguese,
# Russian, Korean, Chinese) so questions asked in any language retrieve Wayne's
# (English) material. Same 384 dims as the old all-MiniLM-L6-v2, so EMBED_DIM is
# unchanged — but the store MUST be re-ingested after changing this, since old
# vectors were built by the previous model.
LOCAL_MODEL   = os.environ.get("EMBED_MODEL", "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
ANSWER_MODEL  = os.environ.get("ANSWER_MODEL", "claude-haiku-4-5-20251001")
ANSWER_TEMPERATURE = os.environ.get("ANSWER_TEMPERATURE", "")  # blank = the model's own default. Set a number (e.g. 0.3) ONLY for models that accept it — Sonnet 5 and later reject any non-default sampling parameter with a 400 error.
_SAMPLING = {"temperature": float(ANSWER_TEMPERATURE)} if str(ANSWER_TEMPERATURE).strip() else {}
TOP_K         = int(os.environ.get("TOP_K", "12"))  # widen recall as the corpus grows
LEX_EXTRA     = int(os.environ.get("LEX_EXTRA", "4"))   # max chunks the lexical rescue may add
LEX_MIN       = float(os.environ.get("LEX_MIN", "4.0")) # min IDF-weighted score to qualify as a rare-term hit
LEX_TERM_IDF  = float(os.environ.get("LEX_TERM_IDF", "5.5"))  # a query term must be at least this rare (IDF) to drive a rescue
MAX_FORSALE   = int(os.environ.get("MAX_FORSALE_PASSAGES", "3"))  # cap paid-book passages per answer
EMBED_DIM     = 384  # all-MiniLM-L6-v2 and the mock backend both use 384

_STORE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "store")
_VEC_PATH  = os.path.join(_STORE_DIR, "vectors.npy")
_META_PATH = os.path.join(_STORE_DIR, "chunks.json")

_model = None  # lazily loaded sentence-transformers model


# ---------------------------------------------------------------------------
# Embeddings
# ---------------------------------------------------------------------------
def _mock_embed(texts):
    """Deterministic hashing embedder. No downloads, no network.

    Good enough to prove the plumbing (chunking -> store -> retrieval) works
    offline. NOT for production quality — swap EMBED_BACKEND=local for that.
    """
    vecs = np.zeros((len(texts), EMBED_DIM), dtype=np.float32)
    for i, text in enumerate(texts):
        for tok in re.findall(r"[a-z0-9]+", text.lower()):
            h = int(hashlib.md5(tok.encode()).hexdigest(), 16)
            vecs[i, h % EMBED_DIM] += 1.0
    norms = np.linalg.norm(vecs, axis=1, keepdims=True)
    norms[norms == 0] = 1.0
    return vecs / norms


def _local_embed(texts):
    global _model
    if _model is None:
        from sentence_transformers import SentenceTransformer
        _model = SentenceTransformer(LOCAL_MODEL)
    return _model.encode(texts, normalize_embeddings=True, convert_to_numpy=True)


def embed(texts):
    if isinstance(texts, str):
        texts = [texts]
    return _mock_embed(texts) if EMBED_BACKEND == "mock" else _local_embed(texts)


# ---------------------------------------------------------------------------
# Chunking
# ---------------------------------------------------------------------------
def chunk_text(text, source, target_words=180, overlap_words=40):
    """Split text into overlapping word windows, keeping a source label so the
    concierge can cite where each passage came from."""
    words = text.split()
    chunks, start, idx = [], 0, 0
    step = max(1, target_words - overlap_words)
    while start < len(words):
        window = words[start:start + target_words]
        if window:
            chunks.append({
                "id": f"{source}#{idx}",
                "source": source,
                "text": " ".join(window),
            })
            idx += 1
        start += step
    return chunks


# ---------------------------------------------------------------------------
# Store (local files — pilot only; production moves to Supabase pgvector)
# ---------------------------------------------------------------------------
def save_store(chunks, vectors):
    os.makedirs(_STORE_DIR, exist_ok=True)
    np.save(_VEC_PATH, vectors.astype(np.float32))
    with open(_META_PATH, "w") as f:
        json.dump(chunks, f)


def load_store():
    if not (os.path.exists(_VEC_PATH) and os.path.exists(_META_PATH)):
        raise FileNotFoundError(
            "No store found. Run `python ingest.py` first to build it."
        )
    vectors = np.load(_VEC_PATH)
    with open(_META_PATH) as f:
        chunks = json.load(f)
    return chunks, vectors


# ---------------------------------------------------------------------------
# Retrieval  (hybrid: dense semantic search + lexical rare-term rescue)
# ---------------------------------------------------------------------------
# A short natural-language question embeds to a "blurry average" of all its
# words, so a rare but decisive term (e.g. "five-fold") can be drowned out by
# conversational filler ("Hi! I'm wondering what Wayne says about ..."). The
# lexical pass rescues chunks that literally contain the question's rare,
# high-signal words — weighted by how rare each word is (IDF) — and merges them
# with the semantic hits so an exact-term match always makes it into context.
# (At the Supabase migration this lexical half becomes Postgres full-text search.)

_STOPWORDS = set(
    "a an and the of to in on for with about is are was were be been being do does "
    "did have has had i you he she it we they me him her us them my your his its our "
    "their this that these those what which who whom whose when where why how not no "
    "yes if then than so as at by from into out up down over under again more most "
    "some any can could would should will shall may might must just only also very "
    "there here own same such per via etc wayne jacobsen say says said tell tells "
    "hello hi hey please thanks thank help wondering curious question anything today "
    "know think want like get go come thing things really much many "
    # the guide's own name and common greeting/politeness filler (rare in this
    # corpus, so IDF alone won't catch them):
    "trail guide hoping hope helping wonder greetings hows hey".split()
)

_lc_cache = {}  # id(chunks) -> list of lowercased chunk texts (built once per store)


def _salient_terms(question):
    """Rare, content-bearing words from the question. Keeps hyphenated terms
    (e.g. 'five-fold') intact; drops stopwords and very short words."""
    raw = re.findall(r"[a-z0-9]+(?:-[a-z0-9]+)*", question.lower())
    return [t for t in raw if len(t) >= 3 and t not in _STOPWORDS]


def _lexical_scores(question, chunks):
    """IDF-weighted lexical score per chunk: rare query terms count for much more
    than common ones, so 'five-fold' outweighs 'ministries'."""
    terms = _salient_terms(question)
    if not terms:
        return {}
    key = id(chunks)
    lc = _lc_cache.get(key)
    if lc is None:
        lc = [c["text"].lower() for c in chunks]
        _lc_cache[key] = lc
    import math
    N = len(lc) or 1
    scores = {}
    for t in terms:
        hits = [i for i, tx in enumerate(lc) if t in tx]
        if not hits:
            continue
        idf = math.log(1 + N / len(hits))   # rarer term -> higher weight
        if idf < LEX_TERM_IDF:              # skip common words (filler, greetings,
            continue                        # the bot's own name) -- only rare,
        for i in hits:                      # decisive terms may drive a rescue
            scores[i] = scores.get(i, 0.0) + idf
    return scores


_EP_NUM_RE = re.compile(r"#?\b(\d{3,4})\b")


def _episode_number_hits(question, chunks, dense, limit=8):
    """A query like "episode 964" names an episode by its NUMBER, which lives in the
    URL/title, not the transcript text — so neither the dense nor the lexical pass finds
    it. Return the indices of the matching episode's chunks, best (most semantically
    relevant) first, so retrieve() can pull them in explicitly. Range-limited so years
    (2024, etc.) aren't mistaken for episode numbers."""
    nums = {n for n in _EP_NUM_RE.findall(question) if 100 <= int(n) <= 1099}
    if not nums:
        return []
    found = []
    for i, c in enumerate(chunks):
        src = c.get("source", "") or ""
        link = c.get("book_link", "") or (src if str(src).startswith("http") else "")
        for n in nums:
            if (f"-{n}/" in link or f"-{n}-" in link
                    or link.rstrip("/").endswith(f"-{n}") or f"(#{n})" in src):
                found.append(i)
                break
    return sorted(found, key=lambda i: float(dense[i]), reverse=True)[:limit]


def retrieve(question, chunks, vectors, top_k=TOP_K):
    qv = embed(question)[0]
    dense = vectors @ qv                       # cosine (all vectors are normalized)
    order = list(np.argsort(-dense)[:top_k])

    # Episode-number lookup (e.g. "episode 964"): the number is in the URL/title, not the
    # spoken text, so pull the matching episode's chunks in explicitly, up front.
    ep = _episode_number_hits(question, chunks, dense)
    if ep:
        order = ep + [i for i in order if i not in set(ep)]

    # Lexical rescue: ensure chunks containing the question's rare terms are
    # represented, even when the semantic average buried them.
    lex = _lexical_scores(question, chunks)
    if lex:
        chosen = set(order)
        floor = max(LEX_MIN, 0.5 * max(lex.values()))
        # rank rescue candidates by lexical score, breaking ties by semantic
        # score so the topically-relevant chunk beats a passing mention
        extra = sorted((i for i in lex if i not in chosen and lex[i] >= floor),
                       key=lambda i: (lex[i], float(dense[i])), reverse=True)
        for i in extra[:LEX_EXTRA]:
            order.append(int(i))

    return [(chunks[i], float(dense[i])) for i in order]


# ---------------------------------------------------------------------------
# Answer (the only step that calls a paid API)
# ---------------------------------------------------------------------------
def _load_system_prompt():
    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "system_prompt.txt")
    with open(path) as f:
        return f.read().strip()


def display_name(source):
    """A readable citation label from a source (URL, book file, or transcript path)."""
    if source.startswith(("http://", "https://")):
        rest = re.sub(r"^https?://", "", source).rstrip("/")
        if "/" not in rest:                 # bare domain, no path -> host as-is, don't Title-Case it
            return rest.lower()
        seg = rest.rsplit("/", 1)[-1].replace("-", " ").replace("_", " ").strip()
        return seg.title() if seg else rest.lower()
    if "/" in source:  # a transcript's "folder/title" label -> use the title part
        seg = re.sub(r"^\d+\.\s*", "", source.rsplit("/", 1)[-1].strip()).replace("_", " ")
        return seg or source
    return source


_BOOK_REF = None


def book_reference():
    """An always-available, authoritative list of Wayne's books (title, author,
    free/for-sale, link, short description) built from book-catalog.csv — so
    'list his books' style questions are answered from fact, not retrieval."""
    global _BOOK_REF
    if _BOOK_REF is not None:
        return _BOOK_REF

    base = os.path.dirname(os.path.abspath(__file__))
    catalog = os.path.join(base, "book-catalog.csv")
    if not os.path.exists(catalog):
        _BOOK_REF = ""
        return _BOOK_REF

    # A real descriptive sentence from each crawled product page (skip store
    # boilerplate like "Looking for quantity discounts?").
    _JUNK = ("quantity discount", "add to cart", "in stock", "out of stock", "sku",
             "select options", "$", "reviews", "related products")

    def _pick(body):
        for s in re.split(r"(?<=[.!?])\s", body):
            s = s.strip()
            if 60 <= len(s) <= 300 and not any(k in s.lower() for k in _JUNK):
                return s
        return ""

    desc = {}
    for p in glob.glob(os.path.join(base, "content", "book-pages", "*.txt")):
        try:
            body = open(p, encoding="utf-8", errors="ignore").read().split("\n", 2)[-1].strip()
            d = _pick(body)
            if d:
                desc[os.path.basename(p)] = d
        except Exception:
            pass

    def slugfile(link):
        s = re.sub(r"^https?://", "", link.lower())
        return re.sub(r"[^a-z0-9]+", "-", s).strip("-")[-120:] + ".txt"

    lines = []
    with open(catalog, encoding="utf-8-sig", newline="") as f:
        for r in csv.DictReader(f):
            status = (r.get("Status") or "").strip().lower()
            title = (r.get("Title") or "").strip()
            if not title or status in ("hold", "exclude", "pending"):
                continue
            author = (r.get("Attribution") or "").strip() or "Wayne Jacobsen"
            link = (r.get("Link") or "").strip()
            line = f"- {title} — by {author}" + (f" — {link}" if link else "")
            d = desc.get(slugfile(link), "")
            if d:
                line += f"\n    {d}"
            lines.append(line)

    _BOOK_REF = (
        "Wayne's books — the COMPLETE, AUTHORITATIVE list, and the single source of truth "
        "for his book titles. Use it for any question about which books he has written, who "
        "he co-authored them with, and where to find them. When you name one of his books, "
        "spell its title EXACTLY as it appears here. Never take a book's title from an "
        "article, blog post, or episode that merely mentions or discusses it — those often "
        "use informal, shortened, or inexact wording. Never invent a title, and do not "
        "present a book that is not on this list as one of Wayne's — if something seems to be "
        "a book but isn't here, say you're not finding it in his book list rather than "
        "guessing:\n"
        + "\n".join(lines)
    )
    return _BOOK_REF


_LINK_NOTES = None


def link_notes():
    """Curated guidance for outdated URLs (from link-corrections.csv) so the guide
    never hands out dead/moved links it finds in older content."""
    global _LINK_NOTES
    if _LINK_NOTES is not None:
        return _LINK_NOTES
    p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "link-corrections.csv")
    rows = []
    if os.path.exists(p):
        with open(p, encoding="utf-8-sig", newline="") as f:
            for r in csv.DictReader(f):
                m = (r.get("Match") or "").strip()
                note = (r.get("Note") or "").strip()
                if m and note:
                    rows.append(f"- {m} → {note}")
    _LINK_NOTES = (
        "LINK NOTES (URLs in older material that are outdated — never share the matched "
        "link; follow the guidance instead):\n" + "\n".join(rows)
    ) if rows else ""
    return _LINK_NOTES


_SERIES_REF = None


def series_reference():
    """An always-available list of Wayne's ongoing podcasts and series (from
    series-catalog.csv), so the guide never denies a real project exists just
    because retrieval didn't surface it."""
    global _SERIES_REF
    if _SERIES_REF is not None:
        return _SERIES_REF
    p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "series-catalog.csv")
    rows = []
    if os.path.exists(p):
        with open(p, encoding="utf-8-sig", newline="") as f:
            for r in csv.DictReader(f):
                title = (r.get("Title") or "").strip()
                if not title:
                    continue
                link = (r.get("Link") or "").strip()
                desc = (r.get("Description") or "").strip()
                line = f"- {title}" + (f" — {desc}" if desc else "") + (f" ({link})" if link else "")
                rows.append(line)
    _SERIES_REF = (
        "Wayne's ongoing podcasts and series (you always know these exist, even if "
        "nothing about them appears in the material below):\n" + "\n".join(rows)
    ) if rows else ""
    return _SERIES_REF


_TRANSLATIONS_REF = None


def translations_reference():
    """An always-available, authoritative list (from translations-catalog.csv,
    which mirrors lifestream.org/international-translations) of which of Wayne's
    works exist in which languages — so 'is there a Spanish version?' questions
    are answered from fact, and the guide never invents a translation. Crucially,
    if something isn't listed here it is NOT translated (e.g. the podcasts)."""
    global _TRANSLATIONS_REF
    if _TRANSLATIONS_REF is not None:
        return _TRANSLATIONS_REF
    p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "translations-catalog.csv")
    langs = {}
    order = []
    if os.path.exists(p):
        with open(p, encoding="utf-8-sig", newline="") as f:
            for r in csv.DictReader(f):
                lang = (r.get("Language") or "").strip()
                work = (r.get("Work") or "").strip()
                if not lang or not work:
                    continue
                title = (r.get("Title") or "").strip()
                note = (r.get("Note") or "").strip()
                item = work + (f" ({title})" if title else "") + (f" — {note}" if note else "")
                if lang not in langs:
                    langs[lang] = []
                    order.append(lang)
                langs[lang].append(item)
    if not order:
        _TRANSLATIONS_REF = ""
        return _TRANSLATIONS_REF
    lines = [f"- {lang}: " + "; ".join(langs[lang]) for lang in order]
    _TRANSLATIONS_REF = (
        "Wayne's works available in other languages (authoritative — from the International "
        "Translations page, https://www.lifestream.org/international-translations/). A work is "
        "available in a language ONLY if it is listed here. If a work — or a podcast such as "
        "My Friend Luis or The God Journey — is not listed for a language, it is NOT available in "
        "that language: say you aren't finding one rather than assuming, and NEVER invent a "
        "translated title. Wayne's audio teaching and podcasts are in English unless a translation "
        "is explicitly named below.\n" + "\n".join(lines)
    )
    return _TRANSLATIONS_REF


_GUIDE_NOTES = None


def guide_notes():
    """Curated facts/corrections (from guide-notes.txt) the guide must always
    honor — e.g. spellings the transcripts get wrong, like 'Sara' not 'Sarah'."""
    global _GUIDE_NOTES
    if _GUIDE_NOTES is not None:
        return _GUIDE_NOTES
    p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "guide-notes.txt")
    lines = []
    if os.path.exists(p):
        for ln in open(p, encoding="utf-8"):
            s = ln.strip()
            if s and not s.startswith("#"):
                lines.append(f"- {s}")
    _GUIDE_NOTES = (
        "IMPORTANT NOTES (curated facts and corrections — always honor these, even "
        "over what the source text says):\n" + "\n".join(lines)
    ) if lines else ""
    return _GUIDE_NOTES


_TOPICS_REF = None


def topics_reference():
    """An always-available list of curated 'start here' pages (from
    topics-catalog.csv) — hand-organized entry points, each built around a
    question or topic, that gather the best of Wayne's content on a theme. These
    are exactly the pages retrieval tends to bury (their topics are Wayne's
    densest themes), so the guide keeps them on hand to point people to the
    matching collection directly."""
    global _TOPICS_REF
    if _TOPICS_REF is not None:
        return _TOPICS_REF
    p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "topics-catalog.csv")
    rows = []
    if os.path.exists(p):
        with open(p, encoding="utf-8-sig", newline="") as f:
            for r in csv.DictReader(f):
                topic = (r.get("Topic") or "").strip()
                if not topic:
                    continue
                link = (r.get("Link") or "").strip()
                desc = (r.get("Description") or "").strip()
                line = f"- {topic}" + (f" — {desc}" if desc else "") + (f" ({link})" if link else "")
                rows.append(line)
    _TOPICS_REF = (
        "Curated 'start here' pages, each built around a question or topic, that organize the "
        "best of Wayne's content on a theme. When a reader's question closely matches one of "
        "these, point them to the matching page as a curated collection (alongside anything else "
        "you'd naturally offer). Use only these exact URLs; never invent or alter them:\n"
        + "\n".join(rows)
    ) if rows else ""
    return _TOPICS_REF


def _source_date(*urls):
    """Pull a creation date (YYYY-MM-DD, else YYYY) out of a source URL when one is
    present — God Journey episode URLs carry /YYYY/MM/DD/. Lets the guide anchor a
    relative time reference ("twenty years ago") to when a piece was actually made,
    not to today. Returns "" when no date can be found."""
    for u in urls:
        if not isinstance(u, str):
            continue
        m = re.search(r"/((?:19|20)\d\d)/(\d\d)/(\d\d)/", u)
        if m:
            return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
        m = re.search(r"/((?:19|20)\d\d)/", u)
        if m:
            return m.group(1)
    return ""


_RECENT_REF = None
RECENT_FILE = "recent-episodes.txt"


def _recent_episodes(docs, n=15):
    """The N newest God Journey episodes (dated from their URL), newest first, as
    (date, title, url). Gives the guide date/recency awareness it otherwise lacks."""
    seen = {}
    for d in docs:
        src = d.get("source", "")
        link = d.get("book_link") or (src if str(src).startswith("http") else "")
        if "thegodjourney.com" not in (link or ""):
            continue
        m = re.search(r"/((?:19|20)\d\d)/(\d\d)/(\d\d)/", link)
        if not m:
            continue
        date = f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
        title = d.get("book_title") or display_name(src)
        if link not in seen or date > seen[link][0]:
            seen[link] = (date, title, link)
    return sorted(seen.values(), key=lambda t: t[0], reverse=True)[:n]


def write_recent_episodes(docs, n=15):
    """Called by ingest.py after the store is built: write the newest episodes to
    recent-episodes.txt so the always-on recent_reference() stays current each ingest."""
    items = _recent_episodes(docs, n)
    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), RECENT_FILE)
    with open(path, "w", encoding="utf-8") as f:
        for date, title, url in items:
            f.write(f"{date} — {title} — {url}\n")
    return len(items)


def recent_reference():
    """Always-on: the newest God Journey episodes in the collection (from
    recent-episodes.txt), so the guide can answer 'what's the latest?' and date
    questions from what it actually has instead of guessing."""
    global _RECENT_REF
    if _RECENT_REF is not None:
        return _RECENT_REF
    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), RECENT_FILE)
    lines = []
    if os.path.exists(path):
        for ln in open(path, encoding="utf-8"):
            ln = ln.strip()
            if ln:
                lines.append(f"- {ln}")
    _RECENT_REF = (
        "MOST RECENT God Journey episodes in your collection, newest first (date — title — link). "
        "This is the freshest content you have. If a reader asks what's newest or most recent, for "
        "'this week's' episode, or whether an episode came out on/around a given date, answer from "
        "THIS list — name and link the matching episode. But it is a snapshot from the last content "
        "update, NOT a live feed: an even newer episode may already be out, so add that they can "
        "check thegodjourney.com for anything newer. Never claim an older item is the most recent "
        "when a newer one is listed here.\n" + "\n".join(lines)
    ) if lines else ""
    return _RECENT_REF


def build_user_message(question, retrieved):
    """Assemble the prompt. Passages from for-sale books are marked, capped in
    number, and accompanied by an instruction to summarize rather than reproduce."""
    free = [c for c, _ in retrieved if not c.get("for_sale")]
    paid = [c for c, _ in retrieved if c.get("for_sale")][:MAX_FORSALE]

    def tag(c):
        fs = "true" if c.get("for_sale") else "false"
        src = c["source"]
        name = c.get("book_title") or display_name(src)
        # Always give the model a link when we can: a catalog link, or for
        # articles/episodes the source URL itself.
        link = c.get("book_link") or (src if src.startswith("http") else "")
        author = c.get("attribution")
        extra = f' link="{link}"' if link else ""
        extra += f' author="{author}"' if author else ""
        date = _source_date(link, src)
        extra += f' date="{date}"' if date else ""
        return f'<content source="{name}" for_sale="{fs}"{extra}>\n{c["text"]}\n</content>'

    excerpts = "\n\n".join(tag(c) for c in free + paid)

    guard = ""
    if paid:
        lines, seen = [], set()
        for c in paid:
            key = c.get("book_title") or c["source"]
            if key in seen:
                continue
            seen.add(key)
            url = c.get("book_link")
            lines.append(f"- {key}" + (f" — get it at {url}" if url else ""))
        guard = (
            "\n\nNote: any item marked for_sale=\"true\" is from a book that is for sale. "
            "Don't reproduce it at length — capture the idea in your own words, quote at most "
            "a short phrase, and warmly point the reader to get the book:\n" + "\n".join(lines)
        )

    notes = link_notes()
    notes_block = f"{notes}\n\n" if notes else ""
    facts = guide_notes()
    facts_block = f"{facts}\n\n" if facts else ""

    return (
        f"Material from Wayne's work that may help:\n\n{excerpts}\n\n"
        f"{book_reference()}\n\n"
        f"{series_reference()}\n\n"
        f"{topics_reference()}\n\n"
        f"{translations_reference()}\n\n"
        f"{recent_reference()}\n\n"
        f"{facts_block}{notes_block}"
        f"Reader's question: {question}\n\n"
        "Answer in your own warm, confident voice, grounded in the material and the book list "
        "above. Do NOT mention this material, 'passages', or that you were given excerpts — "
        "just speak as a guide who knows Wayne's work. For any question about his books — which "
        "he's written, co-authors, or where to get them — rely on the book list above and never "
        "invent titles. Do not label books as 'free' or 'for sale'; when someone asks where to get "
        "a book, point them to its link, and you may note that many of Wayne's books are available "
        "as free downloads on Lifestream as well as for purchase. Name the book, article, or "
        "episode you draw from and share its link when "
        "one is given. If a URL you're about to share matches a LINK NOTES entry above, do not "
        "share that link — follow its guidance instead. When an item has an author, name that "
        "author and treat it as distinct from Wayne's own words. If Wayne's work here doesn't speak to the question, say so "
        "briefly and confidently and point to the nearest relevant theme or where to explore "
        "next — never invent." + guard
    )


def answer(question, retrieved):
    """Ask Claude to answer using only the retrieved passages."""
    from anthropic import Anthropic

    client = Anthropic()  # reads ANTHROPIC_API_KEY from the environment
    resp = client.messages.create(
        model=ANSWER_MODEL,
        max_tokens=2000,
        **_SAMPLING,
        system=_load_system_prompt(),
        messages=[{"role": "user", "content": build_user_message(question, retrieved)}],
    )
    return resp.content[0].text


def answer_stream(question, retrieved, history=None):
    """Yield the answer text incrementally, for a live typing effect. `history`
    is a list of prior {role, content} messages, giving conversation memory."""
    from anthropic import Anthropic

    client = Anthropic()
    messages = list(history or [])
    messages.append({"role": "user", "content": build_user_message(question, retrieved)})
    with client.messages.stream(
        model=ANSWER_MODEL,
        max_tokens=2000,
        **_SAMPLING,
        system=_load_system_prompt(),
        messages=messages,
    ) as stream:
        for text in stream.text_stream:
            yield text
