revamping again

components/handlers/whatsapp_handlers.py

@@ -2,16 +2,17 @@
 import logging
 import os
 import re
-from typing import Optional, Dict
+import itertools
+from typing import Optional, Dict, List, Tuple
 
 from fastapi.responses import JSONResponse
 
 from components.gateways.headlines_to_wa import fetch_cached_headlines, send_to_whatsapp
 from components.LLMs.Mistral import MistralTogetherClient, build_messages
 
-# ------------------------------------------------------------
-# Utilities
-# ------------------------------------------------------------
+# ----------------------------
+# Messaging utility
+# ----------------------------
 
 def _safe_send(text: str, to: str) -> dict:
     """Wrap send_to_whatsapp with logging & safe error handling."""
@@ -26,10 +27,9 @@ def _safe_send(text: str, to: str) -> dict:
         logging.exception(f"Exception while sending WhatsApp message to {to}: {e}")
         return {"status": "error", "error": str(e)}
 
-
-# ------------------------------------------------------------
+# ----------------------------
 # Headlines
-# ------------------------------------------------------------
+# ----------------------------
 
 def handle_headlines(from_number: str) -> JSONResponse:
     full_message_text = fetch_cached_headlines()
@@ -49,10 +49,9 @@ def handle_headlines(from_number: str) -> JSONResponse:
         )
         return JSONResponse(status_code=500, content={"status": "error", "message": "Failed to send digest"})
 
-
-# ------------------------------------------------------------
+# ----------------------------
 # Preferences / Greeting / Help / Unsubscribe / Small Talk
-# ------------------------------------------------------------
+# ----------------------------
 
 def handle_preferences(from_number: str) -> JSONResponse:
     msg = (
@@ -63,7 +62,6 @@ def handle_preferences(from_number: str) -> JSONResponse:
     _safe_send(msg, to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Preferences prompt sent"})
 
-
 def handle_greeting(from_number: str) -> JSONResponse:
     msg = (
         "Hey! 👋 I’m NuseAI.\n"
@@ -74,7 +72,6 @@ def handle_greeting(from_number: str) -> JSONResponse:
     _safe_send(msg, to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Greeting sent"})
 
-
 def handle_help(from_number: str) -> JSONResponse:
     msg = (
         "Here’s how I can help:\n"
@@ -86,51 +83,39 @@ def handle_help(from_number: str) -> JSONResponse:
     _safe_send(msg, to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Help sent"})
 
-
 def handle_unsubscribe(from_number: str) -> JSONResponse:
     _safe_send("You’re unsubscribed. If you change your mind, just say *hi*.", to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Unsubscribed"})
 
-
 def handle_small_talk(from_number: str) -> JSONResponse:
     _safe_send("🙂 Got it. If you’d like the news, just say *headlines*.", to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Small talk"})
 
-
-# ------------------------------------------------------------
-# Chat Question → “Explain by number” flow (structured + quality-guarded)
-# ------------------------------------------------------------
+# ----------------------------
+# Chat Question → ELI5 by number (Hybrid retrieval + better UX)
+# ----------------------------
 
 _HEADLINE_LINE_RE = re.compile(r"^\s*(\d+)\.\s+(.*)$")
+MAX_CONTEXT_CHARS = 4000  # ~2–2.5k tokens budget for LLM
+CONTEXT_LOG_PREVIEW = 1200  # log up to 1.2k chars to avoid huge logs
 
 def _extract_number_ref(text: str) -> Optional[int]:
-    """
-    Find a referenced headline number in free text, e.g.:
-    'explain number 14', 'no. 7 please', '#9', '14', 'explain 14 like I am 5'
-    Returns int or None.
-    """
+    """Detect headline number references like: 'explain #14', 'no. 7', '14'."""
     s = (text or "").lower()
 
-    # explicit forms
     m = re.search(r"(?:number|no\.?|num|#)\s*(\d+)", s)
     if m:
         return int(m.group(1))
 
-    # a bare number (avoid picking up years like 2025; cap at 1..200)
     m2 = re.search(r"\b(\d{1,3})\b", s)
     if m2:
         n = int(m2.group(1))
         if 1 <= n <= 200:
             return n
-
     return None
 
-
 def _parse_rendered_digest(rendered: str) -> Dict[int, str]:
-    """
-    Parse the same rendered digest string you send on WhatsApp and build a map:
-    { number -> headline_line_text }
-    """
+    """Build {number -> headline text} from the rendered WhatsApp digest."""
     mapping: Dict[int, str] = {}
     for line in (rendered or "").splitlines():
         m = _HEADLINE_LINE_RE.match(line)
@@ -141,150 +126,268 @@ def _parse_rendered_digest(rendered: str) -> Dict[int, str]:
         mapping[num] = headline_txt
     return mapping
 
+# ---------- HYBRID RETRIEVAL ----------
 
-def _retrieve_context_for_headline(headline_text: str, top_k: int = 15) -> str:
+def _collect_doc_nodes(index, doc_ids: List[str], max_nodes: int = 300) -> List[str]:
     """
-    Use the vector index to pull contextual passages related to the headline.
-    - Uses a higher top_k to widen coverage (quality over speed).
-    - Gracefully degrades if index is unavailable or not yet built.
+    Given LlamaIndex 'index' and a list of document IDs, fetch lots of node texts
+    from those docs. Fallback-friendly across index versions.
     """
-    # Defer the import so a missing/invalid index module won't break imports
+    texts: List[str] = []
     try:
-        from components.indexers.news_indexer import load_news_index  # type: ignore
-    except Exception as e:
-        logging.warning(f"Index module not available yet: {e}")
-        return ""
+        docstore = getattr(index, "docstore", None) or getattr(index, "storage_context", None).docstore  # type: ignore
+        if not docstore:
+            return texts
 
-    # Try to load the index; if persist_dir is wrong/missing, swallow and return ""
-    try:
-        index = load_news_index()
+        # Try to scan nodes; APIs vary by version
         try:
-            # LlamaIndex v0.10+
-            qe = index.as_query_engine(similarity_top_k=top_k)
+            all_node_ids = list(docstore._ref_doc_info.keys())  # type: ignore
         except Exception:
-            # Older API fallback
-            from llama_index.core.query_engine import RetrievalQueryEngine  # type: ignore
-            qe = RetrievalQueryEngine(index=index, similarity_top_k=top_k)
-
-        query = (
-            "Retrieve concise, factual context that best explains this headline:\n"
-            f"{headline_text}\n"
-            "Focus on who/what/when/where/why, include crucial numbers, avoid speculation."
-        )
-        resp = qe.query(query)
-        return str(resp)
+            all_node_ids = list(getattr(docstore, "docs", {}).keys())  # type: ignore
+
+        count = 0
+        for node_id in all_node_ids:
+            try:
+                node = docstore.get_node(node_id)
+                ref_id = getattr(node, "ref_doc_id", None) or (node.metadata.get("doc_id") if hasattr(node, "metadata") else None)
+                if ref_id in doc_ids:
+                    # get_content() in newer versions, else get_text()
+                    text = str(getattr(node, "get_content", lambda: node.get_text())())
+                    if text:
+                        texts.append(text)
+                        count += 1
+                        if count >= max_nodes:
+                            break
+            except Exception:
+                continue
+    except Exception:
+        pass
+    return texts
+
+def _retrieve_context_for_headline(headline_text: str, first_top_k: int = 8) -> str:
+    """
+    HYBRID retrieval:
+      A) similarity search to get seed nodes
+      B) expand to include full documents of those nodes
+      C) merge/dedupe + progressive widening if too little context
+    Returns a single big string (bounded by MAX_CONTEXT_CHARS).
+    """
+    try:
+        from components.indexers.news_indexer import load_news_index  # deferred import
     except Exception as e:
-        # Avoid noisy tracebacks in normal operation; index may simply not exist yet
-        persist_dir = os.getenv("NEWS_INDEX_PERSIST_DIR") or os.getenv("PERSIST_DIR") or "<unset>"
-        logging.warning(f"Vector retrieval skipped (no index at {persist_dir}): {e}")
+        logging.warning(f"Index module not available yet: {e}")
         return ""
 
+    def _query(top_k: int) -> Tuple[List[str], List[str]]:
+        """Return (seed_texts, seed_doc_ids)."""
+        try:
+            index = load_news_index()
+            try:
+                qe = index.as_query_engine(similarity_top_k=top_k)
+            except Exception:
+                from llama_index.core.query_engine import RetrievalQueryEngine  # type: ignore
+                qe = RetrievalQueryEngine(index=index, similarity_top_k=top_k)
+
+            q = (
+                "Return the most relevant passages that explain this headline:\n"
+                f"{headline_text}\n"
+                "Prioritize who/what/when/where/why and crucial numbers."
+            )
+            resp = qe.query(q)
+
+            seed_texts: List[str] = []
+            seed_doc_ids: List[str] = []
+            source_nodes = getattr(resp, "source_nodes", []) or []
+            for sn in source_nodes:
+                try:
+                    txt = str(getattr(sn.node, "get_content", lambda: sn.node.get_text())())
+                    if txt:
+                        seed_texts.append(txt)
+                    ref_id = getattr(sn.node, "ref_doc_id", None) or (sn.node.metadata.get("doc_id") if hasattr(sn.node, "metadata") else None)
+                    if ref_id:
+                        seed_doc_ids.append(ref_id)
+                except Exception:
+                    continue
+
+            if not seed_texts:
+                seed_texts = [str(resp)] if str(resp).strip() else []
+
+            return seed_texts, list(dict.fromkeys(seed_doc_ids))  # dedupe doc_ids, keep order
+        except Exception as e:
+            logging.warning(f"Similarity retrieval failed (top_k={top_k}): {e}")
+            return [], []
+
+    # Progressive widening
+    for tk in (first_top_k, 15, 25):
+        seed_texts, doc_ids = _query(tk)
+
+        # Expand to all nodes from those documents
+        expanded_texts: List[str] = []
+        try:
+            if doc_ids:
+                index = load_news_index()
+                expanded_texts = _collect_doc_nodes(index, doc_ids, max_nodes=300)
+        except Exception as e:
+            logging.debug(f"Doc expansion failed: {e}")
+
+        # Merge & dedupe
+        combined = []
+        seen = set()
+        for chunk in itertools.chain(seed_texts, expanded_texts):
+            c = (chunk or "").strip()
+            if not c:
+                continue
+            h = hash(c[:512])  # crude dedupe
+            if h in seen:
+                continue
+            seen.add(h)
+            combined.append(c)
+
+        if not combined:
+            continue
+
+        # Truncate to budget
+        out = []
+        total = 0
+        for c in combined:
+            if total + len(c) > MAX_CONTEXT_CHARS:
+                break
+            out.append(c)
+            total += len(c)
+
+        if total > 800:  # enough to answer
+            return "\n\n---\n\n".join(out)
+
+    # Nothing useful
+    return ""
+
+# ---------- ELI5 generation (friendlier UX) ----------
+
+def _confidence_from_context(ctx: str) -> str:
+    """Heuristic confidence, logged (not shown to user)."""
+    n = len(ctx or "")
+    if n > 3000: return "High"
+    if n > 1400: return "Medium"
+    return "Low"
+
+def _format_eli5_output(headline: str, key_points: List[str], facts: List[str], why: List[str], unknowns: List[str], tips: List[str]) -> str:
+    """Format a compact WhatsApp-friendly message WITHOUT a confidence line."""
+    def _top(xs, k): return [x for x in xs if x][:k]
+    key_points = _top(key_points, 3)
+    facts     = _top(facts, 2)
+    why       = _top(why, 2)
+    unknowns  = _top(unknowns, 2)
+    tips      = _top(tips, 1)
+
+    parts = [
+        f"Headline:\n{headline}".strip(),
+        ("In plain words:\n" + " ".join(key_points)) if key_points else None,
+        ("Numbers & facts:\n• " + "\n• ".join(facts)) if facts else "Numbers & facts:\n• We don’t know yet",
+        ("Why it matters:\n• " + "\n• ".join(why)) if why else None,
+        ("What we don’t know yet:\n• " + "\n• ".join(unknowns)) if unknowns else None,
+        ("What you can do:\n• " + tips[0]) if tips else None,
+    ]
+    text = "\n\n".join([p for p in parts if p]).strip()
+    if len(text) > 900:
+        text = text[:900].rstrip() + "…"
+    return text
 
 def _eli5_answer_structured(question: str, context: str, headline_only: Optional[str] = None) -> str:
     """
-    Generate a structured, quality-guarded ELI5 answer.
-    Format:
-    Headline #N — <short title>
-    Key points:
-    • ...
-    • ...
-    Numbers & facts:
-    • ...
-    Why it matters:
-    • ...
-    Caveats:
-    • ...
-    Confidence: High/Medium/Low
-
-    Rules:
-    - 120–180 words total.
-    - Use ONLY the provided context/headline; if missing, write “Not in context”.
-    - No speculation; keep neutral tone; be brief.
+    Generate structured ELI5; we return user-friendly text,
+    and LOG confidence + context preview (not shown to user).
     """
+    # System + user prompts (strict, to prevent hallucinations)
     sys_prompt = (
-        "You are a rigorous, concise explainer for a news assistant. "
-        "Produce clear, structured outputs with bullet points. "
-        "If any detail is not present in context, write 'Not in context'. "
-        "Avoid flowery language; be factual and neutral."
+        "You are a careful explainer. Use ONLY the provided material. "
+        "Prefer simple words and short sentences (aim for a 10-year-old reader). "
+        "Never invent details. If something is missing, say 'We don’t know yet'."
+    )
+
+    json_schema = (
+        '{'
+        '"headline": "<short title or the headline itself>",'
+        '"plain_points": ["<2-3 child-friendly lines>"],'
+        '"facts": ["<up to 2 concrete numbers/dates/entities>"],'
+        '"why": ["<1-2 reasons why this matters>"],'
+        '"unknowns": ["<1-2 things we don’t know yet>"],'
+        '"tips": ["<1 short tip if applicable, else empty>"]'
+        '}'
     )
 
-    if context.strip():
+    if (context or "").strip():
         user_prompt = (
             f"QUESTION:\n{question}\n\n"
-            f"CONTEXT (may be partial, use ONLY this):\n{context}\n\n"
-            "Write 120–180 words in this exact structure:\n"
-            "Headline:\n"
-            "Key points:\n"
-            "• ...\n• ...\n• ...\n"
-            "Numbers & facts:\n"
-            "• ...\n• ...\n"
-            "Why it matters:\n"
-            "• ...\n"
-            "Caveats:\n"
-            "• ...\n"
-            "Confidence: High | Medium | Low\n"
-            "Rules:\n"
-            "- If you can't find a detail in CONTEXT, write 'Not in context'.\n"
-            "- Do NOT add sources or links unless they appear in CONTEXT.\n"
-            "- Keep it short, precise, and neutral.\n"
+            f"CONTEXT (use ONLY this):\n{context}\n\n"
+            "Extract ONLY grounded details. Return STRICT JSON:\n"
+            + json_schema +
+            "\nRules:\n"
+            "- If a field has nothing grounded, use [] or empty string.\n"
+            "- Simple words. No citations or URLs."
         )
     else:
-        # fallback: rely on the headline only
         headline_text = headline_only or question
         user_prompt = (
-            "CONTEXT is empty. You must base the answer ONLY on the HEADLINE below; "
-            "write 'Not in context' for any missing specifics.\n\n"
-            f"HEADLINE:\n{headline_text}\n\n"
-            "Write 90–140 words in this exact structure:\n"
-            "Headline:\n"
-            "Key points:\n"
-            "• ...\n• ...\n"
-            "Numbers & facts:\n"
-            "• Not in context\n"
-            "Why it matters:\n"
-            "• ...\n"
-            "Caveats:\n"
-            "• Limited details available\n"
-            "Confidence: Low\n"
+            "There is NO additional context. Use ONLY this HEADLINE:\n"
+            f"{headline_text}\n\n"
+            "Return STRICT JSON like:\n"
+            + json_schema
         )
 
     try:
         llm = MistralTogetherClient()
         msgs = build_messages(user_prompt, sys_prompt)
-        out, _usage = llm.chat(msgs, temperature=0.2, max_tokens=400)
-        return out.strip()
+        raw, _usage = llm.chat(msgs, temperature=0.2, max_tokens=450)
+
+        # Parse JSON robustly
+        import json as _json
+        try:
+            start = raw.find("{"); end = raw.rfind("}")
+            payload = _json.loads(raw[start:end+1]) if (start != -1 and end != -1) else _json.loads(raw)
+        except Exception:
+            payload = {"headline": headline_only or "", "plain_points": [], "facts": [], "why": [], "unknowns": ["We don’t know yet"], "tips": []}
+
+        headline = (payload.get("headline") or (headline_only or "")).strip()
+        plain    = [p.strip() for p in payload.get("plain_points", []) if p.strip()]
+        facts    = [p.strip() for p in payload.get("facts", []) if p.strip()]
+        why      = [p.strip() for p in payload.get("why", []) if p.strip()]
+        unknowns = [p.strip() for p in payload.get("unknowns", []) if p.strip()]
+        tips     = [p.strip() for p in payload.get("tips", []) if p.strip()]
+
+        # Compute + LOG confidence (not shown to user)
+        conf = _confidence_from_context(context or "")
+        logging.info(f"[ELI5] confidence={conf}; ctx_len={len(context or '')}")
+
+        return _format_eli5_output(headline or (headline_only or "This item"), plain, facts, why, unknowns, tips)
+
     except Exception as e:
         logging.exception(f"Mistral structured ELI5 generation failed: {e}")
-        return (
-            "Headline:\n"
-            "Key points:\n"
-            "• I couldn’t generate an explanation right now.\n"
-            "Numbers & facts:\n"
-            "• Not in context\n"
-            "Why it matters:\n"
-            "• Not in context\n"
-            "Caveats:\n"
-            "• System error\n"
-            "Confidence: Low"
+        return _format_eli5_output(
+            headline_only or "This item",
+            ["I can’t explain more right now."],
+            [],
+            [],
+            ["We don’t know yet"],
+            [],
         )
 
+# ----------------------------
+# Handler
+# ----------------------------
 
 def handle_chat_question(from_number: str, message_text: str) -> JSONResponse:
     """
-    Smart handler:
-      - If the user references a headline number (“explain 14 like I’m 5”),
-        1) Parse the number
-        2) Look up that numbered line from the rendered digest
-        3) Retrieve vector context (top_k widened for coverage)
-        4) Generate a STRUCTURED ELI5 answer (with quality guardrails)
-      - Otherwise, provide a gentle hint (for now).
+    If user references a headline number:
+      1) Map number → headline from rendered digest
+      2) HYBRID retrieve context (seed + doc expansion)
+      3) LOG the retrieved context (length + preview)
+      4) Generate friendly ELI5 (no confidence shown to user)
+    Else: gentle guidance.
     """
     logging.info(f"Chat question from {from_number}: {message_text}")
 
-    # 1) Try to find a headline number reference
     number = _extract_number_ref(message_text or "")
     if number is not None:
-        # 2) Load rendered digest and map numbers to lines
         rendered = fetch_cached_headlines()
         mapping = _parse_rendered_digest(rendered)
         target_line = mapping.get(number)
@@ -297,18 +400,22 @@ def handle_chat_question(from_number: str, message_text: str) -> JSONResponse:
             )
             return JSONResponse(status_code=200, content={"status": "success", "message": "Number not found"})
 
-        # 3) Retrieve broader context from the vector index using the headline line
-        ctx = _retrieve_context_for_headline(target_line, top_k=15)
+        # Retrieve context
+        ctx = _retrieve_context_for_headline(target_line, first_top_k=8)
+
+        # LOG the retrieved context details for debugging
+        ctx_len = len(ctx or "")
+        preview = (ctx or "")[:CONTEXT_LOG_PREVIEW].replace("\n", "\\n")
+        logging.info(f"[ELI5] Retrieved context len={ctx_len} for #{number}")
+        logging.debug(f"[ELI5] Context preview for #{number}: {preview}{'…' if ctx_len > CONTEXT_LOG_PREVIEW else ''}")
 
-        # 4) Generate STRUCTURED ELI5 answer (works even if ctx == "")
+        # Generate answer (no confidence shown to user)
         question = f"Explain headline #{number}: {target_line}"
         answer = _eli5_answer_structured(question, ctx, headline_only=target_line)
 
-        # 5) Send back
         _safe_send(answer, to=from_number)
         return JSONResponse(status_code=200, content={"status": "success", "message": "ELI5 sent"})
 
-    # No number found → for now, guide the user
     _safe_send(
         "Ask me about a specific headline by number, e.g., *explain 7 like I’m 5*.\n"
         "Or type *headlines* for today’s digest.",