massive overhaul and I pray for no fuckups

components/conversation/__init__.py

@@ -0,0 +1,16 @@
+"""Conversation utilities for WhatsApp assistant."""
+
+from .context import (
+    add_message as add_conversation_message,
+    get_recent_messages as get_conversation_history,
+    clear_conversation,
+)
+
+from .llm_responses import generate_reply
+
+__all__ = [
+    "add_conversation_message",
+    "get_conversation_history",
+    "clear_conversation",
+    "generate_reply",
+]

components/conversation/context.py

@@ -0,0 +1,70 @@
+import json
+import logging
+import os
+import time
+from typing import Dict, List
+
+import redis
+
+logger = logging.getLogger(__name__)
+
+REDIS_URL = os.environ.get("UPSTASH_REDIS_URL", "redis://localhost:6379")
+_CONTEXT_MAX_MESSAGES = int(os.environ.get("CONVERSATION_CONTEXT_LIMIT", "12"))
+
+try:
+    _redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
+except Exception as exc:  # pragma: no cover - defensive
+    logger.error("Failed to connect to Redis for conversation context: %s", exc)
+    _redis_client = None
+
+
+def _ctx_key(user_id: str) -> str:
+    return f"conversation:ctx:{user_id}"
+
+
+def add_message(user_id: str, role: str, content: str, limit: int = _CONTEXT_MAX_MESSAGES) -> None:
+    if not user_id or not content or not _redis_client:
+        return
+    entry = json.dumps({
+        "ts": time.time(),
+        "role": role,
+        "content": content,
+    })
+    key = _ctx_key(user_id)
+    try:
+        _redis_client.lpush(key, entry)
+        _redis_client.ltrim(key, 0, max(limit * 2 - 1, limit - 1))
+    except Exception as exc:  # pragma: no cover - network side-effects
+        logger.debug("Failed to append conversation message for %s: %s", user_id, exc)
+
+
+def get_recent_messages(user_id: str, limit: int = _CONTEXT_MAX_MESSAGES) -> List[Dict[str, str]]:
+    if not user_id or not _redis_client:
+        return []
+    key = _ctx_key(user_id)
+    try:
+        raw = _redis_client.lrange(key, 0, limit - 1)
+    except Exception as exc:  # pragma: no cover - network side-effects
+        logger.debug("Failed to fetch conversation messages for %s: %s", user_id, exc)
+        return []
+    messages: List[Dict[str, str]] = []
+    for payload in reversed(raw):
+        try:
+            entry = json.loads(payload)
+            if isinstance(entry, dict) and entry.get("content"):
+                messages.append({
+                    "role": entry.get("role", "user"),
+                    "content": str(entry.get("content")),
+                })
+        except (TypeError, json.JSONDecodeError):
+            continue
+    return messages
+
+
+def clear_conversation(user_id: str) -> None:
+    if not user_id or not _redis_client:
+        return
+    try:
+        _redis_client.delete(_ctx_key(user_id))
+    except Exception as exc:  # pragma: no cover
+        logger.debug("Failed to clear conversation for %s: %s", user_id, exc)

components/conversation/llm_responses.py

@@ -0,0 +1,95 @@
+import logging
+from typing import Dict, List, Optional
+
+from components.LLMs.Mistral import MistralTogetherClient, build_messages
+
+logger = logging.getLogger(__name__)
+
+_CATEGORY_DESCRIPTIONS: Dict[str, str] = {
+    "greeting": "Welcome the user warmly, mention Nuse and offer quick pointers for using the assistant.",
+    "help": "Explain concisely what Nuse can do. Use short bullet-style sentences separated by new lines.",
+    "preferences_prompt": (
+        "Ask the user to choose from the available news topics. State the list clearly and explain how to respond."
+    ),
+    "preferences_confirm": (
+        "Confirm the topics the user selected. Sound upbeat and let them know tailored headlines will follow."
+    ),
+    "small_talk": "Respond briefly and positively to casual chatter while nudging toward news-related help if relevant.",
+    "unsubscribe": "Acknowledge the unsubscribe request politely and explain they can message again to resume.",
+    "headlines_intro": "Introduce the upcoming digest, referencing the user's preferred topics if provided.",
+}
+
+
+def _format_context(context_messages: Optional[List[Dict[str, str]]], limit: int = 8) -> str:
+    if not context_messages:
+        return ""
+    trimmed = context_messages[-limit:]
+    exchanges = []
+    for item in trimmed:
+        role = item.get("role", "user")
+        content = item.get("content", "")
+        if not content:
+            continue
+        label = "User" if role == "user" else "Nuse"
+        exchanges.append(f"{label}: {content}")
+    return "\n".join(exchanges)
+
+
+def generate_reply(
+    category: str,
+    context_messages: Optional[List[Dict[str, str]]],
+    user_message: Optional[str] = None,
+    extra: Optional[Dict[str, str]] = None,
+) -> str:
+    description = _CATEGORY_DESCRIPTIONS.get(category)
+    if not description:
+        logger.debug("No category description for %s; falling back to default prompt", category)
+        description = "Respond helpfully to the user based on the latest exchange."
+
+    context_block = _format_context(context_messages)
+    extras = extra or {}
+    latest = user_message or extras.get("latest_user_message") or ""
+
+    prompt_lines = [
+        "You are Nuse, a concise WhatsApp assistant focused on personalised news.",
+        description,
+    ]
+
+    if extras.get("topics_list"):
+        prompt_lines.append(
+            "Available topics: " + ", ".join(extras["topics_list"])
+        )
+    if extras.get("confirmed_topics"):
+        prompt_lines.append(
+            "User has selected: " + ", ".join(extras["confirmed_topics"])
+        )
+    if extras.get("notes"):
+        prompt_lines.append(extras["notes"])
+
+    system_prompt = " \n".join(prompt_lines)
+
+    user_sections = []
+    if context_block:
+        user_sections.append(f"Conversation so far:\n{context_block}")
+    if latest:
+        user_sections.append(f"Latest user message:\n{latest}")
+
+    if extras.get("headlines_preview"):
+        user_sections.append(
+            "Upcoming digest summary:\n" + extras["headlines_preview"]
+        )
+
+    user_prompt = "\n\n".join(user_sections) or "Respond appropriately."
+
+    llm = MistralTogetherClient()
+    messages = build_messages(user_prompt, system_prompt)
+    try:
+        text, _usage = llm.chat(messages, temperature=0.2, max_tokens=220)
+        return text.strip()
+    except Exception as exc:  # pragma: no cover - network issues
+        logger.error("LLM reply generation failed for %s: %s", category, exc)
+        fallback = {
+            "greeting": "Hi! I'm Nuse. Ask me for headlines or news on any topic when you're ready.",
+            "help": "I can send personalised headlines, answer news questions, and manage your preferences.",
+        }
+        return fallback.get(category, "Thanks for reaching out! How can I help with the news today?")

components/handlers/wa/common.py

@@ -5,6 +5,7 @@ from components.gateways.headlines_to_wa import (
     send_to_whatsapp,
     send_typing_indicator,
 )
+from components.conversation import add_conversation_message
 
 try:  # Memory is optional; degrade gracefully if unavailable
     from components.memory.chat_vector_store import add_chat_turn_to_memory as _store_memory_turn
@@ -33,6 +34,7 @@ def safe_send(text: str, to: str) -> Dict:
         res = send_to_whatsapp(text, destination_number=to)
         if res.get("status") == "success":
             logger.info("Sent message to %s", to)
+            add_conversation_message(to, "assistant", text)
             if _store_memory_turn:
                 try:
                     _store_memory_turn(to, "assistant", text)

components/handlers/wa/research.py

@@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse
 
 from components.LLMs.Mistral import MistralTogetherClient, build_messages
 from components.retrievers.web_qa import get_web_enriched_context
+from components.conversation import get_conversation_history
 
 from .common import safe_send
 
@@ -132,6 +133,8 @@ def _answer_with_context(question: str, context: str, memories: Optional[List[st
 def handle_general_research(from_number: str, message_text: str) -> JSONResponse:
     logger.info("General research question from %s: %s", from_number, message_text)
 
+    conversation_history = get_conversation_history(from_number)
+
     memory_lines: List[str] = []
     if _retrieve_memories:
         try:
@@ -150,7 +153,15 @@ def handle_general_research(from_number: str, message_text: str) -> JSONResponse
     else:
         logger.info("[memory] no preserved lines for %s", from_number)
 
-    related_for_search = memory_lines if memory_lines else None
+    context_snippets: List[str] = [
+        msg.get("content", "")
+        for msg in conversation_history
+        if msg.get("role") == "user" and msg.get("content")
+    ][-4:]
+    related_for_search = [*memory_lines] if memory_lines else []
+    if context_snippets:
+        related_for_search.extend(context_snippets)
+    related_for_search = related_for_search or None
 
     context, sources = get_web_enriched_context(
         message_text,

components/handlers/wa/shortcuts.py

@@ -1,7 +1,13 @@
 import logging
+from typing import Optional
+
 from fastapi.responses import JSONResponse
 
 from components.gateways.headlines_to_wa import fetch_cached_headlines
+from components.conversation import (
+    get_conversation_history,
+    generate_reply,
+)
 from storage.users import get_user
 
 from .common import safe_send
@@ -10,7 +16,7 @@ from .common import safe_send
 logger = logging.getLogger(__name__)
 
 
-def handle_headlines(from_number: str) -> JSONResponse:
+def handle_headlines(from_number: str, latest_user_message: Optional[str] = None) -> JSONResponse:
     user = get_user(from_number) or {}
     prefs = user.get("preferences", {}) if isinstance(user, dict) else {}
     preferred_topics = prefs.get("topics") if isinstance(prefs, dict) else None
@@ -19,67 +25,103 @@ def handle_headlines(from_number: str) -> JSONResponse:
 
     if full_message_text.startswith("❌") or full_message_text.startswith("⚠️"):
         logger.error("Failed to fetch digest for %s: %s", from_number, full_message_text)
-        safe_send(
-            f"Sorry, I couldn't fetch the news digest today. {full_message_text}",
-            to=from_number,
+        apology = generate_reply(
+            "headlines_intro",
+            get_conversation_history(from_number),
+            user_message=latest_user_message,
+            extra={
+                "notes": "Explain that the digest could not be fetched and apologise succinctly.",
+            },
         )
+        safe_send(apology or "Sorry, I couldn't fetch the news digest today.", to=from_number)
         return JSONResponse(
             status_code=500,
             content={"status": "error", "message": "Failed to fetch digest"},
         )
 
+    intro = generate_reply(
+        "headlines_intro",
+        get_conversation_history(from_number),
+        user_message=latest_user_message,
+        extra={
+            "confirmed_topics": [
+                t.title() for t in (preferred_topics or [])
+            ] or None,
+        },
+    )
+    if intro:
+        safe_send(intro, to=from_number)
+
     result = safe_send(full_message_text, to=from_number)
     if result.get("status") == "success":
         return JSONResponse(status_code=200, content={"status": "success", "message": "Digest sent"})
 
+    failure_note = generate_reply(
+        "headlines_intro",
+        get_conversation_history(from_number),
+        user_message=latest_user_message,
+        extra={
+            "notes": "Explain that sending the digest failed and suggest trying again later.",
+        },
+    )
     safe_send(
-        "Sorry, I couldn't send the news digest to you. "
-        f"Error: {result.get('error', 'unknown')}",
+        failure_note
+        or "Sorry, I couldn't send the news digest to you. Please try again in a bit.",
         to=from_number,
     )
     return JSONResponse(status_code=500, content={"status": "error", "message": "Failed to send digest"})
 
 
-def handle_preferences(from_number: str) -> JSONResponse:
-    msg = (
-        "Let’s tune your feed. Reply with topics you like:\n"
-        "• India • World • Technology • Business & Markets • Sports\n\n"
-        "You can send multiple, e.g.: India, Technology"
+def handle_preferences(from_number: str, latest_user_message: Optional[str] = None) -> JSONResponse:
+    reply = generate_reply(
+        "preferences_prompt",
+        get_conversation_history(from_number),
+        user_message=latest_user_message,
+        extra={
+            "topics_list": ["India", "World", "Technology", "Business & Markets", "Sports"],
+        },
     )
-    safe_send(msg, to=from_number)
+    safe_send(reply, to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Preferences prompt sent"})
 
 
-def handle_greeting(from_number: str) -> JSONResponse:
-    msg = (
-        "Hey! 👋 I’m NuseAI.\n"
-        "• Type *headlines* to get today’s digest.\n"
-        "• Type *preferences* to set your topics.\n"
-        "• Type *help* to see what I can do."
+def handle_greeting(from_number: str, latest_user_message: Optional[str] = None) -> JSONResponse:
+    reply = generate_reply(
+        "greeting",
+        get_conversation_history(from_number),
+        user_message=latest_user_message,
     )
-    safe_send(msg, to=from_number)
+    safe_send(reply, to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Greeting sent"})
 
 
-def handle_help(from_number: str) -> JSONResponse:
-    msg = (
-        "Here’s how I can help:\n"
-        "• *Headlines* — today’s 🗞️ Daily Digest 🟡\n"
-        "• *Preferences* — choose topics/regions\n"
-        "• *Unsubscribe* — stop messages\n"
-        "Ask me a question anytime (e.g., “What’s India’s CPI outlook?”)."
+def handle_help(from_number: str, latest_user_message: Optional[str] = None) -> JSONResponse:
+    reply = generate_reply(
+        "help",
+        get_conversation_history(from_number),
+        user_message=latest_user_message,
     )
-    safe_send(msg, to=from_number)
+    safe_send(reply, to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Help sent"})
 
 
-def handle_unsubscribe(from_number: str) -> JSONResponse:
-    safe_send("You’re unsubscribed. If you change your mind, just say *hi*.", to=from_number)
+def handle_unsubscribe(from_number: str, latest_user_message: Optional[str] = None) -> JSONResponse:
+    reply = generate_reply(
+        "unsubscribe",
+        get_conversation_history(from_number),
+        user_message=latest_user_message,
+    )
+    safe_send(reply, to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Unsubscribed"})
 
 
-def handle_small_talk(from_number: str) -> JSONResponse:
-    safe_send("🙂 Got it. If you’d like the news, just say *headlines*.", to=from_number)
+def handle_small_talk(from_number: str, latest_user_message: Optional[str] = None) -> JSONResponse:
+    reply = generate_reply(
+        "small_talk",
+        get_conversation_history(from_number),
+        user_message=latest_user_message,
+    )
+    safe_send(reply, to=from_number)
     return JSONResponse(status_code=200, content={"status": "success", "message": "Small talk"})
 
 

routes/api/whatsapp_webhook.py

@@ -30,6 +30,10 @@ from components.handlers.wa import (
     handle_onboarding_preferences_text,
     handle_onboarding_preferences_flow_submission,
 )
+from components.conversation import (
+    add_conversation_message,
+    get_conversation_history,
+)
 
 try:  # Optional memory integration
     from components.memory.chat_vector_store import add_chat_turn_to_memory as _store_chat_turn
@@ -284,6 +288,15 @@ async def whatsapp_webhook_receiver(request: Request):
 
         logging.info(f"Message from {from_number}: {message_text}")
 
+        if message_text:
+            add_conversation_message(from_number, "user", message_text)
+        elif flow_topic_ids:
+            add_conversation_message(
+                from_number,
+                "user",
+                "User submitted flow topics: " + ", ".join(flow_topic_ids),
+            )
+
         if msg_id:
             send_seen_receipt(from_number, msg_id)
 
@@ -356,17 +369,17 @@ async def whatsapp_webhook_receiver(request: Request):
 
         # Routing
         if label == "headlines_request":
-            return handle_headlines(from_number)
+            return handle_headlines(from_number, latest_user_message=message_text)
         elif label == "preferences_update":
-            return handle_preferences(from_number)
+            return handle_preferences(from_number, latest_user_message=message_text)
         elif label == "greeting":
-            return handle_greeting(from_number)
+            return handle_greeting(from_number, latest_user_message=message_text)
         elif label == "help":
-            return handle_help(from_number)
+            return handle_help(from_number, latest_user_message=message_text)
         elif label == "unsubscribe":
-            return handle_unsubscribe(from_number)
+            return handle_unsubscribe(from_number, latest_user_message=message_text)
         elif label == "small_talk":
-            return handle_small_talk(from_number)
+            return handle_small_talk(from_number, latest_user_message=message_text)
         elif label == "chat_question":
             return handle_chat_question(from_number, message_text)
         elif label == "general_research":