Update app.py

app.py

@@ -6,232 +6,250 @@ import logging
 import time
 import json
 from datetime import datetime
-import hashlib
 from langchain.vectorstores import Chroma
 from langchain.embeddings import HuggingFaceEmbeddings
 
+# --- Basic Flask App Setup ---
 app = Flask(__name__)
-
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+# --- Global State Management ---
 loading_complete = True
-last_update_time = time.time()
-last_data_hash = None
+last_update_time = None 
 
+# --- Embedding and Vector DB Management ---
 def get_embedding_model():
+    """Initializes and returns a singleton HuggingFace embedding model."""
+    # Using a simple hasattr check for a singleton pattern
     if not hasattr(get_embedding_model, "model"):
         get_embedding_model.model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
     return get_embedding_model.model
 
 def get_vector_db():
+    """Initializes and returns a singleton Chroma DB client."""
     if not os.path.exists(LOCAL_DB_DIR):
+        logger.warning(f"Vector DB not found at {LOCAL_DB_DIR}. It may need to be downloaded or created.")
         return None
     try:
-        return Chroma(
-            persist_directory=LOCAL_DB_DIR,
-            embedding_function=get_embedding_model(),
-            collection_name="news_articles"
-        )
+        # Using a simple hasattr check for a singleton pattern
+        if not hasattr(get_vector_db, "db_instance"):
+            get_vector_db.db_instance = Chroma(
+                persist_directory=LOCAL_DB_DIR,
+                embedding_function=get_embedding_model(),
+                collection_name="news_articles"
+            )
+        return get_vector_db.db_instance
     except Exception as e:
         logger.error(f"Failed to load vector DB: {e}")
+        # Invalidate instance on failure
+        if hasattr(get_vector_db, "db_instance"):
+             delattr(get_vector_db, "db_instance")
         return None
 
+# --- Background Processing ---
 def load_feeds_in_background():
+    """Fetches RSS feeds, processes articles, and uploads to Hub in a background thread."""
     global loading_complete, last_update_time
+    # Ensure only one background process runs at a time
+    if not loading_complete:
+        logger.info("An update is already in progress. Skipping.")
+        return
+        
+    loading_complete = False
     try:
-        logger.info("Starting background RSS feed fetch")
+        logger.info("Starting background RSS feed fetch and processing...")
         articles = fetch_rss_feeds()
-        logger.info(f"Fetched {len(articles)} articles")
-        process_and_store_articles(articles)
-        last_update_time = time.time()
-        logger.info("Background feed processing complete")
-        upload_to_hf_hub()
+        logger.info(f"Fetched {len(articles)} articles from RSS feeds.")
+        if articles:
+            process_and_store_articles(articles)
+            upload_to_hf_hub()
+        last_update_time = datetime.now().isoformat()
+        logger.info("Background feed processing complete.")
     except Exception as e:
         logger.error(f"Error in background feed loading: {e}")
     finally:
         loading_complete = True
 
-def get_all_docs_from_db():
-    vector_db = get_vector_db()
-    if not vector_db or vector_db._collection.count() == 0:
-        return {'documents': [], 'metadatas': []}
-    return vector_db.get(include=['documents', 'metadatas'])
-
-def compute_data_hash(categorized_articles):
-    if not categorized_articles: return ""
-    data_str = ""
-    for cat, articles in sorted(categorized_articles.items()):
-        for article in sorted(articles, key=lambda x: x["published"]):
-            data_str += f"{cat}|{article['title']}|{article['link']}|{article['published']}|"
-    return hashlib.sha256(data_str.encode('utf-8')).hexdigest()
-
-def process_docs_into_articles(docs_data):
+# --- Data Transformation Helper ---
+def format_articles_from_db(docs):
+    """
+    Takes ChromaDB documents (with metadata) and formats them into a standardized list of article dictionaries.
+    Handles deduplication based on title and link.
+    """
     enriched_articles = []
     seen_keys = set()
-    for doc, meta in zip(docs_data['documents'], docs_data['metadatas']):
+    
+    # The 'docs' can be a list of (Document, score) tuples or a dict from .get()
+    items = []
+    if isinstance(docs, dict) and 'metadatas' in docs:
+        items = zip(docs['documents'], docs['metadatas'])
+    elif isinstance(docs, list):
+        items = [(doc.page_content, doc.metadata) for doc, score in docs]
+
+    for doc_content, meta in items:
         if not meta: continue
+        
         title = meta.get("title", "No Title")
         link = meta.get("link", "")
-        description = meta.get("original_description", "No Description")
-        published = meta.get("published", "Unknown Date").strip()
+        # Use a composite key to identify unique articles
+        key = f"{title}|{link}"
 
-        key = f"{title}|{link}|{published}"
         if key not in seen_keys:
             seen_keys.add(key)
+            
+            # Safely parse the published date
+            published_str = meta.get("published", "").strip()
             try:
-                published_iso = datetime.strptime(published, "%Y-%m-%d %H:%M:%S").isoformat()
+                published_iso = datetime.strptime(published_str, "%Y-%m-%d %H:%M:%S").isoformat()
             except (ValueError, TypeError):
-                published_iso = "1970-01-01T00:00:00"
+                published_iso = datetime.utcnow().isoformat() # Default to now if format is wrong
             
             enriched_articles.append({
+                "id": meta.get("id", link), # Provide a unique ID
                 "title": title,
                 "link": link,
-                "description": description,
+                "description": meta.get("original_description", "No Description"),
                 "category": meta.get("category", "Uncategorized"),
                 "published": published_iso,
                 "image": meta.get("image", "svg"),
             })
+            
+    # Sort by date descending by default
+    enriched_articles.sort(key=lambda x: x["published"], reverse=True)
     return enriched_articles
 
-@app.route('/')
-def index():
-    global loading_complete, last_update_time, last_data_hash
+# --------------------------------------------------------------------------------
+# --- API v1 Endpoints ---
+# --------------------------------------------------------------------------------
+#
+# API Usage Guide:
+#
+# GET /api/v1/search?q=<query>&limit=<n>
+#   - Performs semantic search.
+#   - `q`: The search term (required).
+#   - `limit`: Max number of results to return (optional, default=20).
+#
+# GET /api/v1/articles/category/<name>?limit=<n>&offset=<o>
+#   - Retrieves all articles for a given category.
+#   - `name`: The category name (e.g., "Technology").
+#   - `limit`: For pagination (optional, default=20).
+#   - `offset`: For pagination (optional, default=0).
+#
+# GET /api/v1/categories
+#   - Returns a list of all unique article categories.
+#
+# GET /api/v1/status
+#   - Checks the status of the background data processing task.
+#
+# --------------------------------------------------------------------------------
+
+@app.route('/api/v1/search', methods=['GET'])
+def api_search():
+    """API endpoint for semantic search."""
+    query = request.args.get('q')
+    limit = request.args.get('limit', default=20, type=int)
 
-    if not os.path.exists(LOCAL_DB_DIR):
-        logger.info(f"No Chroma DB found at '{LOCAL_DB_DIR}', downloading from Hugging Face Hub...")
-        download_from_hf_hub()
+    if not query:
+        return jsonify({"error": "Query parameter 'q' is required."}), 400
 
-    loading_complete = False
-    threading.Thread(target=load_feeds_in_background, daemon=True).start()
+    vector_db = get_vector_db()
+    if not vector_db:
+        return jsonify({"error": "Database not available."}), 503
 
     try:
-        all_docs = get_all_docs_from_db()
-        if not all_docs['metadatas']:
-            logger.info("No articles in the DB yet")
-            return render_template("index.html", categorized_articles={}, has_articles=False, loading=True)
-
-        enriched_articles = process_docs_into_articles(all_docs)
-        enriched_articles.sort(key=lambda x: x["published"], reverse=True)
-        
-        categorized_articles = {}
-        for article in enriched_articles:
-            cat = article["category"]
-            categorized_articles.setdefault(cat, []).append(article)
-        
-        categorized_articles = dict(sorted(categorized_articles.items()))
-
-        for cat in categorized_articles:
-            categorized_articles[cat] = categorized_articles[cat][:10]
-
-        last_data_hash = compute_data_hash(categorized_articles)
-        
-        return render_template("index.html", categorized_articles=categorized_articles, has_articles=True, loading=True)
+        logger.info(f"API: Performing semantic search for: '{query}'")
+        results = vector_db.similarity_search_with_relevance_scores(query, k=limit)
+        formatted_articles = format_articles_from_db(results)
+        return jsonify(formatted_articles)
     except Exception as e:
-        logger.error(f"Error retrieving articles at startup: {e}", exc_info=True)
-        return render_template("index.html", categorized_articles={}, has_articles=False, loading=True)
+        logger.error(f"API Search error: {e}", exc_info=True)
+        return jsonify({"error": "An internal error occurred during search."}), 500
 
-@app.route('/search', methods=['POST'])
-def search():
-    query = request.form.get('search')
-    if not query:
-        return jsonify({"categorized_articles": {}, "has_articles": False, "loading": False})
+@app.route('/api/v1/articles/category/<string:category_name>', methods=['GET'])
+def api_get_articles_by_category(category_name):
+    """API endpoint to get articles filtered by category with pagination."""
+    limit = request.args.get('limit', default=20, type=int)
+    offset = request.args.get('offset', default=0, type=int)
+
+    vector_db = get_vector_db()
+    if not vector_db:
+        return jsonify({"error": "Database not available."}), 503
 
     try:
-        logger.info(f"Performing semantic search for: '{query}'")
-        vector_db = get_vector_db()
-        if not vector_db:
-            return jsonify({"categorized_articles": {}, "has_articles": False, "loading": False})
+        logger.info(f"API: Fetching articles for category '{category_name}'")
+        # Use Chroma's metadata filtering for efficiency
+        results = vector_db.get(
+            where={"category": category_name},
+            include=['documents', 'metadatas']
+        )
         
-        results = vector_db.similarity_search_with_relevance_scores(query, k=50)
+        formatted_articles = format_articles_from_db(results)
+        paginated_results = formatted_articles[offset : offset + limit]
         
-        enriched_articles = []
-        seen_keys = set()
-        for doc, score in results:
-            meta = doc.metadata
-            title = meta.get("title", "No Title")
-            link = meta.get("link", "")
-            key = f"{title}|{link}|{meta.get('published', '')}"
-            if key not in seen_keys:
-                seen_keys.add(key)
-                enriched_articles.append({
-                    "title": title,
-                    "link": link,
-                    "description": meta.get("original_description", "No Description"),
-                    "category": meta.get("category", "Uncategorized"),
-                    "published": meta.get("published", "Unknown Date"),
-                    "image": meta.get("image", "svg"),
-                })
-
-        categorized_articles = {}
-        for article in enriched_articles:
-            cat = article["category"]
-            categorized_articles.setdefault(cat, []).append(article)
-
         return jsonify({
-            "categorized_articles": categorized_articles,
-            "has_articles": bool(enriched_articles),
-            "loading": False
+            "category": category_name,
+            "total_articles": len(formatted_articles),
+            "articles": paginated_results
         })
     except Exception as e:
-        logger.error(f"Semantic search error: {e}", exc_info=True)
-        return jsonify({"categorized_articles": {}, "has_articles": False, "loading": False}), 500
+        logger.error(f"API Category fetch error: {e}", exc_info=True)
+        return jsonify({"error": "An internal error occurred."}), 500
 
-@app.route('/get_all_articles/<category>')
-def get_all_articles(category):
-    try:
-        all_docs = get_all_docs_from_db()
-        enriched_articles = process_docs_into_articles(all_docs)
-        
-        category_articles = [
-            article for article in enriched_articles if article["category"] == category
-        ]
+@app.route('/api/v1/categories', methods=['GET'])
+def api_get_categories():
+    """API endpoint to get a list of all unique categories."""
+    vector_db = get_vector_db()
+    if not vector_db:
+        return jsonify({"error": "Database not available."}), 503
         
-        category_articles.sort(key=lambda x: x["published"], reverse=True)
-        return jsonify({"articles": category_articles, "category": category})
+    try:
+        # Fetch only metadata to be efficient
+        all_metadata = vector_db.get(include=['metadatas'])['metadatas']
+        if not all_metadata:
+            return jsonify([])
+            
+        unique_categories = sorted(list({meta['category'] for meta in all_metadata if 'category' in meta}))
+        return jsonify(unique_categories)
     except Exception as e:
-        logger.error(f"Error fetching all articles for category {category}: {e}")
-        return jsonify({"articles": [], "category": category}), 500
+        logger.error(f"API Categories fetch error: {e}", exc_info=True)
+        return jsonify({"error": "An internal error occurred."}), 500
 
-@app.route('/check_loading')
-def check_loading():
-    global loading_complete, last_update_time
-    return jsonify({"status": "complete" if loading_complete else "loading", "last_update": last_update_time})
+@app.route('/api/v1/status', methods=['GET'])
+def api_get_status():
+    """API endpoint to check the data processing status."""
+    return jsonify({
+        "status": "complete" if loading_complete else "loading",
+        "last_update_time": last_update_time
+    })
 
-@app.route('/get_updates')
-def get_updates():
-    global last_update_time, last_data_hash
-    try:
-        all_docs = get_all_docs_from_db()
-        if not all_docs['metadatas']:
-            return jsonify({"articles": {}, "last_update": last_update_time, "has_updates": False})
-        
-        enriched_articles = process_docs_into_articles(all_docs)
-        categorized_articles = {}
-        for article in enriched_articles:
-            cat = article["category"]
-            categorized_articles.setdefault(cat, []).append(article)
-        
-        for cat in categorized_articles:
-            categorized_articles[cat].sort(key=lambda x: x["published"], reverse=True)
-            categorized_articles[cat] = categorized_articles[cat][:10]
+# --------------------------------------------------------------------------------
+# --- Web Application Routes ---
+# --------------------------------------------------------------------------------
 
-        current_data_hash = compute_data_hash(categorized_articles)
-        has_updates = last_data_hash != current_data_hash
-        
-        if has_updates:
-            logger.info("New RSS data detected, sending updates to frontend")
-            last_data_hash = current_data_hash
-            return jsonify({"articles": categorized_articles, "last_update": last_update_time, "has_updates": True})
-        else:
-            return jsonify({"articles": {}, "last_update": last_update_time, "has_updates": False})
-    except Exception as e:
-        logger.error(f"Error fetching updates: {e}")
-        return jsonify({"articles": {}, "last_update": last_update_time, "has_updates": False}), 500
+@app.route('/')
+def index():
+    """Renders the main web page. Data is fetched by frontend JavaScript."""
+    return render_template("index.html")
 
 @app.route('/card')
 def card_load():
+    """Renders a sample card component."""
     return render_template("card.html")
 
+# --- Main Application Runner ---
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=7860)
+    # On startup, ensure the database exists or download it.
+    if not os.path.exists(LOCAL_DB_DIR):
+        logger.info(f"No local DB found at '{LOCAL_DB_DIR}'. Downloading from Hugging Face Hub...")
+        download_from_hf_hub()
+
+    # Initialize the vector DB instance
+    get_vector_db()
+
+    # Start the first background update immediately.
+    threading.Thread(target=load_feeds_in_background, daemon=True).start()
+
+    # Note: For a production environment, use a proper WSGI server like Gunicorn or uWSGI
+    # instead of Flask's built-in development server.
+    app.run(host="0.0.0.0", port=7860, debug=False)