Upload 31 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,10 @@
+.env*
+!.env.example
+
+__pycache__/
+*.py[cod]
+
+logs/
+
+**__pycache__/
+

API_KEYS_SETUP.md

@@ -0,0 +1,146 @@
+# 🔑 API Keys Setup Guide
+
+## How to Get Pinecone API Key
+
+### Step 1: Create Pinecone Account
+
+1. Go to [https://www.pinecone.io/](https://www.pinecone.io/)
+2. Click **"Sign Up"** or **"Get Started Free"**
+3. Create account with your email or sign up with Google/GitHub
+4. Verify your email address if required
+
+### Step 2: Access Dashboard
+
+1. Log into your Pinecone account
+2. You'll be taken to the Pinecone Console/Dashboard
+3. Look for **"API Keys"** in the left sidebar or navigation menu
+
+### Step 3: Get Your API Key
+
+1. Click on **"API Keys"** in the dashboard
+2. You'll see your default API key listed
+3. Click **"Copy"** or the copy icon next to the API key
+4. Save this key securely - you'll need it for the application
+
+## How to Get Gemini API Key
+
+### Step 1: Go to Google AI Studio
+
+1. Visit [https://aistudio.google.com/](https://aistudio.google.com/)
+2. Sign in with your Google account
+
+### Step 2: Get API Key
+
+1. Click **"Get API Key"** in the top navigation
+2. Click **"Create API Key"**
+3. Select your Google Cloud project (or create a new one)
+4. Copy the generated API key
+
+## How to Get Tavily API Key
+
+### Step 1: Create Tavily Account
+
+1. Go to [https://app.tavily.com/](https://app.tavily.com/)
+2. Click **"Sign Up"** and register with your email or use a social login
+3. Verify your email address if prompted
+
+### Step 2: Access API Keys
+
+1. Log into your Tavily account
+2. Navigate to the **"API Keys"** section in your dashboard
+3. Click **"Create API Key"** if you don't have one yet
+4. Copy the generated API key and store it securely
+
+## 🚀 Quick Start Guide
+
+### Option 1: Set Environment Variables Temporarily
+
+**Windows Command Prompt:**
+
+```cmd
+set PINECONE_API_KEY=pc-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+set GEMINI_API_KEY=AIzaSyxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+set TAVILY_API_KEY=tvly_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+python app.py
+```
+
+**Windows PowerShell:**
+
+```powershell
+$env:PINECONE_API_KEY="pc-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+$env:GEMINI_API_KEY="AIzaSyxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+$env:TAVILY_API_KEY="tvly_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+python app.py
+```
+
+### Option 2: Create .env File (Recommended)
+
+1. Create a file named `.env` in your project root:
+
+```
+PINECONE_API_KEY=pc-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+GEMINI_API_KEY=AIzaSyxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+TAVILY_API_KEY=tvly_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx  # Optional
+```
+
+2. Run the application:
+
+```cmd
+python app.py
+```
+
+## 📋 Free Tier Information
+
+### Pinecone Free Tier:
+
+- ✅ 1 project
+- ✅ 1 index
+- ✅ 100K vectors
+- ✅ Perfect for hackathons and testing
+
+### Gemini Free Tier:
+
+- ✅ 15 requests per minute
+- ✅ 1 million tokens per day
+- ✅ Sufficient for development and demos
+
+### Tavily Free Tier:
+
+- ✅ Generous free tier for testing and development
+- ✅ Check [Tavily Pricing](https://www.tavily.com/pricing) for current limits
+
+## 🔧 Troubleshooting
+
+### If you get "Invalid API Key" errors:
+
+1. Double-check the API key is copied correctly
+2. Make sure there are no extra spaces
+3. Verify the environment variable is set: `echo %PINECONE_API_KEY%`
+
+### If Pinecone connection fails:
+
+1. Check your internet connection
+2. Verify your Pinecone account is active
+3. Make sure you're using the correct region (default is usually fine)
+
+## 🎯 Ready to Launch
+
+Once you have all API keys:
+
+1. **Set the environment variables**
+2. **Run the application:**
+   ```cmd
+   python app.py
+   ```
+3. **Open your browser to:** `http://localhost:7860`
+4. **Start uploading documents and asking questions!**
+
+The application will now have full functionality with:
+
+- ✅ Document processing and embedding
+- ✅ Vector storage in Pinecone
+- ✅ AI-powered question answering
+- ✅ Beautiful Gradio interface
+
+**🎉 Your AI Embedded Knowledge Agent will be fully operational!**

README.md

@@ -1,12 +1,49 @@
----
-title: Payman
-emoji: 📚
-colorFrom: pink
-colorTo: yellow
-sdk: gradio
-sdk_version: 5.34.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🧠 AI Embedded Knowledge Agent
+
+A comprehensive Retrieval-Augmented Generation (RAG) system that allows you to upload documents, process URLs, and ask intelligent questions about your knowledge base. Built with modern AI technologies and optimized for deployment on Hugging Face Spaces.
+
+## 🎥 Demo Video
+
+Watch our comprehensive demo showcasing the live search capabilities and RAG system in action:
+
+[![RAG AI Demo](https://img.shields.io/badge/🎥_Watch_Demo-Loom-00D4AA?style=for-the-badge)](https://www.loom.com/share/f1a3c79b75ad4b65b528b2973612cdd9?sid=a0932972-2926-42ab-a031-3149e40b1b97)
+
+_See how the system processes documents, performs live web searches, and generates intelligent responses with real-time data integration._
+
+## ✨ Features
+
+### 🔥 Core Capabilities
+
+- **📄 Document Processing**: Support for PDF, DOCX, CSV, XLSX, PPTX, TXT, MD, and more
+- **🌐 URL Processing**: Extract content from web pages with intelligent crawling
+- **🔍 Live Web Search**: Real-time web search using Tavily API for up-to-date information
+- **🧠 Smart Q&A**: Ask questions and get contextual answers with source attribution
+- **🎯 High Accuracy**: Advanced embedding and similarity search for precise results
+- **⚡ Real-time Processing**: Fast document ingestion and query processing
+
+### 🚀 Advanced Features
+
+- **🤖 Multiple LLM Support**: Gemini 2.5 Flash, OpenAI GPT models with automatic fallback
+- **📊 Analytics Dashboard**: Query analytics, system metrics, and performance monitoring
+- **🔍 Smart Query Processing**: Query expansion, caching, and suggestion system
+- **📚 Knowledge Base Management**: View, manage, and export your knowledge base
+- **🛡️ Robust Error Handling**: Graceful degradation and comprehensive error recovery
+- **🎨 Beautiful UI**: Modern Gradio interface optimized for user experience
+
+### 🏗️ Technical Excellence
+
+- **🔧 Modular Architecture**: Clean, maintainable, and extensible codebase
+- **⚙️ Configurable**: Comprehensive YAML configuration for all components
+- **🔒 Secure**: Input sanitization, rate limiting, and security best practices
+- **📈 Scalable**: Designed for production deployment with monitoring and health checks
+- **🧪 Well-tested**: Comprehensive test suite and example usage
+
+## 🎯 Project Origin & Agentic Vision
+
+This project was initially conceived and developed as part of the **`agents-mcp-hackathon`**, showcasing a robust **`agent-demo-track`** for intelligent knowledge retrieval and generation. Our vision was to build an autonomous AI agent capable of:
+
+- **🧠 Intelligent Information Retrieval**: Acting as a smart agent to fetch, process, and synthesize information from diverse sources (documents, URLs, live web).
+- **🚀 Dynamic Query Routing**: Intelligently deciding between local knowledge base retrieval and real-time web search based on query intent.
+- **💡 Autonomous Knowledge Management**: Providing a self-contained system for building and querying a dynamic knowledge base.
+
+This system embodies the principles of agentic AI, offering a powerful, self-sufficient solution for complex information needs.

app.py

@@ -0,0 +1,884 @@
+"""
+AI Embedded Knowledge Agent - Main Application Entry Point
+
+This is the main entry point for the RAG AI system that integrates all components
+and launches the Gradio interface for deployment on Hugging Face.
+"""
+
+import nltk
+
+nltk.download("punkt_tab")
+
+
+import spacy.cli
+
+spacy.cli.download("en_core_web_sm")
+nlp = spacy.load("en_core_web_sm")
+
+
+import os
+import sys
+import logging
+from pathlib import Path
+from typing import Optional
+
+# Load environment variables from .env file
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv()
+except ImportError:
+    print(
+        "python-dotenv not installed. Please install it with: pip install python-dotenv"
+    )
+
+# Add src directory to Python path
+src_path = Path(__file__).parent / "src"
+sys.path.insert(0, str(src_path))
+
+# Import all components
+from utils.config_manager import ConfigManager
+from utils.error_handler import ErrorHandler, ErrorType
+from ingestion.document_processor import DocumentProcessor
+from ingestion.url_processor import URLProcessor
+from ingestion.text_extractor import TextExtractor
+from embedding.embedding_generator import EmbeddingGenerator
+from storage.vector_db import VectorDB
+from rag.optimized_query_processor import OptimizedQueryProcessor
+from rag.response_generator import ResponseGenerator
+from rag.live_search import LiveSearchProcessor
+from rag.query_router import QueryRouter
+from ui.gradio_app import GradioApp
+
+
+class RAGSystem:
+    """
+    Main RAG AI system that orchestrates all components.
+
+    This class integrates document processing, embedding generation,
+    vector storage, and query processing into a unified system.
+    """
+
+    def __init__(self, config_path: Optional[str] = None):
+        """
+        Initialize the RAG system with all components.
+
+        Args:
+            config_path: Path to configuration file
+        """
+        # Initialize configuration
+        self.config_manager = ConfigManager(config_path)
+        self.config = self.config_manager.config
+
+        # Setup logging
+        self._setup_logging()
+        self.logger = logging.getLogger(__name__)
+        self.logger.info("Initializing RAG AI System...")
+
+        # Initialize error handler
+        self.error_handler = ErrorHandler()
+
+        # Validate environment and configuration
+        self._validate_environment()
+
+        # Initialize components
+        self._initialize_components()
+
+        # Run health checks
+        self._run_startup_health_checks()
+
+        self.logger.info("RAG AI System initialized successfully! ")
+
+    def _setup_logging(self):
+        """Setup comprehensive logging configuration."""
+        log_config = self.config.get("logging", {})
+        log_level = getattr(logging, log_config.get("level", "INFO").upper())
+        log_format = log_config.get(
+            "format", "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        )
+
+        # Configure root logger with UTF-8 encoding
+        import io
+
+        utf8_stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
+        logging.basicConfig(
+            level=log_level,
+            format=log_format,
+            handlers=[logging.StreamHandler(utf8_stdout)],
+        )
+
+        # Create logs directory if specified
+        log_file = log_config.get("file")
+        if log_file:
+            log_dir = Path(log_file).parent
+            log_dir.mkdir(parents=True, exist_ok=True)
+
+            # Add file handler with rotation
+            try:
+                from logging.handlers import RotatingFileHandler
+
+                file_handler = RotatingFileHandler(
+                    log_file,
+                    maxBytes=log_config.get("max_file_size_mb", 10) * 1024 * 1024,
+                    backupCount=log_config.get("backup_count", 5),
+                )
+                file_handler.setFormatter(logging.Formatter(log_format))
+                logging.getLogger().addHandler(file_handler)
+            except Exception as e:
+                self.logger.warning(f"Could not setup file logging: {e}")
+
+    def _validate_environment(self):
+        """Validate environment variables and configuration."""
+        self.logger.info("Validating environment...")
+
+        # Check required API keys
+        required_keys = ["GEMINI_API_KEY"]
+        optional_keys = ["PINECONE_API_KEY", "OPENAI_API_KEY"]
+
+        missing_required = []
+        for key in required_keys:
+            if not os.getenv(key):
+                missing_required.append(key)
+
+        if missing_required:
+            self.logger.error(
+                f" Missing required environment variables: {missing_required}"
+            )
+            self.logger.error(
+                "Please set the required API keys as environment variables"
+            )
+            # Don't raise error in demo mode, just warn
+            self.logger.warning("Running in demo mode with limited functionality")
+
+        # Check optional keys
+        missing_optional = []
+        for key in optional_keys:
+            if not os.getenv(key):
+                missing_optional.append(key)
+
+        if missing_optional:
+            self.logger.warning(
+                f"Missing optional environment variables: {missing_optional}"
+            )
+            self.logger.warning("Some features may be limited without these keys")
+
+        # Validate configuration
+        self._validate_configuration()
+
+        self.logger.info("Environment validation completed")
+
+    def _validate_configuration(self):
+        """Validate configuration settings."""
+        try:
+            # Check embedding configuration
+            embedding_config = self.config.get("embedding", {})
+            if not embedding_config.get("model"):
+                self.logger.warning("Embedding model not specified, using default")
+
+            # Check vector database configuration
+            vector_db_config = self.config.get("vector_db", {})
+            if not vector_db_config.get("provider"):
+                self.logger.warning(
+                    "Vector database provider not specified, using default"
+                )
+
+            # Check RAG configuration
+            rag_config = self.config.get("rag", {})
+            if rag_config.get("top_k", 5) <= 0:
+                self.logger.warning("Invalid top_k value, using default")
+
+            self.logger.info("Configuration validation completed")
+
+        except Exception as e:
+            self.logger.warning(f"Configuration validation warning: {e}")
+
+    def _initialize_components(self):
+        """Initialize all system components with error handling."""
+        try:
+            self.logger.info("Initializing system components...")
+
+            # Document processing components
+            self.logger.info(" Initializing document processing components...")
+            self.document_processor = DocumentProcessor(
+                self.config_manager.get_section("document_processing")
+            )
+
+            self.url_processor = URLProcessor(
+                self.config_manager.get_section("url_processing")
+            )
+
+            self.text_extractor = TextExtractor(
+                self.config_manager.get_section("document_processing")
+            )
+
+            # Embedding and storage components
+            self.logger.info("Initializing embedding and storage components...")
+            embedding_config = self.config_manager.get_section("embedding")
+            embedding_config["api_key"] = os.getenv("GEMINI_API_KEY")
+
+            self.embedding_generator = EmbeddingGenerator(embedding_config)
+
+            vector_db_config = self.config_manager.get_section("vector_db")
+            vector_db_config["api_key"] = os.getenv("PINECONE_API_KEY")
+
+            self.vector_db = VectorDB(vector_db_config)
+
+            # RAG components
+            self.logger.info("Initializing RAG components...")
+            self.query_processor = OptimizedQueryProcessor(
+                self.embedding_generator,
+                self.vector_db,
+                self.config_manager.get_section("rag"),
+            )
+
+            rag_config = self.config_manager.get_section("rag")
+            # Add API keys to RAG config for LLM initialization
+            rag_config["gemini_api_key"] = os.getenv("GEMINI_API_KEY")
+            rag_config["openai_api_key"] = os.getenv("OPENAI_API_KEY")
+
+            self.response_generator = ResponseGenerator(rag_config)
+
+            # Live Search components
+            self.logger.info("Initializing Live Search components...")
+            live_search_config = self.config_manager.get_section("live_search") or {}
+            self.live_search_processor = LiveSearchProcessor(live_search_config)
+
+            # Query Router for intelligent routing
+            router_config = self.config_manager.get_section("query_router") or {}
+            self.query_router = QueryRouter(
+                self.query_processor, self.live_search_processor, router_config
+            )
+
+            self.logger.info("All components initialized successfully")
+
+        except Exception as e:
+            self.logger.error(f" Failed to initialize components: {str(e)}")
+            # Don't raise in demo mode, continue with limited functionality
+            self.logger.warning("Some components may not be fully functional")
+
+    def _run_startup_health_checks(self):
+        """Run health checks on all components."""
+        self.logger.info("Running startup health checks...")
+
+        health_status = {
+            "document_processor": False,
+            "url_processor": False,
+            "text_extractor": False,
+            "embedding_generator": False,
+            "vector_db": False,
+            "query_processor": False,
+            "response_generator": False,
+        }
+
+        # Check each component
+        try:
+            if hasattr(self, "document_processor"):
+                health_status["document_processor"] = True
+                self.logger.info("Document processor: Healthy")
+        except:
+            self.logger.warning("Document processor: Not available")
+
+        try:
+            if hasattr(self, "url_processor"):
+                health_status["url_processor"] = True
+                self.logger.info("URL processor: Healthy")
+        except:
+            self.logger.warning("URL processor: Not available")
+
+        try:
+            if hasattr(self, "text_extractor"):
+                health_status["text_extractor"] = True
+                self.logger.info("Text extractor: Healthy")
+        except:
+            self.logger.warning("Text extractor: Not available")
+
+        try:
+            if hasattr(self, "embedding_generator"):
+                health_status["embedding_generator"] = True
+                self.logger.info("Embedding generator: Healthy")
+        except:
+            self.logger.warning("Embedding generator: Not available")
+
+        try:
+            if hasattr(self, "vector_db"):
+                health_status["vector_db"] = True
+                self.logger.info("Vector database: Healthy")
+        except:
+            self.logger.warning("Vector database: Not available")
+
+        try:
+            if hasattr(self, "query_processor"):
+                health_status["query_processor"] = True
+                self.logger.info("Query processor: Healthy")
+        except:
+            self.logger.warning("Query processor: Not available")
+
+        try:
+            if hasattr(self, "response_generator"):
+                health_status["response_generator"] = True
+                self.logger.info("Response generator: Healthy")
+        except:
+            self.logger.warning("Response generator: Not available")
+
+        # Overall health
+        healthy_components = sum(health_status.values())
+        total_components = len(health_status)
+
+        self.logger.info(
+            f"Health check complete: {healthy_components}/{total_components} components healthy"
+        )
+
+        if healthy_components < total_components:
+            self.logger.warning("Some components are not fully functional")
+            self.logger.warning("The system will run with limited capabilities")
+
+    def process_document(self, file_path: str) -> dict:
+        """
+        Process a document through the complete pipeline.
+
+        Args:
+            file_path: Path to the document file
+
+        Returns:
+            Dictionary with processing results
+        """
+        try:
+            self.logger.info(f" Processing document: {file_path}")
+
+            # Check if components are available
+            if not all(
+                hasattr(self, attr)
+                for attr in [
+                    "document_processor",
+                    "text_extractor",
+                    "embedding_generator",
+                    "vector_db",
+                ]
+            ):
+                return {
+                    "status": "error",
+                    "error": "Required components not available",
+                    "chunks_processed": 0,
+                }
+
+            # Step 1: Extract content from document
+            doc_result = self.document_processor.process_document(file_path)
+
+            if not doc_result or "content" not in doc_result:
+                return {
+                    "status": "error",
+                    "error": "Failed to extract content from document",
+                    "chunks_processed": 0,
+                }
+
+            # Step 2: Extract and chunk text
+            text_chunks = self.text_extractor.process_text(
+                doc_result["content"], doc_result.get("metadata", {})
+            )
+
+            if not text_chunks:
+                return {
+                    "status": "error",
+                    "error": "No text chunks generated",
+                    "chunks_processed": 0,
+                }
+
+            # Step 3: Generate embeddings
+            embedded_chunks = self.embedding_generator.generate_embeddings(text_chunks)
+
+            if not embedded_chunks:
+                return {
+                    "status": "error",
+                    "error": "Failed to generate embeddings",
+                    "chunks_processed": len(text_chunks),
+                }
+
+            # Step 4: Store in vector database
+            storage_success = self.vector_db.store_embeddings(embedded_chunks)
+
+            return {
+                "status": "success" if storage_success else "partial_success",
+                "chunks_processed": len(text_chunks),
+                "chunks_stored": len(embedded_chunks) if storage_success else 0,
+                "source": file_path,
+            }
+
+        except Exception as e:
+            self.logger.error(f" Error processing document: {str(e)}")
+            error_info = self.error_handler.handle_error(e, {"file_path": file_path})
+            return {
+                "status": "error",
+                "error": str(e),
+                "error_info": error_info,
+                "chunks_processed": 0,
+            }
+
+    def process_url(
+        self, url: str, max_depth: int = 1, follow_links: bool = True
+    ) -> dict:
+        """
+        Process a URL through the complete pipeline with advanced options.
+
+        Args:
+            url: URL to process
+            max_depth: Maximum crawling depth
+            follow_links: Whether to follow links
+
+        Returns:
+            Dictionary with processing results
+        """
+        try:
+            self.logger.info(f"Processing URL: {url}")
+
+            # Check if components are available
+            if not all(
+                hasattr(self, attr)
+                for attr in [
+                    "url_processor",
+                    "text_extractor",
+                    "embedding_generator",
+                    "vector_db",
+                ]
+            ):
+                return {
+                    "status": "error",
+                    "error": "Required components not available",
+                    "chunks_processed": 0,
+                }
+
+            # Step 1: Configure URL processor with advanced options
+            # Update URL processor configuration dynamically
+            self.url_processor.max_depth = max_depth
+            self.url_processor.follow_links = follow_links
+
+            # Reset processor state for fresh crawl
+            self.url_processor.reset()
+
+            # Extract content from URL
+            url_result = self.url_processor.process_url(url)
+
+            if not url_result or "content" not in url_result:
+                return {
+                    "status": "error",
+                    "error": "Failed to extract content from URL",
+                    "chunks_processed": 0,
+                }
+
+            # Step 2: Extract and chunk text
+            text_chunks = self.text_extractor.process_text(
+                url_result["content"], url_result.get("metadata", {})
+            )
+
+            if not text_chunks:
+                return {
+                    "status": "error",
+                    "error": "No text chunks generated",
+                    "chunks_processed": 0,
+                }
+
+            # Step 3: Generate embeddings
+            embedded_chunks = self.embedding_generator.generate_embeddings(text_chunks)
+
+            if not embedded_chunks:
+                return {
+                    "status": "error",
+                    "error": "Failed to generate embeddings",
+                    "chunks_processed": len(text_chunks),
+                }
+
+            # Step 4: Store in vector database
+            storage_success = self.vector_db.store_embeddings(embedded_chunks)
+
+            # Process linked documents if any
+            linked_processed = 0
+            for linked_doc in url_result.get("linked_documents", []):
+                if linked_doc and "content" in linked_doc:
+                    try:
+                        linked_chunks = self.text_extractor.process_text(
+                            linked_doc["content"], linked_doc.get("metadata", {})
+                        )
+                        if linked_chunks:
+                            linked_embedded = (
+                                self.embedding_generator.generate_embeddings(
+                                    linked_chunks
+                                )
+                            )
+                            if linked_embedded and self.vector_db.store_embeddings(
+                                linked_embedded
+                            ):
+                                linked_processed += 1
+                    except Exception as e:
+                        self.logger.warning(f"Failed to process linked document: {e}")
+
+            return {
+                "status": "success" if storage_success else "partial_success",
+                "chunks_processed": len(text_chunks),
+                "chunks_stored": len(embedded_chunks) if storage_success else 0,
+                "linked_documents_processed": linked_processed,
+                "source": url,
+            }
+
+        except Exception as e:
+            self.logger.error(f" Error processing URL: {str(e)}")
+            error_info = self.error_handler.handle_error(e, {"url": url})
+            return {
+                "status": "error",
+                "error": str(e),
+                "error_info": error_info,
+                "chunks_processed": 0,
+            }
+
+    def query(
+        self,
+        question: str,
+        max_results: int = 5,
+        use_live_search: bool = False,
+        search_mode: str = "auto",
+    ) -> dict:
+        """
+        Process a query and generate a response with enhanced search control.
+
+        Args:
+            question: User question
+            max_results: Maximum number of results to retrieve
+            use_live_search: Whether to enable live web search (uses hybrid approach)
+            search_mode: Search mode - "auto", "local_only", "live_only", "hybrid"
+
+        Returns:
+            Dictionary with response and metadata
+        """
+        try:
+            self.logger.info(
+                f"Processing query: {question[:100]}... (live_search: {use_live_search})"
+            )
+
+            # Check if components are available
+            if not all(
+                hasattr(self, attr)
+                for attr in ["query_processor", "response_generator"]
+            ):
+                return {
+                    "query": question,
+                    "response": "Query processing components not available. Please check system configuration.",
+                    "sources": [],
+                    "confidence": 0.0,
+                    "error": "Components not available",
+                }
+
+            # Use Query Router for intelligent routing if available
+            if hasattr(self, "query_router") and (
+                use_live_search or search_mode != "auto"
+            ):
+                self.logger.info(f" Using Query Router with mode: {search_mode}")
+
+                search_options = {"search_depth": "basic", "time_range": "month"}
+
+                router_result = self.query_router.route_query(
+                    question,
+                    use_live_search=use_live_search,
+                    max_results=max_results,
+                    search_options=search_options,
+                    search_mode=search_mode,
+                )
+
+                # Convert router result to standard format
+                if router_result.get("results"):
+                    # Format sources from router results
+                    sources = []
+                    for result in router_result["results"]:
+                        sources.append(
+                            {
+                                "title": result.get("title", ""),
+                                "source": result.get("source", ""),
+                                "content": result.get("content", ""),
+                                "score": result.get("score", 0.0),
+                                "type": result.get("type", "unknown"),
+                            }
+                        )
+
+                    # Generate response using response generator
+                    context_items = []
+                    for result in router_result["results"]:
+                        context_items.append(
+                            {
+                                "text": result.get("content", ""),
+                                "source": result.get("source", ""),
+                                "score": result.get("score", 0.0),
+                                "metadata": result.get("metadata", {}),
+                            }
+                        )
+
+                    response_result = self.response_generator.generate_response(
+                        question, context_items
+                    )
+
+                    return {
+                        "query": question,
+                        "response": response_result.get(
+                            "response", "No response generated"
+                        ),
+                        "sources": sources,
+                        "confidence": response_result.get("confidence", 0.0),
+                        "context_items": len(context_items),
+                        "processing_time": router_result.get("processing_time", 0),
+                        "generation_time": response_result.get("generation_time", 0),
+                        "model_used": response_result.get("model_used", "unknown"),
+                        "routing_decision": router_result.get(
+                            "routing_decision", "unknown"
+                        ),
+                        "search_type": "routed_search",
+                    }
+                else:
+                    # Fallback to local search if router fails
+                    self.logger.warning(
+                        "Router returned no results, falling back to local search"
+                    )
+
+            # Traditional local search path
+            # Step 1: Process query and retrieve context with max_results
+            # Update query processor config temporarily
+            original_top_k = self.query_processor.top_k
+            self.query_processor.top_k = max_results
+
+            query_result = self.query_processor.process_query(question)
+
+            # Restore original top_k
+            self.query_processor.top_k = original_top_k
+
+            if query_result.get("error"):
+                return {
+                    "query": question,
+                    "response": f"Query processing failed: {query_result['error']}",
+                    "sources": [],
+                    "confidence": 0.0,
+                    "error": query_result["error"],
+                }
+
+            # Step 2: Generate response
+            response_result = self.response_generator.generate_response(
+                question, query_result.get("context", [])
+            )
+
+            # Combine results
+            return {
+                "query": question,
+                "response": response_result.get("response", "No response generated"),
+                "sources": response_result.get("sources", []),
+                "confidence": response_result.get("confidence", 0.0),
+                "context_items": query_result.get("total_results", 0),
+                "processing_time": query_result.get("processing_time", 0),
+                "generation_time": response_result.get("generation_time", 0),
+                "model_used": response_result.get("model_used", "unknown"),
+                "search_type": "local_search",
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error processing query: {str(e)}")
+            error_info = self.error_handler.handle_error(e, {"query": question})
+            return {
+                "query": question,
+                "response": "I encountered an error while processing your question. Please try again.",
+                "sources": [],
+                "confidence": 0.0,
+                "error": str(e),
+                "error_info": error_info,
+            }
+
+    def get_system_status(self) -> dict:
+        """
+        Get comprehensive system status.
+
+        Returns:
+            Dictionary with system status information
+        """
+        try:
+            status = {
+                "overall_status": "healthy",
+                "components": {},
+                "configuration": {},
+                "environment": {},
+            }
+
+            # Check component status
+            components = [
+                "document_processor",
+                "url_processor",
+                "text_extractor",
+                "embedding_generator",
+                "vector_db",
+                "query_processor",
+                "response_generator",
+            ]
+
+            for component in components:
+                status["components"][component] = hasattr(self, component)
+
+            # Configuration info
+            status["configuration"] = {
+                "embedding_model": self.config.get("embedding", {}).get(
+                    "model", "unknown"
+                ),
+                "vector_db_provider": self.config.get("vector_db", {}).get(
+                    "provider", "unknown"
+                ),
+                "rag_top_k": self.config.get("rag", {}).get("top_k", 5),
+            }
+
+            # Environment info
+            status["environment"] = {
+                "gemini_api_available": bool(os.getenv("GEMINI_API_KEY")),
+                "pinecone_api_available": bool(os.getenv("PINECONE_API_KEY")),
+                "openai_api_available": bool(os.getenv("OPENAI_API_KEY")),
+            }
+
+            # Overall status
+            healthy_components = sum(status["components"].values())
+            total_components = len(status["components"])
+
+            if healthy_components < total_components * 0.8:
+                status["overall_status"] = "degraded"
+            elif healthy_components < total_components * 0.5:
+                status["overall_status"] = "unhealthy"
+
+            return status
+
+        except Exception as e:
+            self.logger.error(f" Error getting system status: {e}")
+            return {"overall_status": "error", "error": str(e)}
+
+
+def create_app():
+    """
+    Create and configure the RAG application.
+
+    Returns:
+        Tuple of (RAG system instance, Gradio app instance)
+    """
+    try:
+        # Initialize the RAG system
+        rag_system = RAGSystem()
+
+        # Create Gradio interface
+        ui_config = rag_system.config_manager.get_section("ui")
+        gradio_app = GradioApp(rag_system, ui_config)
+
+        return rag_system, gradio_app
+
+    except Exception as e:
+        print(f" Failed to create application: {str(e)}")
+        # Create a minimal system for demo purposes
+        print("Creating minimal demo system...")
+
+        # Create minimal config
+        minimal_config = {
+            "ui": {
+                "title": "AI Embedded Knowledge Agent (Demo Mode)",
+                "description": "Demo mode - some features may be limited. Please configure API keys for full functionality.",
+            }
+        }
+
+        # Create minimal RAG system
+        class MinimalRAGSystem:
+            def __init__(self):
+                self.config_manager = type(
+                    "ConfigManager",
+                    (),
+                    {
+                        "get_section": lambda self, section: minimal_config.get(
+                            section, {}
+                        )
+                    },
+                )()
+
+            def process_document(self, file_path):
+                return {
+                    "status": "error",
+                    "error": "Demo mode - document processing not available",
+                }
+
+            def process_url(self, url):
+                return {
+                    "status": "error",
+                    "error": "Demo mode - URL processing not available",
+                }
+
+            def query(self, question):
+                return {
+                    "query": question,
+                    "response": "Demo mode: Please configure your API keys (GEMINI_API_KEY, PINECONE_API_KEY) to enable full functionality.",
+                    "sources": [],
+                    "confidence": 0.0,
+                }
+
+        rag_system = MinimalRAGSystem()
+        gradio_app = GradioApp(rag_system, minimal_config.get("ui", {}))
+
+        return rag_system, gradio_app
+
+
+def main():
+    """Main function to run the application."""
+    try:
+        print("Starting AI Embedded Knowledge Agent...")
+        print("=" * 50)
+
+        # Create the application
+        rag_system, gradio_app = create_app()
+
+        # Get launch configuration
+        try:
+            ui_config = rag_system.config_manager.get_section("ui")
+        except:
+            ui_config = {}
+
+        # Launch the Gradio interface
+        base_port = ui_config.get("port", 7860)
+        launch_config = {
+            "server_name": ui_config.get("server_name", "0.0.0.0"),
+            "server_port": base_port,
+            "share": ui_config.get("share", False),
+            "show_error": True,
+            "quiet": False,
+        }
+
+        # Try different ports if the default is in use
+        for port_offset in range(10):  # Try ports 7860-7869
+            try:
+                current_port = base_port + port_offset
+                launch_config["server_port"] = current_port
+
+                print(
+                    f"Launching interface on {launch_config['server_name']}:{current_port}"
+                )
+                print("=" * 50)
+
+                gradio_app.launch(**launch_config)
+                break  # If successful, break out of the loop
+
+            except Exception as e:
+                if (
+                    "bind" in str(e).lower()
+                    or "address already in use" in str(e).lower()
+                ):
+                    print(f"Port {current_port} is in use, trying next port...")
+                    continue
+                else:
+                    # If it's a different error, re-raise it
+                    raise e
+        else:
+            # If we've tried all ports without success
+            print(
+                "Could not find an available port. Please close other applications using ports 7860-7869."
+            )
+            raise Exception("No available ports found")
+
+    except KeyboardInterrupt:
+        print("\n👋 Shutting down gracefully...")
+    except Exception as e:
+        print(f" Failed to start application: {str(e)}")
+        print("Please check your configuration and API keys.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

config/config.yaml

@@ -0,0 +1,269 @@
+api_keys:
+  gemini_api_key: ""
+  openai_api_key: ""
+  pinecone_api_key: ""
+backup:
+  enabled: false
+  include_configuration: true
+  include_documents: true
+  include_logs: false
+  include_vector_db: true
+  interval_hours: 24
+  retention_days: 30
+  storage_path: backups/
+customization:
+  custom_css: ""
+  default_query_examples:
+    - What is the main topic of the uploaded documents?
+    - Can you summarize the key points?
+    - What are the important findings mentioned?
+  favicon_url: ""
+  footer_text: ""
+  help_text: ""
+  logo_url: ""
+  welcome_message: ""
+deployment:
+  auto_scale: true
+  development:
+    debug_mode: true
+    enable_profiling: true
+    log_level: DEBUG
+  enable_metrics: true
+  graceful_shutdown_timeout: 30
+  health_check_interval: 60
+  health_endpoint: /health
+  max_cpu_percent: 80
+  max_disk_usage_mb: 5120
+  max_memory_mb: 2048
+  metrics_endpoint: /metrics
+  platform: huggingface
+  production:
+    debug_mode: false
+    enable_profiling: false
+    log_level: WARNING
+  staging:
+    debug_mode: true
+    enable_profiling: true
+    log_level: INFO
+development:
+  debug_mode: false
+  enable_test_endpoints: false
+  mock_apis: false
+  profiling_enabled: false
+  save_intermediate_results: false
+  test_data_path: data/test_data
+  test_mode: false
+document_processing:
+  chunk_overlap: 200
+  chunk_size: 1000
+  detect_language: true
+  extract_images: false
+  extract_metadata: true
+  max_file_size_mb: 50
+  min_chunk_size: 100
+  preserve_formatting: true
+  supported_formats:
+    - .pdf
+    - .docx
+    - .doc
+    - .csv
+    - .xlsx
+    - .xls
+    - .pptx
+    - .txt
+    - .md
+  supported_languages:
+    - en
+    - es
+    - fr
+    - de
+    - it
+    - pt
+    - ru
+    - zh
+    - ja
+    - ko
+embedding:
+  batch_size: 1
+  cache_embeddings: true
+  fallback_model: sentence-transformers
+  max_retries: 3
+  max_tokens: 8192
+  model: gemini-embedding-exp-03-07
+  output_dimensionality: 3072
+  rate_limit_delay: 1.0
+  retry_delay: 2
+  task_type: RETRIEVAL_DOCUMENT
+  title: ""
+features:
+  async_processing: false
+  audio_processing: false
+  auto_summarization: false
+  batch_processing: true
+  content_recommendation: false
+  document_upload: true
+  image_processing: false
+  live_search: true
+  multi_language_support: false
+  query_processing: true
+  question_generation: false
+  real_time_updates: false
+  url_processing: true
+  video_processing: false
+integrations:
+  aws_s3:
+    access_key: ""
+    bucket_name: ""
+    enabled: false
+    secret_key: ""
+  google_analytics:
+    enabled: false
+    tracking_id: ""
+  huggingface:
+    api_key: ""
+    enabled: false
+    models: []
+  postgresql:
+    connection_string: ""
+    enabled: false
+  sentry:
+    dsn: ""
+    enabled: false
+logging:
+  backup_count: 5
+  component_levels:
+    document_processing: INFO
+    embedding: INFO
+    rag: INFO
+    ui: INFO
+    url_processing: INFO
+    vector_db: INFO
+  file: logs/rag_ai.log
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+  level: INFO
+  max_file_size_mb: 10
+notifications:
+  email:
+    enabled: false
+    from_address: ""
+    password: ""
+    smtp_port: 587
+    smtp_server: ""
+    to_addresses: []
+    username: ""
+  enabled: false
+  webhook:
+    enabled: false
+    events:
+      - error
+      - system_health
+      - processing_complete
+    url: ""
+performance:
+  batch_processing_size: 10
+  cache_ttl: 3600
+  enable_caching: true
+  enable_parallel_processing: true
+  garbage_collection_interval: 300
+  max_concurrent_requests: 5
+  max_memory_usage_mb: 1024
+  max_worker_threads: 4
+  request_timeout: 30
+rag:
+  confidence_threshold: 0.3
+  context_window_overlap: 0.1
+  deduplicate_results: true
+  enable_query_caching: true
+  enable_query_expansion: true
+  fallback_model: gpt-3.5-turbo
+  include_sources: true
+  max_context_length: 8000
+  max_response_length: 2000
+  max_tokens: 500
+  model: gemini-2.5-flash-preview-05-20
+  query_cache_ttl: 7200
+  rerank_results: true
+  similarity_threshold: 0.4
+  temperature: 0.7
+  top_k: 10
+  top_p: 0.9
+live_search:
+  enabled: true
+  enable_caching: true
+  include_raw_content: true
+  max_results: 10
+  search_depth: basic
+  time_range: month
+query_router:
+  confidence_threshold: 0.5
+  enable_hybrid_search: true
+  live_weight: 0.4
+  local_weight: 0.6
+  max_hybrid_results: 10
+security:
+  allowed_domains: []
+  blocked_content_types:
+    - executable
+    - script
+  blocked_domains:
+    - localhost
+    - 127.0.0.1
+    - 0.0.0.0
+  enable_content_filtering: true
+  enable_rate_limiting: true
+  max_text_length: 1000000
+  max_upload_size_mb: 100
+  requests_per_hour: 1000
+  requests_per_minute: 60
+  sanitize_input: true
+ui:
+  demo_mode: false
+  description:
+    Upload documents or provide URLs to build your knowledge base, then
+    ask questions!
+  features:
+    analytics_dashboard: true
+    confidence_display: true
+    file_upload: true
+    knowledge_base_management: true
+    query_interface: true
+    source_display: true
+    system_health_monitoring: true
+    url_input: true
+  max_file_uploads: 10
+  max_query_length: 1000
+  port: 7860
+  sample_documents: []
+  server_name: 0.0.0.0
+  share: false
+  show_advanced_options: true
+  theme: default
+  title: "\xF0\u0178\xA7\_ AI Embedded Knowledge Agent"
+url_processing:
+  allowed_domains: []
+  blocked_domains:
+    - localhost
+    - 127.0.0.1
+    - 0.0.0.0
+  delay_between_requests: 0.5
+  extract_main_content: true
+  follow_links: true
+  max_depth: 1
+  max_pages: 10
+  remove_ads: true
+  remove_navigation: true
+  requests_per_second: 2
+  respect_robots_txt: true
+  timeout: 10
+  user_agent: RAG-AI-Bot/1.0
+vector_db:
+  batch_size: 100
+  create_index_if_not_exists: true
+  dimension: 3072
+  environment: us-east-1
+  fallback_provider: memory
+  index_name: rag-ai-index
+  max_retries: 3
+  metric: cosine
+  provider: pinecone
+  retry_delay: 1

docs/architecture.md

@@ -0,0 +1,291 @@
+# AI Embedded Knowledge Agent - Architecture Document
+
+## 1. System Overview
+
+The AI Embedded Knowledge Agent is a versatile knowledge management system designed to ingest, process, and retrieve information from various document types and web sources. Built for a hackathon and deployable on Hugging Face, this system enables users to upload documents or provide URLs, which are then processed, embedded, and stored for intelligent retrieval.
+
+```mermaid
+graph TD
+    A[User Interface - Gradio] --> B[Document Processor]
+    A --> C[URL Processor]
+    B --> D[Text Extractor]
+    C --> D
+    D --> E[Embedding Generator - Gemini]
+    E --> F[Vector Database - Pinecone]
+    A --> G[Query Processor]
+    G --> E
+    G --> F
+    G --> H[Response Generator - LangChain RAG]
+    H --> A
+```
+
+## 2. Core Components
+
+### 2.1 Document Ingestion System
+
+This component handles the intake of various document formats and web content.
+
+#### Document Processor
+
+- **Responsibility**: Process uploaded documents (PDF, DOCX, CSV, PPTX, Excel)
+- **Technologies**: PyMuPDF, python-docx, pandas, python-pptx, pdfplumber
+- **Input**: Raw document files
+- **Output**: Extracted text content
+
+#### URL Processor
+
+- **Responsibility**: Crawl and extract content from provided URLs, including nested documents and links
+- **Technologies**: BeautifulSoup, requests, trafilatura
+- **Input**: URLs
+- **Output**: Extracted text content from web pages and linked documents
+
+### 2.2 Knowledge Processing System
+
+This component transforms raw text into queryable knowledge.
+
+#### Text Extractor
+
+- **Responsibility**: Clean, normalize, and chunk text from various sources
+- **Technologies**: NLTK, spaCy, regex
+- **Input**: Raw text from documents and web pages
+- **Output**: Cleaned, normalized text chunks ready for embedding
+
+#### Embedding Generator
+
+- **Responsibility**: Generate vector embeddings for text chunks
+- **Technology**: Gemini Embedding v3 (gemini-embedding-exp-03-07)
+- **Input**: Processed text chunks
+- **Output**: Vector embeddings
+
+### 2.3 Knowledge Storage System
+
+This component manages the storage and retrieval of vector embeddings.
+
+#### Vector Database
+
+- **Responsibility**: Store and index vector embeddings for efficient retrieval
+- **Technology**: Pinecone
+- **Input**: Vector embeddings with metadata
+- **Output**: Retrieved relevant vectors based on similarity
+
+### 2.4 Query Processing System
+
+This component handles user queries and generates responses.
+
+#### Query Processor
+
+- **Responsibility**: Process user queries and convert them to vector embeddings
+- **Technologies**: Gemini Embedding v3, LangChain
+- **Input**: User queries
+- **Output**: Query vector embeddings
+
+#### Response Generator
+
+- **Responsibility**: Generate coherent responses based on retrieved knowledge
+- **Technology**: LangChain RAG (Retrieval Augmented Generation)
+- **Input**: Retrieved relevant text chunks
+- **Output**: Natural language responses
+
+### 2.5 User Interface System
+
+This component provides the user-facing interface.
+
+#### Gradio UI
+
+- **Responsibility**: Provide intuitive interface for document upload, URL input, and querying
+- **Technology**: Gradio
+- **Features**:
+  - Document upload area
+  - URL input field
+  - Query input and response display
+  - System status indicators
+
+## 3. Data Flow
+
+```mermaid
+sequenceDiagram
+    participant User
+    participant UI as Gradio UI
+    participant DP as Document Processor
+    participant UP as URL Processor
+    participant TE as Text Extractor
+    participant EG as Embedding Generator
+    participant VDB as Vector Database
+    participant QP as Query Processor
+    participant RG as Response Generator
+
+    %% Document Upload Flow
+    User->>UI: Upload Document
+    UI->>DP: Process Document
+    DP->>TE: Extract Text
+    TE->>EG: Generate Embeddings
+    EG->>VDB: Store Embeddings
+
+    %% URL Processing Flow
+    User->>UI: Input URL
+    UI->>UP: Process URL
+    UP->>TE: Extract Text
+    TE->>EG: Generate Embeddings
+    EG->>VDB: Store Embeddings
+
+    %% Query Flow
+    User->>UI: Submit Query
+    UI->>QP: Process Query
+    QP->>EG: Generate Query Embedding
+    QP->>VDB: Retrieve Relevant Embeddings
+    VDB->>QP: Return Relevant Chunks
+    QP->>RG: Generate Response
+    RG->>UI: Display Response
+    UI->>User: Show Answer
+```
+
+## 4. Technical Architecture
+
+### 4.1 Technology Stack
+
+| Component          | Technology                                            | Purpose                                    |
+| ------------------ | ----------------------------------------------------- | ------------------------------------------ |
+| Document Parsing   | PyMuPDF, python-docx, pandas, python-pptx, pdfplumber | Extract text from various document formats |
+| Web Scraping       | BeautifulSoup, requests, trafilatura                  | Extract content from web pages             |
+| Text Processing    | NLTK, spaCy, regex                                    | Clean and chunk text                       |
+| Embedding          | Gemini Embedding v3 (gemini-embedding-exp-03-07)      | Generate vector embeddings                 |
+| Vector Storage     | Pinecone                                              | Store and retrieve vector embeddings       |
+| RAG Implementation | LangChain                                             | Implement retrieval augmented generation   |
+| User Interface     | Gradio                                                | Provide user-friendly interface            |
+
+### 4.2 Integration Points
+
+- **Document Processing → Text Extraction**: Raw text extraction from documents
+- **URL Processing → Text Extraction**: Raw text extraction from web pages
+- **Text Extraction → Embedding Generation**: Processed text chunks for embedding
+- **Embedding Generation → Vector Database**: Storage of embeddings
+- **Query Processing → Embedding Generation**: Query embedding generation
+- **Query Processing → Vector Database**: Retrieval of relevant embeddings
+- **Query Processing → Response Generation**: Generation of coherent responses
+- **Response Generation → UI**: Display of responses to user
+
+## 5. Deployment Architecture
+
+The system is designed to be deployed on Hugging Face using their Spaces feature, which supports Gradio applications.
+
+```mermaid
+graph TD
+    A[User] --> B[Hugging Face Space]
+    B --> C[Gradio Application]
+    C --> D[Document Processing]
+    C --> E[URL Processing]
+    C --> F[Query Processing]
+    D --> G[Gemini API]
+    E --> G
+    F --> G
+    D --> H[Pinecone API]
+    E --> H
+    F --> H
+```
+
+### 5.1 Deployment Considerations
+
+- **API Keys**: Secure storage of Gemini and Pinecone API keys
+- **Rate Limiting**: Handling API rate limits for both Gemini and Pinecone
+- **Memory Management**: Efficient memory usage within Hugging Face constraints
+- **Statelessness**: Designing components to be stateless where possible
+- **Error Handling**: Robust error handling for API failures and timeouts
+
+## 6. Scalability and Performance
+
+For the hackathon version, the focus is on functionality rather than scalability. However, the architecture is designed with the following considerations:
+
+- **Document Size Limits**: Implement reasonable limits on document sizes
+- **Chunking Strategy**: Optimize text chunking for better retrieval performance
+- **Caching**: Implement basic caching for frequently accessed embeddings
+- **Asynchronous Processing**: Use asynchronous processing where appropriate
+
+## 7. Future Enhancements
+
+While not implemented in the hackathon version, the architecture supports future enhancements:
+
+- **Authentication**: User authentication and document access control
+- **Document Versioning**: Track changes to documents over time
+- **Advanced RAG Techniques**: Implement more sophisticated RAG approaches
+- **Multi-Modal Support**: Add support for images and other non-text content
+- **Collaborative Features**: Allow multiple users to collaborate on knowledge bases
+- **Custom Training**: Fine-tune models for specific domains
+
+## 8. Folder Structure
+
+```
+rag-ai/
+├── src/
+│   ├── ingestion/
+│   │   ├── document_processor.py
+│   │   ├── url_processor.py
+│   │   └── text_extractor.py
+│   ├── embedding/
+│   │   └── embedding_generator.py
+│   ├── storage/
+│   │   └── vector_db.py
+│   ├── rag/
+│   │   ├── query_processor.py
+│   │   └── response_generator.py
+│   ├── ui/
+│   │   └── gradio_app.py
+│   └── utils/
+│       ├── config_manager.py
+│       └── error_handler.py
+├── config/
+│   └── config.yaml
+├── docs/
+│   ├── architecture.md
+│   └── api_documentation.md
+├── tests/
+│   ├── test_document_processor.py
+│   ├── test_url_processor.py
+│   └── ...
+├── scripts/
+│   ├── setup.py
+│   └── deploy_to_huggingface.py
+├── data/
+│   ├── sample_documents/
+│   └── test_data/
+├── .gitignore
+├── requirements.txt
+├── README.md
+└── app.py
+```
+
+## 9. Implementation Roadmap
+
+1. **Phase 1: Core Infrastructure**
+
+   - Set up project structure
+   - Implement basic document processing
+   - Set up Pinecone integration
+
+2. **Phase 2: Knowledge Processing**
+
+   - Implement text extraction and chunking
+   - Integrate Gemini embedding API
+   - Develop vector storage and retrieval
+
+3. **Phase 3: Query System**
+
+   - Implement query processing
+   - Develop RAG response generation
+   - Integrate components
+
+4. **Phase 4: User Interface**
+
+   - Develop Gradio UI
+   - Integrate UI with backend components
+   - Add error handling and user feedback
+
+5. **Phase 5: URL Processing**
+
+   - Implement URL crawling
+   - Add nested document extraction
+   - Integrate with existing components
+
+6. **Phase 6: Testing and Deployment**
+   - Comprehensive testing
+   - Optimization for Hugging Face deployment
+   - Documentation and demo preparation

requirements.txt

@@ -0,0 +1,103 @@
+# Core Dependencies
+gradio
+pyyaml
+python-dotenv
+
+# Document Processing
+PyPDF2
+PyMuPDF
+pdfplumber
+python-docx
+pandas
+openpyxl
+python-pptx
+
+# Web Scraping and URL Processing
+requests
+beautifulsoup4
+lxml
+html2text
+trafilatura
+
+# Text Processing
+nltk
+spacy
+textstat
+langdetect
+
+# Embedding and Vector Database
+google-generativeai
+pinecone
+sentence-transformers
+
+# LangChain and LLM Integration
+langchain
+langchain-google-genai
+langchain-openai
+langchain-community
+openai
+
+# Live Search Integration
+tavily-python
+
+# Vector Operations and ML
+numpy
+scikit-learn
+faiss-cpu
+
+# Async and Performance
+aiohttp
+asyncio
+
+# Logging and Monitoring
+structlog
+prometheus-client
+
+# Development and Testing
+pytest
+pytest-asyncio
+black
+flake8
+mypy
+
+# Optional Dependencies for Enhanced Features
+# Uncomment if needed:
+
+# Advanced NLP
+# transformers
+# torch
+
+# Image Processing (if document images need processing)
+# Pillow
+# pytesseract
+
+# Audio Processing (for future features)
+# librosa
+# soundfile
+
+# Database Support
+# psycopg2-binary
+# sqlalchemy
+
+# Cloud Storage
+# boto3
+# google-cloud-storage
+
+# Monitoring and Analytics
+# sentry-sdk
+
+# Additional Text Processing
+# langdetect
+# polyglot
+
+# Web Framework (if API endpoints needed)
+# fastapi
+# uvicorn
+
+# Caching
+# redis
+# diskcache
+
+# Configuration Management
+# hydra-core
+# omegaconf

src/embedding/__init__.py

@@ -0,0 +1,6 @@
+"""
+Embedding module for generating vector embeddings.
+
+This module contains components for generating vector embeddings
+from text chunks using Gemini Embedding v3.
+"""

src/embedding/embedding_generator.py

@@ -0,0 +1,462 @@
+"""
+Embedding Generator Module
+
+This module is responsible for generating vector embeddings for text chunks
+using Gemini Embedding v3 with complete API integration.
+
+Technology: Gemini Embedding v3 (gemini-embedding-exp-03-07)
+"""
+
+import logging
+import os
+import time
+import hashlib
+from datetime import datetime, timedelta
+from typing import Dict, List, Any, Optional, Union
+import json
+
+# Import Gemini API and caching libraries
+try:
+    import google.generativeai as genai
+    from cachetools import TTLCache
+except ImportError as e:
+    logging.warning(f"Some embedding libraries are not installed: {e}")
+
+from utils.error_handler import EmbeddingError, error_handler, ErrorType
+
+
+class EmbeddingGenerator:
+    """
+    Generates vector embeddings for text chunks using Gemini Embedding v3 with full functionality.
+
+    Features:
+    - Gemini Embedding v3 API integration
+    - Batch processing with rate limiting
+    - Intelligent retry logic with exponential backoff
+    - Embedding caching mechanism
+    - Cost optimization
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the EmbeddingGenerator with configuration.
+
+        Args:
+            config: Configuration dictionary with API parameters
+        """
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # API Configuration
+        self.api_key = self.config.get("api_key", os.environ.get("GEMINI_API_KEY"))
+        self.model = self.config.get("model", "gemini-embedding-exp-03-07")
+        self.batch_size = self.config.get("batch_size", 5)
+        self.max_retries = self.config.get("max_retries", 3)
+        self.retry_delay = self.config.get("retry_delay", 1)
+
+        # Performance settings
+        self.rate_limit_delay = self.config.get("rate_limit_delay", 0.1)
+        self.max_text_length = self.config.get(
+            "max_text_length", 8192
+        )  # ✨ 8K token limit for latest model
+        self.enable_caching = self.config.get("enable_caching", True)
+        self.cache_ttl = self.config.get("cache_ttl", 3600)  # 1 hour
+
+        # Statistics tracking
+        self.stats = {
+            "total_requests": 0,
+            "successful_requests": 0,
+            "failed_requests": 0,
+            "cache_hits": 0,
+            "total_tokens_processed": 0,
+            "start_time": datetime.now(),
+        }
+
+        # Initialize cache
+        if self.enable_caching:
+            self.cache = TTLCache(maxsize=1000, ttl=self.cache_ttl)
+        else:
+            self.cache = None
+
+        # Validate and initialize API client
+        self._initialize_client()
+
+    def _initialize_client(self):
+        """Initialize Gemini API client with validation."""
+        if not self.api_key:
+            self.logger.warning(
+                "No Gemini API key provided. Embeddings will not be generated."
+            )
+            self.client = None
+            return
+
+        try:
+            # Configure Gemini API
+            genai.configure(api_key=self.api_key)
+
+            # Test API connection
+            self._test_api_connection()
+
+            self.client = genai
+            self.logger.info("Gemini API client initialized successfully")
+
+        except Exception as e:
+            self.logger.error(f"Failed to initialize Gemini API client: {str(e)}")
+            self.client = None
+
+    def _test_api_connection(self):
+        """Test API connection with a simple request."""
+        try:
+            # Test with a simple embedding request
+            test_result = genai.embed_content(
+                model=self.model,
+                content="test connection",
+                task_type="retrieval_document",
+            )
+
+            if not test_result.get("embedding"):
+                raise Exception("No embedding returned from test request")
+
+            self.logger.info("API connection test successful")
+
+        except Exception as e:
+            raise EmbeddingError(f"API connection test failed: {str(e)}")
+
+    @error_handler(ErrorType.EMBEDDING_GENERATION)
+    def generate_embeddings(self, texts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Generate embeddings for a list of text chunks with full functionality.
+
+        Args:
+            texts: List of dictionaries containing text chunks and metadata
+                Each dict should have 'content' and 'metadata' keys
+
+        Returns:
+            List of dictionaries with original content, metadata, and embeddings
+        """
+        if not self.client or not texts:
+            self.logger.warning("No API client or empty text list")
+            return texts
+
+        self.logger.info(f"Generating embeddings for {len(texts)} text chunks")
+        start_time = time.time()
+
+        # Filter and validate texts
+        valid_texts = self._validate_texts(texts)
+        if not valid_texts:
+            self.logger.warning("No valid texts to process")
+            return texts
+
+        # Process in batches to respect API limits
+        results = []
+        total_batches = (len(valid_texts) + self.batch_size - 1) // self.batch_size
+
+        for i in range(0, len(valid_texts), self.batch_size):
+            batch_num = (i // self.batch_size) + 1
+            batch = valid_texts[i : i + self.batch_size]
+
+            self.logger.info(
+                f"Processing batch {batch_num}/{total_batches} ({len(batch)} items)"
+            )
+
+            try:
+                batch_results = self._process_batch(batch)
+                results.extend(batch_results)
+
+                # Rate limiting between batches
+                if i + self.batch_size < len(valid_texts):
+                    time.sleep(self.rate_limit_delay)
+
+            except Exception as e:
+                self.logger.error(f"Batch {batch_num} failed: {str(e)}")
+                # Add original items without embeddings
+                for item in batch:
+                    item_copy = item.copy()
+                    item_copy["embedding"] = []
+                    item_copy["embedding_error"] = str(e)
+                    results.append(item_copy)
+
+        # Update statistics
+        processing_time = time.time() - start_time
+        self.logger.info(f"Embedding generation completed in {processing_time:.2f}s")
+
+        return results
+
+    def _validate_texts(self, texts: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Validate and filter text inputs.
+
+        Args:
+            texts: List of text dictionaries
+
+        Returns:
+            List of valid text dictionaries
+        """
+        valid_texts = []
+
+        for i, item in enumerate(texts):
+            if not isinstance(item, dict) or "content" not in item:
+                self.logger.warning(f"Invalid item at index {i}: missing 'content' key")
+                continue
+
+            content = item["content"]
+            if not content or not isinstance(content, str):
+                self.logger.warning(
+                    f"Invalid content at index {i}: empty or non-string"
+                )
+                continue
+
+            # Truncate if too long
+            if len(content) > self.max_text_length:
+                self.logger.warning(
+                    f"Truncating text at index {i}: {len(content)} -> {self.max_text_length} chars"
+                )
+                item = item.copy()
+                item["content"] = content[: self.max_text_length]
+                item["metadata"] = item.get("metadata", {})
+                item["metadata"]["truncated"] = True
+                item["metadata"]["original_length"] = len(content)
+
+            valid_texts.append(item)
+
+        return valid_texts
+
+    def _process_batch(self, batch: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Process a batch of text chunks to generate embeddings.
+
+        Args:
+            batch: List of dictionaries containing text chunks and metadata
+
+        Returns:
+            List of dictionaries with original content, metadata, and embeddings
+        """
+        # Extract content and check cache
+        contents = []
+        cache_results = {}
+
+        for i, item in enumerate(batch):
+            content = item["content"]
+
+            # Check cache first
+            if self.cache is not None:
+                cache_key = self._get_cache_key(content)
+                if cache_key in self.cache:
+                    cache_results[i] = self.cache[cache_key]
+                    self.stats["cache_hits"] += 1
+                    continue
+
+            contents.append((i, content))
+
+        # Generate embeddings for non-cached content
+        embeddings_map = {}
+        if contents:
+            content_texts = [content for _, content in contents]
+            embeddings = self._generate_with_retry(content_texts)
+
+            # Map embeddings back to indices
+            for j, (original_index, content) in enumerate(contents):
+                if j < len(embeddings):
+                    embedding = embeddings[j]
+                    embeddings_map[original_index] = embedding
+
+                    # Cache the result
+                    if self.cache is not None:
+                        cache_key = self._get_cache_key(content)
+                        self.cache[cache_key] = embedding
+
+        # 🔗 Combine results
+        results = []
+        for i, item in enumerate(batch):
+            result = item.copy()
+
+            # Add embedding from cache or new generation
+            if i in cache_results:
+                result["embedding"] = cache_results[i]
+                result["embedding_source"] = "cache"
+            elif i in embeddings_map:
+                result["embedding"] = embeddings_map[i]
+                result["embedding_source"] = "api"
+            else:
+                result["embedding"] = []
+                result["embedding_source"] = "failed"
+                self.logger.warning(f"Missing embedding for batch item {i}")
+
+            # Add embedding metadata
+            if result["embedding"]:
+                result["metadata"] = result.get("metadata", {})
+                result["metadata"].update(
+                    {
+                        "embedding_model": self.model,
+                        "embedding_dimension": len(result["embedding"]),
+                        "embedding_generated_at": datetime.now().isoformat(),
+                    }
+                )
+
+            results.append(result)
+
+        return results
+
+    def _generate_with_retry(self, texts: List[str]) -> List[List[float]]:
+        """
+        Generate embeddings with intelligent retry logic.
+
+        Args:
+            texts: List of text strings to embed
+
+        Returns:
+            List of embedding vectors (each is a list of floats)
+        """
+        for attempt in range(self.max_retries):
+            try:
+                self.stats["total_requests"] += 1
+
+                # Generate embeddings using Gemini API
+                embeddings = []
+
+                for text in texts:
+                    try:
+                        # Track tokens
+                        self.stats["total_tokens_processed"] += len(text.split())
+
+                        # Call Gemini API
+                        result = self.client.embed_content(
+                            model=self.model,
+                            content=text,
+                            task_type="retrieval_document",
+                            title="Document chunk for RAG system",
+                        )
+
+                        if result and "embedding" in result:
+                            embeddings.append(result["embedding"])
+                        else:
+                            self.logger.warning(
+                                f"No embedding in API response for text: {text[:50]}..."
+                            )
+                            embeddings.append([])
+
+                    except Exception as e:
+                        self.logger.warning(
+                            f"Failed to embed individual text: {str(e)}"
+                        )
+                        embeddings.append([])
+
+                self.stats["successful_requests"] += 1
+                return embeddings
+
+            except Exception as e:
+                self.stats["failed_requests"] += 1
+                self.logger.warning(
+                    f"Embedding generation failed (attempt {attempt+1}/{self.max_retries}): {str(e)}"
+                )
+
+                if attempt < self.max_retries - 1:
+                    # Exponential backoff with jitter
+                    delay = self.retry_delay * (2**attempt) + (time.time() % 1)
+                    self.logger.info(f"Retrying in {delay:.1f} seconds...")
+                    time.sleep(delay)
+
+        # All retries failed
+        self.logger.error("All embedding generation attempts failed")
+        return [[] for _ in texts]
+
+    @error_handler(ErrorType.EMBEDDING_GENERATION)
+    def generate_query_embedding(self, query: str) -> List[float]:
+        """
+        Generate embedding for a single query string.
+
+        Args:
+            query: Query text to embed
+
+        Returns:
+            Embedding vector as a list of floats
+        """
+        if not self.client or not query:
+            return []
+
+        self.logger.info(f"Generating embedding for query: {query[:50]}...")
+
+        # Check cache first
+        if self.cache is not None:
+            cache_key = self._get_cache_key(query, "query")
+            if cache_key in self.cache:
+                self.stats["cache_hits"] += 1
+                return self.cache[cache_key]
+
+        # Generate embedding
+        embeddings = self._generate_with_retry([query])
+        embedding = embeddings[0] if embeddings else []
+
+        # Cache the result
+        if embedding and self.cache is not None:
+            cache_key = self._get_cache_key(query, "query")
+            self.cache[cache_key] = embedding
+
+        return embedding
+
+    def _get_cache_key(self, text: str, prefix: str = "doc") -> str:
+        """
+        Generate cache key for text.
+
+        Args:
+            text: Text content
+            prefix: Key prefix
+
+        Returns:
+            Cache key string
+        """
+        # 🔐 Create hash of text + model for unique key
+        content_hash = hashlib.md5(f"{self.model}:{text}".encode()).hexdigest()
+        return f"{prefix}:{content_hash}"
+
+    def get_statistics(self) -> Dict[str, Any]:
+        """
+        Get embedding generation statistics.
+
+        Returns:
+            Dictionary with statistics
+        """
+        runtime = datetime.now() - self.stats["start_time"]
+
+        return {
+            **self.stats,
+            "runtime_seconds": runtime.total_seconds(),
+            "cache_hit_rate": (
+                self.stats["cache_hits"] / max(1, self.stats["total_requests"]) * 100
+            ),
+            "success_rate": (
+                self.stats["successful_requests"]
+                / max(1, self.stats["total_requests"])
+                * 100
+            ),
+            "avg_tokens_per_request": (
+                self.stats["total_tokens_processed"]
+                / max(1, self.stats["total_requests"])
+            ),
+            "cache_size": len(self.cache) if self.cache else 0,
+            "model": self.model,
+            "batch_size": self.batch_size,
+        }
+
+    def clear_cache(self):
+        """Clear the embedding cache."""
+        if self.cache:
+            self.cache.clear()
+            self.logger.info("Embedding cache cleared")
+
+    def warm_up_cache(self, sample_texts: List[str]):
+        """
+        🔥 Warm up the cache with sample texts.
+
+        Args:
+            sample_texts: List of sample texts to pre-generate embeddings
+        """
+        if not sample_texts:
+            return
+
+        self.logger.info(f"🔥 Warming up cache with {len(sample_texts)} sample texts")
+
+        sample_items = [{"content": text, "metadata": {}} for text in sample_texts]
+        self.generate_embeddings(sample_items)
+
+        self.logger.info("Cache warm-up completed")

src/ingestion/__init__.py

@@ -0,0 +1,6 @@
+"""
+Ingestion module for processing documents and URLs.
+
+This module contains components for processing various document formats
+and extracting content from web URLs.
+"""

src/ingestion/document_processor.py

@@ -0,0 +1,668 @@
+"""
+Document Processor Module
+
+This module is responsible for processing various document formats including
+PDF, DOCX, CSV, PPTX, and Excel files with complete functionality.
+
+Technologies: PyMuPDF, python-docx, pandas, python-pptx, pdfplumber
+"""
+
+import os
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Union
+import logging
+
+# Import document processing libraries
+try:
+    import fitz  # PyMuPDF
+    import docx
+    import pandas as pd
+    import pptx
+    import pdfplumber
+    from openpyxl import load_workbook
+except ImportError as e:
+    logging.warning(f"Some document processing libraries are not installed: {e}")
+
+from utils.error_handler import DocumentProcessingError, error_handler, ErrorType
+
+
+class DocumentProcessor:
+    """
+    Processes various document formats and extracts text content with full functionality.
+
+    Supported formats:
+    - PDF (using PyMuPDF and pdfplumber)
+    - DOCX (using python-docx)
+    - CSV/Excel (using pandas)
+    - PPTX (using python-pptx)
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the DocumentProcessor with configuration.
+
+        Args:
+            config: Configuration dictionary with processing parameters
+        """
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # Configuration settings
+        self.max_file_size_mb = self.config.get("max_file_size_mb", 50)
+        self.supported_formats = self.config.get(
+            "supported_formats",
+            [".pdf", ".docx", ".csv", ".xlsx", ".xls", ".pptx", ".txt", ".md"],
+        )
+
+    @error_handler(ErrorType.DOCUMENT_PROCESSING)
+    def process_document(self, file_path: str) -> Dict[str, Any]:
+        """
+        Process a document and extract its text content with metadata.
+
+        Args:
+            file_path: Path to the document file
+
+        Returns:
+            Dictionary containing extracted text and metadata
+        """
+        if not os.path.exists(file_path):
+            raise DocumentProcessingError(f"Document not found: {file_path}", file_path)
+
+        # Validate file size
+        file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+        if file_size_mb > self.max_file_size_mb:
+            raise DocumentProcessingError(
+                f"File too large: {file_size_mb:.1f}MB (max: {self.max_file_size_mb}MB)",
+                file_path,
+            )
+
+        file_extension = os.path.splitext(file_path)[1].lower()
+
+        # Validate file format
+        if file_extension not in self.supported_formats:
+            raise DocumentProcessingError(
+                f"Unsupported file format: {file_extension}", file_path
+            )
+
+        self.logger.info(f"Processing document: {file_path} ({file_size_mb:.1f}MB)")
+
+        try:
+            if file_extension == ".pdf":
+                return self._process_pdf(file_path)
+            elif file_extension == ".docx":
+                return self._process_docx(file_path)
+            elif file_extension in [".csv", ".xlsx", ".xls"]:
+                return self._process_spreadsheet(file_path)
+            elif file_extension == ".pptx":
+                return self._process_pptx(file_path)
+            elif file_extension in [".txt", ".md"]:
+                return self._process_text_file(file_path)
+        except Exception as e:
+            raise DocumentProcessingError(
+                f"Error processing document: {str(e)}", file_path
+            )
+
+    def process_batch(self, file_paths: List[str]) -> List[Dict[str, Any]]:
+        """
+        Process multiple documents in batch.
+
+        Args:
+            file_paths: List of file paths to process
+
+        Returns:
+            List of processed document results
+        """
+        results = []
+        self.logger.info(f"Processing batch of {len(file_paths)} documents")
+
+        for i, file_path in enumerate(file_paths):
+            try:
+                result = self.process_document(file_path)
+                results.append(result)
+                self.logger.info(f"Processed {i+1}/{len(file_paths)}: {file_path}")
+            except Exception as e:
+                self.logger.error(f"❌ Failed to process {file_path}: {str(e)}")
+                # Continue with other files
+                continue
+
+        return results
+
+    def _extract_metadata(self, file_path: str) -> Dict[str, Any]:
+        """
+        Extract common metadata from file.
+
+        Args:
+            file_path: Path to the file
+
+        Returns:
+            Dictionary containing file metadata
+        """
+        file_stat = os.stat(file_path)
+        file_path_obj = Path(file_path)
+
+        return {
+            "filename": file_path_obj.name,
+            "file_extension": file_path_obj.suffix.lower(),
+            "file_size_bytes": file_stat.st_size,
+            "file_size_mb": round(file_stat.st_size / (1024 * 1024), 2),
+            "created_time": datetime.fromtimestamp(file_stat.st_ctime).isoformat(),
+            "modified_time": datetime.fromtimestamp(file_stat.st_mtime).isoformat(),
+            "processed_time": datetime.now().isoformat(),
+        }
+
+    def _process_pdf(self, file_path: str) -> Dict[str, Any]:
+        """
+        📄 Extract text from a PDF document using PyMuPDF with fallback to pdfplumber.
+
+        Args:
+            file_path: Path to the PDF file
+
+        Returns:
+            Dictionary with extracted text and metadata
+        """
+        self.logger.info(f"Processing PDF: {file_path}")
+
+        text_content = []
+        metadata = self._extract_metadata(file_path)
+
+        try:
+            # Primary method: PyMuPDF (faster)
+            doc = fitz.open(file_path)
+            metadata.update(
+                {
+                    "page_count": doc.page_count,
+                    "title": doc.metadata.get("title", ""),
+                    "author": doc.metadata.get("author", ""),
+                    "subject": doc.metadata.get("subject", ""),
+                    "creator": doc.metadata.get("creator", ""),
+                }
+            )
+
+            for page_num in range(doc.page_count):
+                page = doc[page_num]
+                text = page.get_text()
+                if text.strip():  # Only add non-empty pages
+                    text_content.append({"page": page_num + 1, "content": text.strip()})
+
+            doc.close()
+
+        except Exception as e:
+            self.logger.warning(f"PyMuPDF failed, trying pdfplumber: {str(e)}")
+
+            # Fallback method: pdfplumber (more robust for complex PDFs)
+            try:
+                with pdfplumber.open(file_path) as pdf:
+                    metadata["page_count"] = len(pdf.pages)
+
+                    for page_num, page in enumerate(pdf.pages):
+                        text = page.extract_text()
+                        if text and text.strip():
+                            text_content.append(
+                                {"page": page_num + 1, "content": text.strip()}
+                            )
+
+            except Exception as fallback_error:
+                raise DocumentProcessingError(
+                    f"Both PDF extraction methods failed: {str(fallback_error)}",
+                    file_path,
+                )
+
+        # Final content processing
+        full_text = "\n\n".join([item["content"] for item in text_content])
+        metadata["total_characters"] = len(full_text)
+        metadata["total_words"] = len(full_text.split())
+
+        return {
+            "content": full_text,
+            "pages": text_content,
+            "metadata": metadata,
+            "source": file_path,
+            "document_type": "pdf",
+        }
+
+    def _process_docx(self, file_path: str) -> Dict[str, Any]:
+        """
+        Extract text from a DOCX document using python-docx.
+
+        Args:
+            file_path: Path to the DOCX file
+
+        Returns:
+            Dictionary with extracted text and metadata
+        """
+        self.logger.info(f"Processing DOCX: {file_path}")
+
+        try:
+            doc = docx.Document(file_path)
+            metadata = self._extract_metadata(file_path)
+
+            # Extract document properties
+            core_props = doc.core_properties
+            metadata.update(
+                {
+                    "title": core_props.title or "",
+                    "author": core_props.author or "",
+                    "subject": core_props.subject or "",
+                    "created": (
+                        core_props.created.isoformat() if core_props.created else ""
+                    ),
+                    "modified": (
+                        core_props.modified.isoformat() if core_props.modified else ""
+                    ),
+                    "paragraph_count": len(doc.paragraphs),
+                }
+            )
+
+            # Extract text content
+            paragraphs = []
+            full_text_parts = []
+
+            for i, paragraph in enumerate(doc.paragraphs):
+                text = paragraph.text.strip()
+                if text:  # Only include non-empty paragraphs
+                    paragraphs.append({"paragraph": i + 1, "content": text})
+                    full_text_parts.append(text)
+
+            #   Extract tables if present
+            tables_content = []
+            for table_idx, table in enumerate(doc.tables):
+                table_data = []
+                for row in table.rows:
+                    row_data = [cell.text.strip() for cell in row.cells]
+                    if any(row_data):  # Only include non-empty rows
+                        table_data.append(row_data)
+
+                if table_data:
+                    tables_content.append({"table": table_idx + 1, "data": table_data})
+                    # Add table content to full text
+                    table_text = "\n".join([" | ".join(row) for row in table_data])
+                    full_text_parts.append(f"\n[Table {table_idx + 1}]\n{table_text}")
+
+            full_text = "\n\n".join(full_text_parts)
+            metadata.update(
+                {
+                    "total_characters": len(full_text),
+                    "total_words": len(full_text.split()),
+                    "table_count": len(tables_content),
+                }
+            )
+
+            return {
+                "content": full_text,
+                "paragraphs": paragraphs,
+                "tables": tables_content,
+                "metadata": metadata,
+                "source": file_path,
+                "document_type": "docx",
+            }
+
+        except Exception as e:
+            raise DocumentProcessingError(f"Error processing DOCX: {str(e)}", file_path)
+
+    def _process_spreadsheet(self, file_path: str) -> Dict[str, Any]:
+        """
+        Extract text from a CSV or Excel file using pandas.
+
+        Args:
+            file_path: Path to the spreadsheet file
+
+        Returns:
+            Dictionary with extracted text and metadata
+        """
+        file_extension = os.path.splitext(file_path)[1].lower()
+        self.logger.info(f"Processing spreadsheet: {file_path}")
+
+        try:
+            metadata = self._extract_metadata(file_path)
+            sheets_data = []
+
+            if file_extension == ".csv":
+                # 📄 Process CSV file
+                df = pd.read_csv(file_path, encoding="utf-8")
+                sheet_content = self._process_dataframe(df, "Sheet1")
+                sheets_data.append(sheet_content)
+                metadata["sheet_count"] = 1
+
+            else:
+                # Process Excel file
+                excel_file = pd.ExcelFile(file_path)
+                metadata["sheet_count"] = len(excel_file.sheet_names)
+
+                for sheet_name in excel_file.sheet_names:
+                    df = pd.read_excel(file_path, sheet_name=sheet_name)
+                    sheet_content = self._process_dataframe(df, sheet_name)
+                    sheets_data.append(sheet_content)
+
+            # 🔗 Combine all sheets content
+            full_text_parts = []
+            for sheet in sheets_data:
+                full_text_parts.append(f"[{sheet['sheet_name']}]\n{sheet['content']}")
+
+            full_text = "\n\n".join(full_text_parts)
+            metadata.update(
+                {
+                    "total_characters": len(full_text),
+                    "total_words": len(full_text.split()),
+                    "total_rows": sum(sheet["row_count"] for sheet in sheets_data),
+                    "total_columns": (
+                        max(sheet["column_count"] for sheet in sheets_data)
+                        if sheets_data
+                        else 0
+                    ),
+                }
+            )
+
+            return {
+                "content": full_text,
+                "sheets": sheets_data,
+                "metadata": metadata,
+                "source": file_path,
+                "document_type": "spreadsheet",
+            }
+
+        except Exception as e:
+            raise DocumentProcessingError(
+                f"Error processing spreadsheet: {str(e)}", file_path
+            )
+
+    def _process_dataframe(self, df: pd.DataFrame, sheet_name: str) -> Dict[str, Any]:
+        """
+        Process a pandas DataFrame into text content.
+
+        Args:
+            df: Pandas DataFrame
+            sheet_name: Name of the sheet
+
+        Returns:
+            Dictionary with processed sheet data
+        """
+        # Clean the dataframe
+        df = df.dropna(how="all")  # Remove completely empty rows
+        df = df.fillna("")  # Fill NaN with empty strings
+
+        #   Create text representation
+        content_parts = []
+
+        # Add headers
+        headers = df.columns.tolist()
+        content_parts.append(" | ".join(str(h) for h in headers))
+        content_parts.append("-" * 50)  # Separator
+
+        # Add data rows
+        for _, row in df.iterrows():
+            row_text = " | ".join(str(cell) for cell in row.values)
+            content_parts.append(row_text)
+
+        content = "\n".join(content_parts)
+
+        return {
+            "sheet_name": sheet_name,
+            "content": content,
+            "headers": headers,
+            "row_count": len(df),
+            "column_count": len(df.columns),
+            "data": df.to_dict("records"),  # For structured access
+        }
+
+    def _process_pptx(self, file_path: str) -> Dict[str, Any]:
+        """
+        🎯 Extract text from a PowerPoint presentation using python-pptx.
+
+        Args:
+            file_path: Path to the PPTX file
+
+        Returns:
+            Dictionary with extracted text and metadata
+        """
+        self.logger.info(f" Processing PPTX: {file_path}")
+
+        try:
+            presentation = pptx.Presentation(file_path)
+            metadata = self._extract_metadata(file_path)
+
+            # Extract presentation metadata
+            core_props = presentation.core_properties
+            metadata.update(
+                {
+                    "title": core_props.title or "",
+                    "author": core_props.author or "",
+                    "subject": core_props.subject or "",
+                    "created": (
+                        core_props.created.isoformat() if core_props.created else ""
+                    ),
+                    "modified": (
+                        core_props.modified.isoformat() if core_props.modified else ""
+                    ),
+                    "slide_count": len(presentation.slides),
+                }
+            )
+
+            # 🎯 Extract content from slides
+            slides_content = []
+            full_text_parts = []
+
+            for slide_idx, slide in enumerate(presentation.slides):
+                slide_text_parts = []
+
+                # Extract text from all shapes in the slide
+                for shape in slide.shapes:
+                    if hasattr(shape, "text") and shape.text.strip():
+                        slide_text_parts.append(shape.text.strip())
+
+                if slide_text_parts:
+                    slide_content = "\n".join(slide_text_parts)
+                    slides_content.append(
+                        {"slide": slide_idx + 1, "content": slide_content}
+                    )
+                    full_text_parts.append(f"[Slide {slide_idx + 1}]\n{slide_content}")
+
+            full_text = "\n\n".join(full_text_parts)
+            metadata.update(
+                {
+                    "total_characters": len(full_text),
+                    "total_words": len(full_text.split()),
+                    "slides_with_content": len(slides_content),
+                }
+            )
+
+            return {
+                "content": full_text,
+                "slides": slides_content,
+                "metadata": metadata,
+                "source": file_path,
+                "document_type": "pptx",
+            }
+
+        except Exception as e:
+            raise DocumentProcessingError(f"Error processing PPTX: {str(e)}", file_path)
+
+    def _process_text_file(self, file_path: str) -> Dict[str, Any]:
+        """
+        📝 Extract text from plain text files (.txt, .md).
+
+        Args:
+            file_path: Path to the text file
+
+        Returns:
+            Dictionary with extracted text and metadata
+        """
+        file_extension = os.path.splitext(file_path)[1].lower()
+        self.logger.info(f" Processing text file: {file_path}")
+
+        try:
+            metadata = self._extract_metadata(file_path)
+
+            # Try different encodings for robust text reading
+            encodings = ["utf-8", "utf-8-sig", "latin-1", "cp1252"]
+            content = None
+
+            for encoding in encodings:
+                try:
+                    with open(file_path, "r", encoding=encoding) as file:
+                        content = file.read()
+                    self.logger.info(
+                        f" Successfully read file with {encoding} encoding"
+                    )
+                    break
+                except UnicodeDecodeError:
+                    continue
+                except Exception as e:
+                    self.logger.warning(f"Failed to read with {encoding}: {str(e)}")
+                    continue
+
+            if content is None:
+                raise DocumentProcessingError(
+                    f"Could not read file with any supported encoding", file_path
+                )
+
+            # Clean and process content
+            content = content.strip()
+            if not content:
+                raise DocumentProcessingError(
+                    f"File is empty or contains no readable text", file_path
+                )
+
+            # Split content into logical sections for better processing
+            sections = []
+            if file_extension == ".md":
+                # 📋 For Markdown files, split by headers
+                sections = self._split_markdown_content(content)
+            else:
+                # 📄 For plain text, split by paragraphs
+                sections = self._split_text_content(content)
+
+            # Update metadata with text-specific information
+            lines = content.split("\n")
+            metadata.update(
+                {
+                    "file_type": (
+                        "markdown" if file_extension == ".md" else "plain_text"
+                    ),
+                    "line_count": len(lines),
+                    "paragraph_count": len(
+                        [p for p in content.split("\n\n") if p.strip()]
+                    ),
+                    "total_characters": len(content),
+                    "total_words": len(content.split()),
+                    "encoding_used": encoding if "encoding" in locals() else "utf-8",
+                    "sections_count": len(sections),
+                }
+            )
+
+            return {
+                "content": content,
+                "sections": sections,
+                "metadata": metadata,
+                "source": file_path,
+                "document_type": "markdown" if file_extension == ".md" else "text",
+            }
+
+        except Exception as e:
+            raise DocumentProcessingError(
+                f"Error processing text file: {str(e)}", file_path
+            )
+
+    def _split_markdown_content(self, content: str) -> List[Dict[str, Any]]:
+        """
+        Split Markdown content by headers for better organization.
+
+        Args:
+            content: Markdown content
+
+        Returns:
+            List of sections with headers and content
+        """
+        sections = []
+        lines = content.split("\n")
+        current_section = {"header": "", "content": [], "level": 0}
+
+        for line in lines:
+            # Check for markdown headers
+            if line.strip().startswith("#"):
+                # Save previous section if it has content
+                if current_section["content"] or current_section["header"]:
+                    section_content = "\n".join(current_section["content"]).strip()
+                    if section_content or current_section["header"]:
+                        sections.append(
+                            {
+                                "header": current_section["header"],
+                                "content": section_content,
+                                "level": current_section["level"],
+                                "section_index": len(sections),
+                            }
+                        )
+
+                # Start new section
+                header_level = len(line) - len(line.lstrip("#"))
+                header_text = line.lstrip("#").strip()
+                current_section = {
+                    "header": header_text,
+                    "content": [],
+                    "level": header_level,
+                }
+            else:
+                current_section["content"].append(line)
+
+        # Add the last section
+        if current_section["content"] or current_section["header"]:
+            section_content = "\n".join(current_section["content"]).strip()
+            if section_content or current_section["header"]:
+                sections.append(
+                    {
+                        "header": current_section["header"],
+                        "content": section_content,
+                        "level": current_section["level"],
+                        "section_index": len(sections),
+                    }
+                )
+
+        # If no headers found, treat entire content as one section
+        if not sections:
+            sections.append(
+                {
+                    "header": "Document Content",
+                    "content": content.strip(),
+                    "level": 1,
+                    "section_index": 0,
+                }
+            )
+
+        return sections
+
+    def _split_text_content(self, content: str) -> List[Dict[str, Any]]:
+        """
+        Split plain text content by paragraphs.
+
+        Args:
+            content: Plain text content
+
+        Returns:
+            List of paragraph sections
+        """
+        sections = []
+        paragraphs = [p.strip() for p in content.split("\n\n") if p.strip()]
+
+        for i, paragraph in enumerate(paragraphs):
+            sections.append(
+                {
+                    "header": f"Paragraph {i + 1}",
+                    "content": paragraph,
+                    "level": 1,
+                    "section_index": i,
+                }
+            )
+
+        # If no clear paragraphs, treat as single section
+        if not sections:
+            sections.append(
+                {
+                    "header": "Document Content",
+                    "content": content.strip(),
+                    "level": 1,
+                    "section_index": 0,
+                }
+            )
+
+        return sections

src/ingestion/pipeline.py

@@ -0,0 +1,495 @@
+"""
+Ingestion Pipeline Module
+
+This module orchestrates the complete document ingestion process,
+integrating all components for a seamless workflow.
+
+Components: DocumentProcessor, URLProcessor, TextExtractor, EmbeddingGenerator, VectorDB
+"""
+
+import logging
+from typing import Dict, List, Any, Optional, Union
+from pathlib import Path
+import asyncio
+from datetime import datetime
+
+from .document_processor import DocumentProcessor
+from .url_processor import URLProcessor
+from ingestion.text_extractor import TextExtractor
+from embedding.embedding_generator import EmbeddingGenerator
+from storage.vector_db import VectorDB
+from utils.config_manager import ConfigManager
+from utils.error_handler import error_handler, ErrorType, RAGError
+
+
+class IngestionPipeline:
+    """
+    Complete ingestion pipeline that orchestrates document processing, text extraction,
+    embedding generation, and vector storage.
+
+    Features:
+    - End-to-end document ingestion 
+    - URL content processing 
+    - Batch processing capabilities 
+    - Progress tracking and statistics 
+    - Error handling and recovery 
+    """
+
+    def __init__(self, config_path: Optional[str] = None):
+        """
+        Initialize the ingestion pipeline.
+
+        Args:
+            config_path: Path to configuration file
+        """
+        self.logger = logging.getLogger(__name__)
+
+        # Load configuration
+        self.config_manager = ConfigManager(config_path)
+        self.config = self.config_manager.config
+
+        # Initialize statistics
+        self.stats = {
+            "documents_processed": 0,
+            "urls_processed": 0,
+            "chunks_created": 0,
+            "embeddings_generated": 0,
+            "vectors_stored": 0,
+            "errors_encountered": 0,
+            "start_time": None,
+            "end_time": None,
+        }
+
+        # Initialize components
+        self._initialize_components()
+
+    def _initialize_components(self):
+        """Initialize all pipeline components."""
+        try:
+            # 📄 Document processor
+            doc_config = self.config.get("document_processing", {})
+            self.document_processor = DocumentProcessor(doc_config)
+
+            # URL processor
+            url_config = self.config.get("url_processing", {})
+            self.url_processor = URLProcessor(url_config)
+
+            # Text extractor
+            text_config = self.config.get("document_processing", {})
+            self.text_extractor = TextExtractor(text_config)
+
+            # 🔮 Embedding generator
+            embedding_config = self.config.get("embedding", {})
+            embedding_config["api_key"] = self.config.get("api_keys", {}).get(
+                "gemini_api_key"
+            )
+            self.embedding_generator = EmbeddingGenerator(embedding_config)
+
+            # Vector database
+            vector_config = self.config.get("vector_db", {})
+            vector_config["api_key"] = self.config.get("api_keys", {}).get(
+                "pinecone_api_key"
+            )
+            self.vector_db = VectorDB(vector_config)
+
+            self.logger.info("All pipeline components initialized successfully")
+
+        except Exception as e:
+            self.logger.error(f"❌ Failed to initialize pipeline components: {str(e)}")
+            raise RAGError(f"Pipeline initialization failed: {str(e)}")
+
+    @error_handler(ErrorType.DOCUMENT_PROCESSING)
+    def process_documents(self, file_paths: List[str]) -> Dict[str, Any]:
+        """
+        Process multiple documents through the complete pipeline.
+
+        Args:
+            file_paths: List of document file paths
+
+        Returns:
+            Processing results and statistics
+        """
+        self.logger.info(
+            f"Starting document processing pipeline for {len(file_paths)} files"
+        )
+        self.stats["start_time"] = datetime.now()
+
+        all_results = []
+
+        for i, file_path in enumerate(file_paths):
+            try:
+                self.logger.info(
+                    f"📄 Processing document {i+1}/{len(file_paths)}: {file_path}"
+                )
+
+                # 📄 Step 1: Process document
+                doc_result = self.document_processor.process_document(file_path)
+                self.stats["documents_processed"] += 1
+
+                # Step 2: Extract and chunk text
+                text_chunks = self.text_extractor.process_text(
+                    doc_result["content"], doc_result["metadata"]
+                )
+                self.stats["chunks_created"] += len(text_chunks)
+
+                # 🔮 Step 3: Generate embeddings
+                embedded_chunks = self.embedding_generator.generate_embeddings(
+                    text_chunks
+                )
+                valid_embeddings = [
+                    chunk for chunk in embedded_chunks if chunk.get("embedding")
+                ]
+                self.stats["embeddings_generated"] += len(valid_embeddings)
+
+                # Step 4: Store in vector database
+                if valid_embeddings:
+                    storage_success = self.vector_db.store_embeddings(valid_embeddings)
+                    if storage_success:
+                        self.stats["vectors_stored"] += len(valid_embeddings)
+
+                # Compile results
+                result = {
+                    "file_path": file_path,
+                    "document_type": doc_result.get("document_type"),
+                    "chunks_created": len(text_chunks),
+                    "embeddings_generated": len(valid_embeddings),
+                    "storage_success": storage_success if valid_embeddings else False,
+                    "metadata": doc_result["metadata"],
+                }
+
+                all_results.append(result)
+                self.logger.info(
+                    f"Document processed: {len(text_chunks)} chunks, {len(valid_embeddings)} embeddings"
+                )
+
+            except Exception as e:
+                self.stats["errors_encountered"] += 1
+                self.logger.error(f"❌ Error processing {file_path}: {str(e)}")
+
+                all_results.append(
+                    {
+                        "file_path": file_path,
+                        "error": str(e),
+                        "chunks_created": 0,
+                        "embeddings_generated": 0,
+                        "storage_success": False,
+                    }
+                )
+
+        self.stats["end_time"] = datetime.now()
+
+        return {
+            "results": all_results,
+            "statistics": self.get_statistics(),
+            "success_rate": self._calculate_success_rate(all_results),
+        }
+
+    @error_handler(ErrorType.URL_PROCESSING)
+    def process_urls(self, urls: List[str]) -> Dict[str, Any]:
+        """
+        Process multiple URLs through the complete pipeline.
+
+        Args:
+            urls: List of URLs to process
+
+        Returns:
+            Processing results and statistics
+        """
+        self.logger.info(f"Starting URL processing pipeline for {len(urls)} URLs")
+        self.stats["start_time"] = datetime.now()
+
+        all_results = []
+
+        for i, url in enumerate(urls):
+            try:
+                self.logger.info(f"Processing URL {i+1}/{len(urls)}: {url}")
+
+                # Step 1: Process URL
+                url_result = self.url_processor.process_url(url)
+                if not url_result:
+                    self.logger.warning(f"No content extracted from URL: {url}")
+                    continue
+
+                self.stats["urls_processed"] += 1
+
+                # Step 2: Extract and chunk text
+                text_chunks = self.text_extractor.process_text(
+                    url_result["content"], url_result["metadata"]
+                )
+                self.stats["chunks_created"] += len(text_chunks)
+
+                # 🔮 Step 3: Generate embeddings
+                embedded_chunks = self.embedding_generator.generate_embeddings(
+                    text_chunks
+                )
+                valid_embeddings = [
+                    chunk for chunk in embedded_chunks if chunk.get("embedding")
+                ]
+                self.stats["embeddings_generated"] += len(valid_embeddings)
+
+                # Step 4: Store in vector database
+                storage_success = False
+                if valid_embeddings:
+                    storage_success = self.vector_db.store_embeddings(valid_embeddings)
+                    if storage_success:
+                        self.stats["vectors_stored"] += len(valid_embeddings)
+
+                # Process linked documents if any
+                linked_results = []
+                for linked_doc in url_result.get("linked_documents", []):
+                    if linked_doc.get("content"):
+                        linked_chunks = self.text_extractor.process_text(
+                            linked_doc["content"], linked_doc["metadata"]
+                        )
+                        linked_embedded = self.embedding_generator.generate_embeddings(
+                            linked_chunks
+                        )
+                        linked_valid = [
+                            chunk for chunk in linked_embedded if chunk.get("embedding")
+                        ]
+
+                        if linked_valid:
+                            self.vector_db.store_embeddings(linked_valid)
+                            linked_results.append(
+                                {
+                                    "url": linked_doc["source"],
+                                    "chunks": len(linked_chunks),
+                                    "embeddings": len(linked_valid),
+                                }
+                            )
+
+                # Compile results
+                result = {
+                    "url": url,
+                    "chunks_created": len(text_chunks),
+                    "embeddings_generated": len(valid_embeddings),
+                    "storage_success": storage_success,
+                    "linked_documents": linked_results,
+                    "metadata": url_result["metadata"],
+                }
+
+                all_results.append(result)
+                self.logger.info(
+                    f"URL processed: {len(text_chunks)} chunks, {len(valid_embeddings)} embeddings"
+                )
+
+            except Exception as e:
+                self.stats["errors_encountered"] += 1
+                self.logger.error(f"❌ Error processing {url}: {str(e)}")
+
+                all_results.append(
+                    {
+                        "url": url,
+                        "error": str(e),
+                        "chunks_created": 0,
+                        "embeddings_generated": 0,
+                        "storage_success": False,
+                    }
+                )
+
+        self.stats["end_time"] = datetime.now()
+
+        return {
+            "results": all_results,
+            "statistics": self.get_statistics(),
+            "success_rate": self._calculate_success_rate(all_results),
+        }
+
+    def process_mixed_sources(
+        self, file_paths: Optional[List[str]] = None, urls: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """
+        Process both documents and URLs in a single pipeline run.
+
+        Args:
+            file_paths: Optional list of document file paths
+            urls: Optional list of URLs
+
+        Returns:
+            Combined processing results
+        """
+        self.logger.info("Starting mixed source processing pipeline")
+
+        results = {
+            "document_results": [],
+            "url_results": [],
+            "combined_statistics": {},
+            "overall_success_rate": 0.0,
+        }
+
+        # 📄 Process documents
+        if file_paths:
+            doc_results = self.process_documents(file_paths)
+            results["document_results"] = doc_results["results"]
+
+        # Process URLs
+        if urls:
+            url_results = self.process_urls(urls)
+            results["url_results"] = url_results["results"]
+
+        # Combine statistics
+        results["combined_statistics"] = self.get_statistics()
+
+        # 🎯 Calculate overall success rate
+        all_items = results["document_results"] + results["url_results"]
+        results["overall_success_rate"] = self._calculate_success_rate(all_items)
+
+        return results
+
+    def _calculate_success_rate(self, results: List[Dict[str, Any]]) -> float:
+        """
+        Calculate success rate from results.
+
+        Args:
+            results: List of processing results
+
+        Returns:
+            Success rate as percentage
+        """
+        if not results:
+            return 0.0
+
+        successful = sum(
+            1 for result in results if result.get("storage_success", False)
+        )
+        return (successful / len(results)) * 100
+
+    def get_statistics(self) -> Dict[str, Any]:
+        """
+        Get comprehensive pipeline statistics.
+
+        Returns:
+            Statistics dictionary
+        """
+        stats = self.stats.copy()
+
+        if stats["start_time"] and stats["end_time"]:
+            runtime = stats["end_time"] - stats["start_time"]
+            stats["runtime_seconds"] = runtime.total_seconds()
+            stats["processing_rate"] = (
+                stats["documents_processed"] + stats["urls_processed"]
+            ) / max(1, runtime.total_seconds())
+
+        # 🔮 Add component statistics
+        stats["embedding_stats"] = self.embedding_generator.get_statistics()
+        stats["vector_db_stats"] = self.vector_db.get_index_stats()
+        stats["url_processor_stats"] = self.url_processor.get_statistics()
+
+        return stats
+
+    def health_check(self) -> Dict[str, Any]:
+        """
+        Perform comprehensive health check on all components.
+
+        Returns:
+            Health check results
+        """
+        health = {
+            "overall_status": "healthy",
+            "timestamp": datetime.now().isoformat(),
+            "components": {},
+        }
+
+        try:
+            # 🔮 Check embedding generator
+            if self.embedding_generator.client:
+                health["components"]["embedding_generator"] = "Ready"
+            else:
+                health["components"]["embedding_generator"] = "❌ Not configured"
+                health["overall_status"] = "degraded"
+
+            # Check vector database
+            vector_health = self.vector_db.health_check()
+            health["components"]["vector_database"] = vector_health["status"]
+            if vector_health["status"] != "healthy":
+                health["overall_status"] = "degraded"
+
+            # Add component details
+            health["details"] = {
+                "vector_db_health": vector_health,
+                "embedding_stats": self.embedding_generator.get_statistics(),
+                "pipeline_stats": self.get_statistics(),
+            }
+
+        except Exception as e:
+            health["overall_status"] = "unhealthy"
+            health["error"] = str(e)
+
+        return health
+
+    def reset_statistics(self):
+        """Reset pipeline statistics."""
+        self.stats = {
+            "documents_processed": 0,
+            "urls_processed": 0,
+            "chunks_created": 0,
+            "embeddings_generated": 0,
+            "vectors_stored": 0,
+            "errors_encountered": 0,
+            "start_time": None,
+            "end_time": None,
+        }
+
+        # Reset component statistics
+        self.embedding_generator.stats = {
+            "total_requests": 0,
+            "successful_requests": 0,
+            "failed_requests": 0,
+            "cache_hits": 0,
+            "total_tokens_processed": 0,
+            "start_time": datetime.now(),
+        }
+
+        self.vector_db.reset_stats()
+        self.url_processor.reset()
+
+        self.logger.info("All pipeline statistics reset")
+
+
+# Convenience function for quick pipeline usage
+def create_pipeline(config_path: Optional[str] = None) -> IngestionPipeline:
+    """
+    Create and return a configured ingestion pipeline.
+
+    Args:
+        config_path: Optional path to configuration file
+
+    Returns:
+        Configured IngestionPipeline instance
+    """
+    return IngestionPipeline(config_path)
+
+
+# 📄 Example usage functions
+def process_documents_simple(
+    file_paths: List[str], config_path: Optional[str] = None
+) -> Dict[str, Any]:
+    """
+    📄 Simple function to process documents with default configuration.
+
+    Args:
+        file_paths: List of document file paths
+        config_path: Optional configuration file path
+
+    Returns:
+        Processing results
+    """
+    pipeline = create_pipeline(config_path)
+    return pipeline.process_documents(file_paths)
+
+
+def process_urls_simple(
+    urls: List[str], config_path: Optional[str] = None
+) -> Dict[str, Any]:
+    """
+    Simple function to process URLs with default configuration.
+
+    Args:
+        urls: List of URLs to process
+        config_path: Optional configuration file path
+
+    Returns:
+        Processing results
+    """
+    pipeline = create_pipeline(config_path)
+    return pipeline.process_urls(urls)

src/ingestion/text_extractor.py

@@ -0,0 +1,526 @@
+"""
+Text Extractor Module
+
+This module is responsible for cleaning, normalizing, and chunking text
+from various sources with complete NLP functionality.
+
+Technologies: NLTK, spaCy, regex, langdetect
+"""
+
+import re
+import logging
+from datetime import datetime
+from typing import Dict, List, Any, Optional, Union
+import unicodedata
+
+# Import NLP libraries
+try:
+    import nltk
+    from nltk.tokenize import sent_tokenize, word_tokenize
+    from nltk.corpus import stopwords
+    from nltk.stem import PorterStemmer
+    import spacy
+    from langdetect import detect
+    from langdetect.lang_detect_exception import LangDetectException as LangDetectError
+
+    # Download required NLTK data
+    try:
+        nltk.data.find("tokenizers/punkt")
+    except LookupError:
+        nltk.download("punkt", quiet=True)
+
+    try:
+        nltk.data.find("corpora/stopwords")
+    except LookupError:
+        nltk.download("stopwords", quiet=True)
+
+except ImportError as e:
+    logging.warning(f"Some NLP libraries are not installed: {e}")
+
+from utils.error_handler import error_handler, ErrorType
+
+
+class TextExtractor:
+    """
+    Cleans, normalizes, and chunks text from various sources with intelligent processing.
+
+    Features:
+    - Advanced text cleaning and normalization 
+    - Language detection 
+    - Intelligent sentence segmentation 
+    - Smart text chunking with overlap 
+    - Metadata preservation 
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the TextExtractor with configuration.
+
+        Args:
+            config: Configuration dictionary with processing parameters
+        """
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # Configuration settings
+        self.chunk_size = self.config.get("chunk_size", 1000)
+        self.chunk_overlap = self.config.get("chunk_overlap", 200)
+        self.min_chunk_size = self.config.get("min_chunk_size", 100)
+        self.max_chunk_size = self.config.get("max_chunk_size", 2000)
+
+        # NLP settings
+        self.enable_language_detection = self.config.get(
+            "enable_language_detection", True
+        )
+        self.preserve_formatting = self.config.get("preserve_formatting", False)
+        self.remove_stopwords = self.config.get("remove_stopwords", False)
+
+        # Initialize NLP components
+        self.nlp = None
+        self.stemmer = None
+        self.stop_words = set()
+
+        self._initialize_nlp_components()
+
+    def _initialize_nlp_components(self):
+        """Initialize NLP components with error handling."""
+        try:
+            # Load spaCy model for advanced processing
+            self.nlp = spacy.load("en_core_web_sm")
+            self.logger.info("spaCy model loaded successfully")
+        except Exception as e:
+            self.logger.warning(f"Could not load spaCy model: {str(e)}")
+
+        try:
+            # Initialize NLTK components
+            self.stemmer = PorterStemmer()
+            self.stop_words = set(stopwords.words("english"))
+            self.logger.info("NLTK components initialized")
+        except Exception as e:
+            self.logger.warning(f"Could not initialize NLTK components: {str(e)}")
+
+    @error_handler(ErrorType.DOCUMENT_PROCESSING)
+    def process_text(
+        self,
+        text: Union[str, List[str]],
+        metadata: Optional[Dict[str, Any]] = None,
+        preserve_structure: bool = False,
+    ) -> List[Dict[str, Any]]:
+        """
+        Process text by cleaning, normalizing, and chunking with intelligence.
+
+        Args:
+            text: Raw text content (string or list of strings)
+            metadata: Optional metadata to include with each chunk
+            preserve_structure: Whether to preserve original text structure
+
+        Returns:
+            List of dictionaries containing processed text chunks and metadata
+        """
+        if not text:
+            return []
+
+        # Convert list to string if needed
+        if isinstance(text, list):
+            text = "\n".join(str(item) for item in text if item)
+
+        if not text.strip():
+            return []
+
+        self.logger.info(f"Processing text: {len(text)} characters")
+
+        # Detect language
+        language = self._detect_language(text)
+
+        # Clean and normalize the text
+        cleaned_text = self._clean_text(text, preserve_structure)
+
+        if len(cleaned_text.strip()) < self.min_chunk_size:
+            self.logger.warning(
+                f"Text too short after cleaning: {len(cleaned_text)} chars"
+            )
+            return []
+
+        # Split text into chunks
+        chunks = self._chunk_text(cleaned_text)
+
+        # Prepare result with enhanced metadata
+        result = []
+        base_metadata = metadata.copy() if metadata else {}
+        base_metadata.update(
+            {
+                "language": language,
+                "original_length": len(text),
+                "cleaned_length": len(cleaned_text),
+                "chunk_count": len(chunks),
+                "processing_time": datetime.now().isoformat(),
+                "chunk_size_config": self.chunk_size,
+                "chunk_overlap_config": self.chunk_overlap,
+            }
+        )
+
+        for i, chunk in enumerate(chunks):
+            chunk_metadata = base_metadata.copy()
+            chunk_stats = self._analyze_chunk(chunk)
+
+            chunk_metadata.update(
+                {
+                    "chunk_index": i,
+                    "chunk_id": f"chunk_{i}_{hash(chunk) % 10000}",
+                    **chunk_stats,
+                }
+            )
+
+            result.append({"content": chunk, "metadata": chunk_metadata})
+
+        self.logger.info(f"Processed text into {len(chunks)} chunks")
+        return result
+
+    def _detect_language(self, text: str) -> str:
+        """
+        Detect the language of the text.
+
+        Args:
+            text: Text to analyze
+
+        Returns:
+            Language code (e.g., 'en', 'es', 'fr')
+        """
+        if not self.enable_language_detection:
+            return "en"  # Default to English
+
+        try:
+            # Use a sample of text for detection (first 1000 chars)
+            sample = text[:1000].strip()
+            if len(sample) < 50:  # Too short for reliable detection
+                return "en"
+
+            language = detect(sample)
+            self.logger.info(f"Detected language: {language}")
+            return language
+
+        except (LangDetectError, Exception) as e:
+            self.logger.warning(f"Language detection failed: {str(e)}")
+            return "en"  # Default to English
+
+    def _clean_text(self, text: str, preserve_structure: bool = False) -> str:
+        """
+        Clean and normalize text with advanced processing.
+
+        Args:
+            text: Raw text to clean
+            preserve_structure: Whether to preserve formatting
+
+        Returns:
+            Cleaned and normalized text
+        """
+        # Unicode normalization
+        text = unicodedata.normalize("NFKC", text)
+
+        if not preserve_structure:
+            # Basic cleaning operations
+            # Remove excessive whitespace but preserve paragraph breaks
+            text = re.sub(r"[ \t]+", " ", text)  # Multiple spaces/tabs to single space
+            text = re.sub(r"\n\s*\n\s*\n+", "\n\n", text)  # Multiple newlines to double
+
+            # Remove or normalize special characters
+            # Keep basic punctuation and common symbols
+            text = re.sub(r'[^\w\s.,;:!?\'"\-()[\]{}/@#$%&*+=<>|\\~`\n]', " ", text)
+
+            # Clean up whitespace again
+            text = re.sub(r"[ \t]+", " ", text)
+            text = re.sub(r"\n\s*\n+", "\n\n", text)
+
+        # Remove common artifacts
+        # Remove page numbers and headers/footers patterns
+        text = re.sub(r"\n\s*\d+\s*\n", "\n", text)  # Standalone page numbers
+        text = re.sub(r"\n\s*Page \d+.*?\n", "\n", text, flags=re.IGNORECASE)
+
+        # Remove excessive punctuation
+        text = re.sub(r"[.]{3,}", "...", text)  # Multiple dots
+        text = re.sub(r"[-]{3,}", "---", text)  # Multiple dashes
+
+        # Final cleanup
+        text = text.strip()
+
+        return text
+
+    def _chunk_text(self, text: str) -> List[str]:
+        """
+        Split text into chunks with intelligent boundary detection.
+
+        Args:
+            text: Cleaned text to chunk
+
+        Returns:
+            List of text chunks
+        """
+        if len(text) <= self.chunk_size:
+            return [text]
+
+        chunks = []
+
+        # Try intelligent chunking with spaCy first
+        if self.nlp:
+            try:
+                return self._chunk_with_spacy(text)
+            except Exception as e:
+                self.logger.warning(f"spaCy chunking failed: {str(e)}")
+
+        # Fallback to NLTK sentence-based chunking
+        try:
+            return self._chunk_with_sentences(text)
+        except Exception as e:
+            self.logger.warning(f"Sentence chunking failed: {str(e)}")
+
+        # Final fallback to character-based chunking
+        return self._chunk_by_characters(text)
+
+    def _chunk_with_spacy(self, text: str) -> List[str]:
+        """
+        Intelligent chunking using spaCy for better semantic boundaries.
+
+        Args:
+            text: Text to chunk
+
+        Returns:
+            List of text chunks
+        """
+        doc = self.nlp(text)
+        chunks = []
+        current_chunk = []
+        current_size = 0
+
+        for sent in doc.sents:
+            sent_text = sent.text.strip()
+            sent_size = len(sent_text)
+
+            # 📏 Check if adding this sentence exceeds chunk size
+            if current_size + sent_size > self.chunk_size and current_chunk:
+                # 📦 Finalize current chunk
+                chunk_text = " ".join(current_chunk)
+                chunks.append(chunk_text)
+
+                # Start new chunk with overlap
+                overlap_chunk, overlap_size = self._create_overlap(current_chunk)
+                current_chunk = overlap_chunk
+                current_size = overlap_size
+
+            current_chunk.append(sent_text)
+            current_size += sent_size
+
+        # 📦 Add the last chunk
+        if current_chunk:
+            chunk_text = " ".join(current_chunk)
+            if len(chunk_text.strip()) >= self.min_chunk_size:
+                chunks.append(chunk_text)
+
+        return chunks
+
+    def _chunk_with_sentences(self, text: str) -> List[str]:
+        """
+        Chunk text using NLTK sentence tokenization.
+
+        Args:
+            text: Text to chunk
+
+        Returns:
+            List of text chunks
+        """
+        sentences = sent_tokenize(text)
+        chunks = []
+        current_chunk = []
+        current_size = 0
+
+        for sentence in sentences:
+            sentence = sentence.strip()
+            sentence_size = len(sentence)
+
+            # 📏 Check chunk size limit
+            if current_size + sentence_size > self.chunk_size and current_chunk:
+                # 📦 Finalize current chunk
+                chunk_text = " ".join(current_chunk)
+                chunks.append(chunk_text)
+
+                # Create overlap
+                overlap_chunk, overlap_size = self._create_overlap(current_chunk)
+                current_chunk = overlap_chunk
+                current_size = overlap_size
+
+            current_chunk.append(sentence)
+            current_size += sentence_size
+
+        # 📦 Add final chunk
+        if current_chunk:
+            chunk_text = " ".join(current_chunk)
+            if len(chunk_text.strip()) >= self.min_chunk_size:
+                chunks.append(chunk_text)
+
+        return chunks
+
+    def _chunk_by_characters(self, text: str) -> List[str]:
+        """
+        Fallback character-based chunking with boundary detection.
+
+        Args:
+            text: Text to chunk
+
+        Returns:
+            List of text chunks
+        """
+        chunks = []
+        start = 0
+
+        while start < len(text):
+            end = start + self.chunk_size
+
+            # Try to find a good boundary
+            if end < len(text):
+                # Look for sentence boundaries first
+                for boundary in [". ", "! ", "? ", "\n\n", "\n", ". "]:
+                    boundary_pos = text.rfind(boundary, start, end)
+                    if boundary_pos > start + self.min_chunk_size:
+                        end = boundary_pos + len(boundary)
+                        break
+
+            chunk = text[start:end].strip()
+            if len(chunk) >= self.min_chunk_size:
+                chunks.append(chunk)
+
+            # Move start position with overlap
+            start = max(start + 1, end - self.chunk_overlap)
+
+        return chunks
+
+    def _create_overlap(self, sentences: List[str]) -> tuple:
+        """
+        Create overlap from previous chunk sentences.
+
+        Args:
+            sentences: List of sentences from previous chunk
+
+        Returns:
+            Tuple of (overlap_sentences, overlap_size)
+        """
+        overlap_sentences = []
+        overlap_size = 0
+
+        # Add sentences from the end for overlap
+        for sentence in reversed(sentences):
+            if overlap_size + len(sentence) <= self.chunk_overlap:
+                overlap_sentences.insert(0, sentence)
+                overlap_size += len(sentence)
+            else:
+                break
+
+        return overlap_sentences, overlap_size
+
+    def _analyze_chunk(self, chunk: str) -> Dict[str, Any]:
+        """
+        Analyze chunk statistics and properties.
+
+        Args:
+            chunk: Text chunk to analyze
+
+        Returns:
+            Dictionary with chunk statistics
+        """
+        words = chunk.split()
+
+        stats = {
+            "character_count": len(chunk),
+            "word_count": len(words),
+            "sentence_count": len(sent_tokenize(chunk)) if chunk else 0,
+            "avg_word_length": (
+                sum(len(word) for word in words) / len(words) if words else 0
+            ),
+        }
+
+        # Advanced analysis with spaCy if available
+        if self.nlp:
+            try:
+                doc = self.nlp(chunk)
+                stats.update(
+                    {
+                        "entity_count": len(doc.ents),
+                        "noun_count": len(
+                            [token for token in doc if token.pos_ == "NOUN"]
+                        ),
+                        "verb_count": len(
+                            [token for token in doc if token.pos_ == "VERB"]
+                        ),
+                    }
+                )
+            except Exception:
+                pass  # Skip advanced analysis if it fails
+
+        return stats
+
+    def extract_keywords(self, text: str, max_keywords: int = 10) -> List[str]:
+        """
+        Extract keywords from text using NLP techniques.
+
+        Args:
+            text: Text to extract keywords from
+            max_keywords: Maximum number of keywords to return
+
+        Returns:
+            List of extracted keywords
+        """
+        if not self.nlp:
+            return []
+
+        try:
+            doc = self.nlp(text)
+
+            # Extract keywords based on POS tags and frequency
+            keywords = []
+            word_freq = {}
+
+            for token in doc:
+                if (
+                    token.pos_ in ["NOUN", "PROPN", "ADJ"]
+                    and not token.is_stop
+                    and not token.is_punct
+                    and len(token.text) > 2
+                ):
+
+                    word = token.lemma_.lower()
+                    word_freq[word] = word_freq.get(word, 0) + 1
+
+            # Sort by frequency and return top keywords
+            sorted_words = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)
+            keywords = [word for word, freq in sorted_words[:max_keywords]]
+
+            return keywords
+
+        except Exception as e:
+            self.logger.warning(f"Keyword extraction failed: {str(e)}")
+            return []
+
+    def get_text_statistics(self, text: str) -> Dict[str, Any]:
+        """
+        Get comprehensive text statistics.
+
+        Args:
+            text: Text to analyze
+
+        Returns:
+            Dictionary with text statistics
+        """
+        words = text.split()
+        sentences = sent_tokenize(text) if text else []
+
+        stats = {
+            "character_count": len(text),
+            "word_count": len(words),
+            "sentence_count": len(sentences),
+            "paragraph_count": len([p for p in text.split("\n\n") if p.strip()]),
+            "avg_words_per_sentence": len(words) / len(sentences) if sentences else 0,
+            "avg_chars_per_word": (
+                sum(len(word) for word in words) / len(words) if words else 0
+            ),
+            "language": self._detect_language(text),
+        }
+
+        return stats

src/ingestion/url_processor.py

@@ -0,0 +1,603 @@
+"""
+URL Processor Module
+
+This module is responsible for crawling and extracting content from provided URLs,
+including nested documents and links with complete web scraping functionality.
+
+Technologies: BeautifulSoup, requests, trafilatura
+"""
+
+import logging
+import time
+import re
+from datetime import datetime
+from typing import Dict, List, Any, Optional, Set
+from urllib.parse import urlparse, urljoin, urlunparse
+from urllib.robotparser import RobotFileParser
+
+# Import web scraping libraries
+try:
+    import requests
+    from bs4 import BeautifulSoup
+    import trafilatura
+    from requests.adapters import HTTPAdapter
+    from urllib3.util.retry import Retry
+except ImportError as e:
+    logging.warning(f"Some web scraping libraries are not installed: {e}")
+
+from utils.error_handler import URLProcessingError, error_handler, ErrorType
+
+
+class URLProcessor:
+    """
+    Processes URLs to extract content from web pages and linked documents with full functionality.
+
+    Features:
+    - Web page content extraction with trafilatura
+    - Recursive link following with depth control
+    - Rate limiting and retry logic
+    - Robots.txt respect
+    - Multiple content type handling
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the URLProcessor with configuration.
+
+        Args:
+            config: Configuration dictionary with processing parameters
+        """
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # Configuration settings
+        self.max_depth = self.config.get("max_depth", 1)
+        self.follow_links = self.config.get("follow_links", True)
+        self.max_pages = self.config.get("max_pages", 10)
+        self.timeout = self.config.get("timeout", 10)
+        self.user_agent = self.config.get("user_agent", "RAG-AI-Bot/1.0")
+        self.respect_robots_txt = self.config.get("respect_robots_txt", True)
+        self.rate_limit_delay = self.config.get("rate_limit_delay", 1.0)
+
+        # Retry configuration
+        self.max_retries = 3
+        self.backoff_factor = 0.3
+
+        # Track visited URLs and robots.txt cache
+        self.visited_urls: Set[str] = set()
+        self.robots_cache: Dict[str, RobotFileParser] = {}
+        self.last_request_time: Dict[str, float] = {}
+
+        # Setup session with retry strategy
+        self.session = self._setup_session()
+
+    def _setup_session(self) -> requests.Session:
+        """
+        Setup requests session with retry strategy and headers.
+
+        Returns:
+            Configured requests session
+        """
+        session = requests.Session()
+
+        # Retry strategy
+        retry_strategy = Retry(
+            total=self.max_retries,
+            backoff_factor=self.backoff_factor,
+            status_forcelist=[429, 500, 502, 503, 504],
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
+
+        # 🏷Default headers
+        session.headers.update(
+            {
+                "User-Agent": self.user_agent,
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.5",
+                "Accept-Encoding": "gzip, deflate",
+                "Connection": "keep-alive",
+            }
+        )
+
+        return session
+
+    @error_handler(ErrorType.URL_PROCESSING)
+    def process_url(self, url: str, depth: int = 0) -> Dict[str, Any]:
+        """
+        Process a URL and extract its content with full functionality.
+
+        Args:
+            url: The URL to process
+            depth: Current crawling depth
+
+        Returns:
+            Dictionary containing extracted text and metadata
+        """
+        # Validation checks
+        if not url or not self._is_valid_url(url):
+            raise URLProcessingError(f"Invalid URL: {url}", url)
+
+        if depth > self.max_depth:
+            self.logger.info(f"🛑 Max depth reached for: {url}")
+            return {}
+
+        if len(self.visited_urls) >= self.max_pages:
+            self.logger.info(f"🛑 Max pages limit reached")
+            return {}
+
+        if url in self.visited_urls:
+            self.logger.info(f"Already visited: {url}")
+            return {}
+
+        # Check robots.txt if enabled
+        if self.respect_robots_txt and not self._can_fetch(url):
+            self.logger.info(f"Robots.txt disallows: {url}")
+            return {}
+
+        self.visited_urls.add(url)
+        self.logger.info(f"Processing URL: {url} (depth: {depth})")
+
+        try:
+            # Rate limiting
+            self._apply_rate_limit(url)
+
+            # Fetch and extract content
+            content, metadata = self._extract_content(url)
+
+            if not content:
+                self.logger.warning(f"No content extracted from: {url}")
+                return {}
+
+            result = {
+                "content": content,
+                "metadata": metadata,
+                "source": url,
+                "depth": depth,
+                "linked_documents": [],
+                "document_type": "webpage",
+                "crawl_stats": {
+                    "max_depth_configured": self.max_depth,
+                    "follow_links_enabled": self.follow_links,
+                    "current_depth": depth,
+                },
+            }
+
+            #  Follow links if configured and not at max depth
+            if (
+                self.follow_links
+                and depth < self.max_depth
+                and len(self.visited_urls) < self.max_pages
+            ):
+                links = self._extract_links(url, content)
+                self.logger.info(f" Found {len(links)} links on {url}")
+
+                for link in links[:5]:  # Limit links per page
+                    try:
+                        linked_content = self.process_url(link, depth + 1)
+                        if linked_content:
+                            result["linked_documents"].append(linked_content)
+                    except Exception as e:
+                        self.logger.warning(
+                            f"Failed to process linked URL {link}: {str(e)}"
+                        )
+                        continue
+
+            return result
+
+        except Exception as e:
+            raise URLProcessingError(f"Error processing URL: {str(e)}", url)
+
+    def process_batch(self, urls: List[str]) -> List[Dict[str, Any]]:
+        """
+        Process multiple URLs in batch.
+
+        Args:
+            urls: List of URLs to process
+
+        Returns:
+            List of processed URL results
+        """
+        results = []
+        self.logger.info(f"Processing batch of {len(urls)} URLs")
+
+        for i, url in enumerate(urls):
+            try:
+                result = self.process_url(url)
+                if result:
+                    results.append(result)
+                self.logger.info(f"Processed {i+1}/{len(urls)}: {url}")
+            except Exception as e:
+                self.logger.error(f"❌ Failed to process {url}: {str(e)}")
+                continue
+
+        return results
+
+    def _is_valid_url(self, url: str) -> bool:
+        """
+        Validate URL format and scheme.
+
+        Args:
+            url: URL to validate
+
+        Returns:
+            True if URL is valid
+        """
+        try:
+            parsed = urlparse(url)
+            return bool(parsed.netloc) and parsed.scheme in ["http", "https"]
+        except Exception:
+            return False
+
+    def _can_fetch(self, url: str) -> bool:
+        """
+        Check if URL can be fetched according to robots.txt.
+
+        Args:
+            url: URL to check
+
+        Returns:
+            True if URL can be fetched
+        """
+        try:
+            parsed_url = urlparse(url)
+            base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
+
+            if base_url not in self.robots_cache:
+                robots_url = urljoin(base_url, "/robots.txt")
+                rp = RobotFileParser()
+                rp.set_url(robots_url)
+
+                try:
+                    rp.read()
+                    self.robots_cache[base_url] = rp
+                except Exception:
+                    # If robots.txt can't be fetched, assume allowed
+                    return True
+
+            return self.robots_cache[base_url].can_fetch(self.user_agent, url)
+
+        except Exception:
+            # If robots.txt check fails, assume allowed
+            return True
+
+    def _apply_rate_limit(self, url: str) -> None:
+        """
+        Apply rate limiting between requests to the same domain.
+
+        Args:
+            url: URL being processed
+        """
+        domain = urlparse(url).netloc
+        current_time = time.time()
+
+        if domain in self.last_request_time:
+            time_since_last = current_time - self.last_request_time[domain]
+            if time_since_last < self.rate_limit_delay:
+                sleep_time = self.rate_limit_delay - time_since_last
+                self.logger.info(
+                    f"Rate limiting: sleeping {sleep_time:.1f}s for {domain}"
+                )
+                time.sleep(sleep_time)
+
+        self.last_request_time[domain] = time.time()
+
+    def _extract_content(self, url: str) -> tuple:
+        """
+        Extract content from a web page using trafilatura with BeautifulSoup fallback.
+
+        Args:
+            url: The URL to extract content from
+
+        Returns:
+            Tuple of (content, metadata)
+        """
+        self.logger.info(f"Extracting content from: {url}")
+
+        try:
+            # Fetch the page
+            response = self.session.get(url, timeout=self.timeout)
+            response.raise_for_status()
+
+            # Basic metadata
+            metadata = {
+                "url": url,
+                "status_code": response.status_code,
+                "content_type": response.headers.get("content-type", ""),
+                "content_length": len(response.content),
+                "extracted_time": datetime.now().isoformat(),
+                "encoding": response.encoding or "utf-8",
+            }
+
+            # Check content type
+            content_type = response.headers.get("content-type", "").lower()
+
+            if "application/pdf" in content_type:
+                return self._handle_pdf_url(response, metadata)
+            elif "text/html" not in content_type and "text/plain" not in content_type:
+                self.logger.warning(f"Unsupported content type: {content_type}")
+                return "", metadata
+
+            # Primary method: trafilatura (best for content extraction)
+            try:
+                content = trafilatura.extract(
+                    response.text,
+                    include_comments=False,
+                    include_tables=True,
+                    include_formatting=False,
+                    favor_precision=True,
+                )
+
+                if content and len(content.strip()) > 50:  # Minimum content threshold
+                    # Extract additional metadata with trafilatura
+                    metadata_extracted = trafilatura.extract_metadata(response.text)
+                    if metadata_extracted:
+                        metadata.update(
+                            {
+                                "title": metadata_extracted.title or "",
+                                "author": metadata_extracted.author or "",
+                                "description": metadata_extracted.description or "",
+                                "sitename": metadata_extracted.sitename or "",
+                                "date": metadata_extracted.date or "",
+                            }
+                        )
+
+                    metadata.update(
+                        {
+                            "extraction_method": "trafilatura",
+                            "word_count": len(content.split()),
+                            "character_count": len(content),
+                        }
+                    )
+
+                    return content.strip(), metadata
+
+            except Exception as e:
+                self.logger.warning(f"Trafilatura failed: {str(e)}")
+
+            # Fallback method: BeautifulSoup
+            return self._extract_with_beautifulsoup(response.text, metadata)
+
+        except requests.RequestException as e:
+            raise URLProcessingError(f"Failed to fetch URL: {str(e)}", url)
+        except Exception as e:
+            raise URLProcessingError(f"Content extraction failed: {str(e)}", url)
+
+    def _extract_with_beautifulsoup(self, html: str, metadata: Dict[str, Any]) -> tuple:
+        """
+        Fallback content extraction using BeautifulSoup.
+
+        Args:
+            html: HTML content
+            metadata: Existing metadata dictionary
+
+        Returns:
+            Tuple of (content, metadata)
+        """
+        try:
+            soup = BeautifulSoup(html, "html.parser")
+
+            # Extract metadata
+            title_tag = soup.find("title")
+            if title_tag:
+                metadata["title"] = title_tag.get_text().strip()
+
+            # Meta tags
+            for meta in soup.find_all("meta"):
+                name = meta.get("name", "").lower()
+                content = meta.get("content", "")
+                if name == "description":
+                    metadata["description"] = content
+                elif name == "author":
+                    metadata["author"] = content
+
+            # Remove unwanted elements
+            for element in soup(
+                ["script", "style", "nav", "header", "footer", "aside"]
+            ):
+                element.decompose()
+
+            # Extract main content
+            content_selectors = [
+                "main",
+                "article",
+                ".content",
+                "#content",
+                ".post",
+                ".entry",
+            ]
+
+            content = ""
+            for selector in content_selectors:
+                content_elem = soup.select_one(selector)
+                if content_elem:
+                    content = content_elem.get_text(separator="\n", strip=True)
+                    break
+
+            # Fallback to body if no main content found
+            if not content:
+                body = soup.find("body")
+                if body:
+                    content = body.get_text(separator="\n", strip=True)
+
+            # Clean and validate content
+            content = re.sub(r"\n\s*\n", "\n\n", content)  # Clean multiple newlines
+            content = content.strip()
+
+            metadata.update(
+                {
+                    "extraction_method": "beautifulsoup",
+                    "word_count": len(content.split()),
+                    "character_count": len(content),
+                }
+            )
+
+            return content, metadata
+
+        except Exception as e:
+            self.logger.error(f"❌ BeautifulSoup extraction failed: {str(e)}")
+            return "", metadata
+
+    def _handle_pdf_url(
+        self, response: requests.Response, metadata: Dict[str, Any]
+    ) -> tuple:
+        """
+        📄 Handle PDF content from URL.
+
+        Args:
+            response: HTTP response containing PDF
+            metadata: Existing metadata
+
+        Returns:
+            Tuple of (content, metadata)
+        """
+        self.logger.info("📄 Detected PDF content, extracting text...")
+
+        try:
+            # Save PDF temporarily and process with document processor
+            import tempfile
+            import os
+            from .document_processor import DocumentProcessor
+
+            with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp_file:
+                tmp_file.write(response.content)
+                tmp_file.flush()
+
+                # Process PDF
+                doc_processor = DocumentProcessor(self.config)
+                result = doc_processor.process_document(tmp_file.name)
+
+                # Cleanup
+                os.unlink(tmp_file.name)
+
+                metadata.update(
+                    {
+                        "document_type": "pdf_from_url",
+                        "extraction_method": "document_processor",
+                    }
+                )
+                metadata.update(result.get("metadata", {}))
+
+                return result.get("content", ""), metadata
+
+        except Exception as e:
+            self.logger.error(f"❌ PDF extraction failed: {str(e)}")
+            return "", metadata
+
+    def _extract_links(self, url: str, content: str) -> List[str]:
+        """
+         Extract links from a web page.
+
+        Args:
+            url: The source URL
+            content: Page content (for context)
+
+        Returns:
+            List of discovered URLs
+        """
+        self.logger.info(f" Extracting links from: {url}")
+
+        try:
+            response = self.session.get(url, timeout=self.timeout)
+            soup = BeautifulSoup(response.text, "html.parser")
+
+            links = []
+            base_domain = urlparse(url).netloc
+
+            for a_tag in soup.find_all("a", href=True):
+                href = a_tag.get("href")
+                if not href:
+                    continue
+
+                #  Convert relative URLs to absolute
+                absolute_url = urljoin(url, href)
+
+                # Filter links
+                if self._should_follow_link(absolute_url, base_domain):
+                    links.append(absolute_url)
+
+            # 🎯 Remove duplicates and limit
+            unique_links = list(dict.fromkeys(links))  # Preserve order
+            return unique_links[:20]  # Limit to prevent explosion
+
+        except Exception as e:
+            self.logger.error(f"❌ Link extraction failed: {str(e)}")
+            return []
+
+    def _should_follow_link(self, url: str, base_domain: str) -> bool:
+        """
+        Determine if a link should be followed.
+
+        Args:
+            url: URL to check
+            base_domain: Base domain of the source page
+
+        Returns:
+            True if link should be followed
+        """
+        try:
+            parsed = urlparse(url)
+
+            # Skip non-HTTP(S) links
+            if parsed.scheme not in ["http", "https"]:
+                return False
+
+            # Skip already visited
+            if url in self.visited_urls:
+                return False
+
+            # Skip file downloads (basic check)
+            path = parsed.path.lower()
+            skip_extensions = [
+                ".pdf",
+                ".doc",
+                ".docx",
+                ".zip",
+                ".exe",
+                ".dmg",
+                ".jpg",
+                ".png",
+                ".gif",
+            ]
+            if any(path.endswith(ext) for ext in skip_extensions):
+                return False
+
+            # Skip fragments and query-heavy URLs
+            if parsed.fragment or len(parsed.query) > 100:
+                return False
+
+            # Prefer same domain (but allow subdomains)
+            link_domain = parsed.netloc
+            if not (
+                link_domain == base_domain or link_domain.endswith("." + base_domain)
+            ):
+                return False
+
+            return True
+
+        except Exception:
+            return False
+
+    def reset(self):
+        """Reset the processor state, clearing visited URLs and caches."""
+        self.visited_urls.clear()
+        self.robots_cache.clear()
+        self.last_request_time.clear()
+        self.logger.info("URL processor state reset")
+
+    def get_statistics(self) -> Dict[str, Any]:
+        """
+        Get processing statistics.
+
+        Returns:
+            Dictionary with processing statistics
+        """
+        return {
+            "urls_processed": len(self.visited_urls),
+            "domains_cached": len(self.robots_cache),
+            "rate_limited_domains": len(self.last_request_time),
+            "max_pages_limit": self.max_pages,
+            "max_depth_limit": self.max_depth,
+        }

src/integrations/__init__.py

@@ -0,0 +1,13 @@
+"""
+Integrations Module
+
+This module contains integrations with external services and APIs
+for enhanced RAG functionality.
+
+Available Integrations:
+- MCP Tavily: Live web search via Model Context Protocol
+"""
+
+from .mcp_tavily_integration import MCPTavilyIntegration, create_mcp_tavily_client
+
+__all__ = ["MCPTavilyIntegration", "create_mcp_tavily_client"]

src/integrations/mcp_tavily_integration.py

@@ -0,0 +1,308 @@
+"""
+MCP Tavily Integration Module
+
+This module demonstrates how to integrate Tavily API via MCP (Model Context Protocol)
+for live web search functionality in the RAG system.
+
+Technology: MCP + Tavily API
+"""
+
+import logging
+import time
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+
+
+class MCPTavilyIntegration:
+    """
+    Handles MCP integration with Tavily API for live web search.
+
+    This class provides the bridge between the RAG system and Tavily's
+    search capabilities through the Model Context Protocol.
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize MCP Tavily integration.
+
+        Args:
+            config: Configuration dictionary
+        """
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # 🔧 MCP Configuration
+        self.server_name = self.config.get("mcp_server_name", "tavily-mcp")
+        self.tool_name = self.config.get("mcp_tool_name", "tavily-search")
+        self.timeout = self.config.get("timeout", 30)
+
+        self.logger.info(" MCP Tavily Integration initialized")
+
+    def search_web(
+        self,
+        query: str,
+        max_results: int = 5,
+        search_depth: str = "basic",
+        time_range: str = "month",
+        topic: str = "general",
+    ) -> Dict[str, Any]:
+        """
+        Perform web search using Tavily API via MCP.
+
+        Args:
+            query: Search query
+            max_results: Maximum number of results
+            search_depth: Search depth (basic/advanced)
+            time_range: Time range for results
+            topic: Search topic category
+
+        Returns:
+            Dictionary with search results
+        """
+        try:
+            self.logger.info(f" MCP Tavily search: '{query}' (depth: {search_depth})")
+
+            # 🚀 Prepare MCP arguments
+            mcp_arguments = {
+                "query": query,
+                "max_results": min(max_results, 20),  # Tavily limit
+                "search_depth": search_depth,
+                "topic": topic,
+                "include_raw_content": True,
+                "time_range": time_range,
+            }
+
+            # 🌐 This is where the actual MCP call would be made
+            # In a real implementation, this would use the MCP client:
+
+            """
+            Example MCP call structure:
+            
+            result = use_mcp_tool(
+                server_name=self.server_name,
+                tool_name=self.tool_name,
+                arguments=mcp_arguments
+            )
+            """
+
+            # 🚧 For demonstration, we'll simulate the MCP response structure
+            simulated_result = self._simulate_tavily_response(query, max_results)
+
+            # 🔄 Process and validate MCP response
+            processed_result = self._process_mcp_response(simulated_result, query)
+
+            self.logger.info(
+                f" MCP search completed: {processed_result.get('total_results', 0)} results"
+            )
+            return processed_result
+
+        except Exception as e:
+            self.logger.error(f" MCP Tavily search failed: {str(e)}")
+            return {
+                "query": query,
+                "results": [],
+                "total_results": 0,
+                "error": str(e),
+                "status": "mcp_error",
+            }
+
+    def _simulate_tavily_response(self, query: str, max_results: int) -> Dict[str, Any]:
+        """
+        Simulate Tavily API response for demonstration.
+
+        In production, this would be replaced by actual MCP call results.
+        """
+        # 🚧 Simulated response structure matching Tavily API
+        return {
+            "query": query,
+            "follow_up_questions": None,
+            "answer": f"Based on web search for '{query}'...",
+            "images": [],
+            "results": [
+                {
+                    "title": f"Example Result 1 for {query}",
+                    "url": "https://example.com/result1",
+                    "content": f"This is example content related to {query}. It provides comprehensive information about the topic.",
+                    "raw_content": f"Raw content for {query} with additional details...",
+                    "published_date": "2024-01-15",
+                    "score": 0.95,
+                },
+                {
+                    "title": f"Example Result 2 for {query}",
+                    "url": "https://example.com/result2",
+                    "content": f"Another relevant result for {query} with different perspective and insights.",
+                    "raw_content": f"Extended raw content for {query}...",
+                    "published_date": "2024-01-14",
+                    "score": 0.87,
+                },
+            ][:max_results],
+            "response_time": 1.2,
+        }
+
+    def _process_mcp_response(
+        self, mcp_result: Dict[str, Any], original_query: str
+    ) -> Dict[str, Any]:
+        """
+        Process and validate MCP response from Tavily.
+
+        Args:
+            mcp_result: Raw MCP response
+            original_query: Original search query
+
+        Returns:
+            Processed search results
+        """
+        try:
+            # 🔍 Extract results from MCP response
+            raw_results = mcp_result.get("results", [])
+
+            # 🔄 Process each result
+            processed_results = []
+            for i, result in enumerate(raw_results):
+                processed_result = {
+                    "title": result.get("title", f"Web Result {i+1}"),
+                    "url": result.get("url", ""),
+                    "content": result.get("content", ""),
+                    "raw_content": result.get("raw_content", ""),
+                    "score": result.get("score", 0.0),
+                    "published_date": result.get("published_date", ""),
+                    "rank": i + 1,
+                    "source": "tavily_web_search",
+                    "search_engine": "tavily",
+                    "metadata": {
+                        "title": result.get("title", ""),
+                        "url": result.get("url", ""),
+                        "content_length": len(result.get("content", "")),
+                        "has_raw_content": bool(result.get("raw_content")),
+                        "search_rank": i + 1,
+                        "published_date": result.get("published_date", ""),
+                    },
+                }
+                processed_results.append(processed_result)
+
+            # 📊 Prepare final response
+            return {
+                "query": original_query,
+                "results": processed_results,
+                "total_results": len(processed_results),
+                "answer": mcp_result.get("answer", ""),
+                "follow_up_questions": mcp_result.get("follow_up_questions", []),
+                "response_time": mcp_result.get("response_time", 0),
+                "timestamp": datetime.now(),
+                "status": "success",
+                "source": "mcp_tavily",
+            }
+
+        except Exception as e:
+            self.logger.error(f" Error processing MCP response: {str(e)}")
+            return {
+                "query": original_query,
+                "results": [],
+                "total_results": 0,
+                "error": f"Response processing failed: {str(e)}",
+                "status": "processing_error",
+            }
+
+    def test_connection(self) -> Dict[str, Any]:
+        """
+        Test MCP connection to Tavily.
+
+        Returns:
+            Connection test results
+        """
+        try:
+            self.logger.info(" Testing MCP Tavily connection...")
+
+            # 🔍 Simple test query
+            test_result = self.search_web(
+                query="test connection", max_results=1, search_depth="basic"
+            )
+
+            if test_result.get("status") == "success":
+                return {
+                    "status": "success",
+                    "message": " MCP Tavily connection successful",
+                    "server_name": self.server_name,
+                    "tool_name": self.tool_name,
+                    "response_time": test_result.get("response_time", 0),
+                }
+            else:
+                return {
+                    "status": "error",
+                    "message": " MCP Tavily connection failed",
+                    "error": test_result.get("error", "Unknown error"),
+                }
+
+        except Exception as e:
+            self.logger.error(f" MCP connection test failed: {str(e)}")
+            return {
+                "status": "error",
+                "message": " MCP connection test failed",
+                "error": str(e),
+            }
+
+    def get_server_info(self) -> Dict[str, Any]:
+        """
+        Get MCP server information.
+
+        Returns:
+            Server information dictionary
+        """
+        return {
+            "server_name": self.server_name,
+            "tool_name": self.tool_name,
+            "timeout": self.timeout,
+            "status": "configured",
+            "description": "MCP integration for Tavily web search API",
+        }
+
+
+# 🔧 Helper function for easy integration
+def create_mcp_tavily_client(
+    config: Optional[Dict[str, Any]] = None,
+) -> MCPTavilyIntegration:
+    """
+    Create and configure MCP Tavily client.
+
+    Args:
+        config: Optional configuration dictionary
+
+    Returns:
+        Configured MCPTavilyIntegration instance
+    """
+    return MCPTavilyIntegration(config)
+
+
+# 📝 Example usage and integration guide
+if __name__ == "__main__":
+    """
+    Example usage of MCP Tavily Integration
+
+    This demonstrates how to use the MCP integration in your RAG system.
+    """
+
+    # 🔧 Configure MCP client
+    config = {
+        "mcp_server_name": "tavily-mcp",
+        "mcp_tool_name": "tavily-search",
+        "timeout": 30,
+    }
+
+    # 🚀 Create client
+    mcp_client = create_mcp_tavily_client(config)
+
+    # 🧪 Test connection
+    connection_test = mcp_client.test_connection()
+    print(f"Connection test: {connection_test}")
+
+    # 🔍 Example search
+    search_result = mcp_client.search_web(
+        query="latest AI developments 2024",
+        max_results=5,
+        search_depth="basic",
+        time_range="month",
+    )
+
+    print(f"Search results: {search_result.get('total_results', 0)} found")
+    for result in search_result.get("results", []):
+        print(f"- {result['title']}: {result['url']}")

src/rag/__init__.py

@@ -0,0 +1,6 @@
+"""
+RAG (Retrieval Augmented Generation) module.
+
+This module contains components for processing queries and
+generating responses using retrieved knowledge.
+"""

src/rag/live_search.py

@@ -0,0 +1,523 @@
+"""
+Live Search Processor using Tavily Python Client.
+Provides real-time web search capabilities for the RAG system.
+"""
+
+import logging
+import os
+import time
+from typing import Dict, List, Any, Optional
+from datetime import datetime, timedelta
+
+logger = logging.getLogger(__name__)
+
+
+class LiveSearchProcessor:
+    """Handles live web search using Tavily Python Client."""
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the LiveSearchProcessor.
+
+        Args:
+            config: Configuration dictionary containing live search settings
+        """
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # Search configuration
+        self.enabled = self.config.get("enabled", False)
+        self.max_results = self.config.get("max_results", 5)
+        self.search_depth = self.config.get("search_depth", "basic")
+        self.include_answer = self.config.get("include_answer", True)
+        self.include_raw_content = self.config.get("include_raw_content", False)
+        self.include_images = self.config.get("include_images", False)
+        self.topic = self.config.get("topic", "general")
+        self.enable_caching = self.config.get("enable_caching", True)
+
+        # Search cache and analytics
+        self.search_cache = {}
+        self.search_history = []
+
+        # Initialize Tavily client
+        self.tavily_client = None
+        self._initialize_client()
+
+        self.logger.info(f"LiveSearchProcessor initialized - Enabled: {self.enabled}")
+
+    def _initialize_client(self):
+        """Initialize the Tavily client."""
+        try:
+            # Get API key from environment variable
+            api_key = os.getenv("TAVILY_API_KEY")
+
+            if not api_key:
+                self.logger.warning("TAVILY_API_KEY not found in environment variables")
+                self.enabled = False
+                return
+
+            # Import and initialize Tavily client
+            from tavily import TavilyClient
+
+            self.tavily_client = TavilyClient(api_key=api_key)
+
+            # ✅ Auto-enable if client initializes successfully and no explicit config
+            if self.tavily_client and not self.config.get(
+                "enabled_explicitly_set", False
+            ):
+                self.enabled = True
+                self.logger.info(
+                    "Tavily client initialized successfully - Auto-enabled live search"
+                )
+            else:
+                self.logger.info("Tavily client initialized successfully")
+
+        except ImportError:
+            self.logger.error(
+                "tavily-python package not installed. Install with: pip install tavily-python"
+            )
+            self.enabled = False
+        except Exception as e:
+            self.logger.error(f"Failed to initialize Tavily client: {str(e)}")
+            self.enabled = False
+
+    def is_enabled(self) -> bool:
+        """Check if live search is enabled."""
+        return self.enabled and self.tavily_client is not None
+
+    def search_web(
+        self,
+        query: str,
+        max_results: Optional[int] = None,
+        search_depth: Optional[str] = None,
+        time_range: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Perform live web search using Tavily API.
+
+        Args:
+            query: Search query string
+            max_results: Maximum number of results to return
+            search_depth: Search depth ('basic' or 'advanced')
+            time_range: Time range for search results
+
+        Returns:
+            Dictionary containing search results and metadata
+        """
+        if not query or not query.strip():
+            return {
+                "query": query,
+                "results": [],
+                "total_results": 0,
+                "error": "Empty query provided",
+                "source": "live_search",
+            }
+
+        if not self.is_enabled():
+            self.logger.warning("Live search is disabled or client not initialized")
+            return {
+                "query": query,
+                "results": [],
+                "total_results": 0,
+                "error": "Live search is disabled or Tavily client not initialized",
+                "source": "live_search",
+            }
+
+        self.logger.info(f"Performing live search: {query[:100]}...")
+        start_time = time.time()
+
+        try:
+            # Use provided parameters or defaults
+            search_max_results = max_results or self.max_results
+            search_depth_param = search_depth or self.search_depth
+
+            # Check cache first
+            cache_key = self._generate_cache_key(
+                query, search_max_results, search_depth_param
+            )
+            if self.enable_caching and cache_key in self.search_cache:
+                cached_result = self.search_cache[cache_key]
+                if self._is_cache_valid(cached_result["timestamp"]):
+                    self.logger.info("Returning cached search result")
+                    cached_result["from_cache"] = True
+                    return cached_result
+
+            # Prepare search parameters
+            search_params = {
+                "query": query,
+                "max_results": min(search_max_results, 20),  # Tavily limit
+                "search_depth": search_depth_param,
+                "include_answer": self.include_answer,
+                "include_raw_content": self.include_raw_content,
+                "include_images": self.include_images,
+                "topic": self.topic,
+            }
+
+            # Add time_range if provided
+            if time_range:
+                search_params["time_range"] = time_range
+
+            # Perform the search
+            response = self.tavily_client.search(**search_params)
+
+            # Process and format results
+            processed_results = self._process_search_results(
+                response.get("results", []), query
+            )
+
+            # Prepare final result
+            result = {
+                "query": query,
+                "results": processed_results,
+                "total_results": len(processed_results),
+                "answer": response.get("answer"),
+                "images": response.get("images", []),
+                "follow_up_questions": response.get("follow_up_questions", []),
+                "search_params": {
+                    "max_results": search_max_results,
+                    "search_depth": search_depth_param,
+                    "time_range": time_range,
+                },
+                "processing_time": time.time() - start_time,
+                "timestamp": datetime.now(),
+                "source": "live_search",
+                "from_cache": False,
+                "search_metadata": {
+                    "source": "tavily",
+                    "timestamp": datetime.now().isoformat(),
+                    "results_count": len(processed_results),
+                    "search_depth": search_depth_param,
+                    "max_results": search_max_results,
+                    "response_time": response.get("response_time"),
+                },
+            }
+
+            # Cache the result
+            if self.enable_caching:
+                self.search_cache[cache_key] = result.copy()
+
+            # Add to search history
+            self._add_to_history(query, len(processed_results))
+
+            self.logger.info(
+                f"Live search completed in {result['processing_time']:.2f}s"
+            )
+            return result
+
+        except Exception as e:
+            self.logger.error(f"Error in live search: {str(e)}")
+            return {
+                "query": query,
+                "results": [],
+                "total_results": 0,
+                "error": str(e),
+                "processing_time": time.time() - start_time,
+                "source": "live_search",
+            }
+
+    def search(self, query: str, **kwargs) -> Dict[str, Any]:
+        """
+        Perform a live web search using Tavily API.
+
+        Args:
+            query: Search query string
+            **kwargs: Additional search parameters
+
+        Returns:
+            Dictionary containing search results
+        """
+        return self.search_web(query, **kwargs)
+
+    def _process_search_results(
+        self, raw_results: List[Dict[str, Any]], query: str
+    ) -> List[Dict[str, Any]]:
+        """
+        Process and format raw search results from Tavily.
+
+        Args:
+            raw_results: Raw results from Tavily API
+            query: Original search query
+
+        Returns:
+            Processed and formatted results
+        """
+        processed_results = []
+        query_words = set(query.lower().split())
+
+        for i, result in enumerate(raw_results):
+            try:
+                # Extract key information
+                title = result.get("title", "")
+                url = result.get("url", "")
+                content = result.get("content", "")
+                raw_content = result.get("raw_content", "")
+                score = result.get("score", 0.0)
+
+                # Calculate relevance score
+                relevance_score = self._calculate_relevance_score(
+                    title, content, query_words, score
+                )
+
+                # Format result
+                formatted_result = {
+                    "title": title,
+                    "url": url,
+                    "content": content[:500] + "..." if len(content) > 500 else content,
+                    "raw_content": raw_content if self.include_raw_content else "",
+                    "score": score,
+                    "relevance_score": relevance_score,
+                    "rank": i + 1,
+                    "source": "web_search",
+                    "search_engine": "tavily",
+                    "published_date": result.get("published_date"),
+                    "metadata": {
+                        "title": title,
+                        "url": url,
+                        "content_length": len(content),
+                        "has_raw_content": bool(raw_content),
+                        "search_rank": i + 1,
+                    },
+                }
+
+                processed_results.append(formatted_result)
+
+            except Exception as e:
+                self.logger.warning(f"Error processing search result {i}: {str(e)}")
+                continue
+
+        # Sort by relevance score
+        processed_results.sort(key=lambda x: x["relevance_score"], reverse=True)
+
+        return processed_results
+
+    def _calculate_relevance_score(
+        self, title: str, content: str, query_words: set, base_score: float
+    ) -> float:
+        """
+        Calculate relevance score for search results.
+
+        Args:
+            title: Result title
+            content: Result content
+            query_words: Set of query words
+            base_score: Base score from search engine
+
+        Returns:
+            Calculated relevance score
+        """
+        try:
+            # Start with base score
+            relevance = base_score
+
+            # Title relevance (higher weight)
+            title_words = set(title.lower().split())
+            title_overlap = len(query_words.intersection(title_words))
+            title_boost = (title_overlap / max(len(query_words), 1)) * 0.3
+
+            # Content relevance
+            content_words = set(content.lower().split())
+            content_overlap = len(query_words.intersection(content_words))
+            content_boost = (content_overlap / max(len(query_words), 1)) * 0.2
+
+            # Exact phrase matching bonus
+            query_phrase = " ".join(query_words).lower()
+            if query_phrase in title.lower():
+                relevance += 0.2
+            elif query_phrase in content.lower():
+                relevance += 0.1
+
+            # Final score calculation
+            final_score = min(relevance + title_boost + content_boost, 1.0)
+
+            return round(final_score, 3)
+
+        except Exception as e:
+            self.logger.warning(f"Error calculating relevance score: {str(e)}")
+            return base_score
+
+    def get_search_context(self, query: str, **kwargs) -> str:
+        """
+        Get search context suitable for RAG applications.
+
+        Args:
+            query: Search query string
+            **kwargs: Additional search parameters
+
+        Returns:
+            Formatted context string
+        """
+        search_results = self.search(query, **kwargs)
+
+        if not search_results.get("results"):
+            error_msg = search_results.get("error", "Unknown error")
+            return f"No live search results found for: {query}. Error: {error_msg}"
+
+        context_parts = []
+
+        # Add answer if available
+        if search_results.get("answer"):
+            context_parts.append(f"Answer: {search_results['answer']}")
+            context_parts.append("")
+
+        # Add search results
+        context_parts.append("Search Results:")
+        for i, result in enumerate(search_results["results"], 1):
+            context_parts.append(f"{i}. {result['title']}")
+            context_parts.append(f"   URL: {result['url']}")
+            context_parts.append(f"   Content: {result['content']}")
+            if result.get("published_date"):
+                context_parts.append(f"   Published: {result['published_date']}")
+            context_parts.append("")
+
+        # Add metadata
+        metadata = search_results.get("search_metadata", {})
+        context_parts.append(
+            f"Search performed at: {metadata.get('timestamp', 'Unknown')}"
+        )
+        context_parts.append(f"Source: {metadata.get('source', 'Unknown')}")
+        context_parts.append(f"Results count: {metadata.get('results_count', 0)}")
+
+        return "\n".join(context_parts)
+
+    def qna_search(self, query: str, **kwargs) -> str:
+        """
+        Get a quick answer to a question using Tavily's QnA search.
+
+        Args:
+            query: Question to answer
+            **kwargs: Additional search parameters
+
+        Returns:
+            Answer string
+        """
+        if not self.is_enabled():
+            return "Live search is disabled or not properly configured."
+
+        try:
+            # Use Tavily's QnA search method
+            answer = self.tavily_client.qna_search(query=query)
+            return answer if answer else "No answer found for the given question."
+
+        except Exception as e:
+            self.logger.error(f"Error in QnA search: {str(e)}")
+            return f"Error getting answer: {str(e)}"
+
+    def _generate_cache_key(
+        self, query: str, max_results: int, search_depth: str
+    ) -> str:
+        """Generate cache key for search results."""
+        import hashlib
+
+        cache_string = f"{query.lower().strip()}{max_results}{search_depth}"
+        return hashlib.md5(cache_string.encode()).hexdigest()
+
+    def _is_cache_valid(self, timestamp: datetime) -> bool:
+        """Check if cached result is still valid (30 minutes for live search)."""
+        return datetime.now() - timestamp < timedelta(minutes=30)
+
+    def _add_to_history(self, query: str, result_count: int):
+        """Add search to history for analytics."""
+        self.search_history.append(
+            {
+                "query": query,
+                "timestamp": datetime.now(),
+                "result_count": result_count,
+                "search_type": "live_web",
+            }
+        )
+
+        # Keep only last 50 searches
+        if len(self.search_history) > 50:
+            self.search_history = self.search_history[-50:]
+
+    def health_check(self) -> Dict[str, Any]:
+        """
+        Perform a health check of the live search service.
+
+        Returns:
+            Dictionary containing health status
+        """
+        try:
+            if not self.enabled:
+                return {
+                    "status": "disabled",
+                    "message": "Live search is disabled in configuration",
+                    "timestamp": datetime.now().isoformat(),
+                }
+
+            if not self.tavily_client:
+                return {
+                    "status": "error",
+                    "message": "Tavily client not initialized. Check TAVILY_API_KEY environment variable.",
+                    "timestamp": datetime.now().isoformat(),
+                }
+
+            # Perform a simple test search
+            test_result = self.search("test health check", max_results=1)
+
+            if test_result.get("error"):
+                return {
+                    "status": "error",
+                    "message": f"Health check failed: {test_result['error']}",
+                    "timestamp": datetime.now().isoformat(),
+                }
+
+            return {
+                "status": "healthy",
+                "message": "Live search service is operational",
+                "timestamp": datetime.now().isoformat(),
+                "config": {
+                    "max_results": self.max_results,
+                    "search_depth": self.search_depth,
+                    "include_answer": self.include_answer,
+                    "topic": self.topic,
+                },
+            }
+
+        except Exception as e:
+            self.logger.error(f"Health check failed: {str(e)}")
+            return {
+                "status": "error",
+                "message": f"Health check failed: {str(e)}",
+                "timestamp": datetime.now().isoformat(),
+            }
+
+    def get_search_analytics(self) -> Dict[str, Any]:
+        """
+        Get analytics about search patterns.
+
+        Returns:
+            Dictionary with search analytics
+        """
+        if not self.search_history:
+            return {"total_searches": 0, "cache_hit_rate": 0.0, "average_results": 0.0}
+
+        total_searches = len(self.search_history)
+        avg_results = (
+            sum(s["result_count"] for s in self.search_history) / total_searches
+        )
+
+        # Recent search trends
+        recent_searches = [s["query"] for s in self.search_history[-10:]]
+
+        return {
+            "total_searches": total_searches,
+            "average_results_per_search": round(avg_results, 2),
+            "recent_searches": recent_searches,
+            "cache_size": len(self.search_cache),
+            "search_type": "live_web",
+        }
+
+    def clear_cache(self):
+        """Clear the search cache."""
+        self.search_cache.clear()
+        self.logger.info("Live search cache cleared")
+
+    def clear_history(self):
+        """Clear the search history."""
+        self.search_history.clear()
+        self.logger.info("Live search history cleared")
+
+
+# 🔄 Compatibility alias for existing imports
+LiveSearchManager = LiveSearchProcessor

src/rag/optimized_query_processor.py

@@ -0,0 +1,275 @@
+"""
+Optimized Query Processor with Rate Limiting and Better Error Handling
+"""
+
+import logging
+import time
+from typing import Dict, List, Any, Optional
+from datetime import datetime, timedelta
+
+
+class OptimizedQueryProcessor:
+    """
+    Optimized QueryProcessor with rate limiting and better error handling
+    """
+
+    def __init__(
+        self, embedding_generator, vector_db, config: Optional[Dict[str, Any]] = None
+    ):
+        self.embedding_generator = embedding_generator
+        self.vector_db = vector_db
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # Optimized configuration settings
+        self.top_k = self.config.get("top_k", 10)  # Increased from 5
+        self.similarity_threshold = self.config.get(
+            "similarity_threshold", 0.4
+        )  # Lowered from 0.7
+        self.max_context_length = self.config.get(
+            "max_context_length", 8000
+        )  # Increased
+        self.enable_caching = self.config.get("enable_caching", True)
+        self.cache_ttl = self.config.get("cache_ttl", 7200)  # 2 hours
+
+        # Rate limiting settings
+        self.last_api_call = 0
+        self.min_api_interval = 1.0  # Minimum 1 second between API calls
+        self.max_retries = 3
+        self.retry_delay = 2.0
+
+        # Query cache and history
+        self.query_cache = {}
+        self.query_history = []
+
+        self.logger.info("OptimizedQueryProcessor initialized")
+
+    def process_query(
+        self, query: str, filter: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        """
+        Process query with optimized rate limiting and error handling
+        """
+        if not query or not query.strip():
+            return {
+                "query": query,
+                "context": [],
+                "total_results": 0,
+                "error": "Empty query provided",
+            }
+
+        self.logger.info(f"Processing query: {query[:100]}...")
+        start_time = time.time()
+
+        try:
+            # Check cache first
+            cache_key = self._generate_cache_key(query, filter)
+            if self.enable_caching and cache_key in self.query_cache:
+                cached_result = self.query_cache[cache_key]
+                if self._is_cache_valid(cached_result["timestamp"]):
+                    self.logger.info("Returning cached result")
+                    cached_result["from_cache"] = True
+                    return cached_result
+
+            # Rate limiting protection
+            self._enforce_rate_limit()
+
+            # Generate query embedding with retry logic
+            query_embedding = self._generate_embedding_with_retry(query)
+
+            if not query_embedding:
+                return {
+                    "query": query,
+                    "context": [],
+                    "total_results": 0,
+                    "error": "Failed to generate query embedding",
+                }
+
+            # Search for similar vectors with increased top_k
+            search_results = self.vector_db.search(
+                query_embedding=query_embedding,
+                top_k=self.top_k * 2,  # Get more results for better filtering
+                filter=filter,
+                include_metadata=True,
+            )
+
+            if not search_results:
+                self.logger.warning("No search results returned from vector database")
+                return {
+                    "query": query,
+                    "context": [],
+                    "total_results": 0,
+                    "error": "No similar documents found",
+                }
+
+            # Apply optimized filtering
+            filtered_results = self._apply_smart_filtering(search_results, query)
+
+            # Extract and format context with better error handling
+            context = self._extract_context_safely(filtered_results)
+
+            # Prepare result
+            result = {
+                "query": query,
+                "context": context,
+                "total_results": len(filtered_results),
+                "processing_time": time.time() - start_time,
+                "timestamp": datetime.now(),
+                "from_cache": False,
+                "similarity_scores": [r.get("score", 0) for r in filtered_results[:5]],
+            }
+
+            # Cache the result
+            if self.enable_caching:
+                self.query_cache[cache_key] = result.copy()
+
+            self.logger.info(
+                f"Query processed in {result['processing_time']:.2f}s, {len(context)} context items"
+            )
+            return result
+
+        except Exception as e:
+            self.logger.error(f"Error processing query: {str(e)}")
+            return {
+                "query": query,
+                "context": [],
+                "total_results": 0,
+                "error": str(e),
+                "processing_time": time.time() - start_time,
+            }
+
+    def _enforce_rate_limit(self):
+        """Enforce rate limiting between API calls"""
+        current_time = time.time()
+        time_since_last_call = current_time - self.last_api_call
+
+        if time_since_last_call < self.min_api_interval:
+            sleep_time = self.min_api_interval - time_since_last_call
+            self.logger.info(f"Rate limiting: sleeping {sleep_time:.1f}s")
+            time.sleep(sleep_time)
+
+        self.last_api_call = time.time()
+
+    def _generate_embedding_with_retry(self, query: str) -> List[float]:
+        """Generate embedding with retry logic and rate limiting"""
+        for attempt in range(self.max_retries):
+            try:
+                self._enforce_rate_limit()
+                embedding = self.embedding_generator.generate_query_embedding(query)
+
+                if embedding:
+                    return embedding
+                else:
+                    self.logger.warning(
+                        f"Attempt {attempt + 1}: Empty embedding returned"
+                    )
+
+            except Exception as e:
+                self.logger.warning(f"Attempt {attempt + 1} failed: {str(e)}")
+
+                if "429" in str(e) or "quota" in str(e).lower():
+                    # Rate limit hit - wait longer
+                    wait_time = self.retry_delay * (2**attempt)
+                    self.logger.info(f"Rate limit hit, waiting {wait_time}s...")
+                    time.sleep(wait_time)
+                elif attempt < self.max_retries - 1:
+                    time.sleep(self.retry_delay)
+
+        self.logger.error("All embedding generation attempts failed")
+        return []
+
+    def _apply_smart_filtering(
+        self, search_results: List[Dict[str, Any]], query: str
+    ) -> List[Dict[str, Any]]:
+        """Apply smart filtering with adaptive threshold"""
+        if not search_results:
+            return []
+
+        # Get score statistics
+        scores = [r.get("score", 0) for r in search_results]
+        max_score = max(scores)
+        avg_score = sum(scores) / len(scores)
+
+        # Adaptive threshold: use lower threshold if max score is low
+        adaptive_threshold = min(self.similarity_threshold, max_score * 0.8)
+
+        self.logger.info(
+            f"Score stats - Max: {max_score:.3f}, Avg: {avg_score:.3f}, Threshold: {adaptive_threshold:.3f}"
+        )
+
+        # Filter results
+        filtered = [
+            result
+            for result in search_results[: self.top_k]
+            if result.get("score", 0) >= adaptive_threshold
+        ]
+
+        # If no results pass threshold, return top 3 anyway
+        if not filtered and search_results:
+            self.logger.warning(
+                f"No results above threshold {adaptive_threshold:.3f}, returning top 3"
+            )
+            filtered = search_results[:3]
+
+        return filtered
+
+    def _extract_context_safely(
+        self, search_results: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """Extract context with better error handling"""
+        context = []
+        total_length = 0
+
+        for i, result in enumerate(search_results):
+            try:
+                # Multiple ways to extract text content
+                text = ""
+                metadata = result.get("metadata", {})
+
+                # Try different text fields
+                for field in ["text", "content", "content_preview", "description"]:
+                    if field in metadata and metadata[field]:
+                        text = str(metadata[field])
+                        break
+
+                if not text:
+                    self.logger.warning(f"No text content found in result {i}")
+                    continue
+
+                # Check length limit
+                if total_length + len(text) > self.max_context_length and context:
+                    break
+
+                # Create context item
+                context_item = {
+                    "text": text,
+                    "score": result.get("score", 0),
+                    "source": metadata.get("source", f"Document {i+1}"),
+                    "chunk_id": result.get("id", ""),
+                    "metadata": metadata,
+                    "relevance_rank": len(context) + 1,
+                }
+
+                context.append(context_item)
+                total_length += len(text)
+
+            except Exception as e:
+                self.logger.warning(f"Error extracting context from result {i}: {e}")
+                continue
+
+        self.logger.info(
+            f"Extracted {len(context)} context items (total length: {total_length})"
+        )
+        return context
+
+    def _generate_cache_key(self, query: str, filter: Optional[Dict[str, Any]]) -> str:
+        """Generate cache key for query"""
+        import hashlib
+
+        filter_str = str(sorted(filter.items())) if filter else ""
+        cache_string = f"{query.lower().strip()}{filter_str}"
+        return hashlib.md5(cache_string.encode()).hexdigest()
+
+    def _is_cache_valid(self, timestamp: datetime) -> bool:
+        """Check if cached result is still valid"""
+        return datetime.now() - timestamp < timedelta(seconds=self.cache_ttl)

src/rag/query_processor.py

@@ -0,0 +1,427 @@
+"""
+Query Processor Module
+
+This module is responsible for processing user queries and converting
+them to vector embeddings for retrieval.
+
+Technologies: Gemini Embedding v3, LangChain, Pinecone
+"""
+
+import logging
+import time
+from typing import Dict, List, Any, Optional
+from datetime import datetime, timedelta
+
+
+class QueryProcessor:
+    """
+    Processes user queries and converts them to vector embeddings.
+
+    Features:
+    - Query preprocessing and normalization
+    - Query embedding generation
+    - Context retrieval from vector database
+    - Query expansion and caching
+    - Metadata filtering and ranking
+    """
+
+    def __init__(
+        self, embedding_generator, vector_db, config: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Initialize the QueryProcessor with dependencies.
+
+        Args:
+            embedding_generator: Instance of EmbeddingGenerator
+            vector_db: Instance of VectorDB
+            config: Configuration dictionary with processing parameters
+        """
+        self.embedding_generator = embedding_generator
+        self.vector_db = vector_db
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # Configuration settings
+        self.top_k = self.config.get("top_k", 5)
+        self.similarity_threshold = self.config.get("similarity_threshold", 0.7)
+        self.max_context_length = self.config.get("max_context_length", 4000)
+        self.enable_caching = self.config.get("enable_caching", True)
+        self.cache_ttl = self.config.get("cache_ttl", 3600)  # 1 hour
+
+        # Query cache and history
+        self.query_cache = {}
+        self.query_history = []
+
+        self.logger.info("QueryProcessor initialized with advanced features")
+
+    def process_query(
+        self, query: str, filter: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        """
+        Process a user query and retrieve relevant context.
+
+        Args:
+            query: User query string
+            filter: Optional metadata filter for search
+
+        Returns:
+            Dictionary containing query, retrieved context, and metadata
+        """
+        if not query or not query.strip():
+            return {
+                "query": query,
+                "context": [],
+                "total_results": 0,
+                "error": "Empty query provided",
+            }
+
+        self.logger.info(f"Processing query: {query[:100]}...")
+        start_time = time.time()
+
+        try:
+            # Check cache first
+            cache_key = self._generate_cache_key(query, filter)
+            if self.enable_caching and cache_key in self.query_cache:
+                cached_result = self.query_cache[cache_key]
+                if self._is_cache_valid(cached_result["timestamp"]):
+                    self.logger.info("  Returning cached result")
+                    cached_result["from_cache"] = True
+                    return cached_result
+
+            # Preprocess the query
+            processed_query = self._preprocess_query(query)
+            expanded_queries = self._expand_query(processed_query)
+
+            # Generate embeddings for all query variations
+            all_results = []
+            for q in expanded_queries:
+                query_embedding = self.embedding_generator.generate_query_embedding(q)
+
+                if query_embedding:
+                    # Search for similar vectors
+                    search_results = self.vector_db.search(
+                        query_embedding=query_embedding,
+                        top_k=self.top_k * 2,  # Get more results for better filtering
+                        filter=filter,
+                    )
+                    all_results.extend(search_results)
+
+            # Deduplicate and rank results
+            unique_results = self._deduplicate_results(all_results)
+            ranked_results = self._rank_results(unique_results, query)
+
+            # Filter results by similarity threshold
+            filtered_results = [
+                result
+                for result in ranked_results[: self.top_k]
+                if result.get("score", 0) >= self.similarity_threshold
+            ]
+
+            # Extract and format context
+            context = self._extract_context(filtered_results)
+
+            # Prepare result
+            result = {
+                "query": query,
+                "processed_query": processed_query,
+                "expanded_queries": expanded_queries,
+                "context": context,
+                "total_results": len(filtered_results),
+                "processing_time": time.time() - start_time,
+                "timestamp": datetime.now(),
+                "from_cache": False,
+            }
+
+            # Cache the result
+            if self.enable_caching:
+                self.query_cache[cache_key] = result.copy()
+
+            # Add to query history
+            self._add_to_history(query, len(filtered_results))
+
+            self.logger.info(f"Query processed in {result['processing_time']:.2f}s")
+            return result
+
+        except Exception as e:
+            self.logger.error(f"❌ Error processing query: {str(e)}")
+            return {
+                "query": query,
+                "context": [],
+                "total_results": 0,
+                "error": str(e),
+                "processing_time": time.time() - start_time,
+            }
+
+    def _preprocess_query(self, query: str) -> str:
+        """
+        Preprocess the query for better embedding generation.
+
+        Args:
+            query: Raw query string
+
+        Returns:
+            Preprocessed query string
+        """
+        # Remove extra whitespace
+        query = " ".join(query.split())
+
+        # Remove special characters that might interfere
+        import re
+
+        query = re.sub(r"[^\w\s\-\?\!]", " ", query)
+
+        # Normalize question words
+        question_words = {
+            "whats": "what is",
+            "hows": "how is",
+            "wheres": "where is",
+            "whos": "who is",
+            "whens": "when is",
+        }
+
+        for abbrev, full in question_words.items():
+            query = query.replace(abbrev, full)
+
+        return query.strip()
+
+    def _expand_query(self, query: str) -> List[str]:
+        """
+        Expand the query with variations for better retrieval.
+
+        Args:
+            query: Preprocessed query
+
+        Returns:
+            List of query variations
+        """
+        expanded = [query]
+
+        # Add question variations
+        if not any(
+            q in query.lower() for q in ["what", "how", "why", "when", "where", "who"]
+        ):
+            expanded.append(f"what is {query}")
+            expanded.append(f"how does {query} work")
+
+        # Add definition variation
+        if "definition" not in query.lower() and "define" not in query.lower():
+            expanded.append(f"{query} definition")
+
+        # Add example variation
+        if "example" not in query.lower():
+            expanded.append(f"{query} examples")
+
+        return expanded[:3]  # Limit to 3 variations
+
+    def _deduplicate_results(
+        self, results: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """
+        Remove duplicate results based on content similarity.
+
+        Args:
+            results: List of search results
+
+        Returns:
+            Deduplicated results
+        """
+        seen_ids = set()
+        unique_results = []
+
+        for result in results:
+            result_id = result.get("id")
+            if result_id and result_id not in seen_ids:
+                seen_ids.add(result_id)
+                unique_results.append(result)
+
+        return unique_results
+
+    def _rank_results(
+        self, results: List[Dict[str, Any]], query: str
+    ) -> List[Dict[str, Any]]:
+        """
+        Rank results based on multiple factors.
+
+        Args:
+            results: List of search results
+            query: Original query
+
+        Returns:
+            Ranked results
+        """
+        query_words = set(query.lower().split())
+
+        for result in results:
+            # Base score from similarity
+            base_score = result.get("score", 0.0)
+
+            # Boost score based on text relevance
+            text = result.get("metadata", {}).get("text", "").lower()
+            text_words = set(text.split())
+            word_overlap = len(query_words.intersection(text_words))
+            relevance_boost = word_overlap / max(len(query_words), 1) * 0.1
+
+            # Boost score based on source type
+            source = result.get("metadata", {}).get("source", "")
+            source_boost = 0.0
+            if source.endswith(".pdf"):
+                source_boost = 0.05  # PDFs often contain structured info
+            elif "http" in source:
+                source_boost = 0.02  # Web content
+
+            # Calculate final score
+            final_score = base_score + relevance_boost + source_boost
+            result["final_score"] = min(final_score, 1.0)
+
+        # Sort by final score
+        return sorted(results, key=lambda x: x.get("final_score", 0), reverse=True)
+
+    def _extract_context(
+        self, search_results: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """
+        Extract and format context from search results.
+
+        Args:
+            search_results: List of search results from vector database
+
+        Returns:
+            List of formatted context items
+        """
+        context = []
+        total_length = 0
+
+        for result in search_results:
+            # Extract text content from metadata
+            text = result.get("metadata", {}).get("text", "")
+
+            # Check if adding this context would exceed the limit
+            if total_length + len(text) > self.max_context_length and context:
+                break
+
+            # Format context item with enhanced metadata
+            context_item = {
+                "text": text,
+                "score": result.get("score", 0),
+                "final_score": result.get("final_score", result.get("score", 0)),
+                "source": result.get("metadata", {}).get("source", "unknown"),
+                "chunk_id": result.get("id", ""),
+                "metadata": result.get("metadata", {}),
+                "relevance_rank": len(context) + 1,
+            }
+
+            context.append(context_item)
+            total_length += len(text)
+
+        self.logger.info(
+            f"Extracted {len(context)} context items (total length: {total_length})"
+        )
+        return context
+
+    def _generate_cache_key(self, query: str, filter: Optional[Dict[str, Any]]) -> str:
+        """Generate a cache key for the query."""
+        import hashlib
+
+        filter_str = str(sorted(filter.items())) if filter else ""
+        cache_string = f"{query.lower().strip()}{filter_str}"
+        return hashlib.md5(cache_string.encode()).hexdigest()
+
+    def _is_cache_valid(self, timestamp: datetime) -> bool:
+        """Check if cached result is still valid."""
+        return datetime.now() - timestamp < timedelta(seconds=self.cache_ttl)
+
+    def _add_to_history(self, query: str, result_count: int):
+        """Add query to history for analytics."""
+        self.query_history.append(
+            {
+                "query": query,
+                "timestamp": datetime.now(),
+                "result_count": result_count,
+            }
+        )
+
+        # Keep only last 100 queries
+        if len(self.query_history) > 100:
+            self.query_history = self.query_history[-100:]
+
+    def get_query_suggestions(self, partial_query: str) -> List[str]:
+        """
+        Generate query suggestions based on partial input and history.
+
+        Args:
+            partial_query: Partial query string
+
+        Returns:
+            List of suggested queries
+        """
+        suggestions = []
+
+        # Add suggestions from query history
+        for hist_item in reversed(self.query_history[-20:]):  # Last 20 queries
+            hist_query = hist_item["query"]
+            if (
+                partial_query.lower() in hist_query.lower()
+                and hist_query not in suggestions
+            ):
+                suggestions.append(hist_query)
+
+        # Add template-based suggestions
+        if len(suggestions) < 3:
+            templates = [
+                f"What is {partial_query}?",
+                f"How does {partial_query} work?",
+                f"Examples of {partial_query}",
+                f"{partial_query} definition",
+                f"{partial_query} best practices",
+            ]
+
+            for template in templates:
+                if template not in suggestions:
+                    suggestions.append(template)
+                if len(suggestions) >= 5:
+                    break
+
+        return suggestions[:5]
+
+    def get_query_analytics(self) -> Dict[str, Any]:
+        """
+        Get analytics about query patterns.
+
+        Returns:
+            Dictionary with query analytics
+        """
+        if not self.query_history:
+            return {"total_queries": 0, "cache_hit_rate": 0.0}
+
+        total_queries = len(self.query_history)
+        recent_queries = [q["query"] for q in self.query_history[-10:]]
+
+        # Calculate average results per query
+        avg_results = sum(q["result_count"] for q in self.query_history) / total_queries
+
+        # Most common query patterns
+        query_words = []
+        for q in self.query_history:
+            query_words.extend(q["query"].lower().split())
+
+        from collections import Counter
+
+        common_words = Counter(query_words).most_common(5)
+
+        return {
+            "total_queries": total_queries,
+            "average_results_per_query": round(avg_results, 2),
+            "recent_queries": recent_queries,
+            "common_query_words": common_words,
+            "cache_size": len(self.query_cache),
+        }
+
+    def clear_cache(self):
+        """Clear the query cache."""
+        self.query_cache.clear()
+        self.logger.info("Query cache cleared")
+
+    def clear_history(self):
+        """Clear the query history."""
+        self.query_history.clear()
+        self.logger.info("Query history cleared")

src/rag/query_router.py

@@ -0,0 +1,587 @@
+"""
+Query Router Module
+
+This module intelligently routes queries between local document search
+and live web search based on query analysis and user preferences.
+
+Technology: Custom routing logic with RAG + Live Search integration
+"""
+
+import logging
+import time
+from typing import Dict, List, Any, Optional, Tuple
+from datetime import datetime
+from enum import Enum
+
+
+class QueryType(Enum):
+    """Enumeration of different query types for routing decisions."""
+
+    FACTUAL = "factual"  # 📊 Current facts, news, data
+    CONCEPTUAL = "conceptual"  # 💡 Definitions, explanations
+    PROCEDURAL = "procedural"  # 🔧 How-to, instructions
+    ANALYTICAL = "analytical"  # 📈 Analysis, comparisons
+    TEMPORAL = "temporal"  # ⏰ Time-sensitive information
+    HYBRID = "hybrid"  # 🔄 Requires both sources
+
+
+class QueryRouter:
+    """
+    Intelligent query router that decides between local docs and live search.
+
+    Features:
+    - Query type classification
+    - Intelligent routing decisions
+    - Hybrid search coordination
+    - Result fusion and ranking
+    - Performance optimization
+    """
+
+    def __init__(
+        self,
+        local_query_processor,
+        live_search_processor,
+        config: Optional[Dict[str, Any]] = None,
+    ):
+        """
+        Initialize the QueryRouter.
+
+        Args:
+            local_query_processor: Local document query processor
+            live_search_processor: Live web search processor
+            config: Configuration dictionary
+        """
+        self.local_processor = local_query_processor
+        self.live_processor = live_search_processor
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # 🎯 Routing configuration
+        self.enable_hybrid_search = self.config.get("enable_hybrid_search", True)
+        self.local_weight = self.config.get("local_weight", 0.6)
+        self.live_weight = self.config.get("live_weight", 0.4)
+        self.confidence_threshold = self.config.get("confidence_threshold", 0.5)
+        self.max_hybrid_results = self.config.get("max_hybrid_results", 10)
+
+        # 📊 Analytics and caching
+        self.routing_history = []
+        self.routing_cache = {}
+
+        # 🔍 Query classification patterns
+        self._init_classification_patterns()
+
+        self.logger.info("QueryRouter initialized with intelligent routing")
+
+    def _init_classification_patterns(self):
+        """Initialize patterns for query classification."""
+        self.temporal_keywords = {
+            "current",
+            "latest",
+            "recent",
+            "today",
+            "now",
+            "2025",
+            "breaking",
+            "news",
+            "update",
+            "trending",
+            "happening",
+        }
+
+        self.factual_keywords = {
+            "what is",
+            "who is",
+            "when did",
+            "where is",
+            "statistics",
+            "data",
+            "facts",
+            "numbers",
+            "rate",
+            "percentage",
+        }
+
+        self.procedural_keywords = {
+            "how to",
+            "steps",
+            "guide",
+            "tutorial",
+            "instructions",
+            "process",
+            "method",
+            "way to",
+            "procedure",
+        }
+
+        self.conceptual_keywords = {
+            "explain",
+            "definition",
+            "meaning",
+            "concept",
+            "theory",
+            "principle",
+            "idea",
+            "understand",
+            "clarify",
+        }
+
+    def route_query(
+        self,
+        query: str,
+        use_live_search: bool = False,
+        max_results: int = 5,
+        search_options: Optional[Dict[str, Any]] = None,
+        search_mode: str = "auto",
+    ) -> Dict[str, Any]:
+        """
+        Route query to appropriate search method(s) with enhanced control.
+
+        Args:
+            query: User query string
+            use_live_search: Enable live search (will use hybrid approach)
+            max_results: Maximum results to return
+            search_options: Additional search options
+            search_mode: Search mode - "auto", "local_only", "live_only", "hybrid"
+
+        Returns:
+            Dictionary with routed results and metadata
+        """
+        if not query or not query.strip():
+            return {
+                "query": query,
+                "results": [],
+                "routing_decision": "error",
+                "error": "Empty query provided",
+            }
+
+        self.logger.info(f" Routing query: {query[:100]}...")
+        start_time = time.time()
+
+        try:
+            # 🎯 Classify query type
+            query_type = self._classify_query(query)
+
+            # 🔄 Make routing decision with enhanced logic
+            routing_decision = self._make_enhanced_routing_decision(
+                query, query_type, use_live_search, search_mode
+            )
+
+            # 🚀 Execute search based on routing decision
+            if routing_decision == "local_only":
+                result = self._search_local_only(query, max_results)
+            elif routing_decision == "live_only":
+                result = self._search_live_only(query, max_results, search_options)
+            elif routing_decision == "hybrid":
+                result = self._search_hybrid(query, max_results, search_options)
+            else:
+                result = self._search_fallback(query, max_results)
+
+            # 📊 Add routing metadata
+            result.update(
+                {
+                    "query_type": query_type.value,
+                    "routing_decision": routing_decision,
+                    "processing_time": time.time() - start_time,
+                    "timestamp": datetime.now(),
+                }
+            )
+
+            # 📈 Track routing decision
+            self._track_routing_decision(query, query_type, routing_decision)
+
+            self.logger.info(
+                f" Query routed via {routing_decision} in {result['processing_time']:.2f}s"
+            )
+            return result
+
+        except Exception as e:
+            self.logger.error(f" Error in query routing: {str(e)}")
+            return {
+                "query": query,
+                "results": [],
+                "routing_decision": "error",
+                "error": str(e),
+                "processing_time": time.time() - start_time,
+            }
+
+    def _classify_query(self, query: str) -> QueryType:
+        """
+        Classify query type for routing decisions.
+
+        Args:
+            query: Query string to classify
+
+        Returns:
+            QueryType enum value
+        """
+        query_lower = query.lower()
+
+        # 🔍 Check for temporal indicators
+        if any(keyword in query_lower for keyword in self.temporal_keywords):
+            return QueryType.TEMPORAL
+
+        # 📊 Check for factual queries
+        if any(keyword in query_lower for keyword in self.factual_keywords):
+            return QueryType.FACTUAL
+
+        # 🔧 Check for procedural queries
+        if any(keyword in query_lower for keyword in self.procedural_keywords):
+            return QueryType.PROCEDURAL
+
+        # 💡 Check for conceptual queries
+        if any(keyword in query_lower for keyword in self.conceptual_keywords):
+            return QueryType.CONCEPTUAL
+
+        # 📈 Default to analytical for complex queries
+        if len(query.split()) > 10:
+            return QueryType.ANALYTICAL
+
+        # 🔄 Default to hybrid for uncertain cases
+        return QueryType.HYBRID
+
+    def _make_routing_decision(
+        self, query: str, query_type: QueryType, force_live: bool
+    ) -> str:
+        """
+        Make intelligent routing decision based on query analysis.
+
+        Args:
+            query: Query string
+            query_type: Classified query type
+            force_live: Whether to enable live search (not force only live)
+
+        Returns:
+            Routing decision string
+        """
+        # 🔄 Smart hybrid approach when live search is enabled
+        if force_live:
+            # ✨ Instead of live_only, use hybrid to combine both sources
+            if query_type == QueryType.TEMPORAL:
+                return "hybrid"  # ⏰ Time-sensitive + stored context
+            else:
+                return "hybrid"  # 🎯 Always combine live + stored data
+
+        # 🎯 Route based on query type (when live search is disabled)
+        if query_type == QueryType.TEMPORAL:
+            return "local_only"  # ⏰ Only stored data when live disabled
+
+        elif query_type == QueryType.FACTUAL:
+            return "local_only"  # 📊 Facts from stored documents
+
+        elif query_type == QueryType.PROCEDURAL:
+            return "local_only"  # 🔧 Procedures likely in documents
+
+        elif query_type == QueryType.CONCEPTUAL:
+            return "local_only"  # 💡 Concepts likely in documents
+
+        elif query_type == QueryType.ANALYTICAL:
+            return "local_only"  # 📈 Analysis from stored data
+
+        else:  # QueryType.HYBRID
+            return "local_only"  # 🔄 Default to local when live disabled
+
+    def _make_enhanced_routing_decision(
+        self, query: str, query_type: QueryType, use_live_search: bool, search_mode: str
+    ) -> str:
+        """
+        Enhanced routing decision with explicit search mode control.
+
+        Args:
+            query: Query string
+            query_type: Classified query type
+            use_live_search: Whether live search is enabled
+            search_mode: Explicit search mode preference
+
+        Returns:
+            Routing decision string
+        """
+        # 🎯 Explicit mode override - user ka choice priority
+        if search_mode == "local_only":
+            return "local_only"
+        elif search_mode == "live_only":
+            return "live_only" if self.live_processor.is_enabled() else "local_only"
+        elif search_mode == "hybrid":
+            return "hybrid" if self.live_processor.is_enabled() else "local_only"
+
+        # 🧠 Auto mode - intelligent decision making
+        elif search_mode == "auto":
+            return self._make_routing_decision(query, query_type, use_live_search)
+
+        # 🔄 Fallback to original logic
+        else:
+            return self._make_routing_decision(query, query_type, use_live_search)
+
+    def _search_local_only(self, query: str, max_results: int) -> Dict[str, Any]:
+        """Search only local documents."""
+        self.logger.info(" Searching local documents only")
+
+        try:
+            local_result = self.local_processor.process_query(query)
+
+            # 🔄 Format results consistently
+            formatted_results = []
+            for item in local_result.get("context", [])[:max_results]:
+                formatted_results.append(
+                    {
+                        "title": f"Document: {item.get('source', 'Unknown')}",
+                        "content": item.get("text", ""),
+                        "score": item.get("score", 0.0),
+                        "source": item.get("source", "local_document"),
+                        "type": "local_document",
+                        "metadata": item.get("metadata", {}),
+                    }
+                )
+
+            return {
+                "query": query,
+                "results": formatted_results,
+                "total_results": len(formatted_results),
+                "sources": ["local_documents"],
+                "local_results": local_result.get("total_results", 0),
+            }
+
+        except Exception as e:
+            self.logger.error(f" Local search error: {str(e)}")
+            return {
+                "query": query,
+                "results": [],
+                "total_results": 0,
+                "error": f"Local search failed: {str(e)}",
+            }
+
+    def _search_live_only(
+        self, query: str, max_results: int, search_options: Optional[Dict[str, Any]]
+    ) -> Dict[str, Any]:
+        """Search only live web sources."""
+        self.logger.info(" Searching live web sources only")
+
+        try:
+            # 🎯 Extract search options
+            options = search_options or {}
+            search_depth = options.get("search_depth", "basic")
+            time_range = options.get("time_range", "month")
+
+            live_result = self.live_processor.search_web(
+                query,
+                max_results=max_results,
+                search_depth=search_depth,
+                time_range=time_range,
+            )
+
+            return {
+                "query": query,
+                "results": live_result.get("results", []),
+                "total_results": live_result.get("total_results", 0),
+                "sources": ["live_web"],
+                "live_results": live_result.get("total_results", 0),
+                "search_params": live_result.get("search_params", {}),
+            }
+
+        except Exception as e:
+            self.logger.error(f" Live search error: {str(e)}")
+            return {
+                "query": query,
+                "results": [],
+                "total_results": 0,
+                "error": f"Live search failed: {str(e)}",
+            }
+
+    def _search_hybrid(
+        self, query: str, max_results: int, search_options: Optional[Dict[str, Any]]
+    ) -> Dict[str, Any]:
+        """Perform hybrid search combining local and live sources."""
+        self.logger.info(" Performing hybrid search")
+
+        try:
+            # 📊 Calculate result distribution
+            local_count = int(max_results * self.local_weight)
+            live_count = max_results - local_count
+
+            # 🚀 Perform both searches concurrently (simplified sequential for now)
+            local_result = self.local_processor.process_query(query)
+
+            options = search_options or {}
+            live_result = self.live_processor.search_web(
+                query,
+                max_results=live_count,
+                search_depth=options.get("search_depth", "basic"),
+                time_range=options.get("time_range", "month"),
+            )
+
+            # 🔄 Combine and rank results
+            combined_results = self._fuse_results(
+                local_result, live_result, local_count, live_count
+            )
+
+            return {
+                "query": query,
+                "results": combined_results[:max_results],
+                "total_results": len(combined_results),
+                "sources": ["local_documents", "live_web"],
+                "local_results": local_result.get("total_results", 0),
+                "live_results": live_result.get("total_results", 0),
+                "fusion_method": "weighted_ranking",
+            }
+
+        except Exception as e:
+            self.logger.error(f" Hybrid search error: {str(e)}")
+            return self._search_fallback(query, max_results)
+
+    def _fuse_results(
+        self,
+        local_result: Dict[str, Any],
+        live_result: Dict[str, Any],
+        local_count: int,
+        live_count: int,
+    ) -> List[Dict[str, Any]]:
+        """
+        Fuse results from local and live searches.
+
+        Args:
+            local_result: Results from local search
+            live_result: Results from live search
+            local_count: Number of local results to include
+            live_count: Number of live results to include
+
+        Returns:
+            Fused and ranked results
+        """
+        fused_results = []
+
+        # 📚 Process local results
+        for item in local_result.get("context", [])[:local_count]:
+            fused_results.append(
+                {
+                    "title": f"Document: {item.get('source', 'Unknown')}",
+                    "content": item.get("text", ""),
+                    "score": item.get("score", 0.0) * self.local_weight,
+                    "source": item.get("source", "local_document"),
+                    "type": "local_document",
+                    "metadata": item.get("metadata", {}),
+                    "fusion_score": item.get("score", 0.0) * self.local_weight,
+                }
+            )
+
+        # 🌐 Process live results
+        for item in live_result.get("results", [])[:live_count]:
+            fused_results.append(
+                {
+                    "title": item.get("title", "Web Result"),
+                    "content": item.get("content", ""),
+                    "score": item.get("relevance_score", 0.0) * self.live_weight,
+                    "source": item.get("url", "web_search"),
+                    "type": "web_result",
+                    "metadata": item.get("metadata", {}),
+                    "fusion_score": item.get("relevance_score", 0.0) * self.live_weight,
+                }
+            )
+
+        # 🔄 Sort by fusion score
+        fused_results.sort(key=lambda x: x.get("fusion_score", 0), reverse=True)
+
+        return fused_results
+
+    def _search_fallback(self, query: str, max_results: int) -> Dict[str, Any]:
+        """Fallback search method when other methods fail."""
+        self.logger.warning(" Using fallback search method")
+
+        try:
+            # 📚 Try local search first
+            local_result = self.local_processor.process_query(query)
+
+            if local_result.get("context"):
+                return self._search_local_only(query, max_results)
+            else:
+                return {
+                    "query": query,
+                    "results": [],
+                    "total_results": 0,
+                    "sources": [],
+                    "error": "No results found in fallback search",
+                }
+
+        except Exception as e:
+            self.logger.error(f" Fallback search failed: {str(e)}")
+            return {
+                "query": query,
+                "results": [],
+                "total_results": 0,
+                "error": f"All search methods failed: {str(e)}",
+            }
+
+    def _track_routing_decision(
+        self, query: str, query_type: QueryType, routing_decision: str
+    ):
+        """Track routing decisions for analytics."""
+        self.routing_history.append(
+            {
+                "query": query[:100],  # Truncate for privacy
+                "query_type": query_type.value,
+                "routing_decision": routing_decision,
+                "timestamp": datetime.now(),
+            }
+        )
+
+        # 📊 Keep only last 100 routing decisions
+        if len(self.routing_history) > 100:
+            self.routing_history = self.routing_history[-100:]
+
+    def get_routing_analytics(self) -> Dict[str, Any]:
+        """
+        Get analytics about routing patterns.
+
+        Returns:
+            Dictionary with routing analytics
+        """
+        if not self.routing_history:
+            return {
+                "total_queries": 0,
+                "routing_distribution": {},
+                "query_type_distribution": {},
+            }
+
+        total_queries = len(self.routing_history)
+
+        # 📊 Calculate routing distribution
+        routing_counts = {}
+        query_type_counts = {}
+
+        for entry in self.routing_history:
+            routing = entry["routing_decision"]
+            query_type = entry["query_type"]
+
+            routing_counts[routing] = routing_counts.get(routing, 0) + 1
+            query_type_counts[query_type] = query_type_counts.get(query_type, 0) + 1
+
+        # 📈 Convert to percentages
+        routing_distribution = {
+            k: round((v / total_queries) * 100, 1) for k, v in routing_counts.items()
+        }
+
+        query_type_distribution = {
+            k: round((v / total_queries) * 100, 1) for k, v in query_type_counts.items()
+        }
+
+        return {
+            "total_queries": total_queries,
+            "routing_distribution": routing_distribution,
+            "query_type_distribution": query_type_distribution,
+            "recent_decisions": [
+                {
+                    "query": entry["query"][:50] + "...",
+                    "type": entry["query_type"],
+                    "routing": entry["routing_decision"],
+                }
+                for entry in self.routing_history[-5:]
+            ],
+        }
+
+    def clear_cache(self):
+        """Clear routing cache."""
+        self.routing_cache.clear()
+        self.logger.info(" Routing cache cleared")
+
+    def clear_history(self):
+        """Clear routing history."""
+        self.routing_history.clear()
+        self.logger.info(" Routing history cleared")

src/rag/response_generator.py

@@ -0,0 +1,591 @@
+"""
+Response Generator Module
+
+This module is responsible for generating coherent responses based on
+retrieved knowledge using LangChain RAG.
+
+Technology: LangChain RAG (Retrieval Augmented Generation)
+"""
+
+import logging
+import time
+import os
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+
+
+class ResponseGenerator:
+    """
+    Generates coherent responses based on retrieved knowledge.
+
+    Features:
+    - Context-aware response generation
+    - Source attribution and confidence scoring
+    - Multiple LLM provider support (Gemini, OpenAI)
+    - Response quality assessment
+    - Template-based fallback generation
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the ResponseGenerator with configuration.
+
+        Args:
+            config: Configuration dictionary with generation parameters
+        """
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # Configuration settings
+        self.model = self.config.get("model", "gpt-3.5-turbo")
+        self.max_tokens = self.config.get("max_tokens", 500)
+        self.temperature = self.config.get("temperature", 0.7)
+        self.include_sources = self.config.get("include_sources", True)
+
+        # Initialize LLM providers
+        self.llm = None
+        self.gemini_client = None
+        self.openai_client = None
+
+        self._initialize_llm_providers()
+
+        # Response templates with markdown formatting
+        self.response_templates = {
+            "no_context": "## ℹ️ No Information Available\n\nI don't have enough information to answer your question. Please try:\n\n- **Uploading relevant documents** using the Upload tab\n- **Adding URLs** using the Add URLs tab\n- **Enabling live search** for real-time web results",
+            "error": "## ⚠️ Error Occurred\n\nI encountered an error while generating the response. Please try again.\n\nIf the problem persists, check your API keys in the Settings tab.",
+            "insufficient_confidence": "## 🤔 Limited Confidence\n\nBased on the available information, I found some relevant content, but I'm **not confident enough** to provide a definitive answer.\n\n**Suggestions:**\n- Try rephrasing your question\n- Add more specific documents\n- Enable live search for additional context",
+        }
+
+        self.logger.info("ResponseGenerator initialized with advanced features")
+
+    def _initialize_llm_providers(self):
+        """Initialize available LLM providers with optimization."""
+        try:
+            # Try to initialize Gemini
+            gemini_api_key = os.getenv("GEMINI_API_KEY")
+            if gemini_api_key:
+                try:
+                    import google.generativeai as genai
+
+                    # Check if settings manager has already initialized Gemini client
+                    # This is an optimization to avoid recreating the client
+                    from utils.settings_manager import SettingsManager
+
+                    if (
+                        hasattr(SettingsManager, "_gemini_client_cache")
+                        and SettingsManager._gemini_client_cache is not None
+                        and SettingsManager._gemini_client_key == gemini_api_key
+                    ):
+
+                        self.logger.info(
+                            "Reusing existing Gemini client from settings manager"
+                        )
+                        genai_client = SettingsManager._gemini_client_cache
+                    else:
+                        # Configure new client
+                        genai.configure(api_key=gemini_api_key)
+                        genai_client = genai
+
+                    # Create model instance
+                    self.gemini_client = genai_client.GenerativeModel(
+                        "gemini-2.5-flash-preview-05-20"
+                    )
+                    self.logger.info("Gemini client initialized")
+                except ImportError:
+                    self.logger.warning("Gemini SDK not available")
+                except Exception as e:
+                    self.logger.warning(f"Failed to initialize Gemini: {e}")
+
+            # Try to initialize OpenAI
+            openai_api_key = os.getenv("OPENAI_API_KEY")
+            if openai_api_key:
+                try:
+                    import openai
+
+                    self.openai_client = openai.OpenAI(api_key=openai_api_key)
+                    self.logger.info("OpenAI client initialized")
+                except ImportError:
+                    self.logger.warning("OpenAI SDK not available")
+                except Exception as e:
+                    self.logger.warning(f"Failed to initialize OpenAI: {e}")
+
+            # Try to initialize LangChain
+            try:
+                if self.gemini_client:
+                    from langchain_google_genai import ChatGoogleGenerativeAI
+
+                    self.llm = ChatGoogleGenerativeAI(
+                        model="gemini-2.5-flash-preview-05-20",
+                        temperature=self.temperature,
+                        google_api_key=gemini_api_key,
+                    )
+                elif self.openai_client:
+                    from langchain_openai import ChatOpenAI
+
+                    self.llm = ChatOpenAI(
+                        model=self.model,
+                        temperature=self.temperature,
+                        max_tokens=self.max_tokens,
+                        openai_api_key=openai_api_key,
+                    )
+
+                if self.llm:
+                    self.logger.info("LangChain LLM initialized")
+
+            except ImportError:
+                self.logger.warning("LangChain not available")
+            except Exception as e:
+                self.logger.warning(f"Failed to initialize LangChain: {e}")
+
+        except Exception as e:
+            self.logger.error(f"❌ Error initializing LLM providers: {e}")
+
+    def generate_response(
+        self, query: str, context: List[Dict[str, Any]]
+    ) -> Dict[str, Any]:
+        """
+        Generate a response based on the query and retrieved context.
+
+        Args:
+            query: Original user query
+            context: List of retrieved context items with text and metadata
+
+        Returns:
+            Dictionary containing the generated response and metadata
+        """
+        if not query:
+            return {
+                "response": "I need a question to answer.",
+                "sources": [],
+                "confidence": 0.0,
+                "error": "No query provided",
+            }
+
+        if not context:
+            return {
+                "response": self.response_templates["no_context"],
+                "sources": [],
+                "confidence": 0.0,
+                "error": "No context available",
+            }
+
+        self.logger.info(f"Generating response for query: {query[:100]}...")
+        start_time = time.time()
+
+        try:
+            # Prepare context for generation
+            formatted_context = self._format_context(context)
+
+            # Calculate initial confidence based on context quality
+            base_confidence = self._calculate_confidence(context)
+
+            # Generate response using available LLM
+            response_result = self._generate_with_llm(query, formatted_context)
+
+            if not response_result["success"]:
+                # Fallback to template-based generation
+                response_result = self._fallback_generation(query, formatted_context)
+
+            # Extract sources from context
+            sources = self._extract_sources(context) if self.include_sources else []
+
+            # Assess response quality
+            quality_score = self._assess_response_quality(
+                response_result["response"], query, context
+            )
+
+            # Calculate final confidence
+            final_confidence = min(base_confidence * quality_score, 1.0)
+
+            # Check if confidence is too low
+            if final_confidence < 0.3:
+                response_text = self.response_templates["insufficient_confidence"]
+                final_confidence = 0.2
+            else:
+                response_text = response_result["response"]
+
+            result = {
+                "response": response_text,
+                "sources": sources,
+                "confidence": final_confidence,
+                "context_items": len(context),
+                "generation_time": time.time() - start_time,
+                "model_used": response_result.get("model", "fallback"),
+                "quality_score": quality_score,
+            }
+
+            self.logger.info(f"Response generated in {result['generation_time']:.2f}s")
+            return result
+
+        except Exception as e:
+            self.logger.error(f"❌ Error generating response: {str(e)}")
+            return {
+                "response": self.response_templates["error"],
+                "sources": [],
+                "confidence": 0.0,
+                "error": str(e),
+                "generation_time": time.time() - start_time,
+            }
+
+    def _generate_with_llm(self, query: str, context: str) -> Dict[str, Any]:
+        """
+        Generate response using available LLM providers.
+
+        Args:
+            query: User query
+            context: Formatted context string
+
+        Returns:
+            Dictionary with generation result
+        """
+        # Create RAG prompt
+        prompt = self._create_rag_prompt(query, context)
+
+        # Try LangChain first
+        if self.llm:
+            try:
+                from langchain.schema import HumanMessage
+
+                messages = [HumanMessage(content=prompt)]
+                response = self.llm.invoke(messages)
+                return {
+                    "success": True,
+                    "response": response.content,
+                    "model": "langchain",
+                }
+            except Exception as e:
+                self.logger.warning(f"LangChain generation failed: {e}")
+
+        # Try Gemini directly
+        if self.gemini_client:
+            try:
+                response = self.gemini_client.generate_content(prompt)
+                return {
+                    "success": True,
+                    "response": response.text,
+                    "model": "gemini-2.5-flash-preview-05-20",
+                }
+            except Exception as e:
+                self.logger.warning(f"Gemini generation failed: {e}")
+
+        # Try OpenAI directly
+        if self.openai_client:
+            try:
+                response = self.openai_client.chat.completions.create(
+                    model=self.model,
+                    messages=[{"role": "user", "content": prompt}],
+                    max_tokens=self.max_tokens,
+                    temperature=self.temperature,
+                )
+                return {
+                    "success": True,
+                    "response": response.choices[0].message.content,
+                    "model": self.model,
+                }
+            except Exception as e:
+                self.logger.warning(f"OpenAI generation failed: {e}")
+
+        return {"success": False, "response": "", "model": "none"}
+
+    def _create_rag_prompt(self, query: str, context: str) -> str:
+        """
+        Create an enhanced prompt template for RAG generation with markdown formatting.
+
+        Args:
+            query: User query
+            context: Formatted context
+
+        Returns:
+            Formatted prompt string
+        """
+        prompt = f"""You are an AI assistant that answers questions based on provided context. Follow these guidelines:
+
+1. Answer the question using ONLY the information provided in the context
+2. If the context doesn't contain enough information, clearly state this
+3. Cite specific sources when making claims
+4. Be concise but comprehensive
+5. If multiple sources provide different information, acknowledge this
+6. Use a professional and helpful tone
+7. **Format your response in clean, readable Markdown**
+
+Context Information:
+{context}
+
+Question: {query}
+
+Instructions:
+- Provide a clear, well-structured answer using **Markdown formatting**
+- Use headers (##, ###) to organize sections
+- Use **bold** for important points
+- Use bullet points (-) or numbered lists (1.) for clarity
+- Use `code blocks` for technical terms or specific data
+- Include relevant details from the context
+- If uncertain, express the level of confidence
+- Do not make up information not present in the context
+
+Format your response in Markdown with proper structure and formatting.
+
+Answer:"""
+
+        return prompt
+
+    def _fallback_generation(self, query: str, context: str) -> Dict[str, Any]:
+        """
+        Fallback response generation when LLM is not available.
+
+        Args:
+            query: User query
+            context: Formatted context
+
+        Returns:
+            Dictionary with generation result
+        """
+        self.logger.info("Using fallback generation")
+
+        # Extract key information from context
+        context_lines = context.split("\n")
+        relevant_lines = [
+            line.strip()
+            for line in context_lines
+            if line.strip() and not line.startswith("[Source:")
+        ]
+
+        if not relevant_lines:
+            return {
+                "success": True,
+                "response": self.response_templates["no_context"],
+                "model": "fallback",
+            }
+
+        # Create a structured markdown response
+        response_parts = [
+            f"## Answer to: {query}",
+            "",
+            "Based on the available information:",
+            "",
+        ]
+
+        # Add key information as markdown list
+        for i, line in enumerate(relevant_lines[:3]):  # Limit to 3 most relevant
+            if len(line) > 50:  # Only include substantial content
+                response_parts.append(f"- {line}")
+
+        response_parts.extend(
+            [
+                "",
+                "---",
+                "",
+                "**Note:** This response is generated using available context. For more detailed analysis, please ensure proper language model integration.",
+            ]
+        )
+
+        response = "\n".join(response_parts)
+
+        return {
+            "success": True,
+            "response": response,
+            "model": "fallback",
+        }
+
+    def _format_context(self, context: List[Dict[str, Any]]) -> str:
+        """
+        Format the retrieved context for use in response generation.
+
+        Args:
+            context: List of context items
+
+        Returns:
+            Formatted context string
+        """
+        formatted_parts = []
+
+        for i, item in enumerate(context):
+            text = item.get("text", "")
+            source = item.get("source", f"Source {i+1}")
+            score = item.get("score", 0.0)
+
+            # Format each context item with metadata
+            formatted_part = f"""[Source {i+1}: {source} (Relevance: {score:.2f})]
+{text}
+---"""
+            formatted_parts.append(formatted_part)
+
+        return "\n\n".join(formatted_parts)
+
+    def _extract_sources(self, context: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Extract source information from context items.
+
+        Args:
+            context: List of context items
+
+        Returns:
+            List of source dictionaries
+        """
+        sources = []
+        seen_sources = set()
+
+        for item in context:
+            source = item.get("source", "Unknown")
+            score = item.get("score", 0.0)
+            final_score = item.get("final_score", score)
+
+            if source not in seen_sources:
+                source_info = {
+                    "source": source,
+                    "relevance_score": round(score, 3),
+                    "final_score": round(final_score, 3),
+                    "metadata": item.get("metadata", {}),
+                }
+
+                # Add source type
+                if source.endswith(".pdf"):
+                    source_info["type"] = "PDF Document"
+                elif source.startswith("http"):
+                    source_info["type"] = "Web Page"
+                elif source.endswith((".docx", ".doc")):
+                    source_info["type"] = "Word Document"
+                else:
+                    source_info["type"] = "Document"
+
+                sources.append(source_info)
+                seen_sources.add(source)
+
+        # Sort by relevance score
+        sources.sort(key=lambda x: x["final_score"], reverse=True)
+        return sources
+
+    def _calculate_confidence(self, context: List[Dict[str, Any]]) -> float:
+        """
+        Calculate confidence score based on context quality.
+
+        Args:
+            context: List of context items
+
+        Returns:
+            Confidence score between 0.0 and 1.0
+        """
+        if not context:
+            return 0.0
+
+        # Calculate average similarity score
+        scores = [item.get("final_score", item.get("score", 0.0)) for item in context]
+        avg_score = sum(scores) / len(scores)
+
+        # Factor in the number of context items
+        context_factor = min(len(context) / 3.0, 1.0)  # Normalize to max of 3 items
+
+        # Factor in score distribution (prefer consistent scores)
+        if len(scores) > 1:
+            score_variance = sum((s - avg_score) ** 2 for s in scores) / len(scores)
+            consistency_factor = max(0.5, 1.0 - score_variance)
+        else:
+            consistency_factor = 1.0
+
+        # Combine factors
+        confidence = (
+            (avg_score * 0.6) + (context_factor * 0.2) + (consistency_factor * 0.2)
+        )
+
+        return min(confidence, 1.0)
+
+    def _assess_response_quality(
+        self, response: str, query: str, context: List[Dict[str, Any]]
+    ) -> float:
+        """
+        Assess the quality of the generated response.
+
+        Args:
+            response: Generated response
+            query: Original query
+            context: Context used for generation
+
+        Returns:
+            Quality score between 0.0 and 1.0
+        """
+        if not response or len(response.strip()) < 10:
+            return 0.1
+
+        quality_score = 0.5  # Base score
+
+        # Check response length (not too short, not too long)
+        response_length = len(response)
+        if 50 <= response_length <= 1000:
+            quality_score += 0.2
+        elif response_length > 1000:
+            quality_score += 0.1
+
+        # Check if response addresses the query
+        query_words = set(query.lower().split())
+        response_words = set(response.lower().split())
+        word_overlap = len(query_words.intersection(response_words))
+        if word_overlap > 0:
+            quality_score += min(word_overlap / len(query_words), 0.2)
+
+        # Check if response uses context information
+        context_texts = [item.get("text", "") for item in context]
+        context_words = set()
+        for text in context_texts:
+            context_words.update(text.lower().split())
+
+        context_usage = len(response_words.intersection(context_words))
+        if context_usage > 5:  # Uses substantial context
+            quality_score += 0.1
+
+        return min(quality_score, 1.0)
+
+    def get_supported_models(self) -> List[str]:
+        """
+        Get list of supported models.
+
+        Returns:
+            List of available model names
+        """
+        models = ["fallback"]
+
+        if self.gemini_client:
+            models.extend(["gemini-2.5-flash-preview-05-20", "gemini-1.5-pro"])
+
+        if self.openai_client:
+            models.extend(["gpt-3.5-turbo", "gpt-4", "gpt-4-turbo"])
+
+        return models
+
+    def update_model(self, model_name: str) -> bool:
+        """
+        Update the model used for generation.
+
+        Args:
+            model_name: Name of the model to use
+
+        Returns:
+            True if model was updated successfully
+        """
+        try:
+            if model_name in self.get_supported_models():
+                self.model = model_name
+                self.logger.info(f"Model updated to: {model_name}")
+                return True
+            else:
+                self.logger.warning(f"Model {model_name} not supported")
+                return False
+        except Exception as e:
+            self.logger.error(f"❌ Error updating model: {e}")
+            return False
+
+    def get_generation_stats(self) -> Dict[str, Any]:
+        """
+        Get statistics about response generation.
+
+        Returns:
+            Dictionary with generation statistics
+        """
+        return {
+            "supported_models": self.get_supported_models(),
+            "current_model": self.model,
+            "gemini_available": self.gemini_client is not None,
+            "openai_available": self.openai_client is not None,
+            "langchain_available": self.llm is not None,
+            "max_tokens": self.max_tokens,
+            "temperature": self.temperature,
+        }

src/storage/__init__.py

@@ -0,0 +1,6 @@
+"""
+Storage module for vector database operations.
+
+This module contains components for storing and retrieving
+vector embeddings using Pinecone.
+"""

src/storage/vector_db.py

@@ -0,0 +1,729 @@
+"""
+Vector Database Module
+
+This module is responsible for storing and indexing vector embeddings
+for efficient retrieval using Pinecone with complete functionality.
+
+Technology: Pinecone
+"""
+
+import logging
+import os
+import time
+import uuid
+import hashlib
+from datetime import datetime
+from typing import Dict, List, Any, Optional, Union
+
+# Import Pinecone and related libraries
+try:
+    import pinecone
+    from pinecone import Pinecone, ServerlessSpec
+except ImportError as e:
+    logging.warning(f"Pinecone library not installed: {e}")
+
+from utils.error_handler import VectorStorageError, error_handler, ErrorType
+
+
+class VectorDB:
+    """
+    Stores and indexes vector embeddings for efficient retrieval using Pinecone with full functionality.
+
+    Features:
+    - Complete Pinecone integration
+    - Index management (create, update, delete)
+    - Batch upsert operations with optimization
+    - Advanced similarity search with metadata filtering
+    - Statistics and monitoring
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the VectorDB with configuration.
+
+        Args:
+            config: Configuration dictionary with Pinecone parameters
+        """
+        self.config = config or {}
+        self.logger = logging.getLogger(__name__)
+
+        # Configuration settings
+        self.api_key = self.config.get("api_key", os.environ.get("PINECONE_API_KEY"))
+        self.environment = self.config.get("environment", "us-west1-gcp")
+        self.index_name = self.config.get("index_name", "rag-ai-index")
+        self.dimension = self.config.get(
+            "dimension", 3072
+        )  # ✅ Fixed: Match Gemini embedding dimension
+        self.metric = self.config.get("metric", "cosine")
+        self.batch_size = self.config.get("batch_size", 100)
+
+        # Performance settings
+        self.max_metadata_size = self.config.get(
+            "max_metadata_size", 40960
+        )  # 40KB limit
+        self.upsert_timeout = self.config.get("upsert_timeout", 60)
+        self.query_timeout = self.config.get("query_timeout", 30)
+
+        # Statistics tracking
+        self.stats = {
+            "vectors_stored": 0,
+            "vectors_queried": 0,
+            "vectors_deleted": 0,
+            "batch_operations": 0,
+            "failed_operations": 0,
+            "start_time": datetime.now(),
+        }
+
+        # Initialize Pinecone client
+        self.pc = None
+        self.index = None
+        self._initialize_client()
+
+    def _initialize_client(self):
+        """Initialize Pinecone client and index with validation."""
+        if not self.api_key:
+            self.logger.warning(
+                "No Pinecone API key provided. Vector storage will not be available."
+            )
+            return
+
+        try:
+            # Initialize Pinecone client
+            self.pc = Pinecone(api_key=self.api_key)
+
+            # Check if index exists, create if not
+            self._ensure_index_exists()
+
+            # Connect to index
+            self.index = self.pc.Index(self.index_name)
+
+            # Test connection
+            self._test_connection()
+
+            self.logger.info(
+                f"Pinecone client initialized successfully with index: {self.index_name}"
+            )
+
+        except Exception as e:
+            self.logger.error(f" Failed to initialize Pinecone client: {str(e)}")
+            self.pc = None
+            self.index = None
+
+    def _ensure_index_exists(self):
+        """Ensure the Pinecone index exists, create if necessary."""
+        try:
+            # List existing indexes
+            existing_indexes = [index.name for index in self.pc.list_indexes()]
+
+            if self.index_name not in existing_indexes:
+                self.logger.info(f"Creating new Pinecone index: {self.index_name}")
+
+                # Create index with serverless spec
+                self.pc.create_index(
+                    name=self.index_name,
+                    dimension=self.dimension,
+                    metric=self.metric,
+                    spec=ServerlessSpec(cloud="aws", region=self.environment),
+                )
+
+                # Wait for index to be ready
+                self._wait_for_index_ready()
+
+                self.logger.info(f"Index {self.index_name} created successfully")
+            else:
+                self.logger.info(f"Index {self.index_name} already exists")
+
+        except Exception as e:
+            raise VectorStorageError(f"Failed to ensure index exists: {str(e)}")
+
+    def _wait_for_index_ready(self, max_wait_time: int = 300):
+        """Wait for index to be ready for operations."""
+        start_time = time.time()
+
+        while time.time() - start_time < max_wait_time:
+            try:
+                index_stats = self.pc.describe_index(self.index_name)
+                if index_stats.status.ready:
+                    self.logger.info(f"Index {self.index_name} is ready")
+                    return
+
+                self.logger.info(f"Waiting for index to be ready...")
+                time.sleep(10)
+
+            except Exception as e:
+                self.logger.warning(f"Error checking index status: {str(e)}")
+                time.sleep(5)
+
+        raise VectorStorageError(
+            f"Index {self.index_name} not ready after {max_wait_time}s"
+        )
+
+    def _test_connection(self):
+        """Test connection to Pinecone index."""
+        try:
+            # Get index stats
+            stats = self.index.describe_index_stats()
+            self.logger.info(f"Connection test successful. Index stats: {stats}")
+
+        except Exception as e:
+            raise VectorStorageError(f"Connection test failed: {str(e)}")
+
+    @error_handler(ErrorType.VECTOR_STORAGE)
+    def store_embeddings(self, items: List[Dict[str, Any]]) -> bool:
+        """
+        Store embeddings in the vector database with full functionality.
+
+        Args:
+            items: List of dictionaries containing content, metadata, and embeddings
+
+        Returns:
+            True if successful, False otherwise
+        """
+        if not self.index or not items:
+            self.logger.warning("No index available or empty items list")
+            return False
+
+        # Filter and validate items
+        valid_items = self._validate_items(items)
+        if not valid_items:
+            self.logger.warning("No valid embeddings to store")
+            return False
+
+        self.logger.info(f"Storing {len(valid_items)} embeddings in Pinecone")
+        start_time = time.time()
+
+        try:
+            # Process in batches
+            total_batches = (len(valid_items) + self.batch_size - 1) // self.batch_size
+            successful_batches = 0
+
+            for i in range(0, len(valid_items), self.batch_size):
+                batch_num = (i // self.batch_size) + 1
+                batch = valid_items[i : i + self.batch_size]
+
+                self.logger.info(
+                    f"Processing batch {batch_num}/{total_batches} ({len(batch)} vectors)"
+                )
+
+                success = self._store_batch(batch)
+                if success:
+                    successful_batches += 1
+                    self.stats["vectors_stored"] += len(batch)
+                else:
+                    self.stats["failed_operations"] += 1
+                    self.logger.error(f" Batch {batch_num} failed")
+
+                # Rate limiting between batches
+                if i + self.batch_size < len(valid_items):
+                    time.sleep(0.1)
+
+            self.stats["batch_operations"] += total_batches
+            processing_time = time.time() - start_time
+
+            success_rate = successful_batches / total_batches * 100
+            self.logger.info(
+                f"Storage completed: {successful_batches}/{total_batches} batches successful ({success_rate:.1f}%) in {processing_time:.2f}s"
+            )
+
+            return successful_batches > 0
+
+        except Exception as e:
+            self.stats["failed_operations"] += 1
+            raise VectorStorageError(f"Failed to store embeddings: {str(e)}")
+
+    def _validate_items(self, items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Validate and prepare items for storage.
+
+        Args:
+            items: List of items to validate
+
+        Returns:
+            List of valid items
+        """
+        valid_items = []
+
+        for i, item in enumerate(items):
+            try:
+                # Check required fields
+                if not isinstance(item, dict):
+                    self.logger.warning(f"Item {i} is not a dictionary")
+                    continue
+
+                if "embedding" not in item or not item["embedding"]:
+                    self.logger.warning(f"Item {i} missing embedding")
+                    continue
+
+                embedding = item["embedding"]
+                if not isinstance(embedding, list) or len(embedding) != self.dimension:
+                    self.logger.warning(
+                        f"Item {i} has invalid embedding dimension: {len(embedding)} != {self.dimension}"
+                    )
+                    continue
+
+                # Prepare item
+                processed_item = self._prepare_item_for_storage(item, i)
+                valid_items.append(processed_item)
+
+            except Exception as e:
+                self.logger.warning(f"Error validating item {i}: {str(e)}")
+                continue
+
+        return valid_items
+
+    def _prepare_item_for_storage(
+        self, item: Dict[str, Any], index: int
+    ) -> Dict[str, Any]:
+        """
+        Prepare item for Pinecone storage.
+
+        Args:
+            item: Item to prepare
+            index: Item index for ID generation
+
+        Returns:
+            Prepared item
+        """
+        # 🆔 Generate unique ID
+        item_id = item.get("id")
+        if not item_id:
+            # Create ID from content hash + timestamp
+            content = item.get("content", "")
+            timestamp = str(int(time.time() * 1000))
+            content_hash = hashlib.md5(content.encode()).hexdigest()[:8]
+            item_id = f"doc_{content_hash}_{timestamp}_{index}"
+
+        # Prepare metadata
+        metadata = item.get("metadata", {}).copy()
+
+        # Add essential fields to metadata
+        metadata.update(
+            {
+                "content_preview": item.get("content", "")[:500],  # First 500 chars
+                "content_length": len(item.get("content", "")),
+                "stored_at": datetime.now().isoformat(),
+                "source": item.get("source", "unknown"),
+                "document_type": item.get("document_type", "text"),
+            }
+        )
+
+        # Ensure metadata size limit
+        metadata = self._truncate_metadata(metadata)
+
+        return {"id": item_id, "values": item["embedding"], "metadata": metadata}
+
+    def _truncate_metadata(self, metadata: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Truncate metadata to fit Pinecone size limits.
+
+        Args:
+            metadata: Original metadata
+
+        Returns:
+            Truncated metadata
+        """
+        import json
+
+        # 📏 Check current size
+        metadata_str = json.dumps(metadata, default=str)
+        if len(metadata_str.encode()) <= self.max_metadata_size:
+            return metadata
+
+        # Truncate large fields
+        truncated = metadata.copy()
+
+        # Truncate text fields progressively
+        text_fields = ["content_preview", "text", "description", "summary"]
+        for field in text_fields:
+            if field in truncated:
+                while (
+                    len(json.dumps(truncated, default=str).encode())
+                    > self.max_metadata_size
+                ):
+                    current_length = len(str(truncated[field]))
+                    if current_length <= 50:
+                        break
+                    truncated[field] = (
+                        str(truncated[field])[: current_length // 2] + "..."
+                    )
+
+        return truncated
+
+    def _store_batch(self, batch: List[Dict[str, Any]]) -> bool:
+        """
+        Store a batch of embeddings in Pinecone.
+
+        Args:
+            batch: List of prepared items
+
+        Returns:
+            True if successful
+        """
+        try:
+            # Upsert vectors to Pinecone
+            upsert_response = self.index.upsert(
+                vectors=batch, timeout=self.upsert_timeout
+            )
+
+            # Verify upsert success
+            if hasattr(upsert_response, "upserted_count"):
+                expected_count = len(batch)
+                actual_count = upsert_response.upserted_count
+
+                if actual_count != expected_count:
+                    self.logger.warning(
+                        f"Upsert count mismatch: {actual_count}/{expected_count}"
+                    )
+                    return False
+
+            self.logger.info(f"Successfully stored batch of {len(batch)} vectors")
+            return True
+
+        except Exception as e:
+            self.logger.error(f" Error storing batch: {str(e)}")
+            return False
+
+    @error_handler(ErrorType.VECTOR_STORAGE)
+    def search(
+        self,
+        query_embedding: List[float],
+        top_k: int = 5,
+        filter: Optional[Dict[str, Any]] = None,
+        include_metadata: bool = True,
+        include_values: bool = False,
+    ) -> List[Dict[str, Any]]:
+        """
+        Search for similar vectors with advanced filtering.
+
+        Args:
+            query_embedding: Query vector to search for
+            top_k: Number of results to return
+            filter: Optional metadata filter
+            include_metadata: Whether to include metadata in results
+            include_values: Whether to include vector values in results
+
+        Returns:
+            List of search results with scores and metadata
+        """
+        if not self.index or not query_embedding:
+            self.logger.warning("No index available or empty query embedding")
+            return []
+
+        # Validate query embedding
+        if len(query_embedding) != self.dimension:
+            raise VectorStorageError(
+                f"Query embedding dimension {len(query_embedding)} != {self.dimension}"
+            )
+
+        self.logger.info(f"Searching for similar vectors (top_k={top_k})")
+        start_time = time.time()
+
+        try:
+            # Perform similarity search
+            search_response = self.index.query(
+                vector=query_embedding,
+                top_k=top_k,
+                filter=filter,
+                include_metadata=include_metadata,
+                include_values=include_values,
+                timeout=self.query_timeout,
+            )
+
+            # Process results
+            results = []
+            if hasattr(search_response, "matches"):
+                for match in search_response.matches:
+                    result = {
+                        "id": match.id,
+                        "score": float(match.score),
+                    }
+
+                    if include_metadata and hasattr(match, "metadata"):
+                        result["metadata"] = (
+                            dict(match.metadata) if match.metadata else {}
+                        )
+
+                    if include_values and hasattr(match, "values"):
+                        result["values"] = match.values
+
+                    results.append(result)
+
+            self.stats["vectors_queried"] += len(results)
+            search_time = time.time() - start_time
+
+            self.logger.info(
+                f"Search completed: {len(results)} results in {search_time:.3f}s"
+            )
+            return results
+
+        except Exception as e:
+            self.stats["failed_operations"] += 1
+            raise VectorStorageError(f"Search failed: {str(e)}")
+
+    @error_handler(ErrorType.VECTOR_STORAGE)
+    def delete(
+        self,
+        ids: Optional[List[str]] = None,
+        filter: Optional[Dict[str, Any]] = None,
+        delete_all: bool = False,
+    ) -> bool:
+        """
+        Delete vectors from the database.
+
+        Args:
+            ids: Optional list of vector IDs to delete
+            filter: Optional metadata filter for vectors to delete
+            delete_all: Whether to delete all vectors
+
+        Returns:
+            True if successful
+        """
+        if not self.index:
+            self.logger.warning("No index available")
+            return False
+
+        try:
+            if delete_all:
+                # Delete all vectors
+                self.index.delete(delete_all=True)
+                self.logger.info("Deleted all vectors from index")
+                self.stats["vectors_deleted"] += 1  # Approximate
+
+            elif ids:
+                # Delete by IDs
+                self.index.delete(ids=ids)
+                self.logger.info(f"Deleted {len(ids)} vectors by ID")
+                self.stats["vectors_deleted"] += len(ids)
+
+            elif filter:
+                # Delete by filter
+                self.index.delete(filter=filter)
+                self.logger.info(f"Deleted vectors by filter: {filter}")
+                self.stats["vectors_deleted"] += 1  # Approximate
+
+            else:
+                self.logger.warning("No deletion criteria provided")
+                return False
+
+            return True
+
+        except Exception as e:
+            self.stats["failed_operations"] += 1
+            raise VectorStorageError(f"Delete operation failed: {str(e)}")
+
+    def get_index_stats(self) -> Dict[str, Any]:
+        """
+        Get comprehensive index statistics.
+
+        Returns:
+            Dictionary with index statistics
+        """
+        if not self.index:
+            return {}
+
+        try:
+            # Get Pinecone index stats
+            pinecone_stats = self.index.describe_index_stats()
+
+            # Combine with internal stats
+            runtime = datetime.now() - self.stats["start_time"]
+
+            return {
+                "pinecone_stats": {
+                    "total_vector_count": pinecone_stats.total_vector_count,
+                    "dimension": pinecone_stats.dimension,
+                    "index_fullness": pinecone_stats.index_fullness,
+                    "namespaces": (
+                        dict(pinecone_stats.namespaces)
+                        if pinecone_stats.namespaces
+                        else {}
+                    ),
+                },
+                "internal_stats": {
+                    **self.stats,
+                    "runtime_seconds": runtime.total_seconds(),
+                    "avg_vectors_per_batch": (
+                        self.stats["vectors_stored"]
+                        / max(1, self.stats["batch_operations"])
+                    ),
+                    "success_rate": (
+                        (
+                            self.stats["batch_operations"]
+                            - self.stats["failed_operations"]
+                        )
+                        / max(1, self.stats["batch_operations"])
+                        * 100
+                    ),
+                },
+                "configuration": {
+                    "index_name": self.index_name,
+                    "dimension": self.dimension,
+                    "metric": self.metric,
+                    "batch_size": self.batch_size,
+                },
+            }
+
+        except Exception as e:
+            self.logger.error(f" Error getting index stats: {str(e)}")
+            return {"error": str(e)}
+
+    def health_check(self) -> Dict[str, Any]:
+        """
+        Perform health check on the vector database.
+
+        Returns:
+            Health check results
+        """
+        health = {
+            "status": "unknown",
+            "timestamp": datetime.now().isoformat(),
+            "checks": {},
+        }
+
+        try:
+            # Check API connection
+            if self.pc:
+                health["checks"]["api_connection"] = "Connected"
+            else:
+                health["checks"]["api_connection"] = " Not connected"
+                health["status"] = "unhealthy"
+                return health
+
+            # Check index availability
+            if self.index:
+                health["checks"]["index_available"] = "Available"
+            else:
+                health["checks"]["index_available"] = " Not available"
+                health["status"] = "unhealthy"
+                return health
+
+            # Test query operation
+            try:
+                test_vector = [0.1] * self.dimension
+                self.index.query(vector=test_vector, top_k=1, timeout=5)
+                health["checks"]["query_operation"] = "Working"
+            except Exception as e:
+                health["checks"]["query_operation"] = f" Failed: {str(e)}"
+                health["status"] = "degraded"
+
+            # Check index stats
+            try:
+                stats = self.index.describe_index_stats()
+                health["checks"]["index_stats"] = f"{stats.total_vector_count} vectors"
+            except Exception as e:
+                health["checks"]["index_stats"] = f" Failed: {str(e)}"
+
+            # 🎯 Overall status
+            if health["status"] == "unknown":
+                health["status"] = "healthy"
+
+        except Exception as e:
+            health["status"] = "unhealthy"
+            health["error"] = str(e)
+
+        return health
+
+    def reset_stats(self):
+        """Reset internal statistics."""
+        self.stats = {
+            "vectors_stored": 0,
+            "vectors_queried": 0,
+            "vectors_deleted": 0,
+            "batch_operations": 0,
+            "failed_operations": 0,
+            "start_time": datetime.now(),
+        }
+        self.logger.info("Statistics reset")
+
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get simplified stats for UI display.
+
+        Returns:
+            Dictionary with basic statistics
+        """
+        try:
+            if not self.index:
+                return {"total_vectors": 0, "status": "disconnected"}
+
+            # Get Pinecone stats
+            pinecone_stats = self.index.describe_index_stats()
+
+            return {
+                "total_vectors": pinecone_stats.total_vector_count,
+                "dimension": pinecone_stats.dimension,
+                "index_fullness": pinecone_stats.index_fullness,
+                "status": "connected",
+            }
+        except Exception as e:
+            self.logger.warning(f"Could not get stats: {e}")
+            return {"total_vectors": 0, "status": "error", "error": str(e)}
+
+    def get_unique_sources(self) -> List[Dict[str, Any]]:
+        """
+        Get unique sources from stored vectors.
+
+        Returns:
+            List of unique sources with metadata
+        """
+        try:
+            if not self.index:
+                return []
+
+            # This is a simplified approach - in a real implementation,
+            # you might want to maintain a separate metadata index
+            # For now, we'll return mock data based on what might be stored
+
+            # Try to get some sample vectors to extract sources
+            test_vector = [0.1] * self.dimension
+            results = self.index.query(
+                vector=test_vector,
+                top_k=100,  # Get more results to find unique sources
+                include_metadata=True,
+            )
+
+            sources = {}
+            for match in results.matches:
+                if hasattr(match, "metadata") and match.metadata:
+                    source = match.metadata.get("source", "Unknown")
+                    if source not in sources:
+                        sources[source] = {
+                            "source": source,
+                            "chunk_count": 1,
+                            "added_date": match.metadata.get("stored_at", "Unknown"),
+                        }
+                    else:
+                        sources[source]["chunk_count"] += 1
+
+            return list(sources.values())
+
+        except Exception as e:
+            self.logger.warning(f"Could not get unique sources: {e}")
+            return []
+
+    def list_documents(self) -> List[Dict[str, Any]]:
+        """
+        List all documents in the vector database.
+
+        Returns:
+            List of document information
+        """
+        try:
+            # Get unique sources and format as documents
+            sources = self.get_unique_sources()
+            documents = []
+
+            for source_info in sources:
+                documents.append(
+                    {
+                        "name": source_info["source"],
+                        "chunks": source_info["chunk_count"],
+                        "date": source_info["added_date"],
+                    }
+                )
+
+            return documents
+
+        except Exception as e:
+            self.logger.warning(f"Could not list documents: {e}")
+            return []

src/ui/__init__.py

@@ -0,0 +1,6 @@
+"""
+UI module for the Gradio user interface.
+
+This module contains components for providing an intuitive
+interface for document upload, URL input, and querying.
+"""

src/utils/__init__.py

@@ -0,0 +1,6 @@
+"""
+Utils module for configuration management and error handling.
+
+This module contains utility components for managing configuration
+and handling errors throughout the application.
+"""

src/utils/config_manager.py

@@ -0,0 +1,279 @@
+"""
+Configuration Manager Module
+
+This module handles loading and managing configuration settings
+for the RAG AI system.
+"""
+
+import os
+import yaml
+import logging
+from typing import Dict, Any, Optional
+from pathlib import Path
+
+
+class ConfigManager:
+    """
+    Manages configuration settings for the RAG AI system.
+
+    Features:
+    - YAML configuration file loading
+    - Environment variable override
+    - Default configuration values
+    - Configuration validation
+    """
+
+    def __init__(self, config_path: Optional[str] = None):
+        """
+        Initialize the ConfigManager.
+
+        Args:
+            config_path: Path to the configuration file (defaults to config/config.yaml)
+        """
+        self.logger = logging.getLogger(__name__)
+
+        # Set default config path
+        if config_path is None:
+            config_path = os.path.join(
+                os.path.dirname(__file__), "..", "..", "config", "config.yaml"
+            )
+
+        self.config_path = Path(config_path)
+        self.config = self._load_config()
+
+    def _load_config(self) -> Dict[str, Any]:
+        """
+        Load configuration from file and environment variables.
+
+        Returns:
+            Configuration dictionary
+        """
+        # Start with default configuration
+        config = self._get_default_config()
+
+        # Load from YAML file if it exists
+        if self.config_path.exists():
+            try:
+                with open(self.config_path, "r", encoding="utf-8") as f:
+                    file_config = yaml.safe_load(f) or {}
+                    config = self._merge_configs(config, file_config)
+                self.logger.info(f"Loaded configuration from {self.config_path}")
+            except Exception as e:
+                self.logger.warning(
+                    f"Failed to load config file {self.config_path}: {str(e)}"
+                )
+        else:
+            self.logger.warning(f"Config file not found: {self.config_path}")
+
+        # Override with environment variables
+        config = self._apply_env_overrides(config)
+
+        # Validate configuration
+        self._validate_config(config)
+
+        return config
+
+    def _get_default_config(self) -> Dict[str, Any]:
+        """
+        Get default configuration values.
+
+        Returns:
+            Default configuration dictionary
+        """
+        return {
+            "api_keys": {
+                "gemini_api_key": "",
+                "pinecone_api_key": "",
+                "openai_api_key": "",
+            },
+            "vector_db": {
+                "provider": "pinecone",
+                "index_name": "rag-ai-index",
+                "dimension": 3072,  # ✅ Fixed: Match Gemini embedding dimension
+                "metric": "cosine",
+                "environment": "us-west1-gcp",
+            },
+            "embedding": {
+                "model": "gemini-embedding-exp-03-07",
+                "batch_size": 5,
+                "max_retries": 3,
+                "retry_delay": 1,
+            },
+            "document_processing": {
+                "chunk_size": 1000,
+                "chunk_overlap": 200,
+                "min_chunk_size": 100,
+                "max_file_size_mb": 50,
+            },
+            "url_processing": {
+                "max_depth": 1,
+                "follow_links": True,
+                "max_pages": 10,
+                "timeout": 10,
+            },
+            "rag": {
+                "top_k": 5,
+                "similarity_threshold": 0.7,
+                "max_context_length": 4000,
+                "model": "gpt-3.5-turbo",
+                "max_tokens": 500,
+                "temperature": 0.7,
+            },
+            "ui": {
+                "title": "AI Embedded Knowledge Agent",
+                "description": "Upload documents or provide URLs to build your knowledge base, then ask questions!",
+                "theme": "default",
+                "share": False,
+                "port": 7860,
+            },
+            "logging": {
+                "level": "INFO",
+                "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            },
+        }
+
+    def _merge_configs(
+        self, base: Dict[str, Any], override: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        Recursively merge two configuration dictionaries.
+
+        Args:
+            base: Base configuration dictionary
+            override: Override configuration dictionary
+
+        Returns:
+            Merged configuration dictionary
+        """
+        result = base.copy()
+
+        for key, value in override.items():
+            if (
+                key in result
+                and isinstance(result[key], dict)
+                and isinstance(value, dict)
+            ):
+                result[key] = self._merge_configs(result[key], value)
+            else:
+                result[key] = value
+
+        return result
+
+    def _apply_env_overrides(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Apply environment variable overrides to configuration.
+
+        Args:
+            config: Configuration dictionary
+
+        Returns:
+            Configuration with environment overrides applied
+        """
+        # API Keys
+        if os.environ.get("GEMINI_API_KEY"):
+            config["api_keys"]["gemini_api_key"] = os.environ["GEMINI_API_KEY"]
+
+        if os.environ.get("PINECONE_API_KEY"):
+            config["api_keys"]["pinecone_api_key"] = os.environ["PINECONE_API_KEY"]
+
+        if os.environ.get("OPENAI_API_KEY"):
+            config["api_keys"]["openai_api_key"] = os.environ["OPENAI_API_KEY"]
+
+        # Pinecone settings
+        if os.environ.get("PINECONE_ENVIRONMENT"):
+            config["vector_db"]["environment"] = os.environ["PINECONE_ENVIRONMENT"]
+
+        if os.environ.get("PINECONE_INDEX_NAME"):
+            config["vector_db"]["index_name"] = os.environ["PINECONE_INDEX_NAME"]
+
+        # UI settings
+        if os.environ.get("GRADIO_SHARE"):
+            config["ui"]["share"] = os.environ["GRADIO_SHARE"].lower() == "true"
+
+        if os.environ.get("PORT"):
+            try:
+                config["ui"]["port"] = int(os.environ["PORT"])
+            except ValueError:
+                self.logger.warning(f"Invalid PORT value: {os.environ['PORT']}")
+
+        return config
+
+    def _validate_config(self, config: Dict[str, Any]) -> None:
+        """
+        Validate configuration values.
+
+        Args:
+            config: Configuration dictionary to validate
+        """
+        # Check required API keys
+        if not config["api_keys"]["gemini_api_key"]:
+            self.logger.warning("Gemini API key not configured")
+
+        if not config["api_keys"]["pinecone_api_key"]:
+            self.logger.warning("Pinecone API key not configured")
+
+        # Validate numeric values
+        if config["document_processing"]["chunk_size"] <= 0:
+            raise ValueError("chunk_size must be positive")
+
+        if config["rag"]["top_k"] <= 0:
+            raise ValueError("top_k must be positive")
+
+        if not 0 <= config["rag"]["similarity_threshold"] <= 1:
+            raise ValueError("similarity_threshold must be between 0 and 1")
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """
+        Get a configuration value using dot notation.
+
+        Args:
+            key: Configuration key (e.g., 'vector_db.index_name')
+            default: Default value if key not found
+
+        Returns:
+            Configuration value
+        """
+        keys = key.split(".")
+        value = self.config
+
+        try:
+            for k in keys:
+                value = value[k]
+            return value
+        except (KeyError, TypeError):
+            return default
+
+    def set(self, key: str, value: Any) -> None:
+        """
+        Set a configuration value using dot notation.
+
+        Args:
+            key: Configuration key (e.g., 'vector_db.index_name')
+            value: Value to set
+        """
+        keys = key.split(".")
+        config = self.config
+
+        for k in keys[:-1]:
+            if k not in config:
+                config[k] = {}
+            config = config[k]
+
+        config[keys[-1]] = value
+
+    def get_section(self, section: str) -> Dict[str, Any]:
+        """
+        Get an entire configuration section.
+
+        Args:
+            section: Section name
+
+        Returns:
+            Configuration section dictionary
+        """
+        return self.config.get(section, {})
+
+    def reload(self) -> None:
+        """Reload configuration from file."""
+        self.config = self._load_config()
+        self.logger.info("Configuration reloaded")

src/utils/error_handler.py

@@ -0,0 +1,383 @@
+"""
+Error Handler Module
+
+This module provides centralized error handling and logging
+for the RAG AI system.
+"""
+
+import logging
+import traceback
+import functools
+from typing import Any, Callable, Dict, Optional, Type, Union
+from enum import Enum
+
+
+class ErrorType(Enum):
+    """Enumeration of error types in the system."""
+
+    DOCUMENT_PROCESSING = "document_processing"
+    URL_PROCESSING = "url_processing"
+    EMBEDDING_GENERATION = "embedding_generation"
+    VECTOR_STORAGE = "vector_storage"
+    QUERY_PROCESSING = "query_processing"
+    RESPONSE_GENERATION = "response_generation"
+    API_ERROR = "api_error"
+    CONFIGURATION = "configuration"
+    UI_ERROR = "ui_error"
+    UNKNOWN = "unknown"
+
+
+class RAGError(Exception):
+    """Base exception class for RAG AI system errors."""
+
+    def __init__(
+        self,
+        message: str,
+        error_type: ErrorType = ErrorType.UNKNOWN,
+        details: Optional[Dict[str, Any]] = None,
+    ):
+        """
+        Initialize RAGError.
+
+        Args:
+            message: Error message
+            error_type: Type of error
+            details: Additional error details
+        """
+        super().__init__(message)
+        self.error_type = error_type
+        self.details = details or {}
+        self.message = message
+
+
+class DocumentProcessingError(RAGError):
+    """Exception for document processing errors."""
+
+    def __init__(
+        self,
+        message: str,
+        file_path: Optional[str] = None,
+        details: Optional[Dict[str, Any]] = None,
+    ):
+        details = details or {}
+        if file_path:
+            details["file_path"] = file_path
+        super().__init__(message, ErrorType.DOCUMENT_PROCESSING, details)
+
+
+class URLProcessingError(RAGError):
+    """Exception for URL processing errors."""
+
+    def __init__(
+        self,
+        message: str,
+        url: Optional[str] = None,
+        details: Optional[Dict[str, Any]] = None,
+    ):
+        details = details or {}
+        if url:
+            details["url"] = url
+        super().__init__(message, ErrorType.URL_PROCESSING, details)
+
+
+class EmbeddingError(RAGError):
+    """Exception for embedding generation errors."""
+
+    def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
+        super().__init__(message, ErrorType.EMBEDDING_GENERATION, details)
+
+
+class VectorStorageError(RAGError):
+    """Exception for vector storage errors."""
+
+    def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
+        super().__init__(message, ErrorType.VECTOR_STORAGE, details)
+
+
+class QueryProcessingError(RAGError):
+    """Exception for query processing errors."""
+
+    def __init__(
+        self,
+        message: str,
+        query: Optional[str] = None,
+        details: Optional[Dict[str, Any]] = None,
+    ):
+        details = details or {}
+        if query:
+            details["query"] = query
+        super().__init__(message, ErrorType.QUERY_PROCESSING, details)
+
+
+class ResponseGenerationError(RAGError):
+    """Exception for response generation errors."""
+
+    def __init__(self, message: str, details: Optional[Dict[str, Any]] = None):
+        super().__init__(message, ErrorType.RESPONSE_GENERATION, details)
+
+
+class APIError(RAGError):
+    """Exception for API-related errors."""
+
+    def __init__(
+        self,
+        message: str,
+        api_name: Optional[str] = None,
+        status_code: Optional[int] = None,
+        details: Optional[Dict[str, Any]] = None,
+    ):
+        details = details or {}
+        if api_name:
+            details["api_name"] = api_name
+        if status_code:
+            details["status_code"] = status_code
+        super().__init__(message, ErrorType.API_ERROR, details)
+
+
+class ConfigurationError(RAGError):
+    """Exception for configuration errors."""
+
+    def __init__(
+        self,
+        message: str,
+        config_key: Optional[str] = None,
+        details: Optional[Dict[str, Any]] = None,
+    ):
+        details = details or {}
+        if config_key:
+            details["config_key"] = config_key
+        super().__init__(message, ErrorType.CONFIGURATION, details)
+
+
+class ErrorHandler:
+    """
+    Centralized error handler for the RAG AI system.
+
+    Features:
+    - Error logging with context
+    - Error categorization
+    - Error recovery suggestions
+    - Performance monitoring
+    """
+
+    def __init__(self, logger_name: str = __name__):
+        """
+        Initialize the ErrorHandler.
+
+        Args:
+            logger_name: Name for the logger instance
+        """
+        self.logger = logging.getLogger(logger_name)
+        self.error_counts = {}
+
+    def handle_error(
+        self, error: Exception, context: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        """
+        Handle an error with logging and context.
+
+        Args:
+            error: The exception that occurred
+            context: Additional context information
+
+        Returns:
+            Dictionary containing error information
+        """
+        context = context or {}
+
+        # Determine error type
+        if isinstance(error, RAGError):
+            error_type = error.error_type
+            error_details = error.details
+        else:
+            error_type = ErrorType.UNKNOWN
+            error_details = {}
+
+        # Create error info
+        error_info = {
+            "type": error_type.value,
+            "message": str(error),
+            "details": error_details,
+            "context": context,
+            "traceback": traceback.format_exc(),
+        }
+
+        # Log the error
+        self._log_error(error_info)
+
+        # Update error counts
+        self._update_error_counts(error_type)
+
+        # Add recovery suggestions
+        error_info["recovery_suggestions"] = self._get_recovery_suggestions(error_type)
+
+        return error_info
+
+    def _log_error(self, error_info: Dict[str, Any]) -> None:
+        """
+        Log error information.
+
+        Args:
+            error_info: Error information dictionary
+        """
+        error_type = error_info["type"]
+        message = error_info["message"]
+        context = error_info.get("context", {})
+
+        log_message = f"[{error_type.upper()}] {message}"
+
+        if context:
+            context_str = ", ".join([f"{k}={v}" for k, v in context.items()])
+            log_message += f" | Context: {context_str}"
+
+        self.logger.error(log_message)
+
+        # Log traceback at debug level
+        if error_info.get("traceback"):
+            self.logger.debug(f"Traceback: {error_info['traceback']}")
+
+    def _update_error_counts(self, error_type: ErrorType) -> None:
+        """
+        Update error count statistics.
+
+        Args:
+            error_type: Type of error that occurred
+        """
+        if error_type not in self.error_counts:
+            self.error_counts[error_type] = 0
+        self.error_counts[error_type] += 1
+
+    def _get_recovery_suggestions(self, error_type: ErrorType) -> list:
+        """
+        Get recovery suggestions for an error type.
+
+        Args:
+            error_type: Type of error
+
+        Returns:
+            List of recovery suggestions
+        """
+        suggestions = {
+            ErrorType.DOCUMENT_PROCESSING: [
+                "Check if the document format is supported",
+                "Verify the document is not corrupted",
+                "Ensure sufficient disk space for processing",
+            ],
+            ErrorType.URL_PROCESSING: [
+                "Verify the URL is accessible",
+                "Check internet connectivity",
+                "Ensure the website allows scraping",
+            ],
+            ErrorType.EMBEDDING_GENERATION: [
+                "Check Gemini API key configuration",
+                "Verify API quota and rate limits",
+                "Ensure text content is not empty",
+            ],
+            ErrorType.VECTOR_STORAGE: [
+                "Check Pinecone API key configuration",
+                "Verify Pinecone index exists",
+                "Check vector dimensions match index configuration",
+            ],
+            ErrorType.QUERY_PROCESSING: [
+                "Ensure query is not empty",
+                "Check if knowledge base has content",
+                "Verify embedding generation is working",
+            ],
+            ErrorType.RESPONSE_GENERATION: [
+                "Check language model configuration",
+                "Verify retrieved context is valid",
+                "Ensure API keys are configured",
+            ],
+            ErrorType.API_ERROR: [
+                "Check API key validity",
+                "Verify network connectivity",
+                "Check API rate limits and quotas",
+            ],
+            ErrorType.CONFIGURATION: [
+                "Check configuration file syntax",
+                "Verify all required settings are present",
+                "Ensure environment variables are set",
+            ],
+            ErrorType.UI_ERROR: [
+                "Refresh the page",
+                "Check browser compatibility",
+                "Verify Gradio is properly installed",
+            ],
+        }
+
+        return suggestions.get(error_type, ["Contact support for assistance"])
+
+    def get_error_statistics(self) -> Dict[str, Any]:
+        """
+        Get error statistics.
+
+        Returns:
+            Dictionary containing error statistics
+        """
+        total_errors = sum(self.error_counts.values())
+
+        return {
+            "total_errors": total_errors,
+            "error_counts": {
+                error_type.value: count
+                for error_type, count in self.error_counts.items()
+            },
+            "most_common_error": (
+                max(self.error_counts.items(), key=lambda x: x[1])[0].value
+                if self.error_counts
+                else None
+            ),
+        }
+
+
+def error_handler(
+    error_type: ErrorType = ErrorType.UNKNOWN, context: Optional[Dict[str, Any]] = None
+):
+    """
+    Decorator for automatic error handling.
+
+    Args:
+        error_type: Type of error to handle
+        context: Additional context information
+
+    Returns:
+        Decorated function
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            handler = ErrorHandler()
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                error_context = context or {}
+                error_context.update(
+                    {
+                        "function": func.__name__,
+                        "args": str(args)[:100],  # Truncate for logging
+                        "kwargs": str(kwargs)[:100],
+                    }
+                )
+
+                error_info = handler.handle_error(e, error_context)
+
+                # Re-raise as appropriate RAG error type
+                if error_type == ErrorType.DOCUMENT_PROCESSING:
+                    raise DocumentProcessingError(str(e), details=error_info)
+                elif error_type == ErrorType.URL_PROCESSING:
+                    raise URLProcessingError(str(e), details=error_info)
+                elif error_type == ErrorType.EMBEDDING_GENERATION:
+                    raise EmbeddingError(str(e), details=error_info)
+                elif error_type == ErrorType.VECTOR_STORAGE:
+                    raise VectorStorageError(str(e), details=error_info)
+                elif error_type == ErrorType.QUERY_PROCESSING:
+                    raise QueryProcessingError(str(e), details=error_info)
+                elif error_type == ErrorType.RESPONSE_GENERATION:
+                    raise ResponseGenerationError(str(e), details=error_info)
+                else:
+                    raise RAGError(str(e), error_type, error_info)
+
+        return wrapper
+
+    return decorator

src/utils/settings_manager.py

@@ -0,0 +1,676 @@
+"""
+Settings Manager Module
+
+This module provides secure environment variable management with UI integration,
+supporting both cache and .env file storage options.
+
+Features:
+- 🔐 Secure API key handling with masking
+- ⚡ Real-time validation and testing
+- 💾 Dual storage backends (cache + .env file)
+- 🛡️ Input sanitization and validation
+- 🔄 Live system updates
+"""
+
+import os
+import re
+import logging
+import json
+import time
+from typing import Dict, Any, Optional, Tuple, List
+from pathlib import Path
+from datetime import datetime
+import tempfile
+
+
+class SettingsManager:
+    """
+    Manages environment variables with secure storage and validation.
+
+    Features:
+    - Secure API key masking and validation
+    - Real-time connection testing
+    - Cache and .env file storage options
+    - Integration with existing ConfigManager
+    """
+
+    def __init__(self, config_manager=None):
+        """
+        Initialize the SettingsManager.
+
+        Args:
+            config_manager: Optional ConfigManager instance for integration
+        """
+        self.logger = logging.getLogger(__name__)
+        self.config_manager = config_manager
+
+        # 🔧 Cache storage for temporary settings
+        self._cache_storage = {}
+
+        # 📁 Project root for .env file
+        self.project_root = Path(__file__).parent.parent.parent
+        self.env_file_path = self.project_root / ".env"
+
+        # 🛡️ Supported environment variables with validation rules
+        self.supported_env_vars = {
+            "GEMINI_API_KEY": {
+                "required": True,
+                "description": "Google Gemini API Key for embeddings and LLM",
+                "format": r"^AIzaSy[A-Za-z0-9_-]{33}$",
+                "mask": True,
+                "test_function": self._test_gemini_connection,
+                "placeholder": "AIzaSy...",
+                "help_url": "https://aistudio.google.com/",
+            },
+            "PINECONE_API_KEY": {
+                "required": False,
+                "description": "Pinecone API Key for vector database",
+                "format": r"^pc-[A-Za-z0-9]{32}$",
+                "mask": True,
+                "test_function": self._test_pinecone_connection,
+                "placeholder": "pc-...",
+                "help_url": "https://www.pinecone.io/",
+            },
+            "OPENAI_API_KEY": {
+                "required": False,
+                "description": "OpenAI API Key for alternative LLM",
+                "format": r"^sk-[A-Za-z0-9]{48}$",
+                "mask": True,
+                "test_function": self._test_openai_connection,
+                "placeholder": "sk-...",
+                "help_url": "https://platform.openai.com/api-keys",
+            },
+            "TAVILY_API_KEY": {
+                "required": False,
+                "description": "Tavily API Key for live web search",
+                "format": r"^tvly-[A-Za-z0-9-]{20,50}$",
+                "mask": True,
+                "test_function": self._test_tavily_connection,
+                "placeholder": "tvly-dev-...",
+                "help_url": "https://app.tavily.com/sign-in",
+            },
+            "PINECONE_ENVIRONMENT": {
+                "required": False,
+                "description": "Pinecone environment region",
+                "format": r"^[a-z0-9-]+$",
+                "mask": False,
+                "default": "us-east-1",
+                "placeholder": "us-east-1",
+                "options": [
+                    "us-east-1",
+                    "us-west1-gcp",
+                    "eu-west1-gcp",
+                    "asia-southeast1-gcp",
+                ],
+            },
+            "PINECONE_INDEX_NAME": {
+                "required": False,
+                "description": "Pinecone index name",
+                "format": r"^[a-z0-9-]+$",
+                "mask": False,
+                "default": "rag-ai-index",
+                "placeholder": "rag-ai-index",
+            },
+            "GRADIO_SHARE": {
+                "required": False,
+                "description": "Enable Gradio public sharing",
+                "format": r"^(true|false)$",
+                "mask": False,
+                "default": "false",
+                "options": ["true", "false"],
+            },
+            "PORT": {
+                "required": False,
+                "description": "Server port number",
+                "format": r"^[1-9][0-9]{3,4}$",
+                "mask": False,
+                "default": "7860",
+                "placeholder": "7860",
+            },
+        }
+
+        self.logger.info("SettingsManager initialized successfully")
+
+    def get_current_settings(self) -> Dict[str, Any]:
+        """
+        Get current environment variable settings with status.
+
+        Returns:
+            Dictionary with current settings and their status
+        """
+        settings = {}
+
+        for var_name, config in self.supported_env_vars.items():
+            # 🔍 Get value from cache, environment, or default
+            value = self._get_env_value(var_name)
+
+            settings[var_name] = {
+                "value": (
+                    self._mask_value(value, config.get("mask", False)) if value else ""
+                ),
+                "raw_value": value or "",
+                "is_set": bool(value),
+                "is_valid": (
+                    self._validate_format(value, config.get("format"))
+                    if value
+                    else False
+                ),
+                "is_required": config.get("required", False),
+                "description": config.get("description", ""),
+                "placeholder": config.get("placeholder", ""),
+                "help_url": config.get("help_url", ""),
+                "options": config.get("options", []),
+                "source": self._get_value_source(var_name),
+                "last_tested": self._cache_storage.get(f"{var_name}_last_tested"),
+                "test_status": self._cache_storage.get(
+                    f"{var_name}_test_status", "untested"
+                ),
+            }
+
+        return settings
+
+    def update_setting(
+        self, var_name: str, value: str, storage_type: str = "cache"
+    ) -> Dict[str, Any]:
+        """
+        Update an environment variable setting.
+
+        Args:
+            var_name: Environment variable name
+            value: New value
+            storage_type: "cache" or "env_file"
+
+        Returns:
+            Dictionary with operation result
+        """
+        try:
+            if var_name not in self.supported_env_vars:
+                return {
+                    "success": False,
+                    "error": f"Unsupported environment variable: {var_name}",
+                    "status": "❌ Invalid variable",
+                }
+
+            config = self.supported_env_vars[var_name]
+
+            # 🛡️ Validate format
+            if value and not self._validate_format(value, config.get("format")):
+                return {
+                    "success": False,
+                    "error": f"Invalid format for {var_name}",
+                    "status": "❌ Invalid format",
+                    "expected_format": config.get("placeholder", ""),
+                }
+
+            # 💾 Store based on storage type
+            if storage_type == "cache":
+                self._cache_storage[var_name] = value
+                os.environ[var_name] = value  # ⚡ Update current session
+                status_msg = "💾 Saved to cache"
+            elif storage_type == "env_file":
+                self._save_to_env_file(var_name, value)
+                os.environ[var_name] = value  # ⚡ Update current session
+                status_msg = "📁 Saved to .env file"
+            else:
+                return {
+                    "success": False,
+                    "error": f"Invalid storage type: {storage_type}",
+                    "status": "❌ Invalid storage type",
+                }
+
+            # 🔄 Update config manager if available
+            if self.config_manager:
+                try:
+                    self.config_manager.reload()
+                except Exception as e:
+                    self.logger.warning(f"Could not reload config manager: {e}")
+
+            self.logger.info(f"Updated {var_name} via {storage_type}")
+
+            return {
+                "success": True,
+                "status": f" {status_msg}",
+                "value": self._mask_value(value, config.get("mask", False)),
+                "storage_type": storage_type,
+                "timestamp": datetime.now().isoformat(),
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error updating {var_name}: {e}")
+            return {"success": False, "error": str(e), "status": " Update failed"}
+
+    def test_connection(self, var_name: str) -> Dict[str, Any]:
+        """
+        Test API connection for a given environment variable.
+
+        Args:
+            var_name: Environment variable name
+
+        Returns:
+            Dictionary with test results
+        """
+        try:
+            if var_name not in self.supported_env_vars:
+                return {
+                    "success": False,
+                    "error": f"Cannot test {var_name}: not supported",
+                    "status": "❌ Not testable",
+                }
+
+            config = self.supported_env_vars[var_name]
+            test_function = config.get("test_function")
+
+            if not test_function:
+                return {
+                    "success": False,
+                    "error": f"No test function available for {var_name}",
+                    "status": "⚠️ No test available",
+                }
+
+            value = self._get_env_value(var_name)
+            if not value:
+                return {
+                    "success": False,
+                    "error": f"{var_name} is not set",
+                    "status": "❌ Not configured",
+                }
+
+            # 🧪 Run the test
+            self.logger.info(f"Testing connection for {var_name}")
+            test_result = test_function(value)
+
+            # 📊 Cache test results
+            timestamp = datetime.now().isoformat()
+            self._cache_storage[f"{var_name}_last_tested"] = timestamp
+            self._cache_storage[f"{var_name}_test_status"] = (
+                "success" if test_result["success"] else "failed"
+            )
+
+            return {**test_result, "timestamp": timestamp, "variable": var_name}
+
+        except Exception as e:
+            self.logger.error(f"Error testing {var_name}: {e}")
+            error_result = {
+                "success": False,
+                "error": str(e),
+                "status": "❌ Test failed",
+                "timestamp": datetime.now().isoformat(),
+            }
+
+            # 📊 Cache failed test
+            self._cache_storage[f"{var_name}_last_tested"] = error_result["timestamp"]
+            self._cache_storage[f"{var_name}_test_status"] = "failed"
+
+            return error_result
+
+    def load_from_env_file(self) -> Dict[str, Any]:
+        """
+        Load settings from .env file.
+
+        Returns:
+            Dictionary with load results
+        """
+        try:
+            if not self.env_file_path.exists():
+                return {
+                    "success": False,
+                    "error": ".env file not found",
+                    "status": "📁 No .env file found",
+                    "loaded_count": 0,
+                }
+
+            loaded_vars = []
+
+            with open(self.env_file_path, "r", encoding="utf-8") as f:
+                for line_num, line in enumerate(f, 1):
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        try:
+                            key, value = line.split("=", 1)
+                            key = key.strip()
+                            value = value.strip().strip("\"'")  # Remove quotes
+
+                            if key in self.supported_env_vars:
+                                os.environ[key] = value
+                                loaded_vars.append(key)
+                        except Exception as e:
+                            self.logger.warning(
+                                f"Error parsing line {line_num} in .env: {e}"
+                            )
+
+            # 🔄 Reload config manager
+            if self.config_manager:
+                try:
+                    self.config_manager.reload()
+                except Exception as e:
+                    self.logger.warning(f"Could not reload config manager: {e}")
+
+            return {
+                "success": True,
+                "status": f" Loaded {len(loaded_vars)} variables from .env",
+                "loaded_count": len(loaded_vars),
+                "loaded_variables": loaded_vars,
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error loading from .env file: {e}")
+            return {
+                "success": False,
+                "error": str(e),
+                "status": " Failed to load .env file",
+                "loaded_count": 0,
+            }
+
+    def clear_cache(self) -> Dict[str, Any]:
+        """
+        Clear cached settings.
+
+        Returns:
+            Dictionary with operation result
+        """
+        try:
+            # 🗑️ Clear cache but preserve test results
+            cached_vars = [
+                key
+                for key in self._cache_storage.keys()
+                if key in self.supported_env_vars
+            ]
+
+            for var in cached_vars:
+                if var in self._cache_storage:
+                    del self._cache_storage[var]
+                    # Remove from current environment if it was cached
+                    if var in os.environ:
+                        del os.environ[var]
+
+            return {
+                "success": True,
+                "status": f"🗑️ Cleared {len(cached_vars)} cached variables",
+                "cleared_count": len(cached_vars),
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error clearing cache: {e}")
+            return {
+                "success": False,
+                "error": str(e),
+                "status": " Failed to clear cache",
+            }
+
+    def export_settings(self, include_sensitive: bool = False) -> Dict[str, Any]:
+        """
+        Export current settings for backup/sharing.
+
+        Args:
+            include_sensitive: Whether to include API keys (masked)
+
+        Returns:
+            Dictionary with exported settings
+        """
+        try:
+            settings = self.get_current_settings()
+            exported = {}
+
+            for var_name, config in settings.items():
+                var_config = self.supported_env_vars[var_name]
+
+                # 🔐 Skip sensitive data if not requested
+                if var_config.get("mask", False) and not include_sensitive:
+                    continue
+
+                exported[var_name] = {
+                    "value": (
+                        config["value"] if include_sensitive else config["raw_value"]
+                    ),
+                    "is_set": config["is_set"],
+                    "source": config["source"],
+                    "description": config["description"],
+                }
+
+            return {
+                "success": True,
+                "settings": exported,
+                "export_timestamp": datetime.now().isoformat(),
+                "include_sensitive": include_sensitive,
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error exporting settings: {e}")
+            return {"success": False, "error": str(e)}
+
+    # 🔧 Private helper methods
+
+    def _get_env_value(self, var_name: str) -> Optional[str]:
+        """Get environment variable value from cache or environment."""
+        # Priority: cache > environment > default
+        if var_name in self._cache_storage:
+            return self._cache_storage[var_name]
+
+        env_value = os.environ.get(var_name)
+        if env_value:
+            return env_value
+
+        return self.supported_env_vars[var_name].get("default")
+
+    def _get_value_source(self, var_name: str) -> str:
+        """Determine the source of an environment variable value."""
+        if var_name in self._cache_storage:
+            return "cache"
+        elif os.environ.get(var_name):
+            return "environment"
+        elif self.supported_env_vars[var_name].get("default"):
+            return "default"
+        else:
+            return "unset"
+
+    def _mask_value(self, value: str, should_mask: bool) -> str:
+        """Mask sensitive values for display."""
+        if not value or not should_mask:
+            return value
+
+        if len(value) <= 8:
+            return "*" * len(value)
+
+        return value[:4] + "*" * (len(value) - 8) + value[-4:]
+
+    def _validate_format(self, value: str, format_pattern: Optional[str]) -> bool:
+        """Validate value against format pattern."""
+        if not format_pattern or not value:
+            return True
+
+        try:
+            return bool(re.match(format_pattern, value))
+        except Exception:
+            return False
+
+    def _save_to_env_file(self, var_name: str, value: str):
+        """Save environment variable to .env file."""
+        env_vars = {}
+
+        # 📖 Read existing .env file
+        if self.env_file_path.exists():
+            with open(self.env_file_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        try:
+                            key, val = line.split("=", 1)
+                            env_vars[key.strip()] = val.strip().strip("\"'")
+                        except Exception as e:
+                            self.logger.warning(f"Error parsing line in .env: {e}")
+
+        # ✏️ Update the variable
+        env_vars[var_name] = value
+
+        # 💾 Write back to file
+        with open(self.env_file_path, "w", encoding="utf-8") as f:
+            f.write("# Environment Variables for RAG AI System\n")
+            f.write(f"# Generated on {datetime.now().isoformat()}\n\n")
+
+            for key, val in env_vars.items():
+                # 🔐 Quote values that contain spaces or special characters
+                if " " in val or any(char in val for char in ["$", '"', "'"]):
+                    f.write(f'{key}="{val}"\n')
+                else:
+                    f.write(f"{key}={val}\n")
+
+    # 🧪 API Testing Functions
+
+    # Cache for Gemini client to avoid recreating it
+    _gemini_client_cache = None
+    _gemini_client_key = None
+    _gemini_last_test_time = None
+    _gemini_test_cooldown = 10  # seconds between tests
+
+    def _test_gemini_connection(self, api_key: str) -> Dict[str, Any]:
+        """Test Gemini API connection with caching and optimization."""
+        try:
+            # Check if we've tested this key recently
+            current_time = time.time()
+            if (
+                self._gemini_last_test_time
+                and api_key == self._gemini_client_key
+                and current_time - self._gemini_last_test_time
+                < self._gemini_test_cooldown
+            ):
+
+                self.logger.info(
+                    "Using cached Gemini test result (within cooldown period)"
+                )
+                return {
+                    "success": True,
+                    "status": "✅ Gemini API connected (cached)",
+                    "details": "Using cached test result",
+                }
+
+            import google.generativeai as genai
+
+            # Use cached client if the API key is the same
+            if api_key == self._gemini_client_key and self._gemini_client_cache:
+                self.logger.info("Using cached Gemini client")
+                client = self._gemini_client_cache
+            else:
+                # Configure new client
+                genai.configure(api_key=api_key)
+                self._gemini_client_cache = genai
+                self._gemini_client_key = api_key
+                client = genai
+
+            # 🧪 Simple test call - use embedding API instead of GenerativeModel
+            # This is faster and more efficient for testing connection
+            test_result = client.embed_content(
+                model="gemini-embedding-exp-03-07",
+                content="test connection",
+                task_type="retrieval_document",
+            )
+
+            # Update last test time
+            self._gemini_last_test_time = current_time
+
+            if test_result and "embedding" in test_result:
+                return {
+                    "success": True,
+                    "status": "✅ Gemini API connected",
+                    "details": "API key is valid and working",
+                }
+            else:
+                return {
+                    "success": False,
+                    "status": "❌ Gemini API failed",
+                    "error": "No embedding in response",
+                }
+
+        except Exception as e:
+            return {
+                "success": False,
+                "status": "❌ Gemini connection failed",
+                "error": str(e),
+            }
+
+    def _test_pinecone_connection(self, api_key: str) -> Dict[str, Any]:
+        """Test Pinecone API connection."""
+        try:
+            from pinecone import Pinecone
+
+            pc = Pinecone(api_key=api_key)
+
+            # 🧪 Test by listing indexes
+            indexes = pc.list_indexes()
+
+            return {
+                "success": True,
+                "status": "✅ Pinecone API connected",
+                "details": f"Found {len(indexes)} indexes",
+            }
+
+        except Exception as e:
+            return {
+                "success": False,
+                "status": "❌ Pinecone connection failed",
+                "error": str(e),
+            }
+
+    def _test_openai_connection(self, api_key: str) -> Dict[str, Any]:
+        """Test OpenAI API connection."""
+        try:
+            import openai
+
+            client = openai.OpenAI(api_key=api_key)
+
+            # 🧪 Test with a simple completion
+            response = client.chat.completions.create(
+                model="gpt-3.5-turbo",
+                messages=[{"role": "user", "content": "Hello"}],
+                max_tokens=5,
+            )
+
+            if response and response.choices:
+                return {
+                    "success": True,
+                    "status": "✅ OpenAI API connected",
+                    "details": "API key is valid and working",
+                }
+            else:
+                return {
+                    "success": False,
+                    "status": "❌ OpenAI API failed",
+                    "error": "No response from API",
+                }
+
+        except Exception as e:
+            return {
+                "success": False,
+                "status": " OpenAI connection failed",
+                "error": str(e),
+            }
+
+    def _test_tavily_connection(self, api_key: str) -> Dict[str, Any]:
+        """Test Tavily API connection."""
+        try:
+            from tavily import TavilyClient
+
+            # 🧪 Initialize client and test with a simple search
+            client = TavilyClient(api_key=api_key)
+
+            # Test with a minimal search query
+            response = client.search(query="test", max_results=1, search_depth="basic")
+
+            if response and isinstance(response, dict):
+                return {
+                    "success": True,
+                    "status": "✅ Tavily API connected",
+                    "details": "API key is valid and working",
+                }
+            else:
+                return {
+                    "success": False,
+                    "status": "❌ Tavily API failed",
+                    "error": "No valid response from API",
+                }
+
+        except Exception as e:
+            return {
+                "success": False,
+                "status": "❌ Tavily connection failed",
+                "error": str(e),
+            }