first commit

.dockerignore

@@ -0,0 +1,46 @@
+# Git
+.git
+.gitignore
+.gitattributes
+
+# Environment files
+.env
+.env.*
+!.env.example
+
+# Python cache files
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+.pytest_cache/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Logs
+*.log
+
+# Tests
+tests/
+
+# Docker related
+Dockerfile
+docker-compose.yml
+.dockerignore
+
+# Other files
+.vscode/
+.idea/
+*.swp
+*.swo
+.DS_Store
+.coverage
+htmlcov/
+.mypy_cache/
+.tox/
+.nox/
+instance/
+.webassets-cache
+main.py 

.env.example

@@ -0,0 +1,26 @@
+# PostgreSQL Configuration
+DB_CONNECTION_MODE=aiven
+AIVEN_DB_URL=postgresql://username:password@host:port/dbname?sslmode=require
+
+# MongoDB Configuration
+MONGODB_URL=mongodb+srv://username:[email protected]/?retryWrites=true&w=majority
+DB_NAME=Telegram
+COLLECTION_NAME=session_chat
+
+# Pinecone configuration
+PINECONE_API_KEY=your-pinecone-api-key
+PINECONE_INDEX_NAME=your-pinecone-index-name
+PINECONE_ENVIRONMENT=gcp-starter
+
+# Google Gemini API key
+GOOGLE_API_KEY=your-google-api-key
+
+# WebSocket configuration
+WEBSOCKET_SERVER=localhost
+WEBSOCKET_PORT=7860
+WEBSOCKET_PATH=/notify
+
+# Application settings
+ENVIRONMENT=production
+DEBUG=false
+PORT=7860 

.gitattributes

@@ -1,35 +1,29 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Auto detect text files and perform LF normalization
+* text=auto eol=lf
+
+# Documents
+*.md text
+*.txt text
+*.ini text
+*.yaml text
+*.yml text
+*.json text
+*.py text
+*.env.example text
+
+# Binary files
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.gif binary
+*.ico binary
+*.db binary
+
+# Git related files
+.gitignore text
+.gitattributes text
+
+# Docker related files
+Dockerfile text
+docker-compose.yml text
+.dockerignore text 

.gitignore

@@ -0,0 +1,79 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+.pytest_cache/
+htmlcov/
+.coverage
+.coverage.*
+.cache/
+coverage.xml
+*.cover
+.mypy_cache/
+
+# Environment
+.env
+.venv
+env/
+venv/
+ENV/
+
+# VSCode
+.vscode/
+*.code-workspace
+.history/
+
+# PyCharm
+.idea/
+*.iml
+*.iws
+*.ipr
+*.iws
+out/
+.idea_modules/
+
+# Logs and databases
+*.log
+*.sql
+*.sqlite
+*.db
+
+# Tests
+tests/
+
+Admin_bot/
+
+# Hugging Face Spaces
+.gitattributes
+
+# OS specific
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+Icon?
+ehthumbs.db
+Thumbs.db
+
+# Project specific
+*.log
+.env
+main.py

Dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Cài đặt các gói hệ thống cần thiết
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    software-properties-common \
+    git \
+    gcc \
+    python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Sao chép các file yêu cầu trước để tận dụng cache của Docker
+COPY requirements.txt .
+
+# Cài đặt các gói Python
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Ensure langchain-core is installed
+RUN pip install --no-cache-dir langchain-core==0.1.19
+
+# Sao chép toàn bộ code vào container
+COPY . .
+
+# Mở cổng mà ứng dụng sẽ chạy
+EXPOSE 7860
+
+# Chạy ứng dụng với uvicorn
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"] 

README.md

@@ -1,10 +1,361 @@
 ---
-title: Pix Agent
-emoji: 🦀
-colorFrom: indigo
-colorTo: indigo
+title: PIX Project Backend
+emoji: 🤖
+colorFrom: blue
+colorTo: green
 sdk: docker
+sdk_version: "3.0.0"
+app_file: app.py
 pinned: false
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# PIX Project Backend
+
+[![FastAPI](https://img.shields.io/badge/FastAPI-0.103.1-009688?style=flat&logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/)
+[![Python 3.11](https://img.shields.io/badge/Python-3.11-3776AB?style=flat&logo=python&logoColor=white)](https://www.python.org/)
+[![HuggingFace Spaces](https://img.shields.io/badge/HuggingFace-Spaces-yellow?style=flat&logo=huggingface&logoColor=white)](https://huggingface.co/spaces)
+
+Backend API for PIX Project with MongoDB, PostgreSQL and RAG integration. This project provides a comprehensive backend solution for managing FAQ items, emergency contacts, events, and a RAG-based question answering system.
+
+## Features
+
+- **MongoDB Integration**: Store user sessions and conversation history
+- **PostgreSQL Integration**: Manage FAQ items, emergency contacts, and events
+- **Pinecone Vector Database**: Store and retrieve vector embeddings for RAG
+- **RAG Question Answering**: Answer questions using relevant information from the vector database
+- **WebSocket Notifications**: Real-time notifications for Admin Bot
+- **API Documentation**: Automatic OpenAPI documentation via Swagger
+- **Docker Support**: Easy deployment using Docker
+- **Auto-Debugging**: Built-in debugging, error tracking, and performance monitoring
+
+## API Endpoints
+
+### MongoDB Endpoints
+
+- `POST /mongodb/session`: Create a new session record
+- `PUT /mongodb/session/{session_id}/response`: Update a session with a response
+- `GET /mongodb/history`: Get user conversation history
+- `GET /mongodb/health`: Check MongoDB connection health
+
+### PostgreSQL Endpoints
+
+- `GET /postgres/health`: Check PostgreSQL connection health
+- `GET /postgres/faq`: Get FAQ items
+- `POST /postgres/faq`: Create a new FAQ item
+- `GET /postgres/faq/{faq_id}`: Get a specific FAQ item
+- `PUT /postgres/faq/{faq_id}`: Update a specific FAQ item
+- `DELETE /postgres/faq/{faq_id}`: Delete a specific FAQ item
+- `GET /postgres/emergency`: Get emergency contact items
+- `POST /postgres/emergency`: Create a new emergency contact item
+- `GET /postgres/emergency/{emergency_id}`: Get a specific emergency contact
+- `GET /postgres/events`: Get event items
+
+### RAG Endpoints
+
+- `POST /rag/chat`: Get answer for a question using RAG
+- `POST /rag/embedding`: Generate embedding for text
+- `GET /rag/health`: Check RAG services health
+
+### WebSocket Endpoints
+
+- `WebSocket /notify`: Receive real-time notifications for new sessions
+
+### Debug Endpoints (Available in Debug Mode Only)
+
+- `GET /debug/config`: Get configuration information
+- `GET /debug/system`: Get system information (CPU, memory, disk usage)
+- `GET /debug/database`: Check all database connections
+- `GET /debug/errors`: View recent error logs
+- `GET /debug/performance`: Get performance metrics
+- `GET /debug/full`: Get comprehensive debug information
+
+## WebSocket API
+
+### Notifications for New Sessions
+
+The backend provides a WebSocket endpoint for receiving notifications about new sessions that match specific criteria.
+
+#### WebSocket Endpoint Configuration
+
+The WebSocket endpoint is configured using environment variables:
+
+```
+# WebSocket configuration
+WEBSOCKET_SERVER=localhost
+WEBSOCKET_PORT=7860
+WEBSOCKET_PATH=/notify
+```
+
+The full WebSocket URL will be:
+```
+ws://{WEBSOCKET_SERVER}:{WEBSOCKET_PORT}{WEBSOCKET_PATH}
+```
+
+For example: `ws://localhost:7860/notify`
+
+#### Notification Criteria
+
+A notification is sent when:
+1. A new session is created with `factor` set to "RAG"
+2. The message content starts with "I don't know"
+
+#### Notification Format
+
+```json
+{
+  "type": "new_session",
+  "timestamp": "2025-04-15 22:30:45",
+  "data": {
+    "session_id": "123e4567-e89b-12d3-a456-426614174000",
+    "factor": "rag",
+    "action": "asking_freely",
+    "created_at": "2025-04-15 22:30:45",
+    "first_name": "John",
+    "last_name": "Doe",
+    "message": "I don't know how to find emergency contacts",
+    "user_id": "12345678",
+    "username": "johndoe"
+  }
+}
+```
+
+#### Usage Example
+
+Admin Bot should establish a WebSocket connection to this endpoint using the configured URL:
+
+```python
+import websocket
+import json
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Get WebSocket configuration from environment variables
+WEBSOCKET_SERVER = os.getenv("WEBSOCKET_SERVER", "localhost")
+WEBSOCKET_PORT = os.getenv("WEBSOCKET_PORT", "7860")
+WEBSOCKET_PATH = os.getenv("WEBSOCKET_PATH", "/notify")
+
+# Create full URL
+ws_url = f"ws://{WEBSOCKET_SERVER}:{WEBSOCKET_PORT}{WEBSOCKET_PATH}"
+
+def on_message(ws, message):
+    data = json.loads(message)
+    print(f"Received notification: {data}")
+    # Forward to Telegram Admin
+
+def on_error(ws, error):
+    print(f"Error: {error}")
+
+def on_close(ws, close_status_code, close_msg):
+    print("Connection closed")
+
+def on_open(ws):
+    print("Connection opened")
+    # Send keepalive message periodically
+    ws.send("keepalive")
+
+# Connect to WebSocket
+ws = websocket.WebSocketApp(
+    ws_url,
+    on_open=on_open,
+    on_message=on_message,
+    on_error=on_error,
+    on_close=on_close
+)
+ws.run_forever()
+```
+
+When a notification is received, Admin Bot should forward the content to the Telegram Admin.
+
+## Environment Variables
+
+Create a `.env` file with the following variables:
+
+```
+# PostgreSQL Configuration
+DB_CONNECTION_MODE=aiven
+AIVEN_DB_URL=postgresql://username:password@host:port/dbname?sslmode=require
+
+# MongoDB Configuration
+MONGODB_URL=mongodb+srv://username:[email protected]/?retryWrites=true&w=majority
+DB_NAME=Telegram
+COLLECTION_NAME=session_chat
+
+# Pinecone configuration
+PINECONE_API_KEY=your-pinecone-api-key
+PINECONE_INDEX_NAME=your-pinecone-index-name
+PINECONE_ENVIRONMENT=gcp-starter
+
+# Google Gemini API key
+GOOGLE_API_KEY=your-google-api-key
+
+# WebSocket configuration
+WEBSOCKET_SERVER=localhost
+WEBSOCKET_PORT=7860
+WEBSOCKET_PATH=/notify
+
+# Application settings
+ENVIRONMENT=production
+DEBUG=false
+PORT=7860
+```
+
+## Installation and Setup
+
+### Local Development
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/ManTT-Data/PixAgent.git
+   cd PixAgent
+   ```
+
+2. Create a virtual environment and install dependencies:
+   ```bash
+   python -m venv venv
+   source venv/bin/activate  # On Windows: venv\Scripts\activate
+   pip install -r requirements.txt
+   ```
+
+3. Create a `.env` file with your configuration (see above)
+
+4. Run the application:
+   ```bash
+   uvicorn app:app --reload --port 7860
+   ```
+
+5. Open your browser and navigate to [http://localhost:7860/docs](http://localhost:7860/docs) to see the API documentation
+
+### Docker Deployment
+
+1. Build the Docker image:
+   ```bash
+   docker build -t pix-project-backend .
+   ```
+
+2. Run the Docker container:
+   ```bash
+   docker run -p 7860:7860 --env-file .env pix-project-backend
+   ```
+
+## Deployment to HuggingFace Spaces
+
+1. Create a new Space on HuggingFace (Dockerfile type)
+2. Link your GitHub repository or push directly to the HuggingFace repo
+3. Add your environment variables in the Space settings
+4. The deployment will use `app.py` as the entry point, which is the standard for HuggingFace Spaces
+
+### Important Notes for HuggingFace Deployment
+
+- The application uses `app.py` with the FastAPI instance named `app` to avoid the "Error loading ASGI app. Attribute 'app' not found in module 'app'" error
+- Make sure all environment variables are set in the Space settings
+- The Dockerfile is configured to expose port 7860, which is the default port for HuggingFace Spaces
+
+## Project Structure
+
+```
+.
+├── app                  # Main application package
+│   ├── api              # API endpoints
+│   │   ├── mongodb_routes.py
+│   │   ├── postgresql_routes.py
+│   │   ├── rag_routes.py
+│   │   └── websocket_routes.py
+│   ├── database         # Database connections
+│   │   ├── mongodb.py
+│   │   ├── pinecone.py
+│   │   └── postgresql.py
+│   ├── models           # Pydantic models
+│   │   ├── mongodb_models.py
+│   │   ├── postgresql_models.py
+│   │   └── rag_models.py
+│   └── utils            # Utility functions
+│       ├── debug_utils.py
+│       └── middleware.py
+├── tests                # Test directory
+│   └── test_api_endpoints.py
+├── .dockerignore        # Docker ignore file
+├── .env.example         # Example environment file
+├── .gitattributes       # Git attributes
+├── .gitignore           # Git ignore file
+├── app.py               # Application entry point
+├── docker-compose.yml   # Docker compose configuration
+├── Dockerfile           # Docker configuration
+├── pytest.ini           # Pytest configuration
+├── README.md            # Project documentation
+├── requirements.txt     # Project dependencies
+└── api_documentation.txt # API documentation for frontend engineers
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+# Advanced Retrieval System
+
+This project now features an enhanced vector retrieval system that improves the quality and relevance of information retrieved from Pinecone using threshold-based filtering and multiple similarity metrics.
+
+## Features
+
+### 1. Threshold-Based Retrieval
+
+The system implements a threshold-based approach to vector retrieval, which:
+- Retrieves a larger candidate set from the vector database
+- Applies a similarity threshold to filter out less relevant results
+- Returns only the most relevant documents that exceed the threshold
+
+### 2. Multiple Similarity Metrics
+
+The system supports multiple similarity metrics:
+- **Cosine Similarity** (default): Measures the cosine of the angle between vectors
+- **Dot Product**: Calculates the dot product between vectors
+- **Euclidean Distance**: Measures the straight-line distance between vectors
+
+Each metric has different characteristics and may perform better for different types of data and queries.
+
+### 3. Score Normalization
+
+For metrics like Euclidean distance where lower values indicate higher similarity, the system automatically normalizes scores to a 0-1 scale where higher values always indicate higher similarity. This makes it easier to compare results across different metrics.
+
+## Configuration
+
+The retrieval system can be configured through environment variables:
+
+```
+# Pinecone retrieval configuration
+PINECONE_DEFAULT_LIMIT_K=10       # Maximum number of candidates to retrieve
+PINECONE_DEFAULT_TOP_K=6          # Number of results to return after filtering
+PINECONE_DEFAULT_SIMILARITY_METRIC=cosine  # Default similarity metric
+PINECONE_DEFAULT_SIMILARITY_THRESHOLD=0.75 # Similarity threshold (0-1)
+PINECONE_ALLOWED_METRICS=cosine,dotproduct,euclidean  # Available metrics
+```
+
+## API Usage
+
+You can customize the retrieval parameters when making API requests:
+
+```json
+{
+  "user_id": "user123",
+  "question": "What are the best restaurants in Da Nang?",
+  "similarity_top_k": 5,
+  "limit_k": 15,
+  "similarity_metric": "cosine",
+  "similarity_threshold": 0.8
+}
+```
+
+## Benefits
+
+1. **Quality Improvement**: Retrieves only the most relevant documents above a certain quality threshold
+2. **Flexibility**: Different similarity metrics can be used for different types of queries
+3. **Efficiency**: Avoids processing irrelevant documents, improving response time
+4. **Configurability**: All parameters can be adjusted via environment variables or at request time
+
+## Implementation Details
+
+The system is implemented as a custom retriever class `ThresholdRetriever` that integrates with LangChain's retrieval infrastructure while providing enhanced functionality. 

api_documentation.txt

@@ -0,0 +1,318 @@
+# Frontend Integration Guide for PixAgent API
+
+This guide provides instructions for integrating with the optimized PostgreSQL-based API endpoints for Event, FAQ, and Emergency data.
+
+## API Endpoints
+
+### Events
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| /postgres/events/ | GET | Fetch all events (with optional filtering) |
+| /postgres/events/{event_id} | GET | Fetch a specific event by ID |
+| /postgres/events/featured | GET | Fetch featured events |
+| /postgres/events/ | POST | Create a new event |
+| /postgres/events/{event_id} | PUT | Update an existing event |
+| /postgres/events/{event_id} | DELETE | Delete an event |
+
+### FAQs
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| /postgres/faqs/ | GET | Fetch all FAQs |
+| /postgres/faqs/{faq_id} | GET | Fetch a specific FAQ by ID |
+| /postgres/faqs/ | POST | Create a new FAQ |
+| /postgres/faqs/{faq_id} | PUT | Update an existing FAQ |
+| /postgres/faqs/{faq_id} | DELETE | Delete a FAQ |
+
+### Emergency Contacts
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| /postgres/emergencies/ | GET | Fetch all emergency contacts |
+| /postgres/emergencies/{emergency_id} | GET | Fetch a specific emergency contact by ID |
+| /postgres/emergencies/ | POST | Create a new emergency contact |
+| /postgres/emergencies/{emergency_id} | PUT | Update an existing emergency contact |
+| /postgres/emergencies/{emergency_id} | DELETE | Delete an emergency contact |
+
+## Response Models
+
+### Event Response Model
+
+interface EventResponse {
+  id: number;
+  name: string;
+  description: string;
+  date_start: string;  // ISO format date
+  date_end: string;    // ISO format date
+  location: string;
+  image_url: string;
+  price: {
+    currency: string;
+    amount: string;
+  };
+  featured: boolean;
+  is_active: boolean;
+  created_at: string;  // ISO format date
+  updated_at: string;  // ISO format date
+}
+
+### FAQ Response Model
+
+interface FaqResponse {
+  id: number;
+  question: string;
+  answer: string;
+  is_active: boolean;
+  created_at: string;  // ISO format date
+  updated_at: string;  // ISO format date
+}
+
+### Emergency Response Model
+
+interface EmergencyResponse {
+  id: number;
+  name: string;
+  phone_number: string;
+  description: string;
+  address: string;
+  priority: number;
+  is_active: boolean;
+  created_at: string;  // ISO format date
+  updated_at: string;  // ISO format date
+}
+
+## Example Usage (React)
+
+### Fetching Events
+
+import { useState, useEffect } from 'react';
+import axios from 'axios';
+
+const API_BASE_URL = 'http://localhost:8000';
+
+function EventList() {
+  const [events, setEvents] = useState([]);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState(null);
+
+  useEffect(() => {
+    const fetchEvents = async () => {
+      try {
+        setLoading(true);
+        const response = await axios.get(`${API_BASE_URL}/postgres/events/`);
+        setEvents(response.data);
+        setLoading(false);
+      } catch (err) {
+        setError('Failed to fetch events');
+        setLoading(false);
+        console.error('Error fetching events:', err);
+      }
+    };
+
+    fetchEvents();
+  }, []);
+
+  if (loading) return <p>Loading events...</p>;
+  if (error) return <p>{error}</p>;
+
+  return (
+    <div>
+      <h1>Events</h1>
+      <div className="event-list">
+        {events.map(event => (
+          <div key={event.id} className="event-card">
+            <h2>{event.name}</h2>
+            <p>{event.description}</p>
+            <p>
+              <strong>When:</strong> {new Date(event.date_start).toLocaleDateString()} - {new Date(event.date_end).toLocaleDateString()}
+            </p>
+            <p><strong>Where:</strong> {event.location}</p>
+            <p><strong>Price:</strong> {event.price.amount} {event.price.currency}</p>
+            {event.featured && <span className="featured-badge">Featured</span>}
+          </div>
+        ))}
+      </div>
+    </div>
+  );
+}
+
+### Creating an Event
+
+import { useState } from 'react';
+import axios from 'axios';
+
+function CreateEvent() {
+  const [eventData, setEventData] = useState({
+    name: '',
+    description: '',
+    date_start: '',
+    date_end: '',
+    location: '',
+    image_url: '',
+    price: {
+      currency: 'USD',
+      amount: '0'
+    },
+    featured: false,
+    is_active: true
+  });
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState(null);
+  const [success, setSuccess] = useState(false);
+
+  const handleChange = (e) => {
+    const { name, value, type, checked } = e.target;
+    
+    if (name === 'price_amount') {
+      setEventData(prev => ({
+        ...prev,
+        price: {
+          ...prev.price,
+          amount: value
+        }
+      }));
+    } else if (name === 'price_currency') {
+      setEventData(prev => ({
+        ...prev,
+        price: {
+          ...prev.price,
+          currency: value
+        }
+      }));
+    } else {
+      setEventData(prev => ({
+        ...prev,
+        [name]: type === 'checkbox' ? checked : value
+      }));
+    }
+  };
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    try {
+      setLoading(true);
+      setError(null);
+      setSuccess(false);
+      
+      const response = await axios.post(`${API_BASE_URL}/postgres/events/`, eventData);
+      setSuccess(true);
+      setEventData({
+        name: '',
+        description: '',
+        date_start: '',
+        date_end: '',
+        location: '',
+        image_url: '',
+        price: {
+          currency: 'USD',
+          amount: '0'
+        },
+        featured: false,
+        is_active: true
+      });
+      setLoading(false);
+    } catch (err) {
+      setError('Failed to create event');
+      setLoading(false);
+      console.error('Error creating event:', err);
+    }
+  };
+
+  return (
+    <div>
+      <h1>Create New Event</h1>
+      {success && <div className="success-message">Event created successfully!</div>}
+      {error && <div className="error-message">{error}</div>}
+      <form onSubmit={handleSubmit}>
+        {/* Form fields would go here */}
+        <button type="submit" disabled={loading}>
+          {loading ? 'Creating...' : 'Create Event'}
+        </button>
+      </form>
+    </div>
+  );
+}
+
+## Performance Optimizations
+
+The API now includes several performance optimizations:
+
+### Caching
+
+The server implements caching for read operations, which significantly improves response times for repeated requests. The average cache improvement is over 70%.
+
+Frontend considerations:
+No need to implement client-side caching for data that doesn't change frequently
+For real-time data, consider adding a refresh button in the UI
+If data might be updated by other users, consider adding a polling mechanism or websocket for updates
+
+
+### Error Handling
+
+The API returns standardized error responses. Example:
+
+async function fetchData(url) {
+  try {
+    const response = await fetch(url);
+    if (!response.ok) {
+      const errorData = await response.json();
+      throw new Error(errorData.detail || 'An error occurred');
+    }
+    return await response.json();
+  } catch (error) {
+    console.error('API request failed:', error);
+    // Handle error in UI
+    return null;
+  }
+}
+
+### Price Field Handling
+
+The price field of events is a JSON object with currency and amount properties. When creating or updating events, ensure this is properly formatted:
+
+// Correct format for price field
+const eventData = {
+  // other fields...
+  price: {
+    currency: 'USD',
+    amount: '10.99'
+  }
+};
+
+// When displaying price
+function formatPrice(price) {
+  if (!price) return 'Free';
+  if (typeof price === 'string') {
+    try {
+      price = JSON.parse(price);
+    } catch {
+      return price;
+    }
+  }
+  return `${price.amount} ${price.currency}`;
+}
+
+## CORS Configuration
+
+The API has CORS enabled for frontend applications. If you're experiencing CORS issues, ensure your frontend domain is allowed in the server configuration.
+
+For local development, the following origins are typically allowed:
+- http://localhost:3000
+- http://localhost:5000
+- http://localhost:8080
+
+## Status Codes
+
+| Status Code | Description |
+|-------------|-------------|
+| 200 | Success - The request was successful |
+| 201 | Created - A new resource was successfully created |
+| 400 | Bad Request - The request could not be understood or was missing required parameters |
+| 404 | Not Found - Resource not found |
+| 422 | Validation Error - Request data failed validation |
+| 500 | Internal Server Error - An error occurred on the server |
+
+## Questions?
+
+For further inquiries about the API, please contact the development team.

app.py

@@ -0,0 +1,197 @@
+from fastapi import FastAPI, Depends, Request, HTTPException, status
+from fastapi.middleware.cors import CORSMiddleware
+from contextlib import asynccontextmanager
+import uvicorn
+import os
+import sys
+import logging
+from dotenv import load_dotenv
+
+# Cấu hình logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+DEBUG = os.getenv("DEBUG", "False").lower() in ("true", "1", "t")
+
+# Kiểm tra các biến môi trường bắt buộc
+required_env_vars = [
+    "AIVEN_DB_URL", 
+    "MONGODB_URL", 
+    "PINECONE_API_KEY", 
+    "PINECONE_INDEX_NAME", 
+    "GOOGLE_API_KEY"
+]
+
+missing_vars = [var for var in required_env_vars if not os.getenv(var)]
+if missing_vars:
+    logger.error(f"Missing required environment variables: {', '.join(missing_vars)}")
+    if not DEBUG:  # Chỉ thoát nếu không ở chế độ debug
+        sys.exit(1)
+
+# Database health checks
+def check_database_connections():
+    """Kiểm tra kết nối các database khi khởi động"""
+    from app.database.postgresql import check_db_connection as check_postgresql
+    from app.database.mongodb import check_db_connection as check_mongodb
+    from app.database.pinecone import check_db_connection as check_pinecone
+    
+    db_status = {
+        "postgresql": check_postgresql(),
+        "mongodb": check_mongodb(),
+        "pinecone": check_pinecone()
+    }
+    
+    all_ok = all(db_status.values())
+    if not all_ok:
+        failed_dbs = [name for name, status in db_status.items() if not status]
+        logger.error(f"Failed to connect to databases: {', '.join(failed_dbs)}")
+        if not DEBUG:  # Chỉ thoát nếu không ở chế độ debug
+            sys.exit(1)
+    
+    return db_status
+
+# Khởi tạo lifespan để kiểm tra kết nối database khi khởi động
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup: kiểm tra kết nối các database
+    logger.info("Starting application...")
+    db_status = check_database_connections()
+    if all(db_status.values()):
+        logger.info("All database connections are working")
+    
+    # Khởi tạo bảng trong cơ sở dữ liệu (nếu chưa tồn tại)
+    if DEBUG:  # Chỉ khởi tạo bảng trong chế độ debug
+        from app.database.postgresql import create_tables
+        if create_tables():
+            logger.info("Database tables created or already exist")
+    
+    yield
+    
+    # Shutdown
+    logger.info("Shutting down application...")
+
+# Import routers
+try:
+    from app.api.mongodb_routes import router as mongodb_router
+    from app.api.postgresql_routes import router as postgresql_router
+    from app.api.rag_routes import router as rag_router
+    from app.api.websocket_routes import router as websocket_router
+    
+    # Import middlewares
+    from app.utils.middleware import RequestLoggingMiddleware, ErrorHandlingMiddleware, DatabaseCheckMiddleware
+    
+    # Import debug utilities
+    from app.utils.debug_utils import debug_view, DebugInfo, error_tracker, performance_monitor
+    
+except ImportError as e:
+    logger.error(f"Error importing routes or middlewares: {e}")
+    raise
+
+# Create FastAPI app
+app = FastAPI(
+    title="PIX Project Backend API",
+    description="Backend API for PIX Project with MongoDB, PostgreSQL and RAG integration",
+    version="1.0.0",
+    docs_url="/docs",
+    redoc_url="/redoc",
+    debug=DEBUG,
+    lifespan=lifespan,
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Thêm middlewares
+app.add_middleware(ErrorHandlingMiddleware)
+app.add_middleware(RequestLoggingMiddleware)
+if not DEBUG:  # Chỉ thêm middleware kiểm tra database trong production
+    app.add_middleware(DatabaseCheckMiddleware)
+
+# Include routers
+app.include_router(mongodb_router)
+app.include_router(postgresql_router)
+app.include_router(rag_router)
+app.include_router(websocket_router)
+
+# Root endpoint
+@app.get("/")
+def read_root():
+    return {
+        "message": "Welcome to PIX Project Backend API",
+        "documentation": "/docs",
+    }
+
+# Health check endpoint
+@app.get("/health")
+def health_check():
+    # Kiểm tra kết nối database
+    db_status = check_database_connections()
+    all_db_ok = all(db_status.values())
+    
+    return {
+        "status": "healthy" if all_db_ok else "degraded",
+        "version": "1.0.0",
+        "environment": os.environ.get("ENVIRONMENT", "production"),
+        "databases": db_status
+    }
+
+# Debug endpoints (chỉ có trong chế độ debug)
+if DEBUG:
+    @app.get("/debug/config")
+    def debug_config():
+        """Hiển thị thông tin cấu hình (chỉ trong chế độ debug)"""
+        config = {
+            "environment": os.environ.get("ENVIRONMENT", "production"),
+            "debug": DEBUG,
+            "db_connection_mode": os.environ.get("DB_CONNECTION_MODE", "aiven"),
+            "databases": {
+                "postgresql": os.environ.get("AIVEN_DB_URL", "").split("@")[1].split("/")[0] if "@" in os.environ.get("AIVEN_DB_URL", "") else "N/A",
+                "mongodb": os.environ.get("MONGODB_URL", "").split("@")[1].split("/?")[0] if "@" in os.environ.get("MONGODB_URL", "") else "N/A",
+                "pinecone": os.environ.get("PINECONE_INDEX_NAME", "N/A"),
+            }
+        }
+        return config
+    
+    @app.get("/debug/system")
+    def debug_system():
+        """Hiển thị thông tin hệ thống (chỉ trong chế độ debug)"""
+        return DebugInfo.get_system_info()
+    
+    @app.get("/debug/database")
+    def debug_database():
+        """Hiển thị trạng thái database (chỉ trong chế độ debug)"""
+        return DebugInfo.get_database_status()
+    
+    @app.get("/debug/errors")
+    def debug_errors(limit: int = 10):
+        """Hiển thị các lỗi gần đây (chỉ trong chế độ debug)"""
+        return error_tracker.get_errors(limit=limit)
+    
+    @app.get("/debug/performance")
+    def debug_performance():
+        """Hiển thị thông tin hiệu suất (chỉ trong chế độ debug)"""
+        return performance_monitor.get_report()
+    
+    @app.get("/debug/full")
+    def debug_full_report(request: Request):
+        """Hiển thị báo cáo debug đầy đủ (chỉ trong chế độ debug)"""
+        return debug_view(request)
+
+# Run the app with uvicorn when executed directly
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 8000))
+    uvicorn.run("app:app", host="0.0.0.0", port=port, reload=DEBUG) 

app/__init__.py

@@ -0,0 +1,23 @@
+# PIX Project Backend
+# Version: 1.0.0
+
+__version__ = "1.0.0"
+
+# Import app từ app.py để tests có thể tìm thấy
+import sys
+import os
+
+# Thêm thư mục gốc vào sys.path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+try:
+    from app.py import app
+except ImportError:
+    # Thử cách khác nếu import trực tiếp không hoạt động
+    import importlib.util
+    spec = importlib.util.spec_from_file_location("app_module", 
+                                                 os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 
+                                                             "app.py"))
+    app_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(app_module)
+    app = app_module.app 

app/api/__init__.py

@@ -0,0 +1 @@
+# API routes package 

app/api/mongodb_routes.py

@@ -0,0 +1,276 @@
+from fastapi import APIRouter, HTTPException, Depends, Query, status, Response
+from typing import List, Optional, Dict
+from pymongo.errors import PyMongoError
+import logging
+from datetime import datetime
+import traceback
+import asyncio
+
+from app.database.mongodb import (
+    save_session, 
+    get_user_history,
+    update_session_response,
+    check_db_connection,
+    session_collection
+)
+from app.models.mongodb_models import (
+    SessionCreate,
+    SessionResponse,
+    HistoryRequest,
+    HistoryResponse,
+    QuestionAnswer
+)
+from app.api.websocket_routes import send_notification
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Create router
+router = APIRouter(
+    prefix="/mongodb",
+    tags=["MongoDB"],
+)
+
+@router.post("/session", response_model=SessionResponse, status_code=status.HTTP_201_CREATED)
+async def create_session(session: SessionCreate, response: Response):
+    """
+    Create a new session record in MongoDB.
+    
+    - **session_id**: Unique identifier for the session (auto-generated if not provided)
+    - **factor**: Factor type (user, rag, etc.)
+    - **action**: Action type (start, events, faq, emergency, help, asking_freely, etc.)
+    - **first_name**: User's first name
+    - **last_name**: User's last name (optional)
+    - **message**: User's message (optional)
+    - **user_id**: User's ID from Telegram
+    - **username**: User's username (optional)
+    - **response**: Response from RAG (optional)
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        if not check_db_connection():
+            logger.error("MongoDB connection failed when trying to create session")
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="MongoDB connection failed"
+            )
+        
+        # Create new session in MongoDB
+        result = save_session(
+            session_id=session.session_id,
+            factor=session.factor,
+            action=session.action,
+            first_name=session.first_name,
+            last_name=session.last_name,
+            message=session.message,
+            user_id=session.user_id,
+            username=session.username,
+            response=session.response
+        )
+        
+        # Chuẩn bị response object
+        session_response = SessionResponse(
+            **session.model_dump(),
+            created_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        )
+        
+        # Kiểm tra nếu session cần gửi thông báo (response bắt đầu bằng "I don't know")
+        if session.response and session.response.strip().lower().startswith("i don't know"):
+            # Gửi thông báo qua WebSocket
+            try:
+                notification_data = {
+                    "session_id": session.session_id,
+                    "factor": session.factor,
+                    "action": session.action,
+                    "message": session.message,
+                    "user_id": session.user_id,
+                    "username": session.username,
+                    "first_name": session.first_name,
+                    "last_name": session.last_name,
+                    "response": session.response,
+                    "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                }
+                
+                # Khởi tạo task để gửi thông báo - sử dụng asyncio.create_task để đảm bảo không block quá trình chính
+                asyncio.create_task(send_notification(notification_data))
+                logger.info(f"Notification queued for session {session.session_id} - response starts with 'I don't know'")
+            except Exception as e:
+                logger.error(f"Error queueing notification: {e}")
+                # Không dừng xử lý chính khi gửi thông báo thất bại
+        
+        # Return response
+        return session_response
+    except PyMongoError as e:
+        logger.error(f"MongoDB error creating session: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"MongoDB error: {str(e)}"
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error creating session: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create session: {str(e)}"
+        )
+
+@router.put("/session/{session_id}/response", status_code=status.HTTP_200_OK)
+async def update_session_with_response(session_id: str, response_text: str):
+    """
+    Update a session with the response.
+    
+    - **session_id**: ID of the session to update
+    - **response_text**: Response to add to the session
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        if not check_db_connection():
+            logger.error("MongoDB connection failed when trying to update session response")
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="MongoDB connection failed"
+            )
+        
+        # Update session in MongoDB
+        result = update_session_response(session_id, response_text)
+        
+        if not result:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=f"Session with ID {session_id} not found"
+            )
+            
+        return {"status": "success", "message": "Response added to session"}
+    except PyMongoError as e:
+        logger.error(f"MongoDB error updating session response: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"MongoDB error: {str(e)}"
+        )
+    except HTTPException:
+        # Re-throw HTTP exceptions
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error updating session response: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to update session: {str(e)}"
+        )
+
+@router.get("/history", response_model=HistoryResponse)
+async def get_history(user_id: str, n: int = Query(3, ge=1, le=10)):
+    """
+    Get user history for a specific user.
+    
+    - **user_id**: User's ID from Telegram
+    - **n**: Number of most recent interactions to return (default: 3, min: 1, max: 10)
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        if not check_db_connection():
+            logger.error("MongoDB connection failed when trying to get user history")
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="MongoDB connection failed"
+            )
+        
+        # Get user history from MongoDB
+        history_data = get_user_history(user_id=user_id, n=n)
+        
+        # Convert to response model
+        return HistoryResponse(history=history_data)
+    except PyMongoError as e:
+        logger.error(f"MongoDB error getting user history: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"MongoDB error: {str(e)}"
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error getting user history: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to get user history: {str(e)}"
+        )
+
+@router.get("/health")
+async def health_check():
+    """
+    Check health of MongoDB connection.
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        is_connected = check_db_connection()
+        
+        if not is_connected:
+            return {
+                "status": "unhealthy", 
+                "message": "MongoDB connection failed", 
+                "timestamp": datetime.now().isoformat()
+            }
+            
+        return {
+            "status": "healthy", 
+            "message": "MongoDB connection is working", 
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        logger.error(f"MongoDB health check failed: {e}")
+        logger.error(traceback.format_exc())
+        return {
+            "status": "error", 
+            "message": f"MongoDB health check error: {str(e)}", 
+            "timestamp": datetime.now().isoformat()
+        }
+
+@router.get("/session/{session_id}")
+async def get_session(session_id: str):
+    """
+    Lấy thông tin session từ MongoDB theo session_id.
+    
+    - **session_id**: ID của session cần lấy
+    """
+    try:
+        # Kiểm tra kết nối MongoDB
+        if not check_db_connection():
+            logger.error("MongoDB connection failed when trying to get session")
+            raise HTTPException(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail="MongoDB connection failed"
+            )
+        
+        # Lấy thông tin từ MongoDB
+        session_data = session_collection.find_one({"session_id": session_id})
+        
+        if not session_data:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=f"Session with ID {session_id} not found"
+            )
+            
+        # Chuyển _id thành string để có thể JSON serialize
+        if "_id" in session_data:
+            session_data["_id"] = str(session_data["_id"])
+            
+        return session_data
+    except PyMongoError as e:
+        logger.error(f"MongoDB error getting session: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"MongoDB error: {str(e)}"
+        )
+    except HTTPException:
+        # Re-throw HTTP exceptions
+        raise
+    except Exception as e:
+        logger.error(f"Unexpected error getting session: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to get session: {str(e)}"
+        ) 

app/api/postgresql_routes.py

@@ -0,0 +1,626 @@
+from fastapi import APIRouter, HTTPException, Depends, Query, Path, Body
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
+from typing import List, Optional, Dict, Any
+import logging
+import traceback
+from datetime import datetime
+from sqlalchemy import text, inspect
+
+from app.database.postgresql import get_db
+from app.database.models import FAQItem, EmergencyItem, EventItem
+from pydantic import BaseModel, Field, ConfigDict
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Create router
+router = APIRouter(
+    prefix="/postgres",
+    tags=["PostgreSQL"],
+)
+
+# --- Pydantic models for request/response ---
+
+# FAQ models
+class FAQBase(BaseModel):
+    question: str
+    answer: str
+    is_active: bool = True
+
+class FAQCreate(FAQBase):
+    pass
+
+class FAQUpdate(BaseModel):
+    question: Optional[str] = None
+    answer: Optional[str] = None
+    is_active: Optional[bool] = None
+
+class FAQResponse(FAQBase):
+    id: int
+    created_at: datetime
+    updated_at: datetime
+    
+    # Sử dụng ConfigDict thay vì class Config cho Pydantic V2
+    model_config = ConfigDict(from_attributes=True)
+
+# Emergency contact models
+class EmergencyBase(BaseModel):
+    name: str
+    phone_number: str
+    description: Optional[str] = None
+    address: Optional[str] = None
+    location: Optional[str] = None
+    priority: int = 0
+    is_active: bool = True
+
+class EmergencyCreate(EmergencyBase):
+    pass
+
+class EmergencyUpdate(BaseModel):
+    name: Optional[str] = None
+    phone_number: Optional[str] = None
+    description: Optional[str] = None
+    address: Optional[str] = None
+    location: Optional[str] = None
+    priority: Optional[int] = None
+    is_active: Optional[bool] = None
+
+class EmergencyResponse(EmergencyBase):
+    id: int
+    created_at: datetime
+    updated_at: datetime
+    
+    # Sử dụng ConfigDict thay vì class Config cho Pydantic V2
+    model_config = ConfigDict(from_attributes=True)
+
+# Event models
+class EventBase(BaseModel):
+    name: str
+    description: str
+    address: str
+    location: Optional[str] = None
+    date_start: datetime
+    date_end: Optional[datetime] = None
+    price: Optional[List[dict]] = None
+    is_active: bool = True
+    featured: bool = False
+
+class EventCreate(EventBase):
+    pass
+
+class EventUpdate(BaseModel):
+    name: Optional[str] = None
+    description: Optional[str] = None
+    address: Optional[str] = None
+    location: Optional[str] = None
+    date_start: Optional[datetime] = None
+    date_end: Optional[datetime] = None
+    price: Optional[List[dict]] = None
+    is_active: Optional[bool] = None
+    featured: Optional[bool] = None
+
+class EventResponse(EventBase):
+    id: int
+    created_at: datetime
+    updated_at: datetime
+    
+    # Sử dụng ConfigDict thay vì class Config cho Pydantic V2
+    model_config = ConfigDict(from_attributes=True)
+
+# --- FAQ endpoints ---
+
+@router.get("/faq", response_model=List[FAQResponse])
+async def get_faqs(
+    skip: int = 0, 
+    limit: int = 100,
+    active_only: bool = False,
+    db: Session = Depends(get_db)
+):
+    """
+    Get all FAQ items.
+    
+    - **skip**: Number of items to skip
+    - **limit**: Maximum number of items to return
+    - **active_only**: If true, only return active items
+    """
+    try:
+        # Log detailed connection info
+        logger.info(f"Attempting to fetch FAQs with skip={skip}, limit={limit}, active_only={active_only}")
+        
+        # Check if the FAQItem table exists
+        inspector = inspect(db.bind)
+        if not inspector.has_table("faq_item"):
+            logger.error("The faq_item table does not exist in the database")
+            raise HTTPException(status_code=500, detail="Table 'faq_item' does not exist")
+        
+        # Log table columns
+        columns = inspector.get_columns("faq_item")
+        logger.info(f"faq_item table columns: {[c['name'] for c in columns]}")
+        
+        # Query the FAQs with detailed logging
+        query = db.query(FAQItem)
+        if active_only:
+            query = query.filter(FAQItem.is_active == True)
+        
+        # Try direct SQL to debug
+        try:
+            test_result = db.execute(text("SELECT COUNT(*) FROM faq_item")).scalar()
+            logger.info(f"SQL test query succeeded, found {test_result} FAQ items")
+        except Exception as sql_error:
+            logger.error(f"SQL test query failed: {sql_error}")
+        
+        # Execute the ORM query
+        faqs = query.offset(skip).limit(limit).all()
+        logger.info(f"Successfully fetched {len(faqs)} FAQ items")
+        
+        # Check what we're returning
+        for i, faq in enumerate(faqs[:3]):  # Log the first 3 items
+            logger.info(f"FAQ item {i+1}: id={faq.id}, question={faq.question[:30]}...")
+        
+        # Convert SQLAlchemy models to Pydantic models - updated for Pydantic v2
+        result = [FAQResponse.model_validate(faq, from_attributes=True) for faq in faqs]
+        return result
+    except SQLAlchemyError as e:
+        logger.error(f"Database error in get_faqs: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
+    except Exception as e:
+        logger.error(f"Unexpected error in get_faqs: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Unexpected error: {str(e)}")
+
+@router.post("/faq", response_model=FAQResponse)
+async def create_faq(
+    faq: FAQCreate,
+    db: Session = Depends(get_db)
+):
+    """
+    Create a new FAQ item.
+    
+    - **question**: Question text
+    - **answer**: Answer text
+    - **is_active**: Whether the FAQ is active (default: True)
+    """
+    try:
+        # Sử dụng model_dump thay vì dict
+        db_faq = FAQItem(**faq.model_dump())
+        db.add(db_faq)
+        db.commit()
+        db.refresh(db_faq)
+        return FAQResponse.model_validate(db_faq, from_attributes=True)
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to create FAQ item")
+
+@router.get("/faq/{faq_id}", response_model=FAQResponse)
+async def get_faq(
+    faq_id: int = Path(..., gt=0),
+    db: Session = Depends(get_db)
+):
+    """
+    Get a specific FAQ item by ID.
+    
+    - **faq_id**: ID of the FAQ item
+    """
+    try:
+        faq = db.query(FAQItem).filter(FAQItem.id == faq_id).first()
+        if not faq:
+            raise HTTPException(status_code=404, detail="FAQ item not found")
+        return FAQResponse.model_validate(faq, from_attributes=True)
+    except SQLAlchemyError as e:
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Database error")
+
+@router.put("/faq/{faq_id}", response_model=FAQResponse)
+async def update_faq(
+    faq_id: int = Path(..., gt=0),
+    faq_update: FAQUpdate = Body(...),
+    db: Session = Depends(get_db)
+):
+    """
+    Update a specific FAQ item.
+    
+    - **faq_id**: ID of the FAQ item to update
+    - **question**: New question text (optional)
+    - **answer**: New answer text (optional)
+    - **is_active**: New active status (optional)
+    """
+    try:
+        faq = db.query(FAQItem).filter(FAQItem.id == faq_id).first()
+        if not faq:
+            raise HTTPException(status_code=404, detail="FAQ item not found")
+        
+        # Sử dụng model_dump thay vì dict
+        update_data = faq_update.model_dump(exclude_unset=True)
+        for key, value in update_data.items():
+            setattr(faq, key, value)
+            
+        db.commit()
+        db.refresh(faq)
+        return FAQResponse.model_validate(faq, from_attributes=True)
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to update FAQ item")
+
+@router.delete("/faq/{faq_id}", response_model=dict)
+async def delete_faq(
+    faq_id: int = Path(..., gt=0),
+    db: Session = Depends(get_db)
+):
+    """
+    Delete a specific FAQ item.
+    
+    - **faq_id**: ID of the FAQ item to delete
+    """
+    try:
+        faq = db.query(FAQItem).filter(FAQItem.id == faq_id).first()
+        if not faq:
+            raise HTTPException(status_code=404, detail="FAQ item not found")
+        
+        db.delete(faq)
+        db.commit()
+        return {"status": "success", "message": f"FAQ item {faq_id} deleted"}
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to delete FAQ item")
+
+# --- Emergency endpoints ---
+
+@router.get("/emergency", response_model=List[EmergencyResponse])
+async def get_emergency_contacts(
+    skip: int = 0, 
+    limit: int = 100,
+    active_only: bool = False,
+    db: Session = Depends(get_db)
+):
+    """
+    Get all emergency contacts.
+    
+    - **skip**: Number of items to skip
+    - **limit**: Maximum number of items to return
+    - **active_only**: If true, only return active items
+    """
+    try:
+        # Log detailed connection info
+        logger.info(f"Attempting to fetch emergency contacts with skip={skip}, limit={limit}, active_only={active_only}")
+        
+        # Check if the EmergencyItem table exists
+        inspector = inspect(db.bind)
+        if not inspector.has_table("emergency_item"):
+            logger.error("The emergency_item table does not exist in the database")
+            raise HTTPException(status_code=500, detail="Table 'emergency_item' does not exist")
+        
+        # Log table columns
+        columns = inspector.get_columns("emergency_item")
+        logger.info(f"emergency_item table columns: {[c['name'] for c in columns]}")
+        
+        # Try direct SQL to debug
+        try:
+            test_result = db.execute(text("SELECT COUNT(*) FROM emergency_item")).scalar()
+            logger.info(f"SQL test query succeeded, found {test_result} emergency contacts")
+        except Exception as sql_error:
+            logger.error(f"SQL test query failed: {sql_error}")
+        
+        # Query the emergency contacts
+        query = db.query(EmergencyItem)
+        if active_only:
+            query = query.filter(EmergencyItem.is_active == True)
+        
+        # Execute the ORM query
+        emergency_contacts = query.offset(skip).limit(limit).all()
+        logger.info(f"Successfully fetched {len(emergency_contacts)} emergency contacts")
+        
+        # Check what we're returning
+        for i, contact in enumerate(emergency_contacts[:3]):  # Log the first 3 items
+            logger.info(f"Emergency contact {i+1}: id={contact.id}, name={contact.name}")
+        
+        return emergency_contacts
+    except SQLAlchemyError as e:
+        logger.error(f"Database error in get_emergency_contacts: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
+    except Exception as e:
+        logger.error(f"Unexpected error in get_emergency_contacts: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Unexpected error: {str(e)}")
+
+@router.post("/emergency", response_model=EmergencyResponse)
+async def create_emergency_contact(
+    emergency: EmergencyCreate,
+    db: Session = Depends(get_db)
+):
+    """
+    Create a new emergency contact.
+    
+    - **name**: Contact name
+    - **phone_number**: Phone number
+    - **description**: Description (optional)
+    - **address**: Address (optional)
+    - **location**: Location coordinates (optional)
+    - **priority**: Priority order (default: 0)
+    - **is_active**: Whether the contact is active (default: True)
+    """
+    try:
+        db_emergency = EmergencyItem(**emergency.model_dump())
+        db.add(db_emergency)
+        db.commit()
+        db.refresh(db_emergency)
+        return db_emergency
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to create emergency contact")
+
+@router.get("/emergency/{emergency_id}", response_model=EmergencyResponse)
+async def get_emergency_contact(
+    emergency_id: int = Path(..., gt=0),
+    db: Session = Depends(get_db)
+):
+    """
+    Get a specific emergency contact by ID.
+    
+    - **emergency_id**: ID of the emergency contact
+    """
+    try:
+        emergency = db.query(EmergencyItem).filter(EmergencyItem.id == emergency_id).first()
+        if not emergency:
+            raise HTTPException(status_code=404, detail="Emergency contact not found")
+        return emergency
+    except SQLAlchemyError as e:
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Database error")
+
+@router.put("/emergency/{emergency_id}", response_model=EmergencyResponse)
+async def update_emergency_contact(
+    emergency_id: int = Path(..., gt=0),
+    emergency_update: EmergencyUpdate = Body(...),
+    db: Session = Depends(get_db)
+):
+    """
+    Update a specific emergency contact.
+    
+    - **emergency_id**: ID of the emergency contact to update
+    - **name**: New name (optional)
+    - **phone_number**: New phone number (optional)
+    - **description**: New description (optional)
+    - **address**: New address (optional)
+    - **location**: New location coordinates (optional)
+    - **priority**: New priority order (optional)
+    - **is_active**: New active status (optional)
+    """
+    try:
+        emergency = db.query(EmergencyItem).filter(EmergencyItem.id == emergency_id).first()
+        if not emergency:
+            raise HTTPException(status_code=404, detail="Emergency contact not found")
+        
+        # Update fields if provided
+        update_data = emergency_update.model_dump(exclude_unset=True)
+        for key, value in update_data.items():
+            setattr(emergency, key, value)
+            
+        db.commit()
+        db.refresh(emergency)
+        return emergency
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to update emergency contact")
+
+@router.delete("/emergency/{emergency_id}", response_model=dict)
+async def delete_emergency_contact(
+    emergency_id: int = Path(..., gt=0),
+    db: Session = Depends(get_db)
+):
+    """
+    Delete a specific emergency contact.
+    
+    - **emergency_id**: ID of the emergency contact to delete
+    """
+    try:
+        emergency = db.query(EmergencyItem).filter(EmergencyItem.id == emergency_id).first()
+        if not emergency:
+            raise HTTPException(status_code=404, detail="Emergency contact not found")
+        
+        db.delete(emergency)
+        db.commit()
+        return {"status": "success", "message": f"Emergency contact {emergency_id} deleted"}
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to delete emergency contact")
+
+# --- Event endpoints ---
+
+@router.get("/events", response_model=List[EventResponse])
+async def get_events(
+    skip: int = 0, 
+    limit: int = 100,
+    active_only: bool = False,
+    featured_only: bool = False,
+    db: Session = Depends(get_db)
+):
+    """
+    Get all events.
+    
+    - **skip**: Number of items to skip
+    - **limit**: Maximum number of items to return
+    - **active_only**: If true, only return active items
+    - **featured_only**: If true, only return featured items
+    """
+    try:
+        # Log detailed connection info
+        logger.info(f"Attempting to fetch events with skip={skip}, limit={limit}, active_only={active_only}, featured_only={featured_only}")
+        
+        # Check if the EventItem table exists
+        inspector = inspect(db.bind)
+        if not inspector.has_table("event_item"):
+            logger.error("The event_item table does not exist in the database")
+            raise HTTPException(status_code=500, detail="Table 'event_item' does not exist")
+        
+        # Log table columns
+        columns = inspector.get_columns("event_item")
+        logger.info(f"event_item table columns: {[c['name'] for c in columns]}")
+        
+        # Try direct SQL to debug
+        try:
+            test_result = db.execute(text("SELECT COUNT(*) FROM event_item")).scalar()
+            logger.info(f"SQL test query succeeded, found {test_result} events")
+        except Exception as sql_error:
+            logger.error(f"SQL test query failed: {sql_error}")
+        
+        # Query the events
+        query = db.query(EventItem)
+        if active_only:
+            query = query.filter(EventItem.is_active == True)
+        if featured_only:
+            query = query.filter(EventItem.featured == True)
+        
+        # Execute the ORM query
+        events = query.offset(skip).limit(limit).all()
+        logger.info(f"Successfully fetched {len(events)} events")
+        
+        # Debug price field of first event
+        if events and len(events) > 0:
+            logger.info(f"First event price type: {type(events[0].price)}, value: {events[0].price}")
+        
+        # Check what we're returning
+        for i, event in enumerate(events[:3]):  # Log the first 3 items
+            logger.info(f"Event {i+1}: id={event.id}, name={event.name}, price={type(event.price)}")
+        
+        return events
+    except SQLAlchemyError as e:
+        logger.error(f"Database error in get_events: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
+    except Exception as e:
+        logger.error(f"Unexpected error in get_events: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Unexpected error: {str(e)}")
+
+@router.post("/events", response_model=EventResponse)
+async def create_event(
+    event: EventCreate,
+    db: Session = Depends(get_db)
+):
+    """
+    Create a new event.
+    
+    - **name**: Event name
+    - **description**: Event description
+    - **address**: Event address
+    - **location**: Location coordinates (optional)
+    - **date_start**: Start date and time
+    - **date_end**: End date and time (optional)
+    - **price**: Price information (optional JSON object)
+    - **is_active**: Whether the event is active (default: True)
+    - **featured**: Whether the event is featured (default: False)
+    """
+    try:
+        db_event = EventItem(**event.model_dump())
+        db.add(db_event)
+        db.commit()
+        db.refresh(db_event)
+        return db_event
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to create event")
+
+@router.get("/events/{event_id}", response_model=EventResponse)
+async def get_event(
+    event_id: int = Path(..., gt=0),
+    db: Session = Depends(get_db)
+):
+    """
+    Get a specific event by ID.
+    
+    - **event_id**: ID of the event
+    """
+    try:
+        event = db.query(EventItem).filter(EventItem.id == event_id).first()
+        if not event:
+            raise HTTPException(status_code=404, detail="Event not found")
+        return event
+    except SQLAlchemyError as e:
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Database error")
+
+@router.put("/events/{event_id}", response_model=EventResponse)
+async def update_event(
+    event_id: int = Path(..., gt=0),
+    event_update: EventUpdate = Body(...),
+    db: Session = Depends(get_db)
+):
+    """
+    Update a specific event.
+    
+    - **event_id**: ID of the event to update
+    - **name**: New name (optional)
+    - **description**: New description (optional)
+    - **address**: New address (optional)
+    - **location**: New location coordinates (optional)
+    - **date_start**: New start date and time (optional)
+    - **date_end**: New end date and time (optional)
+    - **price**: New price information (optional JSON object)
+    - **is_active**: New active status (optional)
+    - **featured**: New featured status (optional)
+    """
+    try:
+        event = db.query(EventItem).filter(EventItem.id == event_id).first()
+        if not event:
+            raise HTTPException(status_code=404, detail="Event not found")
+        
+        # Update fields if provided
+        update_data = event_update.model_dump(exclude_unset=True)
+        for key, value in update_data.items():
+            setattr(event, key, value)
+            
+        db.commit()
+        db.refresh(event)
+        return event
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to update event")
+
+@router.delete("/events/{event_id}", response_model=dict)
+async def delete_event(
+    event_id: int = Path(..., gt=0),
+    db: Session = Depends(get_db)
+):
+    """
+    Delete a specific event.
+    
+    - **event_id**: ID of the event to delete
+    """
+    try:
+        event = db.query(EventItem).filter(EventItem.id == event_id).first()
+        if not event:
+            raise HTTPException(status_code=404, detail="Event not found")
+        
+        db.delete(event)
+        db.commit()
+        return {"status": "success", "message": f"Event {event_id} deleted"}
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise HTTPException(status_code=500, detail="Failed to delete event")
+
+# Health check endpoint
+@router.get("/health")
+async def health_check(db: Session = Depends(get_db)):
+    """
+    Check health of PostgreSQL connection.
+    """
+    try:
+        # Perform a simple database query to check health
+        # Use text() to wrap the SQL query for SQLAlchemy 2.0 compatibility
+        db.execute(text("SELECT 1")).first()
+        return {"status": "healthy", "message": "PostgreSQL connection is working", "timestamp": datetime.now().isoformat()}
+    except Exception as e:
+        logger.error(f"PostgreSQL health check failed: {e}")
+        raise HTTPException(status_code=503, detail=f"PostgreSQL connection failed: {str(e)}") 

app/api/rag_routes.py

@@ -0,0 +1,538 @@
+from fastapi import APIRouter, HTTPException, Depends, Query, BackgroundTasks, Request
+from typing import List, Optional, Dict, Any
+import logging
+import time
+import os
+import json
+import hashlib
+import asyncio
+import traceback
+import google.generativeai as genai
+from datetime import datetime
+from langchain.prompts import PromptTemplate
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from app.utils.utils import cache, timer_decorator
+
+from app.database.mongodb import get_user_history, get_chat_history, get_request_history, save_session, session_collection
+from app.database.pinecone import (
+    search_vectors, 
+    get_chain, 
+    DEFAULT_TOP_K,
+    DEFAULT_LIMIT_K,
+    DEFAULT_SIMILARITY_METRIC,
+    DEFAULT_SIMILARITY_THRESHOLD,
+    ALLOWED_METRICS
+)
+from app.models.rag_models import (
+    ChatRequest,
+    ChatResponse,
+    ChatResponseInternal,
+    SourceDocument,
+    EmbeddingRequest,
+    EmbeddingResponse,
+    UserMessageModel
+)
+
+# Sử dụng bộ nhớ đệm thay vì Redis
+class SimpleCache:
+    def __init__(self):
+        self.cache = {}
+        self.expiration = {}
+    
+    async def get(self, key):
+        if key in self.cache:
+            # Kiểm tra xem cache đã hết hạn chưa
+            if key in self.expiration and self.expiration[key] > time.time():
+                return self.cache[key]
+            else:
+                # Xóa cache đã hết hạn
+                if key in self.cache:
+                    del self.cache[key]
+                if key in self.expiration:
+                    del self.expiration[key]
+        return None
+    
+    async def set(self, key, value, ex=300):  # Mặc định 5 phút
+        self.cache[key] = value
+        self.expiration[key] = time.time() + ex
+
+# Khởi tạo SimpleCache
+redis_client = SimpleCache()
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Configure Google Gemini API
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+genai.configure(api_key=GOOGLE_API_KEY)
+
+# Create router
+router = APIRouter(
+    prefix="/rag",
+    tags=["RAG"],
+)
+
+# Create a prompt template with conversation history
+prompt = PromptTemplate(
+    template = """Goal:
+You are a professional tour guide assistant that assists users in finding information about places in Da Nang, Vietnam.
+You can provide details on restaurants, cafes, hotels, attractions, and other local venues. 
+You have to use core knowledge and conversation history to chat with users, who are Da Nang's tourists. 
+
+Return Format:
+Respond in friendly, natural, concise and use only English like a real tour guide.
+Always use HTML tags (e.g. <b> for bold) so that Telegram can render the special formatting correctly.
+
+Warning:
+Let's support users like a real tour guide, not a bot. The information in core knowledge is your own knowledge.
+Your knowledge is provided in the Core Knowledge. All of information in Core Knowledge is about Da Nang, Vietnam.
+You just care about current time that user mention when user ask about Solana event.
+If you do not have enough information to answer user's question, please reply with "I don't know. I don't have information about that".
+
+Core knowledge:
+{context}
+
+Conversation History:
+{chat_history}
+
+User message:
+{question}
+
+Your message:
+""",
+    input_variables = ["context", "question", "chat_history"],
+)
+
+# Helper for embeddings
+async def get_embedding(text: str):
+    """Get embedding from Google Gemini API"""
+    try:
+        # Initialize embedding model
+        embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+        
+        # Generate embedding
+        result = await embedding_model.aembed_query(text)
+        
+        # Return embedding
+        return {
+            "embedding": result,
+            "text": text,
+            "model": "embedding-001"
+        }
+    except Exception as e:
+        logger.error(f"Error generating embedding: {e}")
+        raise HTTPException(status_code=500, detail=f"Failed to generate embedding: {str(e)}")
+
+# Endpoint for generating embeddings
+@router.post("/embedding", response_model=EmbeddingResponse)
+async def create_embedding(request: EmbeddingRequest):
+    """
+    Generate embedding for text.
+    
+    - **text**: Text to generate embedding for
+    """
+    try:
+        # Get embedding
+        embedding_data = await get_embedding(request.text)
+        
+        # Return embedding
+        return EmbeddingResponse(**embedding_data)
+    except Exception as e:
+        logger.error(f"Error generating embedding: {e}")
+        raise HTTPException(status_code=500, detail=f"Failed to generate embedding: {str(e)}")
+
+@timer_decorator
+@router.post("/chat", response_model=ChatResponse)
+async def chat(request: ChatRequest, background_tasks: BackgroundTasks):
+    """
+    Get answer for a question using RAG.
+    
+    - **user_id**: User's ID from Telegram
+    - **question**: User's question
+    - **include_history**: Whether to include user history in prompt (default: True)
+    - **use_rag**: Whether to use RAG (default: True)
+    - **similarity_top_k**: Number of top similar documents to return after filtering (default: 6)
+    - **limit_k**: Maximum number of documents to retrieve from vector store (default: 10)
+    - **similarity_metric**: Similarity metric to use - cosine, dotproduct, euclidean (default: cosine)
+    - **similarity_threshold**: Threshold for vector similarity (default: 0.75)
+    - **session_id**: Optional session ID for tracking conversations
+    - **first_name**: User's first name
+    - **last_name**: User's last name
+    - **username**: User's username
+    """
+    start_time = time.time()
+    try:
+        # Create cache key for request
+        cache_key = f"rag_chat:{request.user_id}:{request.question}:{request.include_history}:{request.use_rag}:{request.similarity_top_k}:{request.limit_k}:{request.similarity_metric}:{request.similarity_threshold}"
+        
+        # Check cache using redis_client instead of cache
+        cached_response = await redis_client.get(cache_key)
+        if cached_response is not None:
+            logger.info(f"Cache hit for RAG chat request from user {request.user_id}")
+            try:
+                # If cached_response is string (JSON), parse it
+                if isinstance(cached_response, str):
+                    cached_data = json.loads(cached_response)
+                    return ChatResponse(
+                        answer=cached_data.get("answer", ""),
+                        processing_time=cached_data.get("processing_time", 0.0)
+                    )
+                # If cached_response is object with sources, extract answer and processing_time
+                elif hasattr(cached_response, 'sources'):
+                    return ChatResponse(
+                        answer=cached_response.answer,
+                        processing_time=cached_response.processing_time
+                    )
+                # Otherwise, return cached response as is
+                return cached_response
+            except Exception as e:
+                logger.error(f"Error parsing cached response: {e}")
+                # Continue processing if cache parsing fails
+        
+        # Save user message first (so it's available for user history)
+        session_id = request.session_id or f"{request.user_id}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}"
+        logger.info(f"Processing chat request for user {request.user_id}, session {session_id}")
+        
+        # First, save the user's message so it's available for history lookups
+        try:
+            # Save user's question
+            save_session(
+                session_id=session_id,
+                factor="user",
+                action="asking_freely",
+                first_name=getattr(request, 'first_name', "User"),
+                last_name=getattr(request, 'last_name', ""),
+                message=request.question,
+                user_id=request.user_id,
+                username=getattr(request, 'username', ""),
+                response=None  # No response yet
+            )
+            logger.info(f"User message saved for session {session_id}")
+        except Exception as e:
+            logger.error(f"Error saving user message to session: {e}")
+            # Continue processing even if saving fails
+        
+        # Use the RAG pipeline
+        if request.use_rag:
+            # Get the retriever with custom parameters
+            retriever = get_chain(
+                top_k=request.similarity_top_k,
+                limit_k=request.limit_k,
+                similarity_metric=request.similarity_metric,
+                similarity_threshold=request.similarity_threshold
+            )
+            if not retriever:
+                raise HTTPException(status_code=500, detail="Failed to initialize retriever")
+            
+            # Get request history for context
+            context_query = get_request_history(request.user_id) if request.include_history else request.question
+            logger.info(f"Using context query for retrieval: {context_query[:100]}...")
+            
+            # Retrieve relevant documents
+            retrieved_docs = retriever.invoke(context_query)
+            context = "\n".join([doc.page_content for doc in retrieved_docs])
+            
+            # Prepare sources
+            sources = []
+            for doc in retrieved_docs:
+                source = None
+                metadata = {}
+                
+                if hasattr(doc, 'metadata'):
+                    source = doc.metadata.get('source', None)
+                    # Extract score information
+                    score = doc.metadata.get('score', None)
+                    normalized_score = doc.metadata.get('normalized_score', None)
+                    # Remove score info from metadata to avoid duplication
+                    metadata = {k: v for k, v in doc.metadata.items() 
+                               if k not in ['text', 'source', 'score', 'normalized_score']}
+                
+                sources.append(SourceDocument(
+                    text=doc.page_content,
+                    source=source,
+                    score=score,
+                    normalized_score=normalized_score,
+                    metadata=metadata
+                ))
+        else:
+            # No RAG
+            context = ""
+            sources = None
+        
+        # Get chat history
+        chat_history = get_chat_history(request.user_id) if request.include_history else ""
+        logger.info(f"Using chat history: {chat_history[:100]}...")
+        
+        # Initialize Gemini model
+        generation_config = {
+            "temperature": 0.9,
+            "top_p": 1,
+            "top_k": 1,
+            "max_output_tokens": 2048,
+        }
+
+        safety_settings = [
+            {
+                "category": "HARM_CATEGORY_HARASSMENT",
+                "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+            },
+            {
+                "category": "HARM_CATEGORY_HATE_SPEECH",
+                "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+            },
+            {
+                "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+            },
+            {
+                "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+            },
+        ]
+
+        model = genai.GenerativeModel(
+            model_name='models/gemini-2.0-flash',
+            generation_config=generation_config,
+            safety_settings=safety_settings
+        )
+        
+        # Generate the prompt using template
+        prompt_text = prompt.format(
+            context=context,
+            question=request.question,
+            chat_history=chat_history
+        )
+        logger.info(f"Full prompt with history and context: {prompt_text}")
+        
+        # Generate response
+        response = model.generate_content(prompt_text)
+        answer = response.text
+        
+        # Save the RAG response
+        try:
+            # Now save the RAG response with the same session_id
+            save_session(
+                session_id=session_id,
+                factor="rag",
+                action="RAG_response",
+                first_name=getattr(request, 'first_name', "User"),
+                last_name=getattr(request, 'last_name', ""),
+                message=request.question,
+                user_id=request.user_id,
+                username=getattr(request, 'username', ""),
+                response=answer
+            )
+            logger.info(f"RAG response saved for session {session_id}")
+            
+            # Check if the response starts with "I don't know" and trigger notification
+            if answer.strip().lower().startswith("i don't know"):
+                from app.api.websocket_routes import send_notification
+                notification_data = {
+                    "session_id": session_id,
+                    "factor": "rag",
+                    "action": "RAG_response",
+                    "message": request.question,
+                    "user_id": request.user_id,
+                    "username": getattr(request, 'username', ""),
+                    "first_name": getattr(request, 'first_name', "User"),
+                    "last_name": getattr(request, 'last_name', ""),
+                    "response": answer,
+                    "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                }
+                background_tasks.add_task(send_notification, notification_data)
+                logger.info(f"Notification queued for session {session_id} - response starts with 'I don't know'")
+        except Exception as e:
+            logger.error(f"Error saving RAG response to session: {e}")
+            # Continue processing even if saving fails
+        
+        # Calculate processing time
+        processing_time = time.time() - start_time
+        
+        # Create internal response object with sources for logging
+        internal_response = ChatResponseInternal(
+            answer=answer,
+            sources=sources,
+            processing_time=processing_time
+        )
+        
+        # Log full response with sources
+        logger.info(f"Generated response for user {request.user_id}: {answer}")
+        if sources:
+            logger.info(f"Sources used: {len(sources)} documents")
+            for i, source in enumerate(sources):
+                logger.info(f"Source {i+1}: {source.source or 'Unknown'} (score: {source.score})")
+        
+        # Create response object for API (without sources)
+        chat_response = ChatResponse(
+            answer=answer,
+            processing_time=processing_time
+        )
+        
+        # Cache result using redis_client instead of cache
+        try:
+            # Convert to JSON to ensure it can be cached
+            cache_data = {
+                "answer": answer,
+                "processing_time": processing_time
+            }
+            await redis_client.set(cache_key, json.dumps(cache_data), ex=300)
+        except Exception as e:
+            logger.error(f"Error caching response: {e}")
+            # Continue even if caching fails
+        
+        # Return response
+        return chat_response
+    except Exception as e:
+        logger.error(f"Error processing chat request: {e}")
+        import traceback
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Failed to process chat request: {str(e)}")
+
+# Health check endpoint
+@router.get("/health")
+async def health_check():
+    """
+    Check health of RAG services and retrieval system.
+    
+    Returns:
+        - status: "healthy" if all services are working, "degraded" otherwise
+        - services: Status of each service (gemini, pinecone)
+        - retrieval_config: Current retrieval configuration
+        - timestamp: Current time
+    """
+    services = {
+        "gemini": False,
+        "pinecone": False
+    }
+    
+    # Check Gemini
+    try:
+        # Initialize simple model
+        model = genai.GenerativeModel("gemini-2.0-flash")
+        # Test generation
+        response = model.generate_content("Hello")
+        services["gemini"] = True
+    except Exception as e:
+        logger.error(f"Gemini health check failed: {e}")
+    
+    # Check Pinecone
+    try:
+        # Import pinecone function
+        from app.database.pinecone import get_pinecone_index
+        # Get index
+        index = get_pinecone_index()
+        # Check if index exists
+        if index:
+            services["pinecone"] = True
+    except Exception as e:
+        logger.error(f"Pinecone health check failed: {e}")
+    
+    # Get retrieval configuration
+    retrieval_config = {
+        "default_top_k": DEFAULT_TOP_K,
+        "default_limit_k": DEFAULT_LIMIT_K,
+        "default_similarity_metric": DEFAULT_SIMILARITY_METRIC,
+        "default_similarity_threshold": DEFAULT_SIMILARITY_THRESHOLD,
+        "allowed_metrics": ALLOWED_METRICS
+    }
+    
+    # Return health status
+    status = "healthy" if all(services.values()) else "degraded"
+    return {
+        "status": status, 
+        "services": services,
+        "retrieval_config": retrieval_config,
+        "timestamp": datetime.now().isoformat()
+    }
+
+@router.post("/rag")
+async def process_rag(request: Request, user_data: UserMessageModel, background_tasks: BackgroundTasks):
+    """
+    Process a user message through the RAG pipeline and return a response.
+    
+    Parameters:
+    - **user_id**: User ID from the client application
+    - **session_id**: Session ID for tracking the conversation
+    - **message**: User's message/question
+    - **similarity_top_k**: (Optional) Number of top similar documents to return after filtering
+    - **limit_k**: (Optional) Maximum number of documents to retrieve from vector store
+    - **similarity_metric**: (Optional) Similarity metric to use (cosine, dotproduct, euclidean)
+    - **similarity_threshold**: (Optional) Threshold for vector similarity (0-1)
+    """
+    try:
+        # Extract request data
+        user_id = user_data.user_id
+        session_id = user_data.session_id
+        message = user_data.message
+        
+        # Extract retrieval parameters (use defaults if not provided)
+        top_k = user_data.similarity_top_k or DEFAULT_TOP_K
+        limit_k = user_data.limit_k or DEFAULT_LIMIT_K
+        similarity_metric = user_data.similarity_metric or DEFAULT_SIMILARITY_METRIC
+        similarity_threshold = user_data.similarity_threshold or DEFAULT_SIMILARITY_THRESHOLD
+        
+        logger.info(f"RAG request received for user_id={user_id}, session_id={session_id}")
+        logger.info(f"Message: {message[:100]}..." if len(message) > 100 else f"Message: {message}")
+        logger.info(f"Retrieval parameters: top_k={top_k}, limit_k={limit_k}, metric={similarity_metric}, threshold={similarity_threshold}")
+        
+        # Create a cache key for this request to avoid reprocessing identical questions
+        cache_key = f"rag_{user_id}_{session_id}_{hashlib.md5(message.encode()).hexdigest()}_{top_k}_{limit_k}_{similarity_metric}_{similarity_threshold}"
+        
+        # Check if we have this response cached
+        cached_result = await redis_client.get(cache_key)
+        if cached_result:
+            logger.info(f"Cache hit for key: {cache_key}")
+            if isinstance(cached_result, str):  # If stored as JSON string
+                return json.loads(cached_result)
+            return cached_result
+        
+        # Save user message to MongoDB
+        try:
+            # Save user's question
+            save_session(
+                session_id=session_id,
+                factor="user",
+                action="asking_freely",
+                first_name="User",  # You can update this with actual data if available
+                last_name="",
+                message=message,
+                user_id=user_id,
+                username="",
+                response=None  # No response yet
+            )
+            logger.info(f"User message saved to MongoDB with session_id: {session_id}")
+        except Exception as e:
+            logger.error(f"Error saving user message: {e}")
+            # Continue anyway to try to get a response
+        
+        # Create a ChatRequest object to reuse the existing chat endpoint
+        chat_request = ChatRequest(
+            user_id=user_id,
+            question=message,
+            include_history=True,
+            use_rag=True,
+            similarity_top_k=top_k,
+            limit_k=limit_k,
+            similarity_metric=similarity_metric,
+            similarity_threshold=similarity_threshold,
+            session_id=session_id
+        )
+        
+        # Process through the chat endpoint
+        response = await chat(chat_request, background_tasks)
+        
+        # Cache the response
+        try:
+            await redis_client.set(cache_key, json.dumps({
+                "answer": response.answer,
+                "processing_time": response.processing_time
+            }))
+            logger.info(f"Cached response for key: {cache_key}")
+        except Exception as e:
+            logger.error(f"Failed to cache response: {e}")
+        
+        return response
+    except Exception as e:
+        logger.error(f"Error processing RAG request: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Error processing request: {str(e)}") 

app/api/websocket_routes.py

@@ -0,0 +1,306 @@
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Depends, status
+from typing import List, Dict
+import logging
+from datetime import datetime
+import asyncio
+import json
+import os
+from dotenv import load_dotenv
+from app.database.mongodb import session_collection
+from app.utils.utils import get_local_time
+
+# Load environment variables
+load_dotenv()
+
+# Get WebSocket configuration from environment variables
+WEBSOCKET_SERVER = os.getenv("WEBSOCKET_SERVER", "localhost")
+WEBSOCKET_PORT = os.getenv("WEBSOCKET_PORT", "7860")
+WEBSOCKET_PATH = os.getenv("WEBSOCKET_PATH", "/notify")
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Create router
+router = APIRouter(
+    tags=["WebSocket"],
+)
+
+# Store active WebSocket connections
+class ConnectionManager:
+    def __init__(self):
+        self.active_connections: List[WebSocket] = []
+
+    async def connect(self, websocket: WebSocket):
+        await websocket.accept()
+        self.active_connections.append(websocket)
+        client_info = f"{websocket.client.host}:{websocket.client.port}" if hasattr(websocket, 'client') else "Unknown"
+        logger.info(f"New WebSocket connection from {client_info}. Total connections: {len(self.active_connections)}")
+
+    def disconnect(self, websocket: WebSocket):
+        self.active_connections.remove(websocket)
+        logger.info(f"WebSocket connection removed. Total connections: {len(self.active_connections)}")
+
+    async def broadcast(self, message: Dict):
+        if not self.active_connections:
+            logger.warning("No active WebSocket connections to broadcast to")
+            return
+            
+        disconnected = []
+        for connection in self.active_connections:
+            try:
+                await connection.send_json(message)
+                logger.info(f"Message sent to WebSocket connection")
+            except Exception as e:
+                logger.error(f"Error sending message to WebSocket: {e}")
+                disconnected.append(connection)
+                
+        # Remove disconnected connections
+        for conn in disconnected:
+            if conn in self.active_connections:
+                self.active_connections.remove(conn)
+                logger.info(f"Removed disconnected WebSocket. Remaining: {len(self.active_connections)}")
+
+# Initialize connection manager
+manager = ConnectionManager()
+
+# Create full URL of WebSocket server from environment variables
+def get_full_websocket_url(server_side=False):
+    if server_side:
+        # Relative URL (for server side)
+        return WEBSOCKET_PATH
+    else:
+        # Full URL (for client)
+        # Check if should use wss:// for HTTPS
+        is_https = True if int(WEBSOCKET_PORT) == 443 else False
+        protocol = "wss" if is_https else "ws"
+        
+        # If using default port for protocol, don't include in URL
+        if (is_https and int(WEBSOCKET_PORT) == 443) or (not is_https and int(WEBSOCKET_PORT) == 80):
+            return f"{protocol}://{WEBSOCKET_SERVER}{WEBSOCKET_PATH}"
+        else:
+            return f"{protocol}://{WEBSOCKET_SERVER}:{WEBSOCKET_PORT}{WEBSOCKET_PATH}"
+
+# Add GET endpoint to display WebSocket information in Swagger
+@router.get("/notify", 
+    summary="WebSocket notifications for Admin Bot",
+    description=f"""
+    This is documentation for the WebSocket endpoint.
+    
+    To connect to WebSocket:
+    1. Use the path `{get_full_websocket_url()}`
+    2. Connect using a WebSocket client library
+    3. When there are new sessions requiring attention, you will receive notifications through this connection
+    
+    Notifications are sent when:
+    - Session response starts with "I don't know"
+    - The system cannot answer the user's question
+    
+    Make sure to send a "keepalive" message every 5 minutes to maintain the connection.
+    """,
+    status_code=status.HTTP_200_OK
+)
+async def websocket_documentation():
+    """
+    Provides information about how to use the WebSocket endpoint /notify.
+    This endpoint is for documentation purposes only. To use WebSocket, please connect to the WebSocket URL.
+    """
+    ws_url = get_full_websocket_url()
+    return {
+        "websocket_endpoint": WEBSOCKET_PATH,
+        "connection_type": "WebSocket",
+        "protocol": "ws://",
+        "server": WEBSOCKET_SERVER,
+        "port": WEBSOCKET_PORT,
+        "full_url": ws_url,
+        "description": "Endpoint to receive notifications about new sessions requiring attention",
+        "notification_format": {
+            "type": "new_session",
+            "timestamp": "YYYY-MM-DD HH:MM:SS",
+            "data": {
+                "session_id": "session id",
+                "factor": "user",
+                "action": "action type",
+                "message": "User question",
+                "response": "I don't know...",
+                "user_id": "user id",
+                "first_name": "user's first name",
+                "last_name": "user's last name",
+                "username": "username",
+                "created_at": "creation time"
+            }
+        },
+        "client_example": """
+        import websocket
+        import json
+        import os
+        import time
+        import threading
+        from dotenv import load_dotenv
+        
+        # Load environment variables
+        load_dotenv()
+        
+        # Get WebSocket configuration from environment variables
+        WEBSOCKET_SERVER = os.getenv("WEBSOCKET_SERVER", "localhost")
+        WEBSOCKET_PORT = os.getenv("WEBSOCKET_PORT", "7860")
+        WEBSOCKET_PATH = os.getenv("WEBSOCKET_PATH", "/notify")
+        
+        # Create full URL
+        ws_url = f"ws://{WEBSOCKET_SERVER}:{WEBSOCKET_PORT}{WEBSOCKET_PATH}"
+        
+        # If using HTTPS, replace ws:// with wss://
+        # ws_url = f"wss://{WEBSOCKET_SERVER}{WEBSOCKET_PATH}"
+        
+        # Send keepalive periodically
+        def send_keepalive(ws):
+            while True:
+                try:
+                    if ws.sock and ws.sock.connected:
+                        ws.send("keepalive")
+                        print("Sent keepalive message")
+                    time.sleep(300)  # 5 minutes
+                except Exception as e:
+                    print(f"Error sending keepalive: {e}")
+                    time.sleep(60)
+        
+        def on_message(ws, message):
+            try:
+                data = json.loads(message)
+                print(f"Received notification: {data}")
+                # Process notification, e.g.: send to Telegram Admin
+                if data.get("type") == "new_session":
+                    session_data = data.get("data", {})
+                    user_question = session_data.get("message", "")
+                    user_name = session_data.get("first_name", "Unknown User")
+                    print(f"User {user_name} asked: {user_question}")
+                    # Code to send message to Telegram Admin
+            except json.JSONDecodeError:
+                print(f"Received non-JSON message: {message}")
+            except Exception as e:
+                print(f"Error processing message: {e}")
+        
+        def on_error(ws, error):
+            print(f"WebSocket error: {error}")
+        
+        def on_close(ws, close_status_code, close_msg):
+            print(f"WebSocket connection closed: code={close_status_code}, message={close_msg}")
+        
+        def on_open(ws):
+            print(f"WebSocket connection opened to {ws_url}")
+            # Send keepalive messages periodically in a separate thread
+            keepalive_thread = threading.Thread(target=send_keepalive, args=(ws,), daemon=True)
+            keepalive_thread.start()
+        
+        def run_forever_with_reconnect():
+            while True:
+                try:
+                    # Connect WebSocket with ping to maintain connection
+                    ws = websocket.WebSocketApp(
+                        ws_url,
+                        on_open=on_open,
+                        on_message=on_message,
+                        on_error=on_error,
+                        on_close=on_close
+                    )
+                    ws.run_forever(ping_interval=60, ping_timeout=30)
+                    print("WebSocket connection lost, reconnecting in 5 seconds...")
+                    time.sleep(5)
+                except Exception as e:
+                    print(f"WebSocket connection error: {e}")
+                    time.sleep(5)
+        
+        # Start WebSocket client in a separate thread
+        websocket_thread = threading.Thread(target=run_forever_with_reconnect, daemon=True)
+        websocket_thread.start()
+        
+        # Keep the program running
+        try:
+            while True:
+                time.sleep(1)
+        except KeyboardInterrupt:
+            print("Stopping WebSocket client...")
+        """
+    }
+
+@router.websocket("/notify")
+async def websocket_endpoint(websocket: WebSocket):
+    """
+    WebSocket endpoint to receive notifications about new sessions.
+    Admin Bot will connect to this endpoint to receive notifications when there are new sessions requiring attention.
+    """
+    await manager.connect(websocket)
+    try:
+        while True:
+            # Maintain WebSocket connection
+            data = await websocket.receive_text()
+            # Echo back to keep connection active
+            await websocket.send_json({"status": "connected", "echo": data, "timestamp": datetime.now().isoformat()})
+            logger.info(f"Received message from WebSocket: {data}")
+    except WebSocketDisconnect:
+        logger.info("WebSocket client disconnected")
+        manager.disconnect(websocket)
+    except Exception as e:
+        logger.error(f"WebSocket error: {e}")
+        manager.disconnect(websocket)
+
+# Function to send notifications over WebSocket
+async def send_notification(data: dict):
+    """
+    Send notification to all active WebSocket connections.
+    
+    This function is used to notify admin bots about new issues or questions that need attention.
+    It's triggered when the system cannot answer a user's question (response starts with "I don't know").
+    
+    Args:
+        data: The data to send as notification
+    """
+    try:
+        # Log number of active connections and notification attempt
+        logger.info(f"Attempting to send notification. Active connections: {len(manager.active_connections)}")
+        logger.info(f"Notification data: session_id={data.get('session_id')}, user_id={data.get('user_id')}")
+        logger.info(f"Response: {data.get('response', '')[:50]}...")
+        
+        # Check if the response starts with "I don't know"
+        response = data.get('response', '')
+        if not response or not isinstance(response, str):
+            logger.warning(f"Invalid response format in notification data: {response}")
+            return
+            
+        if not response.strip().lower().startswith("i don't know"):
+            logger.info(f"Response doesn't start with 'I don't know', notification not needed: {response[:50]}...")
+            return
+            
+        logger.info(f"Response starts with 'I don't know', sending notification")
+        
+        # Format the notification data for admin
+        notification_data = {
+            "type": "new_session",
+            "timestamp": get_local_time(),
+            "data": {
+                "session_id": data.get('session_id', 'unknown'),
+                "user_id": data.get('user_id', 'unknown'),
+                "message": data.get('message', ''),
+                "response": response,
+                "first_name": data.get('first_name', 'User'),
+                "last_name": data.get('last_name', ''),
+                "username": data.get('username', ''),
+                "created_at": data.get('created_at', get_local_time()),
+                "action": data.get('action', 'unknown'),
+                "factor": "user"  # Always show as user for better readability
+            }
+        }
+        
+        # Check if there are active connections
+        if not manager.active_connections:
+            logger.warning("No active WebSocket connections for notification broadcast")
+            return
+        
+        # Broadcast notification to all active connections
+        logger.info(f"Broadcasting notification to {len(manager.active_connections)} connections")
+        await manager.broadcast(notification_data)
+        logger.info("Notification broadcast completed successfully")
+        
+    except Exception as e:
+        logger.error(f"Error sending notification: {e}")
+        import traceback
+        logger.error(traceback.format_exc())

app/database/__init__.py

@@ -0,0 +1 @@
+# Database connections package 

app/database/models.py

@@ -0,0 +1,165 @@
+from sqlalchemy import Column, Integer, String, DateTime, Boolean, ForeignKey, Float, Text, LargeBinary, JSON
+from sqlalchemy.sql import func
+from sqlalchemy.orm import relationship
+from .postgresql import Base
+import datetime
+
+class FAQItem(Base):
+    __tablename__ = "faq_item"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    question = Column(String, nullable=False)
+    answer = Column(String, nullable=False)
+    is_active = Column(Boolean, default=True)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class EmergencyItem(Base):
+    __tablename__ = "emergency_item"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    phone_number = Column(String, nullable=False)
+    description = Column(String, nullable=True)
+    address = Column(String, nullable=True)
+    location = Column(String, nullable=True)  # Will be converted to/from PostGIS POINT type
+    priority = Column(Integer, default=0)
+    is_active = Column(Boolean, default=True)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class EventItem(Base):
+    __tablename__ = "event_item"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    description = Column(Text, nullable=False)
+    address = Column(String, nullable=False)
+    location = Column(String, nullable=True)  # Will be converted to/from PostGIS POINT type
+    date_start = Column(DateTime, nullable=False)
+    date_end = Column(DateTime, nullable=True)
+    price = Column(JSON, nullable=True)
+    is_active = Column(Boolean, default=True)
+    featured = Column(Boolean, default=False)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+
+class VectorDatabase(Base):
+    __tablename__ = "vector_database"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False, unique=True)
+    description = Column(String, nullable=True)
+    pinecone_index = Column(String, nullable=False)
+    api_key = Column(String, nullable=False)
+    status = Column(String, default="active")
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    
+    # Relationships
+    documents = relationship("Document", back_populates="vector_database")
+    vector_statuses = relationship("VectorStatus", back_populates="vector_database")
+    engine_associations = relationship("EngineVectorDb", back_populates="vector_database")
+
+class Document(Base):
+    __tablename__ = "document"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    file_content = Column(LargeBinary, nullable=True)
+    file_type = Column(String, nullable=True)
+    size = Column(Integer, nullable=True)
+    content_type = Column(String, nullable=True)
+    is_embedded = Column(Boolean, default=False)
+    file_metadata = Column(JSON, nullable=True)
+    vector_database_id = Column(Integer, ForeignKey("vector_database.id"), nullable=False)
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    
+    # Relationships
+    vector_database = relationship("VectorDatabase", back_populates="documents")
+    vector_statuses = relationship("VectorStatus", back_populates="document")
+
+class VectorStatus(Base):
+    __tablename__ = "vector_status"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    document_id = Column(Integer, ForeignKey("document.id"), nullable=False)
+    vector_database_id = Column(Integer, ForeignKey("vector_database.id"), nullable=False)
+    vector_id = Column(String, nullable=True)
+    status = Column(String, default="pending")
+    error_message = Column(String, nullable=True)
+    embedded_at = Column(DateTime, nullable=True)
+    
+    # Relationships
+    document = relationship("Document", back_populates="vector_statuses")
+    vector_database = relationship("VectorDatabase", back_populates="vector_statuses")
+
+class TelegramBot(Base):
+    __tablename__ = "telegram_bot"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    username = Column(String, nullable=False, unique=True)
+    token = Column(String, nullable=False)
+    status = Column(String, default="inactive")
+    created_at = Column(DateTime, server_default=func.now())
+    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    
+    # Relationships
+    bot_engines = relationship("BotEngine", back_populates="bot")
+
+class ChatEngine(Base):
+    __tablename__ = "chat_engine"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String, nullable=False)
+    answer_model = Column(String, nullable=False)
+    system_prompt = Column(Text, nullable=True)
+    empty_response = Column(String, nullable=True)
+    similarity_top_k = Column(Integer, default=3)
+    vector_distance_threshold = Column(Float, default=0.75)
+    grounding_threshold = Column(Float, default=0.2)
+    use_public_information = Column(Boolean, default=False)
+    status = Column(String, default="active")
+    created_at = Column(DateTime, server_default=func.now())
+    last_modified = Column(DateTime, server_default=func.now(), onupdate=func.now())
+    
+    # Relationships
+    bot_engines = relationship("BotEngine", back_populates="engine")
+    engine_vector_dbs = relationship("EngineVectorDb", back_populates="engine")
+
+class BotEngine(Base):
+    __tablename__ = "bot_engine"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    bot_id = Column(Integer, ForeignKey("telegram_bot.id"), nullable=False)
+    engine_id = Column(Integer, ForeignKey("chat_engine.id"), nullable=False)
+    created_at = Column(DateTime, server_default=func.now())
+    
+    # Relationships
+    bot = relationship("TelegramBot", back_populates="bot_engines")
+    engine = relationship("ChatEngine", back_populates="bot_engines")
+
+class EngineVectorDb(Base):
+    __tablename__ = "engine_vector_db"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    engine_id = Column(Integer, ForeignKey("chat_engine.id"), nullable=False)
+    vector_database_id = Column(Integer, ForeignKey("vector_database.id"), nullable=False)
+    priority = Column(Integer, default=0)
+    
+    # Relationships
+    engine = relationship("ChatEngine", back_populates="engine_vector_dbs")
+    vector_database = relationship("VectorDatabase", back_populates="engine_associations")
+
+class ApiKey(Base):
+    __tablename__ = "api_key"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    key = Column(String, nullable=False, unique=True)
+    name = Column(String, nullable=False)
+    description = Column(String, nullable=True)
+    is_active = Column(Boolean, default=True)
+    created_at = Column(DateTime, server_default=func.now())
+    last_used = Column(DateTime, nullable=True) 

app/database/mongodb.py

@@ -0,0 +1,200 @@
+import os
+from pymongo import MongoClient
+from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError
+from dotenv import load_dotenv
+from datetime import datetime, timedelta
+import pytz
+import logging
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+# MongoDB connection string from .env
+MONGODB_URL = os.getenv("MONGODB_URL")
+DB_NAME = os.getenv("DB_NAME", "Telegram")
+COLLECTION_NAME = os.getenv("COLLECTION_NAME", "session_chat")
+
+# Set timeout for MongoDB connection
+MONGODB_TIMEOUT = int(os.getenv("MONGODB_TIMEOUT", "5000"))  # 5 seconds by default
+
+# Create MongoDB connection with timeout
+try:
+    client = MongoClient(MONGODB_URL, serverSelectionTimeoutMS=MONGODB_TIMEOUT)
+    db = client[DB_NAME]
+
+    # Collections
+    session_collection = db[COLLECTION_NAME]
+    logger.info(f"MongoDB connection initialized to {DB_NAME}.{COLLECTION_NAME}")
+    
+except Exception as e:
+    logger.error(f"Failed to initialize MongoDB connection: {e}")
+    # Don't raise exception to avoid crash during startup, error handling will be done in functions
+
+# Check MongoDB connection
+def check_db_connection():
+    """Check MongoDB connection"""
+    try:
+        # Issue a ping to confirm a successful connection
+        client.admin.command('ping')
+        logger.info("MongoDB connection is working")
+        return True
+    except (ConnectionFailure, ServerSelectionTimeoutError) as e:
+        logger.error(f"MongoDB connection failed: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Unknown error when checking MongoDB connection: {e}")
+        return False
+
+# Timezone for Asia/Ho_Chi_Minh
+asia_tz = pytz.timezone('Asia/Ho_Chi_Minh')
+
+def get_local_time():
+    """Get current time in Asia/Ho_Chi_Minh timezone"""
+    return datetime.now(asia_tz).strftime("%Y-%m-%d %H:%M:%S")
+
+def get_local_datetime():
+    """Get current datetime object in Asia/Ho_Chi_Minh timezone"""
+    return datetime.now(asia_tz)
+
+# For backward compatibility
+get_vietnam_time = get_local_time
+get_vietnam_datetime = get_local_datetime
+
+# Utility functions
+def save_session(session_id, factor, action, first_name, last_name, message, user_id, username, response=None):
+    """Save user session to MongoDB"""
+    try:
+        session_data = {
+            "session_id": session_id,
+            "factor": factor,
+            "action": action,
+            "created_at": get_local_time(),
+            "created_at_datetime": get_local_datetime(),
+            "first_name": first_name,
+            "last_name": last_name,
+            "message": message,
+            "user_id": user_id,
+            "username": username,
+            "response": response
+        }
+        result = session_collection.insert_one(session_data)
+        logger.info(f"Session saved with ID: {result.inserted_id}")
+        return {
+            "acknowledged": result.acknowledged,
+            "inserted_id": str(result.inserted_id),
+            "session_data": session_data
+        }
+    except Exception as e:
+        logger.error(f"Error saving session: {e}")
+        raise
+
+def update_session_response(session_id, response):
+    """Update a session with response"""
+    try:
+        result = session_collection.update_one(
+            {"session_id": session_id},
+            {"$set": {"response": response}}
+        )
+        
+        if result.matched_count == 0:
+            logger.warning(f"No session found with ID: {session_id}")
+            return False
+            
+        logger.info(f"Session {session_id} updated with response")
+        return True
+    except Exception as e:
+        logger.error(f"Error updating session response: {e}")
+        raise
+
+def get_recent_sessions(user_id, action, n=3):
+    """Get n most recent sessions for a specific user and action"""
+    try:
+        return list(
+            session_collection.find(
+                {"user_id": user_id, "action": action},
+                {"_id": 0, "message": 1, "response": 1}
+            ).sort("created_at_datetime", -1).limit(n)
+        )
+    except Exception as e:
+        logger.error(f"Error getting recent sessions: {e}")
+        return []
+
+def get_user_history(user_id, n=3):
+    """Get user history for a specific user"""
+    try:
+        # Find all messages of this user
+        user_messages = list(
+            session_collection.find(
+                {
+                    "user_id": user_id, 
+                    "message": {"$exists": True, "$ne": None},
+                    # Include all user messages regardless of action type
+                }
+            ).sort("created_at_datetime", -1).limit(n * 2)  # Get more to ensure we have enough pairs
+        )
+        
+        # Group messages by session_id to find pairs
+        session_dict = {}
+        for msg in user_messages:
+            session_id = msg.get("session_id")
+            if session_id not in session_dict:
+                session_dict[session_id] = {}
+                
+            if msg.get("factor", "").lower() == "user":
+                session_dict[session_id]["question"] = msg.get("message", "")
+                session_dict[session_id]["timestamp"] = msg.get("created_at_datetime")
+            elif msg.get("factor", "").lower() == "rag":
+                session_dict[session_id]["answer"] = msg.get("response", "")
+                
+        # Build history from complete pairs only (with both question and answer)
+        history = []
+        for session_id, data in session_dict.items():
+            if "question" in data and "answer" in data and data.get("answer"):
+                history.append({
+                    "question": data["question"],
+                    "answer": data["answer"]
+                })
+        
+        # Sort by timestamp and limit to n
+        history = sorted(history, key=lambda x: x.get("timestamp", 0), reverse=True)[:n]
+        
+        logger.info(f"Retrieved {len(history)} history items for user {user_id}")
+        return history
+    except Exception as e:
+        logger.error(f"Error getting user history: {e}")
+        return []
+
+# Functions from chatbot.py
+def get_chat_history(user_id, n=5):
+    """Get conversation history for a specific user from MongoDB in format suitable for LLM prompt"""
+    try:
+        history = get_user_history(user_id, n)
+        
+        # Format history for prompt context
+        formatted_history = ""
+        for item in history:
+            formatted_history += f"User: {item['question']}\nAssistant: {item['answer']}\n\n"
+            
+        return formatted_history
+    except Exception as e:
+        logger.error(f"Error getting chat history for prompt: {e}")
+        return ""
+
+def get_request_history(user_id, n=3):
+    """Get the most recent user requests to use as context for retrieval"""
+    try:
+        history = get_user_history(user_id, n)
+        
+        # Just extract the questions for context
+        requests = []
+        for item in history:
+            requests.append(item['question'])
+            
+        # Join all recent requests into a single string for context
+        return " ".join(requests)
+    except Exception as e:
+        logger.error(f"Error getting request history: {e}")
+        return "" 

app/database/pinecone.py

@@ -0,0 +1,581 @@
+import os
+from pinecone import Pinecone
+from dotenv import load_dotenv
+import logging
+from typing import Optional, List, Dict, Any, Union, Tuple
+import time
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+import google.generativeai as genai
+from app.utils.utils import cache
+from langchain_core.retrievers import BaseRetriever
+from langchain.callbacks.manager import Callbacks
+from langchain_core.documents import Document
+from langchain_core.pydantic_v1 import Field
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+# Pinecone API key and index name
+PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
+PINECONE_INDEX_NAME = os.getenv("PINECONE_INDEX_NAME")
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+
+# Pinecone retrieval configuration
+DEFAULT_LIMIT_K = int(os.getenv("PINECONE_DEFAULT_LIMIT_K", "10"))
+DEFAULT_TOP_K = int(os.getenv("PINECONE_DEFAULT_TOP_K", "6"))
+DEFAULT_SIMILARITY_METRIC = os.getenv("PINECONE_DEFAULT_SIMILARITY_METRIC", "cosine")
+DEFAULT_SIMILARITY_THRESHOLD = float(os.getenv("PINECONE_DEFAULT_SIMILARITY_THRESHOLD", "0.75"))
+ALLOWED_METRICS = os.getenv("PINECONE_ALLOWED_METRICS", "cosine,dotproduct,euclidean").split(",")
+
+# Export constants for importing elsewhere
+__all__ = [
+    'get_pinecone_index', 
+    'check_db_connection', 
+    'search_vectors', 
+    'upsert_vectors', 
+    'delete_vectors', 
+    'fetch_metadata',
+    'get_chain',
+    'DEFAULT_TOP_K',
+    'DEFAULT_LIMIT_K',
+    'DEFAULT_SIMILARITY_METRIC',
+    'DEFAULT_SIMILARITY_THRESHOLD',
+    'ALLOWED_METRICS',
+    'ThresholdRetriever'
+]
+
+# Configure Google API
+if GOOGLE_API_KEY:
+    genai.configure(api_key=GOOGLE_API_KEY)
+
+# Initialize global variables to store instances of Pinecone and index
+pc = None
+index = None
+_retriever_instance = None
+
+# Check environment variables
+if not PINECONE_API_KEY:
+    logger.error("PINECONE_API_KEY is not set in environment variables")
+
+if not PINECONE_INDEX_NAME:
+    logger.error("PINECONE_INDEX_NAME is not set in environment variables")
+
+# Initialize Pinecone
+def init_pinecone():
+    """Initialize pinecone connection using new API"""
+    global pc, index
+    
+    try:
+        # Only initialize if not already initialized
+        if pc is None:
+            logger.info(f"Initializing Pinecone connection to index {PINECONE_INDEX_NAME}...")
+            
+            # Initialize Pinecone client using the new API
+            pc = Pinecone(api_key=PINECONE_API_KEY)
+            
+            # Check if index exists
+            index_list = pc.list_indexes()
+            
+            if not hasattr(index_list, 'names') or PINECONE_INDEX_NAME not in index_list.names():
+                logger.error(f"Index {PINECONE_INDEX_NAME} does not exist in Pinecone")
+                return None
+            
+            # Get existing index
+            index = pc.Index(PINECONE_INDEX_NAME)
+            logger.info(f"Pinecone connection established to index {PINECONE_INDEX_NAME}")
+            
+        return index
+    except Exception as e:
+        logger.error(f"Error initializing Pinecone: {e}")
+        return None
+
+# Get Pinecone index singleton
+def get_pinecone_index():
+    """Get Pinecone index"""
+    global index
+    if index is None:
+        index = init_pinecone()
+    return index
+
+# Check Pinecone connection
+def check_db_connection():
+    """Check Pinecone connection"""
+    try:
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            return False
+            
+        # Check index information to confirm connection is working
+        stats = pinecone_index.describe_index_stats()
+        
+        # Get total vector count from the new result structure
+        total_vectors = stats.get('total_vector_count', 0)
+        if hasattr(stats, 'namespaces'):
+            # If there are namespaces, calculate total vector count from namespaces
+            total_vectors = sum(ns.get('vector_count', 0) for ns in stats.namespaces.values())
+            
+        logger.info(f"Pinecone connection is working. Total vectors: {total_vectors}")
+        return True
+    except Exception as e:
+        logger.error(f"Error in Pinecone connection: {e}")
+        return False
+
+# Convert similarity score based on the metric
+def convert_score(score: float, metric: str) -> float:
+    """
+    Convert similarity score to a 0-1 scale based on the metric used.
+    For metrics like euclidean distance where lower is better, we invert the score.
+    
+    Args:
+        score: The raw similarity score
+        metric: The similarity metric used
+        
+    Returns:
+        A normalized score between 0-1 where higher means more similar
+    """
+    if metric.lower() in ["euclidean", "l2"]:
+        # For distance metrics (lower is better), we inverse and normalize
+        # Assuming max reasonable distance is 2.0 for normalized vectors
+        return max(0, 1 - (score / 2.0))
+    else:
+        # For cosine and dot product (higher is better), return as is
+        return score
+
+# Filter results based on similarity threshold
+def filter_by_threshold(results, threshold: float, metric: str) -> List[Dict]:
+    """
+    Filter query results based on similarity threshold.
+    
+    Args:
+        results: The query results from Pinecone
+        threshold: The similarity threshold (0-1)
+        metric: The similarity metric used
+        
+    Returns:
+        Filtered list of matches
+    """
+    filtered_matches = []
+    
+    if not hasattr(results, 'matches'):
+        return filtered_matches
+        
+    for match in results.matches:
+        # Get the score
+        score = getattr(match, 'score', 0)
+        
+        # Convert score based on metric
+        normalized_score = convert_score(score, metric)
+        
+        # Filter based on threshold
+        if normalized_score >= threshold:
+            # Add normalized score as an additional attribute
+            match.normalized_score = normalized_score
+            filtered_matches.append(match)
+    
+    return filtered_matches
+
+# Search vectors in Pinecone with advanced options
+async def search_vectors(
+    query_vector, 
+    top_k: int = DEFAULT_TOP_K,
+    limit_k: int = DEFAULT_LIMIT_K,
+    similarity_metric: str = DEFAULT_SIMILARITY_METRIC,
+    similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
+    namespace: str = "", 
+    filter: Optional[Dict] = None
+) -> Dict:
+    """
+    Search for most similar vectors in Pinecone with advanced filtering options.
+    
+    Args:
+        query_vector: The query vector
+        top_k: Number of results to return (after threshold filtering)
+        limit_k: Maximum number of results to retrieve from Pinecone
+        similarity_metric: Similarity metric to use (cosine, dotproduct, euclidean)
+        similarity_threshold: Threshold for similarity (0-1)
+        namespace: Namespace to search in
+        filter: Filter query
+        
+    Returns:
+        Search results with matches filtered by threshold
+    """
+    try:
+        # Validate parameters
+        if similarity_metric not in ALLOWED_METRICS:
+            logger.warning(f"Invalid similarity metric: {similarity_metric}. Using default: {DEFAULT_SIMILARITY_METRIC}")
+            similarity_metric = DEFAULT_SIMILARITY_METRIC
+            
+        if limit_k < top_k:
+            logger.warning(f"limit_k ({limit_k}) must be greater than or equal to top_k ({top_k}). Setting limit_k to {top_k}")
+            limit_k = top_k
+            
+        # Create cache key from parameters
+        vector_hash = hash(str(query_vector))
+        cache_key = f"pinecone_search:{vector_hash}:{limit_k}:{similarity_metric}:{similarity_threshold}:{namespace}:{filter}"
+        
+        # Check cache first
+        cached_result = cache.get(cache_key)
+        if cached_result is not None:
+            logger.info("Returning cached Pinecone search results")
+            return cached_result
+        
+        # If not in cache, perform search
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            logger.error("Failed to get Pinecone index for search")
+            return None
+        
+        # Query Pinecone with the provided metric and higher limit_k to allow for threshold filtering
+        results = pinecone_index.query(
+            vector=query_vector,
+            top_k=limit_k,  # Retrieve more results than needed to allow for threshold filtering
+            namespace=namespace,
+            filter=filter,
+            include_metadata=True,
+            include_values=False,  # No need to return vector values to save bandwidth
+            metric=similarity_metric  # Specify similarity metric
+        )
+        
+        # Filter results by threshold
+        filtered_matches = filter_by_threshold(results, similarity_threshold, similarity_metric)
+        
+        # Limit to top_k after filtering
+        filtered_matches = filtered_matches[:top_k]
+        
+        # Create a new results object with filtered matches
+        results.matches = filtered_matches
+        
+        # Log search result metrics
+        match_count = len(filtered_matches)
+        logger.info(f"Pinecone search returned {match_count} matches after threshold filtering (metric: {similarity_metric}, threshold: {similarity_threshold})")
+        
+        # Store result in cache with 5 minute TTL
+        cache.set(cache_key, results, ttl=300)
+        
+        return results
+    except Exception as e:
+        logger.error(f"Error searching vectors: {e}")
+        return None
+
+# Upsert vectors to Pinecone
+async def upsert_vectors(vectors, namespace=""):
+    """Upsert vectors to Pinecone index"""
+    try:
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            logger.error("Failed to get Pinecone index for upsert")
+            return None
+            
+        response = pinecone_index.upsert(
+            vectors=vectors,
+            namespace=namespace
+        )
+        
+        # Log upsert metrics
+        upserted_count = response.get('upserted_count', 0)
+        logger.info(f"Upserted {upserted_count} vectors to Pinecone")
+        
+        return response
+    except Exception as e:
+        logger.error(f"Error upserting vectors: {e}")
+        return None
+
+# Delete vectors from Pinecone
+async def delete_vectors(ids, namespace=""):
+    """Delete vectors from Pinecone index"""
+    try:
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            logger.error("Failed to get Pinecone index for delete")
+            return False
+            
+        response = pinecone_index.delete(
+            ids=ids,
+            namespace=namespace
+        )
+        
+        logger.info(f"Deleted vectors with IDs {ids} from Pinecone")
+        return True
+    except Exception as e:
+        logger.error(f"Error deleting vectors: {e}")
+        return False
+
+# Fetch vector metadata from Pinecone
+async def fetch_metadata(ids, namespace=""):
+    """Fetch metadata for specific vector IDs"""
+    try:
+        pinecone_index = get_pinecone_index()
+        if pinecone_index is None:
+            logger.error("Failed to get Pinecone index for fetch")
+            return None
+            
+        response = pinecone_index.fetch(
+            ids=ids,
+            namespace=namespace
+        )
+        
+        return response
+    except Exception as e:
+        logger.error(f"Error fetching vector metadata: {e}")
+        return None
+
+# Create a custom retriever class for Langchain integration
+class ThresholdRetriever(BaseRetriever):
+    """
+    Custom retriever that supports threshold-based filtering and multiple similarity metrics.
+    This integrates with the Langchain ecosystem while using our advanced retrieval logic.
+    """
+    
+    vectorstore: Any = Field(description="Vector store to use for retrieval")
+    embeddings: Any = Field(description="Embeddings model to use for retrieval")
+    search_kwargs: Dict[str, Any] = Field(default_factory=dict, description="Search kwargs for the vectorstore")
+    top_k: int = Field(default=DEFAULT_TOP_K, description="Number of results to return after filtering")
+    limit_k: int = Field(default=DEFAULT_LIMIT_K, description="Maximum number of results to retrieve from Pinecone")
+    similarity_metric: str = Field(default=DEFAULT_SIMILARITY_METRIC, description="Similarity metric to use")
+    similarity_threshold: float = Field(default=DEFAULT_SIMILARITY_THRESHOLD, description="Threshold for similarity")
+    
+    class Config:
+        """Configuration for this pydantic object."""
+        arbitrary_types_allowed = True
+    
+    async def search_vectors_sync(
+        self, query_vector, 
+        top_k: int = DEFAULT_TOP_K,
+        limit_k: int = DEFAULT_LIMIT_K,
+        similarity_metric: str = DEFAULT_SIMILARITY_METRIC,
+        similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
+        namespace: str = "", 
+        filter: Optional[Dict] = None
+    ) -> Dict:
+        """Synchronous wrapper for search_vectors"""
+        import asyncio
+        try:
+            # Get current event loop or create a new one
+            try:
+                loop = asyncio.get_event_loop()
+            except RuntimeError:
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+            
+            # Use event loop to run async function
+            if loop.is_running():
+                # If we're in an event loop, use asyncio.create_task
+                task = asyncio.create_task(search_vectors(
+                    query_vector=query_vector,
+                    top_k=top_k,
+                    limit_k=limit_k,
+                    similarity_metric=similarity_metric,
+                    similarity_threshold=similarity_threshold,
+                    namespace=namespace,
+                    filter=filter
+                ))
+                return await task
+            else:
+                # If not in an event loop, just await directly
+                return await search_vectors(
+                    query_vector=query_vector,
+                    top_k=top_k,
+                    limit_k=limit_k,
+                    similarity_metric=similarity_metric,
+                    similarity_threshold=similarity_threshold,
+                    namespace=namespace,
+                    filter=filter
+                )
+        except Exception as e:
+            logger.error(f"Error in search_vectors_sync: {e}")
+            return None
+
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: Callbacks = None
+    ) -> List[Document]:
+        """
+        Get documents relevant to the query using threshold-based retrieval.
+        
+        Args:
+            query: The query string
+            run_manager: The callbacks manager
+            
+        Returns:
+            List of relevant documents
+        """
+        # Generate embedding for query using the embeddings model
+        try:
+            # Use the embeddings model we stored in the class
+            embedding = self.embeddings.embed_query(query)
+        except Exception as e:
+            logger.error(f"Error generating embedding: {e}")
+            # Fallback to creating a new embedding model if needed
+            embedding_model = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+            embedding = embedding_model.embed_query(query)
+        
+        # Perform search with advanced options - avoid asyncio.run()
+        import asyncio
+        
+        # Get or create event loop
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            
+        # Run asynchronous search in a safe way
+        if loop.is_running():
+            # We're inside an existing event loop (like in FastAPI)
+            # Use a different approach - convert it to a synchronous call
+            from concurrent.futures import ThreadPoolExecutor
+            import functools
+            
+            # Define a wrapper function to run in a thread
+            def run_async_in_thread():
+                # Create a new event loop for this thread
+                thread_loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(thread_loop)
+                # Run the coroutine and return the result
+                return thread_loop.run_until_complete(search_vectors(
+                    query_vector=embedding,
+                    top_k=self.top_k,
+                    limit_k=self.limit_k,
+                    similarity_metric=self.similarity_metric,
+                    similarity_threshold=self.similarity_threshold,
+                    namespace=getattr(self.vectorstore, "namespace", ""),
+                    filter=self.search_kwargs.get("filter", None)
+                ))
+            
+            # Run the async function in a thread
+            with ThreadPoolExecutor() as executor:
+                search_result = executor.submit(run_async_in_thread).result()
+        else:
+            # No event loop running, we can use run_until_complete
+            search_result = loop.run_until_complete(search_vectors(
+                query_vector=embedding,
+                top_k=self.top_k,
+                limit_k=self.limit_k,
+                similarity_metric=self.similarity_metric,
+                similarity_threshold=self.similarity_threshold,
+                namespace=getattr(self.vectorstore, "namespace", ""),
+                filter=self.search_kwargs.get("filter", None)
+            ))
+        
+        # Convert to documents
+        documents = []
+        if search_result and hasattr(search_result, 'matches'):
+            for match in search_result.matches:
+                # Extract metadata
+                metadata = {}
+                if hasattr(match, 'metadata'):
+                    metadata = match.metadata
+                
+                # Add score to metadata
+                score = getattr(match, 'score', 0)
+                normalized_score = getattr(match, 'normalized_score', score)
+                metadata['score'] = score
+                metadata['normalized_score'] = normalized_score
+                
+                # Extract text
+                text = metadata.get('text', '')
+                if 'text' in metadata:
+                    del metadata['text']  # Remove from metadata since it's the content
+                
+                # Create Document
+                doc = Document(
+                    page_content=text,
+                    metadata=metadata
+                )
+                documents.append(doc)
+        
+        return documents
+
+# Get the retrieval chain with Pinecone vector store
+def get_chain(
+    index_name=PINECONE_INDEX_NAME, 
+    namespace="Default", 
+    top_k=DEFAULT_TOP_K, 
+    limit_k=DEFAULT_LIMIT_K,
+    similarity_metric=DEFAULT_SIMILARITY_METRIC, 
+    similarity_threshold=DEFAULT_SIMILARITY_THRESHOLD
+):
+    """
+    Get the retrieval chain with Pinecone vector store using threshold-based retrieval.
+    
+    Args:
+        index_name: Pinecone index name
+        namespace: Pinecone namespace
+        top_k: Number of results to return after filtering
+        limit_k: Maximum number of results to retrieve from Pinecone
+        similarity_metric: Similarity metric to use (cosine, dotproduct, euclidean)
+        similarity_threshold: Threshold for similarity (0-1)
+        
+    Returns:
+        ThresholdRetriever instance
+    """
+    global _retriever_instance
+    try:
+        # If already initialized with same parameters, return cached instance
+        if _retriever_instance is not None:
+            return _retriever_instance
+            
+        # Check if chain has been cached
+        cache_key = f"pinecone_retriever:{index_name}:{namespace}:{top_k}:{limit_k}:{similarity_metric}:{similarity_threshold}"
+        cached_retriever = cache.get(cache_key)
+        if cached_retriever is not None:
+            _retriever_instance = cached_retriever
+            logger.info("Retrieved cached Pinecone retriever")
+            return _retriever_instance
+            
+        start_time = time.time()
+        logger.info("Initializing new retriever chain with threshold-based filtering")
+        
+        # Initialize embeddings model
+        embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001")
+        
+        # Get index
+        pinecone_index = get_pinecone_index()
+        if not pinecone_index:
+            logger.error("Failed to get Pinecone index for retriever chain")
+            return None
+            
+        # Get statistics for logging
+        try:
+            stats = pinecone_index.describe_index_stats()
+            total_vectors = stats.get('total_vector_count', 0)
+            logger.info(f"Pinecone index stats - Total vectors: {total_vectors}")
+        except Exception as e:
+            logger.error(f"Error getting index stats: {e}")
+        
+        # Use Pinecone from langchain_community.vectorstores
+        from langchain_community.vectorstores import Pinecone as LangchainPinecone
+        
+        logger.info(f"Creating Pinecone vectorstore with index: {index_name}, namespace: {namespace}")
+        vectorstore = LangchainPinecone.from_existing_index(
+            embedding=embeddings,
+            index_name=index_name,
+            namespace=namespace,
+            text_key="text" 
+        )
+        
+        # Create threshold-based retriever
+        logger.info(f"Creating ThresholdRetriever with top_k={top_k}, limit_k={limit_k}, " +
+                    f"metric={similarity_metric}, threshold={similarity_threshold}")
+        
+        # Create ThresholdRetriever with both vectorstore and embeddings
+        _retriever_instance = ThresholdRetriever(
+            vectorstore=vectorstore,
+            embeddings=embeddings,  # Pass embeddings separately
+            top_k=top_k,
+            limit_k=limit_k,
+            similarity_metric=similarity_metric,
+            similarity_threshold=similarity_threshold
+        )
+        
+        logger.info(f"Pinecone retriever initialized in {time.time() - start_time:.2f} seconds")
+        
+        # Cache the retriever with longer TTL (1 hour) since it rarely changes
+        cache.set(cache_key, _retriever_instance, ttl=3600)
+        
+        return _retriever_instance
+    except Exception as e:
+        logger.error(f"Error creating retrieval chain: {e}")
+        return None

app/database/postgresql.py

@@ -0,0 +1,104 @@
+import os
+from sqlalchemy import create_engine, text
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.exc import SQLAlchemyError, OperationalError
+from dotenv import load_dotenv
+import logging
+
+# Cấu hình logging
+logger = logging.getLogger(__name__)
+
+# Load environment variables
+load_dotenv()
+
+# Get DB connection mode from environment
+DB_CONNECTION_MODE = os.getenv("DB_CONNECTION_MODE", "aiven")
+
+# Set connection string based on mode
+if DB_CONNECTION_MODE == "aiven":
+    DATABASE_URL = os.getenv("AIVEN_DB_URL")
+else:
+    # Default or other connection modes can be added here
+    DATABASE_URL = os.getenv("AIVEN_DB_URL")
+
+if not DATABASE_URL:
+    logger.error("No database URL configured. Please set AIVEN_DB_URL environment variable.")
+    DATABASE_URL = "postgresql://localhost/test"  # Fallback để không crash khi khởi động
+
+# Create SQLAlchemy engine
+try:
+    engine = create_engine(
+        DATABASE_URL,
+        pool_pre_ping=True,
+        pool_recycle=300,  # Recycle connections every 5 minutes
+        pool_size=10,      # Tăng kích thước pool từ 5 lên 10
+        max_overflow=20,   # Tăng số lượng kết nối tối đa từ 10 lên 20
+        connect_args={
+            "connect_timeout": 3,  # Giảm timeout từ 5 xuống 3 giây
+            "keepalives": 1,      # Bật keepalive
+            "keepalives_idle": 30, # Thời gian idle trước khi gửi keepalive
+            "keepalives_interval": 10, # Khoảng thời gian giữa các gói keepalive
+            "keepalives_count": 5  # Số lần thử lại trước khi đóng kết nối
+        },
+        # Thêm các tùy chọn hiệu suất
+        isolation_level="READ COMMITTED",  # Mức cô lập thấp hơn READ COMMITTED
+        echo=False,  # Tắt echo SQL để giảm overhead logging
+        echo_pool=False  # Tắt echo pool để giảm overhead logging
+    )
+    logger.info("PostgreSQL engine initialized")
+except Exception as e:
+    logger.error(f"Failed to initialize PostgreSQL engine: {e}")
+    # Không raise exception để tránh crash khi khởi động, các xử lý lỗi sẽ được thực hiện ở các function
+
+# Create session factory with optimized settings
+SessionLocal = sessionmaker(
+    autocommit=False, 
+    autoflush=False, 
+    bind=engine,
+    expire_on_commit=False  # Tránh truy vấn lại DB sau khi commit
+)
+
+# Base class for declarative models - use sqlalchemy.orm for SQLAlchemy 2.0 compatibility
+from sqlalchemy.orm import declarative_base
+Base = declarative_base()
+
+# Kiểm tra kết nối PostgreSQL
+def check_db_connection():
+    """Kiểm tra kết nối PostgreSQL"""
+    try:
+        # Thực hiện một truy vấn đơn giản để kiểm tra kết nối
+        with engine.connect() as connection:
+            connection.execute(text("SELECT 1"))
+        logger.info("PostgreSQL connection is working")
+        return True
+    except OperationalError as e:
+        logger.error(f"PostgreSQL connection failed: {e}")
+        return False
+    except Exception as e:
+        logger.error(f"Unknown error when checking PostgreSQL connection: {e}")
+        return False
+
+# Dependency to get DB session
+def get_db():
+    """Get database session dependency for FastAPI endpoints"""
+    db = SessionLocal()
+    try:
+        yield db
+    except SQLAlchemyError as e:
+        logger.error(f"Database session error: {e}")
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+# Tạo các bảng trong cơ sở dữ liệu nếu chưa tồn tại
+def create_tables():
+    """Tạo các bảng trong cơ sở dữ liệu"""
+    try:
+        Base.metadata.create_all(bind=engine)
+        logger.info("Database tables created or already exist")
+        return True
+    except SQLAlchemyError as e:
+        logger.error(f"Failed to create database tables: {e}")
+        return False 

app/models/__init__.py

@@ -0,0 +1 @@
+# Pydantic models package 

app/models/mongodb_models.py

@@ -0,0 +1,55 @@
+from pydantic import BaseModel, Field, ConfigDict
+from typing import Optional, List, Dict, Any
+from datetime import datetime
+import uuid
+
+class SessionBase(BaseModel):
+    """Base model for session data"""
+    session_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    factor: str
+    action: str
+    first_name: str
+    last_name: Optional[str] = None
+    message: Optional[str] = None
+    user_id: str
+    username: Optional[str] = None
+
+class SessionCreate(SessionBase):
+    """Model for creating new session"""
+    response: Optional[str] = None
+
+class SessionResponse(SessionBase):
+    """Response model for session data"""
+    created_at: str
+    response: Optional[str] = None
+
+    model_config = ConfigDict(
+        json_schema_extra={
+            "example": {
+                "session_id": "123e4567-e89b-12d3-a456-426614174000",
+                "factor": "user",
+                "action": "asking_freely",
+                "created_at": "2023-06-01 14:30:45",
+                "first_name": "John",
+                "last_name": "Doe",
+                "message": "How can I find emergency contacts?",
+                "user_id": "12345678",
+                "username": "johndoe",
+                "response": "You can find emergency contacts in the Emergency section..."
+            }
+        }
+    )
+
+class HistoryRequest(BaseModel):
+    """Request model for history"""
+    user_id: str
+    n: int = 3
+
+class QuestionAnswer(BaseModel):
+    """Model for question-answer pair"""
+    question: str
+    answer: str
+
+class HistoryResponse(BaseModel):
+    """Response model for history"""
+    history: List[QuestionAnswer] 

app/models/rag_models.py

@@ -0,0 +1,68 @@
+from pydantic import BaseModel, Field
+from typing import Optional, List, Dict, Any
+
+class ChatRequest(BaseModel):
+    """Request model for chat endpoint"""
+    user_id: str = Field(..., description="User ID from Telegram")
+    question: str = Field(..., description="User's question")
+    include_history: bool = Field(True, description="Whether to include user history in prompt")
+    use_rag: bool = Field(True, description="Whether to use RAG")
+    
+    # Advanced retrieval parameters
+    similarity_top_k: int = Field(6, description="Number of top similar documents to return (after filtering)")
+    limit_k: int = Field(10, description="Maximum number of documents to retrieve from vector store")
+    similarity_metric: str = Field("cosine", description="Similarity metric to use (cosine, dotproduct, euclidean)")
+    similarity_threshold: float = Field(0.75, description="Threshold for vector similarity (0-1)")
+    
+    # User information
+    session_id: Optional[str] = Field(None, description="Session ID for tracking conversations")
+    first_name: Optional[str] = Field(None, description="User's first name")
+    last_name: Optional[str] = Field(None, description="User's last name")
+    username: Optional[str] = Field(None, description="User's username")
+
+class SourceDocument(BaseModel):
+    """Model for source documents"""
+    text: str = Field(..., description="Text content of the document")
+    source: Optional[str] = Field(None, description="Source of the document")
+    score: Optional[float] = Field(None, description="Raw similarity score of the document")
+    normalized_score: Optional[float] = Field(None, description="Normalized similarity score (0-1)")
+    metadata: Optional[Dict[str, Any]] = Field(None, description="Metadata of the document")
+
+class ChatResponse(BaseModel):
+    """Response model for chat endpoint"""
+    answer: str = Field(..., description="Generated answer")
+    processing_time: float = Field(..., description="Processing time in seconds")
+
+class ChatResponseInternal(BaseModel):
+    """Internal model for chat response with sources - used only for logging"""
+    answer: str
+    sources: Optional[List[SourceDocument]] = Field(None, description="Source documents used for generating answer")
+    processing_time: Optional[float] = None
+
+class EmbeddingRequest(BaseModel):
+    """Request model for embedding endpoint"""
+    text: str = Field(..., description="Text to generate embedding for")
+
+class EmbeddingResponse(BaseModel):
+    """Response model for embedding endpoint"""
+    embedding: List[float] = Field(..., description="Generated embedding")
+    text: str = Field(..., description="Text that was embedded")
+    model: str = Field(..., description="Model used for embedding")
+
+class HealthResponse(BaseModel):
+    """Response model for health endpoint"""
+    status: str
+    services: Dict[str, bool]
+    timestamp: str
+
+class UserMessageModel(BaseModel):
+    """Model for user messages sent to the RAG API"""
+    user_id: str = Field(..., description="User ID from the client application")
+    session_id: str = Field(..., description="Session ID for tracking the conversation")
+    message: str = Field(..., description="User's message/question")
+    
+    # Advanced retrieval parameters (optional)
+    similarity_top_k: Optional[int] = Field(None, description="Number of top similar documents to return (after filtering)")
+    limit_k: Optional[int] = Field(None, description="Maximum number of documents to retrieve from vector store")
+    similarity_metric: Optional[str] = Field(None, description="Similarity metric to use (cosine, dotproduct, euclidean)")
+    similarity_threshold: Optional[float] = Field(None, description="Threshold for vector similarity (0-1)") 

app/utils/__init__.py

@@ -0,0 +1 @@
+# Utility functions package 

app/utils/debug_utils.py

@@ -0,0 +1,207 @@
+import os
+import sys
+import logging
+import traceback
+import json
+import time
+from datetime import datetime
+import platform
+
+# Try to import psutil, provide fallback if not available
+try:
+    import psutil
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    PSUTIL_AVAILABLE = False
+    logging.warning("psutil module not available. System monitoring features will be limited.")
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+class DebugInfo:
+    """Class containing debug information"""
+    
+    @staticmethod
+    def get_system_info():
+        """Get system information"""
+        try:
+            info = {
+                "os": platform.system(),
+                "os_version": platform.version(),
+                "python_version": platform.python_version(),
+                "cpu_count": os.cpu_count(),
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            # Add information from psutil if available
+            if PSUTIL_AVAILABLE:
+                info.update({
+                    "total_memory": round(psutil.virtual_memory().total / (1024 * 1024 * 1024), 2),  # GB
+                    "available_memory": round(psutil.virtual_memory().available / (1024 * 1024 * 1024), 2),  # GB
+                    "cpu_usage": psutil.cpu_percent(interval=0.1),
+                    "memory_usage": psutil.virtual_memory().percent,
+                    "disk_usage": psutil.disk_usage('/').percent,
+                })
+            else:
+                info.update({
+                    "total_memory": "psutil not available",
+                    "available_memory": "psutil not available",
+                    "cpu_usage": "psutil not available",
+                    "memory_usage": "psutil not available",
+                    "disk_usage": "psutil not available",
+                })
+            
+            return info
+        except Exception as e:
+            logger.error(f"Error getting system info: {e}")
+            return {"error": str(e)}
+    
+    @staticmethod
+    def get_env_info():
+        """Get environment variable information (masking sensitive information)"""
+        try:
+            # List of environment variables to mask values
+            sensitive_vars = [
+                "API_KEY", "SECRET", "PASSWORD", "TOKEN", "AUTH", "MONGODB_URL", 
+                "AIVEN_DB_URL", "PINECONE_API_KEY", "GOOGLE_API_KEY"
+            ]
+            
+            env_vars = {}
+            for key, value in os.environ.items():
+                # Check if environment variable contains sensitive words
+                is_sensitive = any(s in key.upper() for s in sensitive_vars)
+                
+                if is_sensitive and value:
+                    # Mask value displaying only the first 4 characters
+                    masked_value = value[:4] + "****" if len(value) > 4 else "****"
+                    env_vars[key] = masked_value
+                else:
+                    env_vars[key] = value
+                    
+            return env_vars
+        except Exception as e:
+            logger.error(f"Error getting environment info: {e}")
+            return {"error": str(e)}
+    
+    @staticmethod
+    def get_database_status():
+        """Get database connection status"""
+        try:
+            from app.database.postgresql import check_db_connection as check_postgresql
+            from app.database.mongodb import check_db_connection as check_mongodb
+            from app.database.pinecone import check_db_connection as check_pinecone
+            
+            return {
+                "postgresql": check_postgresql(),
+                "mongodb": check_mongodb(),
+                "pinecone": check_pinecone(),
+                "timestamp": datetime.now().isoformat()
+            }
+        except Exception as e:
+            logger.error(f"Error getting database status: {e}")
+            return {"error": str(e)}
+
+class PerformanceMonitor:
+    """Performance monitoring class"""
+    
+    def __init__(self):
+        self.start_time = time.time()
+        self.checkpoints = []
+    
+    def checkpoint(self, name):
+        """Mark a checkpoint and record the time"""
+        current_time = time.time()
+        elapsed = current_time - self.start_time
+        self.checkpoints.append({
+            "name": name,
+            "time": current_time,
+            "elapsed": elapsed
+        })
+        logger.debug(f"Checkpoint '{name}' at {elapsed:.4f}s")
+        return elapsed
+    
+    def get_report(self):
+        """Generate performance report"""
+        if not self.checkpoints:
+            return {"error": "No checkpoints recorded"}
+            
+        total_time = time.time() - self.start_time
+        
+        # Calculate time between checkpoints
+        intervals = []
+        prev_time = self.start_time
+        
+        for checkpoint in self.checkpoints:
+            interval = checkpoint["time"] - prev_time
+            intervals.append({
+                "name": checkpoint["name"],
+                "interval": interval,
+                "elapsed": checkpoint["elapsed"]
+            })
+            prev_time = checkpoint["time"]
+            
+        return {
+            "total_time": total_time,
+            "checkpoint_count": len(self.checkpoints),
+            "intervals": intervals
+        }
+
+class ErrorTracker:
+    """Class to track and record errors"""
+    
+    def __init__(self, max_errors=100):
+        self.errors = []
+        self.max_errors = max_errors
+    
+    def track_error(self, error, context=None):
+        """Record error information"""
+        error_info = {
+            "error_type": type(error).__name__,
+            "error_message": str(error),
+            "traceback": traceback.format_exc(),
+            "timestamp": datetime.now().isoformat(),
+            "context": context or {}
+        }
+        
+        # Add to error list
+        self.errors.append(error_info)
+        
+        # Limit the number of stored errors
+        if len(self.errors) > self.max_errors:
+            self.errors.pop(0)  # Remove oldest error
+            
+        return error_info
+    
+    def get_errors(self, limit=None):
+        """Get list of recorded errors"""
+        if limit is None or limit >= len(self.errors):
+            return self.errors
+        return self.errors[-limit:]  # Return most recent errors
+
+# Initialize global objects
+error_tracker = ErrorTracker()
+performance_monitor = PerformanceMonitor()
+
+def debug_view(request=None):
+    """Create a full debug report"""
+    debug_data = {
+        "system_info": DebugInfo.get_system_info(),
+        "database_status": DebugInfo.get_database_status(),
+        "performance": performance_monitor.get_report(),
+        "recent_errors": error_tracker.get_errors(limit=10),
+        "timestamp": datetime.now().isoformat()
+    }
+    
+    # Add request information if available
+    if request:
+        debug_data["request"] = {
+            "method": request.method,
+            "url": str(request.url),
+            "headers": dict(request.headers),
+            "client": {
+                "host": request.client.host if request.client else "unknown",
+                "port": request.client.port if request.client else "unknown"
+            }
+        }
+    
+    return debug_data 

app/utils/middleware.py

@@ -0,0 +1,109 @@
+from fastapi import Request, status
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+import logging
+import time
+import traceback
+import uuid
+from .utils import get_local_time
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+class RequestLoggingMiddleware(BaseHTTPMiddleware):
+    """Middleware to log requests and responses"""
+    
+    async def dispatch(self, request: Request, call_next):
+        request_id = str(uuid.uuid4())
+        request.state.request_id = request_id
+        
+        # Log request information
+        client_host = request.client.host if request.client else "unknown"
+        logger.info(f"Request [{request_id}]: {request.method} {request.url.path} from {client_host}")
+        
+        # Measure processing time
+        start_time = time.time()
+        
+        try:
+            # Process request
+            response = await call_next(request)
+            
+            # Calculate processing time
+            process_time = time.time() - start_time
+            logger.info(f"Response [{request_id}]: {response.status_code} processed in {process_time:.4f}s")
+            
+            # Add headers
+            response.headers["X-Request-ID"] = request_id
+            response.headers["X-Process-Time"] = str(process_time)
+            
+            return response
+            
+        except Exception as e:
+            # Log error
+            process_time = time.time() - start_time
+            logger.error(f"Error [{request_id}] after {process_time:.4f}s: {str(e)}")
+            logger.error(traceback.format_exc())
+            
+            # Return error response
+            return JSONResponse(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                content={
+                    "detail": "Internal server error",
+                    "request_id": request_id,
+                    "timestamp": get_local_time()
+                }
+            )
+
+class ErrorHandlingMiddleware(BaseHTTPMiddleware):
+    """Middleware to handle uncaught exceptions in the application"""
+    
+    async def dispatch(self, request: Request, call_next):
+        try:
+            return await call_next(request)
+        except Exception as e:
+            # Get request_id if available
+            request_id = getattr(request.state, "request_id", str(uuid.uuid4()))
+            
+            # Log error
+            logger.error(f"Uncaught exception [{request_id}]: {str(e)}")
+            logger.error(traceback.format_exc())
+            
+            # Return error response
+            return JSONResponse(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                content={
+                    "detail": "Internal server error",
+                    "request_id": request_id,
+                    "timestamp": get_local_time()
+                }
+            )
+
+class DatabaseCheckMiddleware(BaseHTTPMiddleware):
+    """Middleware to check database connections before each request"""
+    
+    async def dispatch(self, request: Request, call_next):
+        # Skip paths that don't need database checks
+        skip_paths = ["/", "/health", "/docs", "/redoc", "/openapi.json"]
+        if request.url.path in skip_paths:
+            return await call_next(request)
+        
+        # Check database connections
+        try:
+            # TODO: Add checks for MongoDB and Pinecone if needed
+            # PostgreSQL check is already done in route handler with get_db() method
+            
+            # Process request normally
+            return await call_next(request)
+            
+        except Exception as e:
+            # Log error
+            logger.error(f"Database connection check failed: {str(e)}")
+            
+            # Return error response
+            return JSONResponse(
+                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                content={
+                    "detail": "Database connection failed",
+                    "timestamp": get_local_time()
+                }
+            ) 

app/utils/utils.py

@@ -0,0 +1,126 @@
+import logging
+import time
+import uuid
+import threading
+from functools import wraps
+from datetime import datetime, timedelta
+import pytz
+from typing import Callable, Any, Dict, Optional
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+)
+logger = logging.getLogger(__name__)
+
+# Asia/Ho_Chi_Minh timezone
+asia_tz = pytz.timezone('Asia/Ho_Chi_Minh')
+
+def generate_uuid():
+    """Generate a unique identifier"""
+    return str(uuid.uuid4())
+
+def get_current_time():
+    """Get current time in ISO format"""
+    return datetime.now().isoformat()
+
+def get_local_time():
+    """Get current time in Asia/Ho_Chi_Minh timezone"""
+    return datetime.now(asia_tz).strftime("%Y-%m-%d %H:%M:%S")
+
+def get_local_datetime():
+    """Get current datetime object in Asia/Ho_Chi_Minh timezone"""
+    return datetime.now(asia_tz)
+
+# For backward compatibility
+get_vietnam_time = get_local_time
+get_vietnam_datetime = get_local_datetime
+
+def timer_decorator(func: Callable) -> Callable:
+    """
+    Decorator to time function execution and log results.
+    """
+    @wraps(func)
+    async def wrapper(*args, **kwargs):
+        start_time = time.time()
+        try:
+            result = await func(*args, **kwargs)
+            elapsed_time = time.time() - start_time
+            logger.info(f"Function {func.__name__} executed in {elapsed_time:.4f} seconds")
+            return result
+        except Exception as e:
+            elapsed_time = time.time() - start_time
+            logger.error(f"Function {func.__name__} failed after {elapsed_time:.4f} seconds: {e}")
+            raise
+    return wrapper
+
+def sanitize_input(text):
+    """Sanitize input text"""
+    if not text:
+        return ""
+    # Remove potential dangerous characters or patterns
+    return text.strip()
+
+def truncate_text(text, max_length=100):
+    """
+    Truncate text to given max length and add ellipsis.
+    """
+    if not text or len(text) <= max_length:
+        return text
+    return text[:max_length] + "..."
+
+# Simple in-memory cache implementation (replaces Redis dependency)
+class SimpleCache:
+    def __init__(self):
+        self._cache = {}
+        self._expiry = {}
+    
+    def get(self, key: str) -> Optional[Any]:
+        """Get value from cache if it exists and hasn't expired"""
+        if key in self._cache:
+            # Check if the key has expired
+            if key in self._expiry and self._expiry[key] > datetime.now():
+                return self._cache[key]
+            else:
+                # Clean up expired keys
+                if key in self._cache:
+                    del self._cache[key]
+                if key in self._expiry:
+                    del self._expiry[key]
+        return None
+    
+    def set(self, key: str, value: Any, ttl: int = 300) -> None:
+        """Set a value in the cache with TTL in seconds"""
+        self._cache[key] = value
+        # Set expiry time
+        self._expiry[key] = datetime.now() + timedelta(seconds=ttl)
+    
+    def delete(self, key: str) -> None:
+        """Delete a key from the cache"""
+        if key in self._cache:
+            del self._cache[key]
+        if key in self._expiry:
+            del self._expiry[key]
+    
+    def clear(self) -> None:
+        """Clear the entire cache"""
+        self._cache.clear()
+        self._expiry.clear()
+
+# Initialize cache
+cache = SimpleCache()
+
+def get_host_url(request) -> str:
+    """
+    Get the host URL from a request object.
+    """
+    host = request.headers.get("host", "localhost")
+    scheme = request.headers.get("x-forwarded-proto", "http")
+    return f"{scheme}://{host}"
+
+def format_time(timestamp):
+    """
+    Format a timestamp into a human-readable string.
+    """
+    return timestamp.strftime("%Y-%m-%d %H:%M:%S") 

docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+
+services:
+  backend:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "7860:7860"
+    env_file:
+      - .env
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7860/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    volumes:
+      - ./app:/app/app
+    command: uvicorn app:app --host 0.0.0.0 --port 7860 --reload 

pytest.ini

@@ -0,0 +1,12 @@
+[pytest]
+# Bỏ qua cảnh báo về anyio module và các cảnh báo vận hành nội bộ
+filterwarnings =
+    ignore::pytest.PytestAssertRewriteWarning:.*anyio
+    ignore:.*general_plain_validator_function.* is deprecated.*:DeprecationWarning
+    ignore:.*with_info_plain_validator_function.*:DeprecationWarning
+
+# Cấu hình cơ bản khác
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_* 

requirements.txt

@@ -0,0 +1,43 @@
+# FastAPI
+fastapi==0.103.1
+uvicorn[standard]==0.23.2
+pydantic==2.4.2
+python-dotenv==1.0.0
+websockets==11.0.3
+
+# MongoDB
+pymongo==4.6.1
+dnspython==2.4.2
+
+# PostgreSQL
+sqlalchemy==2.0.20
+pydantic-settings==2.0.3
+psycopg2-binary==2.9.7
+
+# Pinecone & RAG
+pinecone-client==3.0.0
+langchain==0.1.4
+langchain-core==0.1.19
+langchain-community==0.0.14
+langchain-google-genai==0.0.5
+langchain-pinecone==0.0.1
+faiss-cpu==1.7.4
+google-generativeai==0.3.1
+
+# Extras
+pytz==2023.3
+python-multipart==0.0.6
+httpx==0.25.1
+requests==2.31.0
+beautifulsoup4==4.12.2
+redis==5.0.1
+
+# Testing
+prometheus-client==0.17.1
+pytest==7.4.0
+pytest-cov==4.1.0
+watchfiles==0.21.0
+
+# Core dependencies
+starlette==0.27.0
+psutil==5.9.6