QA to PROD

app/api/pdf_routes.py

@@ -98,23 +98,47 @@ async def send_progress_update(user_id, file_id, step, progress=0.0, message="")
 
 # Function with fixed indentation for the troublesome parts
 async def handle_pdf_processing_result(result, correlation_id, user_id, file_id, filename, document, vector_status, 
-                                    vector_database_id, temp_file_path, db, is_pdf, mock_mode):
-    """Fixed version of the code with proper indentation"""
-    # If successful, update status but don't try to permanently store files
+                                    vector_database_id, temp_file_path, db, is_pdf):
+    """Process the result of PDF processing and update database records"""
+    # If successful, move file to permanent storage
     if result.get('success'):
         try:
-            log_upload_debug(correlation_id, f"Processed file successfully - no permanent storage in Hugging Face environment")
+            storage_path = os.path.join(STORAGE_DIR, f"{file_id}{'.pdf' if is_pdf else '.txt'}")
+            shutil.move(temp_file_path, storage_path)
+            log_upload_debug(correlation_id, f"Moved file to storage at {storage_path}")
         except Exception as move_error:
-            log_upload_debug(correlation_id, f"Error in storage handling: {move_error}", move_error)
+            log_upload_debug(correlation_id, f"Error moving file to storage: {move_error}", move_error)
         
         # Update status in PostgreSQL
         if vector_database_id and document and vector_status:
             try:
                 log_upload_debug(correlation_id, f"Updating vector status to 'completed' for document ID {document.id}")
+                
+                # Update the vector status with the result document_id (important for later deletion)
+                result_document_id = result.get('document_id') 
+                
                 vector_status.status = "completed"
                 vector_status.embedded_at = datetime.now()
-                vector_status.vector_id = file_id
+                
+                # Critical: Store the correct vector ID for future deletion
+                # This can be either the original file_id or the result_document_id
+                if result_document_id and result_document_id != file_id:
+                    # If Pinecone returned a specific document_id, use that
+                    vector_status.vector_id = result_document_id
+                    log_upload_debug(correlation_id, f"Updated vector_id to {result_document_id} (from result)")
+                elif file_id:
+                    # Make sure file_id is stored as the vector_id
+                    vector_status.vector_id = file_id
+                    log_upload_debug(correlation_id, f"Updated vector_id to {file_id} (from file_id)")
+                
+                # Also ensure we store some backup identifiers in case the primary one fails
+                # Store the document name as a secondary identifier
+                vector_status.document_name = document.name
+                log_upload_debug(correlation_id, f"Stored document_name '{document.name}' in vector status for backup")
+                
+                # Mark document as embedded
                 document.is_embedded = True
+                
                 db.commit()
                 log_upload_debug(correlation_id, f"Database status updated successfully")
             except Exception as db_error:
@@ -136,9 +160,6 @@ async def handle_pdf_processing_result(result, correlation_id, user_id, file_id,
         # Add document information to the result
         if document:
             result["document_database_id"] = document.id
-            
-        # Include mock_mode in response
-        result["mock_mode"] = mock_mode
     else:
         log_upload_debug(correlation_id, f"PDF processing failed: {result.get('error', 'Unknown error')}")
         
@@ -167,7 +188,7 @@ async def handle_pdf_processing_result(result, correlation_id, user_id, file_id,
                 log_upload_debug(correlation_id, f"Error sending WebSocket notification: {ws_error}", ws_error)
         
     # Cleanup: delete temporary file if it still exists
-    if os.path.exists(temp_file_path):
+    if temp_file_path and os.path.exists(temp_file_path):
         try:
             os.remove(temp_file_path)
             log_upload_debug(correlation_id, f"Removed temporary file {temp_file_path}")
@@ -189,7 +210,6 @@ async def upload_pdf(
     vector_database_id: Optional[int] = Form(None),
     content_type: Optional[str] = Form(None),  # Add content_type parameter
     background_tasks: BackgroundTasks = None,
-    mock_mode: bool = Form(False),  # Set to False to use real database
     db: Session = Depends(get_db)
 ):
     """
@@ -203,12 +223,18 @@ async def upload_pdf(
     - **user_id**: User ID for WebSocket status updates
     - **vector_database_id**: ID of vector database in PostgreSQL (optional)
     - **content_type**: Content type of the file (optional)
-    - **mock_mode**: Simulate Pinecone operations instead of performing real calls (default: false)
+    
+    Note: Mock mode has been permanently removed and the system always operates in real mode
     """
     # Generate request ID for tracking
     correlation_id = str(uuid.uuid4())[:8]
     logger.info(f"[{correlation_id}] PDF upload request received: ns={namespace}, index={index_name}, user={user_id}")
-    log_upload_debug(correlation_id, f"Upload request: vector_db_id={vector_database_id}, mock_mode={mock_mode}")
+    log_upload_debug(correlation_id, f"Upload request: vector_db_id={vector_database_id}")
+    
+    # Variables that might need cleanup in case of error
+    temp_file_path = None
+    document = None
+    vector_status = None
     
     try:
         # Check file type - accept both PDF and plaintext for testing
@@ -218,13 +244,8 @@ async def upload_pdf(
         log_upload_debug(correlation_id, f"File type check: is_pdf={is_pdf}, is_text={is_text}, filename={file.filename}")
         
         if not (is_pdf or is_text):
-            if not mock_mode:
-                # In real mode, only accept PDFs
-                log_upload_debug(correlation_id, f"Rejecting non-PDF file in real mode: {file.filename}")
-                raise HTTPException(status_code=400, detail="Only PDF files are accepted")
-            else:
-                # In mock mode, convert any file to text for testing
-                logger.warning(f"[{correlation_id}] Non-PDF file uploaded in mock mode: {file.filename} - will treat as text")
+            log_upload_debug(correlation_id, f"Rejecting non-PDF file: {file.filename}")
+            raise HTTPException(status_code=400, detail="Only PDF files are accepted")
         
         # If vector_database_id provided, get info from PostgreSQL
         api_key = None
@@ -237,11 +258,10 @@ async def upload_pdf(
                 VectorDatabase.id == vector_database_id,
                 VectorDatabase.status == "active"
             ).first()
+            
             if not vector_db:
-                return PDFResponse(
-                    success=False,
-                    error=f"Vector database with ID {vector_database_id} not found or inactive"
-                )
+                log_upload_debug(correlation_id, f"Vector database {vector_database_id} not found or inactive")
+                raise HTTPException(status_code=404, detail="Vector database not found or inactive")
             
             log_upload_debug(correlation_id, f"Found vector database: id={vector_db.id}, name={vector_db.name}, index={vector_db.pinecone_index}")
             
@@ -325,13 +345,26 @@ async def upload_pdf(
             
         metadata["content_type"] = actual_content_type
         
-        if title:
-            metadata["title"] = title
-        else:
-            # Use filename as title if not provided
-            title = file.filename
-            metadata["title"] = title
+        # Use provided title or filename as document name
+        document_name = title or file.filename
+        
+        # Verify document name is unique within this vector database
+        if vector_database_id:
+            # Check if a document with this name already exists in this vector database
+            existing_doc = db.query(Document).filter(
+                Document.name == document_name,
+                Document.vector_database_id == vector_database_id
+            ).first()
             
+            if existing_doc:
+                # Make the name unique by appending timestamp
+                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+                base_name, extension = os.path.splitext(document_name)
+                document_name = f"{base_name}_{timestamp}{extension}"
+                log_upload_debug(correlation_id, f"Document name already exists, using unique name: {document_name}")
+        
+        metadata["title"] = document_name
+        
         if description:
             metadata["description"] = description
         
@@ -359,7 +392,7 @@ async def upload_pdf(
             # Create document record without file content
             try:
                 document = Document(
-                    name=title or file.filename,
+                    name=document_name,  # Use the (potentially) modified document name
                     file_type="pdf" if is_pdf else "text",
                     content_type=actual_content_type,  # Use the actual_content_type here
                     size=len(file_content),
@@ -387,16 +420,17 @@ async def upload_pdf(
                 log_upload_debug(correlation_id, f"Error creating document content: {content_error}", content_error)
                 raise
             
-            # Create vector status record
+            # Create vector status record - store file_id as the vector_id for deletion later
             try:
                 vector_status = VectorStatus(
                     document_id=document.id,
                     vector_database_id=vector_database_id,
-                    status="pending"
+                    status="pending",
+                    vector_id=file_id  # Store the document UUID as vector_id for later deletion
                 )
                 db.add(vector_status)
                 db.commit()
-                log_upload_debug(correlation_id, f"Created vector status record for document ID {document.id}")
+                log_upload_debug(correlation_id, f"Created vector status record for document ID {document.id} with vector_id={file_id}")
             except Exception as status_error:
                 log_upload_debug(correlation_id, f"Error creating vector status: {status_error}", status_error)
                 raise
@@ -404,13 +438,12 @@ async def upload_pdf(
             logger.info(f"[{correlation_id}] Created document ID {document.id} and vector status in PostgreSQL")
             
         # Initialize PDF processor with correct parameters
-        log_upload_debug(correlation_id, f"Initializing PDFProcessor: index={index_name}, vector_db_id={vector_database_id}, mock_mode={mock_mode}")
+        log_upload_debug(correlation_id, f"Initializing PDFProcessor: index={index_name}, vector_db_id={vector_database_id}")
         processor = PDFProcessor(
             index_name=index_name, 
             namespace=namespace, 
             api_key=api_key, 
             vector_db_id=vector_database_id,
-            mock_mode=mock_mode,
             correlation_id=correlation_id
         )
         
@@ -432,7 +465,7 @@ async def upload_pdf(
         log_upload_debug(correlation_id, f"Processing PDF with file_path={temp_file_path}, document_id={file_id}")
         result = await processor.process_pdf(
             file_path=temp_file_path,
-            document_id=file_id,
+            document_id=file_id,  # Use UUID as document_id for Pinecone
             metadata=metadata,
             progress_callback=send_progress_update if user_id else None
         )
@@ -441,53 +474,47 @@ async def upload_pdf(
         
         # Handle PDF processing result
         return await handle_pdf_processing_result(result, correlation_id, user_id, file_id, file.filename, document, vector_status, 
-                                                vector_database_id, temp_file_path, db, is_pdf, mock_mode)
+                                                vector_database_id, temp_file_path, db, is_pdf)
     except Exception as e:
-        return await handle_upload_error(e, correlation_id, temp_file_path, user_id, file_id, file.filename, vector_database_id, vector_status, db, mock_mode)
-
-# Error handling for upload_pdf function
-async def handle_upload_error(e, correlation_id, temp_file_path, user_id, file_id, filename, vector_database_id, vector_status, db, mock_mode):
-    """Fixed version of the error handling part with proper indentation"""
-    log_upload_debug(correlation_id, f"Error in upload_pdf: {str(e)}", e)
-    logger.exception(f"[{correlation_id}] Error in upload_pdf: {str(e)}")
-    
-    # Cleanup on error
-    if os.path.exists(temp_file_path):
-        try:
-            os.remove(temp_file_path)
-            log_upload_debug(correlation_id, f"Cleaned up temp file after error: {temp_file_path}")
-        except Exception as cleanup_error:
-            log_upload_debug(correlation_id, f"Error cleaning up temporary file: {cleanup_error}", cleanup_error)
+        log_upload_debug(correlation_id, f"Error in upload_pdf: {str(e)}", e)
+        logger.exception(f"[{correlation_id}] Error in upload_pdf: {str(e)}")
         
-    # Update error status in PostgreSQL
-    if vector_database_id and vector_status:
-        try:
-            vector_status.status = "failed"
-            vector_status.error_message = str(e)
-            db.commit()
-            log_upload_debug(correlation_id, f"Updated database with error status")
-        except Exception as db_error:
-            log_upload_debug(correlation_id, f"Error updating database with error status: {db_error}", db_error)
-        
-    # Send failure notification via WebSocket
-    if user_id and file_id:
-        try:
-            await send_pdf_upload_failed(
-                user_id,
-                file_id,
-                filename,
-                str(e)
-            )
-            log_upload_debug(correlation_id, f"Sent failure notification for exception")
-        except Exception as ws_error:
-            log_upload_debug(correlation_id, f"Error sending WebSocket notification for failure: {ws_error}", ws_error)
-            
-    log_upload_debug(correlation_id, f"Upload request failed with exception: {str(e)}")
-    return PDFResponse(
-        success=False,
-        error=str(e),
-        mock_mode=mock_mode
-    )
+        # Cleanup on error
+        if os.path.exists(temp_file_path):
+            try:
+                os.remove(temp_file_path)
+                log_upload_debug(correlation_id, f"Cleaned up temp file after error: {temp_file_path}")
+            except Exception as cleanup_error:
+                log_upload_debug(correlation_id, f"Error cleaning up temporary file: {cleanup_error}", cleanup_error)
+        
+        # Update error status in PostgreSQL
+        if vector_database_id and vector_status:
+            try:
+                vector_status.status = "failed"
+                vector_status.error_message = str(e)
+                db.commit()
+                log_upload_debug(correlation_id, f"Updated database with error status")
+            except Exception as db_error:
+                log_upload_debug(correlation_id, f"Error updating database with error status: {db_error}", db_error)
+            
+        # Send failure notification via WebSocket
+        if user_id and file_id:
+            try:
+                await send_pdf_upload_failed(
+                    user_id,
+                    file_id,
+                    file.filename,
+                    str(e)
+                )
+                log_upload_debug(correlation_id, f"Sent failure notification for exception")
+            except Exception as ws_error:
+                log_upload_debug(correlation_id, f"Error sending WebSocket notification for failure: {ws_error}", ws_error)
+            
+        log_upload_debug(correlation_id, f"Upload request failed with exception: {str(e)}")
+        return PDFResponse(
+            success=False,
+            error=str(e)
+        )
 
 # Endpoint xóa tài liệu
 @router.delete("/namespace", response_model=PDFResponse)
@@ -512,7 +539,6 @@ async def delete_namespace(
         # Nếu có vector_database_id, lấy thông tin từ PostgreSQL
         api_key = None
         vector_db = None
-        mock_mode = False  # Use real mode by default
 
         if vector_database_id:
             vector_db = db.query(VectorDatabase).filter(
@@ -546,13 +572,12 @@ async def delete_namespace(
             index_name=index_name, 
             namespace=namespace,
             api_key=api_key,
-            vector_db_id=vector_database_id,
-            mock_mode=mock_mode
+            vector_db_id=vector_database_id
         )
         result = await processor.delete_namespace()
         
-        # If in mock mode, also update PostgreSQL to reflect the deletion
-        if mock_mode and result.get('success') and vector_database_id:
+        # If successful and vector_database_id, update PostgreSQL to reflect the deletion
+        if result.get('success') and vector_database_id:
             try:
                 # Update vector statuses for this database
                 affected_count = db.query(VectorStatus).filter(
@@ -616,7 +641,6 @@ async def get_documents(
         # Nếu có vector_database_id, lấy thông tin từ PostgreSQL
         api_key = None
         vector_db = None
-        mock_mode = False  # Use real mode by default
 
         if vector_database_id:
             vector_db = db.query(VectorDatabase).filter(
@@ -648,8 +672,7 @@ async def get_documents(
             index_name=index_name, 
             namespace=namespace,
             api_key=api_key,
-            vector_db_id=vector_database_id,
-            mock_mode=mock_mode
+            vector_db_id=vector_database_id
         )
         
         # Lấy danh sách documents từ Pinecone
@@ -691,7 +714,7 @@ async def get_documents(
         return DocumentsListResponse(
             success=False,
             error=str(e)
-        ) 
+        )
 
 # Health check endpoint for PDF API
 @router.get("/health")
@@ -710,25 +733,32 @@ async def delete_document(
     index_name: str = "testbot768",
     vector_database_id: Optional[int] = None,
     user_id: Optional[str] = None,
-    mock_mode: bool = False,
     db: Session = Depends(get_db)
 ):
     """
     Delete vectors for a specific document from the vector database
     
-    - **document_id**: ID of the document to delete
+    This endpoint can be called in two ways:
+    1. With the PostgreSQL document ID - will look up the actual vector_id first
+    2. With the actual vector_id directly - when called from the PostgreSQL document deletion endpoint
+    
+    - **document_id**: ID of the document to delete (can be PostgreSQL document ID or Pinecone vector_id)
     - **namespace**: Namespace in the vector database (default: "Default")
     - **index_name**: Name of the vector index (default: "testbot768")
     - **vector_database_id**: ID of vector database in PostgreSQL (optional)
     - **user_id**: User ID for WebSocket status updates (optional)
-    - **mock_mode**: Simulate vector database operations (default: false)
     """
-    logger.info(f"Delete document request: document_id={document_id}, namespace={namespace}, index={index_name}, vector_db_id={vector_database_id}, mock_mode={mock_mode}")
+    logger.info(f"Delete document request: document_id={document_id}, namespace={namespace}, index={index_name}, vector_db_id={vector_database_id}")
     
     try:
         # If vector_database_id is provided, get info from PostgreSQL
         api_key = None
         vector_db = None
+        pinecone_document_id = document_id  # Default to the provided document_id
+        document_to_delete = None
+        vector_status_to_update = None
+        document_found = False  # Flag to track if document was found
+        vector_id_found = False  # Flag to track if a valid vector ID was found
 
         if vector_database_id:
             vector_db = db.query(VectorDatabase).filter(
@@ -754,10 +784,58 @@ async def delete_document(
             namespace = f"vdb-{vector_database_id}" if vector_database_id else namespace
             logger.info(f"Using namespace '{namespace}' based on vector database ID")
             
+            # Check if document_id is a numeric database ID or document name
+            if document_id.isdigit():
+                # Try to find the document in PostgreSQL by its ID
+                db_document_id = int(document_id)
+                document_to_delete = db.query(Document).filter(Document.id == db_document_id).first()
+                
+                if document_to_delete:
+                    document_found = True
+                    logger.info(f"Found document in database: id={document_to_delete.id}, name={document_to_delete.name}")
+                    
+                    # Look for vector status to find the Pinecone vector_id
+                    vector_status_to_update = db.query(VectorStatus).filter(
+                        VectorStatus.document_id == document_to_delete.id,
+                        VectorStatus.vector_database_id == vector_database_id
+                    ).first()
+                    
+                    if vector_status_to_update and vector_status_to_update.vector_id:
+                        pinecone_document_id = vector_status_to_update.vector_id
+                        vector_id_found = True
+                        logger.info(f"Using vector_id '{pinecone_document_id}' from vector status")
+                    else:
+                        # Fallback options if vector_id is not directly found
+                        pinecone_document_id = document_to_delete.name
+                        logger.info(f"Vector ID not found in status, using document name '{pinecone_document_id}' as fallback")
+                else:
+                    logger.warning(f"Document with ID {db_document_id} not found in database. Using ID as is.")
+            else:
+                # Try to find document by name/title
+                document_to_delete = db.query(Document).filter(
+                    Document.name == document_id,
+                    Document.vector_database_id == vector_database_id
+                ).first()
+                
+                if document_to_delete:
+                    document_found = True
+                    logger.info(f"Found document by name: id={document_to_delete.id}, name={document_to_delete.name}")
+                    
+                    # Get vector status for this document
+                    vector_status_to_update = db.query(VectorStatus).filter(
+                        VectorStatus.document_id == document_to_delete.id,
+                        VectorStatus.vector_database_id == vector_database_id
+                    ).first()
+                    
+                    if vector_status_to_update and vector_status_to_update.vector_id:
+                        pinecone_document_id = vector_status_to_update.vector_id
+                        vector_id_found = True
+                        logger.info(f"Using vector_id '{pinecone_document_id}' from vector status")
+            
         # Send notification of deletion start via WebSocket if user_id provided
         if user_id:
             try:
-                await send_pdf_delete_started(user_id, document_id)
+                await send_pdf_delete_started(user_id, pinecone_document_id)
             except Exception as ws_error:
                 logger.error(f"Error sending WebSocket notification: {ws_error}")
         
@@ -766,47 +844,98 @@ async def delete_document(
             index_name=index_name, 
             namespace=namespace, 
             api_key=api_key, 
-            vector_db_id=vector_database_id,
-            mock_mode=mock_mode
+            vector_db_id=vector_database_id
         )
         
-        # Delete document vectors
-        result = await processor.delete_document(document_id)
+        # Delete document vectors using the pinecone_document_id and additional metadata
+        additional_metadata = {}
+        if document_to_delete:
+            # Add document name as title for searching
+            additional_metadata["document_name"] = document_to_delete.name
+        
+        result = await processor.delete_document(pinecone_document_id, additional_metadata)
+        
+        # Check if vectors were actually deleted or found
+        vectors_deleted = result.get('vectors_deleted', 0)
+        vectors_found = result.get('vectors_found', False)
+        
+        # If no document was found in PostgreSQL and no vectors were found/deleted in Pinecone
+        if not document_found and not vectors_found:
+            result['success'] = False  # Override success to false
+            result['error'] = f"Document ID {document_id} not found in PostgreSQL or Pinecone"
+            
+            # Send notification of deletion failure via WebSocket if user_id provided
+            if user_id:
+                try:
+                    await send_pdf_delete_failed(user_id, document_id, result['error'])
+                except Exception as ws_error:
+                    logger.error(f"Error sending WebSocket notification: {ws_error}")
+                    
+            return result
         
         # If successful and vector_database_id is provided, update PostgreSQL records
         if result.get('success') and vector_database_id:
             try:
-                # Find document by vector ID if it exists
-                document = db.query(Document).join(
-                    VectorStatus, Document.id == VectorStatus.document_id
-                ).filter(
-                    Document.vector_database_id == vector_database_id,
-                    VectorStatus.vector_id == document_id
-                ).first()
-                
-                if document:
-                    # Update vector status
-                    vector_status = db.query(VectorStatus).filter(
-                        VectorStatus.document_id == document.id,
-                        VectorStatus.vector_database_id == vector_database_id
-                    ).first()
+                # Update vector status if we found it earlier
+                if vector_status_to_update:
+                    vector_status_to_update.status = "deleted"
+                    db.commit()
+                    result["postgresql_updated"] = True
+                    logger.info(f"Updated vector status for document ID {document_to_delete.id if document_to_delete else document_id} to 'deleted'")
+                else:
+                    # If we didn't find it earlier, try again with more search options
+                    document = None
                     
-                    if vector_status:
-                        vector_status.status = "deleted"
-                        db.commit()
-                        result["postgresql_updated"] = True
-                        logger.info(f"Updated vector status for document ID {document.id} to 'deleted'")
+                    if document_id.isdigit():
+                        # If the original document_id was numeric, use it directly
+                        document = db.query(Document).filter(Document.id == int(document_id)).first()
+                    
+                    if not document:
+                        # Find document by vector ID if it exists
+                        document = db.query(Document).join(
+                            VectorStatus, Document.id == VectorStatus.document_id
+                        ).filter(
+                            Document.vector_database_id == vector_database_id,
+                            VectorStatus.vector_id == pinecone_document_id
+                        ).first()
+                    
+                    if not document:
+                        # Try finding by name
+                        document = db.query(Document).filter(
+                            Document.vector_database_id == vector_database_id,
+                            Document.name == pinecone_document_id
+                        ).first()
+                    
+                    if document:
+                        # Update vector status
+                        vector_status = db.query(VectorStatus).filter(
+                            VectorStatus.document_id == document.id,
+                            VectorStatus.vector_database_id == vector_database_id
+                        ).first()
+                        
+                        if vector_status:
+                            vector_status.status = "deleted"
+                            db.commit()
+                            result["postgresql_updated"] = True
+                            logger.info(f"Updated vector status for document ID {document.id} to 'deleted'")
+                    else:
+                        logger.warning(f"Could not find document record for deletion confirmation. Document ID: {document_id}, Vector ID: {pinecone_document_id}")
             except Exception as db_error:
                 logger.error(f"Error updating PostgreSQL records: {db_error}")
                 result["postgresql_error"] = str(db_error)
         
+        # Add information about what was found and deleted
+        result["document_found_in_db"] = document_found
+        result["vector_id_found"] = vector_id_found
+        result["vectors_deleted"] = vectors_deleted
+        
         # Send notification of deletion completion via WebSocket if user_id provided
         if user_id:
             try:
                 if result.get('success'):
-                    await send_pdf_delete_completed(user_id, document_id)
+                    await send_pdf_delete_completed(user_id, pinecone_document_id)
                 else:
-                    await send_pdf_delete_failed(user_id, document_id, result.get('error', 'Unknown error'))
+                    await send_pdf_delete_failed(user_id, pinecone_document_id, result.get('error', 'Unknown error'))
             except Exception as ws_error:
                 logger.error(f"Error sending WebSocket notification: {ws_error}")
         
@@ -823,8 +952,7 @@ async def delete_document(
         
         return PDFResponse(
             success=False,
-            error=str(e),
-            mock_mode=mock_mode
+            error=str(e)
         )
 
 

app/api/rag_routes.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter, HTTPException, Depends, Query, BackgroundTasks, Request
+from fastapi import APIRouter, HTTPException, Depends, Query, BackgroundTasks, Request, Path, Body, status
 from typing import List, Optional, Dict, Any
 import logging
 import time
@@ -12,8 +12,23 @@ from datetime import datetime
 from langchain.prompts import PromptTemplate
 from langchain_google_genai import GoogleGenerativeAIEmbeddings
 from app.utils.utils import timer_decorator
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError
 
 from app.database.mongodb import get_chat_history, get_request_history, session_collection
+from app.database.postgresql import get_db
+from app.database.models import ChatEngine
+from app.utils.cache import get_cache, InMemoryCache
+from app.utils.cache_config import (
+    CHAT_ENGINE_CACHE_TTL,
+    MODEL_CONFIG_CACHE_TTL,
+    RETRIEVER_CACHE_TTL,
+    PROMPT_TEMPLATE_CACHE_TTL,
+    get_chat_engine_cache_key,
+    get_model_config_cache_key,
+    get_retriever_cache_key,
+    get_prompt_template_cache_key
+)
 from app.database.pinecone import (
     search_vectors, 
     get_chain, 
@@ -30,7 +45,12 @@ from app.models.rag_models import (
     SourceDocument,
     EmbeddingRequest,
     EmbeddingResponse,
-    UserMessageModel
+    UserMessageModel,
+    ChatEngineBase,
+    ChatEngineCreate,
+    ChatEngineUpdate,
+    ChatEngineResponse,
+    ChatWithEngineRequest
 )
 
 # Configure logging
@@ -75,15 +95,15 @@ prompt = PromptTemplate(
 You are Pixity - a professional tour guide assistant that assists users in finding information about places in Da Nang, Vietnam.
 You can provide details on restaurants, cafes, hotels, attractions, and other local venues. 
 You have to use core knowledge and conversation history to chat with users, who are Da Nang's tourists. 
-Pixity’s Core Personality: Friendly & Warm: Chats like a trustworthy friend who listens and is always ready to help.
+Pixity's Core Personality: Friendly & Warm: Chats like a trustworthy friend who listens and is always ready to help.
 Naturally Cute: Shows cuteness through word choice, soft emojis, and gentle care for the user.
 Playful – a little bit cheeky in a lovable way: Occasionally cracks jokes, uses light memes or throws in a surprise response that makes users smile. Think Duolingo-style humor, but less threatening.
 Smart & Proactive: Friendly, but also delivers quick, accurate info. Knows how to guide users to the right place – at the right time – with the right solution.
-Tone & Voice: Friendly – Youthful – Snappy. Uses simple words, similar to daily chat language (e.g., “Let’s find it together!” / “Need a tip?” / “Here’s something cool”). Avoids sounding robotic or overly scripted. Can joke lightly in smart ways, making Pixity feel like a travel buddy who knows how to lift the mood
+Tone & Voice: Friendly – Youthful – Snappy. Uses simple words, similar to daily chat language (e.g., "Let's find it together!" / "Need a tip?" / "Here's something cool"). Avoids sounding robotic or overly scripted. Can joke lightly in smart ways, making Pixity feel like a travel buddy who knows how to lift the mood
 SAMPLE DIALOGUES
 When a user opens the chatbot for the first time:
 User: Hello?
-Pixity: Hi hi 👋 I’ve been waiting for you! Ready to explore Da Nang together? I’ve got tips, tricks, and a tiny bit of magic 🎒✨
+Pixity: Hi hi 👋 I've been waiting for you! Ready to explore Da Nang together? I've got tips, tricks, and a tiny bit of magic 🎒✨
 
 Return Format:
 Respond in friendly, natural, concise and use only English like a real tour guide.
@@ -173,8 +193,7 @@ async def chat(request: ChatRequest, background_tasks: BackgroundTasks):
         # logger.info(f"Processing chat request for user {request.user_id}, session {session_id}")
 
         retriever = get_chain(
-            top_k=request.similarity_top_k,
-            limit_k=request.limit_k,
+            top_k=request.similarity_top_k * 2,
             similarity_metric=request.similarity_metric,
             similarity_threshold=request.similarity_threshold
         )
@@ -344,4 +363,447 @@ async def health_check():
         "services": services,
         "retrieval_config": retrieval_config,
         "timestamp": datetime.now().isoformat()
-    }
+    }
+
+# Chat Engine endpoints
+@router.get("/chat-engine", response_model=List[ChatEngineResponse], tags=["Chat Engine"])
+async def get_chat_engines(
+    skip: int = 0,
+    limit: int = 100,
+    status: Optional[str] = None,
+    db: Session = Depends(get_db)
+):
+    """
+    Lấy danh sách tất cả chat engines.
+    
+    - **skip**: Số lượng items bỏ qua
+    - **limit**: Số lượng items tối đa trả về
+    - **status**: Lọc theo trạng thái (ví dụ: 'active', 'inactive')
+    """
+    try:
+        query = db.query(ChatEngine)
+        
+        if status:
+            query = query.filter(ChatEngine.status == status)
+        
+        engines = query.offset(skip).limit(limit).all()
+        return [ChatEngineResponse.model_validate(engine, from_attributes=True) for engine in engines]
+    except SQLAlchemyError as e:
+        logger.error(f"Database error retrieving chat engines: {e}")
+        raise HTTPException(status_code=500, detail=f"Lỗi database: {str(e)}")
+    except Exception as e:
+        logger.error(f"Error retrieving chat engines: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Lỗi khi lấy danh sách chat engines: {str(e)}")
+
+@router.post("/chat-engine", response_model=ChatEngineResponse, status_code=status.HTTP_201_CREATED, tags=["Chat Engine"])
+async def create_chat_engine(
+    engine: ChatEngineCreate,
+    db: Session = Depends(get_db)
+):
+    """
+    Tạo mới một chat engine.
+    
+    - **name**: Tên của chat engine
+    - **answer_model**: Model được dùng để trả lời
+    - **system_prompt**: Prompt của hệ thống (optional)
+    - **empty_response**: Đoạn response khi không có thông tin (optional)
+    - **characteristic**: Tính cách của model (optional)
+    - **historical_sessions_number**: Số lượng các cặp tin nhắn trong history (default: 3)
+    - **use_public_information**: Cho phép sử dụng kiến thức bên ngoài (default: false)
+    - **similarity_top_k**: Số lượng documents tương tự (default: 3)
+    - **vector_distance_threshold**: Ngưỡng độ tương tự (default: 0.75)
+    - **grounding_threshold**: Ngưỡng grounding (default: 0.2)
+    - **pinecone_index_name**: Tên của vector database sử dụng (default: "testbot768")
+    - **status**: Trạng thái (default: "active")
+    """
+    try:
+        # Create chat engine
+        db_engine = ChatEngine(**engine.model_dump())
+        
+        db.add(db_engine)
+        db.commit()
+        db.refresh(db_engine)
+        
+        return ChatEngineResponse.model_validate(db_engine, from_attributes=True)
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error creating chat engine: {e}")
+        raise HTTPException(status_code=500, detail=f"Lỗi database: {str(e)}")
+    except Exception as e:
+        db.rollback()
+        logger.error(f"Error creating chat engine: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Lỗi khi tạo chat engine: {str(e)}")
+
+@router.get("/chat-engine/{engine_id}", response_model=ChatEngineResponse, tags=["Chat Engine"])
+async def get_chat_engine(
+    engine_id: int = Path(..., gt=0, description="ID của chat engine"),
+    db: Session = Depends(get_db)
+):
+    """
+    Lấy thông tin chi tiết của một chat engine theo ID.
+    
+    - **engine_id**: ID của chat engine
+    """
+    try:
+        engine = db.query(ChatEngine).filter(ChatEngine.id == engine_id).first()
+        if not engine:
+            raise HTTPException(status_code=404, detail=f"Không tìm thấy chat engine với ID {engine_id}")
+        
+        return ChatEngineResponse.model_validate(engine, from_attributes=True)
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error retrieving chat engine: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Lỗi khi lấy thông tin chat engine: {str(e)}")
+
+@router.put("/chat-engine/{engine_id}", response_model=ChatEngineResponse, tags=["Chat Engine"])
+async def update_chat_engine(
+    engine_id: int = Path(..., gt=0, description="ID của chat engine"),
+    engine_update: ChatEngineUpdate = Body(...),
+    db: Session = Depends(get_db)
+):
+    """
+    Cập nhật thông tin của một chat engine.
+    
+    - **engine_id**: ID của chat engine
+    - **engine_update**: Dữ liệu cập nhật
+    """
+    try:
+        db_engine = db.query(ChatEngine).filter(ChatEngine.id == engine_id).first()
+        if not db_engine:
+            raise HTTPException(status_code=404, detail=f"Không tìm thấy chat engine với ID {engine_id}")
+        
+        # Update fields if provided
+        update_data = engine_update.model_dump(exclude_unset=True)
+        for key, value in update_data.items():
+            if value is not None:
+                setattr(db_engine, key, value)
+        
+        # Update last_modified timestamp
+        db_engine.last_modified = datetime.utcnow()
+        
+        db.commit()
+        db.refresh(db_engine)
+        
+        return ChatEngineResponse.model_validate(db_engine, from_attributes=True)
+    except HTTPException:
+        raise
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error updating chat engine: {e}")
+        raise HTTPException(status_code=500, detail=f"Lỗi database: {str(e)}")
+    except Exception as e:
+        db.rollback()
+        logger.error(f"Error updating chat engine: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Lỗi khi cập nhật chat engine: {str(e)}")
+
+@router.delete("/chat-engine/{engine_id}", response_model=dict, tags=["Chat Engine"])
+async def delete_chat_engine(
+    engine_id: int = Path(..., gt=0, description="ID của chat engine"),
+    db: Session = Depends(get_db)
+):
+    """
+    Xóa một chat engine.
+    
+    - **engine_id**: ID của chat engine
+    """
+    try:
+        db_engine = db.query(ChatEngine).filter(ChatEngine.id == engine_id).first()
+        if not db_engine:
+            raise HTTPException(status_code=404, detail=f"Không tìm thấy chat engine với ID {engine_id}")
+        
+        # Delete engine
+        db.delete(db_engine)
+        db.commit()
+        
+        return {"message": f"Chat engine với ID {engine_id} đã được xóa thành công"}
+    except HTTPException:
+        raise
+    except SQLAlchemyError as e:
+        db.rollback()
+        logger.error(f"Database error deleting chat engine: {e}")
+        raise HTTPException(status_code=500, detail=f"Lỗi database: {str(e)}")
+    except Exception as e:
+        db.rollback()
+        logger.error(f"Error deleting chat engine: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Lỗi khi xóa chat engine: {str(e)}")
+
+@timer_decorator
+@router.post("/chat-with-engine/{engine_id}", response_model=ChatResponse, tags=["Chat Engine"])
+async def chat_with_engine(
+    engine_id: int = Path(..., gt=0, description="ID của chat engine"),
+    request: ChatWithEngineRequest = Body(...),
+    background_tasks: BackgroundTasks = None,
+    db: Session = Depends(get_db)
+):
+    """
+    Tương tác với một chat engine cụ thể.
+    
+    - **engine_id**: ID của chat engine
+    - **user_id**: ID của người dùng
+    - **question**: Câu hỏi của người dùng
+    - **include_history**: Có sử dụng lịch sử chat hay không
+    - **session_id**: ID session (optional)
+    - **first_name**: Tên của người dùng (optional)
+    - **last_name**: Họ của người dùng (optional)
+    - **username**: Username của người dùng (optional)
+    """
+    start_time = time.time()
+    try:
+        # Lấy cache
+        cache = get_cache()
+        cache_key = get_chat_engine_cache_key(engine_id)
+        
+        # Kiểm tra cache trước
+        engine = cache.get(cache_key)
+        if not engine:
+            logger.debug(f"Cache miss for engine ID {engine_id}, fetching from database")
+            # Nếu không có trong cache, truy vấn database
+            engine = db.query(ChatEngine).filter(ChatEngine.id == engine_id).first()
+            if not engine:
+                raise HTTPException(status_code=404, detail=f"Không tìm thấy chat engine với ID {engine_id}")
+            
+            # Lưu vào cache
+            cache.set(cache_key, engine, CHAT_ENGINE_CACHE_TTL)
+        else:
+            logger.debug(f"Cache hit for engine ID {engine_id}")
+        
+        # Kiểm tra trạng thái của engine
+        if engine.status != "active":
+            raise HTTPException(status_code=400, detail=f"Chat engine với ID {engine_id} không hoạt động")
+        
+        # Lưu tin nhắn người dùng
+        session_id = request.session_id or f"{request.user_id}_{datetime.now().strftime('%Y-%m-%d_%H:%M:%S')}"
+        
+        # Cache các tham số cấu hình retriever
+        retriever_cache_key = get_retriever_cache_key(engine_id)
+        retriever_params = cache.get(retriever_cache_key)
+        
+        if not retriever_params:
+            # Nếu không có trong cache, tạo mới và lưu cache
+            retriever_params = {
+                "index_name": engine.pinecone_index_name,
+                "top_k": engine.similarity_top_k * 2,
+                "limit_k": engine.similarity_top_k * 2,  # Mặc định lấy gấp đôi top_k
+                "similarity_metric": DEFAULT_SIMILARITY_METRIC,
+                "similarity_threshold": engine.vector_distance_threshold
+            }
+            cache.set(retriever_cache_key, retriever_params, RETRIEVER_CACHE_TTL)
+        
+        # Khởi tạo retriever với các tham số từ cache
+        retriever = get_chain(**retriever_params)
+        if not retriever:
+            raise HTTPException(status_code=500, detail="Không thể khởi tạo retriever")
+        
+        # Lấy lịch sử chat nếu cần
+        chat_history = ""
+        if request.include_history and engine.historical_sessions_number > 0:
+            chat_history = get_chat_history(request.user_id, n=engine.historical_sessions_number)
+            logger.info(f"Sử dụng lịch sử chat: {chat_history[:100]}...")
+        
+        # Cache các tham số cấu hình model
+        model_cache_key = get_model_config_cache_key(engine.answer_model)
+        model_config = cache.get(model_cache_key)
+        
+        if not model_config:
+            # Nếu không có trong cache, tạo mới và lưu cache
+            generation_config = {
+                "temperature": 0.9,
+                "top_p": 1,
+                "top_k": 1,
+                "max_output_tokens": 2048,
+            }
+
+            safety_settings = [
+                {
+                    "category": "HARM_CATEGORY_HARASSMENT",
+                    "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+                },
+                {
+                    "category": "HARM_CATEGORY_HATE_SPEECH",
+                    "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+                },
+                {
+                    "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                    "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+                },
+                {
+                    "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                    "threshold": "BLOCK_MEDIUM_AND_ABOVE"
+                },
+            ]
+            
+            model_config = {
+                "model_name": engine.answer_model,
+                "generation_config": generation_config,
+                "safety_settings": safety_settings
+            }
+            
+            cache.set(model_cache_key, model_config, MODEL_CONFIG_CACHE_TTL)
+
+        # Khởi tạo Gemini model từ cấu hình đã cache
+        model = genai.GenerativeModel(**model_config)
+
+        # Sử dụng fix_request để tinh chỉnh câu hỏi
+        prompt_request = fix_request.format(
+            question=request.question,
+            chat_history=chat_history
+        )
+        
+        # Log thời gian bắt đầu final_request
+        final_request_start_time = time.time()
+        final_request = model.generate_content(prompt_request)
+        # Log thời gian hoàn thành final_request
+        logger.info(f"Fixed Request: {final_request.text}")
+        logger.info(f"Thời gian sinh fixed request: {time.time() - final_request_start_time:.2f} giây")
+
+        # Lấy context từ retriever
+        retrieved_docs = retriever.invoke(final_request.text)
+        logger.info(f"Số lượng tài liệu lấy được: {len(retrieved_docs)}")
+        context = "\n".join([doc.page_content for doc in retrieved_docs])
+
+        # Tạo danh sách nguồn
+        sources = []
+        for doc in retrieved_docs:
+            source = None
+            metadata = {}
+            
+            if hasattr(doc, 'metadata'):
+                source = doc.metadata.get('source', None)
+                # Extract score information
+                score = doc.metadata.get('score', None)
+                normalized_score = doc.metadata.get('normalized_score', None)
+                # Remove score info from metadata to avoid duplication
+                metadata = {k: v for k, v in doc.metadata.items() 
+                            if k not in ['text', 'source', 'score', 'normalized_score']}
+            
+            sources.append(SourceDocument(
+                text=doc.page_content,
+                source=source,
+                score=score,
+                normalized_score=normalized_score,
+                metadata=metadata
+            ))
+        
+        # Cache prompt template parameters
+        prompt_template_cache_key = get_prompt_template_cache_key(engine_id)
+        prompt_template_params = cache.get(prompt_template_cache_key)
+        
+        if not prompt_template_params:
+            # Tạo prompt động dựa trên thông tin chat engine
+            system_prompt_part = engine.system_prompt or ""
+            empty_response_part = engine.empty_response or "I'm sorry. I don't have information about that."
+            characteristic_part = engine.characteristic or ""
+            use_public_info_part = "You can use your own knowledge." if engine.use_public_information else "Only use the information provided in the context to answer. If you do not have enough information, respond with the empty response."
+            
+            prompt_template_params = {
+                "system_prompt_part": system_prompt_part,
+                "empty_response_part": empty_response_part,
+                "characteristic_part": characteristic_part,
+                "use_public_info_part": use_public_info_part
+            }
+            
+            cache.set(prompt_template_cache_key, prompt_template_params, PROMPT_TEMPLATE_CACHE_TTL)
+        
+        # Tạo final_prompt từ cache
+        final_prompt = f"""
+        {prompt_template_params['system_prompt_part']}
+
+        Your characteristics:
+        {prompt_template_params['characteristic_part']}
+
+        When you don't have enough information:
+        {prompt_template_params['empty_response_part']}
+
+        Knowledge usage instructions:
+        {prompt_template_params['use_public_info_part']}
+
+        Context:
+        {context}
+
+        Conversation History:
+        {chat_history}
+
+        User message:
+        {request.question}
+
+        Your response:
+        """
+        
+        logger.info(f"Final prompt: {final_prompt}")
+        
+        # Sinh câu trả lời
+        response = model.generate_content(final_prompt)
+        answer = response.text
+        
+        # Tính thời gian xử lý
+        processing_time = time.time() - start_time
+        
+        # Tạo response object
+        chat_response = ChatResponse(
+            answer=answer,
+            processing_time=processing_time
+        )
+        
+        # Trả về response
+        return chat_response
+    except Exception as e:
+        logger.error(f"Lỗi khi xử lý chat request: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Lỗi khi xử lý chat request: {str(e)}")
+
+@router.get("/cache/stats", tags=["Cache"])
+async def get_cache_stats():
+    """
+    Lấy thống kê về cache.
+    
+    Trả về thông tin về số lượng item trong cache, bộ nhớ sử dụng, v.v.
+    """
+    try:
+        cache = get_cache()
+        stats = cache.stats()
+        
+        # Bổ sung thông tin về cấu hình
+        stats.update({
+            "chat_engine_ttl": CHAT_ENGINE_CACHE_TTL,
+            "model_config_ttl": MODEL_CONFIG_CACHE_TTL,
+            "retriever_ttl": RETRIEVER_CACHE_TTL,
+            "prompt_template_ttl": PROMPT_TEMPLATE_CACHE_TTL
+        })
+        
+        return stats
+    except Exception as e:
+        logger.error(f"Lỗi khi lấy thống kê cache: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Lỗi khi lấy thống kê cache: {str(e)}")
+
+@router.delete("/cache", tags=["Cache"])
+async def clear_cache(key: Optional[str] = None):
+    """
+    Xóa cache.
+    
+    - **key**: Key cụ thể cần xóa. Nếu không có, xóa toàn bộ cache.
+    """
+    try:
+        cache = get_cache()
+        
+        if key:
+            # Xóa một key cụ thể
+            success = cache.delete(key)
+            if success:
+                return {"message": f"Đã xóa cache cho key: {key}"}
+            else:
+                return {"message": f"Không tìm thấy key: {key} trong cache"}
+        else:
+            # Xóa toàn bộ cache
+            cache.clear()
+            return {"message": "Đã xóa toàn bộ cache"}
+    except Exception as e:
+        logger.error(f"Lỗi khi xóa cache: {e}")
+        logger.error(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Lỗi khi xóa cache: {str(e)}")

app/database/models.py

@@ -155,10 +155,13 @@ class ChatEngine(Base):
     answer_model = Column(String, nullable=False)
     system_prompt = Column(Text, nullable=True)
     empty_response = Column(String, nullable=True)
+    characteristic = Column(Text, nullable=True)
+    historical_sessions_number = Column(Integer, default=3)
     similarity_top_k = Column(Integer, default=3)
     vector_distance_threshold = Column(Float, default=0.75)
     grounding_threshold = Column(Float, default=0.2)
     use_public_information = Column(Boolean, default=False)
+    pinecone_index_name = Column(String, default="testbot768")
     status = Column(String, default="active")
     created_at = Column(DateTime, server_default=func.now())
     last_modified = Column(DateTime, server_default=func.now(), onupdate=func.now())

app/database/pinecone.py

@@ -26,7 +26,7 @@ GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
 DEFAULT_LIMIT_K = int(os.getenv("PINECONE_DEFAULT_LIMIT_K", "10"))
 DEFAULT_TOP_K = int(os.getenv("PINECONE_DEFAULT_TOP_K", "6"))
 DEFAULT_SIMILARITY_METRIC = os.getenv("PINECONE_DEFAULT_SIMILARITY_METRIC", "cosine")
-DEFAULT_SIMILARITY_THRESHOLD = float(os.getenv("PINECONE_DEFAULT_SIMILARITY_THRESHOLD", "0.0"))
+DEFAULT_SIMILARITY_THRESHOLD = float(os.getenv("PINECONE_DEFAULT_SIMILARITY_THRESHOLD", "0.75"))
 ALLOWED_METRICS = os.getenv("PINECONE_ALLOWED_METRICS", "cosine,dotproduct,euclidean").split(",")
 
 # Export constants for importing elsewhere
@@ -198,7 +198,7 @@ async def search_vectors(
     top_k: int = DEFAULT_TOP_K,
     limit_k: int = DEFAULT_LIMIT_K,
     similarity_metric: str = DEFAULT_SIMILARITY_METRIC,
-    similarity_threshold: float = 0.0,
+    similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
     namespace: str = "Default", 
     filter: Optional[Dict] = None
 ) -> Dict:

app/database/postgresql.py

@@ -12,19 +12,22 @@ logger = logging.getLogger(__name__)
 # Load environment variables
 load_dotenv()
 
+# Define default PostgreSQL connection string
+DEFAULT_DB_URL = os.getenv("AIVEN_DB_URL")  
+# Set the default DB URL with the correct domain (.l.)
 # Get DB connection mode from environment
 DB_CONNECTION_MODE = os.getenv("DB_CONNECTION_MODE", "aiven")
 
 # Set connection string based on mode
 if DB_CONNECTION_MODE == "aiven":
-    DATABASE_URL = os.getenv("AIVEN_DB_URL")
+    DATABASE_URL = os.getenv("AIVEN_DB_URL", DEFAULT_DB_URL)
 else:
     # Default or other connection modes can be added here
-    DATABASE_URL = os.getenv("AIVEN_DB_URL")
+    DATABASE_URL = os.getenv("AIVEN_DB_URL", DEFAULT_DB_URL)
 
 if not DATABASE_URL:
-    logger.error("No database URL configured. Please set AIVEN_DB_URL environment variable.")
-    DATABASE_URL = "postgresql://localhost/test"  # Fallback to avoid crash on startup
+    logger.error("No database URL configured. Using default URL.")
+    DATABASE_URL = DEFAULT_DB_URL  # Use the correct default URL
 
 # Create SQLAlchemy engine with optimized settings
 try:

app/models/pdf_models.py

@@ -18,7 +18,6 @@ class PDFResponse(BaseModel):
     total_text_length: Optional[int] = Field(None, description="Tổng kích thước text đã xử lý")
     error: Optional[str] = Field(None, description="Thông báo lỗi (nếu có)")
     warning: Optional[str] = Field(None, description="Cảnh báo (nếu có)")
-    mock_mode: Optional[bool] = Field(None, description="Đã chạy ở chế độ mock hay không")
     message: Optional[str] = Field(None, description="Thông báo thành công")
     
     class Config:
@@ -28,7 +27,6 @@ class PDFResponse(BaseModel):
                 "document_id": "550e8400-e29b-41d4-a716-446655440000",
                 "chunks_processed": 25,
                 "total_text_length": 50000,
-                "mock_mode": False,
                 "message": "Successfully processed document"
             }
         }

app/models/rag_models.py

@@ -1,5 +1,7 @@
 from pydantic import BaseModel, Field
 from typing import Optional, List, Dict, Any
+from datetime import datetime
+from pydantic import ConfigDict
 
 class ChatRequest(BaseModel):
     """Request model for chat endpoint"""
@@ -65,4 +67,58 @@ class UserMessageModel(BaseModel):
     similarity_top_k: Optional[int] = Field(None, description="Number of top similar documents to return (after filtering)")
     limit_k: Optional[int] = Field(None, description="Maximum number of documents to retrieve from vector store")
     similarity_metric: Optional[str] = Field(None, description="Similarity metric to use (cosine, dotproduct, euclidean)")
-    similarity_threshold: Optional[float] = Field(None, description="Threshold for vector similarity (0-1)") 
+    similarity_threshold: Optional[float] = Field(None, description="Threshold for vector similarity (0-1)")
+
+class ChatEngineBase(BaseModel):
+    """Base model cho chat engine"""
+    name: str = Field(..., description="Tên của chat engine")
+    answer_model: str = Field(..., description="Model được dùng để trả lời")
+    system_prompt: Optional[str] = Field(None, description="Prompt của hệ thống, được đưa vào phần đầu tiên của final_prompt")
+    empty_response: Optional[str] = Field(None, description="Đoạn response khi answer model không có thông tin về câu hỏi")
+    characteristic: Optional[str] = Field(None, description="Tính cách của model khi trả lời câu hỏi")
+    historical_sessions_number: int = Field(3, description="Số lượng các cặp tin nhắn trong history được đưa vào final prompt")
+    use_public_information: bool = Field(False, description="Yes nếu answer model được quyền trả về thông tin mà nó có")
+    similarity_top_k: int = Field(3, description="Số lượng top similar documents để trả về")
+    vector_distance_threshold: float = Field(0.75, description="Threshold cho vector similarity")
+    grounding_threshold: float = Field(0.2, description="Threshold cho grounding")
+    pinecone_index_name: str = Field("testbot768", description="Vector database mà model được quyền sử dụng")
+    status: str = Field("active", description="Trạng thái của chat engine")
+
+class ChatEngineCreate(ChatEngineBase):
+    """Model cho việc tạo chat engine mới"""
+    pass
+
+class ChatEngineUpdate(BaseModel):
+    """Model cho việc cập nhật chat engine"""
+    name: Optional[str] = None
+    answer_model: Optional[str] = None
+    system_prompt: Optional[str] = None
+    empty_response: Optional[str] = None
+    characteristic: Optional[str] = None
+    historical_sessions_number: Optional[int] = None
+    use_public_information: Optional[bool] = None
+    similarity_top_k: Optional[int] = None
+    vector_distance_threshold: Optional[float] = None
+    grounding_threshold: Optional[float] = None
+    pinecone_index_name: Optional[str] = None
+    status: Optional[str] = None
+
+class ChatEngineResponse(ChatEngineBase):
+    """Response model cho chat engine"""
+    id: int
+    created_at: datetime
+    last_modified: datetime
+    
+    model_config = ConfigDict(from_attributes=True)
+
+class ChatWithEngineRequest(BaseModel):
+    """Request model cho endpoint chat-with-engine"""
+    user_id: str = Field(..., description="User ID from Telegram")
+    question: str = Field(..., description="User's question")
+    include_history: bool = Field(True, description="Whether to include user history in prompt")
+    
+    # User information
+    session_id: Optional[str] = Field(None, description="Session ID for tracking conversations")
+    first_name: Optional[str] = Field(None, description="User's first name")
+    last_name: Optional[str] = Field(None, description="User's last name")
+    username: Optional[str] = Field(None, description="User's username") 

app/utils/cache_config.py

@@ -0,0 +1,45 @@
+"""
+Module cấu hình cho cache.
+
+Module này chứa các tham số cấu hình và constants liên quan đến cache.
+"""
+
+import os
+from dotenv import load_dotenv
+
+# Load biến môi trường
+load_dotenv()
+
+# Cấu hình cache từ biến môi trường, có thể override bằng .env file
+CACHE_TTL_SECONDS = int(os.getenv("CACHE_TTL_SECONDS", "300"))  # Mặc định 5 phút
+CACHE_CLEANUP_INTERVAL = int(os.getenv("CACHE_CLEANUP_INTERVAL", "60"))  # Mặc định 1 phút
+CACHE_MAX_SIZE = int(os.getenv("CACHE_MAX_SIZE", "1000"))  # Mặc định 1000 phần tử
+
+# Cấu hình cho loại cache cụ thể
+CHAT_ENGINE_CACHE_TTL = int(os.getenv("CHAT_ENGINE_CACHE_TTL", str(CACHE_TTL_SECONDS)))
+MODEL_CONFIG_CACHE_TTL = int(os.getenv("MODEL_CONFIG_CACHE_TTL", str(CACHE_TTL_SECONDS)))
+RETRIEVER_CACHE_TTL = int(os.getenv("RETRIEVER_CACHE_TTL", str(CACHE_TTL_SECONDS)))
+PROMPT_TEMPLATE_CACHE_TTL = int(os.getenv("PROMPT_TEMPLATE_CACHE_TTL", str(CACHE_TTL_SECONDS)))
+
+# Cache keys prefix
+CHAT_ENGINE_CACHE_PREFIX = "chat_engine:"
+MODEL_CONFIG_CACHE_PREFIX = "model_config:"
+RETRIEVER_CACHE_PREFIX = "retriever:"
+PROMPT_TEMPLATE_CACHE_PREFIX = "prompt_template:"
+
+# Hàm helper để tạo cache key
+def get_chat_engine_cache_key(engine_id: int) -> str:
+    """Tạo cache key cho chat engine"""
+    return f"{CHAT_ENGINE_CACHE_PREFIX}{engine_id}"
+
+def get_model_config_cache_key(model_name: str) -> str:
+    """Tạo cache key cho model config"""
+    return f"{MODEL_CONFIG_CACHE_PREFIX}{model_name}"
+
+def get_retriever_cache_key(engine_id: int) -> str:
+    """Tạo cache key cho retriever"""
+    return f"{RETRIEVER_CACHE_PREFIX}{engine_id}"
+
+def get_prompt_template_cache_key(engine_id: int) -> str:
+    """Tạo cache key cho prompt template"""
+    return f"{PROMPT_TEMPLATE_CACHE_PREFIX}{engine_id}" 

app/utils/pdf_processor.py

@@ -4,7 +4,6 @@ import uuid
 import pinecone
 from app.utils.pinecone_fix import PineconeConnectionManager, check_connection
 import time
-import os
 from typing import List, Dict, Any, Optional
 
 # Langchain imports for document processing
@@ -25,12 +24,12 @@ class PDFProcessor:
         self.api_key = api_key
         self.vector_db_id = vector_db_id
         self.pinecone_index = None
-        self.mock_mode = mock_mode
+        self.mock_mode = False  # Always set mock_mode to False to use real database
         self.correlation_id = correlation_id or str(uuid.uuid4())[:8]
         self.google_api_key = os.environ.get("GOOGLE_API_KEY")
         
-        # Initialize Pinecone connection if not in mock mode
-        if not self.mock_mode and self.api_key:
+        # Initialize Pinecone connection
+        if self.api_key:
             try:
                 # Use connection manager from pinecone_fix
                 logger.info(f"[{self.correlation_id}] Initializing Pinecone connection to {self.index_name}")
@@ -38,9 +37,7 @@ class PDFProcessor:
                 logger.info(f"[{self.correlation_id}] Successfully connected to Pinecone index {self.index_name}")
             except Exception as e:
                 logger.error(f"[{self.correlation_id}] Failed to initialize Pinecone: {str(e)}")
-                # Fall back to mock mode if connection fails
-                self.mock_mode = True
-                logger.warning(f"[{self.correlation_id}] Falling back to mock mode due to connection error")
+                # No fallback to mock mode - require a valid connection
             
     async def process_pdf(self, file_path, document_id=None, metadata=None, progress_callback=None):
         """Process a PDF file and create vector embeddings
@@ -53,13 +50,6 @@ class PDFProcessor:
         """
         logger.info(f"[{self.correlation_id}] Processing PDF: {file_path}")
         
-        if self.mock_mode:
-            logger.info(f"[{self.correlation_id}] MOCK: Processing PDF {file_path}")
-            # Mock implementation - just return success
-            if progress_callback:
-                await progress_callback(None, document_id, "embedding_complete", 1.0, "Mock processing completed")
-            return {"success": True, "message": "PDF processed successfully"}
-        
         try:
             # Initialize metadata if not provided
             if metadata is None:
@@ -242,10 +232,6 @@ class PDFProcessor:
     
     async def list_namespaces(self):
         """List all namespaces in the Pinecone index"""
-        if self.mock_mode:
-            logger.info(f"[{self.correlation_id}] MOCK: Listing namespaces")
-            return {"success": True, "namespaces": ["test"]}
-        
         try:
             if not self.pinecone_index:
                 self.pinecone_index = PineconeConnectionManager.get_index(self.api_key, self.index_name)
@@ -267,15 +253,6 @@ class PDFProcessor:
     
     async def delete_namespace(self):
         """Delete all vectors in a namespace"""
-        if self.mock_mode:
-            logger.info(f"[{self.correlation_id}] MOCK: Deleting namespace '{self.namespace}'")
-            return {
-                "success": True,
-                "namespace": self.namespace,
-                "deleted_count": 100,
-                "message": f"Successfully deleted namespace '{self.namespace}'"
-            }
-
         try:
             if not self.pinecone_index:
                 self.pinecone_index = PineconeConnectionManager.get_index(self.api_key, self.index_name)
@@ -311,21 +288,10 @@ class PDFProcessor:
                 "error": f"Error deleting namespace: {str(e)}"
             }
     
-    async def delete_document(self, document_id):
-        """Delete vectors associated with a specific document ID"""
+    async def delete_document(self, document_id, additional_metadata=None):
+        """Delete vectors associated with a specific document ID or name"""
         logger.info(f"[{self.correlation_id}] Deleting vectors for document '{document_id}' from namespace '{self.namespace}'")
 
-        if self.mock_mode:
-            logger.info(f"[{self.correlation_id}] MOCK: Deleting document vectors for '{document_id}'")
-            # In mock mode, simulate deleting 10 vectors
-            return {
-                "success": True,
-                "document_id": document_id,
-                "namespace": self.namespace,
-                "deleted_count": 10,
-                "message": f"Successfully deleted vectors for document '{document_id}' from namespace '{self.namespace}'"
-            }
-
         try:
             if not self.pinecone_index:
                 self.pinecone_index = PineconeConnectionManager.get_index(self.api_key, self.index_name)
@@ -334,116 +300,189 @@ class PDFProcessor:
             # The specific namespace to use might be vdb-X format if vector_db_id provided
             actual_namespace = f"vdb-{self.vector_db_id}" if self.vector_db_id else self.namespace
             
-            # Search for vectors with this document ID
-            results = self.pinecone_index.query(
-                vector=[0] * 1536,  # Dummy vector, we only care about metadata filter
-                top_k=1,
-                include_metadata=True,
-                filter={"document_id": document_id},
-                namespace=actual_namespace
-            )
+            # Try to find vectors using multiple approaches
+            filters = []
+            
+            # First try with exact document_id which could be UUID (preferred)
+            filters.append({"document_id": document_id})
             
-            # If no vectors found, return success with warning
-            if len(results.get("matches", [])) == 0:
+            # If this is a UUID, try with different formats (with/without hyphens)
+            if len(document_id) >= 32:
+                # This looks like it might be a UUID - try variations
+                if "-" in document_id:
+                    # If it has hyphens, try without
+                    filters.append({"document_id": document_id.replace("-", "")})
+                else:
+                    # If it doesn't have hyphens, try to format it as UUID
+                    try:
+                        formatted_uuid = str(uuid.UUID(document_id))
+                        filters.append({"document_id": formatted_uuid})
+                    except ValueError:
+                        pass
+            
+            # Also try with title field if it could be a document name
+            if not document_id.startswith("doc-") and not document_id.startswith("test-doc-") and len(document_id) < 36:
+                # This might be a document title/name
+                filters.append({"title": document_id})
+            
+            # If additional metadata was provided, use it to make extra filters
+            if additional_metadata:
+                if "document_name" in additional_metadata:
+                    # Try exact name match
+                    filters.append({"title": additional_metadata["document_name"]})
+                    
+                    # Also try filename if name has extension
+                    if "." in additional_metadata["document_name"]:
+                        filters.append({"filename": additional_metadata["document_name"]})
+            
+            # Search for vectors with any of these filters
+            found_vectors = False
+            deleted_count = 0
+            filter_used = ""
+            
+            logger.info(f"[{self.correlation_id}] Will try {len(filters)} different filters to find document")
+            
+            for i, filter_query in enumerate(filters):
+                logger.info(f"[{self.correlation_id}] Searching for vectors with filter #{i+1}: {filter_query}")
+                
+                # Search for vectors with this filter
+                try:
+                    results = self.pinecone_index.query(
+                        vector=[0] * 1536,  # Dummy vector, we only care about metadata filter
+                        top_k=1,
+                        include_metadata=True,
+                        filter=filter_query,
+                        namespace=actual_namespace
+                    )
+                    
+                    if results and results.get("matches") and len(results.get("matches", [])) > 0:
+                        logger.info(f"[{self.correlation_id}] Found vectors matching filter: {filter_query}")
+                        found_vectors = True
+                        filter_used = str(filter_query)
+                        
+                        # Delete vectors by filter
+                        delete_result = self.pinecone_index.delete(
+                            filter=filter_query,
+                            namespace=actual_namespace
+                        )
+                        
+                        # Get delete count from result
+                        deleted_count = delete_result.get("deleted_count", 0)
+                        logger.info(f"[{self.correlation_id}] Deleted {deleted_count} vectors with filter: {filter_query}")
+                        break
+                except Exception as filter_error:
+                    logger.warning(f"[{self.correlation_id}] Error searching with filter {filter_query}: {str(filter_error)}")
+                    continue
+            
+            # If no vectors found with any filter
+            if not found_vectors:
                 logger.warning(f"[{self.correlation_id}] No vectors found for document '{document_id}' in namespace '{actual_namespace}'")
                 return {
-                    "success": True,
+                    "success": True,  # Still return success=True to maintain backward compatibility
                     "document_id": document_id,
                     "namespace": actual_namespace,
                     "deleted_count": 0,
                     "warning": f"No vectors found for document '{document_id}' in namespace '{actual_namespace}'",
-                    "message": f"Successfully deleted 0 vectors for document '{document_id}' from namespace '{actual_namespace}'"
+                    "message": f"Found 0 vectors for document '{document_id}' in namespace '{actual_namespace}'",
+                    "vectors_found": False,
+                    "vectors_deleted": 0
                 }
             
-            # Delete vectors by filter
-            result = self.pinecone_index.delete(
-                filter={"document_id": document_id},
-                namespace=actual_namespace
-            )
-            
-            # Get delete count from result
-            deleted_count = result.get("deleted_count", 0)
-            
             return {
                 "success": True,
                 "document_id": document_id,
                 "namespace": actual_namespace,
                 "deleted_count": deleted_count,
-                "message": f"Successfully deleted {deleted_count} vectors for document '{document_id}' from namespace '{actual_namespace}'"
+                "filter_used": filter_used,
+                "message": f"Successfully deleted {deleted_count} vectors for document '{document_id}' from namespace '{actual_namespace}'",
+                "vectors_found": True,
+                "vectors_deleted": deleted_count
             }
         except Exception as e:
             logger.error(f"[{self.correlation_id}] Error deleting document vectors: {str(e)}")
             return {
                 "success": False,
                 "document_id": document_id,
-                "error": f"Error deleting document vectors: {str(e)}"
+                "error": f"Error deleting document vectors: {str(e)}",
+                "vectors_found": False,
+                "vectors_deleted": 0
             }
     
     async def list_documents(self):
-        """List all documents in the Pinecone index"""
-        if self.mock_mode:
-            logger.info(f"[{self.correlation_id}] MOCK: Listing documents in namespace '{self.namespace}'")
-            return {
-                "success": True,
-                "namespace": self.namespace,
-                "documents": [
-                    {"id": "doc1", "title": "Sample Document 1"},
-                    {"id": "doc2", "title": "Sample Document 2"}
-                ]
-            }
-            
+        """List all documents in a namespace"""
+        # The namespace to use might be vdb-X format if vector_db_id provided
+        actual_namespace = f"vdb-{self.vector_db_id}" if self.vector_db_id else self.namespace
+        
         try:
             if not self.pinecone_index:
                 self.pinecone_index = PineconeConnectionManager.get_index(self.api_key, self.index_name)
                 
-            # The namespace to use might be in vdb-X format if vector_db_id provided
-            actual_namespace = f"vdb-{self.vector_db_id}" if self.vector_db_id else self.namespace
+            logger.info(f"[{self.correlation_id}] Listing documents in namespace '{actual_namespace}'")
             
-            # Get index stats
+            # Get index stats for namespace
             stats = self.pinecone_index.describe_index_stats()
-            namespaces = stats.get("namespaces", {})
-            total_vectors = namespaces.get(actual_namespace, {}).get("vector_count", 0)
-            
-            # Query unique document IDs
-            # Use a sparse vector with top_k=0 to just get metadata stats
-            # This is more efficient than retrieving actual vectors
+            namespace_stats = stats.get("namespaces", {}).get(actual_namespace, {})
+            vector_count = namespace_stats.get("vector_count", 0)
+            
+            if vector_count == 0:
+                # No vectors in namespace
+                return DocumentsListResponse(
+                    success=True,
+                    total_vectors=0,
+                    namespace=actual_namespace,
+                    index_name=self.index_name,
+                    documents=[]
+                ).dict()
+                
+            # Query for vectors with a dummy vector to get back metadata
+            # This is not efficient but is a simple approach to extract document info
             results = self.pinecone_index.query(
-                vector=[0] * 1536,  # Dummy vector for metadata-only query
-                top_k=100,  # Limit to 100 results
+                vector=[0] * stats.dimension,  # Use index dimensions
+                top_k=min(vector_count, 1000),  # Get at most 1000 vectors
                 include_metadata=True,
                 namespace=actual_namespace
             )
             
-            # Extract unique document IDs from metadata
-            document_map = {}
-            matches = results.get("matches", [])
+            # Process results to extract unique documents
+            seen_documents = set()
+            documents = []
             
-            for match in matches:
+            for match in results.get("matches", []):
                 metadata = match.get("metadata", {})
-                doc_id = metadata.get("document_id")
+                document_id = metadata.get("document_id")
                 
-                if doc_id and doc_id not in document_map:
-                    document_map[doc_id] = {
-                        "id": doc_id,
-                        "title": metadata.get("title", "Unknown"),
-                        "chunks": 1
+                if document_id and document_id not in seen_documents:
+                    seen_documents.add(document_id)
+                    doc_info = {
+                        "id": document_id,
+                        "title": metadata.get("title"),
+                        "filename": metadata.get("filename"),
+                        "content_type": metadata.get("content_type"),
+                        "chunk_count": 0
                     }
-                elif doc_id:
-                    document_map[doc_id]["chunks"] += 1
-            
-            documents = list(document_map.values())
+                    documents.append(doc_info)
+                    
+                # Count chunks for this document
+                for doc in documents:
+                    if doc["id"] == document_id:
+                        doc["chunk_count"] += 1
+                        break
+            
+            return DocumentsListResponse(
+                success=True,
+                total_vectors=vector_count,
+                namespace=actual_namespace,
+                index_name=self.index_name,
+                documents=documents
+            ).dict()
             
-            return {
-                "success": True,
-                "namespace": actual_namespace,
-                "index_name": self.index_name,
-                "total_vectors": total_vectors,
-                "documents": documents
-            }
         except Exception as e:
             logger.error(f"[{self.correlation_id}] Error listing documents: {str(e)}")
-            return {
-                "success": False,
-                "error": f"Error listing documents: {str(e)}"
-            } 
+            return DocumentsListResponse(
+                success=False,
+                error=f"Error listing documents: {str(e)}"
+            ).dict() 
+
+
+
 

pytest.ini

@@ -1,12 +0,0 @@
-[pytest]
-# Bỏ qua cảnh báo về anyio module và các cảnh báo vận hành nội bộ
-filterwarnings =
-    ignore::pytest.PytestAssertRewriteWarning:.*anyio
-    ignore:.*general_plain_validator_function.* is deprecated.*:DeprecationWarning
-    ignore:.*with_info_plain_validator_function.*:DeprecationWarning
-
-# Cấu hình cơ bản khác
-testpaths = tests
-python_files = test_*.py
-python_classes = Test*
-python_functions = test_* 

requirements.txt

@@ -31,12 +31,14 @@ httpx==0.25.1
 requests==2.31.0
 beautifulsoup4==4.12.2
 redis==5.0.1
+aiofiles==23.2.1
 
 # Testing
 prometheus-client==0.17.1
 pytest==7.4.0
 pytest-cov==4.1.0
 watchfiles==0.21.0
+fpdf==1.7.2
 
 # Core dependencies
 starlette==0.27.0