Update app.py

app.py

@@ -460,14 +460,28 @@ def get_response_from_pdf(query, model, selected_docs, num_calls=3, temperature=
         yield "No documents available. Please upload PDF documents to answer questions."
         return
 
+    # Log the total number of documents in the database
+    doc_count = len(database.docstore._dict)
+    logging.info(f"Total documents in the database: {doc_count}")
+
+    # Print out some sample documents
+    sample_docs = list(database.docstore._dict.values())[:5]  # Get first 5 documents
+    for i, doc in enumerate(sample_docs):
+        logging.info(f"Sample document {i}:")
+        logging.info(f"  Type: {type(doc)}")
+        logging.info(f"  Attributes: {dir(doc)}")
+        if hasattr(doc, 'metadata'):
+            logging.info(f"  Metadata: {doc.metadata}")
+        if hasattr(doc, 'page_content'):
+            logging.info(f"  Content preview: {doc.page_content[:100]}...")
+
     all_filtered_docs = []
-    k_per_doc = max(5, 10 // len(selected_docs))  # Increased k_per_doc
+    k_per_doc = max(20, 50 // len(selected_docs))  # Increased k_per_doc
 
-    # First, try to retrieve documents with filtering
     for doc_name in selected_docs:
         logging.info(f"Retrieving documents for: {doc_name}")
         try:
-            doc_filter = lambda doc: isinstance(doc, dict) and doc.get('metadata', {}).get('source') == doc_name
+            doc_filter = lambda doc: doc.metadata['source'] == doc_name
             doc_retriever = database.as_retriever(search_kwargs={"k": k_per_doc, "filter": doc_filter})
             relevant_docs = doc_retriever.get_relevant_documents(query)
             all_filtered_docs.extend(relevant_docs)
@@ -475,16 +489,6 @@ def get_response_from_pdf(query, model, selected_docs, num_calls=3, temperature=
         except Exception as e:
             logging.error(f"Error retrieving documents for {doc_name}: {str(e)}")
 
-    # If no documents are found, try retrieving without filtering
-    if not all_filtered_docs:
-        logging.warning("No documents found with filtering. Attempting retrieval without filters.")
-        try:
-            retriever = database.as_retriever(search_kwargs={"k": k_per_doc * len(selected_docs)})
-            all_filtered_docs = retriever.get_relevant_documents(query)
-            logging.info(f"Retrieved {len(all_filtered_docs)} documents without filtering")
-        except Exception as e:
-            logging.error(f"Error retrieving documents without filtering: {str(e)}")
-
     logging.info(f"Total number of filtered documents: {len(all_filtered_docs)}")
     
     if not all_filtered_docs:
@@ -494,14 +498,14 @@ def get_response_from_pdf(query, model, selected_docs, num_calls=3, temperature=
 
     for doc in all_filtered_docs:
         try:
-            source = doc.metadata['source'] if hasattr(doc, 'metadata') else doc.get('metadata', {}).get('source', 'Unknown')
-            content = doc.page_content if hasattr(doc, 'page_content') else doc.get('page_content', '')
+            source = doc.metadata['source']
+            content = doc.page_content
             logging.info(f"Document source: {source}")
             logging.info(f"Document content preview: {content[:100]}...")  # Log first 100 characters of each document
         except Exception as e:
             logging.error(f"Error processing document: {str(e)}")
 
-    context_str = "\n".join([doc.page_content if hasattr(doc, 'page_content') else doc.get('page_content', '') for doc in all_filtered_docs])
+    context_str = "\n".join([doc.page_content for doc in all_filtered_docs])
     logging.info(f"Total context length: {len(context_str)}")
 
     if model == "@cf/meta/llama-3.1-8b-instruct":