Update app.py

app.py

@@ -5,28 +5,45 @@ import streamlit as st
 from twilio.rest import Client
 from sentence_transformers import SentenceTransformer
 from transformers import AutoTokenizer
-
 import faiss
 import numpy as np
 import docx
-
 from groq import Groq
 import requests
 from io import StringIO
 from pdfminer.high_level import extract_text_to_fp
 from pdfminer.layout import LAParams
 from twilio.base.exceptions import TwilioRestException  # Add this at the top
+import pdfplumber
 import datetime
 
 APP_START_TIME = datetime.datetime.now(datetime.timezone.utc)
 
 os.environ["PYTORCH_JIT"] = "0"
 # --- PDF Extraction ---
+# --- PDF Extraction ---
 def extract_text_from_pdf(pdf_path):
-    output_string = StringIO()
-    with open(pdf_path, 'rb') as file:
-        extract_text_to_fp(file, output_string, laparams=LAParams(), output_type='text', codec=None)
-    return output_string.getvalue()
+    text_output = StringIO()
+    tables = []
+    try:
+        with pdfplumber.open(pdf_path) as pdf:
+            for page in pdf.pages:
+                # Extract tables
+                page_tables = page.extract_tables()
+                if page_tables:
+                    tables.extend(page_tables)
+                # Extract text
+                text = page.extract_text()
+                if text:
+                    text_output.write(text + "\n\n")
+    except Exception as e:
+        print(f"Error extracting with pdfplumber: {e}")
+        # Fallback to pdfminer if pdfplumber fails
+        with open(pdf_path, 'rb') as file:
+            extract_text_to_fp(file, text_output, laparams=LAParams(), output_type='text', codec=None)
+    extracted_text = text_output.getvalue()
+    formatted_tables = _format_tables_internal(tables)
+    return f"{extracted_text}\n\n{formatted_tables}"
 
 def clean_extracted_text(text):
     lines = text.splitlines()
@@ -38,12 +55,20 @@ def clean_extracted_text(text):
             cleaned.append(line)
     return '\n'.join(cleaned)
 
+def _format_tables_internal(tables):
+    formatted_tables = []
+    for table in tables:
+        # Basic formatting: joining rows with '|' and cells with ','
+        formatted_table = "\n".join(["|".join(row) for row in table])
+        formatted_tables.append(f"<table data>\n{formatted_table}\n</table>")
+    return "\n\n".join(formatted_tables)
+
 # --- DOCX Extraction ---
 def extract_text_from_docx(docx_path):
     try:
         doc = docx.Document(docx_path)
         return '\n'.join(para.text for para in doc.paragraphs)
-    except:
+    except Exception:
         return ""
 
 # --- Chunking ---
@@ -117,7 +142,7 @@ def fetch_latest_incoming_message(client, conversation_sid):
         else:
             print(f"Twilio error fetching messages for {conversation_sid}:", e)
     except Exception as e:
-        print(f"Unexpected error in fetch_latest_incoming_message for {conversation_sid}:", e)
+        #print(f"Unexpected error in fetch_latest_incoming_message for {conversation_sid}:", e)
 
     return None