Update app.py

app.py

@@ -20,7 +20,20 @@ class AdvancedPDFKnowledgeBase:
         self.bm25 = None
         
     def _split_into_chunks(self, text: str, chunk_size: int = 500) -> List[str]:
-        words = text.split()
+        # Remove linhas duplicadas ou muito semelhantes
+        lines = text.split("\n")
+        unique_lines = []
+        seen = set()
+        for line in lines:
+            if line.strip() and line not in seen:
+                unique_lines.append(line)
+                seen.add(line)
+        
+        # Junta as linhas únicas em um único texto
+        cleaned_text = "\n".join(unique_lines)
+        
+        # Divide o texto limpo em chunks
+        words = cleaned_text.split()
         return [' '.join(words[i:i + chunk_size]) 
                 for i in range(0, len(words), chunk_size)]
     
@@ -76,6 +89,7 @@ class AdvancedPDFKnowledgeBase:
 # Inicializa a base de conhecimento
 knowledge_base = AdvancedPDFKnowledgeBase()
 
+# Função principal de resposta
 def respond(
     message: str,
     history: List[Tuple[str, str]],
@@ -126,7 +140,7 @@ Responda com base no contexto quando relevante."""
                 yield response, context, ""
     except Exception as e:
         yield f"Erro ao gerar resposta: {str(e)}", context, ""
-        
+
 # Função para carregar PDFs
 def load_pdfs(pdf_files: List[gr.File]):
     status = knowledge_base.load_pdfs(pdf_files)
@@ -154,7 +168,7 @@ with gr.Blocks(title="RAG Avançado com PDFs", theme=gr.themes.Soft()) as demo:
     with gr.Accordion("Configurações", open=False):
         with gr.Row():
             with gr.Column():
-                pdf_upload = gr.File(label="Carregar PDFs", file_types=[".pdf"], file_count="multiple", interactive=True)
+                pdf_upload = gr.File(label="Carregar PDFs", file_types=[".pdf", ".txt"], file_count="multiple", interactive=True)
                 load_btn = gr.Button("Carregar PDFs")
             
             with gr.Column():