Update app.py

app.py

@@ -7,6 +7,7 @@ import urllib.parse
 from datetime import datetime, timedelta
 import re
 import os
+
 # List of user agents to rotate through
 _useragent_list = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
@@ -138,6 +139,44 @@ def filter_relevant_content(text):
     
     return filtered_text
 
+def chunk_text(text, max_chunk_size=1000, overlap=100):
+    # List of keywords that might indicate new sections
+    section_keywords = ["revenue", "income", "profit", "loss", "expenses", "outlook", "forecast", "quarter", "year"]
+    
+    # Split text into sentences
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+    
+    chunks = []
+    current_chunk = ""
+    
+    for sentence in sentences:
+        if len(current_chunk) + len(sentence) > max_chunk_size:
+            # If adding this sentence exceeds max_chunk_size, start a new chunk
+            chunks.append(current_chunk.strip())
+            current_chunk = sentence + " "
+        elif any(keyword in sentence.lower() for keyword in section_keywords):
+            # If sentence contains a section keyword, start a new chunk
+            if current_chunk:
+                chunks.append(current_chunk.strip())
+            current_chunk = sentence + " "
+        else:
+            current_chunk += sentence + " "
+    
+    # Add the last chunk if it's not empty
+    if current_chunk:
+        chunks.append(current_chunk.strip())
+    
+    # Add overlap
+    overlapped_chunks = []
+    for i, chunk in enumerate(chunks):
+        if i > 0:
+            chunk = chunks[i-1][-overlap:] + chunk
+        if i < len(chunks) - 1:
+            chunk = chunk + chunks[i+1][:overlap]
+        overlapped_chunks.append(chunk)
+    
+    return overlapped_chunks
+
 def summarize_financial_news(query):
     """Search for financial news, extract relevant content, and summarize"""
     search_results = google_search(query, num_results=3)
@@ -151,15 +190,29 @@ def summarize_financial_news(query):
     if not all_filtered_text:
         return "No relevant financial information found."
     
-    prompt = f"""You are a financial analyst. Summarize the following text from a financial perspective:
-
-{all_filtered_text}
-
+    # Chunk the filtered text
+    chunks = chunk_text(all_filtered_text, max_chunk_size=3000, overlap=200)
+    
+    summaries = []
+    for chunk in chunks:
+        prompt = f"""You are a financial analyst. Summarize the following text from a financial perspective:
+{chunk}
 Provide a detailed, coherent summary focusing on financial implications and analysis."""
 
-    summary = query_llama({"inputs": prompt, "parameters": {"max_length": 5000}})
+        summary = query_llama({"inputs": prompt, "parameters": {"max_length": 1000}})
+        summaries.append(summary[0]['generated_text'])
+    
+    # Combine summaries
+    combined_summary = "\n\n".join(summaries)
+    
+    # Final summarization of combined summaries
+    final_prompt = f"""As a financial analyst, provide a coherent and comprehensive summary of the following financial information:
+{combined_summary}
+Focus on the most important financial implications and analysis."""
+
+    final_summary = query_llama({"inputs": final_prompt, "parameters": {"max_length": 2000}})
     
-    return summary[0]['generated_text']
+    return final_summary[0]['generated_text']
 
 # Gradio Interface
 iface = gr.Interface(