Update app.py

app.py

@@ -20,10 +20,15 @@ from datetime import datetime
 import os
 from dotenv import load_dotenv
 import certifi
-from bs4 import BeautifulSoup
 import requests
-from trafilatura.settings import use_config
-from urllib.request import urlopen, Request
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy import signals
+from scrapy.signalmanager import dispatcher
+from scrapy.utils.log import configure_logging
+from newspaper import Article
+
+
 
 # Load environment variables from a .env file
 load_dotenv()
@@ -37,7 +42,7 @@ SEARXNG_URL = 'https://shreyas094-searxng-local.hf.space/search'
 SEARXNG_KEY = 'f9f07f93b37b8483aadb5ba717f556f3a4ac507b281b4ca01e6c6288aa3e3ae5'
 
 # Use the environment variable
-HF_TOKEN = os.getenv('HF_TOKEN')
+HF_TOKEN = os.getenv("HF_TOKEN")
 client = InferenceClient(
     "mistralai/Mistral-Nemo-Instruct-2407",
     token=HF_TOKEN,
@@ -74,6 +79,51 @@ def is_valid_url(url):
     except ValueError:
         return False
 
+class NewsSpider(scrapy.Spider):
+    name = 'news_spider'
+    
+    def __init__(self, url=None, *args, **kwargs):
+        super(NewsSpider, self).__init__(*args, **kwargs)
+        self.start_urls = [url] if url else []
+    
+    def parse(self, response):
+        content = ' '.join(response.css('p::text').getall())
+        self.logger.info(f"Scraped content length: {len(content)}")
+        return {'content': content}
+
+def scrape_with_scrapy(url, timeout=30):
+    logger.info(f"Starting to scrape with Scrapy: {url}")
+    configure_logging(install_root_handler=False)
+    logging.getLogger('scrapy').setLevel(logging.WARNING)
+    
+    results = []
+    
+    def spider_results(signal, sender, item, response, spider):
+        results.append(item)
+    
+    process = CrawlerProcess(settings={
+        'LOG_ENABLED': True,
+        'LOG_LEVEL': 'WARNING',
+        'DOWNLOAD_TIMEOUT': timeout
+    })
+    
+    dispatcher.connect(spider_results, signal=signals.item_scraped)
+    
+    process.crawl(NewsSpider, url=url)
+    process.start()
+    
+    # Get the content from results
+    if results:
+        return results[0]['content']
+    return ''
+
+def scrape_with_newspaper(url):
+    logger.info(f"Starting to scrape with Newspaper3k: {url}")
+    article = Article(url)
+    article.download()
+    article.parse()
+    return article.text
+
 def scrape_with_bs4(url, session, max_chars=None):
     try:
         response = session.get(url, timeout=5)
@@ -248,7 +298,8 @@ Remember to focus on financial aspects and implications in your assessment and s
         response = llm_client.chat_completion(
             messages=messages,
             max_tokens=150,
-            temperature=temperature
+            temperature=temperature,
+            top_p=0.9
         )
         return response.choices[0].message.content.strip()
     except Exception as e:
@@ -272,8 +323,15 @@ def scrape_full_content(url, scraper="bs4", max_chars=3000, timeout=5):
                 content = main_content.get_text(strip=True, separator='\n')
             else:
                 content = soup.get_text(strip=True, separator='\n')
-        else:  # trafilatura
+        elif scraper == "trafilatura":
             content = scrape_with_trafilatura(url, max_chars, timeout, use_beautifulsoup=True)
+        elif scraper == "scrapy":
+            content = scrape_with_scrapy(url, timeout)
+        elif scraper == "newspaper":
+            content = scrape_with_newspaper(url)
+        else:
+            logger.error(f"Unknown scraper: {scraper}")
+            return ""
         
         # Limit the content to max_chars
         return content[:max_chars] if content else ""
@@ -314,7 +372,10 @@ Your response should be detailed, informative, accurate, and directly relevant t
         response = llm_client.chat_completion(
             messages=messages,
             max_tokens=10000,
-            temperature=temperature
+            temperature=temperature,
+            frequency_penalty=1.1,
+            top_p=0.9,
+            stream=True
         )
         return response.choices[0].message.content.strip()
     except Exception as e:
@@ -408,51 +469,17 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="bs4", max_cha
                 try:
                     logger.info(f"Scraping content from: {url}")
                     
-                    # Implement a retry mechanism with different user agents
-                    user_agents = [
-                        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-                        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15',
-                        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-                    ]
-                    
-                    content = ""
-                    for ua in user_agents:
-                        try:
-                            if scraper == "bs4":
-                                session.headers.update({'User-Agent': ua})
-                                content = scrape_with_bs4(url, session, max_chars)
-                            else:  # trafilatura
-                                # Use urllib to handle custom headers for trafilatura
-                                req = Request(url, headers={'User-Agent': ua})
-                                with urlopen(req) as response:
-                                    downloaded = response.read()
-                                
-                                # Configure trafilatura to use a specific user agent
-                                config = use_config()
-                                config.set("DEFAULT", "USER_AGENT", ua)
-                                
-                                content = scrape_with_trafilatura(url, max_chars, timeout=timeout, use_beautifulsoup=True)
-                            
-                            if content:
-                                break
-                        except requests.exceptions.HTTPError as e:
-                            if e.response.status_code == 403:
-                                logger.warning(f"403 Forbidden error with User-Agent: {ua}. Trying next...")
-                                continue
-                            else:
-                                raise
-                        except Exception as e:
-                            logger.error(f"Error scraping {url} with User-Agent {ua}: {str(e)}")
-                            continue
+                    # MODIFY: Remove the user agent loop and use a single scraping method
+                    content = scrape_full_content(url, scraper, max_chars, timeout)
                     
                     if not content:
-                        logger.warning(f"Failed to scrape content from {url} after trying multiple User-Agents")
+                        logger.warning(f"Failed to scrape content from {url}")
                         continue
                     
                     scraped_content.append({
                         "title": title,
                         "url": url,
-                        "content": content,  # No need to slice here as it's already limited
+                        "content": content,
                         "scraper": scraper
                     })
                     logger.info(f"Successfully scraped content from {url}. Total scraped: {len(scraped_content)}")
@@ -558,7 +585,7 @@ iface = gr.ChatInterface(
     description="Enter your query, and I'll search the web for the most recent and relevant financial news, scrape content, and provide summarized results.",
     additional_inputs=[
         gr.Slider(5, 20, value=10, step=1, label="Number of initial results"),
-        gr.Dropdown(["bs4", "trafilatura"], value="bs4", label="Scraping Method"),
+        gr.Dropdown(["bs4", "trafilatura", "scrapy", "newspaper"], value="bs4", label="Scraping Method"),
         gr.Slider(500, 10000, value=1500, step=100, label="Max characters to retrieve"),
         gr.Dropdown(["", "day", "week", "month", "year"], value="year", label="Time Range"),
         gr.Dropdown(["all", "en", "fr", "de", "es", "it", "nl", "pt", "pl", "ru", "zh"], value="en", label="Language"),