Update app.py

app.py

@@ -299,6 +299,11 @@ Please provide a comprehensive summary based on the above instructions:
         logger.error(f"Error in LLM summarization: {e}")
         return "Error: Unable to generate a summary. Please try again."
 
+import requests
+from trafilatura import extract
+from trafilatura.settings import use_config
+from urllib.request import urlopen, Request
+
 def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura", max_chars=3000, time_range="", language="all", category="", 
                       engines=[], safesearch=2, method="GET", llm_temperature=0.2):
     try:
@@ -314,7 +319,6 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura",
         params = {
             'q': rephrased_query,
             'format': 'json',
-            'num_results': num_results,
             'time_range': time_range,
             'language': language,
             'category': category,
@@ -344,101 +348,111 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura",
             'Sec-Fetch-Site': 'same-origin',
         }
 
-        # Send request to SearXNG
-        logger.info(f"Sending request to SearXNG for query: {rephrased_query}")
-        session = requests_retry_session()
+        scraped_content = []
+        page = 1
+        while len(scraped_content) < num_results:
+            # Update params with current page
+            params['pageno'] = page
 
-        try:
-            if method.upper() == "GET":
-                response = session.get(SEARXNG_URL, params=params, headers=headers, timeout=10, verify=certifi.where())
-            else:  # POST
-                response = session.post(SEARXNG_URL, data=params, headers=headers, timeout=10, verify=certifi.where())
-            
-            response.raise_for_status()
-        except requests.exceptions.RequestException as e:
-            logger.error(f"Error during SearXNG request: {e}")
-            return f"An error occurred during the search request: {e}"
+            # Send request to SearXNG
+            logger.info(f"Sending request to SearXNG for query: {rephrased_query} (Page {page})")
+            session = requests_retry_session()
 
-        search_results = response.json()
-        logger.debug(f"SearXNG Response: {search_results}")
+            try:
+                if method.upper() == "GET":
+                    response = session.get(SEARXNG_URL, params=params, headers=headers, timeout=10, verify=certifi.where())
+                else:  # POST
+                    response = session.post(SEARXNG_URL, data=params, headers=headers, timeout=10, verify=certifi.where())
+                
+                response.raise_for_status()
+            except requests.exceptions.RequestException as e:
+                logger.error(f"Error during SearXNG request: {e}")
+                return f"An error occurred during the search request: {e}"
 
-        num_received = len(search_results.get('results', []))
-        logger.info(f"Received {num_received} results from SearXNG")
+            search_results = response.json()
+            logger.debug(f"SearXNG Response: {search_results}")
 
-        if num_received == 0:
-            logger.warning("No results returned from SearXNG.")
-            return "No results found for the given query."
+            results = search_results.get('results', [])
+            if not results:
+                logger.warning(f"No more results returned from SearXNG on page {page}.")
+                break
 
-        scraped_content = []
+            for result in results:
+                if len(scraped_content) >= num_results:
+                    break
 
-        for result in search_results.get('results', [])[:num_results]:
-            url = result.get('url', '')
-            title = result.get('title', 'No title')
+                url = result.get('url', '')
+                title = result.get('title', 'No title')
 
-            if not is_valid_url(url):
-                logger.warning(f"Invalid URL: {url}")
-                continue
+                if not is_valid_url(url):
+                    logger.warning(f"Invalid URL: {url}")
+                    continue
 
-            try:
-                logger.info(f"Scraping content from: {url}")
-                
-                # Implement a retry mechanism with different user agents
-                user_agents = [
-                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-                    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15',
-                    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-                ]
-                
-                content = ""
-                for ua in user_agents:
-                    try:
-                        if scraper == "bs4":
-                            session.headers.update({'User-Agent': ua})
-                            content = scrape_with_bs4(url, session)
-                        else:  # trafilatura
-                            # Use urllib to handle custom headers for trafilatura
-                            req = Request(url, headers={'User-Agent': ua})
-                            with urlopen(req) as response:
-                                downloaded = response.read()
-                            
-                            # Configure trafilatura to use a specific user agent
-                            config = use_config()
-                            config.set("DEFAULT", "USER_AGENT", ua)
+                try:
+                    logger.info(f"Scraping content from: {url}")
+                    
+                    # Implement a retry mechanism with different user agents
+                    user_agents = [
+                        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+                        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15',
+                        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+                    ]
+                    
+                    content = ""
+                    for ua in user_agents:
+                        try:
+                            if scraper == "bs4":
+                                session.headers.update({'User-Agent': ua})
+                                content = scrape_with_bs4(url, session)
+                            else:  # trafilatura
+                                # Use urllib to handle custom headers for trafilatura
+                                req = Request(url, headers={'User-Agent': ua})
+                                with urlopen(req) as response:
+                                    downloaded = response.read()
+                                
+                                # Configure trafilatura to use a specific user agent
+                                config = use_config()
+                                config.set("DEFAULT", "USER_AGENT", ua)
+                                
+                                content = extract(downloaded, config=config)
                             
-                            content = extract(downloaded, config=config)
-                        
-                        if content:
-                            break
-                    except requests.exceptions.HTTPError as e:
-                        if e.response.status_code == 403:
-                            logger.warning(f"403 Forbidden error with User-Agent: {ua}. Trying next...")
+                            if content:
+                                break
+                        except requests.exceptions.HTTPError as e:
+                            if e.response.status_code == 403:
+                                logger.warning(f"403 Forbidden error with User-Agent: {ua}. Trying next...")
+                                continue
+                            else:
+                                raise
+                        except Exception as e:
+                            logger.error(f"Error scraping {url} with User-Agent {ua}: {str(e)}")
                             continue
-                        else:
-                            raise
-                    except Exception as e:
-                        logger.error(f"Error scraping {url} with User-Agent {ua}: {str(e)}")
+                    
+                    if not content:
+                        logger.warning(f"Failed to scrape content from {url} after trying multiple User-Agents")
                         continue
-                
-                if not content:
-                    logger.warning(f"Failed to scrape content from {url} after trying multiple User-Agents")
-                    continue
-                
-                # Limit content to max_chars
-                scraped_content.append({
-                    "title": title,
-                    "url": url,
-                    "content": content[:max_chars],
-                    "scraper": scraper
-                })
-            except requests.exceptions.RequestException as e:
-                logger.error(f"Error scraping {url}: {e}")
-            except Exception as e:
-                logger.error(f"Unexpected error while scraping {url}: {e}")
+                    
+                    # Limit content to max_chars
+                    scraped_content.append({
+                        "title": title,
+                        "url": url,
+                        "content": content[:max_chars],
+                        "scraper": scraper
+                    })
+                    logger.info(f"Successfully scraped content from {url}. Total scraped: {len(scraped_content)}")
+                except requests.exceptions.RequestException as e:
+                    logger.error(f"Error scraping {url}: {e}")
+                except Exception as e:
+                    logger.error(f"Unexpected error while scraping {url}: {e}")
+
+            page += 1
 
         if not scraped_content:
             logger.warning("No content scraped from search results.")
             return "No content could be scraped from the search results."
 
+        logger.info(f"Successfully scraped {len(scraped_content)} documents.")
+
         # Step 3: Assess relevance, summarize, and check for uniqueness
         relevant_documents = []
         unique_summaries = []
@@ -473,13 +487,13 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="trafilatura",
         
         logger.info(f"Reranked and filtered to top {len(reranked_docs)} unique, finance-related documents.")
 
-        # Step 5: Scrape full content for top 5 documents
-        for doc in reranked_docs[:5]:
+        # Step 5: Scrape full content for top documents (up to num_results)
+        for doc in reranked_docs[:num_results]:
             full_content = scrape_full_content(doc['url'], scraper, max_chars)
             doc['full_content'] = full_content
 
         # Step 6: LLM Summarization
-        llm_summary = llm_summarize(query, reranked_docs[:5], client, temperature=llm_temperature)
+        llm_summary = llm_summarize(query, reranked_docs[:num_results], client, temperature=llm_temperature)
 
         return llm_summary