Update search.py

search.py

@@ -1,36 +1,76 @@
 import requests
 from parsel import Selector
+from bs4 import BeautifulSoup
 
 def extract_structured_data(url):
     """Extract clean, structured content from webpage"""
     try:
-        response = requests.get(url, timeout=10, 
-                               headers={'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)'})
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)',
+            'Accept-Language': 'en-US,en;q=0.9'
+        }
+        response = requests.get(url, timeout=8, headers=headers)
         response.raise_for_status()
         
+        # Using Parsel + BeautifulSoup hybrid for efficiency
         sel = Selector(text=response.text)
+        soup = BeautifulSoup(response.text, 'lxml')
+        
+        # Extract key elements
+        title = sel.css("title::text").get() or url
+        main_content = ""
+        
+        # Try to find article body
+        for selector in ['article', '.article-body', '.post-content', '#content']:
+            if elem := soup.select_one(selector):
+                main_content = elem.get_text(separator=' ', strip=True)[:2000]
+                break
+        
+        # Fallback to paragraph aggregation
+        if not main_content:
+            paragraphs = soup.find_all('p')
+            main_content = " ".join(p.get_text().strip() for p in paragraphs)[:2000]
+        
+        # Extract metadata
+        description = sel.css('meta[name="description"]::attr(content)').get() or ""
         
         return {
             "url": url,
-            "title": sel.css("title::text").get() or url,
-            "content": " ".join(sel.css("p::text, article::text, .main-content::text").getall())[:5000],
-            "key_points": sel.css("li::text, .summary::text, .highlight::text").getall()[:5]
+            "title": title,
+            "content": main_content,
+            "description": description
         }
     except Exception as e:
-        print(f"Error scraping {url}: {e}")
+        print(f"Error scraping {url}: {str(e)[:100]}")
         return None
 
-def search_google(query, num_results=5):    
-    mock_results = [
-        {"url": "https://en.wikipedia.org/wiki/Quantum_computing", "title": "Quantum Computing - Wikipedia"},
-        {"url": "https://www.ibm.com/quantum-computing", "title": "IBM Quantum Computing"},
-        {"url": "https://quantum.microsoft.com", "title": "Microsoft Quantum"}
-    ]
-    
-    # Extract structured data from each URL
-    structured_results = []
-    for result in mock_results[:num_results]:
-        if data := extract_structured_data(result["url"]):
-            structured_results.append(data)
-    
-    return structured_results
+def search_google(query, num_results=5):
+    """Search with enhanced result parsing"""
+    try:
+        # Use custom search API or fallback
+        from googlesearch import search
+        urls = list(search(query, num_results=num_results, advanced=True))
+        
+        results = []
+        for i, result in enumerate(urls[:num_results]):
+            # Use metadata if available
+            base_data = {
+                "url": result.url,
+                "title": result.title,
+                "description": result.description
+            }
+            
+            # Only do full extraction for top 2 results
+            if i < 2:
+                if structured := extract_structured_data(result.url):
+                    results.append(structured)
+                else:
+                    results.append(base_data)
+            else:
+                results.append(base_data)
+        
+        return results
+    except:
+        # Fallback to simple search
+        from googlesearch import search
+        return [{"url": url} for url in search(query, num_results=num_results)]