Update search.py

search.py

@@ -1,76 +1,54 @@
+from googlesearch import search
 import requests
-from parsel import Selector
 from bs4 import BeautifulSoup
+import re
+from functools import lru_cache
 
-def extract_structured_data(url):
-    """Extract clean, structured content from webpage"""
+# Clean HTML tags
+TAG_CLEANER = re.compile(r"<[^>]+>")
+
+@lru_cache(maxsize=500)
+def extract_metadata(url):
+    """Extract title and description from URL"""
     try:
         headers = {
             'User-Agent': 'Mozilla/5.0 (compatible; ResearchBot/1.0)',
             'Accept-Language': 'en-US,en;q=0.9'
         }
-        response = requests.get(url, timeout=8, headers=headers)
+        response = requests.get(url, timeout=5, headers=headers)
         response.raise_for_status()
         
-        # Using Parsel + BeautifulSoup hybrid for efficiency
-        sel = Selector(text=response.text)
         soup = BeautifulSoup(response.text, 'lxml')
         
-        # Extract key elements
-        title = sel.css("title::text").get() or url
-        main_content = ""
-        
-        # Try to find article body
-        for selector in ['article', '.article-body', '.post-content', '#content']:
-            if elem := soup.select_one(selector):
-                main_content = elem.get_text(separator=' ', strip=True)[:2000]
-                break
+        title = soup.title.string.strip() if soup.title else url
+        title = title[:200]
         
-        # Fallback to paragraph aggregation
-        if not main_content:
-            paragraphs = soup.find_all('p')
-            main_content = " ".join(p.get_text().strip() for p in paragraphs)[:2000]
-        
-        # Extract metadata
-        description = sel.css('meta[name="description"]::attr(content)').get() or ""
+        # Try to get description
+        description = ""
+        if meta_desc := soup.find("meta", attrs={"name": "description"}):
+            description = meta_desc.get("content", "")[:300]
         
         return {
             "url": url,
             "title": title,
-            "content": main_content,
             "description": description
         }
     except Exception as e:
-        print(f"Error scraping {url}: {str(e)[:100]}")
-        return None
+        return {"url": url, "title": f"Error: {str(e)[:30]}", "description": ""}
 
 def search_google(query, num_results=5):
     """Search with enhanced result parsing"""
     try:
-        # Use custom search API or fallback
-        from googlesearch import search
-        urls = list(search(query, num_results=num_results, advanced=True))
+        # Get search results
+        urls = list(search(query, num_results=num_results, advanced=False))
         
+        # Extract metadata for each URL
         results = []
-        for i, result in enumerate(urls[:num_results]):
-            # Use metadata if available
-            base_data = {
-                "url": result.url,
-                "title": result.title,
-                "description": result.description
-            }
-            
-            # Only do full extraction for top 2 results
-            if i < 2:
-                if structured := extract_structured_data(result.url):
-                    results.append(structured)
-                else:
-                    results.append(base_data)
-            else:
-                results.append(base_data)
+        for url in urls[:num_results]:
+            if metadata := extract_metadata(url):
+                results.append(metadata)
         
         return results
-    except:
-        # Fallback to simple search
-        from googlesearch import search
-        return [{"url": url} for url in search(query, num_results=num_results)]
+    except Exception as e:
+        print(f"Search error: {e}")
+        return []