Spaces:
Runtime error
Runtime error
| import requests | |
| from bs4 import BeautifulSoup, Comment | |
| import logging | |
| import re | |
| from duckduckgo_search import DDGS | |
| logger = logging.getLogger(__name__) | |
| DEFAULT_TIMEOUT = 10 | |
| MAX_CONTENT_LENGTH_PER_URL = 3000 | |
| MAX_TOTAL_SCRAPED_CONTENT = 9000 | |
| def scrape_url(url_to_scrape): | |
| try: | |
| headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'} | |
| response = requests.get(url_to_scrape, headers=headers, timeout=DEFAULT_TIMEOUT, allow_redirects=True) | |
| response.raise_for_status() | |
| if 'html' not in response.headers.get('content-type', '').lower(): | |
| return {"url": url_to_scrape, "error": "Non-HTML content"} | |
| soup = BeautifulSoup(response.content, 'html.parser') | |
| for el_type in ["script", "style", "nav", "footer", "aside", "form", "header", "menu"]: | |
| for el in soup.select(el_type): el.decompose() | |
| for comment in soup.find_all(string=lambda text: isinstance(text, Comment)): | |
| comment.extract() | |
| main_selectors = ['main', 'article', '.main-content', '.entry-content', 'div[role="main"]'] | |
| content_area = next((soup.select_one(s) for s in main_selectors if soup.select_one(s)), soup.body or soup) | |
| text_parts = [el.get_text(separator=' ', strip=True) for el in content_area.find_all(['p', 'h1', 'h2', 'h3', 'li', 'td'])] | |
| full_text = "\n\n".join(filter(None, text_parts)) | |
| if not full_text: full_text = content_area.get_text(separator='\n', strip=True) | |
| full_text = re.sub(r'\s{3,}', ' ', full_text).strip() | |
| full_text = re.sub(r'(\n\s*){3,}', '\n\n', full_text) | |
| title = soup.find('title').string.strip() if soup.find('title') and soup.find('title').string else url_to_scrape | |
| return {"url": url_to_scrape, "title": title, "content": full_text[:MAX_CONTENT_LENGTH_PER_URL]} | |
| except requests.exceptions.RequestException as e: | |
| logger.error(f"WEBSEARCH: Request failed for {url_to_scrape}: {e}") | |
| return {"url": url_to_scrape, "error": f"Request failed: {e}"} | |
| except Exception as e: | |
| logger.error(f"WEBSEARCH: Scraping error for {url_to_scrape}: {e}", exc_info=True) | |
| return {"url": url_to_scrape, "error": f"Scraping error: {e}"} | |
| def search_and_scrape_duckduckgo(search_query, num_results=3): | |
| scraped_data = [] | |
| total_content_len = 0 | |
| try: | |
| with DDGS() as ddgs: | |
| results = list(ddgs.text(search_query, max_results=num_results + 2, region='wt-wt')) | |
| if not results: return [{"query": search_query, "error": "No search results."}] | |
| for res in results[:num_results]: | |
| if total_content_len >= MAX_TOTAL_SCRAPED_CONTENT: break | |
| if not res.get('href'): continue | |
| scraped_info = scrape_url(res['href']) | |
| if not scraped_info.get("title") or scraped_info.get("title") == res['href']: | |
| scraped_info["title"] = res.get('title', res['href']) | |
| current_content = scraped_info.get("content", "") | |
| if res.get('body') and (not current_content or len(current_content) < 150 or scraped_info.get("error")): | |
| scraped_info["content"] = f"Search snippet: {res['body']}\n\n(Content from page):\n{current_content}" | |
| scraped_data.append(scraped_info) | |
| if scraped_info.get("content") and not scraped_info.get("error"): | |
| total_content_len += len(scraped_info["content"]) | |
| return scraped_data | |
| except Exception as e: | |
| logger.error(f"WEBSEARCH (DDG): Error for '{search_query}': {e}", exc_info=True) | |
| return [{"query": search_query, "error": f"DDG search failed: {e}"}] |