File size: 1,398 Bytes
6d24925 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
import requests
import trafilatura
from newspaper import Article
from typing import Optional
from bs4 import BeautifulSoup
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/115.0.0.0 Safari/537.36"
)
}
def clean_text(text: str) -> str:
# Remove excess whitespace, ads, and headings
cleaned = text.replace("\n", " ").strip()
cleaned = BeautifulSoup(cleaned, "html.parser").text # remove tags
cleaned = " ".join(cleaned.split()) # remove multiple spaces
return cleaned
def scrape_url(url: str, timeout: int = 10) -> Optional[str]:
try:
response = requests.get(url, timeout=timeout, headers=HEADERS)
if response.status_code == 200:
html = response.text
extracted = trafilatura.extract(html, include_comments=False, include_tables=False)
if extracted and len(extracted.split()) > 100:
return clean_text(extracted)
except Exception as e:
print(f"⚠️ Trafilatura failed for {url}: {e}")
try:
article = Article(url)
article.download()
article.parse()
if article.text and len(article.text.split()) > 100:
return clean_text(article.text)
except Exception as e:
print(f"⚠️ Newspaper3k failed for {url}: {e}")
return None
|