app.py

import gradio as gr
import requests
import time
import random
from bs4 import BeautifulSoup
import trafilatura

USER_AGENTS = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1'
]

def get_random_user_agent():
    return random.choice(USER_AGENTS)

def extract_content_bs4(url, max_chars):
    try:
        response = requests.get(url, headers={'User-Agent': get_random_user_agent()}, timeout=10)
        soup = BeautifulSoup(response.content, 'html.parser')
        paragraphs = soup.find_all('p')
        content = ' '.join([p.text for p in paragraphs])
        return content[:max_chars] + "..." if len(content) > max_chars else content
    except Exception as e:
        return f"Error extracting content: {str(e)}"

def extract_content_trafilatura(url, max_chars):
    try:
        downloaded = trafilatura.fetch_url(url, headers={'User-Agent': get_random_user_agent()})
        content = trafilatura.extract(downloaded, include_comments=False, include_tables=False)
        return content[:max_chars] + "..." if content and len(content) > max_chars else content
    except Exception as e:
        return f"Error extracting content: {str(e)}"

def search_searx(query, instance_url='https://searx.org', categories='general', max_retries=3, num_results=10, 
                 use_trafilatura=False, time_range='', language='en', safesearch=0, search_engines='all', 
                 sort_by='relevance', max_chars=1000):
    """
    Perform a search using the SearXNG API with advanced options.
    """
    search_endpoint = f"{instance_url}/search"
    params = {
        'q': query,
        'format': 'json',
        'categories': categories,
        'pageno': 1,
        'time_range': time_range,
        'language': language,
        'safesearch': safesearch,
        'results': str(num_results),
        'engines': ','.join(search_engines) if 'all' not in search_engines else 'all',
        'sort': sort_by
    }
    
    headers = {
        'User-Agent': get_random_user_agent(),
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Accept-Language': 'en-US,en;q=0.5',
        'Referer': instance_url,
        'DNT': '1',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1'
    }
    
    for attempt in range(max_retries):
        try:
            response = requests.get(search_endpoint, params=params, headers=headers, timeout=10)
            response.raise_for_status()
            data = response.json()
            
            if 'results' not in data or not data['results']:
                return "No results found."
            
            formatted_results = ""
            for idx, result in enumerate(data['results'][:num_results], start=1):
                title = result.get('title', 'No Title')
                url = result.get('url', 'No URL')
                
                if use_trafilatura:
                    content = extract_content_trafilatura(url, max_chars)
                else:
                    content = extract_content_bs4(url, max_chars)
                
                formatted_results += f"**{idx}. {title}**\n[{url}]({url})\n{content}\n\n"
            
            return formatted_results
        except requests.exceptions.RequestException as e:
            if response.status_code == 429:
                wait_time = 2 ** attempt + random.uniform(0, 1)
                time.sleep(wait_time)
            else:
                return f"An error occurred while searching: {e}"
    
    return "Max retries reached. Please try again later."

def create_gradio_interface():
    """
    Creates and returns the Gradio interface with advanced SearXNG options and new parameters.
    """
    with gr.Blocks() as demo:
        gr.Markdown("# 🕵️‍♂️ Advanced SearXNG Search with Content Extraction")
        gr.Markdown(
            "This application allows you to perform private searches using SearXNG with advanced options and content extraction."
        )
        with gr.Row():
            with gr.Column():
                query = gr.Textbox(
                    label="Search Query",
                    placeholder="Enter your search query here...",
                    lines=1
                )
                instance_url = gr.Textbox(
                    label="SearXNG Instance URL",
                    value="https://searx.org",
                    placeholder="https://searx.instance.url",
                    lines=1
                )
                categories = gr.Textbox(
                    label="Categories",
                    value="general",
                    placeholder="e.g., general, news, science",
                    lines=1
                )
                num_results = gr.Slider(
                    minimum=1,
                    maximum=20,
                    value=10,
                    step=1,
                    label="Number of Results"
                )
                use_trafilatura = gr.Checkbox(label="Use Trafilatura for extraction (instead of BeautifulSoup)")
                time_range = gr.Dropdown(
                    choices=["", "day", "week", "month", "year"],
                    value="",
                    label="Time Range"
                )
                language = gr.Textbox(
                    label="Language",
                    value="en",
                    placeholder="e.g., en, fr, de",
                    lines=1
                )
                safesearch = gr.Slider(
                    minimum=0,
                    maximum=2,
                    value=0,
                    step=1,
                    label="SafeSearch (0: Off, 1: Moderate, 2: Strict)"
                )
                
                # New parameters
                search_engines = gr.Dropdown(
                    choices=["all", "google", "bing", "duckduckgo", "wikipedia"],
                    value="all",
                    label="Search Engines",
                    multiselect=True
                )
                sort_by = gr.Dropdown(
                    choices=["relevance", "date"],
                    value="relevance",
                    label="Sort Results By"
                )
                max_chars = gr.Slider(
                    minimum=100,
                    maximum=10000,
                    value=1000,
                    step=100,
                    label="Max Characters to Extract"
                )
                
                search_button = gr.Button("Search")
            with gr.Column():
                results = gr.Markdown("### Search Results will appear here...")
        
        def perform_search(q, url, cats, num, use_traf, t_range, lang, safe, engines, sort, chars):
            return search_searx(q, instance_url=url, categories=cats, num_results=int(num), 
                                use_trafilatura=use_traf, time_range=t_range, language=lang, safesearch=int(safe),
                                search_engines=engines, sort_by=sort, max_chars=chars)
        
        search_button.click(
            perform_search,
            inputs=[query, instance_url, categories, num_results, use_trafilatura, time_range, language, safesearch,
                    search_engines, sort_by, max_chars],
            outputs=results
        )
        
        gr.Markdown(
            """
            ---
            **Note:** This application uses SearXNG to fetch results from multiple sources while preserving your privacy. 
            It then attempts to extract content from the original sources, which may be subject to the terms of service of those websites.
            """
        )
    
    return demo

iface = create_gradio_interface()

if __name__ == "__main__":
    iface.launch()