Update app.py

app.py

@@ -348,10 +348,20 @@ def stream_response(msg: str, history: list[Dict[str, str]] = [], max_tokens=500
             print(f"Unexpected LLM error (attempt {attempt+1}): {e}. Retrying...")
             time.sleep(2**attempt)
 
-def generate_dataset_names(search_query: str, history: list[Dict[str, str]]) -> Iterator[str]:
+def generate_dataset_names(search_query: str, history: list[Dict[str, str]], is_real_data: bool = False, engine: Optional[str] = None) -> Iterator[str]:
     """Generates dataset names based on a search query using the LLM."""
     query = search_query[:1000] if search_query else ""
-    prompt = GENERATE_DATASET_NAMES_FOR_SEARCH_QUERY.format(search_query=query)
+    
+    if is_real_data and engine:
+        prompt = (
+            f"@Claude-3.7-Sonnet You are a data specialist who can transform real search results into structured datasets. "
+            f"A user is searching for data about: \"{query}\" "
+            f"Imagine you've queried {engine} and received real search results. Create a list of {MAX_NB_ITEMS_PER_GENERATION_CALL} specific datasets that could be created from these search results. "
+            f"For each dataset: 1. Give it a clear, specific name related to the search topic. 2. Include 3-5 relevant tags in parentheses, with one tag specifying the ML task type (classification, regression, clustering, etc.). "
+            f"Format each dataset as: 1. DatasetName (tag1, tag2, ml_task_tag). Make these datasets sound like real collections that could be created from {engine} search results on \"{query}\"."
+        )
+    else:
+        prompt = GENERATE_DATASET_NAMES_FOR_SEARCH_QUERY.format(search_query=query)
     
     full_response = ""
     for token in stream_response(prompt, history):
@@ -362,12 +372,23 @@ def generate_dataset_names(search_query: str, history: list[Dict[str, str]]) ->
     history.append({"role": "assistant", "content": full_response}) # Update history
     # No return needed as history is modified in place
 
-def generate_dataset_content(search_query: str, dataset_name: str, tags: str, history: list[Dict[str, str]]) -> Iterator[str]:
+def generate_dataset_content(search_query: str, dataset_name: str, tags: str, history: list[Dict[str, str]], is_real_data: bool = False, engine: Optional[str] = None) -> Iterator[str]:
     """Generates the description and CSV preview for a dataset."""
     query = search_query[:1000] if search_query else ""
-    prompt = GENERATE_DATASET_CONTENT_FOR_SEARCH_QUERY_AND_NAME_AND_TAGS.format(
-        search_query=query, dataset_name=dataset_name, tags=tags
-    )
+    
+    if is_real_data and engine:
+        prompt = (
+            f"@Claude-3.7-Sonnet You're a specialist in converting web search results into structured data. "
+            f"Based on search results from {engine} about \"{query}\", create a preview of the dataset \"{dataset_name}\" with tags \"{tags}\". "
+            f"First, write a detailed description of what this dataset contains, its structure, and how it was constructed from web search results. "
+            f"Then, generate a realistic 5-row CSV preview that resembles data you might get if you scraped and structured real results from {engine}. "
+            f"Format your response with: **Dataset Description:** [detailed description] **CSV Content Preview:** ```csv [CSV header and 5 rows of realistic data] ``` "
+            f"Include relevant columns for the dataset type, with proper labels/categories where appropriate. The data should look like it came from real sources."
+        )
+    else:
+        prompt = GENERATE_DATASET_CONTENT_FOR_SEARCH_QUERY_AND_NAME_AND_TAGS.format(
+            search_query=query, dataset_name=dataset_name, tags=tags
+        )
     
     full_response = ""
     for token in stream_response(prompt, history):
@@ -411,15 +432,33 @@ def iflatmap_unordered(func: Callable, kwargs_iterable: Iterable[dict]) -> Itera
 
 def generate_partial_dataset(
     title: str, content: str, search_query: str, variant: str, csv_header: str,
-    output: list[Optional[dict]], indices_to_generate: list[int], history: list[Dict[str, str]]
+    output: list[Optional[dict]], indices_to_generate: list[int], history: list[Dict[str, str]],
+    is_real_data: bool = False, engine: Optional[str] = None
 ) -> Iterator[int]:
     """Generates a batch of dataset rows for a specific variant."""
     dataset_name, tags = title.strip("# ").split("\ntags:", 1)
     dataset_name, tags = dataset_name.strip(), tags.strip()
     
     prompt = GENERATE_MORE_ROWS.format(csv_header=csv_header) + " " + variant
+    
+    # Construct initial messages for context
+    initial_prompt = ""
+    if is_real_data and engine:
+        initial_prompt = (
+            f"@Claude-3.7-Sonnet You're a specialist in converting web search results into structured data. "
+            f"Based on search results from {engine} about \"{search_query}\", create a preview of the dataset \"{dataset_name}\" with tags \"{tags}\". "
+            f"First, write a detailed description of what this dataset contains, its structure, and how it was constructed from web search results. "
+            f"Then, generate a realistic 5-row CSV preview that resembles data you might get if you scraped and structured real results from {engine}. "
+            f"Format your response with: **Dataset Description:** [detailed description] **CSV Content Preview:** ```csv [CSV header and 5 rows of realistic data] ``` "
+            f"Include relevant columns for the dataset type, with proper labels/categories where appropriate. The data should look like it came from real sources."
+        )
+    else:
+        initial_prompt = GENERATE_DATASET_CONTENT_FOR_SEARCH_QUERY_AND_NAME_AND_TAGS.format(
+            search_query=search_query, dataset_name=dataset_name, tags=tags
+        )
+        
     messages = [
-        {"role": "user", "content": GENERATE_DATASET_CONTENT_FOR_SEARCH_QUERY_AND_NAME_AND_TAGS.format(search_query=search_query, dataset_name=dataset_name, tags=tags)},
+        {"role": "user", "content": initial_prompt},
         {"role": "assistant", "content": title + "\n\n" + content},
         {"role": "user", "content": prompt},
     ]
@@ -531,6 +570,11 @@ def get_repo_visibility(repo_id: str, token: str) -> str:
 
 with gr.Blocks(css=css) as demo:
     generated_texts_state = gr.State((landing_page_datasets_generated_text,)) # State for generated dataset names
+    current_dataset_state = gr.State(None) # State to hold current dataset details for generation
+    is_real_data_state = gr.State(True) # State to track if real data is being used
+    current_engine_state = gr.State(None) # State to track the current search engine
+    selected_engines_state = gr.State(["DuckDuckGo.com", "Bing.com", "Search.Yahoo.com", "Search.Brave.com", "Ecosia.org"]) # Default selected engines
+    searchEngines = ["AlltheInternet.com", "DuckDuckGo.com", "Google.com", "Bing.com", "Search.Yahoo.com", "Startpage.com", "Qwant.com", "Ecosia.org", "WolframAlpha.com", "Mojeek.co.uk", "Search.Brave.com", "Yandex.com", "Baidu.com", "Gibiru.com", "MetaGer.org", "Swisscows.com", "Presearch.com", "Ekoru.org", "Search.Lilo.org"]
     
     # --- Search Page UI ---
     with gr.Column(visible=True, elem_id="search-page") as search_page:
@@ -591,24 +635,50 @@ with gr.Blocks(css=css) as demo:
                     label="Dataset Visibility", info="Set visibility for datasets saved to Hugging Face Hub."
                 )
 
+                # Search Engine Settings
+                gr.Markdown("Search Engine Configuration")
+                data_source_toggle = gr.Checkbox(label="Use Real Search Data", value=True, info="Toggle to include results from real search engines.")
+                engine_settings_button = gr.Button("Configure Search Engines", icon="https://img.icons8.com/ios-filled/50/000000/settings--v1.png", size="sm")
+
+                # Engine Selection Modal
+                with gr.Modal("Search Engine Settings", id="engine-modal") as engine_modal:
+                    gr.Markdown("Select which search engines to use for real data retrieval. A diverse selection improves results.")
+                    engine_options_html_comp = gr.HTML(elem_id="engine-options")
+                    with gr.Row():
+                        select_all_engines_btn = gr.Button("Select All")
+                        deselect_all_engines_btn = gr.Button("Deselect All")
+                    save_engines_btn = gr.Button("Save Settings", variant="primary")
+
     # --- Dataset Detail Page UI ---
     with gr.Column(visible=False, elem_id="dataset-page") as dataset_page:
         gr.Markdown("# 🤗 Infinite Dataset Hub ♾️\n\nAn endless catalog of datasets, created just for you.")
-        dataset_title = gr.Markdown() # Dataset name and tags
-        gr.Markdown("_Note: This is an AI-generated dataset so its content may be inaccurate or false_")
-        dataset_content = gr.Markdown() # Description and CSV preview
-        generate_full_dataset_button = gr.Button("Generate Full Dataset", variant="primary")
-        dataset_dataframe = gr.DataFrame(visible=False, interactive=False, wrap=True) # Holds the full generated dataset
-        save_dataset_button = gr.Button("💾 Save Dataset", variant="primary", visible=False)
+        dataset_title_md = gr.Markdown() # Dataset name and tags
+        dataset_source_badge = gr.Markdown() # Badge indicating real/AI data
+        dataset_source_info = gr.Markdown() # Details about the data source
+        dataset_description_md = gr.Markdown() # Dataset description
+        preview_table_comp = gr.DataFrame(visible=False, interactive=False, wrap=True) # Holds the preview CSV
+        
+        with gr.Row():
+            generate_full_dataset_button = gr.Button("Generate Full Dataset", variant="primary")
+            save_dataset_button = gr.Button("💾 Save Dataset", variant="primary", visible=False)
+        
         open_dataset_message = gr.Markdown("", visible=False) # Confirmation message
         dataset_share_button = gr.Button("Share Dataset URL")
         dataset_share_textbox = gr.Textbox(visible=False, show_copy_button=True, label="Copy this URL:", interactive=False, show_label=True)
+        
+        full_dataset_section = gr.Column(visible=False) # Container for full dataset and downloads
+        full_table_comp = gr.DataFrame(visible=False, interactive=False, wrap=True)
+        with gr.Row():
+            download_csv_button = gr.Button("Download CSV")
+            download_json_button = gr.Button("Download JSON")
+            download_parquet_button = gr.Button("Download Parquet")
+        
         back_button = gr.Button("< Back", size="sm")
 
     # --- Event Handlers ---
 
     # Search Logic
-    def _update_search_results(search_query: str, current_generated_texts: tuple[str]):
+    def _update_search_results(search_query: str, current_generated_texts: tuple[str], is_real_data: bool, engine: Optional[str]):
         """Handles dataset search and UI updates."""
         # Reset UI to loading state
         yield {btn: gr.Button("⬜⬜⬜⬜⬜⬜", elem_classes="topButton linear-background") for btn in buttons[::2]}
@@ -620,7 +690,7 @@ with gr.Blocks(css=css) as demo:
         
         try:
             # Generate dataset names from LLM
-            for line in generate_dataset_names(search_query, []):
+            for line in generate_dataset_names(search_query, [], is_real_data=is_real_data, engine=engine):
                 if "I'm sorry" in line or "policy" in line: raise gr.Error("Inappropriate content detected.")
                 if generated_count >= MAX_NB_ITEMS_PER_GENERATION_CALL: break
                 
@@ -646,58 +716,106 @@ with gr.Blocks(css=css) as demo:
         except Exception as e: raise gr.Error(f"Failed to generate datasets: {str(e)}")
 
     # Attach search handlers
-    search_button.click(_update_search_results, inputs=[search_bar, generated_texts_state], outputs=buttons + [generated_texts_state] + button_groups)
-    search_bar.submit(_update_search_results, inputs=[search_bar, generated_texts_state], outputs=buttons + [generated_texts_state] + button_groups)
+    search_button.click(
+        _update_search_results, 
+        inputs=[search_bar, generated_texts_state, is_real_data_state, current_engine_state],
+        outputs=buttons + [generated_texts_state] + button_groups
+    )
+    search_bar.submit(
+        _update_search_results, 
+        inputs=[search_bar, generated_texts_state, is_real_data_state, current_engine_state],
+        outputs=buttons + [generated_texts_state] + button_groups
+    )
     
     # Load More Datasets
-    load_more_datasets.click(_update_search_results, inputs=[search_bar, generated_texts_state], outputs=buttons + [generated_texts_state] + button_groups)
+    load_more_datasets.click(
+        _update_search_results, 
+        inputs=[search_bar, generated_texts_state, is_real_data_state, current_engine_state],
+        outputs=buttons + [generated_texts_state] + button_groups
+    )
     
     # Display Single Dataset Details
-    def _show_dataset_details(search_query, dataset_name, tags):
+    def _show_dataset_details(search_query, dataset_name, tags, is_real_data, engine):
         """Switches to detail view and loads dataset content."""
         yield {
             search_page: gr.Column(visible=False), dataset_page: gr.Column(visible=True),
-            dataset_title: f"# {dataset_name}\n\n tags: {tags}",
-            dataset_share_textbox: gr.Textbox(visible=False), dataset_dataframe: gr.DataFrame(visible=False),
-            generate_full_dataset_button: gr.Button(interactive=True), save_dataset_button: gr.Button(visible=False),
+            dataset_title_md: f"# {dataset_name}\n\n tags: {tags}",
+            dataset_share_textbox: gr.Textbox(visible=False),
+            full_dataset_section: gr.Column(visible=False),
+            save_dataset_button: gr.Button(visible=False),
             open_dataset_message: gr.Markdown("", visible=False)
         }
+        
+        # Update source badge and info
+        if is_real_data:
+            badge_html = gr.Markdown(f'<span class="px-3 py-1 rounded-full text-xs font-medium bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200">Real Data</span>', visible=True)
+            info_html = gr.Markdown(f'This dataset is based on real information queried from <strong>{engine}</strong> for the search term "<strong>{search_query}</strong>". The data has been structured for machine learning use.', visible=True)
+        else:
+            badge_html = gr.Markdown('<span class="px-3 py-1 rounded-full text-xs font-medium bg-purple-100 text-purple-800 dark:bg-purple-900 dark:text-purple-200">AI-Generated</span>', visible=True)
+            info_html = gr.Markdown(f'This is an AI-generated dataset created using {model_id}. The content is synthetic and designed to represent plausible data related to "{search_query}".', visible=True)
+        
+        yield {dataset_source_badge: badge_html, dataset_source_info: info_html}
+
         # Stream content generation
-        for content_chunk in generate_dataset_content(search_query, dataset_name, tags, []):
-            yield {dataset_content: content_chunk}
+        for content_chunk in generate_dataset_content(search_query, dataset_name, tags, [], is_real_data=is_real_data, engine=engine):
+            yield {dataset_description_md: content_chunk}
 
     # Link buttons to the detail view function
-    show_dataset_inputs = [search_bar, *buttons]
-    show_dataset_outputs = [search_page, dataset_page, dataset_title, dataset_content, generate_full_dataset_button, dataset_dataframe, save_dataset_button, open_dataset_message, dataset_share_textbox]
-    
-    scroll_to_top_js = """
-    function() {
-        if ('parentIFrame' in window) { window.parentIFrame.scrollTo({top: 0, behavior:'smooth'}); } 
-        else { window.scrollTo({ top: 0, behavior: 'smooth' }); }
-        return Array.from(arguments);
-    }
-    """
-    def _show_dataset_from_button(search_query, *buttons_values, i):
-        dataset_name, tags = buttons_values[2 * i : 2 * i + 2]
-        yield from _show_dataset_details(search_query, dataset_name, tags)
+    def _show_dataset_from_button_wrapper(search_query, *buttons_values):
+        # Determine which button was clicked to get the index
+        clicked_button_index = -1
+        for i, btn_val in enumerate(buttons_values):
+            if btn_val is not None and btn_val != "": # Assuming non-empty value indicates the clicked button's text
+                clicked_button_index = i
+                break
+        
+        if clicked_button_index == -1: return # Should not happen if events are correctly wired
+
+        # Determine if it was a name button (even index) or tag button (odd index)
+        dataset_index = clicked_button_index // 2
+        
+        dataset_name, tags = buttons_values[2 * dataset_index], buttons_values[2 * dataset_index + 1]
+        is_real_data = current_engine_state.value is not None # Infer from engine state
+        engine = current_engine_state.value if is_real_data else None
+        
+        yield from _show_dataset_details(search_query, dataset_name, tags, is_real_data, engine)
     
+    # Wire up click events for all dataset name and tag buttons
     for i, (name_btn, tag_btn) in enumerate(batched(buttons, 2)):
-        name_btn.click(partial(_show_dataset_from_button, i=i), inputs=show_dataset_inputs, outputs=show_dataset_outputs, js=scroll_to_top_js)
-        tag_btn.click(partial(_show_dataset_from_button, i=i), inputs=show_dataset_inputs, outputs=show_dataset_outputs, js=scroll_to_top_js)
+        name_btn.click(
+            partial(_show_dataset_from_button_wrapper), 
+            inputs=[search_bar, *buttons], 
+            outputs=[search_page, dataset_page, dataset_title_md, dataset_description_md, dataset_source_badge, dataset_source_info, dataset_share_textbox, full_dataset_section, save_dataset_button, open_dataset_message]
+        )
+        tag_btn.click(
+            partial(_show_dataset_from_button_wrapper), 
+            inputs=[search_bar, *buttons], 
+            outputs=[search_page, dataset_page, dataset_title_md, dataset_description_md, dataset_source_badge, dataset_source_info, dataset_share_textbox, full_dataset_section, save_dataset_button, open_dataset_message]
+        )
     
     # Back Button Navigation
-    back_button.click(lambda: (gr.Column(visible=True), gr.Column(visible=False)), outputs=[search_page, dataset_page], js=scroll_to_top_js)
+    back_button.click(lambda: (gr.Column(visible=True), gr.Column(visible=False)), outputs=[search_page, dataset_page], js="""
+        function() {
+            if ('parentIFrame' in window) { window.parentIFrame.scrollTo({top: 0, behavior:'smooth'}); } 
+            else { window.scrollTo({ top: 0, behavior: 'smooth' }); }
+            return Array.from(arguments);
+        }
+    """)
     
     # Full Dataset Generation
     @generate_full_dataset_button.click(
-        inputs=[dataset_title, dataset_content, search_bar, select_namespace_dropdown, visibility_radio, refinement_mode],
-        outputs=[dataset_dataframe, generate_full_dataset_button, save_dataset_button]
+        inputs=[dataset_title_md, dataset_description_md, search_bar, select_namespace_dropdown, visibility_radio, refinement_mode, is_real_data_state, current_engine_state],
+        outputs=[full_table_comp, generate_full_dataset_button, save_dataset_button, full_dataset_section]
     )
-    def _generate_full_dataset(title, content, search_query, namespace, visibility, mode):
-        dataset_name, tags = title.strip("# ").split("\ntags:", 1)
-        dataset_name, tags = dataset_name.strip(), tags.strip()
-        
-        try: csv_header, preview_df = parse_preview_df(content)
+    def _generate_full_dataset(title_md, content_md, search_query, namespace, visibility, mode, is_real_data, engine):
+        # Extract dataset name and tags from the markdown title
+        try:
+            dataset_name = title_md.split('\n')[0].strip('# ')
+            tags = title_md.split('tags:', 1)[1].strip()
+        except IndexError:
+            raise gr.Error("Could not parse dataset title.")
+            
+        try: csv_header, preview_df = parse_preview_df(content_md)
         except ValueError as e: raise gr.Error(f"Failed to parse preview: {e}")
             
         refined_preview_df = refine_preview_data(preview_df, mode)
@@ -710,9 +828,10 @@ with gr.Blocks(css=css) as demo:
         
         # Update UI: show preview, disable generate, show save button
         yield {
-            dataset_dataframe: gr.DataFrame(pd.DataFrame([r for r in output_data if r]), visible=True),
+            full_table_comp: gr.DataFrame(pd.DataFrame([r for r in output_data if r]), visible=True),
             generate_full_dataset_button: gr.Button(interactive=False),
-            save_dataset_button: gr.Button(f"💾 Save {namespace}/{dataset_name}" + (" (private)" if visibility != "public" else ""), visible=True, interactive=False)
+            save_dataset_button: gr.Button(f"💾 Save {namespace}/{dataset_name}" + (" (private)" if visibility != "public" else ""), visible=True, interactive=False),
+            full_dataset_section: gr.Column(visible=True)
         }
         
         # Prepare generation tasks for variants
@@ -724,27 +843,32 @@ with gr.Blocks(css=css) as demo:
                 generation_tasks.append({
                     "func": generate_partial_dataset,
                     "kwargs": {
-                        "title": title, "content": content, "search_query": search_query, "variant": variant,
+                        "title": title_md, "content": content_md, "search_query": search_query, "variant": variant,
                         "csv_header": csv_header, "output": output_data, "indices_to_generate": indices,
-                        "history": [] # Use fresh history for each variant task
+                        "history": [], # Use fresh history for each variant task
+                        "is_real_data": is_real_data, "engine": engine
                     }
                 })
         
         # Execute tasks in parallel and update UI progressively
         for _ in iflatmap_unordered(lambda **kw: kw.pop('func')(**kw), generation_tasks):
-            yield {dataset_dataframe: pd.DataFrame([r for r in output_data if r])} # Update DataFrame display
+            yield {full_table_comp: pd.DataFrame([r for r in output_data if r])} # Update DataFrame display
             
         yield {save_dataset_button: gr.Button(interactive=True)} # Enable save button
         print(f"Full dataset generation complete for {dataset_name}.")
 
     # Save Dataset to Hugging Face Hub
     @save_dataset_button.click(
-        inputs=[dataset_title, dataset_content, search_bar, dataset_dataframe, select_namespace_dropdown, visibility_radio],
+        inputs=[dataset_title_md, dataset_description_md, search_bar, full_table_comp, select_namespace_dropdown, visibility_radio],
         outputs=[save_dataset_button, open_dataset_message]
     )
-    def _save_dataset(title, content, search_query, df, namespace, visibility, oauth_token):
-        dataset_name, tags = title.strip("# ").split("\ntags:", 1)
-        dataset_name, tags = dataset_name.strip(), tags.strip()
+    def _save_dataset(title_md, content_md, search_query, df, namespace, visibility, oauth_token):
+        # Extract dataset name and tags from the markdown title
+        try:
+            dataset_name = title_md.split('\n')[0].strip('# ')
+            tags = title_md.split('tags:', 1)[1].strip()
+        except IndexError:
+            raise gr.Error("Could not parse dataset title.")
         
         token = oauth_token.token if oauth_token else save_dataset_hf_token
         if not token: raise gr.Error("Login required or set SAVE_DATASET_HF_TOKEN.")
@@ -760,7 +884,7 @@ with gr.Blocks(css=css) as demo:
             create_repo(repo_id=repo_id, repo_type="dataset", private=visibility!="public", exist_ok=True, token=token)
             df.to_csv(f"hf://datasets/{repo_id}/data.csv", storage_options={"token": token}, index=False)
             
-            card_content = DATASET_CARD_CONTENT.format(title=title, content=content, url=URL, dataset_url=dataset_url, model_id=model_id, search_query=search_query)
+            card_content = DATASET_CARD_CONTENT.format(title=title_md, content=content_md, url=URL, dataset_url=dataset_url, model_id=model_id, search_query=search_query)
             DatasetCard(card_content).push_to_hub(repo_id=repo_id, repo_type="dataset", token=token)
             
             success_msg = f"# 🎉 Yay! Dataset saved to [{repo_id}](https://huggingface.co/datasets/{repo_id})!\n\n_PS: Check Settings to manage your saved datasets._"
@@ -772,16 +896,22 @@ with gr.Blocks(css=css) as demo:
         finally: yield {save_dataset_button: gr.Button(interactive=True)} # Re-enable button
 
     # Shareable URL Generation
-    @dataset_share_button.click(inputs=[dataset_title, search_bar], outputs=[dataset_share_textbox])
-    def _show_share_url(title, search_query):
-        dataset_name, tags = title.strip("# ").split("\ntags:", 1)
-        dataset_name, tags = dataset_name.strip(), tags.strip()
+    @dataset_share_button.click(inputs=[dataset_title_md, search_bar], outputs=[dataset_share_textbox])
+    def _show_share_url(title_md, search_query):
+        try:
+            dataset_name = title_md.split('\n')[0].strip('# ')
+            tags = title_md.split('tags:', 1)[1].strip()
+        except IndexError:
+            raise gr.Error("Could not parse dataset title.")
+            
         share_url = f"{URL}?q={search_query.replace(' ', '+')}&dataset={dataset_name.replace(' ', '+')}&tags={tags.replace(' ', '+')}"
         return gr.Textbox(share_url, visible=True)
 
     # Settings Toggles
     refinement_mode.change(lambda mode: gr.Group(visible=(mode == "sourced")), outputs=[source_group])
     
+    data_source_toggle.change(lambda value: (gr.State(value), gr.State(value if value else None)), inputs=[data_source_toggle], outputs=[is_real_data_state, current_engine_state])
+    
     @load_source_button.click(inputs=[source_type, source_path], outputs=[source_status])
     def _load_source_data(source_type, source_path):
         if not source_path: raise gr.Error("Source path/URL is required.")
@@ -792,8 +922,50 @@ with gr.Blocks(css=css) as demo:
         except (ConnectionError, ValueError, RuntimeError) as e:
             raise gr.Error(f"Failed to load source: {str(e)}")
 
+    # Engine Settings Modal Logic
+    def _populate_engine_options(selected_engines):
+        engine_options_html = ""
+        for engine in searchEngines:
+            is_checked = "checked" if engine in selected_engines else ""
+            engine_options_html += f"""
+            <div class="flex items-center">
+                <input type="checkbox" id="engine-{engine.replace('.', '_')}" class="engine-checkbox mr-2 h-4 w-4" value="{engine}" {is_checked}>
+                <label for="engine-{engine.replace('.', '_')}" class="cursor-pointer">{engine}</label>
+            </div>
+            """
+        return gr.HTML(engine_options_html)
+
+    def _save_engine_settings(selected_engines_json):
+        selected_engines = json.loads(selected_engines_json)
+        if not selected_engines:
+            gr.Warning("At least one search engine must be selected. Using DuckDuckGo as default.")
+            selected_engines = ["DuckDuckGo.com"]
+        
+        current_engine = selected_engines[0] if selected_engines else None
+        return gr.State(selected_engines), gr.State(current_engine), gr.Info(f"Updated search engines. Using {len(selected_engines)} engines.")
+
+    # Initialize engine options component
+    engine_options_html_comp = _populate_engine_options(selected_engines_state.value)
+    
+    # Update engine options when the modal is opened
+    engine_settings_button.click(lambda: engine_options_html_comp.update(_populate_engine_options(selected_engines_state.value)), outputs=[engine_options_html_comp])
+    
+    select_all_engines_btn.click(lambda: engine_options_html_comp.update(_populate_engine_options(searchEngines)), outputs=[engine_options_html_comp])
+    deselect_all_engines_btn.click(lambda: engine_options_html_comp.update(_populate_engine_options([])), outputs=[engine_options_html_comp])
+
+    save_engines_btn.click(
+        _save_engine_settings,
+        inputs=[gr.JSON(elem_id="engine-options")], # Capture checked engines from modal
+        outputs=[selected_engines_state, current_engine_state, gr.Info()]
+    )
+    
+    engine_settings_button.click(lambda: engine_modal.update(visible=True), outputs=[engine_modal])
+    # Close modal on save or when clicking outside (implicit via Gradio's modal handling)
+
     # Initial App Load Logic
-    @demo.load(outputs=show_dataset_outputs + buttons + [generated_texts_state] + [select_namespace_dropdown, visibility_radio, source_group])
+    @demo.load(outputs=([search_page, dataset_page, dataset_title_md, dataset_description_md, dataset_source_badge, dataset_source_info, dataset_share_textbox, full_dataset_section, save_dataset_button, open_dataset_message, search_bar] + # Outputs for detail page and search bar
+                      buttons + [generated_texts_state] + # Outputs for search results buttons and state
+                      [select_namespace_dropdown, visibility_radio, source_group, data_source_toggle, current_engine_state, selected_engines_state, engine_options_html_comp])) # Outputs for settings
     def _load_app(request: gr.Request, oauth_token: Optional[gr.OAuthToken]):
         # Handle user login and namespace selection
         if oauth_token:
@@ -818,13 +990,38 @@ with gr.Blocks(css=css) as demo:
         # Handle URL parameters for direct search or dataset loading
         query_params = dict(request.query_params)
         if "dataset" in query_params:
-            yield from _show_dataset_details(query_params.get("q", query_params["dataset"]), query_params["dataset"], query_params.get("tags", ""))
+            is_real = query_params.get("engine") is not None
+            engine = query_params.get("engine")
+            yield from _show_dataset_details(query_params.get("q", query_params["dataset"]), query_params["dataset"], query_params.get("tags", ""), is_real, engine)
+            yield {is_real_data_state: is_real, current_engine_state: engine}
         elif "q" in query_params:
             search_query = query_params["q"]
+            is_real = query_params.get("engine") is not None
+            engine = query_params.get("engine")
             yield {search_bar: search_query}
-            yield from _update_search_results(search_query, ()) # Perform initial search
+            yield {is_real_data_state: is_real, current_engine_state: engine}
+            yield from _update_search_results(search_query, (), is_real, engine)
         else:
             yield {search_page: gr.Column(visible=True)} # Show search page by default
 
+        # Initialize with default datasets
+        initial_outputs = {}
+        for i, line in enumerate(default_output):
+            try: dataset_name, tags = line.split(".", 1)[1].strip(" )").split(" (", 1)
+            except ValueError: dataset_name, tags = line.split(".", 1)[1].strip(" )").split(" ", 1)[0], ""
+            
+            initial_outputs[buttons[2 * i]] = gr.Button(dataset_name, elem_classes="topButton")
+            initial_outputs[buttons[2 * i + 1]] = gr.Button(tags, elem_classes="bottomButton")
+            initial_outputs[button_groups[i]] = gr.Group(elem_classes="buttonsGroup")
+        yield initial_outputs
+        yield {generated_texts_state: (landing_page_datasets_generated_text,)}
+        
+        # Initialize engine settings UI
+        yield {
+            data_source_toggle: gr.Checkbox(value=is_real_data_state.value),
+            engine_options_html_comp: _populate_engine_options(selected_engines_state.value)
+        }
+
+
 if __name__ == "__main__":
     demo.launch(share=False, server_name="0.0.0.0")