Infini-d-set

Runtime error

App Files Files Community

acecalisto3 commited on 1 day ago

Commit

5e4f1ce

verified ·

1 Parent(s): afcc0ef

Update app.py

Browse files

Files changed (1) hide show

app.py +935 -169

app.py CHANGED Viewed

@@ -244,7 +244,6 @@ html = """
     </style>
 </head>
 <body class="bg-white dark:bg-gray-900 text-gray-800 dark:text-gray-200 min-h-screen">
-    <!-- Dark mode detection -->
     <script>
         if (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) {
             document.documentElement.classList.add('dark');
@@ -259,15 +258,12 @@ html = """
     </script>
     <div class="container mx-auto px-4 py-8">
-        <!-- Header -->
         <header class="text-center mb-8">
             <h1 class="text-3xl font-bold mb-2">🤗 Infinite Dataset Hub ♾️</h1>
             <p class="text-lg text-gray-600 dark:text-gray-400">Generate datasets from AI and real-world data sources</p>
         </header>
-        <!-- Main Content -->
         <main>
-            <!-- Search Section -->
             <div id="search-page" class="mb-8">
                 <div class="max-w-3xl mx-auto">
                     <div class="mb-4">
@@ -300,7 +296,6 @@ html = """
                         </div>
                     </div>
-                    <!-- Search Engine Selection Modal -->
                     <div id="engine-modal" class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 hidden">
                         <div class="bg-white dark:bg-gray-800 rounded-lg p-6 max-w-lg w-full max-h-[80vh] overflow-y-auto">
                             <div class="flex justify-between items-center mb-4">
@@ -317,8 +312,7 @@ html = """
                             </p>
                             <div id="engine-options" class="space-y-2 mb-6">
-                                <!-- Engine options will be dynamically inserted here -->
-                            </div>
                             <div class="flex justify-between">
                                 <button id="select-all-engines" class="text-primary hover:underline">Select All</button>
@@ -334,8 +328,7 @@ html = """
                     </div>
                     <div id="dataset-results" class="grid grid-cols-1 md:grid-cols-2 gap-4 mt-6">
-                        <!-- Dataset cards will be dynamically inserted here -->
-                    </div>
                     <div id="load-more-container" class="text-center mt-6 hidden">
                         <button id="load-more-button" class="bg-gray-200 dark:bg-gray-700 px-6 py-3 rounded-lg hover:bg-gray-300 dark:hover:bg-gray-600 transition">
@@ -345,7 +338,6 @@ html = """
                 </div>
             </div>
-            <!-- Dataset Detail Page -->
             <div id="dataset-page" class="hidden max-w-4xl mx-auto">
                 <button id="back-button" class="flex items-center text-primary mb-4 hover:underline">
                     <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 mr-1" viewBox="0 0 20 20" fill="currentColor">
@@ -423,7 +415,6 @@ html = """
             </div>
         </main>
-        <!-- Footer -->
         <footer class="mt-12 text-center text-sm text-gray-600 dark:text-gray-400">
             <p>Powered by Claude-3.7-Sonnet • Datasets generated from real sources and AI</p>
         </footer>
@@ -485,115 +476,943 @@ html = """
         const selectAllEngines = document.getElementById('select-all-engines');
         const deselectAllEngines = document.getElementById('deselect-all-engines');
-        // Event Listeners
-        document.addEventListener('DOMContentLoaded', () => {
-            searchButton.addEventListener('click', performSearch);
-            searchInput.addEventListener('keypress', (e) => {
-                if (e.key === 'Enter') performSearch();
-            });
-            loadMoreButton.addEventListener('click', loadMoreDatasets);
-            backButton.addEventListener('click', showSearchPage);
-            generateFullButton.addEventListener('click', generateFullDataset);
-            downloadCsvButton.addEventListener('click', () => downloadData('csv'));
-            downloadJsonButton.addEventListener('click', () => downloadData('json'));
-            downloadParquetButton.addEventListener('click', () => downloadData('parquet'));
-            dataSourceToggle.addEventListener('change', toggleDataSource);
-            engineSettingsButton.addEventListener('click', showEngineModal);
-            closeModalButton.addEventListener('click', hideEngineModal);
-            saveEnginesButton.addEventListener('click', saveEngineSettings);
-            selectAllEngines.addEventListener('click', () => toggleAllEngines(true));
-            deselectAllEngines.addEventListener('click', () => toggleAllEngines(false));
-            // Initialize engine options
-            populateEngineOptions();
-            // Show initial placeholder datasets
-            showPlaceholderDatasets();
-        });
-        // Search Engine Settings
-        function populateEngineOptions() {
-            engineOptions.innerHTML = '';
-            searchEngines.forEach(engine => {
-                const isChecked = selectedEngines.includes(engine);
-                const optionDiv = document.createElement('div');
-                optionDiv.className = 'flex items-center';
-                optionDiv.innerHTML = `
-                    <input type="checkbox" id="engine-${engine}" class="engine-checkbox mr-2 h-4 w-4"
-                           value="${engine}" ${isChecked ? 'checked' : ''}>
-                    <label for="engine-${engine}" class="cursor-pointer">${engine}</label>
-                `;
-                engineOptions.appendChild(optionDiv);
-            });
         }
-        function showEngineModal() {
-            engineModal.classList.remove('hidden');
         }
-        function hideEngineModal() {
-            engineModal.classList.add('hidden');
         }
-        function saveEngineSettings() {
-            const checkboxes = document.querySelectorAll('.engine-checkbox:checked');
-            selectedEngines = Array.from(checkboxes).map(cb => cb.value);
-            if (selectedEngines.length === 0) {
-                // Ensure at least one engine is selected
-                selectedEngines = ["DuckDuckGo.com"];
-                document.getElementById(`engine-DuckDuckGo.com`).checked = true;
-                showNotification("At least one search engine must be selected. Using DuckDuckGo as default.");
-            }
-            hideEngineModal();
-            showNotification(`Updated search engine settings. Using ${selectedEngines.length} engines.`);
         }
-        function toggleAllEngines(select) {
-            const checkboxes = document.querySelectorAll('.engine-checkbox');
-            checkboxes.forEach(cb => {
-                cb.checked = select;
-            });
         }
-        // Toggle data source between real and AI
-        function toggleDataSource() {
-            useRealData = dataSourceToggle.checked;
-            dataSourceText.textContent = useRealData ? "Using: Real + AI Data" : "Using: AI Data Only";
-            // Show or hide engine settings button
-            engineSettingsButton.style.display = useRealData ? "flex" : "none";
-            showNotification(`Switched to ${useRealData ? "combined real and synthetic" : "synthetic-only"} data mode`);
         }
-        // Search functionality
-        function performSearch() {
-            const query = searchInput.value.trim();
-            if (!query) return;
-            currentSearchQuery = query;
-            currentPage = 1;
-            currentDatasets = [];
-            resultsContainer.innerHTML = '';
-            showLoadingSkeletons();
-            if (useRealData) {
-                // Use real data from search engines + AI
-                searchWithRealData(query);
-            } else {
-                // Use only AI-generated data
-                searchWithAIData(query);
-            }
         }
         function searchWithRealData(query) {
             // Randomly select a search engine from the user's selected engines
             currentEngine = selectedEngines[Math.floor(Math.random() * selectedEngines.length)];
@@ -706,65 +1525,6 @@ html = """
             }
         }
-        function parseDatasetResults(content) {
-            const lines = content.split('\n');
-            const datasets = [];
-            lines.forEach(line => {
-                // Match lines that start with a number followed by a period
-                const match = line.match(/^\s*\d+\.\s+(.+?)\s+\((.+?)\)/);
-                if (match) {
-                    const name = match[1].trim();
-                    const tags = match[2].split(',').map(tag => tag.trim());
-                    datasets.push({ name, tags });
-                }
-            });
-            return datasets;
-        }
-        function displayDatasets(datasets) {
-            datasets.forEach(dataset => {
-                const card = document.createElement('div');
-                card.className = 'dataset-card bg-white dark:bg-gray-800 rounded-lg p-4 border border-gray-200 dark:border-gray-700 cursor-pointer relative';
-                const tagsHtml = dataset.tags.map(tag =>
-                    `<span class="inline-block bg-gray-100 dark:bg-gray-700 text-gray-800 dark:text-gray-300 text-xs px-2 py-1 rounded mr-1 mb-1">${tag}</span>`
-                ).join('');
-                // Add a badge for real data
-                let badgeHtml = '';
-                if (dataset.isReal) {
-                    badgeHtml = `<span class="engine-badge" title="Data from ${dataset.engine}">${dataset.engine.split('.')[0]}</span>`;
-                }
-                card.innerHTML = `
-                    ${badgeHtml}
-                    <h3 class="text-lg font-semibold mb-2">${dataset.name}</h3>
-                    <div class="flex flex-wrap mt-2">${tagsHtml}</div>
-                `;
-                card.addEventListener('click', () => showDatasetDetails(dataset));
-                resultsContainer.appendChild(card);
-            });
-        }
-        function showLoadingSkeletons() {
-            for (let i = 0; i < 4; i++) {
-                const skeleton = document.createElement('div');
-                skeleton.className = 'bg-white dark:bg-gray-800 rounded-lg p-4 border border-gray-200 dark:border-gray-700';
-                skeleton.innerHTML = `
-                    <div class="shimmer h-6 w-3/4 mb-2"></div>
-                    <div class="flex flex-wrap mt-2">
-                        <div class="shimmer h-6 w-16 rounded mr-1 mb-1"></div>
-                        <div class="shimmer h-6 w-20 rounded mr-1 mb-1"></div>
-                        <div class="shimmer h-6 w-24 rounded mr-1 mb-1"></div>
-                    </div>
-                `;
-                resultsContainer.appendChild(skeleton);
-            }
-        }
         function loadMoreDatasets() {
             currentPage++;
@@ -1230,6 +1990,8 @@ html = """
                     );
                 } catch (err) {
                     showError("Error sending message: " + err);
                 }
             };
@@ -2041,13 +2803,17 @@ with gr.Blocks(css=css) as demo:
                 engine_settings_button = gr.Button("Configure Search Engines", icon="https://img.icons8.com/ios-filled/50/000000/settings--v1.png", size="sm")
                 # Engine Selection Modal
-                with gr.Modal("Search Engine Settings", id="engine-modal") as engine_modal:
-                    gr.Markdown("Select which search engines to use for real data retrieval. A diverse selection improves results.")
-                    engine_options_html_comp = gr.HTML(elem_id="engine-options")
-                    with gr.Row():
-                        select_all_engines_btn = gr.Button("Select All")
-                        deselect_all_engines_btn = gr.Button("Deselect All")
-                    save_engines_btn = gr.Button("Save Settings", variant="primary")
     # --- Dataset Detail Page UI ---
     with gr.Column(visible=False, elem_id="dataset-page") as dataset_page:
@@ -2094,7 +2860,7 @@ with gr.Blocks(css=css) as demo:
                 if "I'm sorry" in line or "policy" in line: raise gr.Error("Inappropriate content detected.")
                 if generated_count >= MAX_NB_ITEMS_PER_GENERATION_CALL: break
-                match = re.match(r"^\s*\d+\.\s+(.+?)\s+$$(.+?)$$", line) # Parse line format
                 if match:
                     dataset_name, tags = match.groups()
                     dataset_name, tags = dataset_name.strip(), tags.strip()
@@ -2359,9 +3125,9 @@ with gr.Blocks(css=css) as demo:
         outputs=[selected_engines_state, current_engine_state, gr.Info()]
     )
-    engine_settings_button.click(lambda: engine_modal.update(visible=True), outputs=[engine_modal])
-    # Close modal on save or when clicking outside (implicit via Gradio's modal handling)
     # Initial App Load Logic
     @demo.load(outputs=([search_page, dataset_page, dataset_title_md, dataset_description_md, dataset_source_badge, dataset_source_info, dataset_share_textbox, full_dataset_section, save_dataset_button, open_dataset_message, search_bar] + # Outputs for detail page and search bar
                       buttons + [generated_texts_state] + # Outputs for search results buttons and state
@@ -2424,4 +3190,4 @@ with gr.Blocks(css=css) as demo:
 if __name__ == "__main__":
-    demo.launch(share=False, server_name="0.0.0.0")

     </style>
 </head>
 <body class="bg-white dark:bg-gray-900 text-gray-800 dark:text-gray-200 min-h-screen">
     <script>
         if (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) {
             document.documentElement.classList.add('dark');
     </script>
     <div class="container mx-auto px-4 py-8">
         <header class="text-center mb-8">
             <h1 class="text-3xl font-bold mb-2">🤗 Infinite Dataset Hub ♾️</h1>
             <p class="text-lg text-gray-600 dark:text-gray-400">Generate datasets from AI and real-world data sources</p>
         </header>
         <main>
             <div id="search-page" class="mb-8">
                 <div class="max-w-3xl mx-auto">
                     <div class="mb-4">
                         </div>
                     </div>
                     <div id="engine-modal" class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50 hidden">
                         <div class="bg-white dark:bg-gray-800 rounded-lg p-6 max-w-lg w-full max-h-[80vh] overflow-y-auto">
                             <div class="flex justify-between items-center mb-4">
                             </p>
                             <div id="engine-options" class="space-y-2 mb-6">
+                                </div>
                             <div class="flex justify-between">
                                 <button id="select-all-engines" class="text-primary hover:underline">Select All</button>
                     </div>
                     <div id="dataset-results" class="grid grid-cols-1 md:grid-cols-2 gap-4 mt-6">
+                        </div>
                     <div id="load-more-container" class="text-center mt-6 hidden">
                         <button id="load-more-button" class="bg-gray-200 dark:bg-gray-700 px-6 py-3 rounded-lg hover:bg-gray-300 dark:hover:bg-gray-600 transition">
                 </div>
             </div>
             <div id="dataset-page" class="hidden max-w-4xl mx-auto">
                 <button id="back-button" class="flex items-center text-primary mb-4 hover:underline">
                     <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 mr-1" viewBox="0 0 20 20" fill="currentColor">
             </div>
         </main>
         <footer class="mt-12 text-center text-sm text-gray-600 dark:text-gray-400">
             <p>Powered by Claude-3.7-Sonnet • Datasets generated from real sources and AI</p>
         </footer>
         const selectAllEngines = document.getElementById('select-all-engines');
         const deselectAllEngines = document.getElementById('deselect-all-engines');
+        // Event Listeners
+        document.addEventListener('DOMContentLoaded', () => {
+            searchButton.addEventListener('click', performSearch);
+            searchInput.addEventListener('keypress', (e) => {
+                if (e.key === 'Enter') performSearch();
+            });
+            loadMoreButton.addEventListener('click', loadMoreDatasets);
+            backButton.addEventListener('click', showSearchPage);
+            generateFullButton.addEventListener('click', generateFullDataset);
+            downloadCsvButton.addEventListener('click', () => downloadData('csv'));
+            downloadJsonButton.addEventListener('click', () => downloadData('json'));
+            downloadParquetButton.addEventListener('click', () => downloadData('parquet'));
+            dataSourceToggle.addEventListener('change', toggleDataSource);
+            engineSettingsButton.addEventListener('click', showEngineModal);
+            closeModalButton.addEventListener('click', hideEngineModal);
+            saveEnginesButton.addEventListener('click', saveEngineSettings);
+            selectAllEngines.addEventListener('click', () => toggleAllEngines(true));
+            deselectAllEngines.addEventListener('click', () => toggleAllEngines(false));
+            // Initialize engine options
+            populateEngineOptions();
+            // Show initial placeholder datasets
+            showPlaceholderDatasets();
+        });
+        // Search Engine Settings
+        function populateEngineOptions() {
+            engineOptions.innerHTML = '';
+            searchEngines.forEach(engine => {
+                const isChecked = selectedEngines.includes(engine);
+                const optionDiv = document.createElement('div');
+                optionDiv.className = 'flex items-center';
+                optionDiv.innerHTML = `
+                    <input type="checkbox" id="engine-${engine}" class="engine-checkbox mr-2 h-4 w-4"
+                           value="${engine}" ${isChecked ? 'checked' : ''}>
+                    <label for="engine-${engine}" class="cursor-pointer">${engine}</label>
+                `;
+                engineOptions.appendChild(optionDiv);
+            });
+        }
+        function showEngineModal() {
+            engineModal.classList.remove('hidden');
+        }
+        function hideEngineModal() {
+            engineModal.classList.add('hidden');
+        }
+        function saveEngineSettings() {
+            const checkboxes = document.querySelectorAll('.engine-checkbox:checked');
+            selectedEngines = Array.from(checkboxes).map(cb => cb.value);
+            if (selectedEngines.length === 0) {
+                // Ensure at least one engine is selected
+                selectedEngines = ["DuckDuckGo.com"];
+                document.getElementById(`engine-DuckDuckGo.com`).checked = true;
+                showNotification("At least one search engine must be selected. Using DuckDuckGo as default.");
+            }
+            hideEngineModal();
+            showNotification(`Updated search engine settings. Using ${selectedEngines.length} engines.`);
+        }
+        function toggleAllEngines(select) {
+            const checkboxes = document.querySelectorAll('.engine-checkbox');
+            checkboxes.forEach(cb => {
+                cb.checked = select;
+            });
+        }
+        // Toggle data source between real and AI
+        function toggleDataSource() {
+            useRealData = dataSourceToggle.checked;
+            dataSourceText.textContent = useRealData ? "Using: Real + AI Data" : "Using: AI Data Only";
+            // Show or hide engine settings button
+            engineSettingsButton.style.display = useRealData ? "flex" : "none";
+            showNotification(`Switched to ${useRealData ? "combined real and synthetic" : "synthetic-only"} data mode`);
+        }
+        // Search functionality
+        function performSearch() {
+            const query = searchInput.value.trim();
+            if (!query) return;
+            currentSearchQuery = query;
+            currentPage = 1;
+            currentDatasets = [];
+            resultsContainer.innerHTML = '';
+            showLoadingSkeletons();
+            if (useRealData) {
+                // Use real data from search engines + AI
+                searchWithRealData(query);
+            } else {
+                // Use only AI-generated data
+                searchWithAIData(query);
+            }
+        }
+        function searchWithRealData(query) {
+            // Randomly select a search engine from the user's selected engines
+            currentEngine = selectedEngines[Math.floor(Math.random() * selectedEngines.length)];
+            // Register handler for dataset names based on real search results
+            window.Poe.registerHandler("real-search-handler", (result) => {
+                if (result.status === "error") {
+                    showError("Error querying search engines");
+                    return;
+                }
+                const message = result.responses[0];
+                if (message.status === "complete") {
+                    // Parse the dataset names and tags from the response
+                    const datasets = parseDatasetResults(message.content);
+                    datasets.forEach(dataset => {
+                        dataset.isReal = true;
+                        dataset.engine = currentEngine;
+                    });
+                    currentDatasets = datasets;
+                    // Display the datasets
+                    resultsContainer.innerHTML = '';
+                    displayDatasets(datasets);
+                    // Show load more button if we have results
+                    if (datasets.length > 0) {
+                        loadMoreContainer.classList.remove('hidden');
+                    }
+                }
+            });
+            try {
+                window.Poe.sendUserMessage(
+                    `@Claude-3.7-Sonnet You are a data specialist who can transform real search results into structured datasets.
+                    A user is searching for data about: "${query}"
+                    Imagine you've queried ${currentEngine} and received real search results. Create a list of 10 specific datasets that could be created from these search results.
+                    For each dataset:
+                    1. Give it a clear, specific name related to the search topic
+                    2. Include 3-5 relevant tags in parentheses, with one tag specifying the ML task type (classification, regression, clustering, etc.)
+                    Format each dataset as:
+                    1. DatasetName (tag1, tag2, ml_task_tag)
+                    Make these datasets sound like real collections that could be created from ${currentEngine} search results on "${query}".`,
+                    {
+                        handler: "real-search-handler",
+                        stream: false,
+                        openChat: false
+                    }
+                );
+            } catch (err) {
+                showError("Error sending message: " + err);
+                // Fall back to AI data
+                searchWithAIData(query);
+            }
+        }
+        function searchWithAIData(query) {
+            // Register handler for AI-generated dataset names
+            window.Poe.registerHandler("dataset-search-handler", (result) => {
+                if (result.status === "error") {
+                    showError("Error generating datasets");
+                    return;
+                }
+                const message = result.responses[0];
+                if (message.status === "complete") {
+                    // Parse the dataset names and tags from the response
+                    const datasets = parseDatasetResults(message.content);
+                    datasets.forEach(dataset => {
+                        dataset.isReal = false;
+                    });
+                    currentDatasets = datasets;
+                    // Display the datasets
+                    resultsContainer.innerHTML = '';
+                    displayDatasets(datasets);
+                    // Show load more button if we have results
+                    if (datasets.length > 0) {
+                        loadMoreContainer.classList.remove('hidden');
+                    }
+                }
+            });
+            try {
+                window.Poe.sendUserMessage(
+                    `@Claude-3.7-Sonnet A Machine Learning Practioner is looking for a dataset that matches '${query}'.
+                    Generate a list of ${MAX_DATASETS_PER_PAGE} names of quality datasets that don't exist but sound plausible and would
+                    be helpful. Feel free to reuse words from the query '${query}' to name the datasets.
+                    Every dataset should be about '${query}' and have descriptive tags/keywords including the ML task name associated with the dataset (classification, regression, anomaly detection, etc.). Use the following format:
+                    1. DatasetName1 (tag1, tag2, tag3)
+                    2. DatasetName2 (tag1, tag2, tag3)`,
+                    {
+                        handler: "dataset-search-handler",
+                        stream: false,
+                        openChat: false
+                    }
+                );
+            } catch (err) {
+                showError("Error sending message: " + err);
+            }
+        }
+        function loadMoreDatasets() {
+            currentPage++;
+            // Use the same data source (real or AI) as the initial search
+            if (useRealData) {
+                loadMoreRealDatasets();
+            } else {
+                loadMoreAIDatasets();
+            }
+        }
+        function loadMoreRealDatasets() {
+            // Rotate to a different search engine for variety
+            const previousEngine = currentEngine;
+            while (currentEngine === previousEngine && selectedEngines.length > 1) {
+                currentEngine = selectedEngines[Math.floor(Math.random() * selectedEngines.length)];
+            }
+            // Register handler for more datasets
+            window.Poe.registerHandler("more-real-datasets-handler", (result) => {
+                if (result.status === "error") {
+                    showError("Error generating more datasets");
+                    return;
+                }
+                const message = result.responses[0];
+                if (message.status === "complete") {
+                    // Parse the dataset names and tags from the response
+                    const datasets = parseDatasetResults(message.content);
+                    datasets.forEach(dataset => {
+                        dataset.isReal = true;
+                        dataset.engine = currentEngine;
+                    });
+                    currentDatasets = [...currentDatasets, ...datasets];
+                    // Display the datasets
+                    displayDatasets(datasets);
+                }
+            });
+            try {
+                window.Poe.sendUserMessage(
+                    `@Claude-3.7-Sonnet You're a data specialist who can transform real search results into structured datasets.
+                    Continue our previous search for data about: "${currentSearchQuery}"
+                    Now let's use a different search engine: ${currentEngine}
+                    Create 10 more specific datasets that could be created from these search results. Make sure these are different from the previous datasets.
+                    Use the same format:
+                    1. DatasetName (tag1, tag2, ml_task_tag)
+                    Make these datasets sound like real collections that could be created from ${currentEngine} search results on "${currentSearchQuery}".`,
+                    {
+                        handler: "more-real-datasets-handler",
+                        stream: false,
+                        openChat: false
+                    }
+                );
+            } catch (err) {
+                showError("Error sending message: " + err);
+                // Fall back to AI data
+                loadMoreAIDatasets();
+            }
+        }
+        function loadMoreAIDatasets() {
+            // Register handler for more AI datasets
+            window.Poe.registerHandler("more-datasets-handler", (result) => {
+                if (result.status === "error") {
+                    showError("Error generating more datasets");
+                    return;
+                }
+                const message = result.responses[0];
+                if (message.status === "complete") {
+                    // Parse the dataset names and tags from the response
+                    const datasets = parseDatasetResults(message.content);
+                    datasets.forEach(dataset => {
+                        dataset.isReal = false;
+                    });
+                    currentDatasets = [...currentDatasets, ...datasets];
+                    // Display the datasets
+                    displayDatasets(datasets);
+                }
+            });
+            try {
+                window.Poe.sendUserMessage(
+                    `@Claude-3.7-Sonnet Please generate ${MAX_DATASETS_PER_PAGE} more dataset names about '${currentSearchQuery}'. Use the same format as before:
+                    1. DatasetName1 (tag1, tag2, tag3)
+                    Make sure these are completely different from previous suggestions.`,
+                    {
+                        handler: "more-datasets-handler",
+                        stream: false,
+                        openChat: false
+                    }
+                );
+            } catch (err) {
+                showError("Error sending message: " + err);
+            }
+        }
+        function showDatasetDetails(dataset) {
+            currentDataset = dataset;
+            searchPage.classList.add('hidden');
+            datasetPage.classList.remove('hidden');
+            // Update UI with dataset info
+            datasetTitle.textContent = dataset.name;
+            datasetTags.innerHTML = dataset.tags.map(tag =>
+                `<span class="inline-block bg-gray-100 dark:bg-gray-700 text-gray-800 dark:text-gray-300 text-xs px-2 py-1 rounded mr-1 mb-1">${tag}</span>`
+            ).join('');
+            // Update source badge
+            if (dataset.isReal) {
+                dataSourceBadge.textContent = "Real Data";
+                dataSourceBadge.className = "px-3 py-1 rounded-full text-xs font-medium bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200";
+                sourceDetails.innerHTML = `This dataset is based on real information queried from <strong>${dataset.engine}</strong> for the search term "<strong>${currentSearchQuery}</strong>". The data has been structured for machine learning use.`;
+            } else {
+                dataSourceBadge.textContent = "AI-Generated";
+                dataSourceBadge.className = "px-3 py-1 rounded-full text-xs font-medium bg-purple-100 text-purple-800 dark:bg-purple-900 dark:text-purple-200";
+                sourceDetails.innerHTML = `This is an AI-generated dataset created using Claude-3.7-Sonnet. The content is synthetic and designed to represent plausible data related to "${currentSearchQuery}".`;
+            }
+            // Clear previous content
+            datasetDescription.innerHTML = '<div class="shimmer h-4 w-full mb-2"></div>'.repeat(3);
+            previewTable.innerHTML = '';
+            fullDatasetSection.classList.add('hidden');
+            generateStatus.classList.add('hidden');
+            generateFullButton.disabled = false;
+            // Reset full dataset
+            fullDatasetRows = [];
+            // Generate dataset preview - different approach for real vs AI data
+            if (dataset.isReal) {
+                generateRealDatasetPreview(dataset);
+            } else {
+                generateAIDatasetPreview(dataset);
+            }
+            // Scroll to top
+            window.scrollTo(0, 0);
+        }
+        function generateRealDatasetPreview(dataset) {
+            window.Poe.registerHandler("real-preview-handler", (result) => {
+                if (result.status === "error") {
+                    datasetDescription.innerHTML = '<p class="text-red-500">Error generating dataset preview</p>';
+                    return;
+                }
+                const message = result.responses[0];
+                if (message.status === "complete") {
+                    const content = message.content;
+                    // Extract description and CSV
+                    const parts = content.split('**CSV Content Preview:**');
+                    let description = "";
+                    let csvContent = "";
+                    if (parts.length > 1) {
+                        description = parts[0].replace('**Dataset Description:**', '').trim();
+                        csvContent = parts[1].trim();
+                        // Clean up CSV content (remove markdown code block markers)
+                        csvContent = csvContent.replace(/```csv\n|```\n|```/g, '').trim();
+                    } else {
+                        description = "No description available";
+                        csvContent = content;
+                    }
+                    // Display description
+                    datasetDescription.innerHTML = marked.parse(description);
+                    // Parse and display CSV preview
+                    try {
+                        const results = Papa.parse(csvContent, {
+                            header: true,
+                            skipEmptyLines: true
+                        });
+                        if (results.data && results.data.length > 0) {
+                            // Create table from CSV data
+                            createTable(previewTable, results.data, results.meta.fields);
+                        } else {
+                            previewTable.innerHTML = '<p class="p-4 text-red-500">No preview data available</p>';
+                        }
+                    } catch (err) {
+                        previewTable.innerHTML = `<p class="p-4 text-red-500">Error parsing CSV: ${err.message}</p>`;
+                    }
+                }
+            });
+            try {
+                const tagsStr = dataset.tags.join(', ');
+                window.Poe.sendUserMessage(
+                    `@Claude-3.7-Sonnet You're a specialist in converting web search results into structured data.
+                    Based on search results from ${dataset.engine} about "${currentSearchQuery}",
+                    create a preview of the dataset "${dataset.name}" with tags "${tagsStr}".
+                    First, write a detailed description of what this dataset contains, its structure, and how it was constructed from web search results.
+                    Then, generate a realistic 5-row CSV preview that resembles data you might get if you scraped and structured real results from ${dataset.engine}.
+                    Format your response with:
+                    **Dataset Description:** [detailed description]
+                    **CSV Content Preview:**
+                    \`\`\`csv
+                    [CSV header and 5 rows of realistic data]
+                    \`\`\`
+                    Include relevant columns for the dataset type, with proper labels/categories where appropriate. The data should look like it came from real sources.`,
+                    {
+                        handler: "real-preview-handler",
+                        stream: false,
+                        openChat: false
+                    }
+                );
+            } catch (err) {
+                datasetDescription.innerHTML = `<p class="text-red-500">Error: ${err.message}</p>`;
+            }
+        }
+        function generateAIDatasetPreview(dataset) {
+            window.Poe.registerHandler("dataset-preview-handler", (result) => {
+                if (result.status === "error") {
+                    datasetDescription.innerHTML = '<p class="text-red-500">Error generating dataset preview</p>';
+                    return;
+                }
+                const message = result.responses[0];
+                if (message.status === "complete") {
+                    const content = message.content;
+                    // Extract description and CSV
+                    const parts = content.split('**CSV Content Preview:**');
+                    let description = "";
+                    let csvContent = "";
+                    if (parts.length > 1) {
+                        description = parts[0].replace('**Dataset Description:**', '').trim();
+                        csvContent = parts[1].trim();
+                        // Clean up CSV content (remove markdown code block markers)
+                        csvContent = csvContent.replace(/```csv\n|```\n|```/g, '').trim();
+                    } else {
+                        description = "No description available";
+                        csvContent = content;
+                    }
+                    // Display description
+                    datasetDescription.innerHTML = marked.parse(description);
+                    // Parse and display CSV preview
+                    try {
+                        const results = Papa.parse(csvContent, {
+                            header: true,
+                            skipEmptyLines: true
+                        });
+                        if (results.data && results.data.length > 0) {
+                            // Create table from CSV data
+                            createTable(previewTable, results.data, results.meta.fields);
+                        } else {
+                            previewTable.innerHTML = '<p class="p-4 text-red-500">No preview data available</p>';
+                        }
+                    } catch (err) {
+                        previewTable.innerHTML = `<p class="p-4 text-red-500">Error parsing CSV: ${err.message}</p>`;
+                    }
+                }
+            });
+            try {
+                const tagsStr = dataset.tags.join(', ');
+                window.Poe.sendUserMessage(
+                    `@Claude-3.7-Sonnet An ML practitioner is looking for a dataset CSV after the query '${currentSearchQuery}'.
+                    Generate the first 5 rows of a plausible and quality CSV for the dataset '${dataset.name}'.
+                    You can get inspiration from related keywords '${tagsStr}' but most importantly the dataset should correspond to the query '${currentSearchQuery}'.
+                    Focus on quality text content and use a 'label' or 'labels' column if it makes sense (invent labels, avoid reusing the keywords, be accurate while labelling texts).
+                    Reply using a short description of the dataset with title **Dataset Description:** followed by the CSV content in a code block and with title **CSV Content Preview:**`,
+                    {
+                        handler: "dataset-preview-handler",
+                        stream: false,
+                        openChat: false
+                    }
+                );
+            } catch (err) {
+                datasetDescription.innerHTML = `<p class="text-red-500">Error: ${err.message}</p>`;
+            }
+        }
+        function createTable(container, data, headers) {
+            container.innerHTML = '';
+            const table = document.createElement('table');
+            table.className = 'w-full';
+            // Create header
+            const thead = document.createElement('thead');
+            const headerRow = document.createElement('tr');
+            headers.forEach(header => {
+                const th = document.createElement('th');
+                th.textContent = header;
+                headerRow.appendChild(th);
+            });
+            thead.appendChild(headerRow);
+            table.appendChild(thead);
+            // Create body
+            const tbody = document.createElement('tbody');
+            data.forEach(row => {
+                const tr = document.createElement('tr');
+                headers.forEach(header => {
+                    const td = document.createElement('td');
+                    td.textContent = row[header] || '';
+                    tr.appendChild(td);
+                });
+                tbody.appendChild(tr);
+            });
+            table.appendChild(tbody);
+            container.appendChild(table);
+        }
+        function generateFullDataset() {
+            // Disable button and show status
+            generateFullButton.disabled = true;
+            generateStatus.classList.remove('hidden');
+            rowsCount.textContent = '0';
+            progressBar.style.width = '0%';
+            // Set up variables for tracking generation
+            let csvHeader = '';
+            const targetRows = MAX_FULL_DATASET_ROWS;
+            let currentRows = 0;
+            fullDatasetRows = [];
+            // Get the CSV header from the preview table
+            const previewHeaders = Array.from(previewTable.querySelectorAll('thead th')).map(th => th.textContent);
+            csvHeader = previewHeaders.join(',');
+            // Add initial rows from preview
+            const previewRows = Array.from(previewTable.querySelectorAll('tbody tr')).map(tr => {
+                const row = {};
+                Array.from(tr.querySelectorAll('td')).forEach((td, index) => {
+                    row[previewHeaders[index]] = td.textContent;
+                });
+                return row;
+            });
+            fullDatasetRows = [...previewRows];
+            currentRows = previewRows.length;
+            updateGenerationProgress(currentRows, targetRows);
+            // Choose generation method based on dataset type
+            if (currentDataset.isReal) {
+                generateFullRealDataset(previewHeaders, csvHeader, currentRows, targetRows);
+            } else {
+                generateFullAIDataset(previewHeaders, csvHeader, currentRows, targetRows);
+            }
+        }
+        function generateFullRealDataset(previewHeaders, csvHeader, currentRows, targetRows) {
+            // Function to generate more rows in batches from "real" search results
+            const generateBatch = (batchIndex) => {
+                const batchSize = 15; // Larger batches for efficiency
+                const startRow = currentRows + batchIndex * batchSize;
+                if (startRow >= targetRows) {
+                    // We've reached the target, show the full dataset
+                    showFullDataset();
+                    return;
+                }
+                window.Poe.registerHandler(`real-batch-${batchIndex}-handler`, (result) => {
+                    if (result.status === "error") {
+                        showError("Error generating dataset rows");
+                        return;
+                    }
+                    const message = result.responses[0];
+                    if (message.status === "complete") {
+                        const content = message.content;
+                        // Extract CSV content (remove markdown code block markers)
+                        let csvContent = content.replace(/```csv\n|```\n|```/g, '').trim();
+                        // If there are multiple code blocks, try to find one with CSV data
+                        if (csvContent.includes('```')) {
+                            const codeBlocks = content.match(/```(?:csv)?\n([\s\S]*?)```/g) || [];
+                            if (codeBlocks.length > 0) {
+                                csvContent = codeBlocks[0].replace(/```(?:csv)?\n|```/g, '').trim();
+                            }
+                        }
+                        try {
+                            // Parse the CSV
+                            const results = Papa.parse(csvContent, {
+                                header: true,
+                                skipEmptyLines: true
+                            });
+                            if (results.data && results.data.length > 0) {
+                                // Add the new rows
+                                fullDatasetRows = [...fullDatasetRows, ...results.data];
+                                currentRows += results.data.length;
+                                // Update progress
+                                updateGenerationProgress(currentRows, targetRows);
+                                // Generate next batch
+                                generateBatch(batchIndex + 1);
+                            } else {
+                                // Try again with a different prompt
+                                generateBatch(batchIndex);
+                            }
+                        } catch (err) {
+                            console.error("Error parsing CSV:", err);
+                            // Try again
+                            generateBatch(batchIndex);
+                        }
+                    }
+                });
+                try {
+                    // For variation, rotate through engines for each batch
+                    const engineForBatch = selectedEngines[batchIndex % selectedEngines.length] || currentDataset.engine;
+                    window.Poe.sendUserMessage(
+                        `@Claude-3.7-Sonnet You're expanding a dataset based on search results from ${engineForBatch}.
+                        For the dataset "${currentDataset.name}" about "${currentSearchQuery}", please generate ${batchSize} more rows of data.
+                        Use this exact CSV header: ${csvHeader}
+                        The data should look realistic, as if it came from actual ${engineForBatch} search results for "${currentSearchQuery}".
+                        Include appropriate values for each field, maintaining the same patterns and types as seen in the existing data.
+                        Only include the CSV data in your response (header + ${batchSize} rows), no explanations or additional text.`,
+                        {
+                            handler: `real-batch-${batchIndex}-handler`,
+                            stream: false,
+                            openChat: false
+                        }
+                    );
+                } catch (err) {
+                    showError("Error sending message: " + err);
+                    // Fall back to AI data
+                    generateBatch(batchIndex + 1);
+                }
+            };
+            // Start generating batches
+            generateBatch(0);
+        }
+        function generateFullAIDataset(previewHeaders, csvHeader, currentRows, targetRows) {
+            // Function to generate more rows in batches from AI
+            const generateBatch = (batchIndex) => {
+                const batchSize = 10;
+                const startRow = currentRows + batchIndex * batchSize;
+                if (startRow >= targetRows) {
+                    // We've reached the target, show the full dataset
+                    showFullDataset();
+                    return;
+                }
+                window.Poe.registerHandler(`batch-${batchIndex}-handler`, (result) => {
+                    if (result.status === "error") {
+                        showError("Error generating dataset rows");
+                        return;
+                    }
+                    const message = result.responses[0];
+                    if (message.status === "complete") {
+                        const content = message.content;
+                        // Extract CSV content (remove markdown code block markers)
+                        let csvContent = content.replace(/```csv\n|```\n|```/g, '').trim();
+                        // If there are multiple code blocks, try to find one with CSV data
+                        if (csvContent.includes('```')) {
+                            const codeBlocks = content.match(/```(?:csv)?\n([\s\S]*?)```/g) || [];
+                            if (codeBlocks.length > 0) {
+                                csvContent = codeBlocks[0].replace(/```(?:csv)?\n|```/g, '').trim();
+                            }
+                        }
+                        try {
+                            // Parse the CSV
+                            const results = Papa.parse(csvContent, {
+                                header: true,
+                                skipEmptyLines: true
+                            });
+                            if (results.data && results.data.length > 0) {
+                                // Add the new rows
+                                fullDatasetRows = [...fullDatasetRows, ...results.data];
+                                currentRows += results.data.length;
+                                // Update progress
+                                updateGenerationProgress(currentRows, targetRows);
+                                // Generate next batch
+                                generateBatch(batchIndex + 1);
+                            } else {
+                                // Try again with a different prompt
+                                generateBatch(batchIndex);
+                            }
+                        } catch (err) {
+                            console.error("Error parsing CSV:", err);
+                            // Try again
+                            generateBatch(batchIndex);
+                        }
+                    }
+                });
+                try {
+                    const tagsStr = currentDataset.tags.join(', ');
+                    window.Poe.sendUserMessage(
+                        `@Claude-3.7-Sonnet For the dataset '${currentDataset.name}' about '${currentSearchQuery}' with tags '${tagsStr}',
+                        please generate ${batchSize} more sample rows in CSV format. Use the same CSV header: ${csvHeader}
+                        Only include the CSV data in your response, no explanations or additional text.`,
+                        {
+                            handler: `batch-${batchIndex}-handler`,
+                            stream: false,
+                            openChat: false
+                        }
+                    );
+                } catch (err) {
+                    showError("Error sending message: " + err);
+                }
+            };
+            // Start generating batches
+            generateBatch(0);
+        }
+        function updateGenerationProgress(current, total) {
+            rowsCount.textContent = current;
+            const percentage = Math.min(100, Math.floor((current / total) * 100));
+            progressBar.style.width = `${percentage}%`;
+        }
+        function showFullDataset() {
+            // Hide generation status
+            generateStatus.classList.add('hidden');
+            // Show full dataset section
+            fullDatasetSection.classList.remove('hidden');
+            // Get headers from the data
+            const headers = Object.keys(fullDatasetRows[0] || {});
+            // Create and display the table
+            createTable(fullTable, fullDatasetRows.slice(0, 10), headers);
+            // Add a note about showing limited rows
+            const note = document.createElement('p');
+            note.className = 'text-sm text-gray-600 dark:text-gray-400 mt-2';
+            note.textContent = `Showing 10 of ${fullDatasetRows.length} rows. Use the download buttons to get the complete dataset.`;
+            fullTable.appendChild(note);
         }
+        function downloadData(format) {
+            if (fullDatasetRows.length === 0) return;
+            const filename = `${currentDataset.name.replace(/\s+/g, '_')}_dataset`;
+            switch(format) {
+                case 'csv':
+                    downloadCsv(filename);
+                    break;
+                case 'json':
+                    downloadJson(filename);
+                    break;
+                case 'parquet':
+                    // Show a notification that this format is simulated
+                    showNotification("Parquet format download simulated - actual conversion would require a server component");
+                    downloadJson(filename + "_parquet_simulated");
+                    break;
+            }
         }
+        function downloadCsv(filename) {
+            // Convert data to CSV
+            const csv = Papa.unparse(fullDatasetRows);
+            // Create a blob and download link
+            const blob = new Blob([csv], { type: 'text/csv' });
+            const url = URL.createObjectURL(blob);
+            const a = document.createElement('a');
+            a.href = url;
+            a.download = `${filename}.csv`;
+            document.body.appendChild(a);
+            a.click();
+            // Clean up
+            setTimeout(() => {
+                document.body.removeChild(a);
+                URL.revokeObjectURL(url);
+            }, 100);
         }
+        function downloadJson(filename) {
+            // Convert data to JSON
+            const json = JSON.stringify(fullDatasetRows, null, 2);
+            // Create a blob and download link
+            const blob = new Blob([json], { type: 'application/json' });
+            const url = URL.createObjectURL(blob);
+            const a = document.createElement('a');
+            a.href = url;
+            a.download = `${filename}.json`;
+            document.body.appendChild(a);
+            a.click();
+            // Clean up
+            setTimeout(() => {
+                document.body.removeChild(a);
+                URL.revokeObjectURL(url);
+            }, 100);
         }
+        function showSearchPage() {
+            searchPage.classList.remove('hidden');
+            datasetPage.classList.add('hidden');
         }
+        function showError(message) {
+            console.error(message);
+            showNotification(message, true);
         }
+        function showNotification(message, isError = false) {
+            const notification = document.createElement('div');
+            notification.className = `fixed bottom-4 right-4 px-6 py-3 rounded-lg shadow-lg ${
+                isError
+                    ? 'bg-red-500 text-white'
+                    : 'bg-green-500 text-white'
+            } z-50 transition-opacity duration-300`;
+            notification.textContent = message;
+            document.body.appendChild(notification);
+            setTimeout(() => {
+                notification.style.opacity = '0';
+                setTimeout(() => {
+                    document.body.removeChild(notification);
+                }, 300);
+            }, 3000);
         }
+        function showPlaceholderDatasets() {
+            const placeholders = [
+                {
+                    name: "NewsEventsPredict",
+                    tags: ["classification", "media", "trend"],
+                    isReal: true,
+                    engine: "AlltheInternet.com"
+                },
+                {
+                    name: "FinancialForecast",
+                    tags: ["economy", "stocks", "regression"],
+                    isReal: false
+                },
+                {
+                    name: "HealthMonitor",
+                    tags: ["science", "real-time", "anomaly detection"],
+                    isReal: true,
+                    engine: "DuckDuckGo.com"
+                },
+                {
+                    name: "SportsAnalysis",
+                    tags: ["classification", "performance", "player tracking"],
+                    isReal: false
+                },
+                {
+                    name: "RetailSalesAnalyzer",
+                    tags: ["consumer behavior", "sales trend", "segmentation"],
+                    isReal: true,
+                    engine: "Bing.com"
+                },
+                {
+                    name: "SocialMediaSentiment",
+                    tags: ["text classification", "opinion mining", "NLP"],
+                    isReal: false
+                }
+            ];
+            currentDatasets = placeholders;
+            displayDatasets(placeholders);
+            loadMoreContainer.classList.remove('hidden');
+        }
         function searchWithRealData(query) {
             // Randomly select a search engine from the user's selected engines
             currentEngine = selectedEngines[Math.floor(Math.random() * selectedEngines.length)];
             }
         }
         function loadMoreDatasets() {
             currentPage++;
                     );
                 } catch (err) {
                     showError("Error sending message: " + err);
+                    // Fall back to AI data
+                    generateBatch(batchIndex + 1);
                 }
             };
                 engine_settings_button = gr.Button("Configure Search Engines", icon="https://img.icons8.com/ios-filled/50/000000/settings--v1.png", size="sm")
                 # Engine Selection Modal
+                with gr.Column(visible=False, id="engine-modal-container") as engine_modal:
+                    with gr.Blocks():
+                        with gr.Row():
+                            gr.Markdown("### Search Engine Settings", elem_classes="text-xl font-bold")
+                            close_modal_btn = gr.Button("❌")
+                        gr.Markdown("Select which search engines to use for real data retrieval. A diverse selection improves results.")
+                        engine_options_html_comp = gr.HTML(elem_id="engine-options")
+                        with gr.Row():
+                            select_all_engines_btn = gr.Button("Select All")
+                            deselect_all_engines_btn = gr.Button("Deselect All")
+                        save_engines_btn = gr.Button("Save Settings", variant="primary")
     # --- Dataset Detail Page UI ---
     with gr.Column(visible=False, elem_id="dataset-page") as dataset_page:
                 if "I'm sorry" in line or "policy" in line: raise gr.Error("Inappropriate content detected.")
                 if generated_count >= MAX_NB_ITEMS_PER_GENERATION_CALL: break
+                match = re.match(r"^\s*\d+\.\s+(.+?)\s+\((.+?)\)", line) # Parse line format
                 if match:
                     dataset_name, tags = match.groups()
                     dataset_name, tags = dataset_name.strip(), tags.strip()
         outputs=[selected_engines_state, current_engine_state, gr.Info()]
     )
+    engine_settings_button.click(lambda: gr.Column(visible=True), outputs=[engine_modal])
+    close_modal_btn.click(lambda: gr.Column(visible=False), outputs=[engine_modal])
     # Initial App Load Logic
     @demo.load(outputs=([search_page, dataset_page, dataset_title_md, dataset_description_md, dataset_source_badge, dataset_source_info, dataset_share_textbox, full_dataset_section, save_dataset_button, open_dataset_message, search_bar] + # Outputs for detail page and search bar
                       buttons + [generated_texts_state] + # Outputs for search results buttons and state
 if __name__ == "__main__":
+    demo.launch(share=False, server_name="0.0.0.0")