Update app.py

app.py

@@ -2,6 +2,7 @@ import io
 import os
 import re
 import time
+from typing import Any, Dict, List
 from itertools import islice
 from functools import partial
 from multiprocessing.pool import ThreadPool
@@ -79,17 +80,11 @@ tags:
 - infinite-dataset-hub
 - synthetic
 ---
-
 {title}
-
 _Note: This is an AI-generated dataset so its content may be inaccurate or false_
-
 {content}
-
 **Source of the data:**
-
 The dataset was generated using the [Infinite Dataset Hub]({url}) and {model_id} using the query '{search_query}':
-
 - **Dataset Generation Page**: {dataset_url}
 - **Model**: https://huggingface.co/{model_id}
 - **More Datasets**: https://huggingface.co/datasets?other=infinite-dataset-hub
@@ -99,7 +94,6 @@ css = """
 a {
     color: var(--body-text-color);
 }
-
 .datasetButton {
     justify-content: start;
     justify-content: left;
@@ -149,7 +143,6 @@ a {
 .insivibleButtonGroup {
     display: none;
 }
-
 @keyframes placeHolderShimmer{
     0%{
         background-position: -468px 0
@@ -377,7 +370,16 @@ with gr.Blocks(css=css) as demo:
                             try:
                                 generated_df = parse_csv_df(generated_csv.strip(), csv_header=csv_header)
                                 if len(generated_df) > nb_samples:
-                                    output[indices_to_generate[nb_samples]] = generated_df.iloc[-1].to_dict()
+                                    # Convert latest record to dict and refine it
+                                    record = generated_df.iloc[-1].to_dict()
+                                    refined_record = refine_data_generic([record])[0]
+                                    
+                                    # Add quality flags if any
+                                    flags = detect_anomalies(refined_record)
+                                    if flags:
+                                        refined_record['_quality_flags'] = flags
+                                    
+                                    output[indices_to_generate[nb_samples]] = refined_record
                                     nb_samples += 1
                                     yield 1
                             except Exception:
@@ -409,6 +411,48 @@ with gr.Blocks(css=css) as demo:
         ]
 
 
+    def refine_data_generic(dataset: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Universally refine any dataset.
+        Works on list of dicts. Detects field types and applies general cleanup.
+        """
+        def normalize_value(value):
+            if isinstance(value, str):
+                # Trim, title-case common descriptors, remove duplicate whitespace
+                value = re.sub(r'\s+', ' ', value.strip())
+                value = value.replace('_', ' ')
+                if any(k in value.lower() for k in ['color', 'material', 'type', 'status']):
+                    value = value.title()
+            return value
+
+        def clean_record(record):
+            cleaned = {}
+            for key, value in record.items():
+                # Normalize key and value
+                clean_key = key.strip().lower().replace(" ", "_")
+                if isinstance(value, list):
+                    cleaned[clean_key] = [normalize_value(v) for v in value]
+                elif isinstance(value, dict):
+                    cleaned[clean_key] = clean_record(value)
+                else:
+                    cleaned[clean_key] = normalize_value(value)
+            return cleaned
+
+        return [clean_record(entry) for entry in dataset]
+
+    def detect_anomalies(record: Dict[str, Any]) -> List[str]:
+        """
+        Detect potential anomalies in a record.
+        Returns a list of flags for any detected issues.
+        """
+        flags = []
+        for k, v in record.items():
+            if isinstance(v, str) and len(v) > 300:
+                flags.append(f"{k} looks too verbose.")
+            if isinstance(v, str) and v.lower() in ['n/a', 'none', 'undefined']:
+                flags.append(f"{k} is missing or undefined.")
+        return flags
+
     def parse_preview_df(content: str) -> tuple[str, pd.DataFrame]:
         _in_csv = False
         csv = "\n".join(
@@ -418,7 +462,27 @@ with gr.Blocks(css=css) as demo:
         )
         if not csv:
             raise gr.Error("Failed to parse CSV Preview")
-        return csv.split("\n")[0], parse_csv_df(csv)
+        
+        # Get header and parse initial DataFrame
+        csv_header = csv.split("\n")[0]
+        df = parse_csv_df(csv)
+        
+        # Convert DataFrame to list of dicts for refinement
+        records = df.to_dict('records')
+        
+        # Apply refinement
+        refined_records = refine_data_generic(records)
+        
+        # Add quality flags
+        for record in refined_records:
+            flags = detect_anomalies(record)
+            if flags:
+                record['_quality_flags'] = flags
+        
+        # Convert back to DataFrame
+        refined_df = pd.DataFrame(refined_records)
+        
+        return csv_header, refined_df
 
 
     def parse_csv_df(csv: str, csv_header: Optional[str] = None) -> pd.DataFrame: