Update app.py

app.py

@@ -10,12 +10,14 @@ import time
 import random
 from typing import List, Tuple, Optional
 import io
+import concurrent.futures
+from tqdm import tqdm
 
 # NuExtract API configuration
 API_URL = "https://api-inference.huggingface.co/models/numind/NuExtract-1.5"
 headers = {"Authorization": f"Bearer {os.environ.get('HF_TOKEN', '')}"}
 
-# Geocoding Service
+# Geocoding Service with improved performance
 class GeocodingService:
     def __init__(self, user_agent: str = None, timeout: int = 10, rate_limit: float = 1.1):
         if user_agent is None:
@@ -27,6 +29,7 @@ class GeocodingService:
         )
         self.rate_limit = rate_limit
         self.last_request = 0
+        self.cache = {}  # Simple in-memory cache for geocoding results
 
     def _rate_limit_wait(self):
         current_time = time.time()
@@ -36,29 +39,72 @@ class GeocodingService:
         self.last_request = time.time()
 
     def geocode_location(self, location: str, max_retries: int = 3) -> Optional[Tuple[float, float]]:
+        # Check cache first
+        if location in self.cache:
+            return self.cache[location]
+            
         for attempt in range(max_retries):
             try:
                 self._rate_limit_wait()
                 location_data = self.geolocator.geocode(location)
                 if location_data:
-                    return (location_data.latitude, location_data.longitude)
+                    # Store in cache and return
+                    self.cache[location] = (location_data.latitude, location_data.longitude)
+                    return self.cache[location]
+                # Cache None results too
+                self.cache[location] = None
                 return None
             except (GeocoderTimedOut, GeocoderServiceError) as e:
                 if attempt == max_retries - 1:
                     print(f"Failed to geocode '{location}' after {max_retries} attempts: {e}")
+                    self.cache[location] = None
                     return None
                 time.sleep(2 ** attempt)  # Exponential backoff
             except Exception as e:
                 print(f"Error geocoding '{location}': {e}")
+                self.cache[location] = None
                 return None
         return None
 
-    def process_locations(self, locations: str) -> List[Optional[Tuple[float, float]]]:
+    def process_locations(self, locations: str, progress_callback=None) -> List[Optional[Tuple[float, float]]]:
         if pd.isna(locations) or not locations:
             return []
 
         location_list = [loc.strip() for loc in locations.split(',')]
-        return [self.geocode_location(loc) for loc in location_list]
+        
+        # Process locations in parallel with a limited number of workers
+        return self.process_locations_parallel(location_list, progress_callback)
+        
+    def process_locations_parallel(self, location_list, progress_callback=None, max_workers=4) -> List[Optional[Tuple[float, float]]]:
+        """Process locations in parallel with progress tracking"""
+        results = [None] * len(location_list)
+        
+        # Use a ThreadPoolExecutor for parallel processing
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # Submit all tasks
+            future_to_index = {executor.submit(self.geocode_location, loc): i 
+                              for i, loc in enumerate(location_list)}
+            
+            # Process as they complete with progress updates
+            total = len(future_to_index)
+            completed = 0
+            
+            for future in concurrent.futures.as_completed(future_to_index):
+                index = future_to_index[future]
+                try:
+                    results[index] = future.result()
+                except Exception as e:
+                    print(f"Error processing location: {e}")
+                    results[index] = None
+                
+                # Update progress
+                completed += 1
+                if progress_callback:
+                    progress_callback(completed, total)
+                else:
+                    print(f"Geocoded {completed}/{total} locations")
+                    
+        return results
 
 # Mapping Functions
 def create_location_map(df: pd.DataFrame,
@@ -105,13 +151,17 @@ def create_location_map(df: pd.DataFrame,
 
     return m
 
-# Processing Functions
-def process_excel(file, places_column):
+# Processing Functions with progress updates
+def process_excel(file, places_column, progress=None):
     # Check if file is None
     if file is None:
         return None, "No file uploaded", None
     
     try:
+        # Update progress
+        if progress:
+            progress(0.1, "Reading Excel file...")
+        
         # Handle various file object types that Gradio might provide
         if hasattr(file, 'name'):
             # Gradio file object
@@ -126,11 +176,36 @@ def process_excel(file, places_column):
         if places_column not in df.columns:
             return None, f"Column '{places_column}' not found in the Excel file. Available columns: {', '.join(df.columns)}", None
         
+        if progress:
+            progress(0.2, "Initializing geocoding...")
+        
         # Initialize the geocoding service
         geocoder = GeocodingService(user_agent="gradio_map_visualization_app")
         
-        # Process locations and add coordinates
-        df['coordinates'] = df[places_column].apply(geocoder.process_locations)
+        # Function to update progress during geocoding
+        def geocoding_progress(completed, total):
+            if progress:
+                # Scale progress between 20% and 80%
+                progress_value = 0.2 + (0.6 * (completed / total))
+                progress(progress_value, f"Geocoding {completed}/{total} locations...")
+        
+        # Process locations and add coordinates with progress tracking
+        print("Starting geocoding process...")
+        
+        # Process each row with progress updates
+        coordinates_list = []
+        total_rows = len(df)
+        
+        for idx, row in df.iterrows():
+            locations = row[places_column]
+            coords = geocoder.process_locations(locations, geocoding_progress)
+            coordinates_list.append(coords)
+            print(f"Processed row {idx+1}/{total_rows}")
+            
+        df['coordinates'] = coordinates_list
+        
+        if progress:
+            progress(0.8, "Creating map...")
         
         # Create the map
         map_obj = create_location_map(df, coordinates_col='coordinates', places_col=places_column)
@@ -140,6 +215,9 @@ def process_excel(file, places_column):
         map_obj.save(temp_map_path)
         
         # Save the processed DataFrame to Excel
+        if progress:
+            progress(0.9, "Saving results...")
+            
         processed_file_path = "processed_data.xlsx"
         df.to_excel(processed_file_path, index=False)
         
@@ -152,8 +230,13 @@ def process_excel(file, places_column):
         stats += f"Successfully geocoded: {successful_geocodes}\n"
         stats += f"Failed to geocode: {failed_geocodes}"
         
+        if progress:
+            progress(1.0, "Processing complete!")
+            
         return temp_map_path, stats, processed_file_path
     except Exception as e:
+        if progress:
+            progress(1.0, f"Error: {str(e)}")
         return None, f"Error processing file: {str(e)}", None
 
 # NuExtract Functions
@@ -251,16 +334,21 @@ with gr.Blocks() as demo:
                     process_btn = gr.Button("Process and Map", variant="primary")
                 
                 with gr.Column():
+                    progress_bar = gr.Progress()
                     map_output = gr.HTML(label="Map Visualization")
                     stats_output = gr.Textbox(label="Statistics", lines=3)
                     processed_file = gr.File(label="Processed Data", visible=True, interactive=False)
             
-            def process_and_map(file, column):
+            def process_and_map(file, column, progress=gr.Progress()):
                 if file is None:
                     return None, "Please upload an Excel file", None
                 
                 try:
-                    map_path, stats, processed_path = process_excel(file, column)
+                    # Initialize progress
+                    progress(0, "Starting process...")
+                    
+                    # Process the file with progress updates
+                    map_path, stats, processed_path = process_excel(file, column, progress)
                     
                     if map_path and processed_path:
                         with open(map_path, "r") as f: