better execption handling

app.py

@@ -1,5 +1,3 @@
-# --- START OF FILE app.py ---
-
 import json
 import gradio as gr
 import pandas as pd
@@ -162,7 +160,6 @@ def process_tags_for_series(series_of_tags_values, tqdm_cls=None): # Added tqdm_
 
 
 def load_models_data(force_refresh=False, tqdm_cls=None): # tqdm_cls for Gradio progress
-    # ... (initial part of load_models_data for loading pre-processed parquet is the same) ...
     if tqdm_cls is None: tqdm_cls = tqdm # Default to standard tqdm if None
     overall_start_time = time.time()
     print(f"Gradio load_models_data called with force_refresh={force_refresh}")
@@ -242,11 +239,10 @@ def load_models_data(force_refresh=False, tqdm_cls=None): # tqdm_cls for Gradio
     if output_filesize_col_name in df_raw.columns and pd.api.types.is_numeric_dtype(df_raw[output_filesize_col_name]):
         df[output_filesize_col_name] = pd.to_numeric(df_raw[output_filesize_col_name], errors='coerce').fillna(0.0)
     elif 'safetensors' in df.columns:
-        # Use tqdm_cls for progress tracking if available (Gradio's gr.Progress.tqdm)
         safetensors_iter = df['safetensors']
-        if tqdm_cls and tqdm_cls != tqdm: # Check if it's Gradio's progress tracker
+        if tqdm_cls and tqdm_cls != tqdm: 
              safetensors_iter = tqdm_cls(df['safetensors'], desc="Extracting model sizes (GB)", unit="row")
-        elif tqdm_cls == tqdm: # For direct console tqdm if passed
+        elif tqdm_cls == tqdm: 
             safetensors_iter = tqdm(df['safetensors'], desc="Extracting model sizes (GB)", unit="row", leave=False)
             
         df[output_filesize_col_name] = [extract_model_size(s) for s in safetensors_iter]
@@ -266,7 +262,6 @@ def load_models_data(force_refresh=False, tqdm_cls=None): # tqdm_cls for Gradio
         else: return "Small (<1GB)" # Default
     df['size_category'] = df[output_filesize_col_name].apply(get_size_category_gradio)
 
-    # >>> USE THE CORRECTED process_tags_for_series HERE <<<
     df['tags'] = process_tags_for_series(df['tags'], tqdm_cls=tqdm_cls) 
     
     df['temp_tags_joined'] = df['tags'].apply(
@@ -293,7 +288,6 @@ def load_models_data(force_refresh=False, tqdm_cls=None): # tqdm_cls for Gradio
     df['is_biomed'] = df['has_bio'] | df['has_med']
     df['organization'] = df['id'].apply(extract_org_from_id)
 
-    # Drop safetensors if params was calculated from it, and params didn't pre-exist as numeric
     if 'safetensors' in df.columns and \
        not (output_filesize_col_name in df_raw.columns and pd.api.types.is_numeric_dtype(df_raw[output_filesize_col_name])):
         df = df.drop(columns=['safetensors'], errors='ignore')
@@ -310,7 +304,6 @@ def load_models_data(force_refresh=False, tqdm_cls=None): # tqdm_cls for Gradio
     return df, True, final_msg
 
 
-# ... (make_treemap_data, create_treemap functions remain unchanged) ...
 def make_treemap_data(df, count_by, top_k=25, tag_filter=None, pipeline_filter=None, size_filter=None, skip_orgs=None):
     if df is None or df.empty: return pd.DataFrame()
     filtered_df = df.copy()
@@ -320,19 +313,10 @@ def make_treemap_data(df, count_by, top_k=25, tag_filter=None, pipeline_filter=N
     
     if 'has_robot' in filtered_df.columns:
         initial_robot_count = filtered_df['has_robot'].sum()
-        # print(f"DIAGNOSTIC (make_treemap_data entry): Input df has {initial_robot_count} 'has_robot' models.") # Can be noisy
-    # else:
-        # print("DIAGNOSTIC (make_treemap_data entry): 'has_robot' column NOT in input df.")
-
     if tag_filter and tag_filter in col_map:
         target_col = col_map[tag_filter]
         if target_col in filtered_df.columns:
-            # if tag_filter == "Robotics":
-                # count_before_robot_filter = filtered_df[target_col].sum()
-                # print(f"DIAGNOSTIC (make_treemap_data): Applying 'Robotics' filter. Models with '{target_col}'=True: {count_before_robot_filter}")
             filtered_df = filtered_df[filtered_df[target_col]]
-            # if tag_filter == "Robotics":
-                 # print(f"DIAGNOSTIC (make_treemap_data): After 'Robotics' filter ({target_col}), df rows: {len(filtered_df)}")
         else:
             print(f"Warning: Tag filter column '{col_map[tag_filter]}' not found in DataFrame.")
     if pipeline_filter:
@@ -351,94 +335,111 @@ def make_treemap_data(df, count_by, top_k=25, tag_filter=None, pipeline_filter=N
         else:
             print("Warning: 'organization' column not found for filtering.")
     if filtered_df.empty: return pd.DataFrame()
-    # Ensure count_by column is numeric, coercing if necessary
     if count_by not in filtered_df.columns or not pd.api.types.is_numeric_dtype(filtered_df[count_by]):
-        # print(f"Warning: Column '{count_by}' for treemap values is not numeric or missing. Coercing to numeric, filling NaNs with 0.")
         filtered_df[count_by] = pd.to_numeric(filtered_df.get(count_by), errors="coerce").fillna(0.0)
         
-    org_totals = filtered_df.groupby("organization")[count_by].sum().nlargest(top_k, keep='first') # Default keep='first'
+    org_totals = filtered_df.groupby("organization")[count_by].sum().nlargest(top_k, keep='first') 
     top_orgs_list = org_totals.index.tolist()
     treemap_data = filtered_df[filtered_df["organization"].isin(top_orgs_list)][["id", "organization", count_by]].copy()
-    treemap_data["root"] = "models" # For treemap structure
-    treemap_data[count_by] = pd.to_numeric(treemap_data[count_by], errors="coerce").fillna(0.0) # Ensure numeric again after subsetting
+    treemap_data["root"] = "models" 
+    treemap_data[count_by] = pd.to_numeric(treemap_data[count_by], errors="coerce").fillna(0.0) 
     return treemap_data
 
 def create_treemap(treemap_data, count_by, title=None):
     if treemap_data.empty:
-        fig = px.treemap(names=["No data matches filters"], parents=[""], values=[1]) # Placeholder for empty data
+        fig = px.treemap(names=["No data matches filters"], parents=[""], values=[1]) 
         fig.update_layout(title="No data matches the selected filters", margin=dict(t=50, l=25, r=25, b=25))
         return fig
     fig = px.treemap(
         treemap_data, path=["root", "organization", "id"], values=count_by,
         title=title or f"HuggingFace Models - {count_by.capitalize()} by Organization",
-        color_discrete_sequence=px.colors.qualitative.Plotly # Example color sequence
+        color_discrete_sequence=px.colors.qualitative.Plotly
     )
     fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))
     fig.update_traces(textinfo="label+value+percent root", hovertemplate="<b>%{label}</b><br>%{value:,} " + count_by + "<br>%{percentRoot:.2%} of total<extra></extra>")
     return fig
 
-# --- Gradio UI and Controllers ---
 with gr.Blocks(title="HuggingFace Model Explorer") as demo:
     models_data_state = gr.State(pd.DataFrame())
-    loading_complete_state = gr.State(False) # To control button interactivity
+    loading_complete_state = gr.State(False) 
 
     with gr.Row():
         gr.Markdown("# HuggingFace Models TreeMap Visualization")
     with gr.Row():
-        with gr.Column(scale=1): # Controls column
+        with gr.Column(scale=1): 
             count_by_dropdown = gr.Dropdown(label="Metric", choices=[("Downloads (last 30 days)", "downloads"), ("Downloads (All Time)", "downloadsAllTime"), ("Likes", "likes")], value="downloads")
             filter_choice_radio = gr.Radio(label="Filter Type", choices=["None", "Tag Filter", "Pipeline Filter"], value="None")
             tag_filter_dropdown = gr.Dropdown(label="Select Tag", choices=TAG_FILTER_CHOICES, value=None, visible=False)
             pipeline_filter_dropdown = gr.Dropdown(label="Select Pipeline Tag", choices=PIPELINE_TAGS, value=None, visible=False)
             size_filter_dropdown = gr.Dropdown(label="Model Size Filter", choices=["None"] + list(MODEL_SIZE_RANGES.keys()), value="None")
             top_k_slider = gr.Slider(label="Number of Top Organizations", minimum=5, maximum=50, value=25, step=5)
-            skip_orgs_textbox = gr.Textbox(label="Organizations to Skip (comma-separated)", value="TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski") # Common large orgs
+            skip_orgs_textbox = gr.Textbox(label="Organizations to Skip (comma-separated)", value="TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski") 
             
-            generate_plot_button = gr.Button(value="Generate Plot", variant="primary", interactive=False) # Starts disabled
+            generate_plot_button = gr.Button(value="Generate Plot", variant="primary", interactive=False) 
             refresh_data_button = gr.Button(value="Refresh Data from Hugging Face", variant="secondary")
 
-        with gr.Column(scale=3): # Plot and info column
+        with gr.Column(scale=3): 
             plot_output = gr.Plot()
-            status_message_md = gr.Markdown("Initializing...") # For general status updates
-            data_info_md = gr.Markdown("") # For detailed data stats
+            status_message_md = gr.Markdown("Initializing...") 
+            data_info_md = gr.Markdown("") 
 
-    # Enable generate button only after data is loaded
     def _update_button_interactivity(is_loaded_flag):
         return gr.update(interactive=is_loaded_flag)
     loading_complete_state.change(fn=_update_button_interactivity, inputs=loading_complete_state, outputs=generate_plot_button)
 
-    # Show/hide tag/pipeline filters based on radio choice
     def _toggle_filters_visibility(choice):
         return gr.update(visible=choice == "Tag Filter"), gr.update(visible=choice == "Pipeline Filter")
     filter_choice_radio.change(fn=_toggle_filters_visibility, inputs=filter_choice_radio, outputs=[tag_filter_dropdown, pipeline_filter_dropdown])
 
 
-    def ui_load_data_controller(force_refresh_ui_trigger=False, progress=gr.Progress(track_tqdm=True)): # Gradio progress tracker
+    def ui_load_data_controller(force_refresh_ui_trigger=False, progress=gr.Progress(track_tqdm=True)): 
         print(f"ui_load_data_controller called with force_refresh_ui_trigger={force_refresh_ui_trigger}")
         status_msg_ui = "Loading data..."
         data_info_text = ""
         current_df = pd.DataFrame()
         load_success_flag = False
-        data_as_of_date_display = "N/A"
+        # data_as_of_date_display = "N/A" # Will be set inside the logic
 
         try:
-            # Pass gr.Progress.tqdm to load_models_data if it's a Gradio call
             current_df, load_success_flag, status_msg_from_load = load_models_data(
                 force_refresh=force_refresh_ui_trigger, tqdm_cls=progress.tqdm if progress else tqdm
             )
 
             if load_success_flag:
+                # Default value for data_as_of_date_display
+                data_as_of_date_display = "Pre-processed (date unavailable or invalid)"
+
                 if force_refresh_ui_trigger: # Data was just fetched by Gradio
                     data_as_of_date_display = pd.Timestamp.now(tz='UTC').strftime('%B %d, %Y, %H:%M:%S %Z')
                 # If loaded from pre-processed parquet, check for its timestamp column
-                elif 'data_download_timestamp' in current_df.columns and not current_df.empty and pd.notna(current_df['data_download_timestamp'].iloc[0]):
-                    timestamp_from_parquet = pd.to_datetime(current_df['data_download_timestamp'].iloc[0])
-                    if timestamp_from_parquet.tzinfo is None: # If no timezone, assume UTC from preprocessor
-                        timestamp_from_parquet = timestamp_from_parquet.tz_localize('UTC')
-                    data_as_of_date_display = timestamp_from_parquet.strftime('%B %d, %Y, %H:%M:%S %Z')
-                else: # Pre-processed data but no timestamp column or it's NaT
-                    data_as_of_date_display = "Pre-processed (date unavailable)"
-
+                elif 'data_download_timestamp' in current_df.columns and not current_df.empty:
+                    try:
+                        # Step 1: Safely get the value from the DataFrame's first row for the timestamp column
+                        raw_val_from_df = current_df['data_download_timestamp'].iloc[0]
+
+                        # Step 2: Process if raw_val_from_df is a list/array 
+                        scalar_timestamp_val = None
+                        if isinstance(raw_val_from_df, (list, tuple, np.ndarray)):
+                            if len(raw_val_from_df) > 0:
+                                scalar_timestamp_val = raw_val_from_df[0] 
+                        else:
+                            scalar_timestamp_val = raw_val_from_df 
+
+                        # Step 3: Check for NA and convert the scalar value to datetime
+                        if pd.notna(scalar_timestamp_val):
+                            dt_obj = pd.to_datetime(scalar_timestamp_val)
+                            if pd.notna(dt_obj): 
+                                if dt_obj.tzinfo is None: 
+                                    dt_obj = dt_obj.tz_localize('UTC')
+                                data_as_of_date_display = dt_obj.strftime('%B %d, %Y, %H:%M:%S %Z')
+                            
+                    except IndexError:
+                        print(f"DEBUG: IndexError encountered while processing 'data_download_timestamp'. DF empty: {current_df.empty}")
+                        if 'data_download_timestamp' in current_df.columns and not current_df.empty:
+                             print(f"DEBUG: Head of 'data_download_timestamp': {str(current_df['data_download_timestamp'].head(1))}") # Ensure string conversion for print
+                    except Exception as e_ts_proc:
+                        print(f"Error processing 'data_download_timestamp' from parquet: {e_ts_proc}")
+                
                 # Build data info string
                 size_dist_lines = []
                 if 'size_category' in current_df.columns:
@@ -466,13 +467,13 @@ with gr.Blocks(title="HuggingFace Model Explorer") as demo:
                 status_msg_ui = "Data loaded successfully. Ready to generate plot."
             else: # load_success_flag is False
                 data_info_text = f"### Data Load Failed\n- {status_msg_from_load}"
-                status_msg_ui = status_msg_from_load # Pass error message from load_models_data
+                status_msg_ui = status_msg_from_load 
         
         except Exception as e:
             status_msg_ui = f"An unexpected error occurred in ui_load_data_controller: {str(e)}"
             data_info_text = f"### Critical Error\n- {status_msg_ui}"
-            print(f"Critical error in ui_load_data_controller: {e}")
-            load_success_flag = False # Ensure this is false on error
+            print(f"Critical error in ui_load_data_controller: {e}") # This is the original error print
+            load_success_flag = False 
 
         return current_df, load_success_flag, data_info_text, status_msg_ui
 
@@ -481,18 +482,14 @@ with gr.Blocks(title="HuggingFace Model Explorer") as demo:
         if df_current_models is None or df_current_models.empty:
             empty_fig = create_treemap(pd.DataFrame(), metric_choice, "Error: Model Data Not Loaded")
             error_msg = "Model data is not loaded or is empty. Please load or refresh data first."
-            gr.Warning(error_msg) # Display a Gradio warning
+            gr.Warning(error_msg) 
             return empty_fig, error_msg
 
         tag_to_use = tag_choice if filter_type == "Tag Filter" else None
         pipeline_to_use = pipeline_choice if filter_type == "Pipeline Filter" else None
-        size_to_use = size_choice if size_choice != "None" else None # Handle "None" string
+        size_to_use = size_choice if size_choice != "None" else None 
         orgs_to_skip = [org.strip() for org in skip_orgs_input.split(',') if org.strip()] if skip_orgs_input else []
 
-        # if 'has_robot' in df_current_models.columns:
-        #     robot_count_before_treemap = df_current_models['has_robot'].sum()
-            # print(f"DIAGNOSTIC (ui_generate_plot_controller): df_current_models entering make_treemap_data has {robot_count_before_treemap} 'has_robot' models.")
-
         treemap_df = make_treemap_data(df_current_models, metric_choice, k_orgs, tag_to_use, pipeline_to_use, size_to_use, orgs_to_skip)
         
         title_labels = {"downloads": "Downloads (last 30 days)", "downloadsAllTime": "Downloads (All Time)", "likes": "Likes"}
@@ -502,33 +499,29 @@ with gr.Blocks(title="HuggingFace Model Explorer") as demo:
         if treemap_df.empty:
             plot_stats_md = "No data matches the selected filters. Try adjusting your filters."
         else:
-            total_items_in_plot = len(treemap_df['id'].unique()) # Count unique models in plot
-            total_value_in_plot = treemap_df[metric_choice].sum() # Sum of metric in plot
+            total_items_in_plot = len(treemap_df['id'].unique()) 
+            total_value_in_plot = treemap_df[metric_choice].sum() 
             plot_stats_md = (f"## Plot Statistics\n- **Models shown**: {total_items_in_plot:,}\n- **Total {metric_choice}**: {int(total_value_in_plot):,}")
         
         return plotly_fig, plot_stats_md
 
-    # --- Event Handlers ---
-    # Initial data load on app start
     demo.load(
         fn=lambda progress=gr.Progress(track_tqdm=True): ui_load_data_controller(force_refresh_ui_trigger=False, progress=progress),
-        inputs=[], # No inputs for initial load
+        inputs=[], 
         outputs=[models_data_state, loading_complete_state, data_info_md, status_message_md]
     )
 
-    # Refresh data button
     refresh_data_button.click(
         fn=lambda progress=gr.Progress(track_tqdm=True): ui_load_data_controller(force_refresh_ui_trigger=True, progress=progress),
         inputs=[], 
         outputs=[models_data_state, loading_complete_state, data_info_md, status_message_md]
     )
 
-    # Generate plot button
     generate_plot_button.click(
         fn=ui_generate_plot_controller,
         inputs=[count_by_dropdown, filter_choice_radio, tag_filter_dropdown, pipeline_filter_dropdown,
                 size_filter_dropdown, top_k_slider, skip_orgs_textbox, models_data_state],
-        outputs=[plot_output, status_message_md] # Update plot and status message
+        outputs=[plot_output, status_message_md] 
     )
 
 if __name__ == "__main__":
@@ -537,5 +530,4 @@ if __name__ == "__main__":
         print("It is highly recommended to run the preprocessing script (preprocess.py) first.")
     else:
         print(f"Found pre-processed data file: '{PROCESSED_PARQUET_FILE_PATH}'.")
-    demo.launch()
-# --- END OF FILE app.py ---
+    demo.launch()