update data loading (#1)

- update data loading (4a6a9087b9c9112279a8d1d77794d2c80e2f4e70)


Co-authored-by: Kai <[email protected]>

utils/data_loader.py

@@ -5,11 +5,20 @@ import random
 import re
 from .context_processor import process_highlights
 
+# Global data store - loaded once at import time
+_ARENA_DATA = None
+
 def load_arena_data():
     """
     Loads the arena data from the arena_df.csv file in the utils directory.
     Returns the data in a format compatible with the application.
     """
+    global _ARENA_DATA
+    
+    # If data is already loaded, return it
+    if _ARENA_DATA is not None:
+        return _ARENA_DATA
+    
     try:
         # Define the path to the CSV file
         csv_path = os.path.join('utils', 'arena_df.csv')
@@ -17,6 +26,9 @@ def load_arena_data():
         # Read the CSV file
         df = pd.read_csv(csv_path)
         print(f"Loaded arena data with {len(df)} examples")
+        
+        # Store the data globally
+        _ARENA_DATA = df
         return df
     except Exception as e:
         print(f"Error loading arena data: {e}")
@@ -39,7 +51,7 @@ def get_random_example():
     Selects a random example from the loaded arena data.
     Returns the example data in a format compatible with the application.
     """
-    # Load the arena data
+    # Get the globally stored data - won't reload from disk
     df = load_arena_data()
     
     if df.empty:
@@ -89,7 +101,7 @@ def get_random_example():
             
             if isinstance(example['contexts_highlighted'], str):
                 try:
-                    # Try direct parsing, assuming it's a valid JSON array
+                    # Try direct JSON parsing first
                     raw_str = example['contexts_highlighted']
                     
                     # First, manually parse the highlighted contexts using regex