Update app.py

app.py

@@ -1,23 +1,22 @@
 import gradio as gr
 import datetime
 import plotly.graph_objects as go
-from utils import *
+import threading
+import time
+from utils import *  # Ensure get_wandb_runs and get_scores are defined here.
 from chain import get_model_info
 
 # Global history list to record the lowest avg_loss over time.
 loss_history = []
 
-# Set your project name and filter for runs.
+# Set your project name and filter.
 project_name = 'ai-factory-validators'
 filters = {"State": {"$eq": "running"}}
 
 window_size = 32
-# NOTE: Ensure that get_wandb_runs and get_scores are defined in your environment.
-# For example:
-# def get_wandb_runs(project_name, filters):
-#     # Your implementation here
-# def get_scores(ids, runs):
-#     # Your implementation here
+
+# Create a global lock so that update_results runs in mutual exclusion.
+update_lock = threading.Lock()
 
 def moving_average(data, window=10):
     """Compute the moving average of data using a sliding window."""
@@ -31,115 +30,130 @@ def moving_average(data, window=10):
     return ma
 
 def update_results():
-    """
-    Fetches the latest runs and scores using your provided snippet,
-    groups data by competition, highlights the best (lowest avg_loss) entry,
-    records the current lowest avg_loss (skipping duplicate values and limiting the history length),
-    computes the moving average, and returns both an HTML table and a Plotly graph.
-    """
-    # Load new results using your provided snippet.
-    runs = get_wandb_runs(project_name, filters)
-    scores = get_scores(list(range(256)), runs)
-    
-    # Group scores by competition_id and filter columns: uid, avg_loss, win_rate.
-    tables = {}
-    for uid, data in scores.items():
-        comp_id = data.get("competition_id", "unknown")
-        if comp_id not in tables:
-            tables[comp_id] = []
-        tables[comp_id].append({
-            "uid": uid,
-            "avg_loss": data.get("avg_loss"),
-            "win_rate": data.get("win_rate"),
-            "model": get_model_info(uid)
-        })
-    # Sort each table by uid.
-    for comp_id in tables:
-        tables[comp_id] = sorted(tables[comp_id], key=lambda x: x["uid"])
-    
-    # Determine the lowest avg_loss in the current scores.
-    try:
-        min_loss = min(data.get("avg_loss", float("inf")) for data in scores.values())
-    except ValueError:
-        min_loss = None  # if scores is empty
-
-    # Record the current time.
-    now = datetime.datetime.now()
-    # Only append if loss_history is empty or the last value is different from current min_loss.
-    if not loss_history or loss_history[-1][1] != min_loss:
-        loss_history.append((now, min_loss))
-        # Limit loss_history length to 10000
-        if len(loss_history) > 10000:
-            loss_history[:] = loss_history[-10000:]
-    
-    # Create time series data.
-    times = [t[0] for t in loss_history]
-    losses = [t[1] for t in loss_history]
-    
-    # Compute the moving average with window of 10.
-    ma_losses = moving_average(losses, window=window_size)
-    
-    # Create a Plotly line graph with both the raw lowest avg_loss and its moving average.
-    fig = go.Figure()
-    fig.add_trace(go.Scatter(x=times, y=losses, mode='lines+markers', name='Lowest avg_loss'))
-    fig.add_trace(go.Scatter(x=times, y=ma_losses, mode='lines', name=f'Moving Average (window={window_size})'))
-    fig.update_layout(
-        title="Lowest Avg Loss Over Time",
-        xaxis_title="Time",
-        yaxis_title="Lowest Avg Loss",
-        template="plotly_white",
-        height=400
-    )
-    
-    # Build HTML content: one table per competition_id.
-    html_content = "<h1>AI Factory Leaderboard</h1>"
-    for comp_id, rows in tables.items():
-        # Identify best (lowest avg_loss) entry in the current competition.
-        best_loss = min(row["avg_loss"] for row in rows)
-        # For competition 0, mark it as Research Track.
-        comp_title = f"Competition ID: {comp_id}"
-        if comp_id == 0:
-            comp_title += " (Research Track)"
-        html_content += f"<h3>{comp_title}</h3>"
-        html_content += """
-        <table border='1' style='border-collapse: collapse; width: 100%;'>
-            <tr>
-                <th>UID</th>
-                <th>Avg Loss</th>
-                <th>Win Rate</th>
-                <th>Model Name</th>
-            </tr>
-        """
-        for row in rows:
-            # Highlight the row if it has the best avg_loss.
-            style = "background-color: #d4edda;" if row["avg_loss"] == best_loss else ""
-            html_content += f"<tr style='{style}'><td>{row['uid']}</td><td>{row['avg_loss']:.4f}</td><td>{row['win_rate']:.2f}</td><td>{row['model']}</td></tr>"
-        html_content += "</table><br>"
+    """Fetch runs and scores, update the leaderboard and plot, ensuring that only one call runs at a time."""
+    with update_lock:
+        # Load new results using provided snippets.
+        runs = get_wandb_runs(project_name, filters)
+        scores = get_scores(list(range(256)), runs)
+        
+        # Group scores by competition_id with required fields.
+        tables = {}
+        for uid, data in scores.items():
+            comp_id = data.get("competition_id", "unknown")
+            if comp_id not in tables:
+                tables[comp_id] = []
+            tables[comp_id].append({
+                "uid": uid,
+                "avg_loss": data.get("avg_loss"),
+                "win_rate": data.get("win_rate"),
+                "model": get_model_info(uid)
+            })
+        # Sort each table by UID.
+        for comp_id in tables:
+            tables[comp_id] = sorted(tables[comp_id], key=lambda x: x["uid"])
         
-    return html_content, fig
+        # Determine the current lowest avg_loss (for plotting).
+        try:
+            min_loss = min(data.get("avg_loss", float("inf")) for data in scores.values())
+        except ValueError:
+            min_loss = None
+        
+        # Record the current time and update loss_history.
+        now = datetime.datetime.now()
+        if not loss_history or loss_history[-1][1] != min_loss:
+            loss_history.append((now, min_loss))
+            if len(loss_history) > 10000:
+                loss_history[:] = loss_history[-10000:]
+        
+        # Create time series and compute moving average.
+        times = [t[0] for t in loss_history]
+        losses = [t[1] for t in loss_history]
+        ma_losses = moving_average(losses, window=window_size)
+        
+        # Build the Plotly graph.
+        fig = go.Figure()
+        fig.add_trace(go.Scatter(x=times, y=losses, mode='lines+markers', name='Lowest avg_loss'))
+        fig.add_trace(go.Scatter(x=times, y=ma_losses, mode='lines', name=f'Moving Average (window={window_size})'))
+        fig.update_layout(
+            title="Lowest Avg Loss Over Time",
+            xaxis_title="Time",
+            yaxis_title="Lowest Avg Loss",
+            template="plotly_white",
+            height=400
+        )
+        
+        # Build the HTML content for the leaderboard.
+        html_content = "<h1>AI Factory Leaderboard</h1>"
+        for comp_id, rows in tables.items():
+            # Identify the row with the highest win_rate.
+            best_win_rate = max(row["win_rate"] for row in rows)
+            comp_title = f"Competition ID: {comp_id}"
+            if comp_id == 0:
+                comp_title += " (Research Track)"
+            html_content += f"<h3>{comp_title}</h3>"
+            html_content += """
+            <table border='1' style='border-collapse: collapse; width: 100%;'>
+                <tr>
+                    <th>UID</th>
+                    <th>Avg Loss</th>
+                    <th>Win Rate</th>
+                    <th>Model Name</th>
+                </tr>
+            """
+            for row in rows:
+                if row["win_rate"] == best_win_rate:
+                    style = "background-color: #ffeb99;"  # Light yellow background.
+                    crown = " 👑"
+                else:
+                    style = ""
+                    crown = ""
+                html_content += f"<tr style='{style}'><td>{row['uid']}</td><td>{row['avg_loss']:.4f}</td><td>{row['win_rate']:.2f}</td><td>{row['model']}{crown}</td></tr>"
+            html_content += "</table><br>"
+            
+        return html_content, fig
+
+# Global variables to store the latest outputs.
+latest_html = ""
+latest_fig = None
+
+def background_update():
+    """Background thread that runs update_results every 10 seconds and stores its outputs."""
+    global latest_html, latest_fig
+    while True:
+        try:
+            html_content, fig = update_results()
+            latest_html, latest_fig = html_content, fig
+        except Exception as e:
+            print("Error during background update:", e)
+        time.sleep(10)
+
+# Start the background update thread.
+threading.Thread(target=background_update, daemon=True).start()
+
+def get_latest_results():
+    """Return the latest HTML and Plotly graph."""
+    return latest_html, latest_fig
 
 with gr.Blocks() as demo:
-    # Add CSS to hide the refresh button but keep it in the DOM.
+    # Hide any unwanted refresh button in the DOM.
     gr.HTML("<style>#refresh_button {display: none;}</style>")
     
-    # Display the title at the top.
+    # Display the title.
     gr.HTML("<h1 style='text-align:center;'>AI Factory Leaderboard</h1>")
     
-    # Outputs: HTML tables and Plotly graph.
+    # Define the outputs.
     tables_output = gr.HTML()
     graph_output = gr.Plot()
     
-     # Hidden component to trigger periodic updates
+    # A hidden textbox triggers periodic updates every 10 seconds.
     trigger = gr.Textbox(visible=False, every=10)
-
-    # Set up the function to run every 10 seconds
-    trigger.change(fn=update_results, inputs=[], outputs=[tables_output, graph_output])
+    trigger.change(fn=get_latest_results, inputs=[], outputs=[tables_output, graph_output])
     
-    # Provide a manual refresh button as well.
+    # Manual refresh button that also calls update_results.
     manual_refresh = gr.Button("Refresh Now")
     manual_refresh.click(fn=update_results, inputs=[], outputs=[tables_output, graph_output])
     
-    # Load the results once on startup.
-    demo.load(fn=update_results, inputs=[], outputs=[tables_output, graph_output])
+    # Load results once on startup.
+    demo.load(fn=get_latest_results, inputs=[], outputs=[tables_output, graph_output])
     
-demo.launch()
+demo.launch()