updated api to work for new dashboard

api.py

@@ -2,20 +2,20 @@
 import atexit
 import datetime
 
-from apscheduler.schedulers.background import BackgroundScheduler
-from fastapi import FastAPI
-import utils
 import pandas as pd
 import uvicorn
+from apscheduler.schedulers.background import BackgroundScheduler
+from fastapi import FastAPI
 
-from classes import Productivity, ProductivityData, Last24hProductivityData, Throughput
-
+import utils
+from classes import Metagraph, Productivity, Throughput
 
 # Global variables (saves time on loading data)
 state_vars = None
 reload_timestamp = datetime.datetime.now().strftime('%D %T')
 
 data_all = None
+data_30d = None
 data_24h = None
 
 app = FastAPI()
@@ -24,12 +24,14 @@ def load_data():
     """
     Reload the state variables
     """
-    global data_all, data_24h, reload_timestamp
+    global data_all, data_30d ,data_24h, reload_timestamp
     
-    utils.fetch_new_runs()
+    # utils.fetch_new_runs()
     
     data_all = utils.preload_data()
     
+    data_30d = data_all[(pd.Timestamp.now() -  data_all['updated_at'].apply(lambda x: pd.Timestamp(x)) < pd.Timedelta('30 days'))]
+    
     data_24h = data_all[(pd.Timestamp.now() -  data_all['updated_at'].apply(lambda x: pd.Timestamp(x)) < pd.Timedelta('1 days'))]
     reload_timestamp = datetime.datetime.now().strftime('%D %T')
 
@@ -61,13 +63,31 @@ def productivity_metrics():
     Get the productivity metrics
     """
 
-    # Unpack the metrics using the correct keys
-    result = utils.get_productivity(df_all=data_all, df_24h=data_24h)
-    all_time = ProductivityData(**result['all_time'])
-    last_24h = Last24hProductivityData(**result['last_24h'])
+    result = utils.get_productivity(df_all=data_all, df_24h=data_24h, df_30d=data_30d)
 
-    return {"all_time": all_time, "last_24h": last_24h}
 
+    return result
+@app.get("/metagraph", response_model=Metagraph)
+def get_metagraph():
+    """
+    Get the metagraph
+    """
+    
+    df_m = utils.get_metagraph()
+    df_miners = df_m.sort_values('I', ascending=False).reset_index()
+    incentives = df_miners['I'].astype(float).values
+    emissions = df_miners['E'].astype(float).values
+    identities = df_miners['identity']
+    hotkeys = df_miners['hotkey']
+    coldkeys = df_miners['coldkey']
+    trusts = df_miners['trust'].astype(float).values
+    results = {'incentives': incentives,
+               'emissions': emissions,
+               'identities': identities,
+               'hotkeys': hotkeys,
+               'coldkeys': coldkeys,
+               'trusts': trusts}
+    return results
 
 @app.get("/throughput", response_model=Throughput)
 def throughput_metrics():

classes.py

@@ -1,17 +1,23 @@
 from pydantic import BaseModel
+from datetime import datetime
+from typing import List
+
+
+class Data(BaseModel):
+    last_event_at: List[datetime]
+    cumulative_jobs: List[int]
 
 class ProductivityData(BaseModel):
-    total_completed_jobs: dict[str, dict[int, str]]
-    
-    
-    
-class Last24hProductivityData(BaseModel):
     unique_folded: int
     total_completed_jobs: int
+    unique_folded_data: Data
+    total_completed_jobs_data: Data
     
+
 class Productivity(BaseModel):
     all_time: ProductivityData
-    last_24h: Last24hProductivityData
+    last_24h: ProductivityData
+    last_30d: ProductivityData
     
 class ThroughputData(BaseModel):
     validator_sent: float
@@ -20,4 +26,12 @@ class ThroughputData(BaseModel):
 class Throughput(BaseModel):
     all_time: ThroughputData
     last_24h: ThroughputData
-    data: dict
+    data: dict
+    
+class Metagraph(BaseModel):
+    incentives: List[float]
+    emissions: List[float]
+    identities: List[str]
+    hotkeys: List[str]
+    coldkeys: List[str]
+    trusts: List[float]

utils.py

@@ -8,7 +8,6 @@ import pandas as pd
 import streamlit as st
 import tqdm
 import wandb
-
 # TODO: Store the runs dataframe (as in sn1 dashboard) and top up with the ones created since the last snapshot
 # TODO: Store relevant wandb data in a database for faster access
 
@@ -192,40 +191,60 @@ def get_data_transferred(df, df_24h, unit='GB'):
     'data': df[['md_inputs_sum', 'md_outputs_sum', 'updated_at']].to_dict()
     }
 
+def calculate_productivity_data(df):
+    completed_jobs = df[df['updated_count'] == 10]
+    completed_jobs['last_event_at'] = pd.to_datetime(completed_jobs['updated_at'])
+    unique_folded = completed_jobs.drop_duplicates(subset=['pdb_id'], keep='first')
+    completed_jobs = completed_jobs.sort_values(by='last_event_at').reset_index()
+    completed_jobs['cumulative_jobs'] = completed_jobs.index + 1
+    unique_folded = unique_folded.sort_values(by='last_event_at').reset_index()
+    unique_folded['cumulative_jobs'] = unique_folded.index + 1
+    return {
+        'unique_folded': len(unique_folded),
+        'total_completed_jobs': len(completed_jobs),
+        'unique_folded_data': {'last_event_at': unique_folded['last_event_at'].dt.to_pydatetime(), 'cumulative_jobs':unique_folded['cumulative_jobs'].values},
+        'total_completed_jobs_data': {'last_event_at': completed_jobs['last_event_at'].dt.to_pydatetime(), 'cumulative_jobs':completed_jobs['cumulative_jobs'].values}
+    }
 
-def get_productivity(df_all, df_24h): 
+def get_productivity(df_all, df_24h, df_30d): 
     result = {
         'all_time': {
-            'total_completed_jobs': 0
+            'unique_folded': 0,
+            'total_completed_jobs': 0,
+            'unique_folded_data': {},
+            'total_completed_jobs_data': {}
         },
         'last_24h': {
             'unique_folded': 0,
-            'total_completed_jobs': 0
+            'total_completed_jobs': 0,
+            "unique_folded_data": {},
+            'total_completed_jobs_data': {}
+        },
+        'last_30d': {
+            'unique_folded': 0,
+            'total_completed_jobs': 0,
+            "unique_folded_data": {},
+            'total_completed_jobs_data': {}
         }
     }
-    if df_all is not None:
-  
-        
-        completed_jobs = df_all[df_all['updated_count'] == 10]
-        
-        result['all_time'].update({
-            'total_completed_jobs': completed_jobs[["updated_at", "pdb_id"]].to_dict(),
-        })
+
+
+
+    if df_all is not None:      
+        result['all_time'].update(calculate_productivity_data(df_all))
         
     if df_24h is not None:
-        completed_jobs_24h = df_24h[df_24h['updated_count'] == 10]
-        unique_completed_jobs_24h = completed_jobs_24h.drop_duplicates(subset=['pdb_id'], keep='first')
-        result['last_24h'].update({
-            'unique_folded': len(unique_completed_jobs_24h),
-            'total_completed_jobs': len(completed_jobs_24h)
-        })
+        result['last_24h'].update(calculate_productivity_data(df_24h))
+        
+    if df_30d is not None:
+        result['last_30d'].update(calculate_productivity_data(df_30d))
     return result
 
-def get_leaderboard(df, ntop=10, entity_choice='identity'):
+def get_leaderboard(df, entity_choice='identity'):
 
     df = df.loc[df.validator_permit==False]
     df.index = range(df.shape[0])
-    return df.groupby(entity_choice).I.sum().sort_values().reset_index().tail(ntop)
+    return df.groupby(entity_choice).I.sum().sort_values().reset_index()
 
 
 
@@ -305,8 +324,7 @@ def preload_data():
     return combined_df
 
 @st.cache_data()
-def get_metagraph(time):
-    print(f'Loading metagraph with time {time}')
+def get_metagraph():
     subtensor = bt.subtensor(network=NETWORK)
     m = subtensor.metagraph(netuid=NETUID)
     meta_cols = ['I','stake','trust','validator_trust','validator_permit','C','R','E','dividends','last_update']