add webhook to fetch data properly

server/.env.example

@@ -4,7 +4,7 @@ HUGGING_FACE_HUB_TOKEN=your_token_here
 
 # Repository ID for storing leaderboard data (required)
 # Format: username/repo-name
-HUGGING_FACE_STORAGE_REPO=tfrere/leaderboard-explorer
+HUGGING_FACE_STORAGE_REPO=leaderboard-explorer/leaderboard_explorer
 
 # File path in the repository (required)
 HUGGING_FACE_STORAGE_FILE_PATH=final_leaderboards.json

server/server.py

@@ -1,4 +1,4 @@
-from fastapi import FastAPI, HTTPException
+from fastapi import FastAPI, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -7,6 +7,13 @@ import os
 from dotenv import load_dotenv
 from huggingface_hub import HfApi
 import json
+import logging
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
 
 # Load environment variables
 load_dotenv()
@@ -49,31 +56,41 @@ hf_api = HfApi(token=HF_TOKEN)
 def fetch_leaderboards():
     """Fetch leaderboards data from Hugging Face"""
     try:
-        # Download the JSON file directly
+        logging.info(f"Fetching leaderboards from {REPO_ID}/{FILE_PATH}")
+        # Download the JSON file directly with force_download to ensure we get the latest version
         json_path = hf_api.hf_hub_download(
             repo_id=REPO_ID,
             filename=FILE_PATH,
-            repo_type="dataset"
+            repo_type="dataset",
+            force_download=True,  # Force download to ensure we get the latest version
+            force_filename="leaderboards_latest.json"  # Force a specific filename to avoid caching issues
         )
         
+        logging.info(f"File downloaded to: {json_path}")
+        
         with open(json_path, 'r') as f:
-            cache["data"] = json.load(f)
+            new_data = json.load(f)
+            old_data = cache["data"]
+            cache["data"] = new_data
             cache["last_updated"] = datetime.now()
-            print(f"Cache updated at {cache['last_updated']}")
+            
+            # Log the differences
+            old_len = len(old_data) if old_data and isinstance(old_data, list) else 0
+            new_len = len(new_data) if isinstance(new_data, list) else 0
+            logging.info(f"Cache updated: Old entries: {old_len}, New entries: {new_len}")
+            logging.info(f"Cache update timestamp: {cache['last_updated']}")
             
     except Exception as e:
-        print(f"Error fetching data: {str(e)}")
+        logging.error(f"Error fetching data: {str(e)}", exc_info=True)
         if not cache["data"]:  # Only raise if we don't have any cached data
             raise HTTPException(status_code=500, detail="Failed to fetch leaderboards data")
 
-# Initialize scheduler
-scheduler = BackgroundScheduler()
-scheduler.add_job(fetch_leaderboards, 'interval', minutes=CACHE_DURATION_MINUTES)
-scheduler.start()
-
 # Initial fetch
 fetch_leaderboards()
 
+# Mount static files
+app.mount("/static", StaticFiles(directory="static", html=True), name="static")
+
 @app.get("/api/leaderboards")
 async def get_leaderboards():
     """Get leaderboards data from cache"""
@@ -94,8 +111,38 @@ async def health_check():
         "last_updated": cache["last_updated"].isoformat() if cache["last_updated"] else None
     }
 
-# Mount static files
-app.mount("/", StaticFiles(directory="static", html=True), name="static")
+@app.post("/api/webhook")
+async def handle_webhook(request: Request):
+    """Handle webhook notifications from Hugging Face Hub"""
+    try:
+        body = await request.json()
+        logging.info(f"Received webhook with payload: {body}")
+        
+        # Get the event details
+        event = body.get("event", {})
+        
+        # Verify if it's a relevant update (repo content update)
+        if event.get("action") == "update" and event.get("scope") == "repo.content":
+            try:
+                logging.info(f"Dataset update detected for repo {REPO_ID}, file {FILE_PATH}")
+                # Force a clean fetch
+                fetch_leaderboards()
+                if cache["last_updated"]:
+                    logging.info(f"Cache successfully updated at {cache['last_updated']}")
+                    return {"status": "success", "message": "Cache updated"}
+                else:
+                    logging.error("Cache update failed: last_updated is None")
+                    return {"status": "error", "message": "Cache update failed"}
+            except Exception as fetch_error:
+                logging.error(f"Error during fetch_leaderboards: {str(fetch_error)}", exc_info=True)
+                return {"status": "error", "message": f"Failed to update cache: {str(fetch_error)}"}
+        
+        logging.info(f"Ignoring webhook event: action={event.get('action')}, scope={event.get('scope')}")
+        return {"status": "ignored", "message": "Event type not relevant"}
+        
+    except Exception as e:
+        logging.error(f"Error processing webhook: {str(e)}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"Failed to process webhook: {str(e)}")
 
 if __name__ == "__main__":
     import uvicorn