Retries for the HF snapshot download

src/leaderboard/populate.py

@@ -1,7 +1,10 @@
 import json
+import os
+import time
 
 import pandas as pd
 from huggingface_hub import snapshot_download
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from src.envs import RESULTS_REPO_ID, RESULTS_REPO_PATH, TOKEN
 from src.leaderboard.utils import COLUMNS
@@ -9,15 +12,29 @@ from src.leaderboard.utils import COLUMNS
 
 def download_result_data():
     print(f"Downloading {RESULTS_REPO_ID} to {RESULTS_REPO_PATH}")
-    snapshot_download(
-        repo_id=RESULTS_REPO_ID,
-        local_dir=RESULTS_REPO_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-        allow_patterns=["results/*"],
-    )
+    # Set env vars to configure huggingface_hub
+    os.environ["HF_HUB_ETAG_TIMEOUT"] = "30"
+    os.environ["HF_HUB_DOWNLOAD_TIMEOUT"] = "30"
+
+    while True:
+        try:
+            snapshot_download(
+                repo_id=RESULTS_REPO_ID,
+                local_dir=RESULTS_REPO_PATH,
+                repo_type="dataset",
+                tqdm_class=None,
+                etag_timeout=30,
+                token=TOKEN,
+                allow_patterns=["results/*"],
+            )
+            break
+        except Exception as e:
+            if isinstance(e, (ReadTimeout, ConnectionError)):
+                time.sleep(3)
+                print(f"Connection error: {e}. Retrying...")
+                continue
+            else:
+                raise e
 
 
 def load_results() -> pd.DataFrame: