Spaces:
Running
Running
Shiyu Zhao
commited on
Commit
·
e65749f
1
Parent(s):
85d9e2b
Update space
Browse files- app.py +11 -4
- utils/hub_storage.py +23 -28
app.py
CHANGED
|
@@ -225,11 +225,12 @@ def scan_submissions_directory():
|
|
| 225 |
|
| 226 |
# Get submissions directory content from HuggingFace hub
|
| 227 |
try:
|
| 228 |
-
repo_files = api.
|
| 229 |
repo_id=REPO_ID,
|
| 230 |
-
repo_type="space"
|
| 231 |
-
paths=["submissions"]
|
| 232 |
)
|
|
|
|
|
|
|
| 233 |
except Exception as e:
|
| 234 |
print(f"Error listing repository contents: {str(e)}")
|
| 235 |
return
|
|
@@ -294,7 +295,12 @@ def scan_submissions_directory():
|
|
| 294 |
|
| 295 |
# Read metadata file
|
| 296 |
try:
|
| 297 |
-
metadata_content =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
submission_data = json.loads(metadata_content)
|
| 299 |
except Exception as e:
|
| 300 |
print(f"Error reading metadata for {folder_name}: {str(e)}")
|
|
@@ -306,6 +312,7 @@ def scan_submissions_directory():
|
|
| 306 |
|
| 307 |
# Update corresponding DataFrame
|
| 308 |
update_leaderboard_data(submission_data)
|
|
|
|
| 309 |
|
| 310 |
except Exception as e:
|
| 311 |
print(f"Error processing folder {folder_name}: {str(e)}")
|
|
|
|
| 225 |
|
| 226 |
# Get submissions directory content from HuggingFace hub
|
| 227 |
try:
|
| 228 |
+
repo_files = api.list_repo_files(
|
| 229 |
repo_id=REPO_ID,
|
| 230 |
+
repo_type="space"
|
|
|
|
| 231 |
)
|
| 232 |
+
# Filter for files in submissions directory
|
| 233 |
+
repo_files = [f for f in repo_files if f.startswith('submissions/')]
|
| 234 |
except Exception as e:
|
| 235 |
print(f"Error listing repository contents: {str(e)}")
|
| 236 |
return
|
|
|
|
| 295 |
|
| 296 |
# Read metadata file
|
| 297 |
try:
|
| 298 |
+
metadata_content = api.hf_hub_download(
|
| 299 |
+
repo_id=REPO_ID,
|
| 300 |
+
repo_type="space",
|
| 301 |
+
filename=metadata_file,
|
| 302 |
+
text=True
|
| 303 |
+
)
|
| 304 |
submission_data = json.loads(metadata_content)
|
| 305 |
except Exception as e:
|
| 306 |
print(f"Error reading metadata for {folder_name}: {str(e)}")
|
|
|
|
| 312 |
|
| 313 |
# Update corresponding DataFrame
|
| 314 |
update_leaderboard_data(submission_data)
|
| 315 |
+
print(f"Added submission from {folder_name} to {split} leaderboard")
|
| 316 |
|
| 317 |
except Exception as e:
|
| 318 |
print(f"Error processing folder {folder_name}: {str(e)}")
|
utils/hub_storage.py
CHANGED
|
@@ -4,43 +4,38 @@ from .token_handler import TokenHandler
|
|
| 4 |
|
| 5 |
class HubStorage:
|
| 6 |
def __init__(self, repo_id):
|
| 7 |
-
self.token_handler = TokenHandler()
|
| 8 |
-
self.token = self.token_handler.get_verified_token()
|
| 9 |
-
self.api = HfApi()
|
| 10 |
self.repo_id = repo_id
|
|
|
|
| 11 |
|
| 12 |
-
def save_to_hub(self, file_content, path_in_repo, commit_message):
|
| 13 |
-
"""Save file to HuggingFace Hub"""
|
| 14 |
-
return self.api.upload_file(
|
| 15 |
-
path_or_fileobj=file_content,
|
| 16 |
-
path_in_repo=path_in_repo,
|
| 17 |
-
repo_id=self.repo_id,
|
| 18 |
-
repo_type="space",
|
| 19 |
-
commit_message=commit_message,
|
| 20 |
-
token=self.token
|
| 21 |
-
)
|
| 22 |
-
|
| 23 |
-
def load_from_hub(self, path_in_repo):
|
| 24 |
-
"""Load file from HuggingFace Hub"""
|
| 25 |
-
return self.api.hf_hub_download(
|
| 26 |
-
repo_id=self.repo_id,
|
| 27 |
-
filename=path_in_repo,
|
| 28 |
-
repo_type="space",
|
| 29 |
-
token=self.token
|
| 30 |
-
)
|
| 31 |
-
|
| 32 |
def get_file_content(self, file_path):
|
| 33 |
"""
|
| 34 |
Get content of a file from the repository
|
| 35 |
"""
|
| 36 |
try:
|
| 37 |
-
|
| 38 |
-
content = api.file_download(
|
| 39 |
repo_id=self.repo_id,
|
| 40 |
repo_type="space",
|
| 41 |
-
filename=file_path
|
|
|
|
| 42 |
)
|
| 43 |
-
return content
|
| 44 |
except Exception as e:
|
| 45 |
print(f"Error reading file {file_path}: {str(e)}")
|
| 46 |
-
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
class HubStorage:
|
| 6 |
def __init__(self, repo_id):
|
|
|
|
|
|
|
|
|
|
| 7 |
self.repo_id = repo_id
|
| 8 |
+
self.api = HfApi()
|
| 9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
def get_file_content(self, file_path):
|
| 11 |
"""
|
| 12 |
Get content of a file from the repository
|
| 13 |
"""
|
| 14 |
try:
|
| 15 |
+
content = self.api.hf_hub_download(
|
|
|
|
| 16 |
repo_id=self.repo_id,
|
| 17 |
repo_type="space",
|
| 18 |
+
filename=file_path,
|
| 19 |
+
text=True
|
| 20 |
)
|
| 21 |
+
return content
|
| 22 |
except Exception as e:
|
| 23 |
print(f"Error reading file {file_path}: {str(e)}")
|
| 24 |
+
return None
|
| 25 |
+
|
| 26 |
+
def save_to_hub(self, file_content, path_in_repo, commit_message):
|
| 27 |
+
"""
|
| 28 |
+
Save a file to the hub
|
| 29 |
+
"""
|
| 30 |
+
try:
|
| 31 |
+
self.api.upload_file(
|
| 32 |
+
path_or_fileobj=file_content,
|
| 33 |
+
path_in_repo=path_in_repo,
|
| 34 |
+
repo_id=self.repo_id,
|
| 35 |
+
repo_type="space",
|
| 36 |
+
commit_message=commit_message
|
| 37 |
+
)
|
| 38 |
+
return True
|
| 39 |
+
except Exception as e:
|
| 40 |
+
print(f"Error saving file to hub: {str(e)}")
|
| 41 |
+
return False
|