Spaces:
Sleeping
Sleeping
File size: 3,644 Bytes
3b3db42 8b1f7a0 3b3db42 91e8a06 8b1f7a0 3b3db42 8b1f7a0 54eae7e 8b1f7a0 17f029a 8b1f7a0 54eae7e 3d8dbe8 54eae7e 8b1f7a0 17f029a 8b1f7a0 fefe31a 8b1f7a0 9f4fde3 8b1f7a0 17f029a 8b1f7a0 6be74f5 8b1f7a0 54eae7e 8b1f7a0 fefe31a 8b1f7a0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 |
import json
import os
from datetime import datetime, timezone
from src.display.formatting import styled_error, styled_message, styled_warning
from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
from src.submission.check_validity import (
already_submitted_models,
check_model_card,
get_model_size,
is_model_on_hub,
)
REQUESTED_MODELS = None
USERS_TO_SUBMISSION_DATES = None
def add_new_eval(
model: str,
revision: str,
precision: str,
model_type: str,
):
global REQUESTED_MODELS
global USERS_TO_SUBMISSION_DATES
if not REQUESTED_MODELS:
REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
user_name = ""
model_path = model
if "/" in model:
user_name = model.split("/")[0]
model_path = model.split("/")[1]
precision = precision.split(" ")[0]
# Extract just the emoji from model_type (e.g., "🔤 Encoder (BERT-like)" -> "🔤")
model_type_symbol = model_type.split(" ")[0] if model_type else "?"
current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
# Does the model actually exist?
if revision == "":
revision = "main"
# Is the model on the hub?
model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
if not model_on_hub:
return styled_error(f'Model "{model}" {error}')
# Is the model info correctly filled?
try:
model_info = API.model_info(repo_id=model, revision=revision)
except Exception:
return styled_error("Could not get your model information. Please fill it up properly.")
model_size = get_model_size(model_info=model_info, precision=precision)
# Were the model card and license filled?
try:
license = "Unknown"
if hasattr(model_info, 'cardData') and model_info.cardData:
license = model_info.cardData.get("license", "Unknown")
if license == "Unknown":
return styled_error("Please select a license for your model")
except Exception:
return styled_error("Please select a license for your model")
modelcard_OK, error_msg = check_model_card(model)
if not modelcard_OK:
return styled_error(error_msg)
# Seems good, creating the eval
print("Adding new eval")
eval_entry = {
"model": model,
"revision": revision,
"precision": precision,
"model_type": model_type_symbol,
"status": "PENDING",
"submitted_time": current_time,
"likes": model_info.likes,
"params": model_size,
"license": license,
"private": False,
}
# Check for duplicate submission
if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
return styled_warning("This model has been already submitted.")
print("Creating eval file")
OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
os.makedirs(OUT_DIR, exist_ok=True)
out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_Original.json"
with open(out_path, "w") as f:
f.write(json.dumps(eval_entry))
print("Uploading eval file")
API.upload_file(
path_or_fileobj=out_path,
path_in_repo=out_path.split("eval-queue/")[1],
repo_id=QUEUE_REPO,
repo_type="dataset",
commit_message=f"Add {model} to eval queue",
)
# Remove the local file
os.remove(out_path)
return styled_message(
"Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
)
|