src/submission/submit.py

import json
import os
import re
from datetime import datetime, timezone
from pathlib import Path

from src.display.formatting import styled_error, styled_message, styled_warning
from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO, PROMPT_VERSIONS, PREDICTIONS_REPO
from src.submission.check_validity import already_submitted_models, is_model_on_hub, get_model_properties

REQUESTED_MODELS = None


def read_configuration(file_paths):
    configuration_file_paths = list(filter(lambda file_path: file_path.name.endswith(".json"), file_paths or []))
    if len(configuration_file_paths) != 1:
        return None, None, None, None, None, styled_error(f"Expected exactly one configuration file but found {len(configuration_file_paths)}!")

    configuration_file_path = file_paths.pop(file_paths.index(configuration_file_paths[0]))

    try:
        with open(configuration_file_path.name, "r", encoding="utf-8") as f:
            data = json.load(f)
    except Exception:
        return None, None, None, None, None, styled_error("Failed to read configuration file!")

    try:
        model_name = data["model_name"]
        model_args = {
            **dict({tuple(arg.split("=")) for arg in data["config"].get("model_args", "").split(",") if len(arg) > 0}),
            "revision": data["config"]["model_revision"],
            "trust_remote_code": True,
            "cache_dir": None
        }
        base_model = model_args.pop("pretrained")
        model_on_hub, error, _ = is_model_on_hub(model_name=base_model, model_args=model_args, token=TOKEN, test_tokenizer=True)
        if not model_on_hub:
            return None, None, model_name, None, None, styled_error(f"Model {model_name} {error}")

        limit = data["config"]["limit"]
        if limit is not None:
            return None, None, model_name, None, None, styled_error(f"Only full results are accepted but found a specified limit of {limit}!")

        prediction_files = {}
        versions = {}
        n_shots = {}
        for task_name, _ in data["configs"].items():
            sample_files = list(filter(lambda file_path: re.search(rf"samples_{task_name}_.*\.jsonl", file_path.name), file_paths))
            if len(sample_files) == 0:
                return None, None, model_name, None, None, styled_error(f"No prediction file found for configured task {task_name}!")

            prediction_files[task_name] = str(file_paths.pop(file_paths.index(sample_files[0])))

            versions[task_name] = data["versions"][task_name]
            n_shots[task_name] = data["n-shot"][task_name]
        if len(prediction_files) == 0:
            return None, None, model_name, None, None, styled_error("No tasks found in configuration!")

        versions = set(versions.values())
        if len(versions) != 1:
            return None, None, model_name, None, None, styled_error(f"All tasks should have the same version but found {versions}!")
        version = list(versions)[0]
        if version not in PROMPT_VERSIONS:
            return None, None, model_name, None, None, styled_error(f"Unknown version {version}, should be one of {PROMPT_VERSIONS}!")

        n_shots = set(n_shots.values())
        if len(n_shots) != 1:
            return None, None, model_name, version, None, styled_error(f"All tasks should have the same number of shots but found {n_shots}!")
        n_shot = list(n_shots)[0]
    except KeyError:
        return None, None, model_name, None, None, styled_error("Wrong configuration file format!")

    if len(file_paths) > 0:
        ignored_files = [Path(file_path).name for file_path in file_paths]
        return data, prediction_files, model_name, version, n_shot, styled_warning(f"The following files will be ignored: {ignored_files}")
    return data, prediction_files, model_name, version, n_shot, styled_message("Files parsed successfully, verify that read metadata is correct before submitting")


def add_new_eval(
    model_training: str,
    maltese_training: str,
    language_count: int,
    configuration: dict,
    prediction_files: dict[str, str],
):
    global REQUESTED_MODELS
    if not REQUESTED_MODELS:
        REQUESTED_MODELS = already_submitted_models(EVAL_REQUESTS_PATH)

    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f")

    if configuration is None or configuration == {} or prediction_files is None or prediction_files == {}:
        return styled_error("No files selected for upload, please upload an output folder (or wait for the files to finish uploading).")

    if model_training is None or model_training == "":
        return styled_error("Please select the model's overall training.")

    if maltese_training is None or maltese_training == "":
        return styled_error("Please select the model's Maltese training.")

    if language_count is None or language_count < 1:
        language_count = None

    model_name, revision, precision, seed, prompt_version, n_shot = get_model_properties(configuration)
    model_id = configuration["model_name"]

    # Seems good, creating the eval
    print("Adding new eval")

    # Check for duplicate submission
    if f"{model_name}_{revision}_{precision}_{seed}_{prompt_version}_{n_shot}" in REQUESTED_MODELS:
        return styled_warning("This model has been already submitted.")

    request = {
        "model": model_id,
        "model_args": dict({tuple(arg.split("=")) for arg in configuration["config"].get("model_args", "").split(",") if len(arg) > 0}),
        "revision": revision,
        "precision": precision,
        "seed": seed,
        "n_shot": n_shot,
        "prompt_version": prompt_version,
        "tasks": list(configuration["configs"].keys()),
        "model_training": model_training,
        "maltese_training": maltese_training,
        "language_count": language_count,
        "submitted_time": current_time,
        "status": "PENDING",
    }

    for task_name, file_path in prediction_files.items():
        print(f"Uploading {model_id} {task_name} prediction file")
        API.upload_file(
            path_or_fileobj=file_path,
            path_in_repo=f"{n_shot}-shot_{prompt_version}/{model_name}_{revision}_{precision}/{seed}-seed/samples_{task_name}_{current_time}.jsonl",
            repo_id=PREDICTIONS_REPO,
            repo_type="dataset",
            commit_message=f"Add {configuration['model_name']} {task_name} {n_shot}-shot outputs",
        )

    print(f"Creating {model_id} configruation file")
    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{model_name}"
    os.makedirs(OUT_DIR, exist_ok=True)
    out_path = f"{OUT_DIR}/requests_{model_name}_{revision}_{precision}_{n_shot}shot_{prompt_version}_{seed}seed_{current_time}.json"

    with open(out_path, "w") as f:
        f.write(json.dumps({"leaderboard": request, "configuration": configuration}, ensure_ascii=False, indent=2))

    print(f"Uploading {model_id} configuration file")
    API.upload_file(
        path_or_fileobj=out_path,
        path_in_repo=out_path.split("eval-queue/")[1],
        repo_id=QUEUE_REPO,
        repo_type="dataset",
        commit_message=f"Add {configuration['model_name']} {n_shot}-shot to eval queue",
    )

    # Remove the local file
    os.remove(out_path)

    return styled_message(
        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
    )