submissions work

.env.example

@@ -0,0 +1,6 @@
+HF_ACCESS_TOKEN=hf_app_XXX
+AUTOTRAIN_USERNAME=autoevaluator
+AUTOTRAIN_TOKEN=hf_XXX
+AUTOTRAIN_BACKEND_API=https://api.autotrain.huggingface.co
+MOONLANDING_URL=https://huggingface.co
+SUBMISSION_LIMIT=5

__init__.py

@@ -1 +1,3 @@
+import config
+
 __version__ = "0.0.1"

config.py

@@ -0,0 +1,16 @@
+import os
+from dotenv import load_dotenv
+from pathlib import Path
+
+if Path(".env").is_file():
+    load_dotenv(".env")
+
+
+MOONLANDING_URL = os.getenv("MOONLANDING_URL")
+COMPETITION_ID = os.getenv("COMPETITION_ID")
+DUMMY_DATA_PATH = os.getenv("DUMMY_DATA_PATH")
+AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
+AUTOTRAIN_TOKEN = os.getenv("AUTOTRAIN_TOKEN")
+HF_ACCESS_TOKEN = os.getenv("HF_ACCESS_TOKEN")
+AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
+SUBMISSION_LIMIT = int(os.getenv("SUBMISSION_LIMIT"))

pages/3_🔥_New Submission.py

@@ -0,0 +1,89 @@
+import streamlit as st
+from huggingface_hub import HfApi, CommitOperationAdd
+import uuid
+import os
+import pandas as pd
+import utils
+import config
+
+SUBMISSION_TEXT = """You can make upto 5 submissions per day.
+The test data has been divided into public and private splits.
+Your score on the public split will be shown on the leaderboard.
+Your final score will be based on your private split performance.
+The final rankings will be based on the private split performance.
+"""
+
+SUBMISSION_ERROR = """Submission is not in a proper format.
+Please check evaluation instructions for more details."""
+
+
+def app():
+    st.set_page_config(page_title="New Submissions", page_icon="🤗")
+    st.write("## New Submission")
+    st.markdown(SUBMISSION_TEXT)
+    uploaded_file = st.file_uploader("Choose a file")
+    # user token
+    user_token = st.text_input("Enter your token", value="", type="password")
+    user_token = user_token.strip()
+    if uploaded_file is not None and user_token != "":
+        # verify token
+        user_info = utils.user_authentication(token=user_token)
+        print(user_info)
+        if "error" in user_info:
+            st.error("Invalid token")
+            return
+
+        if user_info["emailVerified"] is False:
+            st.error("Please verify your email on Hugging Face Hub")
+            return
+
+        # check if user can submit to the competition
+        if utils.check_user_submission_limit(user_info) is False:
+            st.error("You have reached your submission limit for today")
+            return
+
+        bytes_data = uploaded_file.getvalue()
+        # verify file is valid
+        if not utils.verify_submission(bytes_data):
+            st.error("Invalid submission")
+            st.write(SUBMISSION_ERROR)
+            # write a horizontal html line
+            st.markdown("<hr/>", unsafe_allow_html=True)
+        else:
+            with st.spinner("Uploading submission..."):
+                user_id = user_info["id"]
+                submission_id = str(uuid.uuid4())
+                file_extension = uploaded_file.name.split(".")[-1]
+                # upload file to hf hub
+                api = HfApi()
+                api.upload_file(
+                    path_or_fileobj=bytes_data,
+                    path_in_repo=f"submissions/{user_id}-{submission_id}.{file_extension}",
+                    repo_id=config.COMPETITION_ID,
+                    repo_type="dataset",
+                    token=config.AUTOTRAIN_TOKEN,
+                )
+            with st.spinner("Creating submission..."):
+                # update submission limit
+                submissions_made = utils.increment_submissions(
+                    user_id=user_id,
+                    submission_id=submission_id,
+                    submission_comment="",
+                )
+            st.success(
+                f"Upload successful! You have {config.SUBMISSION_LIMIT - submissions_made} submissions left for today."
+            )
+
+            with st.spinner("Scheuling submission for evaluation..."):
+                # schedule submission for evaluation
+                utils.create_project(
+                    project_id=submission_id,
+                    submission_dataset=f"{config.COMPETITION_ID}",
+                    model="dummy",
+                    dataset=user_id,
+                )
+            st.success("Submission scheduled for evaluation")
+
+
+if __name__ == "__main__":
+    app()

pages/3_🔥_Submissions.py

@@ -1,92 +0,0 @@
-import streamlit as st
-from huggingface_hub import HfApi, CommitOperationAdd
-import uuid
-import os
-import pandas as pd
-
-SUBMISSION_TEXT = """You can make upto 5 submissions per day.
-The test data has been divided into public and private splits.
-Your score on the public split will be shown on the leaderboard.
-Your final score will be based on your private split performance.
-The final rankings will be based on the private split performance.
-"""
-
-SUBMISSION_ERROR = """Submission is not in a proper format.
-Please check evaluation instructions for more details."""
-
-COMPETITION_ID = os.getenv("COMPETITION_ID", "sample_competition")
-USER_ID = os.getenv("USER_ID", "sample_user")
-DUMMY_DATA_PATH = os.getenv("DUMMY_DATA_PATH", "autoevaluator/benchmark-dummy-data")
-
-
-def verify_submission(submission):
-    # verify submission is valid
-    return True
-
-
-def fetch_submissions():
-    submissions = [
-        {
-            "submission_id": "72836-23423",
-            "score": 0.7,
-            "created_at": "2021-01-01T00:00:00Z",
-        },
-        {
-            "submission_id": "23-42332",
-            "score": 0.5,
-            "created_at": "2021-01-01T00:00:00Z",
-        },
-    ]
-    df = pd.DataFrame(submissions)
-    return df
-
-
-def app():
-    st.set_page_config(page_title="Submissions", page_icon="🤗")
-    st.write("## Submissions")
-    uploaded_file = st.sidebar.file_uploader("Choose a file")
-    if uploaded_file is not None:
-        bytes_data = uploaded_file.getvalue()
-        # verify file is valid
-        if not verify_submission(bytes_data):
-            st.error("Invalid submission")
-            st.write(SUBMISSION_ERROR)
-            # write a horizontal html line
-            st.markdown("<hr/>", unsafe_allow_html=True)
-        else:
-            # start progress bar
-            progress_bar = st.progress(0)
-            submission_id = str(uuid.uuid4())
-            api = HfApi()
-            operations = [
-                CommitOperationAdd(
-                    path_in_repo="submission.csv",
-                    path_or_fileobj=bytes_data,
-                ),
-            ]
-            # update progress bar
-            progress_bar.progress(0.5)
-            api.create_repo(
-                repo_id=submission_id,
-                private=True,
-                repo_type="dataset",
-            )
-            api.create_commit(
-                repo_id=f"{USER_ID}/{submission_id}",
-                operations=operations,
-                commit_message="add submission.csv",
-                repo_type="dataset",
-            )
-
-    st.markdown(SUBMISSION_TEXT)
-    # add submissions history table
-    st.write("### Submissions History")
-    submissions = fetch_submissions()
-    if len(submissions) == 0:
-        st.write("You have not made any submissions yet.")
-    else:
-        st.write(submissions)
-
-
-if __name__ == "__main__":
-    app()

pages/4_⭐️_Submission History.py

@@ -0,0 +1,38 @@
+import streamlit as st
+from huggingface_hub import HfApi, CommitOperationAdd
+import uuid
+import os
+import pandas as pd
+import utils
+import config
+
+SUBMISSION_TEXT = """You can select upto 2 submissions for private leaderboard.
+"""
+
+
+def app():
+    st.set_page_config(page_title="Submission History", page_icon="🤗")
+    st.write("## Your Submissions")
+    st.markdown(SUBMISSION_TEXT)
+    # user token
+    user_token = st.text_input("Enter your token", value="", type="password")
+    user_token = user_token.strip()
+    if user_token != "":
+        user_info = utils.user_authentication(token=user_token)
+        if "error" in user_info:
+            st.error("Invalid token")
+            return
+
+        if user_info["emailVerified"] is False:
+            st.error("Please verify your email on Hugging Face Hub")
+            return
+
+        # get user submissions
+        user_id = user_info["id"]
+        user_submissions = utils.fetch_submissions(user_id)
+        submissions_df = pd.DataFrame(user_submissions)
+        st.write(submissions_df)
+
+
+if __name__ == "__main__":
+    app()

utils.py

@@ -0,0 +1,243 @@
+import time
+import requests
+import config
+from huggingface_hub import HfApi, hf_hub_download
+import json
+import datetime
+import io
+from huggingface_hub.utils._errors import EntryNotFoundError
+
+
+def get_auth_headers(token: str, prefix: str = "Bearer"):
+    return {"Authorization": f"{prefix} {token}"}
+
+
+def http_post(path: str, token: str, payload=None, domain: str = None, params=None) -> requests.Response:
+    """HTTP POST request to the AutoNLP API, raises UnreachableAPIError if the API cannot be reached"""
+    try:
+        response = requests.post(
+            url=domain + path, json=payload, headers=get_auth_headers(token=token), allow_redirects=True, params=params
+        )
+    except requests.exceptions.ConnectionError:
+        print("❌ Failed to reach AutoNLP API, check your internet connection")
+    response.raise_for_status()
+    return response
+
+
+def http_get(path: str, token: str, domain: str = None) -> requests.Response:
+    """HTTP POST request to the AutoNLP API, raises UnreachableAPIError if the API cannot be reached"""
+    try:
+        response = requests.get(url=domain + path, headers=get_auth_headers(token=token), allow_redirects=True)
+    except requests.exceptions.ConnectionError:
+        print("❌ Failed to reach AutoNLP API, check your internet connection")
+    response.raise_for_status()
+    return response
+
+
+def create_project(project_id, submission_dataset, model, dataset):
+    project_config = {}
+    project_config["dataset_name"] = "lewtun/imdb-dummy"
+    project_config["dataset_config"] = "lewtun--imdb-dummy"
+    project_config["dataset_split"] = "train"
+    project_config["col_mapping"] = {"text": "text", "label": "target"}
+
+    payload = {
+        "username": config.AUTOTRAIN_USERNAME,
+        "proj_name": project_id,
+        "task": 1,
+        "config": {
+            "language": "en",
+            "max_models": 5,
+            "benchmark": {
+                "dataset": dataset,
+                "model": model,
+                "submission_dataset": submission_dataset,
+            },
+        },
+    }
+
+    project_json_resp = http_post(
+        path="/projects/create", payload=payload, token=config.AUTOTRAIN_TOKEN, domain=config.AUTOTRAIN_BACKEND_API
+    ).json()
+    print(project_json_resp)
+    time.sleep(5)
+    # Upload data
+    payload = {
+        "split": 4,
+        "col_mapping": project_config["col_mapping"],
+        "load_config": {"max_size_bytes": 0, "shuffle": False},
+        "dataset_id": project_config["dataset_name"],
+        "dataset_config": project_config["dataset_config"],
+        "dataset_split": project_config["dataset_split"],
+    }
+
+    data_json_resp = http_post(
+        path=f"/projects/{project_json_resp['id']}/data/dataset",
+        payload=payload,
+        token=config.AUTOTRAIN_TOKEN,
+        domain=config.AUTOTRAIN_BACKEND_API,
+    ).json()
+    print("💾💾💾 Dataset creation 💾💾💾")
+    print(data_json_resp)
+
+    # Process data
+    data_proc_json_resp = http_post(
+        path=f"/projects/{project_json_resp['id']}/data/start_processing",
+        token=config.AUTOTRAIN_TOKEN,
+        domain=config.AUTOTRAIN_BACKEND_API,
+    ).json()
+    print(f"🍪 Start data processing response: {data_proc_json_resp}")
+
+    print("⏳ Waiting for data processing to complete ...")
+    is_data_processing_success = False
+    while is_data_processing_success is not True:
+        project_status = http_get(
+            path=f"/projects/{project_json_resp['id']}",
+            token=config.AUTOTRAIN_TOKEN,
+            domain=config.AUTOTRAIN_BACKEND_API,
+        ).json()
+        # See database.database.enums.ProjectStatus for definitions of `status`
+        if project_status["status"] == 3:
+            is_data_processing_success = True
+            print("✅ Data processing complete!")
+        time.sleep(10)
+
+    # Approve training job
+    train_job_resp = http_post(
+        path=f"/projects/{project_json_resp['id']}/start_training",
+        token=config.AUTOTRAIN_TOKEN,
+        domain=config.AUTOTRAIN_BACKEND_API,
+    ).json()
+    print(f"🏃 Training job approval response: {train_job_resp}")
+
+
+def user_authentication(token):
+    headers = {}
+    cookies = {}
+    if token.startswith("hf_"):
+        headers["Authorization"] = f"Bearer {token}"
+    else:
+        cookies = {"token": token}
+    try:
+        response = requests.get(
+            config.MOONLANDING_URL + "/api/whoami-v2",
+            headers=headers,
+            cookies=cookies,
+            timeout=3,
+        )
+    except (requests.Timeout, ConnectionError) as err:
+        print(f"Failed to request whoami-v2 - {repr(err)}")
+        raise Exception("Hugging Face Hub is unreachable, please try again later.")
+    return response.json()
+
+
+def add_new_user(user_info):
+    api = HfApi()
+    user_submission_info = {}
+    user_submission_info["name"] = user_info["name"]
+    user_submission_info["id"] = user_info["id"]
+    user_submission_info["submissions"] = []
+    # convert user_submission_info to BufferedIOBase file object
+    user_submission_info_json = json.dumps(user_submission_info)
+    user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
+    user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+
+    api.upload_file(
+        path_or_fileobj=user_submission_info_json_buffer,
+        path_in_repo=f"{user_info['id']}.json",
+        repo_id=config.COMPETITION_ID,
+        repo_type="dataset",
+        token=config.AUTOTRAIN_TOKEN,
+    )
+
+
+def check_user_submission_limit(user_info):
+    user_id = user_info["id"]
+    try:
+        user_fname = hf_hub_download(
+            repo_id=config.COMPETITION_ID,
+            filename=f"{user_id}.json",
+            use_auth_token=config.AUTOTRAIN_TOKEN,
+            repo_type="dataset",
+        )
+    except EntryNotFoundError:
+        add_new_user(user_info)
+        user_fname = hf_hub_download(
+            repo_id=config.COMPETITION_ID,
+            filename=f"{user_id}.json",
+            use_auth_token=config.AUTOTRAIN_TOKEN,
+            repo_type="dataset",
+        )
+    except Exception as e:
+        print(e)
+        raise Exception("Hugging Face Hub is unreachable, please try again later.")
+
+    with open(user_fname, "r") as f:
+        user_submission_info = json.load(f)
+
+    todays_date = datetime.datetime.now().strftime("%Y-%m-%d")
+    if len(user_submission_info["submissions"]) == 0:
+        user_submission_info["submissions"] = []
+
+    # count the number of times user has submitted today
+    todays_submissions = 0
+    for sub in user_submission_info["submissions"]:
+        if sub["date"] == todays_date:
+            todays_submissions += 1
+    if todays_submissions >= config.SUBMISSION_LIMIT:
+        return False
+    return True
+
+
+def increment_submissions(user_id, submission_id, submission_comment):
+    user_fname = hf_hub_download(
+        repo_id=config.COMPETITION_ID,
+        filename=f"{user_id}.json",
+        use_auth_token=config.AUTOTRAIN_TOKEN,
+        repo_type="dataset",
+    )
+    with open(user_fname, "r") as f:
+        user_submission_info = json.load(f)
+    todays_date = datetime.datetime.now().strftime("%Y-%m-%d")
+    user_submission_info["submissions"].append(
+        {
+            "date": todays_date,
+            "submission_id": submission_id,
+            "submission_comment": submission_comment,
+            "status": "pending",
+        }
+    )
+    # count the number of times user has submitted today
+    todays_submissions = 0
+    for sub in user_submission_info["submissions"]:
+        if sub["date"] == todays_date:
+            todays_submissions += 1
+    # convert user_submission_info to BufferedIOBase file object
+    user_submission_info_json = json.dumps(user_submission_info)
+    user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
+    user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+    api = HfApi()
+    api.upload_file(
+        path_or_fileobj=user_submission_info_json_buffer,
+        path_in_repo=f"{user_id}.json",
+        repo_id=config.COMPETITION_ID,
+        repo_type="dataset",
+        token=config.AUTOTRAIN_TOKEN,
+    )
+    return todays_submissions
+
+
+def verify_submission(bytes_data):
+    return True
+
+
+def fetch_submissions(user_id):
+    user_fname = hf_hub_download(
+        repo_id=config.COMPETITION_ID,
+        filename=f"{user_id}.json",
+        use_auth_token=config.AUTOTRAIN_TOKEN,
+        repo_type="dataset",
+    )
+    with open(user_fname, "r") as f:
+        user_submission_info = json.load(f)
+    return user_submission_info["submissions"]