private test dataset

competitions/create.py

@@ -163,6 +163,7 @@ def _create(
         "SUBMISSION_ROWS": int(submission_rows),
         "EVAL_METRIC": metric,
         "LOGO": competition_logo,
+        "DATASET": "",
     }
     teams_json = {}
     user_team_json = {}

competitions/evaluate.py

@@ -4,7 +4,7 @@ import os
 import shutil
 import subprocess
 
-from huggingface_hub import HfApi, snapshot_download
+from huggingface_hub import HfApi, Repository, snapshot_download
 from loguru import logger
 
 from competitions import utils
@@ -81,6 +81,7 @@ def run(params):
     utils.update_submission_status(params, "processing")
 
     if params.competition_type == "script":
+        _ = Repository(local_dir="/tmp/data", clone_from=params.dataset)
         generate_submission_file(params)
 
     evaluation = compute_metrics(params)

competitions/info.py

@@ -129,3 +129,7 @@ class CompetitionInfo:
     @property
     def hardware(self):
         return self.config["HARDWARE"]
+
+    @property
+    def dataset(self):
+        return self.config.get("DATASET", "")

competitions/params.py

@@ -17,6 +17,7 @@ class EvalParams(BaseModel):
     output_path: str
     submission_repo: str
     time_limit: int
+    dataset: str
 
     class Config:
         protected_namespaces = ()

docs/source/competition_repo.mdx

@@ -47,7 +47,8 @@ conf.json is the configuration file for the competition. An example conf.json is
    "SUBMISSION_COLUMNS":"id,pred",
    "SUBMISSION_ROWS":10000,
    "EVAL_METRIC":"roc_auc_score",
-   "LOGO":"https://github.com/abhishekkrthakur/public_images/blob/main/song.png?raw=true"
+   "LOGO":"https://github.com/abhishekkrthakur/public_images/blob/main/song.png?raw=true",
+   "DATASET": ""
 }
 ```
 
@@ -69,6 +70,7 @@ as it would require you to re-evaluate all the submissions.
 - SUBMISSION_ROWS: This field is used to specify the number of rows in the submission file without the header.
 - EVAL_METRIC: This field is used to specify the evaluation metric. We support all the scikit-learn metrics and even custom metrics.
 - LOGO: This field is used to specify the logo of the competition. The logo must be a png file. The logo is shown on the all pages of the competition.
+- DATASET: This field is used to specify the PRIVATE dataset used in the competition. The dataset is available to the users only during the script run. This is only used for script competitions.
 
 
 ### solution.csv

docs/source/custom_metric.mdx

@@ -48,6 +48,7 @@ class EvalParams(BaseModel):
     output_path: str
     submission_repo: str
     time_limit: int
+    dataset: str  # private test dataset, used only for script competitions
 ```
 
 You are free to do whatever you want to in the `compute` function.