Commit
·
5e53d23
1
Parent(s):
a475949
remove unused constants and redundant imports; update Dockerfile dependencies
Browse files- Dockerfile +1 -0
- src/config.py +0 -6
- src/eval.py +11 -36
Dockerfile
CHANGED
|
@@ -3,6 +3,7 @@ FROM python:3.12-slim
|
|
| 3 |
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 4 |
wget \
|
| 5 |
ca-certificates \
|
|
|
|
| 6 |
&& rm -rf /var/lib/apt/lists/*
|
| 7 |
|
| 8 |
# Set the working directory in the container
|
|
|
|
| 3 |
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 4 |
wget \
|
| 5 |
ca-certificates \
|
| 6 |
+
libgl1-mesa-glx \
|
| 7 |
&& rm -rf /var/lib/apt/lists/*
|
| 8 |
|
| 9 |
# Set the working directory in the container
|
src/config.py
CHANGED
|
@@ -1,8 +1,3 @@
|
|
| 1 |
-
# File and directory paths
|
| 2 |
-
import os
|
| 3 |
-
|
| 4 |
-
EVAL_SCRIPT_PATH = "src/eval.py"
|
| 5 |
-
|
| 6 |
LOCAL_TEMP_SUBMISSIONS_DIR = "temp_submissions_app"
|
| 7 |
|
| 8 |
# Hugging Face Dataset Configuration
|
|
@@ -27,7 +22,6 @@ SUPPORTED_FRAMEWORKS = [
|
|
| 27 |
def print_config():
|
| 28 |
"""Print the configuration settings."""
|
| 29 |
print("Configuration Settings:")
|
| 30 |
-
print(f"EVAL_SCRIPT_PATH: {EVAL_SCRIPT_PATH}")
|
| 31 |
print(f"LOCAL_TEMP_SUBMISSIONS_DIR: {LOCAL_TEMP_SUBMISSIONS_DIR}")
|
| 32 |
print(f"DATASET_REPO_ID: {DATASET_REPO_ID}")
|
| 33 |
print(f"DS_SUBMISSIONS_PATH: {DS_SUBMISSIONS_PATH}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
LOCAL_TEMP_SUBMISSIONS_DIR = "temp_submissions_app"
|
| 2 |
|
| 3 |
# Hugging Face Dataset Configuration
|
|
|
|
| 22 |
def print_config():
|
| 23 |
"""Print the configuration settings."""
|
| 24 |
print("Configuration Settings:")
|
|
|
|
| 25 |
print(f"LOCAL_TEMP_SUBMISSIONS_DIR: {LOCAL_TEMP_SUBMISSIONS_DIR}")
|
| 26 |
print(f"DATASET_REPO_ID: {DATASET_REPO_ID}")
|
| 27 |
print(f"DS_SUBMISSIONS_PATH: {DS_SUBMISSIONS_PATH}")
|
src/eval.py
CHANGED
|
@@ -1,17 +1,19 @@
|
|
| 1 |
-
# eval.py
|
| 2 |
import datetime
|
| 3 |
-
import sys
|
| 4 |
-
import os
|
| 5 |
import time
|
| 6 |
import json
|
| 7 |
-
import subprocess
|
| 8 |
import tempfile
|
| 9 |
-
from pathlib import Path
|
| 10 |
|
| 11 |
import minizinc
|
| 12 |
-
from datasets import load_dataset
|
| 13 |
-
from huggingface_hub import HfApi, hf_hub_download
|
| 14 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
# --- Configuration ---
|
| 17 |
|
|
@@ -24,37 +26,10 @@ GT_MODEL_CODE_COLUMN = "model"
|
|
| 24 |
# Timeout for running individual model scripts (both generated and modified ground-truth)
|
| 25 |
SCRIPT_EXECUTION_TIMEOUT = 60 # seconds
|
| 26 |
|
| 27 |
-
"""Handles evaluation of submissions."""
|
| 28 |
-
|
| 29 |
-
import os
|
| 30 |
-
import sys
|
| 31 |
-
import subprocess
|
| 32 |
-
import threading
|
| 33 |
-
from pathlib import Path
|
| 34 |
-
|
| 35 |
-
from src.config import EVAL_SCRIPT_PATH, DATASET_REPO_ID, DS_RESULTS_PATH, CPMPY_FRAMEWORK, ORTOOLS_FRAMEWORK, \
|
| 36 |
-
MINIZINC_FRAMEWORK
|
| 37 |
-
|
| 38 |
|
| 39 |
def run_evaluation(submission_path):
|
| 40 |
-
|
| 41 |
-
# if not Path(EVAL_SCRIPT_PATH).exists():
|
| 42 |
-
# print(f"ERROR: Eval script '{EVAL_SCRIPT_PATH}' not found")
|
| 43 |
-
# return
|
| 44 |
-
|
| 45 |
print(f"Starting evaluation for: {submission_path}")
|
| 46 |
-
|
| 47 |
-
# command = [
|
| 48 |
-
# sys.executable,
|
| 49 |
-
# EVAL_SCRIPT_PATH,
|
| 50 |
-
# DATASET_REPO_ID,
|
| 51 |
-
# submission_path,
|
| 52 |
-
# DS_RESULTS_PATH
|
| 53 |
-
# ]
|
| 54 |
-
|
| 55 |
-
|
| 56 |
main_eval(DATASET_REPO_ID, submission_path, DS_RESULTS_PATH)
|
| 57 |
-
|
| 58 |
print(f"Evaluation process complete for: {submission_path}", flush=True)
|
| 59 |
|
| 60 |
|
|
@@ -67,6 +42,7 @@ def start_background_evaluation(submission_path):
|
|
| 67 |
thread.start()
|
| 68 |
return True
|
| 69 |
|
|
|
|
| 70 |
def extract_json_from_code_output(output: str):
|
| 71 |
try:
|
| 72 |
start_index = output.find('{')
|
|
@@ -78,7 +54,6 @@ def extract_json_from_code_output(output: str):
|
|
| 78 |
return None
|
| 79 |
|
| 80 |
|
| 81 |
-
|
| 82 |
def exec_code_minizinc(code: str, timeout_sec):
|
| 83 |
"""
|
| 84 |
Executes a MiniZinc model string using the minizinc-python library.
|
|
|
|
|
|
|
| 1 |
import datetime
|
|
|
|
|
|
|
| 2 |
import time
|
| 3 |
import json
|
|
|
|
| 4 |
import tempfile
|
|
|
|
| 5 |
|
| 6 |
import minizinc
|
| 7 |
+
from datasets import load_dataset
|
| 8 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
import subprocess
|
| 12 |
+
import threading
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
from src.config import DATASET_REPO_ID, DS_RESULTS_PATH, CPMPY_FRAMEWORK, ORTOOLS_FRAMEWORK, \
|
| 16 |
+
MINIZINC_FRAMEWORK
|
| 17 |
|
| 18 |
# --- Configuration ---
|
| 19 |
|
|
|
|
| 26 |
# Timeout for running individual model scripts (both generated and modified ground-truth)
|
| 27 |
SCRIPT_EXECUTION_TIMEOUT = 60 # seconds
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
|
| 30 |
def run_evaluation(submission_path):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
print(f"Starting evaluation for: {submission_path}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
main_eval(DATASET_REPO_ID, submission_path, DS_RESULTS_PATH)
|
|
|
|
| 33 |
print(f"Evaluation process complete for: {submission_path}", flush=True)
|
| 34 |
|
| 35 |
|
|
|
|
| 42 |
thread.start()
|
| 43 |
return True
|
| 44 |
|
| 45 |
+
|
| 46 |
def extract_json_from_code_output(output: str):
|
| 47 |
try:
|
| 48 |
start_index = output.find('{')
|
|
|
|
| 54 |
return None
|
| 55 |
|
| 56 |
|
|
|
|
| 57 |
def exec_code_minizinc(code: str, timeout_sec):
|
| 58 |
"""
|
| 59 |
Executes a MiniZinc model string using the minizinc-python library.
|