add minizinc

app.py

@@ -1,6 +1,21 @@
+import os
+import subprocess
+
 from src.ui import create_ui
 from src.utils import setup_directories
 
+
+# Add MiniZinc to PATH just in case
+os.environ["PATH"] = "/home/user/minizinc/bin:" + os.environ["PATH"]
+
+# Check MiniZinc is working
+try:
+    output = subprocess.check_output(["minizinc", "--version"], text=True)
+    print("MiniZinc version:", output)
+except Exception as e:
+    print("MiniZinc not found:", e)
+
+
 setup_directories()
 
 if __name__ == "__main__":

requirements.txt

@@ -14,4 +14,5 @@ tqdm
 transformers
 tokenizers>=0.15.0
 sentencepiece
-cpmpy
+cpmpy
+minizinc

setup.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Stop if any command fails
+set -e
+
+# Define version and paths
+VERSION=2.9.3
+MZN_DIR=/home/user/minizinc
+
+# Download and extract MiniZinc
+wget https://github.com/MiniZinc/MiniZincIDE/releases/download/${VERSION}/MiniZincIDE-${VERSION}-bundle-linux-x86_64.tgz
+mkdir -p $MZN_DIR
+tar -xzf MiniZincIDE-${VERSION}-bundle-linux-x86_64.tgz -C $MZN_DIR --strip-components=1
+rm MiniZincIDE-${VERSION}-bundle-linux-x86_64.tgz
+
+# Export path for the runtime
+echo "export PATH=$MZN_DIR/bin:\$PATH" >> ~/.bashrc
+echo "export PATH=$MZN_DIR/bin:\$PATH" >> ~/.profile
+export PATH=$MZN_DIR/bin:$PATH
+
+echo "✅ setup.sh finished running" > setup_log.txt

small_sample_submission.jsonl

@@ -0,0 +1,5 @@
+{"id": "csplib__csplib_001_car_sequencing", "model": "# Data\nat_most = [1, 2, 2, 2, 1]  # The amount of times a property can be present\n# in a group of consecutive timeslots (see next variable)\nper_slots = [2, 3, 3, 5, 5]  # The amount of consecutive timeslots\ndemand = [1, 1, 2, 2, 2, 2]  # The demand per type of car\nrequires = [[1, 0, 1, 1, 0],\n            [0, 0, 0, 1, 0],\n            [0, 1, 0, 0, 1],\n            [0, 1, 0, 1, 0],\n            [1, 0, 1, 0, 0],\n            [1, 1, 0, 0, 0]]  # The properties per type of car\n# End of data\n\n# Import libraries\nfrom cpmpy import *\nimport json\n\n# Parameters\nn_cars = sum(demand)  # The amount of cars to sequence\nn_options = len(at_most)  # The amount of different options\nn_types = len(demand)  # The amount of different car types\nrequires = cpm_array(requires)  # For element constraint\n\n# Decision Variables\nsequence = intvar(0, n_types - 1, shape=n_cars, name=\"sequence\")  # The sequence of car types\nsetup = boolvar(shape=(n_cars, n_options), name=\"setup\")  # Sequence of different options based on the car type\n\n# Model\nmodel = Model()\n\n# The amount of each type of car in the sequence has to be equal to the demand for that type\nmodel += [sum(sequence == t) == demand[t] for t in range(n_types)]\n\n# Make sure that the options in the setup table correspond to those of the car type\nfor s in range(n_cars):\n    model += [setup[s, o] == requires[sequence[s], o] for o in range(n_options)]\n\n# Check that no more than \"at most\" car options are used per \"per_slots\" slots\nfor o in range(n_options):\n    for s in range(n_cars - per_slots[o]):\n        slot_range = range(s, s + per_slots[o])\n        model += (sum(setup[slot_range, o]) <= at_most[o])\n\n# Solve\nmodel.solve()\n\n# Print\nsolution = {\"sequence\": sequence.value().tolist()}\nprint(json.dumps(solution))\n# End of CPMPy script"}
+{"id": "csplib__csplib_002_template_design", "model": "# Data\nn_slots = 9  # The amount of slots on a template\nn_templates = 2  # The amount of templates\nn_var = 7  # The amount of different variations\ndemand = [250, 255, 260, 500, 500, 800, 1100]  # The demand per variation\n# End of data\n\n# Import libraries\nfrom cpmpy import *\nimport json\n\n# Parameters\nub = max(demand)  # The upper bound for the production\n\n# create model\nmodel = Model()\n\n# decision variables\nproduction = intvar(1, ub, shape=n_templates, name=\"production\")\nlayout = intvar(0, n_var, shape=(n_templates, n_var), name=\"layout\")\n\n# all slots are populated in a template\nmodel += all(sum(layout[i]) == n_slots for i in range(n_templates))\n\n# meet demand\nfor var in range(n_var):\n    model += sum(production * layout[:, var]) >= demand[var]\n\n# break symmetry\n# equal demand\nfor i in range(n_var - 1):\n    if demand[i] == demand[i + 1]:\n        model += layout[0, i] <= layout[0, i + 1]\n        for j in range(n_templates - 1):\n            model += (layout[j, i] == layout[j, i + 1]).implies(layout[j + 1, i] <= layout[j + 1, i + 1])\n\n# distinguish templates\nfor i in range(n_templates - 1):\n    model += production[i] <= production[i + 1]\n\n# static symmetry\nfor i in range(n_var - 1):\n    if demand[i] < demand[i + 1]:\n        model += sum(production * layout[:, i]) <= sum(production * layout[:, i + 1])\n\n# minimize number of printed sheets\nmodel.minimize(sum(production))\n\n# Solve\nmodel.solve()\n\n# Print\nsolution = {\"production\": production.value().tolist(), \"layout\": layout.value().tolist()}\nprint(json.dumps(solution))\n# End of CPMPy script"}
+{"id": "csplib__csplib_005_autocorrelation", "model": "# Data\nn = 10  # Length of the binary sequence\n# End of data\n\n# Import libraries\nfrom cpmpy import *\nimport numpy as np\nimport json\n\n\n# periodic auto correlation\ndef PAF(arr, s):\n    # roll the array 's' indices\n    return sum(arr * np.roll(arr, -s))\n\n\n# Decision Variables\nsequence = intvar(-1, 1, shape=n, name=\"sequence\")  # binary sequence\nE = sum([PAF(sequence, s) ** 2 for s in range(1, n)])  # energy value\n\nmodel = Model()\n\n# exclude 0\nmodel += sequence != 0\n\n# minimize sum of squares\nmodel.minimize(E)\n\n# Solve\nmodel.solve()\n\n# Print\nsolution = {\"sequence\": sequence.value().tolist(), \"E\": E.value()}\nprint(json.dumps(solution))\n# End of CPMPy script"}
+{"id": "csplib__csplib_006_golomb_rulers", "model": "# Data\nsize = 10  # Number of marks on the Golomb ruler\n# End of data\n\n# Import libraries\nfrom cpmpy import *\nimport json\n\n# Decision variables\nmarks = intvar(0, size * size, shape=size, name=\"marks\")\nlength = marks[-1]\n\n# Model\nmodel = Model()\n\n# first mark is 0\nmodel += (marks[0] == 0)\n\n# marks must be increasing\nmodel += marks[:-1] < marks[1:]\n\n# golomb constraint\ndiffs = [marks[j] - marks[i] for i in range(0, size - 1) for j in range(i + 1, size)]\nmodel += AllDifferent(diffs)\n\n# Symmetry breaking\nmodel += (marks[size - 1] - marks[size - 2] > marks[1] - marks[0])\nmodel += (diffs[0] < diffs[size - 1])\n\n# find optimal ruler\nmodel.minimize(length)\n\n# Solve\nmodel.solve()\n\n# Print\nsolution = {\"marks\": marks.value().tolist(), \"length\": length.value()}\nprint(json.dumps(solution))\n# End of CPMPy script"}
+{"id": "csplib__csplib_007_all_interval", "model": "# Data\nn = 12  # Number of pitch-classes\n# End of data\n\n# Import libraries\nfrom cpmpy import *\nimport numpy as np\nimport json\n\n# Create the solver\nmodel = Model()\n\n# Declare variables\nx = intvar(0, n - 1, shape=n, name=\"x\")  # Pitch-classes\ndiffs = intvar(1, n - 1, shape=n - 1, name=\"diffs\")  # Intervals\n\n# Constraints\nmodel += [AllDifferent(x),\n          AllDifferent(diffs)]\n\n# Differences between successive values\nmodel += diffs == np.abs(x[1:] - x[:-1])\n\n# Symmetry breaking\nmodel += [x[0] < x[-1]]  # Mirroring array is equivalent solution\nmodel += [diffs[0] < diffs[1]]  # Further symmetry breaking\n\n# Solve\nmodel.solve()\n\n# Print\nsolution = {\"x\": x.value().tolist(), \"diffs\": diffs.value().tolist()}\nprint(json.dumps(solution))\n# End of CPMPy script"}

src/config.py

@@ -8,4 +8,14 @@ DS_SUBMISSIONS_PATH = "submissions"
 DS_RESULTS_PATH = "results"
 
 # leaderboard
-LDB_COLS = ["Submission Name", "Solution Found (%)", "Consistency (%)", "Final Solution Accuracy (%)", "# of Models submitted"]
+LDB_COLS = ["Submission Name", "Solution Found (%)", "Consistency (%)", "Final Solution Accuracy (%)", "# of Models submitted"]
+
+# modelling frameworks
+CPMPY_FRAMEWORK = "CPMpy"
+MINIZINC_FRAMEWORK = "MiniZinc"
+ORTOOLS_FRAMEWORK = "OR-Tools"
+SUPPORTED_FRAMEWORKS = [
+    CPMPY_FRAMEWORK,
+    MINIZINC_FRAMEWORK,
+    ORTOOLS_FRAMEWORK,
+]

src/eval.py

@@ -1,4 +1,5 @@
 # eval.py
+import datetime
 import sys
 import os
 import time
@@ -6,6 +7,8 @@ import json
 import subprocess
 import tempfile
 from pathlib import Path
+
+import minizinc
 from datasets import load_dataset  # Hugging Face datasets library
 from huggingface_hub import HfApi, hf_hub_download, snapshot_download # For user data dataset
 from huggingface_hub.utils import RepositoryNotFoundError
@@ -29,7 +32,8 @@ import subprocess
 import threading
 from pathlib import Path
 
-from src.config import EVAL_SCRIPT_PATH, DATASET_REPO_ID, DS_RESULTS_PATH
+from src.config import EVAL_SCRIPT_PATH, DATASET_REPO_ID, DS_RESULTS_PATH, CPMPY_FRAMEWORK, ORTOOLS_FRAMEWORK, \
+    MINIZINC_FRAMEWORK
 
 
 def run_evaluation(submission_path):
@@ -92,6 +96,68 @@ def extract_json_from_code_output(output: str):
         return None
 
 
+
+def exec_code_minizinc(code: str, timeout_sec):
+    """
+    Executes a MiniZinc model string using the minizinc-python library.
+
+    :param code: The MiniZinc model code as a string.
+    :param timeout_sec: The maximum time to wait for the solver in seconds.
+    :return: A tuple of (success, output, timeout_occured)
+    """
+    successfully_executed = False
+    output = ""
+    timeout_occurred = False
+    timeout_duration = datetime.timedelta(seconds=timeout_sec)
+
+    try:
+        # 1. Create a MiniZinc model instance
+        model = minizinc.Model()
+        model.add_string(code)
+
+        # 2. Find a default solver configured with MiniZinc
+        # You can be more specific, e.g., solver = minizinc.Solver.lookup("gecode")
+        # If the default solver isn't found or suitable, this will raise an error.
+        gecode = minizinc.Solver.lookup("gecode")
+        if gecode is None:
+            raise RuntimeError("No suitable solver found. Please install a MiniZinc solver.")
+
+        # 3. Create an Instance to solve
+        instance = minizinc.Instance(gecode, model)
+
+        # 4. Solve the instance with the specified timeout
+        # The solve() method handles the timeout internally.
+        result = instance.solve(timeout=timeout_duration)
+
+        # 5. Process the result
+        if result.status in {minizinc.Status.SATISFIED, minizinc.Status.OPTIMAL_SOLUTION}:
+            successfully_executed = True
+            output = str(result.solution) if result.solution is not None else ""
+            timeout_occurred = False
+        elif result.status == minizinc.Status.UNKNOWN:
+             successfully_executed = False
+             output = f"Timeout Error: Solver stopped after {timeout_sec} seconds (Status: UNKNOWN)."
+             timeout_occurred = True
+        else:
+            # Handle other non-success statuses (UNSAT, ERROR, etc.)
+            successfully_executed = False
+            output = f"Solving failed. Status: {result.status}"
+            timeout_occurred = False
+
+    except minizinc.MiniZincError as e:
+        # Catch MiniZinc specific errors (e.g., syntax errors, solver not found)
+        successfully_executed = False
+        output = f"MiniZinc Error: {e}"
+        timeout_occurred = False
+    except Exception as e:
+        # Catch other unexpected errors
+        successfully_executed = False
+        output = f"Unexpected Error during MiniZinc execution: {e}"
+        timeout_occurred = False
+
+    return successfully_executed, output, timeout_occurred
+
+
 def exec_code(code: str, timeout=10, modelling_language='cpmpy'):
     """
     Execute the given code and return the output
@@ -108,21 +174,21 @@ def exec_code(code: str, timeout=10, modelling_language='cpmpy'):
     os.makedirs(temp_dir, exist_ok=True)
 
     # write the code to a temporary file
-    suffix = '.__hidden_py__' if modelling_language == "cpmpy" or modelling_language == "or-tools" else '.mzn'
+    suffix = '.__hidden_py__' if modelling_language == CPMPY_FRAMEWORK or modelling_language == ORTOOLS_FRAMEWORK else '.mzn'
     with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix=suffix, dir=temp_dir, encoding='utf-8') as temp_file:
         temp_instance_path = temp_file.name
         temp_file.write(code)
 
     try:
         # execute the code
-        if modelling_language == "cpmpy" or modelling_language == "or-tools":
+        if modelling_language == CPMPY_FRAMEWORK or modelling_language == ORTOOLS_FRAMEWORK:
             command = [sys.executable, temp_instance_path]
             result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, encoding='utf-8')
 
             successfully_executed = (result.returncode == 0)
             output = result.stdout if successfully_executed else result.stderr
             timeout_occurred = False
-        elif modelling_language == "minizinc":
+        elif modelling_language == MINIZINC_FRAMEWORK:
             successfully_executed, output, timeout_occurred = exec_code_minizinc(code, timeout)
         else:
             raise ValueError(f"MODELLING_LANGUAGE not supported: {modelling_language}")
@@ -259,8 +325,14 @@ def main(
                     submitted_models.append(json_obj)
                 except json.JSONDecodeError as e:
                     print(f"  ERROR: Failed to parse JSON object from line: {line}. Error: {e}", flush=True)
+
+        # load metadata file
+        with open(os.path.join(local_submission_dir, submission_path_in_dataset, "metadata.json"), "r", encoding="utf-8") as f:
+            metadata = json.load(f)
+
         print(f"  Loaded {len(submitted_models)} generated models.", flush=True)
 
+
         # Statistics
         total_submitted_models = 0
         models_ran_successfully = 0
@@ -287,7 +359,7 @@ def main(
 
                 summary_f.write("    1. Running submitted model...\n")
 
-                succ_exec, output, timeout_occurred = exec_code(curr_model, timeout=SCRIPT_EXECUTION_TIMEOUT)
+                succ_exec, output, timeout_occurred = exec_code(curr_model, timeout=SCRIPT_EXECUTION_TIMEOUT, modelling_language=metadata["modelling_framework"])
 
                 if timeout_occurred:
                     summary_f.write(f"      - TIMEOUT: Execution time exceeded {SCRIPT_EXECUTION_TIMEOUT} seconds.\n")

src/hf_utils.py

@@ -1,11 +1,12 @@
 """Utilities for interacting with the Hugging Face Hub."""
-
+import io
+import json
 import os
 import shutil
 from pathlib import Path
+
 import pandas as pd
 from huggingface_hub import HfApi, hf_hub_download, list_repo_files
-from huggingface_hub.utils import RepositoryNotFoundError, HFValidationError
 
 from src.config import DATASET_REPO_ID, DS_RESULTS_PATH, DS_SUBMISSIONS_PATH, LDB_COLS
 
@@ -98,7 +99,7 @@ def load_leaderboard_data():
     return pd.DataFrame(leaderboard_entries)
 
 
-def upload_submission(uploaded_file, dir_name, report_file):
+def upload_submission(uploaded_file, dir_name, report_file, model_framework):
     """Upload submission to Hugging Face Dataset."""
     if not HF_API:
         return False, "Hugging Face API not initialized"
@@ -120,6 +121,19 @@ def upload_submission(uploaded_file, dir_name, report_file):
                 repo_type="dataset",
                 commit_message=f"Upload report for submission: {dir_name}"
             )
+
+        # create a file for metadata
+        metadata = {
+            "submission_name": dir_name,
+            "modelling_framework": model_framework,
+        }
+        HF_API.upload_file(
+            path_or_fileobj=io.BytesIO(json.dumps(metadata, indent=4).encode('utf-8')),
+            path_in_repo=f"{submission_path}/metadata.json",
+            repo_id=DATASET_REPO_ID,
+            repo_type="dataset",
+            commit_message=f"Upload metadata for submission: {dir_name}"
+        )
         
         return True, submission_path
     except Exception as e:

src/ui.py

@@ -3,12 +3,16 @@ import json
 import gradio as gr
 from pathlib import Path
 
+from src.config import SUPPORTED_FRAMEWORKS
 from src.hf_utils import load_leaderboard_data, upload_submission, check_name_exists
 from src.eval import start_background_evaluation
 
 
-def handle_upload(submission_name, uploaded_file, report_file, progress=gr.Progress()):
+def handle_upload(submission_name, uploaded_file, report_file, model_framework, progress=gr.Progress()):
     """Handle file upload and start evaluation."""
+    if model_framework not in SUPPORTED_FRAMEWORKS:
+        return f"Unsupported modelling framework: {model_framework}. Supported frameworks are: {', '.join(SUPPORTED_FRAMEWORKS)}"
+
     if not uploaded_file:
         return "No file uploaded. Please upload a valid submission file."
 
@@ -46,7 +50,7 @@ def handle_upload(submission_name, uploaded_file, report_file, progress=gr.Progr
             if not found_one:
                 return "Empty file. Please upload a valid JSONL file."
 
-        success, result = upload_submission(uploaded_file, submission_name, report_file)
+        success, result = upload_submission(uploaded_file, submission_name, report_file, model_framework)
         if not success:
             return f"Upload failed: {result}"
 
@@ -77,7 +81,7 @@ def create_ui():
             "## How to Submit\n"
             "1. **Name your submission**: Choose a unique name for your submission (e.g., `my_cool_submission`). "
             "This name will be used to identify your submission on the leaderboard.\n"
-            "2. **Upload a report (PDF)**: This is optional, but we highly encourage you to upload a report "
+            "2. **Upload a PDF report**: This is optional, but we highly encourage you to upload a report "
             "   (in PDF format) describing your approach. As this is an open competition, we want to avoid submissions "
             "   that just copy the models from the dataset. The report can be a short description of your approach, "
             "   the models you generated, and any other relevant information.\n"
@@ -90,14 +94,14 @@ def create_ui():
             "It may take a few minutes for a submission to be evaluated and appear on the leaderboard.\n"
             "\n\n"
             "## Important Notes\n"
-            "1. **Submission Name**: The submission name must be unique. If you try to upload a submission with a name that already exists, you will receive an error message.\n"
-            "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a PDF.\n"
+            "1. **Submission Name**: The submission name must be different from any existing submission names.\n"
+            "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a `pdf` file.\n"
             "3. **Evaluation Script**: It is highly recommended to use the evaluation script provided [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/user_eval.py) to check your results before submission. You can run the script as follows:\n"
             "   ```bash\n"
             "   python user_eval.py --submission_file path/to/my/submission.jsonl\n"
             "   ```\n"
             "   This will evaluate your submission locally and print the results to the console.\n"
-            "4. **Modelling Language**: For now, only CPMpy is supported. More languages will be added in the future.\n"
+            "4. **Modelling Frameworks**: Currently, the supported modelling frameworks are MiniZinc, CPMpy and OR-Tools. More frameworks will be added.\n"
             "\n\n"
             "### If you have any questions or issues, please feel free to reach out to us TODO\n"
             "---\n"
@@ -113,6 +117,17 @@ def create_ui():
                     interactive=True,
                     info="This name will appear on the leaderboard"
                 )
+                model_framework = gr.Dropdown(
+                    label="Modelling Framework (required)",
+                    choices=SUPPORTED_FRAMEWORKS,
+                    value=None,
+                    multiselect=False,
+                    interactive=True,
+                    info="Select the modelling framework used for your submission.",
+                    allow_custom_value=False,
+                    filterable=False,
+                )
+
                 with gr.Row():
                     report_file = gr.File(
                         label="Upload PDF Report (optional, but recommended)",
@@ -137,7 +152,7 @@ def create_ui():
         # Event handlers
         upload_button.click(
             fn=handle_upload,
-            inputs=[submission_name, submission_file, report_file],
+            inputs=[submission_name, submission_file, report_file, model_framework],
             outputs=[status_box],
             show_progress="full",
         )