add user eval

src/eval.py

@@ -231,7 +231,7 @@ def main(
             print(f"  CRITICAL ERROR - Failed to download submission files: {e_download}", flush=True)
             return 1
 
-        # 2. Load ground-truth dataset (remains the same)
+        # 2. Load ground-truth dataset
         print(f"  Loading ground-truth dataset '{GT_DATASET_NAME}'...", flush=True)
         try:
             gt_dataset = load_dataset(GT_DATASET_NAME, split="train", trust_remote_code=True)

src/hf_utils.py

@@ -36,31 +36,39 @@ def load_leaderboard_data():
             if f.endswith("summary.txt") and f.startswith(DS_RESULTS_PATH + "/")
         ]
         summary_files.sort(reverse=True)
-        
-        for file_path in summary_files:
+
+        submissions = [
+            f for f in repo_files
+            if f.endswith("submission.jsonl") and f.startswith(DS_SUBMISSIONS_PATH + "/")
+        ]
+
+        # for file_path in summary_files:
+        for file_path in submissions:
             dir_name = Path(file_path).parent.name
             if dir_name in processed_result_dirs:
                 continue
-                
+
             processed_result_dirs.add(dir_name)
-            entry = {LDB_COLS[0]: dir_name, LDB_COLS[1]: 'N/A', LDB_COLS[2]: 'N/A', LDB_COLS[3]: 'N/A', LDB_COLS[4]: 0}
-            
+            entry = {LDB_COLS[0]: dir_name,
+                     LDB_COLS[1]: 'In Progress...',
+                     LDB_COLS[2]: 'In Progress...',
+                     LDB_COLS[3]: 'In Progress...',
+                     LDB_COLS[4]: 'In Progress...'}
+
+            # check if summary file exists, otherwise skip
+            if f"{DS_RESULTS_PATH}/{dir_name}/summary.txt" not in repo_files:
+                leaderboard_entries.append(entry)
+                continue
+
             # Download summary file
             temp_dir = os.path.join("temp_hf_downloads", dir_name)
             local_summary_path = hf_hub_download(
                 repo_id=DATASET_REPO_ID,
-                filename=file_path,
+                filename=f"{DS_RESULTS_PATH}/{dir_name}/summary.txt",
                 repo_type="dataset",
                 local_dir=temp_dir,
             )
-            
-            # Count files
-            files_in_result_dir = [
-                f for f in repo_files
-                if f.startswith(f"{DS_RESULTS_PATH}/{dir_name}/") and not f.endswith("/")
-            ]
-            
-            # Parse score from summary
+
             if Path(local_summary_path).exists():
                 with open(local_summary_path, "r", encoding="utf-8") as f:
                     for line in f:
@@ -90,15 +98,13 @@ def load_leaderboard_data():
     return pd.DataFrame(leaderboard_entries)
 
 
-def upload_submission(uploaded_file, dir_name):
+def upload_submission(uploaded_file, dir_name, report_file):
     """Upload submission to Hugging Face Dataset."""
     if not HF_API:
         return False, "Hugging Face API not initialized"
 
     try:
         submission_path = f"{DS_SUBMISSIONS_PATH}/{dir_name}"
-
-        # file_name = os.path.basename(uploaded_file.name)
         HF_API.upload_file(
             path_or_fileobj=uploaded_file,
             path_in_repo=f"{submission_path}/submission.jsonl",
@@ -106,6 +112,14 @@ def upload_submission(uploaded_file, dir_name):
             repo_type="dataset",
             commit_message=f"Upload submission: {dir_name}"
         )
+        if report_file:
+            HF_API.upload_file(
+                path_or_fileobj=report_file,
+                path_in_repo=f"{submission_path}/report.pdf",
+                repo_id=DATASET_REPO_ID,
+                repo_type="dataset",
+                commit_message=f"Upload report for submission: {dir_name}"
+            )
         
         return True, submission_path
     except Exception as e:

src/ui.py

@@ -7,11 +7,14 @@ from src.hf_utils import load_leaderboard_data, upload_submission, check_name_ex
 from src.eval import start_background_evaluation
 
 
-def handle_upload(submission_name, uploaded_file, progress=gr.Progress()):
+def handle_upload(submission_name, uploaded_file, report_file, progress=gr.Progress()):
     """Handle file upload and start evaluation."""
     if not uploaded_file:
         return "No file uploaded. Please upload a valid submission file."
 
+    if report_file and not report_file.name.endswith(".pdf"):
+        return "Invalid report format. Please upload a PDF file."
+
     # normalize the submission name
     submission_name = submission_name.strip().replace(" ", "_").lower()
     # keep only alphanumeric characters and underscores, restrict to 30 characters
@@ -43,7 +46,7 @@ def handle_upload(submission_name, uploaded_file, progress=gr.Progress()):
             if not found_one:
                 return "Empty file. Please upload a valid JSONL file."
 
-        success, result = upload_submission(uploaded_file, submission_name)
+        success, result = upload_submission(uploaded_file, submission_name, report_file)
         if not success:
             return f"Upload failed: {result}"
 
@@ -53,7 +56,11 @@ def handle_upload(submission_name, uploaded_file, progress=gr.Progress()):
         start_background_evaluation(result)
 
         progress(1.0, "Process complete")
-        return f"Upload complete. Evaluation started for: {submission_name}. Refresh the leaderboard to see results. Do not worry if the leaderboard does not update immediately; it may take some time for the results to appear."
+        return (
+            f"✅ Submission '{submission_name}' uploaded successfully!\n"
+            f"Do not worry if the leaderboard does not update immediately; "
+            f"it may take some time for the results to appear (around 5-10 minutes). "
+            f"Feel free to close the tab and check back later.")
 
     except Exception as e:
         return f"Error processing upload: {str(e)}"
@@ -61,8 +68,40 @@ def handle_upload(submission_name, uploaded_file, progress=gr.Progress()):
 
 def create_ui():
     """Create and return Gradio UI."""
-    with gr.Blocks(title="CP-Bench Leaderboard") as demo:
+    with gr.Blocks(title="Welcome to the CP-Bench leaderboard!") as demo:
         gr.Markdown("# CP-Bench Leaderboard")
+        gr.Markdown(
+            "This leaderboard is designed to evaluate LLM-generated constraint models for the problems "
+            "in the [CP-Bench](https://huggingface.co/datasets/kostis-init/CP-Bench) dataset."
+            "\n\n"
+            "## How to Submit\n"
+            "1. **Name your submission**: Choose a unique name for your submission (e.g., `my_cool_submission`). "
+            "This name will be used to identify your submission on the leaderboard.\n"
+            "2. **Upload a report (PDF)**: This is optional, but we highly encourage you to upload a report "
+            "   (in PDF format) describing your approach. As this is an open competition, we want to avoid submissions "
+            "   that just copy the models from the dataset. The report can be a short description of your approach, "
+            "   the models you generated, and any other relevant information.\n"
+            "3. **Upload your submission**: Upload a **single** `.jsonl` file containing the generated models. "
+            "   **Each line in the file should be a JSON object with two keys: `id` and `model`.**\n"
+            "   * `id`: The ID of the problem exactly as it appears in the original dataset (e.g., `csplib__csplib_001_car_sequencing`).\n"
+            "   * `model`: The generated model for the problem (as a string representing runnable code). Make sure that it eventually outputs the solution as a json with key(s) as described in the `decision_variables` entry and values as would be expected in the problem. This is part of the evaluation as well: unexpected keys, or value types are considered incorrect. This is because our automatic evaluation is based on the solution printed by the submitted models.\n" 
+            "   * An example submission file can be found [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/sample_submission.jsonl).\n"
+            "3. **Check the leaderboard**: After uploading, you can check the leaderboard to see your results. "
+            "It may take a few minutes for a submission to be evaluated and appear on the leaderboard.\n"
+            "\n\n"
+            "## Important Notes\n"
+            "1. **Submission Name**: The submission name must be unique. If you try to upload a submission with a name that already exists, you will receive an error message.\n"
+            "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a PDF.\n"
+            "3. **Evaluation Script**: It is highly recommended to use the evaluation script provided [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/user_eval.py) to check your results before submission. You can run the script as follows:\n"
+            "   ```bash\n"
+            "   python user_eval.py --submission_file path/to/my/submission.jsonl\n"
+            "   ```\n"
+            "   This will evaluate your submission locally and print the results to the console.\n"
+            "4. **Modelling Language**: For now, only CPMpy is supported. More languages will be added in the future.\n"
+            "\n\n"
+            "### If you have any questions or issues, please feel free to reach out to us TODO\n"
+            "---\n"
+        )
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -74,18 +113,31 @@ def create_ui():
                     interactive=True,
                     info="This name will appear on the leaderboard"
                 )
-                upload_button = gr.UploadButton("Click to Upload Submission", file_count="single")
+                with gr.Row():
+                    report_file = gr.File(
+                        label="Upload PDF Report (optional, but recommended)",
+                        file_types=[".pdf"],
+                        file_count="single",
+                        interactive=True,
+                    )
+                    submission_file = gr.File(
+                        label="Upload Submission File (required, .jsonl)",
+                        file_types=[".jsonl"],
+                        file_count="single",
+                        interactive=True,
+                    )
+                upload_button = gr.Button("Click to Upload Submission")
                 status_box = gr.Textbox(label="Status", interactive=False)
 
-            with gr.Column(scale=3):
+            with gr.Column(scale=2):
                 gr.Markdown("## 🏆 Results Leaderboard")
-                leaderboard = gr.DataFrame(value=load_leaderboard_data, label="Leaderboard", interactive=False)
+                leaderboard = gr.DataFrame(value=load_leaderboard_data, interactive=False)
                 refresh_button = gr.Button("🔄 Refresh Leaderboard")
 
         # Event handlers
-        upload_button.upload(
+        upload_button.click(
             fn=handle_upload,
-            inputs=[submission_name, upload_button],
+            inputs=[submission_name, submission_file, report_file],
             outputs=[status_box],
             show_progress="full",
         )
@@ -96,4 +148,4 @@ def create_ui():
             outputs=[leaderboard]
         )
 
-    return demo
+    return demo

user_eval.py

@@ -0,0 +1,293 @@
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+import click
+from pathlib import Path
+
+from datasets import load_dataset
+
+GT_DATASET_NAME = "kostis-init/CP-Bench"
+GT_PROBLEM_NAME_COLUMN = "id"
+GT_MODEL_CODE_COLUMN = "model"
+
+
+def exec_code(code: str, timeout=10, modelling_language='cpmpy'):
+    """
+    Execute the given code and return the output
+
+    :param code: The code to execute as a string
+    :param timeout: The maximum time to wait for the code to execute in seconds
+    :param modelling_language: The language to use for execution (cpmpy, minizinc, or-tools)
+    :return: A tuple of (success, output, timeout_occured)
+    """
+
+    # create a temp directory to store the temporary file
+    temp_dir_name = "_temp_dir_for_exec_code"
+    temp_dir = os.path.join(os.getcwd(), temp_dir_name)
+    os.makedirs(temp_dir, exist_ok=True)
+
+    # write the code to a temporary file
+    suffix = '.__hidden_py__' if modelling_language == "cpmpy" or modelling_language == "or-tools" else '.mzn'
+    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix=suffix, dir=temp_dir, encoding='utf-8') as temp_file:
+        temp_instance_path = temp_file.name
+        temp_file.write(code)
+
+    try:
+        # execute the code
+        if modelling_language == "cpmpy" or modelling_language == "or-tools":
+            command = [sys.executable, temp_instance_path]
+            result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, encoding='utf-8')
+
+            successfully_executed = (result.returncode == 0)
+            output = result.stdout if successfully_executed else result.stderr
+            timeout_occurred = False
+        # elif modelling_language == "minizinc":
+        #     successfully_executed, output, timeout_occurred = exec_code_minizinc(code, timeout)
+        else:
+            raise ValueError(f"MODELLING_LANGUAGE not supported: {modelling_language}")
+
+    except subprocess.TimeoutExpired as e:
+        successfully_executed = False
+        output = f"Timeout Error: Execution time exceeded {timeout} seconds"
+        timeout_occurred = True
+    except Exception as e:
+        successfully_executed = False
+        output = f"Error: {e}"
+        timeout_occurred = False
+
+    os.remove(temp_instance_path)
+
+    return successfully_executed, output, timeout_occurred
+
+
+def validate_submission_file(file_path: Path) -> tuple[bool, str]:
+    """Validate the submission file format and content.
+    
+    Args:
+        file_path: Path to the submission file
+    
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    if not file_path.exists():
+        return False, f"File {file_path} does not exist"
+
+    if not file_path.name.endswith('.jsonl'):
+        return False, "Invalid file format. Please provide a .jsonl file"
+
+    try:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            found_one = False
+            for line_num, line in enumerate(file, 1):
+                found_one = True
+                try:
+                    json_object = json.loads(line)
+                    if not all(key in json_object for key in ["id", "model"]):
+                        return False, f"Line {line_num}: Missing required keys 'id' and/or 'model'"
+                except json.JSONDecodeError:
+                    return False, f"Line {line_num}: Invalid JSON format"
+
+            if not found_one:
+                return False, "Empty file. Please provide a valid JSONL file"
+
+    except Exception as e:
+        return False, f"Error reading file: {str(e)}"
+
+    return True, "File is valid"
+
+
+def extract_json_from_code_output(output: str):
+    try:
+        start_index = output.find('{')
+        end_index = output.rfind('}') + 1
+        # Extract the JSON part
+        json_part = output[start_index:end_index]
+        return json.loads(json_part)
+    except json.JSONDecodeError:
+        return None
+
+
+def add_constraints_as_string(solution):
+    """Generate constraints as a string to be added to the original script."""
+    constraints = ""
+    if solution:  # Ensure solution is not None
+        for key, value in solution.items():
+            # Basic escaping for string values if they occur, though typically solutions are numeric/boolean
+            if isinstance(value, str):
+                constraints += f"\nmodel += ({key} == \"{value}\")"
+            else:
+                constraints += f"\nmodel += ({key} == {value})"
+    return constraints
+
+
+def get_modified_script(script_content, solution):
+    """Add constraints to the script content and self-consistency checks."""
+    constraints_str = add_constraints_as_string(solution)
+    modified_script = f"{script_content}\n{constraints_str}"
+    modified_script += """
+# Print the absolute path of the current directory along with the script name
+import os
+print(os.path.abspath(__file__))
+
+# Keep old objective
+old_objective = None
+if hasattr(model, 'objective_is_min') and model.objective_is_min is not None:
+    old_objective = model.objective_value()
+
+# Check self-consistency
+if not model.solve():
+    print('ERROR: The model is unsatisfiable with the self-consistency constraints')
+else:
+    print('SUCCESS: Model is consistent')
+
+# Check if the objective value is the same
+if old_objective is None:
+    print('SUCCESS: No objective defined')
+elif model.objective_value() != old_objective:
+    print('ERROR: The objective value has changed')
+else:
+    print('SUCCESS: Objective value is consistent')
+"""
+    return modified_script
+
+@click.command()
+@click.option('--submission_file', required=True, type=click.Path(exists=True, path_type=Path),
+              help='Path to the submission JSONL file')
+def main(submission_file: Path):
+    """Evaluate a submission file for the CP-Bench competition."""
+    is_valid, message = validate_submission_file(submission_file)
+    if not is_valid:
+        click.echo(f"Error: {message}")
+        return
+
+    click.echo("Starting evaluation...")
+
+    # load generated models from jsonl to memory
+    print(f"  Loading models from file...", flush=True)
+    submitted_models = []
+    with open(submission_file, "r", encoding="utf-8") as f:
+        for line in f:
+            try:
+                json_obj = json.loads(line)
+                submitted_models.append(json_obj)
+            except json.JSONDecodeError as e:
+                print(f"  ERROR: Failed to parse JSON object from line: {line}. Error: {e}", flush=True)
+    print(f"  Loaded {len(submitted_models)} generated models.", flush=True)
+
+
+    # eval
+    total_submitted_models = 0
+    models_ran_successfully = 0
+    consistency_checks_passed = 0
+    objective_checks_passed = 0
+    all_checks_passed = 0
+    gt_models_found = 0
+    
+    # Load ground-truth models
+    print(f"  Loading ground-truth dataset '{GT_DATASET_NAME}'...", flush=True)
+    try:
+        gt_dataset = load_dataset(GT_DATASET_NAME, split="train", trust_remote_code=True)
+        ground_truth_models = {
+            item[GT_PROBLEM_NAME_COLUMN]: item[GT_MODEL_CODE_COLUMN]
+            for item in gt_dataset if
+            GT_PROBLEM_NAME_COLUMN in item and GT_MODEL_CODE_COLUMN in item and item[GT_MODEL_CODE_COLUMN]
+        }
+        if not ground_truth_models: raise ValueError("No models in GT dataset.")
+        print(f"  Loaded {len(ground_truth_models)} ground-truth models.", flush=True)
+    except Exception as e_gt:
+        print(f"  CRITICAL ERROR - Failed to load ground-truth dataset: {e_gt}", flush=True)
+        return
+
+    # Iterate through downloaded submitted models
+    for submitted_model in submitted_models:
+        curr_model = submitted_model[GT_MODEL_CODE_COLUMN]
+
+        total_submitted_models += 1
+        problem_name = submitted_model[GT_PROBLEM_NAME_COLUMN]
+        print(f"\n  Processing model: {problem_name}", flush=True)
+        print(f"\n--- Model: {problem_name} ---\n")
+
+        print("    1. Running submitted model...\n")
+
+        succ_exec, output, timeout_occurred = exec_code(curr_model, timeout=60)
+
+        if timeout_occurred:
+            print(f"      - TIMEOUT: Execution time exceeded 60 seconds.\n")
+            continue
+        if not succ_exec:
+            print(f"      - FAILED: Execution failed with error: {output}\n")
+            continue
+        if output is None or not output.strip():
+            print(f"      - FAILED: No output from execution.\n")
+            continue
+        # Attempt to extract JSON from stdout
+        generated_solution = extract_json_from_code_output(output)
+        if generated_solution is None:
+            print(f"      - FAILED: Could not extract JSON solution from output: {output}\n")
+            continue
+
+        models_ran_successfully += 1
+        print(f"      - SUCCESS: Got solution: {generated_solution}\n")
+
+        print(f"    2. Checking against ground-truth for '{problem_name}'...\n")
+        if problem_name not in ground_truth_models:
+            print(f"      - FAILED: Ground-truth model for '{problem_name}' not found in dataset.\n")
+            continue
+        gt_models_found += 1
+        ground_truth_script_content = ground_truth_models[problem_name]
+        print("      - SUCCESS: Found ground-truth model.\n")
+
+        print("    3. Performing self-consistency check on ground-truth model...\n")
+        modified_gt_script = get_modified_script(ground_truth_script_content, generated_solution)
+
+        try:
+            with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding='utf-8') as tmp_file:
+                tmp_file.write(modified_gt_script)
+                tmp_file_path_str = tmp_file.name
+
+            gt_check_result = subprocess.run(
+                [sys.executable, tmp_file_path_str],
+                capture_output=True, text=True, timeout=60, encoding='utf-8',
+            )
+            os.unlink(tmp_file_path_str)
+
+            gt_stdout = gt_check_result.stdout
+            if "SUCCESS: Model is consistent" in gt_stdout:
+                print("      - CONSISTENCY: PASSED\n")
+                consistency_checks_passed += 1
+            else:
+                print(
+                    "      - CONSISTENCY: FAILED (Details in logs or stdout)\n")
+
+            if "SUCCESS: No objective defined" in gt_stdout or "SUCCESS: Objective value is consistent" in gt_stdout:
+                print("      - OBJECTIVE: PASSED\n")
+                objective_checks_passed += 1
+            else:
+                print("      - OBJECTIVE: FAILED (Details in logs or stdout)\n")
+
+            if "SUCCESS: Model is consistent" in gt_stdout and (
+                    "SUCCESS: No objective defined" in gt_stdout or "SUCCESS: Objective value is consistent" in gt_stdout):
+                print("      - SELF-CONSISTENCY CHECK: PASSED fully\n")
+                all_checks_passed += 1
+
+        except Exception as e_gt_run:
+            print(f"      - SELF-CONSISTENCY CHECK: FAILED (Error: {e_gt_run})\n")
+
+    # Final statistics (write to summary_f)
+    print("\n" + "=" * 30 + "\n")
+    print("Overall Evaluation:\n")
+    print(f"  Total Submitted Models Parsed: {total_submitted_models}\n")
+    print(f"  Execution perc: {models_ran_successfully / len(ground_truth_models) * 100:.2f}%\n")
+    print(f"  Consistency perc: {consistency_checks_passed / len(ground_truth_models) * 100:.2f}%\n")
+    print(f"  Objective perc: {objective_checks_passed / len(ground_truth_models) * 100:.2f}%\n")
+    print(f"  Final Solution Accuracy perc: {all_checks_passed / len(ground_truth_models) * 100:.2f}%\n")
+    print("-" * 30 + "\n")
+
+    click.echo("Evaluation complete!")
+
+
+if __name__ == "__main__":
+    main()