deb

app.py

@@ -5,5 +5,5 @@ print(f"[{time.time()}] app.py: Script execution has STARTED.")
 import os
 import subprocess
 import src.ui
-print(f"[{time.time()}] app.py: Imported all.")
+print(f"[{time.time()}] app.py: Imported all.", flush=True)
 # DO NOT add any other imports or code yet.

src/ui.py

@@ -2,166 +2,165 @@ import json
 
 import gradio as gr
 from pathlib import Path
-print("importedddd1 1", flush=True)
 from src.config import SUPPORTED_FRAMEWORKS
 
 from src.hf_utils import load_leaderboard_data, upload_submission, check_name_exists
 from src.eval import start_background_evaluation
-print("importedddd")
-#
-# def handle_upload(submission_name, uploaded_file, report_file, model_framework, progress=gr.Progress()):
-#     """Handle file upload and start evaluation."""
-#     if model_framework not in SUPPORTED_FRAMEWORKS:
-#         return f"Unsupported modelling framework: {model_framework}. Supported frameworks are: {', '.join(SUPPORTED_FRAMEWORKS)}"
-#
-#     if not uploaded_file:
-#         return "No file uploaded. Please upload a valid submission file."
-#
-#     if report_file and not report_file.name.endswith(".pdf"):
-#         return "Invalid report format. Please upload a PDF file."
-#
-#     # normalize the submission name
-#     submission_name = submission_name.strip().replace(" ", "_").lower()
-#     # keep only alphanumeric characters and underscores, restrict to 30 characters
-#     submission_name = "".join(
-#         c for c in submission_name if c.isalnum() or c == "_"
-#     )[:30]
-#
-#     if not submission_name or submission_name.strip() == "":
-#         return "Submission name is required"
-#
-#     if check_name_exists(submission_name):
-#         return f"Submission name '{submission_name}' already exists. Please choose a different name."
-#
-#     try:
-#         progress(0.3, "Uploading to Hugging Face...")
-#
-#         # Check if the file is a valid JSONL file
-#         if not uploaded_file.name.endswith(".jsonl"):
-#             return "Invalid file format. Please upload a .jsonl file."
-#
-#         # Check that the keys in the JSONL file are correct ('id' and 'model')
-#         with open(uploaded_file.name, "r") as file:
-#             found_one = False
-#             for line in file:
-#                 found_one = True
-#                 json_object = json.loads(line)
-#                 if not all(key in json_object for key in ["id", "model"]):
-#                     return "Invalid content. Each line must contain 'id' and 'model' keys."
-#             if not found_one:
-#                 return "Empty file. Please upload a valid JSONL file."
-#
-#         success, result = upload_submission(uploaded_file, submission_name, report_file, model_framework)
-#         if not success:
-#             return f"Upload failed: {result}"
-#
-#         progress(0.7, "Starting evaluation...")
-#
-#         # Start evaluation
-#         start_background_evaluation(result)
-#
-#         progress(1.0, "Process complete")
-#         return (
-#             f"✅ Submission '{submission_name}' uploaded successfully!\n"
-#             f"Do not worry if the leaderboard does not update immediately; "
-#             f"it may take some time for the results to appear (around 5-10 minutes). "
-#             f"Feel free to close the tab and check back later.")
-#
-#     except Exception as e:
-#         return f"Error processing upload: {str(e)}"
-#
-#
-# def create_ui():
-#     """Create and return Gradio UI."""
-#     with gr.Blocks(title="Welcome to the CP-Bench leaderboard!") as demo:
-#         gr.Markdown("# CP-Bench Leaderboard")
-#         gr.Markdown(
-#             "This leaderboard is designed to evaluate LLM-generated constraint models for the problems "
-#             "in the [CP-Bench](https://huggingface.co/datasets/kostis-init/CP-Bench) dataset."
-#             "\n\n"
-#             "## How to Submit\n"
-#             "1. **Name your submission**: Choose a unique name for your submission (e.g., `my_cool_submission`). "
-#             "This name will be used to identify your submission on the leaderboard.\n"
-#             "2. **Upload a PDF report**: This is optional, but we highly encourage you to upload a report "
-#             "   (in PDF format) describing your approach. As this is an open competition, we want to avoid submissions "
-#             "   that just copy the models from the dataset. The report can be a short description of your approach, "
-#             "   the models you generated, and any other relevant information.\n"
-#             "3. **Upload your submission**: Upload a **single** `.jsonl` file containing the generated models. "
-#             "   **Each line in the file should be a JSON object with two keys: `id` and `model`.**\n"
-#             "   * `id`: The ID of the problem exactly as it appears in the original dataset (e.g., `csplib__csplib_001_car_sequencing`).\n"
-#             "   * `model`: The generated model for the problem (as a string representing runnable code). Make sure that it eventually outputs the solution as a json with key(s) as described in the `decision_variables` entry and values as would be expected in the problem. This is part of the evaluation as well: unexpected keys, or value types are considered incorrect. This is because our automatic evaluation is based on the solution printed by the submitted models.\n"
-#             "   * An example submission file can be found [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/sample_submission.jsonl).\n"
-#             "3. **Check the leaderboard**: After uploading, you can check the leaderboard to see your results. "
-#             "It may take a few minutes for a submission to be evaluated and appear on the leaderboard.\n"
-#             "\n\n"
-#             "## Important Notes\n"
-#             "1. **Submission Name**: The submission name must be different from any existing submission names.\n"
-#             "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a `pdf` file.\n"
-#             "3. **Evaluation Script**: It is highly recommended to use the evaluation script provided [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/user_eval.py) to check your results before submission. You can run the script as follows:\n"
-#             "   ```bash\n"
-#             "   python user_eval.py --submission_file path/to/my/submission.jsonl\n"
-#             "   ```\n"
-#             "   This will evaluate your submission locally and print the results to the console.\n"
-#             "4. **Modelling Frameworks**: Currently, the supported modelling frameworks are MiniZinc, CPMpy and OR-Tools. More frameworks will be added.\n"
-#             "\n\n"
-#             "### If you have any questions or issues, please feel free to reach out to us TODO\n"
-#             "---\n"
-#         )
-#
-#         with gr.Row():
-#             with gr.Column(scale=1):
-#                 gr.Markdown("## 📤 Upload Submission")
-#
-#                 submission_name = gr.Textbox(
-#                     label="Submission Name (required)",
-#                     placeholder="Enter a unique name for your submission",
-#                     interactive=True,
-#                     info="This name will appear on the leaderboard"
-#                 )
-#                 model_framework = gr.Dropdown(
-#                     label="Modelling Framework (required)",
-#                     choices=SUPPORTED_FRAMEWORKS,
-#                     value=None,
-#                     multiselect=False,
-#                     interactive=True,
-#                     info="Select the modelling framework used for your submission.",
-#                     allow_custom_value=False,
-#                     filterable=False,
-#                 )
-#
-#                 with gr.Row():
-#                     report_file = gr.File(
-#                         label="Upload PDF Report (optional, but recommended)",
-#                         file_types=[".pdf"],
-#                         file_count="single",
-#                         interactive=True,
-#                     )
-#                     submission_file = gr.File(
-#                         label="Upload Submission File (required, .jsonl)",
-#                         file_types=[".jsonl"],
-#                         file_count="single",
-#                         interactive=True,
-#                     )
-#                 upload_button = gr.Button("Click to Upload Submission")
-#                 status_box = gr.Textbox(label="Status", interactive=False)
-#
-#             with gr.Column(scale=2):
-#                 gr.Markdown("## 🏆 Results Leaderboard")
-#                 leaderboard = gr.DataFrame(value=load_leaderboard_data, interactive=False)
-#                 refresh_button = gr.Button("🔄 Refresh Leaderboard")
-#
-#         # Event handlers
-#         upload_button.click(
-#             fn=handle_upload,
-#             inputs=[submission_name, submission_file, report_file, model_framework],
-#             outputs=[status_box],
-#             show_progress="full",
-#         )
-#
-#         refresh_button.click(
-#             fn=load_leaderboard_data,
-#             inputs=None,
-#             outputs=[leaderboard]
-#         )
-#
-#     return demo
+
+
+def handle_upload(submission_name, uploaded_file, report_file, model_framework, progress=gr.Progress()):
+    """Handle file upload and start evaluation."""
+    if model_framework not in SUPPORTED_FRAMEWORKS:
+        return f"Unsupported modelling framework: {model_framework}. Supported frameworks are: {', '.join(SUPPORTED_FRAMEWORKS)}"
+
+    if not uploaded_file:
+        return "No file uploaded. Please upload a valid submission file."
+
+    if report_file and not report_file.name.endswith(".pdf"):
+        return "Invalid report format. Please upload a PDF file."
+
+    # normalize the submission name
+    submission_name = submission_name.strip().replace(" ", "_").lower()
+    # keep only alphanumeric characters and underscores, restrict to 30 characters
+    submission_name = "".join(
+        c for c in submission_name if c.isalnum() or c == "_"
+    )[:30]
+
+    if not submission_name or submission_name.strip() == "":
+        return "Submission name is required"
+
+    if check_name_exists(submission_name):
+        return f"Submission name '{submission_name}' already exists. Please choose a different name."
+
+    try:
+        progress(0.3, "Uploading to Hugging Face...")
+
+        # Check if the file is a valid JSONL file
+        if not uploaded_file.name.endswith(".jsonl"):
+            return "Invalid file format. Please upload a .jsonl file."
+
+        # Check that the keys in the JSONL file are correct ('id' and 'model')
+        with open(uploaded_file.name, "r") as file:
+            found_one = False
+            for line in file:
+                found_one = True
+                json_object = json.loads(line)
+                if not all(key in json_object for key in ["id", "model"]):
+                    return "Invalid content. Each line must contain 'id' and 'model' keys."
+            if not found_one:
+                return "Empty file. Please upload a valid JSONL file."
+
+        success, result = upload_submission(uploaded_file, submission_name, report_file, model_framework)
+        if not success:
+            return f"Upload failed: {result}"
+
+        progress(0.7, "Starting evaluation...")
+
+        # Start evaluation
+        start_background_evaluation(result)
+
+        progress(1.0, "Process complete")
+        return (
+            f"✅ Submission '{submission_name}' uploaded successfully!\n"
+            f"Do not worry if the leaderboard does not update immediately; "
+            f"it may take some time for the results to appear (around 5-10 minutes). "
+            f"Feel free to close the tab and check back later.")
+
+    except Exception as e:
+        return f"Error processing upload: {str(e)}"
+
+
+def create_ui():
+    """Create and return Gradio UI."""
+    with gr.Blocks(title="Welcome to the CP-Bench leaderboard!") as demo:
+        gr.Markdown("# CP-Bench Leaderboard")
+        gr.Markdown(
+            "This leaderboard is designed to evaluate LLM-generated constraint models for the problems "
+            "in the [CP-Bench](https://huggingface.co/datasets/kostis-init/CP-Bench) dataset."
+            "\n\n"
+            "## How to Submit\n"
+            "1. **Name your submission**: Choose a unique name for your submission (e.g., `my_cool_submission`). "
+            "This name will be used to identify your submission on the leaderboard.\n"
+            "2. **Upload a PDF report**: This is optional, but we highly encourage you to upload a report "
+            "   (in PDF format) describing your approach. As this is an open competition, we want to avoid submissions "
+            "   that just copy the models from the dataset. The report can be a short description of your approach, "
+            "   the models you generated, and any other relevant information.\n"
+            "3. **Upload your submission**: Upload a **single** `.jsonl` file containing the generated models. "
+            "   **Each line in the file should be a JSON object with two keys: `id` and `model`.**\n"
+            "   * `id`: The ID of the problem exactly as it appears in the original dataset (e.g., `csplib__csplib_001_car_sequencing`).\n"
+            "   * `model`: The generated model for the problem (as a string representing runnable code). Make sure that it eventually outputs the solution as a json with key(s) as described in the `decision_variables` entry and values as would be expected in the problem. This is part of the evaluation as well: unexpected keys, or value types are considered incorrect. This is because our automatic evaluation is based on the solution printed by the submitted models.\n"
+            "   * An example submission file can be found [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/sample_submission.jsonl).\n"
+            "3. **Check the leaderboard**: After uploading, you can check the leaderboard to see your results. "
+            "It may take a few minutes for a submission to be evaluated and appear on the leaderboard.\n"
+            "\n\n"
+            "## Important Notes\n"
+            "1. **Submission Name**: The submission name must be different from any existing submission names.\n"
+            "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a `pdf` file.\n"
+            "3. **Evaluation Script**: It is highly recommended to use the evaluation script provided [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/user_eval.py) to check your results before submission. You can run the script as follows:\n"
+            "   ```bash\n"
+            "   python user_eval.py --submission_file path/to/my/submission.jsonl\n"
+            "   ```\n"
+            "   This will evaluate your submission locally and print the results to the console.\n"
+            "4. **Modelling Frameworks**: Currently, the supported modelling frameworks are MiniZinc, CPMpy and OR-Tools. More frameworks will be added.\n"
+            "\n\n"
+            "### If you have any questions or issues, please feel free to reach out to us TODO\n"
+            "---\n"
+        )
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown("## 📤 Upload Submission")
+
+                submission_name = gr.Textbox(
+                    label="Submission Name (required)",
+                    placeholder="Enter a unique name for your submission",
+                    interactive=True,
+                    info="This name will appear on the leaderboard"
+                )
+                model_framework = gr.Dropdown(
+                    label="Modelling Framework (required)",
+                    choices=SUPPORTED_FRAMEWORKS,
+                    value=None,
+                    multiselect=False,
+                    interactive=True,
+                    info="Select the modelling framework used for your submission.",
+                    allow_custom_value=False,
+                    filterable=False,
+                )
+
+                with gr.Row():
+                    report_file = gr.File(
+                        label="Upload PDF Report (optional, but recommended)",
+                        file_types=[".pdf"],
+                        file_count="single",
+                        interactive=True,
+                    )
+                    submission_file = gr.File(
+                        label="Upload Submission File (required, .jsonl)",
+                        file_types=[".jsonl"],
+                        file_count="single",
+                        interactive=True,
+                    )
+                upload_button = gr.Button("Click to Upload Submission")
+                status_box = gr.Textbox(label="Status", interactive=False)
+
+            with gr.Column(scale=2):
+                gr.Markdown("## 🏆 Results Leaderboard")
+                leaderboard = gr.DataFrame(value=load_leaderboard_data, interactive=False)
+                refresh_button = gr.Button("🔄 Refresh Leaderboard")
+
+        # Event handlers
+        upload_button.click(
+            fn=handle_upload,
+            inputs=[submission_name, submission_file, report_file, model_framework],
+            outputs=[status_box],
+            show_progress="full",
+        )
+
+        refresh_button.click(
+            fn=load_leaderboard_data,
+            inputs=None,
+            outputs=[leaderboard]
+        )
+
+    return demo