deb

src/ui.py

@@ -2,165 +2,166 @@ import json
 
 import gradio as gr
 from pathlib import Path
-
+print("importedddd1 1", flush=True)
 from src.config import SUPPORTED_FRAMEWORKS
-from src.hf_utils import load_leaderboard_data, upload_submission, check_name_exists
-from src.eval import start_background_evaluation
-
-
-def handle_upload(submission_name, uploaded_file, report_file, model_framework, progress=gr.Progress()):
-    """Handle file upload and start evaluation."""
-    if model_framework not in SUPPORTED_FRAMEWORKS:
-        return f"Unsupported modelling framework: {model_framework}. Supported frameworks are: {', '.join(SUPPORTED_FRAMEWORKS)}"
-
-    if not uploaded_file:
-        return "No file uploaded. Please upload a valid submission file."
-
-    if report_file and not report_file.name.endswith(".pdf"):
-        return "Invalid report format. Please upload a PDF file."
-
-    # normalize the submission name
-    submission_name = submission_name.strip().replace(" ", "_").lower()
-    # keep only alphanumeric characters and underscores, restrict to 30 characters
-    submission_name = "".join(
-        c for c in submission_name if c.isalnum() or c == "_"
-    )[:30]
-
-    if not submission_name or submission_name.strip() == "":
-        return "Submission name is required"
-
-    if check_name_exists(submission_name):
-        return f"Submission name '{submission_name}' already exists. Please choose a different name."
-
-    try:
-        progress(0.3, "Uploading to Hugging Face...")
-
-        # Check if the file is a valid JSONL file
-        if not uploaded_file.name.endswith(".jsonl"):
-            return "Invalid file format. Please upload a .jsonl file."
-
-        # Check that the keys in the JSONL file are correct ('id' and 'model')
-        with open(uploaded_file.name, "r") as file:
-            found_one = False
-            for line in file:
-                found_one = True
-                json_object = json.loads(line)
-                if not all(key in json_object for key in ["id", "model"]):
-                    return "Invalid content. Each line must contain 'id' and 'model' keys."
-            if not found_one:
-                return "Empty file. Please upload a valid JSONL file."
-
-        success, result = upload_submission(uploaded_file, submission_name, report_file, model_framework)
-        if not success:
-            return f"Upload failed: {result}"
-
-        progress(0.7, "Starting evaluation...")
-
-        # Start evaluation
-        start_background_evaluation(result)
-
-        progress(1.0, "Process complete")
-        return (
-            f"✅ Submission '{submission_name}' uploaded successfully!\n"
-            f"Do not worry if the leaderboard does not update immediately; "
-            f"it may take some time for the results to appear (around 5-10 minutes). "
-            f"Feel free to close the tab and check back later.")
-
-    except Exception as e:
-        return f"Error processing upload: {str(e)}"
-
-
-def create_ui():
-    """Create and return Gradio UI."""
-    with gr.Blocks(title="Welcome to the CP-Bench leaderboard!") as demo:
-        gr.Markdown("# CP-Bench Leaderboard")
-        gr.Markdown(
-            "This leaderboard is designed to evaluate LLM-generated constraint models for the problems "
-            "in the [CP-Bench](https://huggingface.co/datasets/kostis-init/CP-Bench) dataset."
-            "\n\n"
-            "## How to Submit\n"
-            "1. **Name your submission**: Choose a unique name for your submission (e.g., `my_cool_submission`). "
-            "This name will be used to identify your submission on the leaderboard.\n"
-            "2. **Upload a PDF report**: This is optional, but we highly encourage you to upload a report "
-            "   (in PDF format) describing your approach. As this is an open competition, we want to avoid submissions "
-            "   that just copy the models from the dataset. The report can be a short description of your approach, "
-            "   the models you generated, and any other relevant information.\n"
-            "3. **Upload your submission**: Upload a **single** `.jsonl` file containing the generated models. "
-            "   **Each line in the file should be a JSON object with two keys: `id` and `model`.**\n"
-            "   * `id`: The ID of the problem exactly as it appears in the original dataset (e.g., `csplib__csplib_001_car_sequencing`).\n"
-            "   * `model`: The generated model for the problem (as a string representing runnable code). Make sure that it eventually outputs the solution as a json with key(s) as described in the `decision_variables` entry and values as would be expected in the problem. This is part of the evaluation as well: unexpected keys, or value types are considered incorrect. This is because our automatic evaluation is based on the solution printed by the submitted models.\n" 
-            "   * An example submission file can be found [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/sample_submission.jsonl).\n"
-            "3. **Check the leaderboard**: After uploading, you can check the leaderboard to see your results. "
-            "It may take a few minutes for a submission to be evaluated and appear on the leaderboard.\n"
-            "\n\n"
-            "## Important Notes\n"
-            "1. **Submission Name**: The submission name must be different from any existing submission names.\n"
-            "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a `pdf` file.\n"
-            "3. **Evaluation Script**: It is highly recommended to use the evaluation script provided [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/user_eval.py) to check your results before submission. You can run the script as follows:\n"
-            "   ```bash\n"
-            "   python user_eval.py --submission_file path/to/my/submission.jsonl\n"
-            "   ```\n"
-            "   This will evaluate your submission locally and print the results to the console.\n"
-            "4. **Modelling Frameworks**: Currently, the supported modelling frameworks are MiniZinc, CPMpy and OR-Tools. More frameworks will be added.\n"
-            "\n\n"
-            "### If you have any questions or issues, please feel free to reach out to us TODO\n"
-            "---\n"
-        )
-
-        with gr.Row():
-            with gr.Column(scale=1):
-                gr.Markdown("## 📤 Upload Submission")
-
-                submission_name = gr.Textbox(
-                    label="Submission Name (required)",
-                    placeholder="Enter a unique name for your submission",
-                    interactive=True,
-                    info="This name will appear on the leaderboard"
-                )
-                model_framework = gr.Dropdown(
-                    label="Modelling Framework (required)",
-                    choices=SUPPORTED_FRAMEWORKS,
-                    value=None,
-                    multiselect=False,
-                    interactive=True,
-                    info="Select the modelling framework used for your submission.",
-                    allow_custom_value=False,
-                    filterable=False,
-                )
-
-                with gr.Row():
-                    report_file = gr.File(
-                        label="Upload PDF Report (optional, but recommended)",
-                        file_types=[".pdf"],
-                        file_count="single",
-                        interactive=True,
-                    )
-                    submission_file = gr.File(
-                        label="Upload Submission File (required, .jsonl)",
-                        file_types=[".jsonl"],
-                        file_count="single",
-                        interactive=True,
-                    )
-                upload_button = gr.Button("Click to Upload Submission")
-                status_box = gr.Textbox(label="Status", interactive=False)
-
-            with gr.Column(scale=2):
-                gr.Markdown("## 🏆 Results Leaderboard")
-                leaderboard = gr.DataFrame(value=load_leaderboard_data, interactive=False)
-                refresh_button = gr.Button("🔄 Refresh Leaderboard")
-
-        # Event handlers
-        upload_button.click(
-            fn=handle_upload,
-            inputs=[submission_name, submission_file, report_file, model_framework],
-            outputs=[status_box],
-            show_progress="full",
-        )
-
-        refresh_button.click(
-            fn=load_leaderboard_data,
-            inputs=None,
-            outputs=[leaderboard]
-        )
-
-    return demo
+print("importedddd")
+# from src.hf_utils import load_leaderboard_data, upload_submission, check_name_exists
+# from src.eval import start_background_evaluation
+#
+#
+# def handle_upload(submission_name, uploaded_file, report_file, model_framework, progress=gr.Progress()):
+#     """Handle file upload and start evaluation."""
+#     if model_framework not in SUPPORTED_FRAMEWORKS:
+#         return f"Unsupported modelling framework: {model_framework}. Supported frameworks are: {', '.join(SUPPORTED_FRAMEWORKS)}"
+#
+#     if not uploaded_file:
+#         return "No file uploaded. Please upload a valid submission file."
+#
+#     if report_file and not report_file.name.endswith(".pdf"):
+#         return "Invalid report format. Please upload a PDF file."
+#
+#     # normalize the submission name
+#     submission_name = submission_name.strip().replace(" ", "_").lower()
+#     # keep only alphanumeric characters and underscores, restrict to 30 characters
+#     submission_name = "".join(
+#         c for c in submission_name if c.isalnum() or c == "_"
+#     )[:30]
+#
+#     if not submission_name or submission_name.strip() == "":
+#         return "Submission name is required"
+#
+#     if check_name_exists(submission_name):
+#         return f"Submission name '{submission_name}' already exists. Please choose a different name."
+#
+#     try:
+#         progress(0.3, "Uploading to Hugging Face...")
+#
+#         # Check if the file is a valid JSONL file
+#         if not uploaded_file.name.endswith(".jsonl"):
+#             return "Invalid file format. Please upload a .jsonl file."
+#
+#         # Check that the keys in the JSONL file are correct ('id' and 'model')
+#         with open(uploaded_file.name, "r") as file:
+#             found_one = False
+#             for line in file:
+#                 found_one = True
+#                 json_object = json.loads(line)
+#                 if not all(key in json_object for key in ["id", "model"]):
+#                     return "Invalid content. Each line must contain 'id' and 'model' keys."
+#             if not found_one:
+#                 return "Empty file. Please upload a valid JSONL file."
+#
+#         success, result = upload_submission(uploaded_file, submission_name, report_file, model_framework)
+#         if not success:
+#             return f"Upload failed: {result}"
+#
+#         progress(0.7, "Starting evaluation...")
+#
+#         # Start evaluation
+#         start_background_evaluation(result)
+#
+#         progress(1.0, "Process complete")
+#         return (
+#             f"✅ Submission '{submission_name}' uploaded successfully!\n"
+#             f"Do not worry if the leaderboard does not update immediately; "
+#             f"it may take some time for the results to appear (around 5-10 minutes). "
+#             f"Feel free to close the tab and check back later.")
+#
+#     except Exception as e:
+#         return f"Error processing upload: {str(e)}"
+#
+#
+# def create_ui():
+#     """Create and return Gradio UI."""
+#     with gr.Blocks(title="Welcome to the CP-Bench leaderboard!") as demo:
+#         gr.Markdown("# CP-Bench Leaderboard")
+#         gr.Markdown(
+#             "This leaderboard is designed to evaluate LLM-generated constraint models for the problems "
+#             "in the [CP-Bench](https://huggingface.co/datasets/kostis-init/CP-Bench) dataset."
+#             "\n\n"
+#             "## How to Submit\n"
+#             "1. **Name your submission**: Choose a unique name for your submission (e.g., `my_cool_submission`). "
+#             "This name will be used to identify your submission on the leaderboard.\n"
+#             "2. **Upload a PDF report**: This is optional, but we highly encourage you to upload a report "
+#             "   (in PDF format) describing your approach. As this is an open competition, we want to avoid submissions "
+#             "   that just copy the models from the dataset. The report can be a short description of your approach, "
+#             "   the models you generated, and any other relevant information.\n"
+#             "3. **Upload your submission**: Upload a **single** `.jsonl` file containing the generated models. "
+#             "   **Each line in the file should be a JSON object with two keys: `id` and `model`.**\n"
+#             "   * `id`: The ID of the problem exactly as it appears in the original dataset (e.g., `csplib__csplib_001_car_sequencing`).\n"
+#             "   * `model`: The generated model for the problem (as a string representing runnable code). Make sure that it eventually outputs the solution as a json with key(s) as described in the `decision_variables` entry and values as would be expected in the problem. This is part of the evaluation as well: unexpected keys, or value types are considered incorrect. This is because our automatic evaluation is based on the solution printed by the submitted models.\n"
+#             "   * An example submission file can be found [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/sample_submission.jsonl).\n"
+#             "3. **Check the leaderboard**: After uploading, you can check the leaderboard to see your results. "
+#             "It may take a few minutes for a submission to be evaluated and appear on the leaderboard.\n"
+#             "\n\n"
+#             "## Important Notes\n"
+#             "1. **Submission Name**: The submission name must be different from any existing submission names.\n"
+#             "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a `pdf` file.\n"
+#             "3. **Evaluation Script**: It is highly recommended to use the evaluation script provided [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/user_eval.py) to check your results before submission. You can run the script as follows:\n"
+#             "   ```bash\n"
+#             "   python user_eval.py --submission_file path/to/my/submission.jsonl\n"
+#             "   ```\n"
+#             "   This will evaluate your submission locally and print the results to the console.\n"
+#             "4. **Modelling Frameworks**: Currently, the supported modelling frameworks are MiniZinc, CPMpy and OR-Tools. More frameworks will be added.\n"
+#             "\n\n"
+#             "### If you have any questions or issues, please feel free to reach out to us TODO\n"
+#             "---\n"
+#         )
+#
+#         with gr.Row():
+#             with gr.Column(scale=1):
+#                 gr.Markdown("## 📤 Upload Submission")
+#
+#                 submission_name = gr.Textbox(
+#                     label="Submission Name (required)",
+#                     placeholder="Enter a unique name for your submission",
+#                     interactive=True,
+#                     info="This name will appear on the leaderboard"
+#                 )
+#                 model_framework = gr.Dropdown(
+#                     label="Modelling Framework (required)",
+#                     choices=SUPPORTED_FRAMEWORKS,
+#                     value=None,
+#                     multiselect=False,
+#                     interactive=True,
+#                     info="Select the modelling framework used for your submission.",
+#                     allow_custom_value=False,
+#                     filterable=False,
+#                 )
+#
+#                 with gr.Row():
+#                     report_file = gr.File(
+#                         label="Upload PDF Report (optional, but recommended)",
+#                         file_types=[".pdf"],
+#                         file_count="single",
+#                         interactive=True,
+#                     )
+#                     submission_file = gr.File(
+#                         label="Upload Submission File (required, .jsonl)",
+#                         file_types=[".jsonl"],
+#                         file_count="single",
+#                         interactive=True,
+#                     )
+#                 upload_button = gr.Button("Click to Upload Submission")
+#                 status_box = gr.Textbox(label="Status", interactive=False)
+#
+#             with gr.Column(scale=2):
+#                 gr.Markdown("## 🏆 Results Leaderboard")
+#                 leaderboard = gr.DataFrame(value=load_leaderboard_data, interactive=False)
+#                 refresh_button = gr.Button("🔄 Refresh Leaderboard")
+#
+#         # Event handlers
+#         upload_button.click(
+#             fn=handle_upload,
+#             inputs=[submission_name, submission_file, report_file, model_framework],
+#             outputs=[status_box],
+#             show_progress="full",
+#         )
+#
+#         refresh_button.click(
+#             fn=load_leaderboard_data,
+#             inputs=None,
+#             outputs=[leaderboard]
+#         )
+#
+#     return demo