| import json | |
| import gradio as gr | |
| from pathlib import Path | |
| print("importedddd1 1", flush=True) | |
| from src.config import SUPPORTED_FRAMEWORKS | |
| print("importedddd") | |
| # from src.hf_utils import load_leaderboard_data, upload_submission, check_name_exists | |
| # from src.eval import start_background_evaluation | |
| # | |
| # | |
| # def handle_upload(submission_name, uploaded_file, report_file, model_framework, progress=gr.Progress()): | |
| # """Handle file upload and start evaluation.""" | |
| # if model_framework not in SUPPORTED_FRAMEWORKS: | |
| # return f"Unsupported modelling framework: {model_framework}. Supported frameworks are: {', '.join(SUPPORTED_FRAMEWORKS)}" | |
| # | |
| # if not uploaded_file: | |
| # return "No file uploaded. Please upload a valid submission file." | |
| # | |
| # if report_file and not report_file.name.endswith(".pdf"): | |
| # return "Invalid report format. Please upload a PDF file." | |
| # | |
| # # normalize the submission name | |
| # submission_name = submission_name.strip().replace(" ", "_").lower() | |
| # # keep only alphanumeric characters and underscores, restrict to 30 characters | |
| # submission_name = "".join( | |
| # c for c in submission_name if c.isalnum() or c == "_" | |
| # )[:30] | |
| # | |
| # if not submission_name or submission_name.strip() == "": | |
| # return "Submission name is required" | |
| # | |
| # if check_name_exists(submission_name): | |
| # return f"Submission name '{submission_name}' already exists. Please choose a different name." | |
| # | |
| # try: | |
| # progress(0.3, "Uploading to Hugging Face...") | |
| # | |
| # # Check if the file is a valid JSONL file | |
| # if not uploaded_file.name.endswith(".jsonl"): | |
| # return "Invalid file format. Please upload a .jsonl file." | |
| # | |
| # # Check that the keys in the JSONL file are correct ('id' and 'model') | |
| # with open(uploaded_file.name, "r") as file: | |
| # found_one = False | |
| # for line in file: | |
| # found_one = True | |
| # json_object = json.loads(line) | |
| # if not all(key in json_object for key in ["id", "model"]): | |
| # return "Invalid content. Each line must contain 'id' and 'model' keys." | |
| # if not found_one: | |
| # return "Empty file. Please upload a valid JSONL file." | |
| # | |
| # success, result = upload_submission(uploaded_file, submission_name, report_file, model_framework) | |
| # if not success: | |
| # return f"Upload failed: {result}" | |
| # | |
| # progress(0.7, "Starting evaluation...") | |
| # | |
| # # Start evaluation | |
| # start_background_evaluation(result) | |
| # | |
| # progress(1.0, "Process complete") | |
| # return ( | |
| # f"✅ Submission '{submission_name}' uploaded successfully!\n" | |
| # f"Do not worry if the leaderboard does not update immediately; " | |
| # f"it may take some time for the results to appear (around 5-10 minutes). " | |
| # f"Feel free to close the tab and check back later.") | |
| # | |
| # except Exception as e: | |
| # return f"Error processing upload: {str(e)}" | |
| # | |
| # | |
| # def create_ui(): | |
| # """Create and return Gradio UI.""" | |
| # with gr.Blocks(title="Welcome to the CP-Bench leaderboard!") as demo: | |
| # gr.Markdown("# CP-Bench Leaderboard") | |
| # gr.Markdown( | |
| # "This leaderboard is designed to evaluate LLM-generated constraint models for the problems " | |
| # "in the [CP-Bench](https://huggingface.co/datasets/kostis-init/CP-Bench) dataset." | |
| # "\n\n" | |
| # "## How to Submit\n" | |
| # "1. **Name your submission**: Choose a unique name for your submission (e.g., `my_cool_submission`). " | |
| # "This name will be used to identify your submission on the leaderboard.\n" | |
| # "2. **Upload a PDF report**: This is optional, but we highly encourage you to upload a report " | |
| # " (in PDF format) describing your approach. As this is an open competition, we want to avoid submissions " | |
| # " that just copy the models from the dataset. The report can be a short description of your approach, " | |
| # " the models you generated, and any other relevant information.\n" | |
| # "3. **Upload your submission**: Upload a **single** `.jsonl` file containing the generated models. " | |
| # " **Each line in the file should be a JSON object with two keys: `id` and `model`.**\n" | |
| # " * `id`: The ID of the problem exactly as it appears in the original dataset (e.g., `csplib__csplib_001_car_sequencing`).\n" | |
| # " * `model`: The generated model for the problem (as a string representing runnable code). Make sure that it eventually outputs the solution as a json with key(s) as described in the `decision_variables` entry and values as would be expected in the problem. This is part of the evaluation as well: unexpected keys, or value types are considered incorrect. This is because our automatic evaluation is based on the solution printed by the submitted models.\n" | |
| # " * An example submission file can be found [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/sample_submission.jsonl).\n" | |
| # "3. **Check the leaderboard**: After uploading, you can check the leaderboard to see your results. " | |
| # "It may take a few minutes for a submission to be evaluated and appear on the leaderboard.\n" | |
| # "\n\n" | |
| # "## Important Notes\n" | |
| # "1. **Submission Name**: The submission name must be different from any existing submission names.\n" | |
| # "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a `pdf` file.\n" | |
| # "3. **Evaluation Script**: It is highly recommended to use the evaluation script provided [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/user_eval.py) to check your results before submission. You can run the script as follows:\n" | |
| # " ```bash\n" | |
| # " python user_eval.py --submission_file path/to/my/submission.jsonl\n" | |
| # " ```\n" | |
| # " This will evaluate your submission locally and print the results to the console.\n" | |
| # "4. **Modelling Frameworks**: Currently, the supported modelling frameworks are MiniZinc, CPMpy and OR-Tools. More frameworks will be added.\n" | |
| # "\n\n" | |
| # "### If you have any questions or issues, please feel free to reach out to us TODO\n" | |
| # "---\n" | |
| # ) | |
| # | |
| # with gr.Row(): | |
| # with gr.Column(scale=1): | |
| # gr.Markdown("## 📤 Upload Submission") | |
| # | |
| # submission_name = gr.Textbox( | |
| # label="Submission Name (required)", | |
| # placeholder="Enter a unique name for your submission", | |
| # interactive=True, | |
| # info="This name will appear on the leaderboard" | |
| # ) | |
| # model_framework = gr.Dropdown( | |
| # label="Modelling Framework (required)", | |
| # choices=SUPPORTED_FRAMEWORKS, | |
| # value=None, | |
| # multiselect=False, | |
| # interactive=True, | |
| # info="Select the modelling framework used for your submission.", | |
| # allow_custom_value=False, | |
| # filterable=False, | |
| # ) | |
| # | |
| # with gr.Row(): | |
| # report_file = gr.File( | |
| # label="Upload PDF Report (optional, but recommended)", | |
| # file_types=[".pdf"], | |
| # file_count="single", | |
| # interactive=True, | |
| # ) | |
| # submission_file = gr.File( | |
| # label="Upload Submission File (required, .jsonl)", | |
| # file_types=[".jsonl"], | |
| # file_count="single", | |
| # interactive=True, | |
| # ) | |
| # upload_button = gr.Button("Click to Upload Submission") | |
| # status_box = gr.Textbox(label="Status", interactive=False) | |
| # | |
| # with gr.Column(scale=2): | |
| # gr.Markdown("## 🏆 Results Leaderboard") | |
| # leaderboard = gr.DataFrame(value=load_leaderboard_data, interactive=False) | |
| # refresh_button = gr.Button("🔄 Refresh Leaderboard") | |
| # | |
| # # Event handlers | |
| # upload_button.click( | |
| # fn=handle_upload, | |
| # inputs=[submission_name, submission_file, report_file, model_framework], | |
| # outputs=[status_box], | |
| # show_progress="full", | |
| # ) | |
| # | |
| # refresh_button.click( | |
| # fn=load_leaderboard_data, | |
| # inputs=None, | |
| # outputs=[leaderboard] | |
| # ) | |
| # | |
| # return demo | |