File size: 8,949 Bytes
21ed616 180f9fe 03bca85 e67d561 03bca85 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 |
import json
import gradio as gr
from pathlib import Path
print("importedddd1 1", flush=True)
from src.config import SUPPORTED_FRAMEWORKS
print("importedddd")
# from src.hf_utils import load_leaderboard_data, upload_submission, check_name_exists
# from src.eval import start_background_evaluation
#
#
# def handle_upload(submission_name, uploaded_file, report_file, model_framework, progress=gr.Progress()):
# """Handle file upload and start evaluation."""
# if model_framework not in SUPPORTED_FRAMEWORKS:
# return f"Unsupported modelling framework: {model_framework}. Supported frameworks are: {', '.join(SUPPORTED_FRAMEWORKS)}"
#
# if not uploaded_file:
# return "No file uploaded. Please upload a valid submission file."
#
# if report_file and not report_file.name.endswith(".pdf"):
# return "Invalid report format. Please upload a PDF file."
#
# # normalize the submission name
# submission_name = submission_name.strip().replace(" ", "_").lower()
# # keep only alphanumeric characters and underscores, restrict to 30 characters
# submission_name = "".join(
# c for c in submission_name if c.isalnum() or c == "_"
# )[:30]
#
# if not submission_name or submission_name.strip() == "":
# return "Submission name is required"
#
# if check_name_exists(submission_name):
# return f"Submission name '{submission_name}' already exists. Please choose a different name."
#
# try:
# progress(0.3, "Uploading to Hugging Face...")
#
# # Check if the file is a valid JSONL file
# if not uploaded_file.name.endswith(".jsonl"):
# return "Invalid file format. Please upload a .jsonl file."
#
# # Check that the keys in the JSONL file are correct ('id' and 'model')
# with open(uploaded_file.name, "r") as file:
# found_one = False
# for line in file:
# found_one = True
# json_object = json.loads(line)
# if not all(key in json_object for key in ["id", "model"]):
# return "Invalid content. Each line must contain 'id' and 'model' keys."
# if not found_one:
# return "Empty file. Please upload a valid JSONL file."
#
# success, result = upload_submission(uploaded_file, submission_name, report_file, model_framework)
# if not success:
# return f"Upload failed: {result}"
#
# progress(0.7, "Starting evaluation...")
#
# # Start evaluation
# start_background_evaluation(result)
#
# progress(1.0, "Process complete")
# return (
# f"✅ Submission '{submission_name}' uploaded successfully!\n"
# f"Do not worry if the leaderboard does not update immediately; "
# f"it may take some time for the results to appear (around 5-10 minutes). "
# f"Feel free to close the tab and check back later.")
#
# except Exception as e:
# return f"Error processing upload: {str(e)}"
#
#
# def create_ui():
# """Create and return Gradio UI."""
# with gr.Blocks(title="Welcome to the CP-Bench leaderboard!") as demo:
# gr.Markdown("# CP-Bench Leaderboard")
# gr.Markdown(
# "This leaderboard is designed to evaluate LLM-generated constraint models for the problems "
# "in the [CP-Bench](https://huggingface.co/datasets/kostis-init/CP-Bench) dataset."
# "\n\n"
# "## How to Submit\n"
# "1. **Name your submission**: Choose a unique name for your submission (e.g., `my_cool_submission`). "
# "This name will be used to identify your submission on the leaderboard.\n"
# "2. **Upload a PDF report**: This is optional, but we highly encourage you to upload a report "
# " (in PDF format) describing your approach. As this is an open competition, we want to avoid submissions "
# " that just copy the models from the dataset. The report can be a short description of your approach, "
# " the models you generated, and any other relevant information.\n"
# "3. **Upload your submission**: Upload a **single** `.jsonl` file containing the generated models. "
# " **Each line in the file should be a JSON object with two keys: `id` and `model`.**\n"
# " * `id`: The ID of the problem exactly as it appears in the original dataset (e.g., `csplib__csplib_001_car_sequencing`).\n"
# " * `model`: The generated model for the problem (as a string representing runnable code). Make sure that it eventually outputs the solution as a json with key(s) as described in the `decision_variables` entry and values as would be expected in the problem. This is part of the evaluation as well: unexpected keys, or value types are considered incorrect. This is because our automatic evaluation is based on the solution printed by the submitted models.\n"
# " * An example submission file can be found [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/sample_submission.jsonl).\n"
# "3. **Check the leaderboard**: After uploading, you can check the leaderboard to see your results. "
# "It may take a few minutes for a submission to be evaluated and appear on the leaderboard.\n"
# "\n\n"
# "## Important Notes\n"
# "1. **Submission Name**: The submission name must be different from any existing submission names.\n"
# "2. **File Format**: Ensure that the uploaded files are in the correct format. The submission file must be a `.jsonl` file, and the report must be a `pdf` file.\n"
# "3. **Evaluation Script**: It is highly recommended to use the evaluation script provided [here](https://huggingface.co/spaces/kostis-init/CP-Bench-competition/blob/main/user_eval.py) to check your results before submission. You can run the script as follows:\n"
# " ```bash\n"
# " python user_eval.py --submission_file path/to/my/submission.jsonl\n"
# " ```\n"
# " This will evaluate your submission locally and print the results to the console.\n"
# "4. **Modelling Frameworks**: Currently, the supported modelling frameworks are MiniZinc, CPMpy and OR-Tools. More frameworks will be added.\n"
# "\n\n"
# "### If you have any questions or issues, please feel free to reach out to us TODO\n"
# "---\n"
# )
#
# with gr.Row():
# with gr.Column(scale=1):
# gr.Markdown("## 📤 Upload Submission")
#
# submission_name = gr.Textbox(
# label="Submission Name (required)",
# placeholder="Enter a unique name for your submission",
# interactive=True,
# info="This name will appear on the leaderboard"
# )
# model_framework = gr.Dropdown(
# label="Modelling Framework (required)",
# choices=SUPPORTED_FRAMEWORKS,
# value=None,
# multiselect=False,
# interactive=True,
# info="Select the modelling framework used for your submission.",
# allow_custom_value=False,
# filterable=False,
# )
#
# with gr.Row():
# report_file = gr.File(
# label="Upload PDF Report (optional, but recommended)",
# file_types=[".pdf"],
# file_count="single",
# interactive=True,
# )
# submission_file = gr.File(
# label="Upload Submission File (required, .jsonl)",
# file_types=[".jsonl"],
# file_count="single",
# interactive=True,
# )
# upload_button = gr.Button("Click to Upload Submission")
# status_box = gr.Textbox(label="Status", interactive=False)
#
# with gr.Column(scale=2):
# gr.Markdown("## 🏆 Results Leaderboard")
# leaderboard = gr.DataFrame(value=load_leaderboard_data, interactive=False)
# refresh_button = gr.Button("🔄 Refresh Leaderboard")
#
# # Event handlers
# upload_button.click(
# fn=handle_upload,
# inputs=[submission_name, submission_file, report_file, model_framework],
# outputs=[status_box],
# show_progress="full",
# )
#
# refresh_button.click(
# fn=load_leaderboard_data,
# inputs=None,
# outputs=[leaderboard]
# )
#
# return demo
|