import gradio as gr from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns import pandas as pd import os import json import tempfile import shutil import zipfile from huggingface_hub import snapshot_download # Constants for PhysicalCodeBench TITLE = """

PhysicalCodeBench Leaderboard

Evaluating LLMs on Physics-based Simulation Code Generation

""" INTRODUCTION_TEXT = """ PhysicalCodeBench evaluates the abilities of Large Language Models (LLMs) to generate code for physics-based simulations. The benchmark consists of user instructions that describe physical scenarios to be simulated, reference code implementations, and resulting simulation videos generated using the [Genesis](https://github.com/Genesis-Embodied-AI/Genesis) physics engine. This leaderboard showcases model performance on the PhysicalCodeBench-50 dataset, measuring both text-based execution success and visual quality of the generated simulations. """ ABOUT_TEXT = """ ## About PhysicalCodeBench PhysicalCodeBench evaluates an LLM's ability to: - Understand natural language descriptions of physical scenarios - Generate executable code that correctly implements the physics simulation - Produce visually accurate and physically plausible results The benchmark covers a variety of physical phenomena including: - Rigid body dynamics (collisions, rolling, bouncing, etc.) - Fluid and particle simulations - Soft body physics - Controlled environments (robotic arms, drones, etc.) - Chain reactions and complex interactions ## Evaluation Metrics PhysicalCodeBench uses two main evaluation dimensions: 1. **Text Score (50 points)**: Evaluates code execution success - Code runs without errors (25 points) - Generates proper output files (10 points) - Output files meet required specifications (15 points) 2. **Visual Score (50 points)**: Evaluates simulation quality - CLIP Score: Measures text-video alignment (25 points) - Motion Smoothness: Evaluates physics simulation quality (25 points) Total score is the sum of Text and Visual scores (maximum 100 points). """ SUBMISSION_TEXT = """ ## How to Submit Your Model Results 1. Fork the [PhysicalCodeBench repository](https://github.com/Sealical/PhysicalCodeBench) 2. Generate code for all 50 tasks in the benchmark using your model 3. Run the evaluation pipeline with your generated code 4. Create a submission folder with the following structure: ``` submission/ ├── model_info.json # Contains model details (name, size, etc.) ├── evaluation_results/ # Directory containing all result files └── PhysCodeEval_results.json # Main evaluation results file ``` 5. Zip your submission folder and upload it below along with your model details Your submission will be verified and added to the leaderboard once approved. """ CITATION_TEXT = """ @article{PhysicalCodeBench2025, title={PhysicalCodeBench: Evaluating LLMs on Physics-based Simulation Code Generation}, author={Your Name and Co-authors}, journal={arXiv preprint arXiv:XXXX.XXXXX}, year={2025} } """ # Custom CSS for the interface custom_css = """ .markdown-text { font-size: 16px !important; text-align: left !important; } .tab-button { font-size: 16px !important; } """ # Define column structure for the leaderboard class PhysCodeColumn: def __init__(self, name, type, displayed_by_default=True, never_hidden=False, hidden=False): self.name = name self.type = type self.displayed_by_default = displayed_by_default self.never_hidden = never_hidden self.hidden = hidden # Define the columns for our leaderboard COLUMNS = [ PhysCodeColumn("rank", "number", True, True, False), PhysCodeColumn("model", "str", True, True, False), PhysCodeColumn("model_type", "str", True, False, False), PhysCodeColumn("organization", "str", True, False, False), PhysCodeColumn("text_score", "number", True, False, False), PhysCodeColumn("visual_score", "number", True, False, False), PhysCodeColumn("total_score", "number", True, False, False), PhysCodeColumn("clip_score", "number", False, False, False), PhysCodeColumn("motion_smooth_score", "number", False, False, False), PhysCodeColumn("execution_success", "number", False, False, False), PhysCodeColumn("file_generation", "number", False, False, False), PhysCodeColumn("file_quality", "number", False, False, False), PhysCodeColumn("submission_date", "date", False, False, False) ] # Enums for model metadata class ModelType: Proprietary = "Proprietary" OpenSource = "Open Source" CloseSource = "Close Source" API = "API" Unknown = "Unknown" @staticmethod def to_str(model_type): return model_type # Load sample data (replace with your actual data loading logic) def get_leaderboard_df(): # Sample data based on your README data = [ { "rank": 1, "model": "GPT4o", "model_type": ModelType.CloseSource, "organization": "OpenAI", "text_score": 16.0, "visual_score": 18.262, "total_score": 34.262, "clip_score": 10.2, "motion_smooth_score": 8.062, "execution_success": 10.0, "file_generation": 3.0, "file_quality": 3.0, "submission_date": "2025-01-15" }, { "rank": 2, "model": "Gemini-2.0-flash", "model_type": ModelType.CloseSource, "organization": "Google", "text_score": 15.0, "visual_score": 16.963, "total_score": 31.963, "clip_score": 9.5, "motion_smooth_score": 7.463, "execution_success": 9.0, "file_generation": 3.0, "file_quality": 3.0, "submission_date": "2025-01-20" }, { "rank": 3, "model": "DS-R1", "model_type": ModelType.OpenSource, "organization": "DeepSeek", "text_score": 14.0, "visual_score": 15.815, "total_score": 29.815, "clip_score": 8.9, "motion_smooth_score": 6.915, "execution_success": 8.5, "file_generation": 3.0, "file_quality": 2.5, "submission_date": "2025-01-25" }, { "rank": 4, "model": "DeepSeek-R1-Distill-Qwen-32B", "model_type": ModelType.OpenSource, "organization": "DeepSeek", "text_score": 12.2, "visual_score": 15.82, "total_score": 28.02, "clip_score": 8.8, "motion_smooth_score": 7.02, "execution_success": 7.2, "file_generation": 2.5, "file_quality": 2.5, "submission_date": "2025-01-28" }, { "rank": 5, "model": "QwQ-32B", "model_type": ModelType.OpenSource, "organization": "QwQ Team", "text_score": 7.1, "visual_score": 8.964, "total_score": 16.064, "clip_score": 4.964, "motion_smooth_score": 4.0, "execution_success": 4.1, "file_generation": 1.5, "file_quality": 1.5, "submission_date": "2025-02-05" }, { "rank": 6, "model": "Qwen-2.5-32B", "model_type": ModelType.OpenSource, "organization": "Alibaba", "text_score": 0.7, "visual_score": 1.126, "total_score": 1.826, "clip_score": 0.626, "motion_smooth_score": 0.5, "execution_success": 0.5, "file_generation": 0.1, "file_quality": 0.1, "submission_date": "2025-02-10" } ] return pd.DataFrame(data) # Function to load submission from JSON file def load_submissions_from_json(json_path): if os.path.exists(json_path): with open(json_path, 'r') as f: data = json.load(f) return pd.DataFrame(data) return None # Initialize the leaderboard def init_leaderboard(dataframe): if dataframe is None or dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") return Leaderboard( value=dataframe, datatype=[c.type for c in COLUMNS], select_columns=SelectColumns( default_selection=[c.name for c in COLUMNS if c.displayed_by_default], cant_deselect=[c.name for c in COLUMNS if c.never_hidden], label="Select Columns to Display:", ), search_columns=["model", "organization"], hide_columns=[c.name for c in COLUMNS if c.hidden], filter_columns=[ ColumnFilter("model_type", type="checkboxgroup", label="Model types"), ColumnFilter("organization", type="checkboxgroup", label="Organizations"), ], interactive=False, ) # Function to handle ZIP file upload and extraction def process_zip_submission(zip_file): if zip_file is None: return "No file uploaded. Please upload a ZIP file containing your submission." # Create temp directory for extraction temp_dir = tempfile.mkdtemp() try: # Extract the zip file with zipfile.ZipFile(zip_file.name, 'r') as zip_ref: zip_ref.extractall(temp_dir) # Check for required files model_info_path = os.path.join(temp_dir, "model_info.json") results_json_path = os.path.join(temp_dir, "PhysCodeEval_results.json") if not os.path.exists(model_info_path): return "Error: model_info.json not found in the ZIP file." if not os.path.exists(results_json_path): return "Error: PhysCodeEval_results.json not found in the ZIP file." # Load model info with open(model_info_path, 'r') as f: model_info = json.load(f) # Check for required model info fields required_fields = ["model_name", "model_type", "organization"] missing_fields = [field for field in required_fields if field not in model_info] if missing_fields: return f"Error: Missing required fields in model_info.json: {', '.join(missing_fields)}" # TODO: Process the submission files (this would involve your validation logic) return f"Successfully processed submission for {model_info['model_name']} by {model_info['organization']}. Your submission will be reviewed and added to the leaderboard once approved." except zipfile.BadZipFile: return "Error: Invalid ZIP file." except Exception as e: return f"Error processing submission: {str(e)}" finally: # Clean up shutil.rmtree(temp_dir) # Submission form handling def process_submission(model_name, model_type, organization, team_name, email, submission_link): # Check for required fields if not model_name: return "Error: Model name is required." if not model_type: return "Error: Model type is required." if not email: return "Error: Contact email is required." # This would be implemented to handle actual submission processing return f"Thank you for submitting {model_name} from {organization or team_name}! Your submission will be reviewed and added to the leaderboard once verified. We will contact you at {email} if we need additional information." # Main application def create_demo(): # Load the leaderboard data leaderboard_df = get_leaderboard_df() # Create the Gradio interface demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") with gr.Tabs() as tabs: with gr.TabItem("🏅 Leaderboard", id=0): leaderboard = init_leaderboard(leaderboard_df) with gr.TabItem("📊 Visualizations", id=1): gr.Markdown("## Performance Comparisons") with gr.Row(): with gr.Column(): gr.Markdown("### Text vs. Visual Scores") # Add a visualization component here (e.g., scatter plot) with gr.Column(): gr.Markdown("### Score Breakdown by Task Type") # Add a visualization component here (e.g., bar chart) with gr.Row(): model_selector = gr.Dropdown( choices=leaderboard_df["model"].tolist(), label="Select Model for Detailed Analysis", multiselect=False, ) with gr.TabItem("📝 About", id=2): gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text") with gr.TabItem("🚀 Submit", id=3): gr.Markdown(SUBMISSION_TEXT, elem_classes="markdown-text") gr.Markdown("### Submission Details") with gr.Row(): zip_file_input = gr.File(label="Upload submission ZIP file*") with gr.Row(): with gr.Column(): model_name_input = gr.Textbox(label="Model Name*") model_type_input = gr.Dropdown( choices=["Open Source", "Close Source", "API", "Proprietary"], label="Model Type*", multiselect=False, ) organization_input = gr.Textbox(label="Organization (if applicable)") with gr.Column(): team_name_input = gr.Textbox(label="Team Name (if applicable)") email_input = gr.Textbox(label="Contact Email*") submission_link_input = gr.Textbox(label="GitHub Pull Request URL") submit_button = gr.Button("Submit") submission_result = gr.Markdown() # Combined submission function that processes both ZIP and form data def combined_submission(zip_file, model_name, model_type, organization, team_name, email, submission_link): if zip_file is None: return "Error: Please upload a ZIP file containing your submission." if not model_name or not model_type or not email: return "Error: Model name, model type, and email are required fields." # Process ZIP file zip_result = process_zip_submission(zip_file) if zip_result.startswith("Error:"): return zip_result # Process form data return f"Thank you for submitting {model_name} from {organization or team_name}! Your submission ZIP has been processed successfully. We will contact you at {email} if we need additional information." submit_button.click( combined_submission, [zip_file_input, model_name_input, model_type_input, organization_input, team_name_input, email_input, submission_link_input], submission_result, ) with gr.Row(): with gr.Accordion("📙 Citation", open=False): citation_button = gr.Textbox( value=CITATION_TEXT, label="Citation", lines=8, elem_id="citation-button", show_copy_button=True, ) return demo # Launch the application if __name__ == "__main__": demo = create_demo() demo.launch()