import gradio as gr from pathlib import Path import uuid import random from utils.data_utils import generate_leaderboard from utils.plot_utils import plot_ratings from utils.utils import simulate, submit_rating, generate_matchup from config import MODE, VIDEOS, MODELS, CRITERIA, default_beta head = f""" """ with gr.Blocks(title='3D Animation Arena', head=head, css_paths='static/style.css') as arena: sessionState = gr.State({ 'video': None, 'modelLeft': None, 'modelRight': None, 'darkMode': False, 'videos': VIDEOS, 'currentTab': CRITERIA[0], 'uuid': None }) frontState = gr.JSON(sessionState, visible=False) with gr.Row(): with gr.Column(scale=1): gr.HTML('') with gr.Column(scale=12): gr.HTML("

3D Animation Arena

") with gr.Column(scale=1): toggle_dark = gr.Button(value="Dark Mode") def update_toggle_dark(state): state['darkMode'] = not state['darkMode'] if state['darkMode']: return gr.update(value="Light Mode"), state else: return gr.update(value="Dark Mode"), state toggle_dark.click( inputs=[sessionState], js=""" () => { document.body.classList.toggle('dark'); } """, fn=update_toggle_dark, outputs=[toggle_dark, sessionState] ) with gr.Tab(label='Arena'): models = gr.HTML('''
''', render=False) with gr.Row(): with gr.Column(scale=1): gr.HTML(f"

1. Choose a video below:

") video = gr.Video( label='Input Video', interactive=False, autoplay=True, show_download_button=False, loop=True, elem_id='gradioVideo', ) triggerButtons = {} for vid in sessionState.value['videos']: triggerButtons[vid] = gr.Button(elem_id=f'triggerBtn_{vid}', visible=False) triggerButtons[vid].click( fn=lambda vid=vid: gr.update(value=f'https://gradio-model-viewer.s3.eu-west-1.amazonaws.com/sample+videos/{vid}.mp4'), outputs=[video] ) examples = gr.HTML(visible=False) with gr.Column(scale=4): gr.HTML("""

2. Play around with the models: You can control the playback in both viewers at the same time by using the video, or control both viewers independently by using mouse and GUI!

""") with gr.Row(): models.render() with gr.Row(): gr.HTML(f"

3. Choose your favorite model for each criteria:

") ratingButtons = {} for criteria in CRITERIA: with gr.Row(): with gr.Column(): with gr.Row(): match criteria: case 'Global_Appreciation': instructions = "Your overall appreciation of the models, including general aesthetics and self-contacts if applicable." case 'Ground_Contacts': instructions = "The quality of the models' contacts with the ground, including ground penetration and foot sliding." case 'Fidelity': instructions = "The fidelity of the models compared to the motion of the original video." case 'Fluidity': instructions = "The smoothness and temporal coherence of the models." gr.HTML(f"""

{criteria.replace('_', ' ')} {instructions}

""") with gr.Row(): ratingButtons[criteria] = [] with gr.Column(scale=2): ratingButtons[criteria].append(gr.Button('Left Model', variant='primary', interactive=False)) with gr.Column(scale=1, min_width=2): ratingButtons[criteria].append(gr.Button('Skip', min_width=2, interactive=False)) with gr.Column(scale=2): ratingButtons[criteria].append(gr.Button('Right Model', variant='primary', interactive=False)) # Leaderboard per criteria with gr.Tab(label='Leaderboards') as leaderboard_tab: if MODE == 'testing': # Simulation controls with gr.Row(): simulate_btn = gr.Button('Simulate Matches', variant='primary') add_model_btn = gr.Button('Add Model', variant='secondary') with gr.Row(): gr.Markdown(''' ## Probability of each model to be chosen is updated after each vote following: \ $$ p_i = \\frac{e^{-\\frac{Matches_i}{\\beta}}}{\\sum_{j=1}^{N} e^{-\\frac{Matches_j}{\\beta}}} $$ ''') iterate = gr.Number(label='Number of iterations', value=100, minimum=1, maximum=2000, precision=0, interactive=True) beta = gr.Number(label='Beta', value=default_beta, minimum=1, maximum=1000, precision=0, step=10, interactive=True) else: beta = gr.Number(label='Beta', value=default_beta, render=False) leaderboards = {} tabs = {} for criteria in CRITERIA: with gr.Tab(label=criteria.replace('_', ' ')) as tabs[criteria]: with gr.Row(): gr.HTML(f"

{criteria.replace('_', ' ')}

") with gr.Row(): leaderboards[criteria] = gr.Dataframe(value=None, row_count=(len(MODELS), 'fixed'), headers=['Model', 'Elo', 'Wins', 'Matches', 'Win Rate'], interactive=False) # Plots if MODE == 'testing': with gr.Row(): elo_plot = gr.Plot(value=None, label='Elo Ratings', format='plotly', elem_id='plot') with gr.Row(): wr_plot = gr.Plot(value=None, label='Win Rates', format='plotly', elem_id='plot') with gr.Row(): matches_plot = gr.Plot(value=None, label='Matches played', format='plotly', elem_id='plot') elif MODE == 'production': elo_plot = gr.Plot(value=None, label='Elo Ratings', format='plotly', elem_id='plot', visible=False) wr_plot = gr.Plot(value=None, label='Win Rates', format='plotly', elem_id='plot', visible=False) matches_plot = gr.Plot(value=None, label='Matches played', format='plotly', elem_id='plot', visible=False) with gr.Tab(label='About'): gr.Markdown(''' ## Thank you for using the 3D Animation Arena! This app is designed to compare different models based on human preferences, inspired by dylanebert's [3D Arena](https://huggingface.co/spaces/dylanebert/3d-arena) on Hugging Face. Current rankings often use metrics to assess the quality of a model, but these metrics may not always reflect the complexity behind human preferences. The current models competing in the arena are: - 4DHumans (https://github.com/shubham-goel/4D-Humans) - CLIFF (https://github.com/haofanwang/CLIFF) - GVHMR (https://github.com/zju3dv/GVHMR) - HybrIK (https://github.com/jeffffffli/HybrIK) - WHAM (https://github.com/yohanshin/WHAM) All inferences are precomputed following the code in the associated GitHub repository. Some post-inference modifications have been made to some models in order to make the comparison possible. These modifications include: * Adjusting height to a common ground * Fixing the root depth of certain models, when depth was extremely jittery * Fixing the root position of certain models, when no root position was available All models use the SMPL body model to discard the influence of the body model on the comparison. These choices were made without any intention to favor or harm any model. All matchups are generated randomly, don't hesitate to rate the same videos multiple times as the matchups will probably be different! --- If you have comments, complaints or suggestions, please contact me at 3danimationarena@gmail.com. New models and videos will be added over time, feel free to share your ideas! Keep in mind that I will not add raw inferences from other people to keep it fair. ''') # Event handlers def randomize_videos(state): state['uuid'] = str(uuid.uuid4()) random.shuffle(state['videos']) gallery = "" return state, gallery async def display_leaderboards(): return [await generate_leaderboard(criteria) for criteria in CRITERIA] arena.load( inputs=[sessionState], fn=lambda state: randomize_videos(state), outputs=[sessionState, examples], ).then( inputs=[], fn=lambda: gr.update(visible=True), outputs=[examples] ).then( inputs=[gr.State(CRITERIA[0])], fn=plot_ratings, outputs=[elo_plot, wr_plot, matches_plot] ).then( inputs=[], fn=display_leaderboards, outputs=[leaderboards[criteria] for criteria in CRITERIA] ) async def update_models(video, state): leaderboard = await generate_leaderboard(CRITERIA[0]) video_name = video.split('/')[-1].split('.')[0] modelLeft, modelRight = generate_matchup(leaderboard=leaderboard, beta=beta.value) state['video'] = video_name state['modelLeft'] = MODELS[modelLeft] state['modelRight'] = MODELS[modelRight] return state, state video.change( inputs=[video, sessionState], fn=update_models, outputs=[sessionState, frontState] ) # Weird workaround to run JS function on state change, from https://github.com/gradio-app/gradio/issues/3525#issuecomment-2348596861 frontState.change( inputs=[frontState], js='(state) => updateViewers(state)', fn=lambda state: None, ).then( inputs=None, fn=lambda: tuple(gr.update(interactive=True) for _ in sum(ratingButtons.values(), [])), outputs= sum(ratingButtons.values(), []) ) leaderboard_tab.select( inputs=None, js='() => resetPlots()', fn=None, ).then( fn=lambda: [gr.update(value=None) for _ in range(3)], outputs=[elo_plot, wr_plot, matches_plot] ).then( inputs=[sessionState], fn=lambda state: plot_ratings(state['currentTab']), outputs=[elo_plot, wr_plot, matches_plot] ) async def process_rating(state, i, criteria): return gr.update(value=await submit_rating( criteria=criteria, video=state['video'], winner=state['modelLeft'] if i == 0 else state['modelRight'] if i == 2 else None, loser=state['modelRight'] if i == 0 else state['modelLeft'] if i == 2 else None, uuid=state['uuid'] )) def update_tab(state, criteria): state['currentTab'] = criteria return state for criteria in CRITERIA: for i, button in enumerate(ratingButtons[criteria]): button.click( # fn=lambda i=i, criteria=criteria: gr.Info(f'{"You chose Left Model for " if i == 0 else "You chose Right Model for " if i == 2 else "You skipped "} {criteria.replace("_", " ")}!'), # ).then( fn=lambda: tuple(gr.update(interactive=False) for _ in range(len(ratingButtons[criteria]))), outputs=ratingButtons[criteria] ).then( inputs=[sessionState, gr.State(i), gr.State(criteria)], fn=process_rating, outputs=[leaderboards[criteria]], ) tabs[criteria].select( fn=lambda: [gr.update(value=None) for _ in range(3)], outputs=[elo_plot, wr_plot, matches_plot] ).then( inputs=[gr.State(criteria)], fn=plot_ratings, outputs=[elo_plot, wr_plot, matches_plot] ).then( inputs=[sessionState, gr.State(criteria)], fn=update_tab, outputs=[sessionState] ) if MODE == 'testing': for criteria in CRITERIA: simulate_btn.click( inputs=[iterate, beta, gr.State(criteria)], fn=simulate, outputs=[leaderboards[criteria]], ).then(fn=lambda: [gr.update(value=None) for _ in range(3)], outputs=[elo_plot, wr_plot, matches_plot] ).then( inputs=[gr.State(criteria)], fn=plot_ratings, outputs=[elo_plot, wr_plot, matches_plot] ) add_model_btn.click( fn=lambda: MODELS.append(f'model_{len(MODELS)}'), ) if __name__ == '__main__': gr.set_static_paths(['static']) arena.queue(default_concurrency_limit=50).launch(inbrowser=True, allowed_paths=['static/'])