update fal api models

model/model_manager.py

@@ -28,6 +28,11 @@ class ModelManager:
         result = pipe(prompt=prompt)
         return result
 
+    def generate_image_ig_api(self, prompt, model_name):
+        pipe = self.load_model_pipe(model_name)
+        result = pipe(prompt=prompt)
+        return result
+
     def generate_image_ig_parallel_anony(self, prompt, model_A, model_B):
         if model_A == "" and model_B == "":
             model_names = random.sample([model for model in self.model_ig_list], 2)
@@ -35,14 +40,16 @@ class ModelManager:
             model_names = [model_A, model_B]
 
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [executor.submit(self.generate_image_ig, prompt, model) for model in model_names]
+            futures = [executor.submit(self.generate_image_ig, prompt, model) if model.startswith("imagenhub")
+                       else executor.submit(self.generate_image_ig_api, prompt, model) for model in model_names]
             results = [future.result() for future in futures]
         return results[0], results[1], model_names[0], model_names[1]
 
     def generate_image_ig_parallel(self, prompt, model_A, model_B):
         model_names = [model_A, model_B]
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [executor.submit(self.generate_image_ig, prompt, model) for model in model_names]
+            futures = [executor.submit(self.generate_image_ig, prompt, model) if model.startswith("imagenhub")
+                       else executor.submit(self.generate_image_ig_api, prompt, model) for model in model_names]
             results = [future.result() for future in futures]
         return results[0], results[1]
 
@@ -77,6 +84,11 @@ class ModelManager:
         result = pipe(prompt=prompt)
         return result
 
+    def generate_video_vg_api(self, prompt, model_name):
+        pipe = self.load_model_pipe(model_name)
+        result = pipe(prompt=prompt)
+        return result
+
     def generate_video_vg_parallel_anony(self, prompt, model_A, model_B):
         if model_A == "" and model_B == "":
             model_names = random.sample([model for model in self.model_vg_list], 2)
@@ -84,13 +96,15 @@ class ModelManager:
             model_names = [model_A, model_B]
 
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [executor.submit(self.generate_video_vg, prompt, model) for model in model_names]
+            futures = [executor.submit(self.generate_video_vg, prompt, model) if model.startswith("videogenhub")
+                       else executor.submit(self.generate_video_vg_api, prompt, model) for model in model_names]
             results = [future.result() for future in futures]
         return results[0], results[1], model_names[0], model_names[1]
 
     def generate_video_vg_parallel(self, prompt, model_A, model_B):
         model_names = [model_A, model_B]
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [executor.submit(self.generate_video_vg, prompt, model) for model in model_names]
+            futures = [executor.submit(self.generate_video_vg, prompt, model) if model.startswith("videogenhub")
+                       else executor.submit(self.generate_video_vg_api, prompt, model) for model in model_names]
             results = [future.result() for future in futures]
         return results[0], results[1]

model/model_registry.py

@@ -46,21 +46,21 @@ def get_model_description_md(model_list):
 # regist image generation models
 
 register_model_info(
-    ["imagenhub_LCM_generation"],
+    ["imagenhub_LCM_generation", "fal_LCM_text2image"],
     "LCM",
     "https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7",
     "Latent Consistency Models.",
 )
 
 register_model_info(
-    ["imagenhub_PlayGroundV2_generation"],
+    ["imagenhub_PlayGroundV2_generation", 'playground_PlayGroundV2_generation'],
     "Playground v2",
     "https://huggingface.co/playgroundai/playground-v2-1024px-aesthetic",
     "Playground v2 – 1024px Aesthetic Model",
 )
 
 register_model_info(
-    ["imagenhub_PlayGroundV2.5_generation"],
+    ["imagenhub_PlayGroundV2.5_generation", 'playground_PlayGroundV2.5_generation'],
     "Playground v2.5",
     "https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic",
     "Playground v2.5 is the state-of-the-art open-source model in aesthetic quality",
@@ -74,14 +74,14 @@ register_model_info(
 )
 
 register_model_info(
-    ["imagenhub_SDXLTurbo_generation"],
+    ["imagenhub_SDXLTurbo_generation", "fal_SDXLTurbo_text2image"],
     "SDXLTurbo",
     "https://huggingface.co/stabilityai/sdxl-turbo",
     "SDXL-Turbo is a fast generative text-to-image model.",
 )
 
 register_model_info(
-    ["imagenhub_SDXL_generation"],
+    ["imagenhub_SDXL_generation", "fal_SDXL_text2image"],
     "SDXL",
     "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
     "SDXL is a Latent Diffusion Model that uses two fixed, pretrained text encoders.",
@@ -95,21 +95,21 @@ register_model_info(
 )
 
 register_model_info(
-    ["imagenhub_PixArtSigma_generation"],
+    ["imagenhub_PixArtSigma_generation", "fal_PixArtSigma_text2image"],
     "PixArtSigma",
     "https://github.com/PixArt-alpha/PixArt-sigma",
     "Improved version of Pixart-α.",
 )
 
 register_model_info(
-    ["imagenhub_SDXLLightning_generation"],
+    ["imagenhub_SDXLLightning_generation", "fal_SDXLLightning_text2image"],
     "SDXL-Lightning",
     "https://huggingface.co/ByteDance/SDXL-Lightning",
     "SDXL-Lightning is a lightning-fast text-to-image generation model.",
 )
 
 register_model_info(
-    ["imagenhub_StableCascade_generation"],
+    ["imagenhub_StableCascade_generation", "fal_StableCascade_text2image"],
     "StableCascade",
     "https://huggingface.co/stabilityai/stable-cascade",
     "StableCascade is built upon the Würstchen architecture and working at a much smaller latent space.",
@@ -137,12 +137,6 @@ register_model_info(
     "Image Editing with Cross-Attention Control.",
 )
 
-# register_model_info(
-#     ["imagenhub_SDEdit_edition"],
-#     "SDEdit",
-#     "",
-#     "xxx",
-# )
 
 register_model_info(
     ["imagenhub_InstructPix2Pix_edition"],
@@ -221,13 +215,6 @@ register_model_info(
     "ModelScope is a a T2V synthesis model that evolves from a T2I synthesis model.",
 )
 
-#register_model_info(
-#    ["videogenhub_CogVideo_generation"],
-#    "CogVideo",
-#    "https://arxiv.org/abs/2205.15868",
-#    "Text-to-Video Generation via Transformers",
-#)
-
 register_model_info(
     ["videogenhub_OpenSora_generation"],
     "OpenSora",

model/models/__init__.py

@@ -3,8 +3,9 @@ from .playground_api import load_playground_model
 from .fal_api_models import load_fal_model
 from .videogenhub_models import load_videogenhub_model
 
-IMAGE_GENERATION_MODELS = ['imagenhub_LCM_generation','imagenhub_SDXLTurbo_generation','imagenhub_SDXL_generation', 'imagenhub_PixArtAlpha_generation', 'imagenhub_PixArtSigma_generation',
-                            'imagenhub_OpenJourney_generation','imagenhub_SDXLLightning_generation', 'imagenhub_StableCascade_generation',
+
+IMAGE_GENERATION_MODELS = ['fal_LCM_text2image','fal_SDXLTurbo_text2image','fal_SDXL_text2image', 'imagenhub_PixArtAlpha_generation', 'fal_PixArtSigma_text2image',
+                            'imagenhub_OpenJourney_generation','fal_SDXLLightning_text2image', 'fal_StableCascade_text2image',
                             'playground_PlayGroundV2_generation', 'playground_PlayGroundV2.5_generation']
 IMAGE_EDITION_MODELS = ['imagenhub_CycleDiffusion_edition', 'imagenhub_Pix2PixZero_edition', 'imagenhub_Prompt2prompt_edition',
                         'imagenhub_SDEdit_edition', 'imagenhub_InstructPix2Pix_edition', 

model/models/fal_api_models.py

@@ -3,7 +3,10 @@ from PIL import Image
 import requests
 import io
 import os
+import base64
 
+FAL_MODEl_NAME_MAP = {"SDXL": "fast-sdxl", "SDXLTurbo": "fast-turbo-diffusion", "SDXLLightning": "fast-lightning-sdxl",
+                      "LCM": "fast-lcm-diffusion", "PixArtSigma": "pixart-sigma", "StableCascade": "stable-cascade"}
 
 class FalModel():
     def __init__(self, model_name, model_type):
@@ -12,10 +15,24 @@ class FalModel():
         os.environ['FAL_KEY'] = os.environ['FalAPI']
     
     def __call__(self, *args, **kwargs):
+        def decode_data_url(data_url):
+            # Find the start of the Base64 encoded data
+            base64_start = data_url.find(",") + 1
+            if base64_start == 0:
+                raise ValueError("Invalid data URL provided")
+
+            # Extract the Base64 encoded data
+            base64_string = data_url[base64_start:]
+
+            # Decode the Base64 string
+            decoded_bytes = base64.b64decode(base64_string)
+
+            return decoded_bytes
+
         if self.model_type == "text2image":
             assert "prompt" in kwargs, "prompt is required for text2image model"
             handler = fal_client.submit(
-                f"fal-ai/{self.model_name}",
+                f"fal-ai/{FAL_MODEl_NAME_MAP[self.model_name]}",
                 arguments={
                     "prompt": kwargs["prompt"]
                 },
@@ -25,9 +42,14 @@ class FalModel():
                     print('Request in progress')
                     print(event.logs)
             result = handler.get()
+            print(result)
             result_url = result['images'][0]['url']
-            response = requests.get(result_url)
-            result = Image.open(io.BytesIO(response.content))
+            if self.model_name in ["SDXLTurbo", "LCM"]:
+                result_url = io.BytesIO(decode_data_url(result_url))
+                result = Image.open(result_url)
+            else:
+                response = requests.get(result_url)
+                result = Image.open(io.BytesIO(response.content))
             return result
         elif self.model_type == "image2image":
             raise NotImplementedError("image2image model is not implemented yet")

serve/gradio_web.py

@@ -36,6 +36,7 @@ Find out who is the 🥇conditional image generation models! More models are goi
 
     state0 = gr.State()
     state1 = gr.State()
+    anony = True
     gen_func = partial(generate_igm_annoy, models.generate_image_ig_parallel_anony)
     
     gr.Markdown(notice_markdown, elem_id="notice_markdown")
@@ -187,6 +188,7 @@ def build_side_by_side_ui_named(models):
 
     state0 = gr.State()
     state1 = gr.State()
+    anony = False
     gen_func = partial(generate_igm, models.generate_image_ig_parallel)
     gr.Markdown(notice_markdown, elem_id="notice_markdown")
 
@@ -199,6 +201,7 @@ def build_side_by_side_ui_named(models):
                     interactive=True,
                     show_label=False,
                     container=False,
+                    allow_custom_value=True
                     )
             with gr.Column():
                 model_selector_right = gr.Dropdown(
@@ -207,6 +210,7 @@ def build_side_by_side_ui_named(models):
                     interactive=True,
                     show_label=False,
                     container=False,
+                    allow_custom_value=True
                     )
         with gr.Row():
             with gr.Accordion("🔍 Expand to see all model descriptions", open=False):
@@ -253,8 +257,8 @@ def build_side_by_side_ui_named(models):
                   ["A futuristic hopeful busy city, purple and green color scheme", os.path.join("./examples", "city.jpg")]],
         inputs = [textbox, dummy_img_output])
 
-    model_selector_left.change(clear_history_side_by_side, inputs=None, outputs=[state0, state1, textbox, chatbot_left, chatbot_right], api_name="model_selector_left_side_by_side")
-    model_selector_right.change(clear_history_side_by_side, inputs=None, outputs=[state0, state1, textbox, chatbot_left, chatbot_right], api_name="model_selector_right_side_by_side")
+    # model_selector_left.change(clear_history_side_by_side, inputs=None, outputs=[state0, state1, textbox, chatbot_left, chatbot_right], api_name="model_selector_left_side_by_side")
+    # model_selector_right.change(clear_history_side_by_side, inputs=None, outputs=[state0, state1, textbox, chatbot_left, chatbot_right], api_name="model_selector_right_side_by_side")
     
     btn_list = [leftvote_btn, rightvote_btn, tie_btn, bothbad_btn, regenerate_btn, clear_btn]
 

serve/gradio_web_image_editing.py

@@ -27,7 +27,7 @@ def build_side_by_side_ui_anony_ie(models):
 - Wait to see the results after edition.
 - Click "New Round" to start a new round.
 - Vote won't be counted if model identity is revealed during generation.
-- The model could output a totally black image or noise. It's not a bug but the failure case of the model.
+- The model coulmodel_selector_left.changed output a totally black image or noise. It's not a bug but the failure case of the model.
 
 ## 🏆 Arena Elo 
 Find out who is the 🥇conditional image edition models!
@@ -303,14 +303,14 @@ def build_side_by_side_ui_named_ie(models):
         inputs=[textbox_source, textbox_target, textbox_instruct, source_image])
     
 
-    model_selector_left.change(
-        clear_history_side_by_side_ie, 
-        inputs=None, 
-        outputs=[state0, state1, textbox_source, textbox_target, textbox_instruct, 
-                 source_image, chatbot_left, chatbot_right
-                 ], 
-        api_name="model_selector_left_side_by_side")
-    model_selector_right.change(clear_history_side_by_side_ie, inputs=None, outputs=[state0, state1, textbox_source, textbox_target, textbox_instruct, source_image, chatbot_left, chatbot_right], api_name="model_selector_right_side_by_side")
+    # model_selector_left.change(
+    #     clear_history_side_by_side_ie,
+    #     inputs=None,
+    #     outputs=[state0, state1, textbox_source, textbox_target, textbox_instruct,
+    #              source_image, chatbot_left, chatbot_right
+    #              ],
+    #     api_name="model_selector_left_side_by_side")
+    # model_selector_right.change(clear_history_side_by_side_ie, inputs=None, outputs=[state0, state1, textbox_source, textbox_target, textbox_instruct, source_image, chatbot_left, chatbot_right], api_name="model_selector_right_side_by_side")
 
 
     btn_list = [leftvote_btn, rightvote_btn, tie_btn, bothbad_btn, regenerate_btn, clear_btn]

serve/gradio_web_video_generation.py

@@ -259,12 +259,12 @@ def build_side_by_side_ui_named_vg(models):
                    'https://fal-cdn.batuhan-941.workers.dev/files/monkey/5-vUtY4_bHAhTtevZx16K.mp4']],
         inputs=[textbox, dummy_video_output])
 
-    model_selector_left.change(clear_history_side_by_side, inputs=None,
-                               outputs=[state0, state1, textbox, chatbot_left, chatbot_right],
-                               api_name="model_selector_left_side_by_side")
-    model_selector_right.change(clear_history_side_by_side, inputs=None,
-                                outputs=[state0, state1, textbox, chatbot_left, chatbot_right],
-                                api_name="model_selector_right_side_by_side")
+    # model_selector_left.change(clear_history_side_by_side, inputs=None,
+    #                            outputs=[state0, state1, textbox, chatbot_left, chatbot_right],
+    #                            api_name="model_selector_left_side_by_side")
+    # model_selector_right.change(clear_history_side_by_side, inputs=None,
+    #                             outputs=[state0, state1, textbox, chatbot_left, chatbot_right],
+    #                             api_name="model_selector_right_side_by_side")
 
     btn_list = [leftvote_btn, rightvote_btn, tie_btn, bothbad_btn, regenerate_btn, clear_btn]
 

serve/vote_utils.py

@@ -172,8 +172,12 @@ def leftvote_last_response_igm(
     vote_last_response_igm(
         [state0, state1], "leftvote", [model_selector0, model_selector1], request
     )
-    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
-
+    if model_selector0 == gr.State(""):
+        return ("",) + (disable_btn,) * 4 + (
+        gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True),
+        gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
+    else:
+        return ("",) + (disable_btn,) * 4 + (gr.Markdown('', visible=True), gr.Markdown('', visible=True))
 
 def rightvote_last_response_igm(
     state0, state1, model_selector0, model_selector1, request: gr.Request
@@ -182,7 +186,10 @@ def rightvote_last_response_igm(
     vote_last_response_igm(
         [state0, state1], "rightvote", [model_selector0, model_selector1], request
     )
-    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    if model_selector0 == gr.State(""):
+        return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
+    else:
+        return ("",) + (disable_btn,) * 4 + (gr.Markdown('', visible=True), gr.Markdown('', visible=True))
 
 
 def tievote_last_response_igm(
@@ -192,7 +199,12 @@ def tievote_last_response_igm(
     vote_last_response_igm(
         [state0, state1], "tievote", [model_selector0, model_selector1], request
     )
-    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    if model_selector0 == gr.State(""):
+        return ("",) + (disable_btn,) * 4 + (
+        gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True),
+        gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
+    else:
+        return ("",) + (disable_btn,) * 4 + (gr.Markdown('', visible=True), gr.Markdown('', visible=True))
 
 
 def bothbad_vote_last_response_igm(
@@ -202,7 +214,12 @@ def bothbad_vote_last_response_igm(
     vote_last_response_igm(
         [state0, state1], "bothbad_vote", [model_selector0, model_selector1], request
     )
-    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    if model_selector0 == gr.State(""):
+        return ("",) + (disable_btn,) * 4 + (
+            gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True),
+            gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
+    else:
+        return ("",) + (disable_btn,) * 4 + (gr.Markdown('', visible=True), gr.Markdown('', visible=True))
 
 ## Image Editing (IE) Single Model Direct Chat
 
@@ -237,7 +254,7 @@ def leftvote_last_response_iem(
     #     "### Model B: " + state1.model_name,
     # )
     # names = (state0.model_name, state1.model_name)
-    names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    names = (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
     return names + ("", "", gr.Image(height=512, width=512, type="pil"), "") + (disable_btn,) * 4
         
 def rightvote_last_response_iem(
@@ -251,7 +268,7 @@ def rightvote_last_response_iem(
     #     "### Model A: " + state0.model_name,
     #     "### Model B: " + state1.model_name,
     # )
-    names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    names = (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
     return names + ("", "", gr.Image(height=512, width=512, type="pil"), "") + (disable_btn,) * 4
 
 def tievote_last_response_iem(
@@ -261,7 +278,7 @@ def tievote_last_response_iem(
     vote_last_response_iem(
         [state0, state1], "tievote", [model_selector0, model_selector1], request
     )
-    names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    names = (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
     return names + ("", "", gr.Image(height=512, width=512, type="pil"), "") + (disable_btn,) * 4
 
 def bothbad_vote_last_response_iem(
@@ -271,7 +288,7 @@ def bothbad_vote_last_response_iem(
     vote_last_response_iem(
         [state0, state1], "bothbad_vote", [model_selector0, model_selector1], request
     )
-    names = (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    names = (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
     return names + ("", "", gr.Image(height=512, width=512, type="pil"), "") + (disable_btn,) * 4
 
 
@@ -304,7 +321,7 @@ def leftvote_last_response_vgm(
     vote_last_response_vgm(
         [state0, state1], "leftvote", [model_selector0, model_selector1], request
     )
-    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
 
 
 def rightvote_last_response_vgm(
@@ -314,7 +331,7 @@ def rightvote_last_response_vgm(
     vote_last_response_vgm(
         [state0, state1], "rightvote", [model_selector0, model_selector1], request
     )
-    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
 
 
 def tievote_last_response_vgm(
@@ -324,7 +341,7 @@ def tievote_last_response_vgm(
     vote_last_response_vgm(
         [state0, state1], "tievote", [model_selector0, model_selector1], request
     )
-    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
 
 
 def bothbad_vote_last_response_vgm(
@@ -334,7 +351,7 @@ def bothbad_vote_last_response_vgm(
     vote_last_response_vgm(
         [state0, state1], "bothbad_vote", [model_selector0, model_selector1], request
     )
-    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name}", visible=True), gr.Markdown(f"### Model B: {state1.model_name}", visible=True))
+    return ("",) + (disable_btn,) * 4 + (gr.Markdown(f"### Model A: {state0.model_name.split('_')[1]}", visible=True), gr.Markdown(f"### Model B: {state1.model_name.split('_')[1]}", visible=True))
 
 share_js = """
 function (a, b, c, d) {