Update app.py

app.py

@@ -4,14 +4,12 @@ from transformers import AutoConfig, AutoModelForCausalLM
 from janus.models import MultiModalityCausalLM, VLChatProcessor
 from janus.utils.io import load_pil_images
 from PIL import Image
-
 import numpy as np
 import os
 import time
 from Upsample import RealESRGAN
 import spaces  # Import spaces for ZeroGPU compatibility
 
-
 # Load model and processor
 model_path = "deepseek-ai/Janus-Pro-7B"
 config = AutoConfig.from_pretrained(model_path)
@@ -31,16 +29,15 @@ cuda_device = 'cuda' if torch.cuda.is_available() else 'cpu'
 
 # SR model
 sr_model = RealESRGAN(torch.device('cuda' if torch.cuda.is_available() else 'cpu'), scale=2)
-sr_model.load_weights(f'weights/RealESRGAN_x2.pth', download=False)
+sr_model.load_weights('weights/RealESRGAN_x2.pth', download=False)
 
 @torch.inference_mode()
-@spaces.GPU(duration=120) 
-# Multimodal Understanding function
+@spaces.GPU(duration=120)
 def multimodal_understanding(image, question, seed, top_p, temperature):
     # Clear CUDA cache before generating
     torch.cuda.empty_cache()
     
-    # set seed
+    # Set seed
     torch.manual_seed(seed)
     np.random.seed(seed)
     torch.cuda.manual_seed(seed)
@@ -54,12 +51,11 @@ def multimodal_understanding(image, question, seed, top_p, temperature):
         {"role": "<|Assistant|>", "content": ""},
     ]
     
-    pil_images = [Image.fromarray(image)]
+    pil_images = [Image.fromarray(image)] if isinstance(image, np.ndarray) else [image]
     prepare_inputs = vl_chat_processor(
         conversations=conversation, images=pil_images, force_batchify=True
     ).to(cuda_device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16)
     
-    
     inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
     
     outputs = vl_gpt.language_model.generate(
@@ -78,16 +74,9 @@ def multimodal_understanding(image, question, seed, top_p, temperature):
     answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
     return answer
 
-
-def generate(input_ids,
-             width,
-             height,
-             temperature: float = 1,
-             parallel_size: int = 5,
-             cfg_weight: float = 5,
-             image_token_num_per_image: int = 576,
-             patch_size: int = 16):
-    # Clear CUDA cache before generating
+def generate(input_ids, width, height, temperature: float = 1,
+             parallel_size: int = 5, cfg_weight: float = 5,
+             image_token_num_per_image: int = 576, patch_size: int = 16):
     torch.cuda.empty_cache()
     
     tokens = torch.zeros((parallel_size * 2, len(input_ids)), dtype=torch.int).to(cuda_device)
@@ -102,8 +91,8 @@ def generate(input_ids,
     for i in range(image_token_num_per_image):
         with torch.no_grad():
             outputs = vl_gpt.language_model.model(inputs_embeds=inputs_embeds,
-                                                use_cache=True,
-                                                past_key_values=pkv)
+                                                  use_cache=True,
+                                                  past_key_values=pkv)
             pkv = outputs.past_key_values
             hidden_states = outputs.last_hidden_state
             logits = vl_gpt.gen_head(hidden_states[:, -1, :])
@@ -117,34 +106,22 @@ def generate(input_ids,
 
             img_embeds = vl_gpt.prepare_gen_img_embeds(next_token)
             inputs_embeds = img_embeds.unsqueeze(dim=1)
-
     
-
     patches = vl_gpt.gen_vision_model.decode_code(generated_tokens.to(dtype=torch.int),
-                                                 shape=[parallel_size, 8, width // patch_size, height // patch_size])
-
+                                                  shape=[parallel_size, 8, width // patch_size, height // patch_size])
     return generated_tokens.to(dtype=torch.int), patches
 
 def unpack(dec, width, height, parallel_size=5):
     dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)
     dec = np.clip((dec + 1) / 2 * 255, 0, 255)
-
     visual_img = np.zeros((parallel_size, width, height, 3), dtype=np.uint8)
     visual_img[:, :, :] = dec
-
     return visual_img
 
-
-
 @torch.inference_mode()
-@spaces.GPU(duration=120)  # Specify a duration to avoid timeout
-def generate_image(prompt,
-                   seed=None,
-                   guidance=5,
-                   t2i_temperature=1.0):
-    # Clear CUDA cache and avoid tracking gradients
+@spaces.GPU(duration=120)
+def generate_image(prompt, seed=None, guidance=5, t2i_temperature=1.0):
     torch.cuda.empty_cache()
-    # Set the seed for reproducible results
     if seed is not None:
         torch.manual_seed(seed)
         torch.cuda.manual_seed(seed)
@@ -156,11 +133,12 @@ def generate_image(prompt,
     with torch.no_grad():
         messages = [{'role': '<|User|>', 'content': prompt},
                     {'role': '<|Assistant|>', 'content': ''}]
-        text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(conversations=messages,
-                                                                   sft_format=vl_chat_processor.sft_format,
-                                                                   system_prompt='')
+        text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(
+            conversations=messages,
+            sft_format=vl_chat_processor.sft_format,
+            system_prompt=''
+        )
         text = text + vl_chat_processor.image_start_tag
-        
         input_ids = torch.LongTensor(tokenizer.encode(text))
         output, patches = generate(input_ids,
                                    width // 16 * 16,
@@ -173,95 +151,125 @@ def generate_image(prompt,
                         height // 16 * 16,
                         parallel_size=parallel_size)
 
-        # return [Image.fromarray(images[i]).resize((768, 768), Image.LANCZOS) for i in range(parallel_size)]
         stime = time.time()
         ret_images = [image_upsample(Image.fromarray(images[i])) for i in range(parallel_size)]
         print(f'upsample time: {time.time() - stime}')
         return ret_images
 
-
 @spaces.GPU(duration=60)
 def image_upsample(img: Image.Image) -> Image.Image:
     if img is None:
         raise Exception("Image not uploaded")
-    
     width, height = img.size
-    
     if width >= 5000 or height >= 5000:
         raise Exception("The image is too large.")
-
     global sr_model
     result = sr_model.predict(img.convert('RGB'))
     return result
-        
 
-# Gradio interface
-with gr.Blocks() as demo:
-    gr.Markdown(value="# Multimodal Understanding")
-    with gr.Row():
-        image_input = gr.Image()
-        with gr.Column():
-            question_input = gr.Textbox(label="Question")
-            und_seed_input = gr.Number(label="Seed", precision=0, value=42)
-            top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
-            temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature")
-        
-    understanding_button = gr.Button("Chat")
-    understanding_output = gr.Textbox(label="Response")
+# Custom CSS for a sleek, modern and highly readable interface
+custom_css = """
+body {
+    background: #f0f2f5;
+    font-family: 'Segoe UI', sans-serif;
+    color: #333;
+}
+h1, h2, h3 {
+    font-weight: 600;
+}
+.gradio-container {
+    padding: 20px;
+}
+header {
+    text-align: center;
+    padding: 20px;
+    margin-bottom: 20px;
+}
+header h1 {
+    font-size: 3em;
+    color: #2c3e50;
+}
+.gr-button {
+    background-color: #3498db !important;
+    color: #fff !important;
+    border: none !important;
+    padding: 10px 20px !important;
+    border-radius: 5px !important;
+    font-size: 1em !important;
+}
+.gr-button:hover {
+    background-color: #2980b9 !important;
+}
+.gr-input, .gr-slider, .gr-number, .gr-textbox {
+    border-radius: 5px;
+}
+.gr-gallery-item {
+    border-radius: 10px;
+    overflow: hidden;
+    box-shadow: 0 2px 10px rgba(0,0,0,0.1);
+}
+"""
 
-    examples_inpainting = gr.Examples(
-        label="Multimodal Understanding examples",
-        examples=[
-            [
-                "explain this meme",
-                "doge.png",
-            ],
-            [
-                "Convert the formula into latex code.",
-                "equation.png",
-            ],
-        ],
-        inputs=[question_input, image_input],
-    )
-    
+# Gradio Interface
+with gr.Blocks(css=custom_css, title="Multimodal & T2I Demo") as demo:
+    with gr.Column(variant="panel"):
+        gr.Markdown("<header><h1>Janus Multimodal Demo</h1></header>")
         
-    gr.Markdown(value="# Text-to-Image Generation")
-
-    
-    
-    with gr.Row():
-        cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=5, step=0.5, label="CFG Weight")
-        t2i_temperature = gr.Slider(minimum=0, maximum=1, value=1.0, step=0.05, label="temperature")
-
-    prompt_input = gr.Textbox(label="Prompt. (Prompt in more detail can help produce better images!")
-    seed_input = gr.Number(label="Seed (Optional)", precision=0, value=1234)
-
-    generation_button = gr.Button("Generate Images")
-
-    image_output = gr.Gallery(label="Generated Images", columns=2, rows=2, height=300)
-
-    examples_t2i = gr.Examples(
-        label="Text to image generation examples.",
-        examples=[
-            "Master shifu racoon wearing drip attire as a street gangster.",
-            "The face of a beautiful girl",
-            "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k",
-            "A cute and adorable baby fox with big brown eyes, autumn leaves in the background enchanting,immortal,fluffy, shiny mane,Petals,fairyism,unreal engine 5 and Octane Render,highly detailed, photorealistic, cinematic, natural colors.",
-            "The image features an intricately designed eye set against a circular backdrop adorned with ornate swirl patterns that evoke both realism and surrealism. At the center of attention is a strikingly vivid blue iris surrounded by delicate veins radiating outward from the pupil to create depth and intensity. The eyelashes are long and dark, casting subtle shadows on the skin around them which appears smooth yet slightly textured as if aged or weathered over time.\n\nAbove the eye, there's a stone-like structure resembling part of classical architecture, adding layers of mystery and timeless elegance to the composition. This architectural element contrasts sharply but harmoniously with the organic curves surrounding it. Below the eye lies another decorative motif reminiscent of baroque artistry, further enhancing the overall sense of eternity encapsulated within each meticulously crafted detail. \n\nOverall, the atmosphere exudes a mysterious aura intertwined seamlessly with elements suggesting timelessness, achieved through the juxtaposition of realistic textures and surreal artistic flourishes. Each component\u2014from the intricate designs framing the eye to the ancient-looking stone piece above\u2014contributes uniquely towards creating a visually captivating tableau imbued with enigmatic allure.",
-        ],
-        inputs=prompt_input,
-    )
-    
-    understanding_button.click(
-        multimodal_understanding,
-        inputs=[image_input, question_input, und_seed_input, top_p, temperature],
-        outputs=understanding_output
-    )
-    
-    generation_button.click(
-        fn=generate_image,
-        inputs=[prompt_input, seed_input, cfg_weight_input, t2i_temperature],
-        outputs=image_output
-    )
+        with gr.Tabs():
+            with gr.TabItem("Multimodal Understanding"):
+                gr.Markdown("### Chat with Images")
+                with gr.Row():
+                    image_input = gr.Image(label="Upload Image", type="numpy", tool="editor")
+                    with gr.Column():
+                        question_input = gr.Textbox(label="Question", placeholder="Enter your question about the image here...", lines=4)
+                        und_seed_input = gr.Number(label="Seed", precision=0, value=42)
+                        top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="Top_p")
+                        temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="Temperature")
+                understanding_button = gr.Button("Chat", elem_id="understanding-button")
+                understanding_output = gr.Textbox(label="Response", lines=6)
+                with gr.Accordion("Examples", open=False):
+                    gr.Examples(
+                        label="Multimodal Understanding Examples",
+                        examples=[
+                            ["explain this meme", "doge.png"],
+                            ["Convert the formula into LaTeX code.", "equation.png"],
+                        ],
+                        inputs=[question_input, image_input],
+                    )
+                understanding_button.click(
+                    multimodal_understanding,
+                    inputs=[image_input, question_input, und_seed_input, top_p, temperature],
+                    outputs=understanding_output,
+                )
+            
+            with gr.TabItem("Text-to-Image Generation"):
+                gr.Markdown("### Generate Images from Text")
+                with gr.Row():
+                    prompt_input = gr.Textbox(label="Prompt", placeholder="Enter detailed prompt for image generation...", lines=4)
+                with gr.Row():
+                    seed_input = gr.Number(label="Seed (Optional)", precision=0, value=1234)
+                    cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=5, step=0.5, label="CFG Weight")
+                    t2i_temperature = gr.Slider(minimum=0, maximum=1, value=1.0, step=0.05, label="Temperature")
+                generation_button = gr.Button("Generate Images", elem_id="generation-button")
+                image_output = gr.Gallery(label="Generated Images", columns=2, rows=2, height=300)
+                with gr.Accordion("Examples", open=False):
+                    gr.Examples(
+                        label="Text-to-Image Examples",
+                        examples=[
+                            "Master shifu racoon wearing drip attire as a street gangster.",
+                            "The face of a beautiful girl",
+                            "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k",
+                            "A cute and adorable baby fox with big brown eyes, autumn leaves in the background enchanting, immortal, fluffy, shiny mane, petals, fairyism, unreal engine 5 and Octane Render, highly detailed, photorealistic, cinematic, natural colors.",
+                            "An intricately designed eye with ornate swirl patterns, vivid blue iris, and classical architectural motifs, exuding mysterious timelessness."
+                        ],
+                        inputs=prompt_input,
+                    )
+                generation_button.click(
+                    fn=generate_image,
+                    inputs=[prompt_input, seed_input, cfg_weight_input, t2i_temperature],
+                    outputs=image_output,
+                )
+        
+    gr.Markdown("<footer style='text-align:center; padding:20px 0;'>Join our community on <a href='https://discord.gg/openfreeai' target='_blank'>Discord</a></footer>")
 
-demo.launch(share=True)
+demo.launch(share=True)