Update app.py

app.py

@@ -1,18 +1,20 @@
 import gradio as gr
 import peft
 from peft import LoraConfig
-from transformers import AutoTokenizer,BitsAndBytesConfig, AutoModelForCausalLM, CLIPVisionModel, AutoProcessor
+from transformers import AutoTokenizer, BitsAndBytesConfig, AutoModelForCausalLM, CLIPVisionModel, AutoProcessor
 import torch
 from peft import PeftModel
 import torch.nn as nn
 import whisperx
 import os
+
+# Load models
 clip_model_name = "openai/clip-vit-base-patch32"
 phi_model_name  = "microsoft/phi-2"
 tokenizer  = AutoTokenizer.from_pretrained(phi_model_name, trust_remote_code=True)
 processor  = AutoProcessor.from_pretrained(clip_model_name)
 tokenizer.pad_token = tokenizer.eos_token
-IMAGE_TOKEN_ID = 23893 # token for word comment
+IMAGE_TOKEN_ID = 23893  # token for word comment
 device = "cuda" if torch.cuda.is_available() else "cpu"
 clip_embed = 768
 phi_embed  = 2560
@@ -28,35 +30,32 @@ class SimpleResBlock(nn.Module):
             nn.GELU(),
             nn.Linear(phi_embed, phi_embed)
         )
+    
     def forward(self, x):
         x = self.pre_norm(x)
         return x + self.proj(x)
-        
-# models
+
+# Initialize models
 clip_model = CLIPVisionModel.from_pretrained(clip_model_name).to(device)
 projection = torch.nn.Linear(clip_embed, phi_embed).to(device)
 resblock = SimpleResBlock(phi_embed).to(device)
-phi_model = AutoModelForCausalLM.from_pretrained(phi_model_name,trust_remote_code=True).to(device)
-# Assuming you have defined 'device' and 'compute_type' elsewhere
+phi_model = AutoModelForCausalLM.from_pretrained(phi_model_name, trust_remote_code=True).to(device)
 audio_model = whisperx.load_model("tiny", device, compute_type=compute_type, asr_options={'max_new_tokens': 2048, 'clip_timestamps': True, 'hallucination_silence_threshold': 0.25})
 
-# load weights
-model_to_merge = PeftModel.from_pretrained(phi_model,os.path.join(os.getcwd(), 'model_chkpt/lora_adaptor'))
-merged_model   = model_to_merge.merge_and_unload()
-projection.load_state_dict(torch.load(os.path.join(os.getcwd(),'model_chkpt/finetunned_projection.pth'),map_location=torch.device(device)))
-resblock.load_state_dict(torch.load(os.path.join(os.getcwd(),'model_chkpt/finetuned_resblock.pth'),map_location=torch.device(device)))
+# Load weights
+model_to_merge = PeftModel.from_pretrained(phi_model, os.path.join(os.getcwd(), 'model_chkpt/lora_adaptor'))
+merged_model = model_to_merge.merge_and_unload()
+projection.load_state_dict(torch.load(os.path.join(os.getcwd(), 'model_chkpt/finetunned_projection.pth'), map_location=device))
+resblock.load_state_dict(torch.load(os.path.join(os.getcwd(), 'model_chkpt/finetuned_resblock.pth'), map_location=device))
 
-def model_generate_ans(img=None,img_audio=None,val_q=None):
-
-    max_generate_length = 100
+def model_generate_ans(img=None, img_audio=None, val_q=None, max_length=100):
     val_combined_embeds = []
     
     with torch.no_grad():
-    
-        # image
+        # Image processing
         if img is not None:
-            image_processed  = processor(images=img, return_tensors="pt").to(device)
-            clip_val_outputs = clip_model(**image_processed).last_hidden_state[:,1:,:]
+            image_processed = processor(images=img, return_tensors="pt").to(device)
+            clip_val_outputs = clip_model(**image_processed).last_hidden_state[:, 1:, :]
             val_image_embeds = projection(clip_val_outputs)
             val_image_embeds = resblock(val_image_embeds).to(torch.float16)
             
@@ -66,60 +65,54 @@ def model_generate_ans(img=None,img_audio=None,val_q=None):
             val_combined_embeds.append(val_image_embeds)
             val_combined_embeds.append(img_token_embeds)
 
-        # audio
+        # Audio processing
         if img_audio is not None:
             audio_result = audio_model.transcribe(img_audio)
-            audio_text = ''
-            for seg in audio_result['segments']:
-                audio_text += seg['text']
-            audio_text = audio_text.strip()
+            audio_text = ' '.join(seg['text'] for seg in audio_result['segments']).strip()
             audio_tokens = tokenizer(audio_text, return_tensors="pt", return_attention_mask=False)['input_ids'].squeeze(0).to(device)
-            audio_embeds    = merged_model.model.embed_tokens(audio_tokens).unsqueeze(0)
+            audio_embeds = merged_model.model.embed_tokens(audio_tokens).unsqueeze(0)
             val_combined_embeds.append(audio_embeds)
-            
-        # text question
-        if len(val_q) != 0:
-            val_q_tokenised = tokenizer(val_q, return_tensors="pt", return_attention_mask=False)['input_ids'].squeeze(0).to(device)
-            val_q_embeds    = merged_model.model.embed_tokens(val_q_tokenised).unsqueeze(0)
+
+        # Text question processing
+        if val_q:
+            val_q_tokenized = tokenizer(val_q, return_tensors="pt", return_attention_mask=False)['input_ids'].squeeze(0).to(device)
+            val_q_embeds = merged_model.model.embed_tokens(val_q_tokenized).unsqueeze(0)
             val_combined_embeds.append(val_q_embeds)
 
-        val_combined_embeds = torch.cat(val_combined_embeds,dim=1)
-        
-        #val_combined_embeds = torch.cat([val_image_embeds, img_token_embeds, val_q_embeds], dim=1) # 4, 69, 2560
-        predicted_caption = torch.full((1,max_generate_length),50256).to(device)
+        val_combined_embeds = torch.cat(val_combined_embeds, dim=1)
+        predicted_caption = torch.full((1, max_length), 50256).to(device)
     
-        for g in range(max_generate_length):
-            phi_output_logits = merged_model(inputs_embeds=val_combined_embeds)['logits'] # 4, 69, 51200
-            predicted_word_token_logits = phi_output_logits[:, -1, :].unsqueeze(1) # 4,1,51200
-            predicted_word_token = torch.argmax(predicted_word_token_logits, dim = -1) # 4,1
-            predicted_caption[:,g] = predicted_word_token.view(1,-1)
-            next_token_embeds = phi_model.model.embed_tokens(predicted_word_token) # 4,1,2560
-            val_combined_embeds   = torch.cat([val_combined_embeds, next_token_embeds], dim=1)
+        for g in range(max_length):
+            phi_output_logits = merged_model(inputs_embeds=val_combined_embeds)['logits']
+            predicted_word_token_logits = phi_output_logits[:, -1, :].unsqueeze(1)
+            predicted_word_token = torch.argmax(predicted_word_token_logits, dim=-1)
+            predicted_caption[:, g] = predicted_word_token.view(1, -1)
+            next_token_embeds = phi_model.model.embed_tokens(predicted_word_token)
+            val_combined_embeds = torch.cat([val_combined_embeds, next_token_embeds], dim=1)
             
-        predicted_captions_decoded = tokenizer.batch_decode(predicted_caption,ignore_index = 50256)[0]
+        predicted_captions_decoded = tokenizer.batch_decode(predicted_caption, skip_special_tokens=True)[0]
     
     return predicted_captions_decoded
-    
 
+# Gradio Interface
 with gr.Blocks() as demo:
-
     gr.Markdown(
-    """
-    # Chat with MultiModal GPT !
-    Build using combining clip model and phi-2 model.
-    """
+        """
+        # Chat with MultiModal GPT!
+        Combining CLIP model and Phi-2 model for multimodal understanding.
+        """
     )
 
-    # app GUI
     with gr.Row():
         with gr.Column():
-            img_input    = gr.Image(label='Image',type="pil")
-            img_audio    = gr.Audio(label="Audio Query", sources=['microphone', 'upload'], type='filepath')
-            img_question = gr.Text(label ='Text Query')
+            img_input = gr.Image(label='Image', type="pil", tool="editor")
+            img_audio = gr.Audio(label="Audio Query", sources=['microphone', 'upload'], type='filepath')
+            img_question = gr.Textbox(label='Text Query', placeholder='Type your question here...')
+            max_length = gr.Slider(label='Max Length of Response', minimum=1, maximum=300, value=100, step=1)
         with gr.Column():
-            img_answer   = gr.Text(label ='Answer')
+            img_answer = gr.Textbox(label='Answer', interactive=False)
 
     section_btn = gr.Button("Submit")
-    section_btn.click(model_generate_ans, inputs=[img_input,img_audio,img_question], outputs=[img_answer])
+    section_btn.click(model_generate_ans, inputs=[img_input, img_audio, img_question, max_length], outputs=[img_answer])
     
 demo.launch()