Update app.py

app.py

@@ -1,4 +1,6 @@
 import gradio as gr
+from model.config import *
+from PIL import Image
 # import pickle
 # import numpy as np
 # from fastapi import FastAPI,Response
@@ -38,52 +40,61 @@ import torch
 
 #     f1_metric.set(f1)
 
-feature_extractor = ViTImageProcessor.from_pretrained("model")
 
-cap_model = VisionEncoderDecoderModel.from_pretrained("model")
 
-tokenizer = AutoTokenizer.from_pretrained("model")
-print("tokenizer --",tokenizer)
 
-device = "cuda" if torch.cuda.is_available() else "cpu"
+model = VisionEncoderDecoderModel.from_encoder_decoder_pretrained(encoder._name_or_path, decoder._name_or_path)
 
-cap_model.to(device)
 
-def generate_caption(processor, model, image, tokenizer=None):
-  # max_length = 16
-  # num_beams = 4
-  # gen_kwargs = {"max_length": max_length, "num_beams": num_beams}
+tokenizer = AutoTokenizer.from_pretrained(decoder._name_or_path)
+tokenizer.pad_token = tokenizer.unk_token
 
-  # pixel_values = feature_extractor(images=image, return_tensors="pt").pixel_values
-  # pixel_values = pixel_values.to(device)
 
-  # output_ids = model.generate(pixel_values, **gen_kwargs)
 
-  # preds = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
-  # preds = [pred.strip() for pred in preds]
-  # return preds
-    inputs = processor(images=image, return_tensors="pt").to(device)
-    print("inputs",inputs)
+# feature_extractor = ViTImageProcessor.from_pretrained("model")
+
+# cap_model = VisionEncoderDecoderModel.from_pretrained("model")
+
+# tokenizer = AutoTokenizer.from_pretrained("model")
+
+
+
+
+
+
+
+
+
+# device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# cap_model.to(device)
+
+# def generate_caption(model, image, tokenizer=None):
+
     
-    generated_ids = model.generate(pixel_values=inputs.pixel_values)
-    print("generated_ids",generated_ids)
-
-    if tokenizer is not None:
-        print("tokenizer not null--",tokenizer)
-        generated_caption = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
-    else:
-        print("tokenizer null--",tokenizer)
-        generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+#     generated_ids = model.generate(pixel_values=inputs.pixel_values)
+#     print("generated_ids",generated_ids)
+
+#     if tokenizer is not None:
+#         print("tokenizer not null--",tokenizer)
+#         generated_caption = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+#     else:
+#         print("tokenizer null--",tokenizer)
+#         generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
    
-    return generated_caption
+#     return generated_caption
 
 def predict_event(image):
     
 
+    img =  Image.open(image).convert("RGB")
+
+    generated_caption = tokenizer.decode(model.generate(feature_extractor(img, return_tensors="pt").pixel_values.to("cuda"))[0])
 
-    caption_vitgpt = generate_caption(feature_extractor, cap_model, image, tokenizer)
+    # caption_vitgpt = generate_caption(model, image)
+    #caption_vitgpt = generate_caption(feature_extractor, cap_model, image, tokenizer)
 
-    return caption_vitgpt
+    return '\033[96m' +generated_caption[:85]+ '\033[0m'