revamped

app.py

@@ -1,34 +1,70 @@
 import tensorflow as tf
 import numpy as np
+import pickle
 from PIL import Image
 import gradio as gr
+from tensorflow.keras.models import Model
+from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2, preprocess_input
+from tensorflow.keras.preprocessing.sequence import pad_sequences
 
-# CNN+LSTM model loaded
+# Load MobileNetV2 model for feature extraction
+mobilenet_model = MobileNetV2(weights="imagenet")
+mobilenet_model = Model(inputs=mobilenet_model.inputs, outputs=mobilenet_model.layers[-2].output)
+
+# Load the trained captioning model
 model = tf.keras.models.load_model("model.h5")
 
-# Define the preprocessing function for the image
+# Load the tokenizer
+with open("tokenizer.pkl", "rb") as tokenizer_file:
+    tokenizer = pickle.load(tokenizer_file)
+
+# Set maximum caption length and start/end tokens
+max_caption_length = 33  # Adjust based on your model's training
+start_token = "startseq"
+end_token = "endseq"
+
+# Define a function to get a word from an index
+def get_word_from_index(index, tokenizer):
+    for word, idx in tokenizer.word_index.items():
+        if idx == index:
+            return word
+    return None
+
+# Preprocess image and extract features
 def preprocess_image(image):
-    # Resize and normalize image to match model's expected input shape
-    image = image.resize((224, 224))  # Modify size based on your model
-    image_array = np.array(image) / 255.0  # Normalize
-    image_array = np.expand_dims(image_array, axis=0)  # Add batch dimension
-    return image_array
+    image = image.resize((224, 224))
+    image_array = np.array(image)
+    image_array = np.expand_dims(image_array, axis=0)
+    image_array = preprocess_input(image_array)
+    return mobilenet_model.predict(image_array, verbose=0)
 
-# Define the function that generates a caption from the image
+# Generate caption from the image features
 def generate_caption(image):
-    preprocessed_image = preprocess_image(image)
+    image_features = preprocess_image(image)
     
-    # Generate a caption from the model
-    # Note: Adjust this if your model requires a sequence start token or has a decoding loop
-    caption_tokens = model.predict(preprocessed_image)
+    caption = start_token
+    for _ in range(max_caption_length):
+        sequence = tokenizer.texts_to_sequences([caption])[0]
+        sequence = pad_sequences([sequence], maxlen=max_caption_length)
+
+        yhat = model.predict([image_features, sequence], verbose=0)
+        predicted_index = np.argmax(yhat)
+        predicted_word = get_word_from_index(predicted_index, tokenizer)
+        
+        # If no valid word or end token is predicted, stop generation
+        if predicted_word is None or predicted_word == end_token:
+            break
+        caption += " " + predicted_word
 
-    return caption_tokens
+    # Remove start and end tokens for final output
+    final_caption = caption.replace(start_token, "").replace(end_token, "").strip()
+    return final_caption
 
-# Define the Gradio interface
+# Define Gradio interface
 iface = gr.Interface(
-    fn=generate_caption,          # The function that generates captions
-    inputs=gr.Image(type="pil"),  # Accept an image input
-    outputs="text",                # Output a text caption
+    fn=generate_caption,                   # Function to generate caption
+    inputs=gr.Image(type="pil"),           # Input an image
+    outputs="text",                        # Output a text caption
     title="Image Captioning Model",
     description="Upload an image, and the model will generate a caption describing it."
 )

requirements.txt

@@ -3,6 +3,5 @@ gradio==5.3.0
 pandas==2.0.3
 Pillow==9.5.0
 torch==2.1.0
-keras==2.15.0
 nltk==3.8.1
 numpy==1.23.5

tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:287c34d4b6f6db43caaba67747a6674377e26f366e19f1c821a5f5be5067a178
+size 347726