Update app.py

app.py

@@ -9,7 +9,7 @@ import tensorflow as tf
 import tensorflow_hub as hub
 import io
 from sklearn.metrics.pairwise import cosine_similarity
-import tempfile # Add this import
+import tempfile
 import logging
 
 # Configure logging
@@ -73,34 +73,44 @@ def save_dataframe_to_csv(df):
     # Return the file path (no need to reopen the file with "rb" mode)
     return temp_file_path
 
-# Main function to perform image captioning and image-text matching
-def process_images_and_statements(image, file_name):
-    logging.info('Starting process_images_and_statements')
-
-    # Generate image caption for the uploaded image using git-large-r-textcaps
-    caption = generate_caption(git_processor_large_textcaps, git_model_large_textcaps, image)
-
-    # Initialize an empty list to store the results
-    results_list = []
-
-    # Loop through each predefined statement
-    for statement in statements:
-        # Compute textual similarity and ITM score, and combine the scores ...
-
-        # Append the result to the results_list
-        results_list.append({
-            'Image File Name': file_name,  # Include the image file name
-            'Statement': statement,
-            'Generated Caption': caption,
-            'Textual Similarity Score': f"{textual_similarity_score:.2f}%",
-            'ITM Score': f"{itm_score_statement:.2f}%",
-            'Final Combined Score': f"{final_score:.2f}%"
-        })
-
-    # Convert the results_list to a DataFrame using pandas.concat
-    results_df = pd.concat([pd.DataFrame([result]) for result in results_list], ignore_index=True)
-
-    logging.info('Finished process_images_and_statements')
+# Main function to perform image captioning and image-text matching for multiple images
+def process_images_and_statements(files):
+    # Initialize an empty list to store the results for all images
+    all_results_list = []
+
+    # Loop through each uploaded file (image)
+    for file_name, image in files.items():
+        # Generate image caption for the uploaded image using git-large-r-textcaps
+        caption = generate_caption(git_processor_large_textcaps, git_model_large_textcaps, image)
+
+        # Loop through each predefined statement
+        for statement in statements:
+            # Compute textual similarity between caption and statement
+            textual_similarity_score = (compute_textual_similarity(caption, statement) * 100) # Multiply by 100
+
+            # Compute ITM score for the image-statement pair
+            itm_score_statement = (compute_itm_score(image, statement) * 100) # Multiply by 100
+
+            # Define weights for combining textual similarity score and image-statement ITM score (adjust as needed)
+            weight_textual_similarity = 0.5
+            weight_statement = 0.5
+
+            # Combine the two scores using a weighted average
+            final_score = ((weight_textual_similarity * textual_similarity_score) +
+                           (weight_statement * itm_score_statement))
+
+            # Append the result to the all_results_list
+            all_results_list.append({
+                'Image File Name': file_name,  # Include the image file name
+                'Statement': statement,
+                'Generated Caption': caption,
+                'Textual Similarity Score': f"{textual_similarity_score:.2f}%", # Format as percentage with two decimal places
+                'ITM Score': f"{itm_score_statement:.2f}%", # Format as percentage with two decimal places
+                'Final Combined Score': f"{final_score:.2f}%" # Format as percentage with two decimal places
+            })
+
+    # Convert the all_results_list to a DataFrame using pandas.concat
+    results_df = pd.concat([pd.DataFrame([result]) for result in all_results_list], ignore_index=True)
 
     # Save results_df to a CSV file
     csv_results = save_dataframe_to_csv(results_df)
@@ -108,9 +118,8 @@ def process_images_and_statements(image, file_name):
     # Return both the DataFrame and the CSV data for the Gradio interface
     return results_df, csv_results
 
-# Gradio interface with File input to receive both the image and the file name
-#image_input = gr.inputs.File(type="image", label="Upload Image")
-image_input = gr.inputs.Image(label="Upload Images", multiple=True)
+# Gradio interface with File input to receive multiple images and file names
+image_input = gr.inputs.File(file_count="multiple", type="file", label="Upload Images")
 output_df = gr.outputs.Dataframe(type="pandas", label="Results")
 output_csv = gr.outputs.File(label="Download CSV")
 
@@ -123,4 +132,5 @@ iface = gr.Interface(
     css=".output { flex-direction: column; } .output .outputs { width: 100%; }" # Custom CSS
 )
 
-iface.launch()
+iface.launch()
+