Update app.py

app.py

@@ -2,6 +2,23 @@ import torch
 from PIL import Image
 from transformers import AutoProcessor, AutoModelForImageClassification
 import gradio as gr
+import pytesseract
+
+def classify_meme(image: Image.Image):
+    # OCR: extract text from image
+    extracted_text = pytesseract.image_to_string(image)
+
+    # Process image with SigLIP2 model
+    inputs = processor(images=image, return_tensors="pt").to(model.device)
+    with torch.no_grad():
+        outputs = model(**inputs)
+        probs = torch.nn.functional.softmax(outputs.logits, dim=-1)
+    predictions = {labels[i]: float(probs[0][i]) for i in range(len(labels))}
+
+    return {
+        "Predictions": predictions,
+        "Extracted Text": extracted_text.strip()
+    }
 
 # Load model and processor from Hugging Face
 model = AutoModelForImageClassification.from_pretrained("google/siglip2-base-patch16-naflex")