Upload 5 files

BharatCaptioner.py

@@ -0,0 +1,148 @@
+import os
+import tensorflow as tf
+
+class_labels = ['The Agra Fort', 'Ajanta Caves', 'Alai Darwaza', 'Amarnath Temple', 'The Amber Fort', 'Basilica of Bom Jesus', 'Brihadisvara Temple', 'Charar-e-Sharief shrine', 'Charminar', 'Chhatrapati Shivaji Terminus', 'Chota Imambara', 'Dal Lake', 'The Elephanta Caves', 'Ellora Caves', 'Fatehpur Sikri', 'Gateway of India', 'Ghats in Varanasi', 'Gol Gumbaz', 'Golden Temple', 'Group of Monuments at Mahabalipuram', 'Hampi', 'Hawa Mahal', "Humayun's Tomb", 'The India gate', 'Iron Pillar', 'Jagannath Temple, Puri', 'Jageshwar', 'Jama Masjid', 'Jamali Kamali Tomb', 'Jantar Mantar, Jaipur', 'Jantar Mantar, New Delhi', 'Kedarnath Temple', 'Khajuraho Temple', 'Konark Sun Temple', 'Mahabodhi Temple', 'Meenakshi Temple', 'Nalanda mahavihara', 'Parliament House, New Delhi', 'Qutb Minar', 'Qutb Minar Complex', 'Ram Mandir', 'Rani ki Vav', 'Rashtrapati Bhavan', 'The Red Fort', 'Sanchi', 'Supreme Court of India', 'Swaminarayan Akshardham (Delhi)', 'Taj Hotels', 'The Lotus Temple', 'The Mysore Palace', 'The Statue of Unity', 'The Taj Mahal', 'Vaishno Devi Temple', 'Venkateswara Temple, Tirumala', 'Victoria Memorial, Kolkata', 'Vivekananda Rock Memorial']
+
+import gdown
+import os
+import tensorflow as tf
+
+# URL with the file ID
+#v3 - https://drive.google.com/file/d/1psuUF80mDO3EZkx2Rl9F5EkqbVHSxFqI/view?usp=sharing
+#v4 - https://drive.google.com/file/d/10weM6bZ9R8xVl0KxKig9WRzss4OpIa7f/view?usp=sharing
+#v5 - https://drive.google.com/file/d/1PXixJsrUaVcHEEC-jDlv4tHT2qrCrf5c/view?usp=sharing
+file_id = '1PXixJsrUaVcHEEC-jDlv4tHT2qrCrf5c'  # Replace with your file ID
+url = f'https://drive.google.com/uc?id={file_id}'
+
+# Output path to save the model
+output = 'LandmarkClassifierV5.h5'  # Replace with your model file name
+
+# Download the file if it doesn't exist
+if not os.path.exists(output):
+    gdown.download(url, output, quiet=False)
+
+# Load your model
+def load_model():
+    # Replace with code to load your actual model
+    loaded_model = tf.keras.models.load_model(output)
+    return loaded_model
+
+model = load_model()
+
+# Print model summary to verify
+model.summary()
+
+# Predict on a new image
+from tensorflow.keras.preprocessing import image
+import numpy as np
+
+import requests
+import numpy as np
+from PIL import Image
+from io import BytesIO
+import matplotlib.pyplot as plt
+
+def identify_landmark(img):
+    # Preprocess the image
+    img = img.resize((224, 224))
+    img_array = image.img_to_array(img)
+    img_array = np.expand_dims(img_array, axis=0)
+    img_array /= 255.0
+
+    # Get predictions
+    predictions = model.predict(img_array)
+
+    # Get the index of the class with the highest probability
+    predicted_class_index = np.argmax(predictions[0])
+
+    # Get the probability of the predicted class
+    predicted_probability = predictions[0][predicted_class_index]
+
+    # Map the predicted class index to the class label
+    predicted_class_label = class_labels[predicted_class_index]
+    
+    return predicted_class_label,predicted_probability
+
+
+# def identify_landmark(img):
+#     img1 = img.copy()
+
+#     # Preprocess the image
+#     img = img.resize((224, 224))
+#     img_array = image.img_to_array(img)
+#     img_array = np.expand_dims(img_array, axis=0)
+#     img_array /= 255.0
+
+#     # Get predictions
+#     predictions = model.predict(img_array)
+
+#     # Get the index of the class with the highest probability
+#     predicted_class_index = np.argmax(predictions[0])
+
+#     # Map the predicted class index to the class label
+#     return class_labels[predicted_class_index]
+
+
+# def generate_landmark_path(img_path):
+#     img = image.load_img(img_path, target_size=(224, 224))
+#     img1 = img.copy()
+
+#     # Preprocess the image
+#     img = img.resize((224, 224))
+#     img_array = image.img_to_array(img)
+#     img_array = np.expand_dims(img_array, axis=0)
+#     img_array /= 255.0
+
+#     # Get predictions
+#     predictions = model.predict(img_array)
+
+#     # Get the index of the class with the highest probability
+#     predicted_class_index = np.argmax(predictions[0])
+
+#     # Map the predicted class index to the class label
+#     plt.imshow(img1)
+#     plt.axis("off")
+#     plt.title(class_labels[predicted_class_index])
+#     plt.show()
+#     return wikipedia.summary(class_labels[predicted_class_index])
+
+
+# def generate_landmark_url(img_url):
+#     try:
+#         # Download the image from the URL
+#         response = requests.get(img_url)
+#         response.raise_for_status()  # Check if the request was successful
+
+#         # Open the image
+#         img = Image.open(BytesIO(response.content))
+#         img1 = img.copy()
+
+#         # Preprocess the image
+#         img = img.resize((224, 224))
+#         img_array = image.img_to_array(img)
+#         img_array = np.expand_dims(img_array, axis=0)
+#         img_array /= 255.0
+
+#         # Get predictions
+#         predictions = model.predict(img_array)
+
+#         # Get the index of the class with the highest probability
+#         predicted_class_index = np.argmax(predictions[0])
+
+#         # Map the predicted class index to the class label
+#         plt.imshow(img1)
+#         plt.axis("off")
+#         plt.title(class_labels[predicted_class_index])
+#         plt.show()
+
+#         return wikipedia.summary(class_labels[predicted_class_index])
+
+#     except requests.exceptions.RequestException as e:
+#         print(f"Error downloading the image: {e}")
+#         return "Invalid image URL."
+#     except IOError as e:
+#         print(f"Error opening the image: {e}")
+#         return "Invalid image file."
+#     except Exception as e:
+#         print(f"An error occurred: {e}")
+#         return "An error occurred while processing the image."

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Harshal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app.py

@@ -0,0 +1,183 @@
+import os
+import json
+import streamlit as st
+from PIL import Image, UnidentifiedImageError, ExifTags
+import requests
+from io import BytesIO
+import wikipedia
+from easygoogletranslate import EasyGoogleTranslate
+from BharatCaptioner import identify_landmark
+from groq import Groq
+import hashlib
+
+# Initialize EasyGoogleTranslate
+translator = EasyGoogleTranslate(source_language="en", target_language="hi", timeout=10)
+
+# Load configuration for Groq API key
+working_dir = os.path.dirname(os.path.abspath(__file__))
+config_data = json.load(open(f"{working_dir}/config.json"))
+GROQ_API_KEY = config_data["GROQ_API_KEY"]
+os.environ["GROQ_API_KEY"] = GROQ_API_KEY
+
+client = Groq()
+
+# Title of the Streamlit app
+st.title("BharatCaptioner with Conversational Chatbot")
+st.write(
+    "A tool to identify/describe Indian Landmarks in Indic Languages and chat about the image."
+)
+
+# Sidebar details
+st.sidebar.title("Developed by Harshal and Harsh Pandey")
+st.sidebar.write(
+    "**For the Model that I trained**: [Mail me here](mailto:[email protected])"
+)
+st.sidebar.write(
+    "**For the Code**: [GitHub Repo](https://github.com/justharshal2023/BharatCaptioner)"
+)
+st.sidebar.write(
+    "**Connect with me**: [LinkedIn](https://www.linkedin.com/in/harshal-123a90250/)"
+)
+
+# Image upload or URL input
+uploaded_file = st.file_uploader("Choose an image...", type=["jpg", "jpeg", "png"])
+url = st.text_input("Or enter a valid image URL...")
+
+image = None
+error_message = None
+landmark = None
+summary = None
+caption = None
+
+
+# Function to correct image orientation
+def correct_image_orientation(img):
+    try:
+        for orientation in ExifTags.TAGS.keys():
+            if ExifTags.TAGS[orientation] == "Orientation":
+                break
+        exif = img._getexif()
+        if exif is not None:
+            orientation = exif[orientation]
+            if orientation == 3:
+                img = img.rotate(180, expand=True)
+            elif orientation == 6:
+                img = img.rotate(270, expand=True)
+            elif orientation == 8:
+                img = img.rotate(90, expand=True)
+    except (AttributeError, KeyError, IndexError):
+        pass
+    return img
+
+
+# Function to get a unique hash for the image
+def get_image_hash(image):
+    img_bytes = image.tobytes()
+    return hashlib.md5(img_bytes).hexdigest()
+
+
+# Check if new image or URL is uploaded and reset the chat history
+def reset_chat_if_new_image():
+    if "last_uploaded_hash" not in st.session_state:
+        st.session_state["last_uploaded_hash"] = None
+
+    # Process the new image or URL
+    if uploaded_file:
+        image = Image.open(uploaded_file)
+        image = correct_image_orientation(image)
+        new_image_hash = get_image_hash(image)
+    elif url:
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+            image = Image.open(BytesIO(response.content))
+            image = correct_image_orientation(image)
+            new_image_hash = get_image_hash(image)
+        except (requests.exceptions.RequestException, UnidentifiedImageError):
+            image = None
+            new_image_hash = None
+            error_message = (
+                "Error: The provided URL is invalid or the image could not be loaded."
+            )
+            st.error(error_message)
+    else:
+        image = None
+        new_image_hash = None
+
+    # If the image is new, reset the chat and session state
+    if new_image_hash and new_image_hash != st.session_state["last_uploaded_hash"]:
+        st.session_state.clear()
+        st.session_state["last_uploaded_hash"] = new_image_hash
+        st.experimental_rerun()
+
+    return image
+
+
+# Call the reset function to check for new images or URL
+image = reset_chat_if_new_image()
+
+# If an image is provided
+if image is not None:
+    # Resize image for processing
+    image = image.resize((256, 256))
+
+    # Identify the landmark using BharatCaptioner
+    landmark, prob = identify_landmark(image)
+    summary = wikipedia.summary(landmark, sentences=3)  # Shortened summary
+    st.write(f"**Landmark Identified:** {landmark} (Confidence: {prob:.2f})")
+
+    # Display image and landmark name in the sidebar
+    with st.sidebar:
+        st.image(image, caption="Current Image", use_column_width=True)
+        st.write(f"**Landmark:** {landmark}")
+
+    # Chatbot functionality
+    st.write("### Chat with the Chatbot about the Image")
+    caption = f"The landmark in the image is {landmark}. {summary}"
+
+    # Initialize chat history in session state if not present
+    if "chat_history" not in st.session_state:
+        st.session_state["chat_history"] = []
+
+    # Chatbot introduction message with bold text for landmark and question
+    if not st.session_state.get("chatbot_started"):
+        chatbot_intro = f"Hello! I see the image is of **{landmark}**. {summary} **Would you like to know more** about this landmark?"
+        st.session_state["chat_history"].append(
+            {"role": "assistant", "content": chatbot_intro}
+        )
+        st.session_state["chatbot_started"] = True
+
+    # Display chat history
+    for message in st.session_state.chat_history:
+        with st.chat_message(message["role"]):
+            st.markdown(message["content"])
+
+    # User input
+    user_prompt = st.chat_input("Ask the Chatbot about the image...")
+
+    if user_prompt:
+        st.chat_message("user").markdown(user_prompt)
+        st.session_state.chat_history.append({"role": "user", "content": user_prompt})
+
+        # Send the user's message to the LLaMA chatbot
+        messages = [
+            {
+                "role": "system",
+                "content": "You are a helpful image conversational assistant. "
+                + f"The caption of the image is: {caption}",
+            },
+            *st.session_state.chat_history,
+        ]
+
+        response = client.chat.completions.create(
+            model="llama-3.1-8b-instant", messages=messages
+        )
+
+        assistant_response = response.choices[0].message.content
+        st.session_state.chat_history.append(
+            {"role": "assistant", "content": assistant_response}
+        )
+
+        # Display chatbot response
+        with st.chat_message("assistant"):
+            st.markdown(assistant_response)

requirements.txt

@@ -0,0 +1,12 @@
+streamlit==1.8.0 
+easygoogletranslate 
+wikipedia
+pillow 
+requests
+matplotlib
+tensorflow
+gdown
+transformers
+torch
+tf-keras
+groq