app.py

import streamlit as st
import requests
import io


# Designing the interface
st.title("Medical Image Captioning")

st.sidebar.markdown(
    """
    This project features 3 different Medical image captioning models.
    Two of the use the InceptionV3 architecture to do feature extraction and then generate the captions using an LSTM model.
    The difference between these two is that the first one uses InceptionV3 trained on ImageNet data and outputs 2048 features.
    The second one is based on a retrained version of InceptionV3 that uses the CUI data from the ROCO dataset to extract 745 features from the images.
    The final model is transformer based on...
    """
)

with st.spinner('Loading objects ...'):
    from model import *

random_image_id = get_random_image_id()

st.sidebar.title("Select a sample image")
sample_image_id = st.sidebar.selectbox(
    "Please choose a sample image",
    sample_image_ids
)

st.sidebar.title("Select a model Type")
model_type = st.sidebar.selectbox(
    "Please choose a model",
    ['Pretrained Inception', 'Retrained Inception', 'Transformer']
)

inception, lstm = fetch_model(model_type)
word2Index, index2Word, variable_params = fetch_auxiliary_files(model_type)
max_len = variable_params['max_caption_len']

if st.sidebar.button("Random ROCO (test) images"):
    random_image_id = get_random_image_id()
    sample_image_id = "None"

bytes_data = None
with st.sidebar.form("file-uploader-form", clear_on_submit=True):
    uploaded_file = st.file_uploader("Choose a file")
    submitted = st.form_submit_button("Upload")
    if submitted and uploaded_file is not None:
        bytes_data = io.BytesIO(uploaded_file.getvalue())

if (bytes_data is None) and submitted:

    st.write("No file is selected to upload")

else:

    image_id = random_image_id
    if sample_image_id != "None":
        assert type(sample_image_id) == int
        image_id = sample_image_id

    sample_name = f"ROCO_{str(image_id).zfill(5)}.jpg"
    sample_path = os.path.join(sample_dir, sample_name)

    if bytes_data is not None:
        image = Image.open(bytes_data)
    elif os.path.isfile(sample_path):
        image = Image.open(sample_path)

    width, height = 299, 299
    resized = image.resize(size=(width, height))

    if bytes_data is None:
        st.markdown(f"ROCO_{str(image_id).zfill(5)}.jpg")
    show = st.image(resized)
    show.image(resized, '\n\nSelected Image')

    # For newline
    st.sidebar.write('\n')

    with st.spinner('Generating image caption ...'):
        st.header(f'Predicted caption:\n\n')

        preprocessed_img = preprocess_image_inception(resized)
        features = extract_features(inception, preprocessed_img)
        caption = generate_caption(lstm, features, max_len, word2Index, index2Word)
        st.subheader(caption)

    st.sidebar.header("Model predicts: ")
    st.sidebar.write(f"{caption}")

    image.close()