app.py

import time
import json
import random
import io
import ast
from PIL import Image, ImageDraw, ImageFont
from PIL import ImageColor
import xml.etree.ElementTree as ET

import spaces
import torch
import gradio as gr

from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor


### Load the model and helper functions ### 

model_path = "Qwen/Qwen2.5-VL-7B-Instruct"
model = Qwen2_5_VLForConditionalGeneration.from_pretrained(model_path,
                                                           torch_dtype=torch.bfloat16,
                                                           # attn_implementation="flash_attention_2", # note: can use flash attention (if possible) for faster inference
                                                           attn_implementation="sdpa").to("cuda")
processor = AutoProcessor.from_pretrained(model_path)

# Note: many of the helper functions come from the Qwen2.5-VL spatial understanding cookbook: https://github.com/QwenLM/Qwen2.5-VL/blob/main/cookbooks/spatial_understanding.ipynb

additional_colors = [colorname for (colorname, colorcode) in ImageColor.colormap.items()]

def decode_xml_points(text):
    try:
        root = ET.fromstring(text)
        num_points = (len(root.attrib) - 1) // 2
        points = []
        for i in range(num_points):
            x = root.attrib.get(f'x{i+1}')
            y = root.attrib.get(f'y{i+1}')
            points.append([x, y])
        alt = root.attrib.get('alt')
        phrase = root.text.strip() if root.text else None
        return {
            "points": points,
            "alt": alt,
            "phrase": phrase
        }
    except Exception as e:
        print(e)
        return None

def plot_bounding_boxes(im,
                        bounding_boxes,
                        input_width,
                        input_height):
    """
    Plots bounding boxes on an image with markers for each a name, using PIL, normalized coordinates, and different colors.

    Args:
        img_path: The path to the image file.
        bounding_boxes: A list of bounding boxes containing the name of the object
         and their positions in normalized [y1 x1 y2 x2] format.
    """

    # Load the image
    img = im
    width, height = img.size
    print(f"Plotting on image with size: {img.size}")
    # Create a drawing object
    draw = ImageDraw.Draw(img)

    # Define a list of colors
    colors = [
    'red',
    'green',
    'blue',
    'yellow',
    'orange',
    'pink',
    'purple',
    'brown',
    'gray',
    'beige',
    'turquoise',
    'cyan',
    'magenta',
    'lime',
    'navy',
    'maroon',
    'teal',
    'olive',
    'coral',
    'lavender',
    'violet',
    'gold',
    'silver',
    ] + additional_colors

    # Parsing out the markdown fencing
    bounding_boxes = parse_json(bounding_boxes)

    font = ImageFont.load_default(size=16.0)

    try:
      json_output = ast.literal_eval(bounding_boxes)
    except Exception as e:
      end_idx = bounding_boxes.rfind('"}') + len('"}')
      truncated_text = bounding_boxes[:end_idx] + "]"
      json_output = ast.literal_eval(truncated_text)

    print(json_output)

    # Iterate over the bounding boxes
    for i, bounding_box in enumerate(json_output):
      # Select a color from the list
      color = colors[i % len(colors)]

      # Convert normalized coordinates to absolute coordinates
      abs_y1 = int(bounding_box["bbox_2d"][1]/input_height * height)
      abs_x1 = int(bounding_box["bbox_2d"][0]/input_width * width)
      abs_y2 = int(bounding_box["bbox_2d"][3]/input_height * height)
      abs_x2 = int(bounding_box["bbox_2d"][2]/input_width * width)

      if abs_x1 > abs_x2:
        abs_x1, abs_x2 = abs_x2, abs_x1

      if abs_y1 > abs_y2:
        abs_y1, abs_y2 = abs_y2, abs_y1

      target_box = [abs_x1, abs_y1, abs_x2, abs_y2]
      print(f"[INFO] Target box (XYXY): {target_box} | Label: {bounding_box['label']}")

      # Draw the bounding box
      draw.rectangle(
          ((abs_x1, abs_y1),
           (abs_x2, abs_y2)),
          outline=color,
          width=4
      )

      # Draw the text
      if "label" in bounding_box:
        draw.text((abs_x1 + 8, abs_y1 + 6),
                  bounding_box["label"],
                  fill=color,
                  font=font)

    # Display the image
    return img


def plot_points(im, text, input_width, input_height):
  img = im
  width, height = img.size
  draw = ImageDraw.Draw(img)
  colors = [
    'red', 'green', 'blue', 'yellow', 'orange', 'pink', 'purple', 'brown', 'gray',
    'beige', 'turquoise', 'cyan', 'magenta', 'lime', 'navy', 'maroon', 'teal',
    'olive', 'coral', 'lavender', 'violet', 'gold', 'silver',
  ] + additional_colors
  xml_text = text.replace('```xml', '')
  xml_text = xml_text.replace('```', '')
  data = decode_xml_points(xml_text)
  if data is None:
    img.show()
    return
  points = data['points']
  description = data['phrase']

  font = ImageFont.truetype("NotoSansCJK-Regular.ttc", size=14)

  for i, point in enumerate(points):
    color = colors[i % len(colors)]
    abs_x1 = int(point[0])/input_width * width
    abs_y1 = int(point[1])/input_height * height
    radius = 2
    draw.ellipse([(abs_x1 - radius, abs_y1 - radius), (abs_x1 + radius, abs_y1 + radius)], fill=color)
    draw.text((abs_x1 + 8, abs_y1 + 6), description, fill=color, font=font)

  img.show()


# @title Parsing JSON output
def parse_json(json_output):
    # Parsing out the markdown fencing
    lines = json_output.splitlines()
    for i, line in enumerate(lines):
        if line == "```json":
            json_output = "\n".join(lines[i+1:])  # Remove everything before "```json"
            json_output = json_output.split("```")[0]  # Remove everything after the closing "```"
            break  # Exit the loop once "```json" is found
    return json_output

def resize_image_to_max_dimension(image, max_dimension=1024):
    """
    Resize an image so that its longest dimension is max_dimension pixels,
    but only if either dimension exceeds max_dimension.
    Maintains the original aspect ratio.

    Args:
        image: PIL Image object
        max_dimension: Maximum allowed dimension (default: 1024)

    Returns:
        Resized PIL Image object
    """
    # Get current dimensions
    width, height = image.size

    # Check if resizing is needed
    if width <= max_dimension and height <= max_dimension:
        return image  # No resizing needed

    # Calculate new dimensions maintaining aspect ratio
    if width >= height:
        # Width is the longest dimension
        new_width = max_dimension
        new_height = int((height / width) * max_dimension)
    else:
        # Height is the longest dimension
        new_height = max_dimension
        new_width = int((width / height) * max_dimension)

    # Resize the image
    resized_image = image.resize((new_width, new_height), Image.LANCZOS)

    print(f"Image resized from {width}x{height} to {new_width}x{new_height}")

    return resized_image

def inference(img_url,
              prompt,
              system_prompt="You are a helpful assistant who is very knowledgable about foods from all over the world. You always respond with high quality answers about what kinds of edible foods and drinks are visible in an image.",
              max_new_tokens=1024):
  start_time = time.time()

  if isinstance(img_url, str):
    image = Image.open(img_url)
  else:
    image = resize_image_to_max_dimension(img_url)

  messages = [
    {
      "role": "system",
      "content": system_prompt
    },
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": prompt
        },
        {
          "image": img_url
        }
      ]
    }
  ]
  text = processor.apply_chat_template(messages,
                                       tokenize=False,
                                       add_generation_prompt=True)
  # print("input:\n",text)
  inputs = processor(text=[text],
                     images=[image],
                     padding=True,
                     return_tensors="pt").to('cuda')

  output_ids = model.generate(**inputs, max_new_tokens=1024)
  generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs.input_ids, output_ids)]
  output_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)
  print("[INFO] output:\n",output_text[0])
  end_time = time.time()
  total_time = end_time - start_time

  print(f"[INFO] Total time for inference: {round(total_time, 4)} seconds")

  input_height = inputs['image_grid_thw'][0][1]*14
  input_width = inputs['image_grid_thw'][0][2]*14

  return output_text[0], input_height, input_width

import numpy as np
from PIL import Image

def numpy_to_pil(numpy_array):
    """
    Convert a NumPy array to a PIL Image.

    Parameters:
    -----------
    numpy_array : numpy.ndarray
        The NumPy array to convert. Should be of shape (height, width) for
        grayscale or (height, width, channels) for RGB/RGBA images.
        Data type should preferably be np.uint8.

    Returns:
    --------
    PIL.Image.Image
        The converted PIL Image.
    """
    # Ensure array is uint8 type (required for most standard images)
    if numpy_array.dtype != np.uint8:
        # If array is float in [0,1] range
        if numpy_array.dtype in [np.float32, np.float64] and numpy_array.max() <= 1.0:
            numpy_array = (numpy_array * 255).astype(np.uint8)
        # Otherwise just convert to uint8
        else:
            numpy_array = numpy_array.astype(np.uint8)

    # Convert to PIL Image
    pil_image = Image.fromarray(numpy_array)

    return pil_image

### Prompt section ###

prompt_thinking = """Outline the bounding box coordinates and names of each unique edible food and drink item and output all the coordinates in JSON format.
Only outline food and drink items that are edible.
Ignore hard to see items in the background. Only focus on the foreground.
Output a max of one box per unique food or drink group.
Use a visually description name for the target item.
Make sure to list each unique food/drink item.
Start with <think></think> tags to visually inspect the whole image and list every unique edible food/drink item you are going to locate. 
  * Keep the think tags succinct, just say "The image contains the following unique foods... I will create boundiung box coordines for those."
  * If there are no visible edible foods or drinks, say "The image contains the following unique foods: None. ... I will set the label to 'no foods found'."
Make sure to output the bounding box coordinates in valid JSON in the format [{"bbox_2d": ..., "label": ...}...].
Do not miss any foods/drinks. It's important to get them all.
If no edible foods or drinks are found in the image, return a single bbox_2d with the coordinates of the whole image with the label 'no foods found'.
  * For example: [{"bbox_2d": ..., "label": "no foods found"}]

For example:

<examples>
<example_1>
<think>
The image contains the following unique foods/drinks: A pint of dark beer (Guinness), roasted pork slices with gravy, pork crackling, a green salad, roasted potato wedges, roasted sweet potato wedges, coleslaw/shredded cabbage salad, and roasted cubed root vegetable (possibly pumpkin or sweet potato) with cheese. I will create bounding box coordinates for those.
</think>

```json
[
  {"bbox_2d": [27, 271, 365, 531], "label": "Pint of Guinness"},
  {"bbox_2d": [343, 300, 609, 711], "label": "Green Salad"},
  {"bbox_2d": [524, 600, 706, 956], "label": "Roasted Potato Wedges"},
  {"bbox_2d": [425, 619, 530, 864], "label": "Roasted Sweet Potato Wedge"},
  {"bbox_2d": [556, 372, 968, 799], "label": "Coleslaw/Shredded Cabbage Salad"},
  {"bbox_2d": [661, 228, 919, 514], "label": "Roasted Root Vegetable (possibly Sweet Potato or Pumpkin)"},
  {"bbox_2d": [488, 58, 747, 394], "label": "Roasted Pork with Gravy"},
  {"bbox_2d": [348, 163, 488, 330], "label": "Small piece of pork crackling"}
]
```
</example_1>

<example_2>
<think>
The image contains the following unique foods/drinks: A bottle of beer, a bottle of red wine, sliced beef brisket, potato salad, corn salad, spinach salad, pickled red vegetable (possibly onion or beet), and pickle slices. I will create bounding box coordinates for those.
</think>

```json
[
  {"box_2d": [0, 146, 271, 390], "label": "Bottle of Red Wine"},
  {"box_2d": [502, 363, 711, 677], "label": "Sliced Beef Brisket"},
  {"box_2d": [363, 237, 519, 486], "label": "Corn Salad"},
  {"box_2d": [428, 621, 532, 811], "label": "Pickled Red Vegetable"},
  {"box_2d": [518, 647, 638, 767], "label": "Pickle Slices"},
  {"box_2d": [341, 449, 541, 704], "label": "Spinach Salad"},
  {"box_2d": [504, 220, 639, 411], "label": "Potato Salad"},
  {"box_2d": [0, 0, 234, 181], "label": "Bottle of Beer"}
]
```
</example_2>
<example_3>
<think>
The image contains the following unique foods: Scrambled eggs on toast, pea shoots (microgreens), an orange dip/spread (likely romesco sauce), and grilled bacon (likely bacon or ham). I will create bounding box coordinates for those.
</think>

```json
[
  {"box_2d": [241, 383, 536, 685], "label": "Scrambled Eggs"},
  {"box_2d": [348, 630, 623, 879], "label": "Grilled Toast"},
  {"box_2d": [67, 107, 506, 516], "label": "Pea Shoots"},
  {"box_2d": [485, 198, 645, 440], "label": "Romesco Sauce"},
  {"box_2d": [512, 657, 702, 959], "label": "Grilled Bacon"}
]
```
</example_3>

<example_4>
<think>
The image contains the following unique foods: Sliced roasted duck, thin pancakes or crepes (like for Peking duck), sliced cucumber, cilantro, pickled red onions, sliced red chilies, hoisin sauce, and crispy fried shallots or garlic. I will create bounding box coordinates for those.
</think>

```json
[
  {"box_2d": [294, 462, 367, 645], "label": "Crispy Fried Shallots"},
  {"box_2d": [325, 657, 425, 843], "label": "Hoisin Sauce"},
  {"box_2d": [393, 401, 643, 830], "label": "Thin Crepes"},
  {"box_2d": [593, 363, 808, 800], "label": "Sliced Roasted Duck"},
  {"box_2d": [283, 195, 443, 479], "label": "Cilantro"},
  {"box_2d": [398, 119, 641, 310], "label": "Sliced Cucumber"},
  {"box_2d": [412, 298, 556, 440], "label": "Pickled Red Onions"},
  {"box_2d": [539, 265, 704, 419], "label": "Sliced Red Chilies"}
]
```
</example_4>
<example_5>
<think>
The image contains the following unique foods: Sliced cooked chicken, halved hard-boiled eggs, cashew nuts, peanut sauce/dressing, and a noodle salad (rice noodles, shredded carrots, cucumber, cilantro, red cabbage/onion). I will create bounding box coordinates for those.
</think>

```json
[
  {"box_2d": [191, 217, 403, 496], "label": "Sliced Cooked Chicken"},
  {"box_2d": [173, 436, 338, 683], "label": "Halved Hard-Boiled Eggs"},
  {"box_2d": [330, 250, 565, 460], "label": "Cashew Nuts"},
  {"box_2d": [306, 388, 491, 729], "label": "Peanut Sauce/Dressing"},
  {"box_2d": [445, 280, 641, 779], "label": "Noodle Salad With Carrots, Cilantro and Red Onion"}
]
```
</example_5>
<example_6>
<think>
The image contains the following unique food: A milk chocolate bar. I will create bounding box coordinates for this.
</think>

```json
[
  {"bbox_2d": [256, 196, 495, 770], "label": "Milk Chocolate Bar"}
]
```
</example_6>
<example_7>
<think>
The image contains the following unique food: A glass of water. I will create bounding box coordinates for this.
</think>

```json
[
  {"bbox_2d": [180, 150, 702, 630], "label": "Glass of Water"}
]
```
</example_7>
<example_8>
<think>
The image contains the following unique food: An iced coffee with milk beverage. I will create bounding box coordinates for this.
</think>

```json
[
  {"bbox_2d": [121, 214, 430, 624], "label": "Iced Coffee with Milk"}
]
```
</example_8>
<example_9>
<think>
The image contains the following unique foods: Three fried eggs on top of rice cakes and topped with grated cheese, a jar of pickled vegetables. I will create bounding box coordinates for these items.
</think>

```json
[
  {"bbox_2d": [144, 385, 382, 622], "label": "Fried Egg with Grated Cheese on a Rice Cake"},
  {"bbox_2d": [400, 433, 660, 728], "label": "Fried Egg with Grated Cheese on a Rice Cake"},
  {"bbox_2d": [176, 593, 510, 878], "label": "Fried Egg with Grated Cheese on a Rice Cake"},
  {"bbox_2d": [115, 22, 330, 348], "label": "Jar of Pickled Vegetables"},
]
```
</example_9>
<example_10>
<think>
The image contains the following unique foods: Halved hard-boiled eggs, sliced cucumber sticks, celery sticks, almond butter, chopped tomatoes, and beef jerky/biltong. I will create bounding box coordinates for those.
</think>

```json
[
  {"box_2d": [259, 130, 484, 388], "label": "Halved Hard-Boiled Eggs"},
  {"box_2d": [179, 328, 410, 570], "label": "Sliced Cucumber Sticks"},
  {"box_2d": [225, 530, 445, 845], "label": "Celery Sticks"},
  {"box_2d": [259, 537, 382, 724], "label": "Almond Butter"},
  {"box_2d": [471, 115, 766, 440], "label": "Chopped Tomatoes"},
  {"box_2d": [388, 409, 674, 954], "label": "Beef Biltong"}
]
```
</example_10>

<example_11>
<think>
The image contains the following unique foods: None. It is an image of a step bin in a store. I will set the label to 'no foods found'.
</think>

```json
[
  {"bbox_2d": [252, 630, 680, 1151], "label": "no foods found"}
]
<example_11>
</examples>
"""

# Make a prompt with no thinking (output less but better)
prompt_no_thinking = """Outline the bounding box coordinates and names of each unique edible food and drink item and output all the coordinates in JSON format.
Only outline food and drink items that are edible.
Ignore hard to see items in the background. Only focus on the foreground.
Output a max of one box per unique food or drink group.
Use a visually description name for the target item.
Make sure to list each unique food/drink item.
Make sure to output the bounding box coordinates in valid JSON in the format [{"bbox_2d": ..., "label": ...}...].
Do not miss any foods/drinks. It's important to get them all.
If no edible foods or drinks are found in the image, return a single bbox_2d with the coordinates of the whole image with the label 'no foods found'.
  * For example: [{"bbox_2d": ..., "label": "no foods found"}]

For example:

<examples>
<example_1>
```json
[
  {"bbox_2d": [27, 271, 365, 531], "label": "Pint of Guinness"},
  {"bbox_2d": [343, 300, 609, 711], "label": "Green Salad"},
  {"bbox_2d": [524, 600, 706, 956], "label": "Roasted Potato Wedges"},
  {"bbox_2d": [425, 619, 530, 864], "label": "Roasted Sweet Potato Wedge"},
  {"bbox_2d": [556, 372, 968, 799], "label": "Coleslaw/Shredded Cabbage Salad"},
  {"bbox_2d": [661, 228, 919, 514], "label": "Roasted Root Vegetable (possibly Sweet Potato or Pumpkin)"},
  {"bbox_2d": [488, 58, 747, 394], "label": "Roasted Pork with Gravy"},
  {"bbox_2d": [348, 163, 488, 330], "label": "Small piece of pork crackling"}
]
```
</example_1>

<example_2>
```json
[
  {"box_2d": [0, 146, 271, 390], "label": "Bottle of Red Wine"},
  {"box_2d": [502, 363, 711, 677], "label": "Sliced Beef Brisket"},
  {"box_2d": [363, 237, 519, 486], "label": "Corn Salad"},
  {"box_2d": [428, 621, 532, 811], "label": "Pickled Red Vegetable"},
  {"box_2d": [518, 647, 638, 767], "label": "Pickle Slices"},
  {"box_2d": [341, 449, 541, 704], "label": "Spinach Salad"},
  {"box_2d": [504, 220, 639, 411], "label": "Potato Salad"},
  {"box_2d": [0, 0, 234, 181], "label": "Bottle of Beer"}
]
```
</example_2>

<example_3>
```json
[
  {"bbox_2d": [192, 211, 594, 496], "label": "Baked Zucchini"},
  {"bbox_2d": [87, 439, 454, 839], "label": "Creamy Broccoli Casserole"},
  {"bbox_2d": [388, 300, 803, 672], "label": "Roasted Meat with Gravy"},
  {"bbox_2d": [306, 588, 735, 975], "label": "Peas"},
  {"bbox_2d": [634, 596, 826, 818], "label": "Roasted Potato"}
]
```
</example_3>

<example_4>
```json
[
  {"box_2d": [294, 462, 367, 645], "label": "Crispy Fried Shallots"},
  {"box_2d": [325, 657, 425, 843], "label": "Hoisin Sauce"},
  {"box_2d": [393, 401, 643, 830], "label": "Thin Crepes"},
  {"box_2d": [593, 363, 808, 800], "label": "Sliced Roasted Duck"},
  {"box_2d": [283, 195, 443, 479], "label": "Cilantro"},
  {"box_2d": [398, 119, 641, 310], "label": "Sliced Cucumber"},
  {"box_2d": [412, 298, 556, 440], "label": "Pickled Red Onions"},
  {"box_2d": [539, 265, 704, 419], "label": "Sliced Red Chilies"}
]
```
</example_4>

<example_5>
```json
[
  {"box_2d": [191, 217, 403, 496], "label": "Sliced Cooked Chicken"},
  {"box_2d": [173, 436, 338, 683], "label": "Halved Hard-Boiled Eggs"},
  {"box_2d": [330, 250, 565, 460], "label": "Cashew Nuts"},
  {"box_2d": [306, 388, 491, 729], "label": "Peanut Sauce/Dressing"},
  {"box_2d": [445, 280, 641, 779], "label": "Noodle Salad With Carrots, Cilantro and Red Onion"}
]
```
</example_5>

<example_6>
```json
[
  {"bbox_2d": [256, 196, 495, 770], "label": "Milk Chocolate Bar"}
]
```
</example_6>

<example_7>
```json
[
  {"bbox_2d": [180, 150, 702, 630], "label": "Glass of Water"}
]
```
</example_7>

<example_8>
```json
[
  {"bbox_2d": [121, 214, 430, 624], "label": "Iced Coffee with Milk"}
]
```
</example_8>

<example_9>
```json
[
  {"bbox_2d": [144, 385, 382, 622], "label": "Fried Egg with Grated Cheese on a Rice Cake"},
  {"bbox_2d": [400, 433, 660, 728], "label": "Fried Egg with Grated Cheese on a Rice Cake"},
  {"bbox_2d": [176, 593, 510, 878], "label": "Fried Egg with Grated Cheese on a Rice Cake"},
  {"bbox_2d": [115, 22, 330, 348], "label": "Jar of Pickled Vegetables"},
]
```
</example_9>

<example_10>
```json
[
  {"box_2d": [259, 130, 484, 388], "label": "Halved Hard-Boiled Eggs"},
  {"box_2d": [179, 328, 410, 570], "label": "Sliced Cucumber Sticks"},
  {"box_2d": [225, 530, 445, 845], "label": "Celery Sticks"},
  {"box_2d": [259, 537, 382, 724], "label": "Almond Butter"},
  {"box_2d": [471, 115, 766, 440], "label": "Chopped Tomatoes"},
  {"box_2d": [388, 409, 674, 954], "label": "Beef Biltong"}
]
```
</example_10>

<example_11>
```json
[
  {"bbox_2d": [252, 630, 680, 1151], "label": "no foods found"}
]
<example_11>
</examples>
"""

### Load the app ###

# Note: Depending on how many foods are in an image, the following may take longer then the default 60 seconds to run on a Hugging Face Zero GPU.
# So we set the duration to 120 (120 seconds).
@spaces.GPU(duration=120)
def infer_on_image(input_image):

  print(f"[INFO] Image type: {type(input_image)}, shape: {input_image.shape}")

  # Convert image to PIL (it gets uploaded as NumPy)
  image_1 = numpy_to_pil(numpy_array=input_image)

  ### Thinking prompt ###
  start_time_thinking = time.time()
  response, input_height, input_width = inference(img_url=image_1,
                                                  prompt=prompt_thinking)
  end_time_thinking = time.time()
  total_time_thinking = round(end_time_thinking - start_time_thinking,
                              4)

  output_image_1 = plot_bounding_boxes(image_1,
                                       response,
                                       input_width,
                                       input_height)

  ### Non-thinking prompt ###

  # Make a second image to prevent plotting duplicates
  # Convert image to PIL (it gets uploaded as NumPy)
  image_2 = numpy_to_pil(numpy_array=input_image)

  start_time_no_thinking = time.time()
  response_no_thinking, input_height, input_width = inference(img_url=image_2,
                                                              prompt=prompt_no_thinking)
  end_time_no_thinking = time.time()
  total_time_no_thinking = round(end_time_no_thinking - start_time_no_thinking,
                                 4)

  output_image_2 = plot_bounding_boxes(image_2,
                                       response_no_thinking,
                                       input_width,
                                       input_height)

  return output_image_1, response, total_time_thinking, output_image_2, response_no_thinking, total_time_no_thinking

# Want input to be a single image
# Outputs to be time + image

description = f"""Demo based on example [Qwen2.5-VL spatial notebook](https://github.com/QwenLM/Qwen2.5-VL/blob/main/cookbooks/spatial_understanding.ipynb) for detecting foods and drinks in images with bounding boxes. Input an image of food/drink for bounding boxes to be detected. If no food is present in an image the model should return 'no foods found'.\n
One prediction will use thinking tags, e.g. <think>...</think> to try an describe what's in the image. The other will directly predict a JSON of bounding box coordinates and labels.
Boxes may not be as accurate as a dedicated object detection model but the benefit here is that they are class agnostic (e.g. the model can detect a wide range of items despite never being explicitly trained on them).
The foundation knowledge in Qwen2.5-VL (we are using [Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) in this demo) means it can detect a wide range of foods and drinks.
See the app.py file for the different prompts used."""

demo = gr.Interface(fn=infer_on_image,
                    inputs=gr.Image(label="Input image"),
                    outputs=[gr.Image(label="Image w/ thinking tags"),
                             gr.Text(label="Raw output w/ thinking tags"),
                             gr.Text(label="Inference time w/ thinking tags"),
                             gr.Image(label="Image w/o thinking tags"),
                             gr.Text(label="Raw output w/o thinking tags"),
                             gr.Text(label="Inference time w/o thinking tags")],
                    title="Qwen2.5-VL Food Detection 👁️🍔",
                    description=description,
                    # Examples come in the form of a list of lists, where each inner list contains elements to prefill the `inputs` parameter with
                    examples=[
                        ["examples/example_1.jpeg"],
                        ["examples/example_2.jpeg"],
                        ["examples/example_3.jpeg"]
                    ],
                    cache_examples=True)

demo.launch(debug=True)