Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -1,47 +1,40 @@
|
|
1 |
import gradio as gr
|
2 |
from transformers import pipeline
|
3 |
import numpy as np
|
4 |
-
import torch
|
5 |
|
6 |
-
# Load
|
7 |
-
|
8 |
|
9 |
-
# Load
|
10 |
-
|
11 |
-
torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
|
12 |
|
13 |
-
|
14 |
-
model = pipeline("text-to-speech", model=model_id, device=device)
|
15 |
-
|
16 |
-
|
17 |
-
def process_image(input_image):
|
18 |
try:
|
19 |
-
# Step 1: Generate caption
|
20 |
-
caption =
|
21 |
|
22 |
-
# Step 2: Convert caption to speech
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
return (audio_data, sample_rate), caption
|
28 |
|
|
|
|
|
29 |
except Exception as e:
|
30 |
-
return str(e)
|
31 |
-
|
32 |
|
33 |
-
#
|
34 |
iface = gr.Interface(
|
35 |
fn=process_image,
|
36 |
-
inputs=gr.Image(type='pil', label="Upload Image"),
|
37 |
outputs=[
|
38 |
-
gr.Audio(label="Generated
|
39 |
gr.Textbox(label="Generated Caption")
|
40 |
],
|
41 |
-
title="
|
42 |
description="Upload an image to generate a caption and hear it described with speech."
|
43 |
)
|
44 |
|
45 |
-
iface.launch(
|
46 |
|
47 |
|
|
|
1 |
import gradio as gr
|
2 |
from transformers import pipeline
|
3 |
import numpy as np
|
|
|
4 |
|
5 |
+
# Load image captioning model
|
6 |
+
caption_model = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base")
|
7 |
|
8 |
+
# Load Whisper TTS model
|
9 |
+
speech_model = pipeline("text-to-speech", model="openai/whisper-large-v3-turbo")
|
|
|
10 |
|
11 |
+
def process_image(image):
|
|
|
|
|
|
|
|
|
12 |
try:
|
13 |
+
# Step 1: Generate caption from image
|
14 |
+
caption = caption_model(image)[0]['generated_text']
|
15 |
|
16 |
+
# Step 2: Convert caption to speech using Whisper
|
17 |
+
speech = speech_model(caption)
|
18 |
+
audio = np.array(speech["audio"])
|
19 |
+
rate = speech["sampling_rate"]
|
|
|
|
|
20 |
|
21 |
+
# Return both the audio and the caption
|
22 |
+
return (audio, rate), caption
|
23 |
except Exception as e:
|
24 |
+
return str(e), "Error generating caption or audio."
|
|
|
25 |
|
26 |
+
# Gradio Interface
|
27 |
iface = gr.Interface(
|
28 |
fn=process_image,
|
29 |
+
inputs=gr.Image(type='pil', label="Upload an Image"),
|
30 |
outputs=[
|
31 |
+
gr.Audio(label="Generated Audio"),
|
32 |
gr.Textbox(label="Generated Caption")
|
33 |
],
|
34 |
+
title="SeeSay",
|
35 |
description="Upload an image to generate a caption and hear it described with speech."
|
36 |
)
|
37 |
|
38 |
+
iface.launch()
|
39 |
|
40 |
|