whisper audio question input

.gitignore

@@ -1,3 +1,4 @@
 .idea
 .env
 /venv/
+/rough.py

app.py

@@ -14,6 +14,8 @@ from monai.transforms import (
     Orientationd,
     EnsureChannelFirstd,
 )
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
+import librosa
 import torch
 
 title = 'Detect and Segment Brain Tumors 🧠'
@@ -36,6 +38,9 @@ preproc_transforms = Compose(
 )
 post_trans = Compose([Activations(sigmoid=True), AsDiscrete(threshold=0.5)])
 
+processor_whisper = WhisperProcessor.from_pretrained("whisper-tiny")
+model_whisper = WhisperForConditionalGeneration.from_pretrained("whisper-tiny")
+
 model_tumor_seg = SegResNet(
     blocks_down=[1, 2, 2, 4],
     blocks_up=[1, 1, 1],
@@ -62,13 +67,31 @@ def inference(input):
 
 
 examples = [
-    ['examples/BRATS_225.nii.gz', 83, 2],
-    ['examples/BRATS_485.nii.gz', 90, 1],
-    ['examples/BRATS_485.nii.gz', 110, 0]
+    ['examples/BRATS_225.nii.gz', 83, 2, 'english', 'examples/sample1_en.mp3'],
+    ['examples/BRATS_485.nii.gz', 90, 1, 'japanese', 'examples/sample2_jp.mp3'],
+    ['examples/BRATS_485.nii.gz', 110, 0, 'german', 'examples/sample3_gr.mp3'],
 ]
 
 
-def detector(tumor_file, slice_number, channel, audio_question):
+def process_audio(sampling_rate, waveform):
+    waveform = waveform / 32678.0
+    if len(waveform.shape) > 1:
+        waveform = librosa.to_mono(waveform.T)
+    if sampling_rate != 16000:
+        waveform = librosa.resample(waveform, orig_sr=sampling_rate, target_sr=16000)
+    waveform = waveform[:16000 * 30]
+    waveform = torch.tensor(waveform)
+    return waveform
+
+
+def detector(tumor_file, slice_number, channel, language, audio_question):
+    sampling_rate, waveform = audio_question
+    forced_decoder_ids = processor_whisper.get_decoder_prompt_ids(language=language, task="transcribe")
+    waveform = process_audio(sampling_rate, waveform)
+    audio_inputs = processor_whisper(audio=waveform, sampling_rate=16000, return_tensors="pt")
+    predicted_ids = model_whisper.generate(**audio_inputs, max_length=400, forced_decoder_ids=forced_decoder_ids)
+    transcription = processor_whisper.batch_decode(predicted_ids, skip_special_tokens=True)
+    output_text = transcription[0]
     tumor_file_path = tumor_file.name
     processed_data = preproc_transforms({'image': [tumor_file_path]})
     tensor_3d_input = processed_data['image'].unsqueeze(0).to('cpu')
@@ -87,18 +110,19 @@ def detector(tumor_file, slice_number, channel, audio_question):
     plt.savefig(output_image_path, bbox_inches='tight', pad_inches=0)
     segment_image = np.asarray(Image.open(output_image_path))
     os.remove(output_image_path)
-    return (channel_image, segment_image, "Question answer")
+    return (channel_image, segment_image, output_text)
 
 
 interface = gr.Interface(fn=detector, inputs=[gr.File(label="Tumor File"),
                                               gr.Slider(0, 200, 50, step=1, label="Slice Number"),
-                                              gr.Radio([0, 1, 2], label="Channel"),
+                                              gr.Radio((0, 1, 2), label="Channel"),
+                                              gr.Radio(("english", "japanese", "german", "spanish"), label="Language"),
                                               gr.Audio(source="microphone"), ],
                          outputs=[gr.Image(label='channel', shape=(1, 1)),
                                   gr.Image(label='Segmented Tumor', shape=(1, 1)),
                                   gr.Textbox(label="Medical Summary")], title=title,
                          examples=examples,
-                         description=description, outputs_layout="row", theme='dark')
+                         description=description, theme='dark')
 
 theme = gr.themes.Default().set(
     button_primary_background_fill="#FF0000",

requirements.txt

@@ -4,4 +4,5 @@ torchvision
 torchaudio
 nibabel
 monai
-matplotlib
+matplotlib
+librosa