llm added

app.py

@@ -2,6 +2,8 @@ import gradio as gr
 import numpy as np
 import matplotlib.pyplot as plt
 from PIL import Image
+import openai
+from dotenv import load_dotenv
 import os
 from monai.networks.nets import SegResNet
 from monai.inferers import sliding_window_inference
@@ -18,6 +20,8 @@ from transformers import WhisperProcessor, WhisperForConditionalGeneration
 import librosa
 import torch
 
+load_dotenv()
+
 title = 'Detect and Segment Brain Tumors 🧠'
 description = '''
 '''
@@ -84,14 +88,30 @@ def process_audio(sampling_rate, waveform):
     return waveform
 
 
+openai.api_key = os.environ.get("OPENAI_KEY")
+
+
+def make_llm_call(prompt,
+                  context="You are a text generation model DR-Brain Developed by team brute force team consist of HARSHA VARDHAN V , SAWIN KUMAR Y , CHARAN TEJA P, KISHORE S. Your specialized in medical stuff"):
+    messages = [{"role": "user", "content": prompt}]
+    if context:
+        messages.insert(0, {"role": "system", "content": context})
+    response_obj = openai.ChatCompletion.create(model="gpt-3.5-turbo", messages=messages)
+    response_message = dict(dict(response_obj)['choices'][0])["message"]["content"]
+    return response_message
+
+
 def detector(tumor_file, slice_number, channel, language, audio_question):
-    sampling_rate, waveform = audio_question
-    forced_decoder_ids = processor_whisper.get_decoder_prompt_ids(language=language, task="transcribe")
-    waveform = process_audio(sampling_rate, waveform)
-    audio_inputs = processor_whisper(audio=waveform, sampling_rate=16000, return_tensors="pt")
-    predicted_ids = model_whisper.generate(**audio_inputs, max_length=400, forced_decoder_ids=forced_decoder_ids)
-    transcription = processor_whisper.batch_decode(predicted_ids, skip_special_tokens=True)
-    output_text = transcription[0]
+    llm_answer = "Hi I'm Dr brain please enter a question to answer"
+    if audio_question:
+        sampling_rate, waveform = audio_question
+        forced_decoder_ids = processor_whisper.get_decoder_prompt_ids(language=language, task="transcribe")
+        waveform = process_audio(sampling_rate, waveform)
+        audio_inputs = processor_whisper(audio=waveform, sampling_rate=16000, return_tensors="pt")
+        predicted_ids = model_whisper.generate(**audio_inputs, max_length=400, forced_decoder_ids=forced_decoder_ids)
+        transcription = processor_whisper.batch_decode(predicted_ids, skip_special_tokens=True)
+        llm_quesion = transcription[0]
+        llm_answer = make_llm_call(llm_quesion)
     tumor_file_path = tumor_file.name
     processed_data = preproc_transforms({'image': [tumor_file_path]})
     tensor_3d_input = processed_data['image'].unsqueeze(0).to('cpu')
@@ -110,17 +130,17 @@ def detector(tumor_file, slice_number, channel, language, audio_question):
     plt.savefig(output_image_path, bbox_inches='tight', pad_inches=0)
     segment_image = np.asarray(Image.open(output_image_path))
     os.remove(output_image_path)
-    return (channel_image, segment_image, output_text)
+    return (channel_image, segment_image, llm_answer)
 
 
 interface = gr.Interface(fn=detector, inputs=[gr.File(label="Tumor File"),
                                               gr.Slider(0, 200, 50, step=1, label="Slice Number"),
                                               gr.Radio((0, 1, 2), label="Channel"),
                                               gr.Radio(("english", "japanese", "german", "spanish"), label="Language"),
-                                              gr.Audio(source="microphone"), ],
+                                              gr.Audio(info="Ask our medical specialist", source="microphone"), ],
                          outputs=[gr.Image(label='channel', shape=(1, 1)),
                                   gr.Image(label='Segmented Tumor', shape=(1, 1)),
-                                  gr.Textbox(label="Medical Summary")], title=title,
+                                  gr.Textbox(label="Dr brain response")], title=title,
                          examples=examples,
                          description=description, theme='dark')
 

requirements.txt

@@ -5,4 +5,7 @@ torchaudio
 nibabel
 monai
 matplotlib
-librosa
+librosa
+python-dotenv
+requests
+openai