File size: 7,851 Bytes
2754fd7 bf93411 2754fd7 bf93411 2754fd7 bf93411 2754fd7 bf93411 2754fd7 bf93411 2754fd7 bf93411 2754fd7 bf93411 2754fd7 bf93411 2754fd7 bf93411 2754fd7 bf93411 2754fd7 9ded30d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 |
import gradio as gr
import numpy as np
import cv2
import pandas as pd
from datetime import datetime
import time
from transformers import pipeline
from deepface import DeepFace
import librosa
from python_speech_features import mfcc
# Initialize models
voice_classifier = pipeline("audio-classification", model="superb/hubert-base-superb-er")
# Global variables to store results
emotion_history = []
current_emotions = {"face": "neutral", "voice": "neutral"}
last_update_time = time.time()
def analyze_face(frame):
"""Analyze facial expressions in the frame using DeepFace"""
try:
# Convert frame to RGB (DeepFace expects RGB)
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Detect emotions
result = DeepFace.analyze(rgb_frame, actions=['emotion'], enforce_detection=False)
if result and isinstance(result, list):
emotions = result[0]['emotion']
dominant_emotion = max(emotions, key=emotions.get)
return dominant_emotion, emotions
return "neutral", {"angry": 0, "disgust": 0, "fear": 0, "happy": 0, "sad": 0, "surprise": 0, "neutral": 1}
except Exception as e:
print(f"Face analysis error: {e}")
return "neutral", {"angry": 0, "disgust": 0, "fear": 0, "happy": 0, "sad": 0, "surprise": 0, "neutral": 1}
def analyze_voice(audio):
"""Analyze voice tone from audio"""
try:
sr, y = audio
y = y.astype(np.float32)
# Convert to mono if stereo
if len(y.shape) > 1:
y = np.mean(y, axis=0)
# Resample to 16kHz if needed
if sr != 16000:
y = librosa.resample(y, orig_sr=sr, target_sr=16000)
sr = 16000
# Classify emotion
result = voice_classifier({"sampling_rate": sr, "raw": y})
dominant_emotion = result[0]['label']
return dominant_emotion, result
except Exception as e:
print(f"Voice analysis error: {e}")
return "neutral", [{"label": "neutral", "score": 1.0}]
def update_emotion_history(face_emotion, voice_emotion):
"""Update the emotion history and current emotions"""
global current_emotions, emotion_history, last_update_time
current_time = datetime.now().strftime("%H:%M:%S")
# Update current emotions
current_emotions = {
"face": face_emotion,
"voice": voice_emotion,
"timestamp": current_time
}
# Add to history (every 5 seconds or when emotion changes significantly)
if (time.time() - last_update_time) > 5 or not emotion_history:
emotion_history.append({
"timestamp": current_time,
"face": face_emotion,
"voice": voice_emotion
})
last_update_time = time.time()
# Keep only last 20 entries
if len(emotion_history) > 20:
emotion_history = emotion_history[-20:]
def get_emotion_timeline():
"""Create a timeline DataFrame for display"""
if not emotion_history:
return pd.DataFrame(columns=["Time", "Facial Emotion", "Voice Emotion"])
df = pd.DataFrame(emotion_history)
df = df.rename(columns={
"timestamp": "Time",
"face": "Facial Emotion",
"voice": "Voice Emotion"
})
return df
def get_practitioner_advice(face_emotion, voice_emotion):
"""Generate suggestions based on detected emotions"""
advice = []
# Facial emotion advice
if face_emotion in ["sad", "fear"]:
advice.append("Patient appears distressed. Consider speaking more slowly and with reassurance.")
elif face_emotion == "angry":
advice.append("Patient seems frustrated. Acknowledge their concerns and maintain calm demeanor.")
elif face_emotion == "disgust":
advice.append("Patient may be uncomfortable. Check if they're experiencing any discomfort.")
elif face_emotion == "surprise":
advice.append("Patient seems surprised. Ensure they understand all information.")
# Voice emotion advice
if voice_emotion in ["sad", "fear"]:
advice.append("Patient's tone suggests anxiety. Provide clear explanations and emotional support.")
elif voice_emotion == "angry":
advice.append("Patient sounds upset. Practice active listening and validate their feelings.")
elif voice_emotion == "happy":
advice.append("Patient seems positive. This may be a good time to discuss treatment options.")
return "\n".join(advice) if advice else "Patient appears neutral. Continue with consultation."
def process_input(video, audio):
"""Process video and audio inputs to detect emotions"""
try:
# Process video frame
if video is not None:
frame = cv2.cvtColor(video, cv2.COLOR_RGB2BGR)
face_emotion, face_details = analyze_face(frame)
else:
face_emotion, face_details = "neutral", {}
# Process audio
if audio is not None:
voice_emotion, voice_details = analyze_voice(audio)
else:
voice_emotion, voice_details = "neutral", {}
# Update history and get outputs
update_emotion_history(face_emotion, voice_emotion)
timeline_df = get_emotion_timeline()
advice = get_practitioner_advice(face_emotion, voice_emotion)
# Prepare outputs
outputs = {
"current_face": face_emotion,
"current_voice": voice_emotion,
"timeline": timeline_df,
"advice": advice,
"face_details": str(face_details),
"voice_details": str(voice_details)
}
return outputs
except Exception as e:
print(f"Processing error: {e}")
return {
"current_face": "Error",
"current_voice": "Error",
"timeline": pd.DataFrame(),
"advice": "System error occurred",
"face_details": "",
"voice_details": ""
}
# Gradio interface
with gr.Blocks(title="Patient Emotion Recognition", theme="soft") as demo:
gr.Markdown("# Real-Time Patient Emotion Recognition")
gr.Markdown("Analyze facial expressions and voice tone during medical consultations")
with gr.Row():
with gr.Column():
video_input = gr.Image(label="Live Camera Feed", source="webcam", streaming=True)
audio_input = gr.Audio(label="Voice Input", source="microphone", type="numpy")
submit_btn = gr.Button("Analyze Emotions")
with gr.Column():
current_face = gr.Textbox(label="Current Facial Emotion")
current_voice = gr.Textbox(label="Current Voice Emotion")
advice_output = gr.Textbox(label="Practitioner Suggestions", lines=3)
timeline_output = gr.Dataframe(label="Emotion Timeline", interactive=False)
face_details = gr.Textbox(label="Face Analysis Details", visible=False)
voice_details = gr.Textbox(label="Voice Analysis Details", visible=False)
# Live processing
video_input.change(
process_input,
inputs=[video_input, audio_input],
outputs=[current_face, current_voice, timeline_output, advice_output, face_details, voice_details],
show_progress="hidden"
)
audio_input.change(
process_input,
inputs=[video_input, audio_input],
outputs=[current_face, current_voice, timeline_output, advice_output, face_details, voice_details],
show_progress="hidden"
)
submit_btn.click(
process_input,
inputs=[video_input, audio_input],
outputs=[current_face, current_voice, timeline_output, advice_output, face_details, voice_details]
)
if __name__ == "__main__":
demo.launch(debug=True) |