1

README.md

@@ -4,10 +4,56 @@ emoji: 😀😲😐😥🥴😱😡
 colorFrom: blue
 colorTo: pink
 sdk: gradio
-sdk_version: '4.24.0'
+sdk_version: 4.24.0
 app_file: app.py
 pinned: false
 license: mit
+short_description: A tool to detect Stress, Anxiety and Depression
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Technologies
+
+This project utilizes various Python scripts for different aspects of analysis and recognition:
+
+- `blink_detection.py`: Detects and analyzes blinking patterns.
+- `body_movement_analysis.py`: Analyzes body movements.
+- `emotion_analysis.py`: Analyzes emotional states.
+- `face_expressions.py`: Recognizes facial expressions.
+- `FACS_analysis_sad.py`: Performs Facial Action Coding System analysis for sadness.
+- `gaze_estimation.py`: Estimates gaze direction.
+- `head_posture_detection.py`: Detects head posture.
+- `heart_rate_variability.py`: Analyzes heart rate variability.
+- `posture_analysis.py`: Analyzes posture.
+- `roberta_chatbot.py`: Chatbot using the RoBERTa model.
+- `sentiment_analysis.py`: Performs sentiment analysis.
+- `skin_analysis.py`: Analyzes skin conditions.
+- `sleep_quality.py`: Evaluates sleep quality.
+- `speech_emotion_recognition.py`: Recognizes emotions from speech.
+- `speech_stress_analysis.py`: Analyzes stress levels from speech.
+
+These scripts combine to provide comprehensive analysis capabilities for various aspects of human behavior and physiology.
+
+## Upload Trick to HG
+
+# Track large files with Git LFS
+
+git lfs track "assets/models/_.dat"
+git lfs track "assets/models/_.pt"
+
+# Add the .gitattributes file and commit it
+
+git add .gitattributes
+git commit -m "Track large files with Git LFS"
+
+# Add your large files and commit them
+
+git add assets/models/shape_predictor_68_face_landmarks.dat
+git add assets/models/FER_dinamic_LSTM_IEMOCAP.pt
+git add assets/models/FER_static_ResNet50_AffectNet.pt
+git commit -m "Add large files"
+
+# Add remaining files, commit, and push
+
+git add .
+git commit -m 'pre-launch'
+git push origin main --force

app.py

@@ -4,13 +4,6 @@ from tabs.FACS_analysis import create_facs_analysis_tab
 from ui_components import CUSTOM_CSS, HEADER_HTML, DISCLAIMER_HTML
 import spaces  # Importing spaces to utilize Zero GPU
 
-# Initialize Zero GPU
-if torch.cuda.is_available():
-    zero = torch.Tensor([0]).cuda()
-    print(f"Initial device: {zero.device}")
-else:
-    zero = torch.Tensor([0])
-    print("CUDA is not available. Using CPU.")
 
 # Define the tab structure
 TAB_STRUCTURE = [
@@ -22,10 +15,6 @@ TAB_STRUCTURE = [
 # Decorate GPU-dependent function with Zero GPU
 @spaces.GPU(duration=120)  # Allocates GPU for 120 seconds when needed
 def create_demo():
-    if torch.cuda.is_available():
-        print(f"Device inside create_demo: {zero.device}")
-    else:
-        print("CUDA is not available inside create_demo.")
     
     # Gradio blocks to create the interface
     with gr.Blocks(css=CUSTOM_CSS) as demo:

app/au_processing.py

@@ -0,0 +1,64 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import cv2
+import torch
+from PIL import Image
+from app.model import pth_model_static, cam, pth_processing
+from app.face_utils import get_box
+import mediapipe as mp
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_frame_and_predict_aus(frame):
+    if len(frame.shape) == 2:
+        frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
+    elif frame.shape[2] == 4:
+        frame = cv2.cvtColor(frame, cv2.COLOR_RGBA2RGB)
+
+    with mp_face_mesh.FaceMesh(
+        max_num_faces=1,
+        refine_landmarks=False,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5
+    ) as face_mesh:
+        results = face_mesh.process(frame)
+        
+        if results.multi_face_landmarks:
+            h, w = frame.shape[:2]
+            for fl in results.multi_face_landmarks:
+                startX, startY, endX, endY = get_box(fl, w, h)
+                cur_face = frame[startY:endY, startX:endX]
+                cur_face_n = pth_processing(Image.fromarray(cur_face))
+                
+                with torch.no_grad():
+                    features = pth_model_static(cur_face_n)
+                    au_intensities = features_to_au_intensities(features)
+                
+                grayscale_cam = cam(input_tensor=cur_face_n)
+                grayscale_cam = grayscale_cam[0, :]
+                cur_face_hm = cv2.resize(cur_face, (224, 224))
+                cur_face_hm = np.float32(cur_face_hm) / 255
+                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)
+                
+                return cur_face, au_intensities, heatmap
+    
+    return None, None, None
+
+def features_to_au_intensities(features):
+    features_np = features.detach().cpu().numpy()[0]
+    au_intensities = (features_np - features_np.min()) / (features_np.max() - features_np.min())
+    return au_intensities[:24]  # Assuming we want 24 AUs
+
+def au_statistics_plot(frames, au_intensities_list):
+    fig, ax = plt.subplots(figsize=(12, 6))
+    au_intensities_array = np.array(au_intensities_list)
+    
+    for i in range(au_intensities_array.shape[1]):
+        ax.plot(frames, au_intensities_array[:, i], label=f'AU{i+1}')
+    
+    ax.set_xlabel('Frame')
+    ax.set_ylabel('AU Intensity')
+    ax.set_title('Action Unit Intensities Over Time')
+    ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
+    plt.tight_layout()
+    return fig

app/authors.py

@@ -0,0 +1,34 @@
+"""
+File: authors.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: About the authors.
+License: MIT License
+"""
+
+
+AUTHORS = """
+    Authors: [Elena Ryumina](https://github.com/ElenaRyumina), [Dmitry Ryumin](https://github.com/DmitryRyumin), [Denis Dresvyanskiy](https://www.uni-ulm.de/en/nt/staff/research-assistants/dresvyanskiy/), [Maxim Markitantov](https://hci.nw.ru/en/employees/10) and [Alexey Karpov](https://hci.nw.ru/en/employees/1)
+
+    Authorship contribution:
+
+    App developers: ``Elena Ryumina`` and ``Dmitry Ryumin``
+
+    Methodology developers: ``Elena Ryumina``, ``Denis Dresvyanskiy`` and ``Alexey Karpov``
+
+    Model developer: ``Elena Ryumina``
+
+    TensorFlow to PyTorch model converters: ``Maxim Markitantov`` and ``Elena Ryumina``
+
+    Citation
+
+    If you are using EMO-AffectNetModel in your research, please consider to cite research [paper](https://www.sciencedirect.com/science/article/pii/S0925231222012656). Here is an example of BibTeX entry:
+
+    <div class="highlight highlight-text-bibtex notranslate position-relative overflow-auto" dir="auto"><pre><span class="pl-k">@article</span>{<span class="pl-en">RYUMINA2022</span>,
+        <span class="pl-s">title</span>        = <span class="pl-s"><span class="pl-pds">{</span>In Search of a Robust Facial Expressions Recognition Model: A Large-Scale Visual Cross-Corpus Study<span class="pl-pds">}</span></span>,
+        <span class="pl-s">author</span>       = <span class="pl-s"><span class="pl-pds">{</span>Elena Ryumina and Denis Dresvyanskiy and Alexey Karpov<span class="pl-pds">}</span></span>,
+        <span class="pl-s">journal</span>      = <span class="pl-s"><span class="pl-pds">{</span>Neurocomputing<span class="pl-pds">}</span></span>,
+        <span class="pl-s">year</span>         = <span class="pl-s"><span class="pl-pds">{</span>2022<span class="pl-pds">}</span></span>,
+        <span class="pl-s">doi</span>          = <span class="pl-s"><span class="pl-pds">{</span>10.1016/j.neucom.2022.10.013<span class="pl-pds">}</span></span>,
+        <span class="pl-s">url</span>          = <span class="pl-s"><span class="pl-pds">{</span>https://www.sciencedirect.com/science/article/pii/S0925231222012656<span class="pl-pds">}</span></span>,
+    }</div>
+"""

app/config.py

@@ -0,0 +1,49 @@
+"""
+File: config.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Configuration file.
+License: MIT License
+"""
+
+import toml
+from typing import Dict
+from types import SimpleNamespace
+
+
+def flatten_dict(prefix: str, d: Dict) -> Dict:
+    result = {}
+
+    for k, v in d.items():
+        if isinstance(v, dict):
+            result.update(flatten_dict(f"{prefix}{k}_", v))
+        else:
+            result[f"{prefix}{k}"] = v
+
+    return result
+
+
+config = toml.load("config.toml")
+
+config_data = flatten_dict("", config)
+
+config_data = SimpleNamespace(**config_data)
+
+DICT_EMO = {
+    0: "Neutral",
+    1: "Happiness",
+    2: "Sadness",
+    3: "Surprise",
+    4: "Fear",
+    5: "Disgust",
+    6: "Anger",
+}
+
+COLORS = {
+    0: 'blue', 
+    1: 'orange', 
+    2: 'green', 
+    3: 'red', 
+    4: 'purple', 
+    5: 'brown', 
+    6: 'pink'
+}

app/description.py

@@ -0,0 +1,46 @@
+"""
+File: description.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Project description for the Gradio app.
+License: MIT License
+"""
+
+# Importing necessary components for the Gradio app
+from app.config import config_data
+
+DESCRIPTION_STATIC = f"""\
+# Static Facial Expression Recognition
+<div class="app-flex-container">
+    <img src="https://img.shields.io/badge/version-v{config_data.APP_VERSION}-rc0" alt="Version">
+    <a href="https://visitorbadge.io/status?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition"><img src="https://api.visitorbadge.io/api/combined?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition&countColor=%23263759&style=flat" /></a>
+    <a href="https://paperswithcode.com/paper/in-search-of-a-robust-facial-expressions"><img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/in-search-of-a-robust-facial-expressions/facial-expression-recognition-on-affectnet" /></a>
+    </div>
+"""
+
+DESCRIPTION_DYNAMIC = f"""\
+# Dynamic Facial Expression Recognition
+<div class="app-flex-container">
+    <img src="https://img.shields.io/badge/version-v{config_data.APP_VERSION}-rc0" alt="Version">
+    <a href="https://paperswithcode.com/paper/in-search-of-a-robust-facial-expressions"><img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/in-search-of-a-robust-facial-expressions/facial-expression-recognition-on-affectnet" /></a>
+    </div>
+"""
+
+DESCRIPTION_SLEEP_QUALITY = """
+# Sleep Quality Analysis
+
+This tab analyzes sleep quality based on facial features, focusing on skin tone and eye bags. 
+
+## How to use:
+1. Upload a video of a person's face.
+2. Click 'Submit' to process the video.
+3. View the results, including:
+   - Original video
+   - Processed face video
+   - Sleep quality analysis video
+   - Eye bags detection image
+   - Sleep quality statistics over time
+
+The analysis provides insights into potential sleep issues based on visual cues.
+
+Note: This analysis is for informational purposes only and should not be considered a medical diagnosis. Always consult with a healthcare professional for sleep-related concerns.
+"""

app/image_processing.py

@@ -0,0 +1,49 @@
+import numpy as np
+import cv2
+from PIL import Image
+import torch
+from app.model import pth_model_static, cam, pth_processing
+from app.face_utils import get_box
+from app.config import DICT_EMO
+from pytorch_grad_cam.utils.image import show_cam_on_image
+import mediapipe as mp
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_image_and_predict(inp):
+    inp = np.array(inp)
+
+    if inp is None:
+        return None, None, None
+
+    try:
+        h, w = inp.shape[:2]
+    except Exception:
+        return None, None, None
+
+    with mp_face_mesh.FaceMesh(
+        max_num_faces=1,
+        refine_landmarks=False,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5,
+    ) as face_mesh:
+        results = face_mesh.process(inp)
+        if results.multi_face_landmarks:
+            for fl in results.multi_face_landmarks:
+                startX, startY, endX, endY = get_box(fl, w, h)
+                cur_face = inp[startY:endY, startX:endX]
+                cur_face_n = pth_processing(Image.fromarray(cur_face))
+                with torch.no_grad():
+                    prediction = (
+                        torch.nn.functional.softmax(pth_model_static(cur_face_n), dim=1)
+                        .detach()
+                        .numpy()[0]
+                    )
+                confidences = {DICT_EMO[i]: float(prediction[i]) for i in range(7)}
+                grayscale_cam = cam(input_tensor=cur_face_n)
+                grayscale_cam = grayscale_cam[0, :]
+                cur_face_hm = cv2.resize(cur_face,(224,224))
+                cur_face_hm = np.float32(cur_face_hm) / 255
+                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)
+
+    return cur_face, heatmap, confidences

app/plot.py

@@ -0,0 +1,29 @@
+"""
+File: config.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Plotting statistical information.
+License: MIT License
+"""
+import matplotlib.pyplot as plt
+import numpy as np 
+
+# Importing necessary components for the Gradio app
+from app.config import DICT_EMO, COLORS
+
+
+def statistics_plot(frames, probs):
+    fig, ax = plt.subplots(figsize=(10, 4))
+    fig.subplots_adjust(left=0.07, bottom=0.14, right=0.98, top=0.8, wspace=0, hspace=0)
+    # Установка параметров left, bottom, right, top, чтобы выделить место для легенды и названий осей
+    probs = np.array(probs)
+    for i in range(7):
+        try:
+            ax.plot(frames, probs[:, i], label=DICT_EMO[i], c=COLORS[i]) 
+        except Exception:
+            return None
+            
+    ax.legend(loc='upper center', bbox_to_anchor=(0.47, 1.2), ncol=7, fontsize=12) 
+    ax.set_xlabel('Frames', fontsize=12)  # Добавляем подпись к оси X
+    ax.set_ylabel('Probability', fontsize=12)  # Добавляем подпись к оси Y
+    ax.grid(True)
+    return plt

app/sleep_quality_processing.py

@@ -0,0 +1,94 @@
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import mediapipe as mp
+from app.face_utils import get_box
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_video_and_predict_sleep_quality(video):
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video_original = 'result_original.mp4'
+    path_save_video_face = 'result_face.mp4'
+    path_save_video_sleep = 'result_sleep.mp4'
+    
+    vid_writer_original = cv2.VideoWriter(path_save_video_original, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))
+    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+    vid_writer_sleep = cv2.VideoWriter(path_save_video_sleep, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    frames = []
+    sleep_quality_scores = []
+    eye_bags_images = []
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_rgb)
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY = get_box(fl, w, h)
+                    cur_face = frame_rgb[startY:endY, startX:endX]
+                    
+                    sleep_quality_score, eye_bags_image = analyze_sleep_quality(cur_face)
+                    sleep_quality_scores.append(sleep_quality_score)
+                    eye_bags_images.append(cv2.resize(eye_bags_image, (224, 224)))
+
+                    sleep_quality_viz = create_sleep_quality_visualization(cur_face, sleep_quality_score)
+                    
+                    cur_face = cv2.resize(cur_face, (224, 224))
+                    
+                    vid_writer_face.write(cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR))
+                    vid_writer_sleep.write(sleep_quality_viz)
+
+            vid_writer_original.write(frame)
+            frames.append(len(frames) + 1)
+
+    cap.release()
+    vid_writer_original.release()
+    vid_writer_face.release()
+    vid_writer_sleep.release()
+
+    sleep_stat = sleep_quality_statistics_plot(frames, sleep_quality_scores)
+    
+    if eye_bags_images:
+        average_eye_bags_image = np.mean(np.array(eye_bags_images), axis=0).astype(np.uint8)
+    else:
+        average_eye_bags_image = np.zeros((224, 224, 3), dtype=np.uint8)
+
+    return (path_save_video_original, path_save_video_face, path_save_video_sleep, 
+            average_eye_bags_image, sleep_stat)
+
+def analyze_sleep_quality(face_image):
+    # Placeholder function - implement your sleep quality analysis here
+    sleep_quality_score = np.random.random()
+    eye_bags_image = cv2.resize(face_image, (224, 224))
+    return sleep_quality_score, eye_bags_image
+
+def create_sleep_quality_visualization(face_image, sleep_quality_score):
+    viz = face_image.copy()
+    cv2.putText(viz, f"Sleep Quality: {sleep_quality_score:.2f}", (10, 30), 
+                cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
+    return cv2.cvtColor(viz, cv2.COLOR_RGB2BGR)
+
+def sleep_quality_statistics_plot(frames, sleep_quality_scores):
+    fig, ax = plt.subplots()
+    ax.plot(frames, sleep_quality_scores)
+    ax.set_xlabel('Frame')
+    ax.set_ylabel('Sleep Quality Score')
+    ax.set_title('Sleep Quality Over Time')
+    plt.tight_layout()
+    return fig

app/video_processing.py

@@ -0,0 +1,132 @@
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+import mediapipe as mp
+from app.model import pth_model_static, pth_model_dynamic, cam, pth_processing
+from app.face_utils import get_box, display_info
+from app.config import config_data
+from app.plot import statistics_plot
+from .au_processing import features_to_au_intensities, au_statistics_plot
+
+mp_face_mesh = mp.solutions.face_mesh
+
+def preprocess_video_and_predict(video):
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video_face = 'result_face.mp4'
+    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    path_save_video_hm = 'result_hm.mp4'
+    vid_writer_hm = cv2.VideoWriter(path_save_video_hm, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    lstm_features = []
+    count_frame = 1
+    count_face = 0
+    probs = []
+    frames = []
+    au_intensities_list = []
+    last_output = None
+    last_heatmap = None 
+    last_au_intensities = None
+    cur_face = None
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            _, frame = cap.read()
+            if frame is None: break
+
+            frame_copy = frame.copy()
+            frame_copy.flags.writeable = False
+            frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_copy)
+            frame_copy.flags.writeable = True
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY  = get_box(fl, w, h)
+                    cur_face = frame_copy[startY:endY, startX: endX]
+
+                    if count_face%config_data.FRAME_DOWNSAMPLING == 0:
+                        cur_face_copy = pth_processing(Image.fromarray(cur_face))
+                        with torch.no_grad():
+                            features = torch.nn.functional.relu(pth_model_static.extract_features(cur_face_copy)).detach().numpy()
+                            au_intensities = features_to_au_intensities(pth_model_static(cur_face_copy))
+
+                        grayscale_cam = cam(input_tensor=cur_face_copy)
+                        grayscale_cam = grayscale_cam[0, :]
+                        cur_face_hm = cv2.resize(cur_face,(224,224), interpolation = cv2.INTER_AREA)
+                        cur_face_hm = np.float32(cur_face_hm) / 255
+                        heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=False)
+                        last_heatmap = heatmap
+                        last_au_intensities = au_intensities
+        
+                        if len(lstm_features) == 0:
+                            lstm_features = [features]*10
+                        else:
+                            lstm_features = lstm_features[1:] + [features]
+
+                        lstm_f = torch.from_numpy(np.vstack(lstm_features))
+                        lstm_f = torch.unsqueeze(lstm_f, 0)
+                        with torch.no_grad():
+                            output = pth_model_dynamic(lstm_f).detach().numpy()
+                        last_output = output
+
+                        if count_face == 0:
+                            count_face += 1
+
+                    else:
+                        if last_output is not None:
+                            output = last_output
+                            heatmap = last_heatmap
+                            au_intensities = last_au_intensities
+
+                        elif last_output is None:
+                            output = np.empty((1, 7))
+                            output[:] = np.nan
+                            au_intensities = np.empty(24)
+                            au_intensities[:] = np.nan
+                            
+                    probs.append(output[0])
+                    frames.append(count_frame)
+                    au_intensities_list.append(au_intensities)
+            else:
+                if last_output is not None:
+                    lstm_features = []
+                    empty = np.empty((7))
+                    empty[:] = np.nan
+                    probs.append(empty)
+                    frames.append(count_frame)
+                    au_intensities_list.append(np.full(24, np.nan))
+
+            if cur_face is not None:
+                heatmap_f = display_info(heatmap, 'Frame: {}'.format(count_frame), box_scale=.3)
+
+                cur_face = cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR)
+                cur_face = cv2.resize(cur_face, (224,224), interpolation = cv2.INTER_AREA)
+                cur_face = display_info(cur_face, 'Frame: {}'.format(count_frame), box_scale=.3)
+                vid_writer_face.write(cur_face)
+                vid_writer_hm.write(heatmap_f)
+
+            count_frame += 1
+            if count_face != 0:
+                count_face += 1
+
+        vid_writer_face.release()
+        vid_writer_hm.release()
+
+        stat = statistics_plot(frames, probs)
+        au_stat = au_statistics_plot(frames, au_intensities_list)
+
+        if not stat or not au_stat:
+            return None, None, None, None, None
+        
+    return video, path_save_video_face, path_save_video_hm, stat, au_stat

requirements.txt

@@ -1,49 +1,139 @@
-# CUDA-enabled PyTorch packages
-torch
-torchvision
-torchaudio
-
-# Core dependencies
-gradio==4.38.1
-gradio_client==1.1.0
-
-# Additional dependencies
 absl-py==2.1.0
 aiofiles==23.2.1
 altair==5.3.0
+annotated-types==0.7.0
 anyio==4.4.0
+astunparse==1.6.3
 attrs==23.2.0
 audioread==3.0.1
 certifi==2024.7.4
+cffi==1.16.0
 charset-normalizer==3.3.2
 click==8.1.7
+contourpy==1.2.1
+cycler==0.12.1
 decorator==4.4.2
+dlib==19.24.4
+dnspython==2.6.1
+email_validator==2.2.0
+exceptiongroup==1.2.2
 fastapi==0.111.1
+fastapi-cli==0.0.4
+ffmpy==0.3.2
+filelock==3.15.4
+flatbuffers==24.3.25
+fonttools==4.53.1
+fsspec==2024.6.1
+gast==0.6.0
+google-pasta==0.2.0
+grad-cam
+gradio==4.38.1
+gradio_client==1.1.0
+grpcio==1.64.1
+h11==0.14.0
 h5py==3.11.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
 huggingface-hub==0.23.5
 idna==3.7
+imageio==2.34.2
+imageio-ffmpeg==0.5.1
+importlib_resources==6.4.0
+imutils==0.5.4
+jax==0.4.30
+jaxlib==0.4.30
 Jinja2==3.1.4
 joblib==1.4.2
 jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+keras==3.4.1
 kiwisolver==1.4.5
+lazy_loader==0.4
+libclang==18.1.1
 librosa==0.10.2.post1
+llvmlite==0.43.0
+Markdown==3.6
+markdown-it-py==3.0.0
 MarkupSafe==2.1.5
 matplotlib==3.9.1
+mdurl==0.1.2
+mediapipe==0.10.14
+ml-dtypes==0.4.0
+moviepy==1.0.3
+mpmath==1.3.0
+msgpack==1.0.8
+namex==0.0.8
+networkx==3.3
+numba==0.60.0
 numpy==1.26.4
+opencv-contrib-python==4.10.0.84
+opencv-python==4.10.0.84
+opt-einsum==3.3.0
+optree==0.12.1
+orjson==3.10.6
+packaging==24.1
 pandas==2.2.2
-Pillow==10.4.0
+pillow==10.4.0
+platformdirs==4.2.2
+pooch==1.8.2
+proglog==0.1.10
+protobuf==4.25.3
+pycparser==2.22
 pydantic==2.8.2
+pydantic_core==2.20.1
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
 python-multipart==0.0.9
 pytz==2024.1
 PyYAML==6.0.1
+referencing==0.35.1
+regex==2024.5.15
 requests==2.32.3
+rich==13.7.1
+rpds-py==0.19.0
+ruff==0.5.2
+safetensors==0.4.3
 scikit-learn==1.5.1
 scipy==1.14.0
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+sounddevice==0.4.7
 soundfile==0.12.1
+soxr==0.3.7
 starlette==0.37.2
+sympy==1.13.0
+tensorboard==2.17.0
+tensorboard-data-server==0.7.2
+tensorflow==2.17.0
+tensorflow-io-gcs-filesystem==0.37.1
+termcolor==2.4.0
+tf_keras==2.17.0
+threadpoolctl==3.5.0
+tokenizers==0.19.1
+toml==0.10.2
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.3.1
+torchaudio==2.3.1
+torchvision==0.18.1
 tqdm==4.66.4
 transformers==4.42.4
+ttach==0.0.3
+typer==0.12.3
+typing_extensions==4.12.2
+tzdata==2024.1
+ujson==5.10.0
+urllib3==2.2.2
 uvicorn==0.30.1
-
-# Any other necessary dependencies
-# Add your additional dependencies here
+uvloop==0.19.0
+watchfiles==0.22.0
+wavio==0.0.9
+websockets==11.0.3
+Werkzeug==3.0.3
+wrapt==1.16.0

tabs/__emotion_analysis.py

@@ -0,0 +1,36 @@
+import os
+import torch
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import gradio as gr
+
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
+
+emotion_tokenizer = AutoTokenizer.from_pretrained("j-hartmann/emotion-english-distilroberta-base")
+emotion_model = AutoModelForSequenceClassification.from_pretrained("j-hartmann/emotion-english-distilroberta-base")
+emotion_labels = ["anger", "disgust", "fear", "joy", "neutral", "sadness", "surprise"]
+
+def analyze_emotion(text):
+    try:
+        inputs = emotion_tokenizer(text, return_tensors="pt", truncation=True, padding=True, max_length=512)
+        outputs = emotion_model(**inputs)
+        probs = torch.nn.functional.softmax(outputs.logits, dim=-1)
+        max_prob, max_index = torch.max(probs, dim=1)
+        return emotion_labels[max_index.item()], f"{max_prob.item():.4f}"
+    except Exception as e:
+        print(f"Error in emotion analysis: {e}")
+        return "Error", "N/A"
+
+def create_emotion_tab():
+    with gr.Row():
+        with gr.Column(scale=2):
+            input_text = gr.Textbox(value='I actually speak to the expets myself to give you the best value you can get', lines=5, placeholder="Enter text here...", label="Input Text")
+            with gr.Row():
+                clear_btn = gr.Button("Clear", scale=1)
+                submit_btn = gr.Button("Analyze", scale=1, elem_classes="submit")
+        with gr.Column(scale=1):
+            output_emotion = gr.Textbox(label="Detected Emotion")
+            output_confidence = gr.Textbox(label="Emotion Confidence Score")
+    
+    submit_btn.click(analyze_emotion, inputs=[input_text], outputs=[output_emotion, output_confidence])
+    clear_btn.click(lambda: ("", "", ""), outputs=[input_text, output_emotion, output_confidence])
+    gr.Examples(["I am so happy today!", "I feel terrible and sad.", "This is a neutral statement."], inputs=[input_text])

tabs/__sentiment_analysis.py

@@ -0,0 +1,36 @@
+import os
+import torch
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import gradio as gr
+
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
+
+sentiment_tokenizer = AutoTokenizer.from_pretrained("nlptown/bert-base-multilingual-uncased-sentiment")
+sentiment_model = AutoModelForSequenceClassification.from_pretrained("nlptown/bert-base-multilingual-uncased-sentiment")
+sentiment_labels = ["very negative", "negative", "neutral", "positive", "very positive"]
+
+def analyze_sentiment(text):
+    try:
+        inputs = sentiment_tokenizer(text, return_tensors="pt", truncation=True, padding=True, max_length=512)
+        outputs = sentiment_model(**inputs)
+        probs = torch.nn.functional.softmax(outputs.logits, dim=-1)
+        max_prob, max_index = torch.max(probs, dim=1)
+        return sentiment_labels[max_index.item()], f"{max_prob.item():.4f}"
+    except Exception as e:
+        print(f"Error in sentiment analysis: {e}")
+        return "Error", "N/A"
+
+def create_sentiment_tab():
+    with gr.Row():
+        with gr.Column(scale=2):
+            input_text = gr.Textbox(value="I actually speak to the expets myself to give you the best value you can get", lines=5, placeholder="Enter text here...", label="Input Text")
+            with gr.Row():
+                clear_btn = gr.Button("Clear", scale=1)
+                submit_btn = gr.Button("Analyze", scale=1, elem_classes="submit")
+        with gr.Column(scale=1):
+            output_sentiment = gr.Textbox(label="Detected Sentiment")
+            output_confidence = gr.Textbox(label="Sentiment Confidence Score")
+    
+    submit_btn.click(analyze_sentiment, inputs=[input_text], outputs=[output_sentiment, output_confidence], queue=True)
+    clear_btn.click(lambda: ("", "", ""), outputs=[input_text, output_sentiment, output_confidence], queue=True)
+    gr.Examples(["I am so happy today!", "I feel terrible and sad.", "This is a neutral statement."], inputs=[input_text])

tabs/speech_emotion_recognition.py

@@ -0,0 +1,206 @@
+# tabs/speech_emotion_recognition.py
+
+import gradio as gr
+import numpy as np
+import librosa
+import librosa.display
+import matplotlib.pyplot as plt
+from transformers import pipeline
+import torch
+import tempfile
+import warnings
+import os
+
+# Suppress specific warnings from transformers if needed
+warnings.filterwarnings("ignore", category=UserWarning, module='transformers')
+
+# Determine the device
+def get_device():
+    if torch.backends.mps.is_available():
+        device = torch.device("mps")
+        print("Using MPS device for inference.")
+    elif torch.cuda.is_available():
+        device = torch.device("cuda")
+        print("Using CUDA device for inference.")
+    else:
+        device = torch.device("cpu")
+        print("Using CPU for inference.")
+    return device
+
+device = get_device()
+
+# Initialize the pipelines with the specified device
+try:
+    emotion_model = pipeline(
+        "audio-classification",
+        model="ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition",
+        device=0 if device.type == "cuda" else ("mps" if device.type == "mps" else -1)
+    )
+    print("Emotion model loaded successfully.")
+except Exception as e:
+    print(f"Error loading emotion model: {e}")
+    emotion_model = None
+
+try:
+    transcription_model = pipeline(
+        "automatic-speech-recognition",
+        model="facebook/wav2vec2-base-960h",
+        device=0 if device.type == "cuda" else ("mps" if device.type == "mps" else -1)
+    )
+    print("Transcription model loaded successfully.")
+except Exception as e:
+    print(f"Error loading transcription model: {e}")
+    transcription_model = None
+
+# Emotion Mapping
+emotion_mapping = {
+    "angry": (0.8, 0.8, -0.5),
+    "happy": (0.6, 0.6, 0.8),
+    "sad": (-0.6, -0.4, -0.6),
+    "neutral": (0, 0, 0),
+    "fear": (0.3, -0.3, -0.7),
+    "surprise": (0.4, 0.2, 0.2),
+    "disgust": (0.2, 0.5, -0.6),
+    "calm": (-0.2, 0.1, 0.3),
+    "excited": (0.7, 0.5, 0.7),
+    "frustrated": (0.6, 0.5, -0.4)
+}
+
+def process_audio_emotion(audio_file):
+    """
+    Processes the input audio file to perform transcription and emotion recognition.
+    Generates waveform and mel spectrogram plots.
+
+    Returns:
+        A tuple containing:
+        - Transcription (str)
+        - Emotion (str)
+        - Confidence (%) (float)
+        - Arousal (float)
+        - Dominance (float)
+        - Valence (float)
+        - Waveform Plot (str: filepath)
+        - Mel Spectrogram Plot (str: filepath)
+    """
+    if not audio_file:
+        return (
+            "No audio file provided.",  # Transcription (textbox)
+            None,                       # Emotion (textbox)
+            None,                       # Confidence (%) (number)
+            None,                       # Arousal (number)
+            None,                       # Dominance (number)
+            None,                       # Valence (number)
+            None,                       # Waveform Plot (image)
+            None                        # Mel Spectrogram Plot (image)
+        )
+
+    try:
+        y, sr = librosa.load(audio_file, sr=None)
+
+        # Transcription
+        if transcription_model:
+            transcription_result = transcription_model(audio_file)
+            transcription = transcription_result.get("text", "N/A")
+        else:
+            transcription = "Transcription model not loaded."
+
+        # Emotion Recognition
+        if emotion_model:
+            emotion_results = emotion_model(audio_file)
+            if emotion_results:
+                emotion_result = emotion_results[0]
+                emotion = emotion_result.get("label", "Unknown").lower()
+                confidence = emotion_result.get("score", 0.0) * 100  # Convert to percentage
+                arousal, dominance, valence = emotion_mapping.get(emotion, (0.0, 0.0, 0.0))
+            else:
+                emotion = "No emotion detected."
+                confidence = 0.0
+                arousal, dominance, valence = 0.0, 0.0, 0.0
+        else:
+            emotion = "Emotion model not loaded."
+            confidence = 0.0
+            arousal, dominance, valence = 0.0, 0.0, 0.0
+
+        # Plotting Waveform
+        plt.figure(figsize=(10, 4))
+        librosa.display.waveshow(y, sr=sr)
+        plt.title("Waveform")
+        plt.xlabel("Time (s)")
+        plt.ylabel("Amplitude")
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as tmp_waveform:
+            plt.savefig(tmp_waveform.name, bbox_inches='tight')
+            waveform_plot_path = tmp_waveform.name
+        plt.close()
+
+        # Plotting Mel Spectrogram
+        mel_spec = librosa.feature.melspectrogram(y=y, sr=sr)
+        plt.figure(figsize=(10, 4))
+        librosa.display.specshow(librosa.power_to_db(mel_spec, ref=np.max), sr=sr, x_axis='time', y_axis='mel')
+        plt.colorbar(format='%+2.0f dB')
+        plt.title("Mel Spectrogram")
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as tmp_mel:
+            plt.savefig(tmp_mel.name, bbox_inches='tight')
+            mel_spec_plot_path = tmp_mel.name
+        plt.close()
+
+        return (
+            transcription,                  # Transcription (textbox)
+            emotion.capitalize(),           # Emotion (textbox)
+            confidence,                     # Confidence (%) (number)
+            arousal,                        # Arousal (number)
+            dominance,                      # Dominance (number)
+            valence,                        # Valence (number)
+            waveform_plot_path,             # Waveform Plot (image)
+            mel_spec_plot_path              # Mel Spectrogram Plot (image)
+        )
+    except Exception as e:
+        return (
+            f"Error: {str(e)}",  # Transcription (textbox)
+            None,                 # Emotion (textbox)
+            None,                 # Confidence (%) (number)
+            None,                 # Arousal (number)
+            None,                 # Dominance (number)
+            None,                 # Valence (number)
+            None,                 # Waveform Plot (image)
+            None                  # Mel Spectrogram Plot (image)
+        )
+
+def create_emotion_recognition_tab():
+    """
+    Creates the Emotion Recognition tab in the Gradio interface.
+    """
+    with gr.Row():
+        with gr.Column(scale=2):
+            input_audio = gr.Audio(label="Input Audio", type="filepath")
+            gr.Examples(
+                examples=["./assets/audio/fitness.wav"],
+                inputs=[input_audio],
+                label="Examples"
+            )
+        with gr.Column(scale=1):
+            transcription_output = gr.Textbox(label="Transcription", interactive=False)
+            emotion_output = gr.Textbox(label="Emotion", interactive=False)
+            confidence_output = gr.Number(label="Confidence (%)", interactive=False)
+            arousal_output = gr.Number(label="Arousal (Level of Energy)", interactive=False)
+            dominance_output = gr.Number(label="Dominance (Degree of Control)", interactive=False)
+            valence_output = gr.Number(label="Valence (Positivity/Negativity)", interactive=False)
+        with gr.Column(scale=1):
+            waveform_plot = gr.Image(label="Waveform")
+            mel_spec_plot = gr.Image(label="Mel Spectrogram")
+
+    input_audio.change(
+        fn=process_audio_emotion,
+        inputs=[input_audio],
+        outputs=[
+            transcription_output,
+            emotion_output,
+            confidence_output,
+            arousal_output,
+            dominance_output,
+            valence_output,
+            waveform_plot,
+            mel_spec_plot
+        ]
+    )
+
+# Call create_emotion_recognition_tab to create the Gradio interface

tabs/speech_stress_analysis.py

@@ -0,0 +1,126 @@
+# tabs/speech_stress_analysis.py
+
+import gradio as gr
+import librosa
+import librosa.display
+import numpy as np
+import matplotlib.pyplot as plt
+import tempfile
+import warnings
+
+# Suppress specific warnings from transformers if needed
+warnings.filterwarnings("ignore", category=UserWarning, module='transformers')
+
+def extract_audio_features(audio_file):
+    y, sr = librosa.load(audio_file, sr=None)
+    mfccs = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=13)
+    pitches, magnitudes = librosa.piptrack(y=y, sr=sr)
+    pitches = pitches[(magnitudes > np.median(magnitudes)) & (pitches > 0)]
+    energy = librosa.feature.rms(y=y)[0]
+    return mfccs, pitches, energy, y, sr
+
+def analyze_voice_stress(audio_file):
+    if not audio_file:
+        return "No audio file provided.", None
+
+    try:
+        mfccs, pitches, energy, y, sr = extract_audio_features(audio_file)
+
+        # Calculate variances
+        var_mfccs = np.var(mfccs, axis=1).mean()  # Mean variance across MFCC coefficients
+        var_energy = np.var(energy)               # Variance of RMS energy
+        var_pitches = np.var(pitches) if len(pitches) > 0 else 0  # Variance of pitches if present
+
+        # Debugging: Print individual variances
+        print(f"Variance MFCCs (mean across coefficients): {var_mfccs}")
+        print(f"Variance Energy: {var_energy}")
+        print(f"Variance Pitches: {var_pitches}")
+
+        # Normalize each variance using Z-Score Standardization
+        mfccs_mean = 1000
+        mfccs_std = 500
+        energy_mean = 0.005
+        energy_std = 0.005
+        pitches_mean = 500000
+        pitches_std = 200000
+
+        norm_var_mfccs = (var_mfccs - mfccs_mean) / mfccs_std
+        norm_var_energy = (var_energy - energy_mean) / energy_std
+        norm_var_pitches = (var_pitches - pitches_mean) / pitches_std if var_pitches > 0 else 0
+
+        # Debugging: Print normalized variances
+        print(f"Normalized Variance MFCCs: {norm_var_mfccs}")
+        print(f"Normalized Variance Energy: {norm_var_energy}")
+        print(f"Normalized Variance Pitches: {norm_var_pitches}")
+
+        # Combine normalized variances
+        stress_level = np.mean([
+            norm_var_mfccs,
+            norm_var_energy,
+            norm_var_pitches
+        ]) if var_pitches > 0 else np.mean([norm_var_mfccs, norm_var_energy])
+
+        # Debugging: Print stress_level before normalization
+        print(f"Calculated Stress Level (before scaling): {stress_level}")
+
+        # Scale to 0-100%
+        normalized_stress = (stress_level + 3) / 6 * 100  # Maps -3 to 0%, +3 to 100%
+        normalized_stress = np.clip(normalized_stress, 0, 100)  # Ensure within 0-100%
+
+        # Debugging: Print normalized_stress
+        print(f"Normalized Stress Level: {normalized_stress}")
+
+        # Plotting
+        fig, axs = plt.subplots(3, 1, figsize=(10, 12))
+
+        # MFCCs
+        img_mfcc = librosa.display.specshow(mfccs, sr=sr, x_axis='time', ax=axs[0])
+        axs[0].set_title('MFCCs')
+        axs[0].set_ylabel('MFCC Coefficient')
+        fig.colorbar(img_mfcc, ax=axs[0])
+
+        # Pitch
+        axs[1].plot(pitches)
+        axs[1].set_title('Pitch')
+        axs[1].set_ylabel('Frequency (Hz)')
+
+        # Energy
+        axs[2].plot(energy)
+        axs[2].set_title('Energy (RMS)')
+        axs[2].set_ylabel('RMS Energy')
+        axs[2].set_xlabel('Frames')
+
+        plt.tight_layout()
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.png') as temp_file:
+            plt.savefig(temp_file.name)
+            plot_path = temp_file.name
+        plt.close()
+
+        # Interpretation
+        if normalized_stress < 33:
+            stress_interpretation = "Low"
+        elif normalized_stress < 66:
+            stress_interpretation = "Medium"
+        else:
+            stress_interpretation = "High"
+
+        return f"{normalized_stress:.2f}% - {stress_interpretation} Stress", plot_path
+    except Exception as e:
+        return f"Error: {str(e)}", None
+
+def create_voice_stress_tab():
+    with gr.Row():
+        with gr.Column(scale=2):
+            input_audio = gr.Audio(label="Input Audio", type="filepath")
+            clear_btn = gr.Button("Clear", scale=1)
+        with gr.Column(scale=1):
+            output_stress = gr.Label(label="Stress Level")
+            output_plot = gr.Image(label="Stress Analysis Plot")
+
+    # Automatically trigger analysis when an audio file is uploaded
+    input_audio.change(analyze_voice_stress, inputs=[input_audio], outputs=[output_stress, output_plot])
+
+    clear_btn.click(lambda: (None, None), outputs=[input_audio, output_stress, output_plot])
+
+    gr.Examples(["./assets/audio/fitness.wav"], inputs=[input_audio])
+