gr state on human3d

main_noweb.py

@@ -32,9 +32,9 @@ print("[INFO]: Imported modules!")
 human = MMPoseInferencer("simcc_mobilenetv2_wo-deconv-8xb64-210e_coco-256x192") # simcc_mobilenetv2_wo-deconv-8xb64-210e_coco-256x192 dekr_hrnet-w32_8xb10-140e_coco-512x512
 hand = MMPoseInferencer("hand")
 #model3d = gr.State()
-human3d = MMPoseInferencer(device=device, 
+human3d = gr.State(MMPoseInferencer(device=device, 
                                pose3d="human3d", 
-                               scope="mmpose")
+                               scope="mmpose"))
 
 
 #"https://github.com/open-mmlab/mmpose/blob/main/configs/body_3d_keypoint/pose_lift/h36m/pose-lift_simplebaseline3d_8xb64-200e_h36m.py", 
@@ -94,16 +94,14 @@ def check_extension(video):
     return video
 
 
-def pose3d(video, kpt_threshold, ):
+def pose3d(video, kpt_threshold, model):
 
     video = check_extension(video)
     print(device)
     
     #human3d = MMPoseInferencer(device=device,   pose3d="human3d",  scope="mmpose")#"pose-lift_videopose3d-243frm-supv-cpn-ft_8xb128-200e_h36m")
 
-    print("HUMAN 3d downloaded!!")
-    human3dst = gr.State(value=human3d)
-    
+    print("HUMAN 3d downloaded!!")    
     # Define new unique folder
     add_dir = str(uuid.uuid4())
     vis_out_dir = os.path.join("/".join(video.split("/")[:-1]), add_dir)
@@ -111,7 +109,7 @@ def pose3d(video, kpt_threshold, ):
     os.makedirs(add_dir)
     print(check_fps(video))
     #video = human3d.preprocess(video, batch_size=8)
-    result_generator = human3dst(video, 
+    result_generator = model(video, 
                             vis_out_dir = add_dir,
                             radius = 8,
                             thickness = 5,
@@ -158,11 +156,11 @@ def pose2d(video, kpt_threshold):
 
     return "".join(out_file), "".join(kpoints)
 
-def pose3dbatch(video, kpt_threshold):
+def pose3dbatch(video, kpt_threshold, model):
     kpoints=[]
     outvids=[]
-    for v, t in zip(video, kpt_threshold):
-        vname, kname = pose3d(v, t)
+    for v, t in zip(video, kpt_threshold, model):
+        vname, kname = pose3d(v, t, model)
         outvids.append(vname) 
         kpoints.append(kname)
     return [outvids]#kpoints, outvids
@@ -195,61 +193,60 @@ def pose2dhand(video, kpt_threshold):
    
     return "".join(out_file), "".join(kpoints)
 
-def UI():
-    block = gr.Blocks()
-
-    with block:
-        with gr.Column():       
-            with gr.Tab("Upload video"):
-                with gr.Column():
-                    with gr.Row():
-                        with gr.Column():
-                            with gr.Row():
-                                video_input = gr.Video(source="upload", type="filepath", height=256, width=192)
-                                # Insert slider with kpt_thr
-                                with gr.Column():
-                                    gr.Markdown("Drag the keypoint threshold to filter out lower probability keypoints:")
-                                    file_kpthr = gr.Slider(0, 1, value=0.3, label='Keypoint threshold')
-                            with gr.Row():
-                                submit_pose_file = gr.Button("Make 2d pose estimation")
-                                submit_pose3d_file = gr.Button("Make 3d pose estimation")
-                                submit_hand_file = gr.Button("Make 2d hand estimation")
-
-                    with gr.Row():
-                        video_output1 = gr.PlayableVideo(label = "Estimate human 2d poses", show_label=True, height=256)
-                        video_output2 = gr.PlayableVideo(label = "Estimate human 3d poses", show_label=True, height=256)
-                        video_output3 = gr.PlayableVideo(label = "Estimate human hand poses", show_label=True, height=256)
+block = gr.Blocks()
+
+with block:
+    with gr.Column():       
+        with gr.Tab("Upload video"):
+            with gr.Column():
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Row():
+                            video_input = gr.Video(source="upload", type="filepath", height=256, width=192)
+                            # Insert slider with kpt_thr
+                            with gr.Column():
+                                gr.Markdown("Drag the keypoint threshold to filter out lower probability keypoints:")
+                                file_kpthr = gr.Slider(0, 1, value=0.3, label='Keypoint threshold')
+                        with gr.Row():
+                            submit_pose_file = gr.Button("Make 2d pose estimation")
+                            submit_pose3d_file = gr.Button("Make 3d pose estimation")
+                            submit_hand_file = gr.Button("Make 2d hand estimation")
+
+                with gr.Row():
+                    video_output1 = gr.PlayableVideo(label = "Estimate human 2d poses", show_label=True, height=256)
+                    video_output2 = gr.PlayableVideo(label = "Estimate human 3d poses", show_label=True, height=256)
+                    video_output3 = gr.PlayableVideo(label = "Estimate human hand poses", show_label=True, height=256)
+                
+                gr.Markdown("Download the .json file that contains the keypoint positions for each frame in the video.")
+                jsonoutput = gr.File(file_types=[".json"])
+                gr.Markdown("""There are multiple ways to interact with these keypoints. 
+                            \n The example below shows how you can calulate the angle on the elbow for example.
+                            \n Copy the code into your own preferred interpreter and experiment with the keypoint file. 
+                            \n If you choose to run the code, start by installing the packages json and numpy. The complete overview of the keypoint indices can be seen in the tab 'General information'. """)
+                gr.Code(
+                    value="""
                     
-                    gr.Markdown("Download the .json file that contains the keypoint positions for each frame in the video.")
-                    jsonoutput = gr.File(file_types=[".json"])
-                    gr.Markdown("""There are multiple ways to interact with these keypoints. 
-                                \n The example below shows how you can calulate the angle on the elbow for example.
-                                \n Copy the code into your own preferred interpreter and experiment with the keypoint file. 
-                                \n If you choose to run the code, start by installing the packages json and numpy. The complete overview of the keypoint indices can be seen in the tab 'General information'. """)
-                    gr.Code(
-                        value="""
-                        
 # Importing packages needed  
 import json
 import numpy as np
 
 # First we load the data
 with open(file_path, 'r') as json_file:
-    data = json.load(json_file)
+data = json.load(json_file)
 
 # The we define a function for calculating angles    
 def calculate_angle(a, b, c):
-    a = np.array(a) # First point
-    b = np.array(b) # Middle point
-    c = np.array(c) # End point
-    
-    radians = np.arctan2(c[1]-b[1], c[0]-b[0]) - np.arctan2(a[1]-b[1], a[0]-b[0])
-    angle = np.abs(radians*180.0/np.pi)
+a = np.array(a) # First point
+b = np.array(b) # Middle point
+c = np.array(c) # End point
+
+radians = np.arctan2(c[1]-b[1], c[0]-b[0]) - np.arctan2(a[1]-b[1], a[0]-b[0])
+angle = np.abs(radians*180.0/np.pi)
+
+if angle >180.0:
+    angle = 360-angle
     
-    if angle >180.0:
-        angle = 360-angle
-        
-    return angle 
+return angle 
 
 
 # COCO keypoint indices
@@ -266,78 +263,76 @@ wrist_point = data[0]['instances'][0]['keypoints'][wrist_index]
 angle = calculate_angle(shoulder_point, elbow_point, wrist_point)
 print("Angle is: ", angle)    
 
-                        """,
-                        language="python",
-                        interactive=False,
-                        show_label=False,
-                    )
-
-
-
-            with gr.Tab("General information"):
-                gr.Markdown(""" 
-                            \n # Information about the models 
-
-                            \n ## Pose models: 
-                            
-                            \n All the pose estimation models come from the library [MMpose](https://github.com/open-mmlab/mmpose). It is a library for human pose estimation that provides pre-trained models for 2D and 3D pose estimation. 
-
-                            \n The 2D pose model is used for estimating the 2D coordinates of human body joints from an image or a video frame. The model uses a convolutional neural network (CNN) to predict the joint locations and their confidence scores. 
-                            
-                            \n The 2D hand model is a specialized version of the 2D pose model that is designed for hand pose estimation. It uses a similar CNN architecture to the 2D pose model but is trained specifically for detecting the joints in the hand. 
-                            
-                            \n The 3D pose model is used for estimating the 3D coordinates of human body joints from an image or a video frame. The model uses a combination of 2D pose estimation and depth estimation to infer the 3D joint locations. 
-                            
-                            \n The keypoints in the 2D pose model has the following order:
-
-                            \n ``` 
-                            0: Nose
-                            1: Left Eye
-                            2: Right Eye
-                            3: Left Ear
-                            4: Right Ear
-                            5: Left Shoulder
-                            6: Right Shoulder
-                            7: Left Elbow
-                            8: Right Elbow
-                            9: Left Wrist
-                            10: Right Wrist
-                            11: Left Hip
-                            12: Right Hip
-                            13: Left Knee
-                            14: Right Knee
-                            15: Left Ankle
-                            16: Right Ankle 
-                            ```
-
-                            \n Below, you can see a visualization of the poses of the 2d, 3d and hand keypoint locations: """)
-                gr.Image("./cocoposes.png", width="200")
-                gr.Image("./cocohand.png", width="200")
-                            
+                    """,
+                    language="python",
+                    interactive=False,
+                    show_label=False,
+                )
 
-                
 
-            # From file
-            submit_pose_file.click(fn=pose2d, 
-                                inputs=  [video_input, file_kpthr], 
-                                outputs = [video_output1, jsonoutput],
-                                queue=True)
-            
-            submit_pose3d_file.click(fn=pose3dbatch, 
-                                    inputs= [video_input, file_kpthr], 
-                                    outputs = video_output2,#[video_output2, jsonoutput],
-                                    batch=True, 
-                                    max_batch_size=16,
-                                    queue=True) # Sometimes it worked with queue false? But still slow
+
+        with gr.Tab("General information"):
+            gr.Markdown(""" 
+                        \n # Information about the models 
+
+                        \n ## Pose models: 
+                        
+                        \n All the pose estimation models come from the library [MMpose](https://github.com/open-mmlab/mmpose). It is a library for human pose estimation that provides pre-trained models for 2D and 3D pose estimation. 
+
+                        \n The 2D pose model is used for estimating the 2D coordinates of human body joints from an image or a video frame. The model uses a convolutional neural network (CNN) to predict the joint locations and their confidence scores. 
+                        
+                        \n The 2D hand model is a specialized version of the 2D pose model that is designed for hand pose estimation. It uses a similar CNN architecture to the 2D pose model but is trained specifically for detecting the joints in the hand. 
+                        
+                        \n The 3D pose model is used for estimating the 3D coordinates of human body joints from an image or a video frame. The model uses a combination of 2D pose estimation and depth estimation to infer the 3D joint locations. 
+                        
+                        \n The keypoints in the 2D pose model has the following order:
+
+                        \n ``` 
+                        0: Nose
+                        1: Left Eye
+                        2: Right Eye
+                        3: Left Ear
+                        4: Right Ear
+                        5: Left Shoulder
+                        6: Right Shoulder
+                        7: Left Elbow
+                        8: Right Elbow
+                        9: Left Wrist
+                        10: Right Wrist
+                        11: Left Hip
+                        12: Right Hip
+                        13: Left Knee
+                        14: Right Knee
+                        15: Left Ankle
+                        16: Right Ankle 
+                        ```
+
+                        \n Below, you can see a visualization of the poses of the 2d, 3d and hand keypoint locations: """)
+            gr.Image("./cocoposes.png", width="200")
+            gr.Image("./cocohand.png", width="200")
+                        
+
             
-            submit_hand_file.click(fn=pose2dhand, 
-                                inputs= [video_input, file_kpthr], 
-                                outputs = [video_output3, jsonoutput],
-                                queue=True)
-        return block
+
+        # From file
+        submit_pose_file.click(fn=pose2d, 
+                            inputs=  [video_input, file_kpthr], 
+                            outputs = [video_output1, jsonoutput],
+                            queue=True)
+        
+        submit_pose3d_file.click(fn=pose3dbatch, 
+                                inputs= [video_input, file_kpthr, human3d], 
+                                outputs = video_output2,#[video_output2, jsonoutput],
+                                batch=True, 
+                                max_batch_size=16,
+                                queue=True) # Sometimes it worked with queue false? But still slow
+        
+        submit_hand_file.click(fn=pose2dhand, 
+                            inputs= [video_input, file_kpthr], 
+                            outputs = [video_output3, jsonoutput],
+                            queue=True)
         
 if __name__ == "__main__":
-    block = UI()
     block.queue(max_size=60, 
                 concurrency_count=40, # When you increase the concurrency_count parameter in queue(), max_threads() in launch() is automatically increased as well.
                 #max_size=25, # Maximum number of requests that the queue processes