Update Application File

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/air_draw.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.9 (rtgesrec) (2)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,12 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="N813" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (rtgesrec) (2)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/air_draw.iml" filepath="$PROJECT_DIR$/.idea/air_draw.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

app.py

@@ -1,7 +1,180 @@
+import cv2
+import numpy as np
+from PIL import Image
+import mediapipe as mp
+import time
 import gradio as gr
 
-def greet(name):
-    return "Hello " + name + "!!"
+DOMINANT_HAND = "Right"
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()
+# width, height = 1280, 720
+width_, height_, = 256, 144
+
+drawing_flag = False
+sleepy_time = time.time()
+
+output_frames = []
+
+
+def find_hands(brain, img):
+    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # opencv image is in BGR form but mp is trained with RGB
+    results = brain.process(
+        img_rgb)  # process finds the hands and outputs classification and 21 landmarks for each hand
+    all_hands = []  # initializing array to hold the dictionary for the hands
+    h, w, _ = img.shape  # get height and width of image for scaling
+    if results.multi_hand_landmarks:
+        for hand_type, hand_lms in zip(results.multi_handedness,
+                                       results.multi_hand_landmarks):  # elegant solution for mp list object traversal
+            hand = {}  # initializing dict for each hand
+            lm_list = []  # landmarks array for all 21 point of the hand
+            for lm in hand_lms.landmark:
+                px, py, pz = int(lm.x * w), int(lm.y * h), int(
+                    lm.z * w)  # scaling landmark points to image size for frame coordinates
+                lm_list.append([px, py, pz])
+
+            hand["lm_list"] = lm_list  # add "lm_list" key for all landmark points of the hand
+            hand["type"] = hand_type.classification[0].label  # adds the label (left/right) for the hand
+            all_hands.append(hand)  # appends the dict
+    return all_hands
+
+
+def is_drawing(index, thumb):  # proximity function with arbitrary threshold
+    npindex = np.array((index[0], index[1]))
+    npthumb = np.array((thumb[0], thumb[1]))
+    if np.linalg.norm(npindex - npthumb) < 30:
+        return True
+    else:
+        return False
+
+
+def save(landmarks):  # brute force finger orientation checking
+    if landmarks[8][1] < landmarks[6][1]:
+        if landmarks[12][1] < landmarks[10][1]:
+            if landmarks[16][1] < landmarks[14][1]:
+                if landmarks[20][1] < landmarks[18][1]:
+                    return True
+    else:
+        return False
+
+
+def clear(landmarks):  # brute force finger orientation checking
+    if landmarks[4][1] < landmarks[3][1] < landmarks[2][1] < landmarks[8][1]:
+        return True
+    else:
+        return False
+
+
+def show(video): # main
+    cam = cv2.VideoCapture(video)  # get the video file from path
+    width = cam.get(cv2.CAP_PROP_FRAME_WIDTH)
+    height = cam.get(cv2.CAP_PROP_FRAME_HEIGHT)
+
+    detector = mp.solutions.hands.Hands(min_detection_confidence=0.8)  # initialize detector
+    # paper = np.zeros((width, height, 4), np.uint8)
+    paper = np.zeros((int(height), int(width), 3), dtype=np.uint8)  # create blank page
+    paper.fill(255)
+
+    past_holder = ()  # hold previous index coordinates
+    palette = cv2.imread('palette.jpg')
+
+    page_num = 0  # iterating for saving (not a viable function for gradio)
+
+    color = (0, 0, 0)
+
+    global sleepy_time  # get sleep time for multiple gestures
+
+    # runny = 1
+    while cam.isOpened():
+        # runny -= 1
+        x, rgb_image = cam.read()
+        rgb_image_f = cv2.flip(np.asanyarray(rgb_image), 1)  # mirrored video
+
+        hands = find_hands(detector, rgb_image_f)
+
+        if x:  # return flag for cv2
+            try:  # for error handling
+                if hands:
+                    hand1 = hands[0] if hands[0]["type"] == DOMINANT_HAND else hands[1]
+                    lm_list1 = hand1["lm_list"]  # List of 21 Landmarks
+                    handedness = hand1["type"]
+
+                    if handedness == DOMINANT_HAND:
+                        idx_coords = lm_list1[8][0], lm_list1[8][1]  # 0 is width (bigger)
+                        # print(idx_coords)
+                        cv2.circle(rgb_image_f, idx_coords, 5, color, cv2.FILLED)
+
+                        if idx_coords[1] < 72: # brute force but should be extremely marginally faster lol
+                            if idx_coords[0] < 142:  # red
+                                color = (0, 0, 255)
+                            if 142 < idx_coords[0] < 285:  # orange
+                                color = (0, 115, 255)
+                            if 285 < idx_coords[0] < 426:  # yellow
+                                color = (0, 229, 255)
+                            if 426 < idx_coords[0] < 569:  # green
+                                color = (0, 195, 88)
+                            if 569 < idx_coords[0] < 711:  # blue
+                                color = (195, 85, 0)
+                            if 711 < idx_coords[0] < 853:  # indigo
+                                color = (195, 0, 68)
+                            if 853 < idx_coords[0] < 996:  # violet
+                                color = (195, 0, 143)
+                            if 996 < idx_coords[0] < 1137:  # black
+                                color = (0, 0, 0)
+                            if 1137 < idx_coords[0]:  # white / eraser
+                                color = (255, 255, 255)
+
+                        if len(past_holder) and drawing_flag: # start drawing
+                            cv2.line(paper, past_holder, idx_coords, color, 5)
+                            cv2.line(rgb_image_f, past_holder, idx_coords, color, 5)
+                            # paper[idx_coords[0]][idx_coords[1]][0] = 255
+                            # paper[idx_coords[0]][idx_coords[1]][3] = 255
+                            cv2.circle(rgb_image_f, idx_coords, 5, color, cv2.FILLED)
+
+                        if save(lm_list1) and time.time() - sleepy_time > 3: # save / output
+                            paper[0:height_, w - width_: w] = 255
+                            paper = cv2.cvtColor(paper, cv2.COLOR_BGR2RGB)
+                            im = Image.fromarray(paper)
+                            im.save("paper%s.png" % page_num)
+                            print("saved")
+                            sleepy_time = time.time()
+                            paper = cv2.cvtColor(paper, cv2.COLOR_RGB2BGR)
+                            page_num += 1
+                            return paper
+
+                        if clear(lm_list1) and time.time() - sleepy_time > 3: # reset paper
+                            paper = np.zeros((height, width, 3), dtype=np.uint8)
+                            paper.fill(255)
+                            print("page cleared")
+                            sleepy_time = time.time()
+
+                        past_holder = idx_coords
+
+                if is_drawing(idx_coords, lm_list1[4]):  # 4 is thumb
+                    drawing_flag = True
+                else:
+                    drawing_flag = False
+
+            except:
+                pass
+
+            finally:
+                rgb_image_f[0:72, ] = palette
+                presenter = cv2.resize(rgb_image_f, (width_, height_))
+                h, w, _ = rgb_image_f.shape
+                paper[0:height_, w - width_: w] = presenter
+
+                # output_frames.append(paper)
+
+                # cv2.imshow("Image", rgb_image_f)
+                # cv2.imshow("paper", paper)
+                # key = cv2.waitKey(1)
+                # if key & 0xFF == ord('q') or key == 27:  # Press esc or 'q' to close the image window
+                #     break
+
+        else:
+            break
+
+
+iface = gr.Interface(fn=show, inputs=gr.inputs.Video(source="webcam", type="mp4"), outputs='image')
+
+iface.launch(share=True)