Add demo for spoof detection

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

app.py

@@ -0,0 +1,30 @@
+import cv2
+import gradio as gr
+
+from spoofynet import SpoofyNet
+
+spoofynet = SpoofyNet()
+
+
+def find_spoofs(input_img):
+    spoofs = spoofynet.find_spoof(input_img)
+    for spoof in spoofs:
+        (startX, startY, endX, endY) = spoof["coords"]
+        label = "Real" if spoof["is_real"] else "Spoofed"
+        color = (0, 255, 0) if spoof["is_real"] else (0, 0, 255)
+        cv2.putText(
+            input_img,
+            f"{label}: {spoof['probs']:.2f}",
+            (startX, startY - 10),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.5,
+            color,
+            2,
+        )
+
+        cv2.rectangle(input_img, (startX, startY), (endX, endY), color, 4)
+    return input_img
+
+
+demo = gr.Interface(find_spoofs, gr.Image(), "image")
+demo.launch()

auto_rotate.py

@@ -0,0 +1,75 @@
+import math
+
+import cv2
+import numpy as np
+import onnxruntime
+from PIL import Image
+
+session = onnxruntime.InferenceSession("models/slim-facelandmark.onnx")
+
+
+def EuclideanDistance(source_representation, test_representation):
+    euclidean_distance = source_representation - test_representation
+    euclidean_distance = np.sum(np.multiply(euclidean_distance, euclidean_distance))
+    euclidean_distance = np.sqrt(euclidean_distance)
+    return euclidean_distance
+
+
+def alignment_procedure(img, left_eye, right_eye):
+    left_eye_x, left_eye_y = left_eye
+    right_eye_x, right_eye_y = right_eye
+
+    if left_eye_y > right_eye_y:
+        point_3rd = (right_eye_x, left_eye_y)
+        direction = -1  # rotate same direction to clock
+    else:
+        point_3rd = (left_eye_x, right_eye_y)
+        direction = 1  # rotate inverse direction of clock
+
+    a = EuclideanDistance(np.array(left_eye), np.array(point_3rd))
+    b = EuclideanDistance(np.array(right_eye), np.array(point_3rd))
+    c = EuclideanDistance(np.array(right_eye), np.array(left_eye))
+
+    if (
+        b != 0 and c != 0
+    ):  # this multiplication causes division by zero in cos_a calculation
+
+        cos_a = (b * b + c * c - a * a) / (2 * b * c)
+        angle = np.arccos(cos_a)  # angle in radian
+        angle = (angle * 180) / math.pi  # radian to degree
+
+        # -----------------------
+        # rotate base image
+
+        if direction == -1:
+            angle = 90 - angle
+
+        img = Image.fromarray(img)
+        img = np.array(img.rotate(direction * angle))
+
+    # -----------------------
+
+    return img  # return img anyway
+
+
+def align_face(image):
+    inputs = cv2.resize(image, (112, 112))
+    inputs = cv2.cvtColor(inputs, cv2.COLOR_BGR2RGB)
+    inputs = inputs.transpose(2, 0, 1).astype(np.float32)
+    inputs = inputs / 255.0
+    inputs = np.expand_dims(inputs, axis=0)
+    landmarks = session.run(None, {"input": inputs})[0]
+    pre_landmark = landmarks[0]
+    pre_landmark = pre_landmark.reshape(-1, 2)
+    left_eyex, left_eyey = pre_landmark[96]
+    right_eyex, right_eyey = pre_landmark[97]
+
+    img = alignment_procedure(image, (left_eyex, left_eyey), (right_eyex, right_eyey))
+    return img
+
+
+if __name__ == "__main__":
+    img = cv2.imread("obamaface.jpg")
+    img = align_face(img)
+    cv2.imshow("img", img)
+    cv2.waitKey(0)

box_utils_numpy.py

@@ -0,0 +1,74 @@
+# MIT LICENSE Copyright (c) 2019 linzai
+# Originally comes from https://github.com/Linzaer/Ultra-Light-Fast-Generic-Face-Detector-1MB
+
+import numpy as np
+
+
+def area_of(left_top, right_bottom):
+    """Compute the areas of rectangles given two corners.
+
+    Args:
+        left_top (N, 2): left top corner.
+        right_bottom (N, 2): right bottom corner.
+
+    Returns:
+        area (N): return the area.
+    """
+    hw = np.clip(right_bottom - left_top, 0.0, None)
+    return hw[..., 0] * hw[..., 1]
+
+
+def iou_of(boxes0, boxes1, eps=1e-5):
+    """Return intersection-over-union (Jaccard index) of boxes.
+
+    Args:
+        boxes0 (N, 4): ground truth boxes.
+        boxes1 (N or 1, 4): predicted boxes.
+        eps: a small number to avoid 0 as denominator.
+    Returns:
+        iou (N): IoU values.
+    """
+    overlap_left_top = np.maximum(boxes0[..., :2], boxes1[..., :2])
+    overlap_right_bottom = np.minimum(boxes0[..., 2:], boxes1[..., 2:])
+
+    overlap_area = area_of(overlap_left_top, overlap_right_bottom)
+    area0 = area_of(boxes0[..., :2], boxes0[..., 2:])
+    area1 = area_of(boxes1[..., :2], boxes1[..., 2:])
+    return overlap_area / (area0 + area1 - overlap_area + eps)
+
+
+def hard_nms(box_scores, iou_threshold, top_k=-1, candidate_size=200):
+    """
+
+    Args:
+        box_scores (N, 5): boxes in corner-form and probabilities.
+        iou_threshold: intersection over union threshold.
+        top_k: keep top_k results. If k <= 0, keep all the results.
+        candidate_size: only consider the candidates with the highest scores.
+    Returns:
+         picked: a list of indexes of the kept boxes
+    """
+    scores = box_scores[:, -1]
+    boxes = box_scores[:, :-1]
+    picked = []
+    # _, indexes = scores.sort(descending=True)
+    indexes = np.argsort(scores)
+    # indexes = indexes[:candidate_size]
+    indexes = indexes[-candidate_size:]
+    while len(indexes) > 0:
+        # current = indexes[0]
+        current = indexes[-1]
+        picked.append(current)
+        if 0 < top_k == len(picked) or len(indexes) == 1:
+            break
+        current_box = boxes[current, :]
+        # indexes = indexes[1:]
+        indexes = indexes[:-1]
+        rest_boxes = boxes[indexes, :]
+        iou = iou_of(
+            rest_boxes,
+            np.expand_dims(current_box, axis=0),
+        )
+        indexes = indexes[iou <= iou_threshold]
+
+    return box_scores[picked, :]

models/slim-facedetect.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd7b4b72b9572f51df26812112ec1b38cb3ff0ef1caad9977ce5e2bf950efbd9
+size 419735

models/slim-facelandmark.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70a984e0fff8c8bc8acbbfa8671b6159e9fdb7e141aa47d3441a2f777459970e
+size 427876

models/spoof.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba5b74238233864eab57be35223076f03b6ea2f6231b50516c4531d2e900e1e6
+size 16808373

requirements.txt

@@ -0,0 +1,4 @@
+Pillow
+numpy
+opencv-python-headless
+onnxruntime

spoofynet.py

@@ -0,0 +1,134 @@
+import cv2
+import numpy as np
+from onnxruntime import InferenceSession
+
+import box_utils_numpy
+from auto_rotate import align_face
+
+
+def softmax(x):
+    e_x = np.exp(x - np.max(x))
+    return e_x / e_x.sum(axis=0)
+
+
+def crop_square(img, size, interpolation=cv2.INTER_AREA):
+    h, w = img.shape[:2]
+    min_size = np.amin([h, w])
+
+    # Centralize and crop
+    crop_img = img[
+        int(h / 2 - min_size / 2) : int(h / 2 + min_size / 2),
+        int(w / 2 - min_size / 2) : int(w / 2 + min_size / 2),
+    ]
+    resized = cv2.resize(crop_img, (size, size), interpolation=interpolation)
+
+    return resized
+
+
+class SpoofyNet:
+    def __init__(self):
+        self.face_model = InferenceSession("models/slim-facedetect.onnx")
+        self.face_inputname = self.face_model.get_inputs()[0].name
+        self.classifier = InferenceSession("models/spoof.onnx")
+
+    def find_boxes(
+        self,
+        width,
+        height,
+        confidences,
+        boxes,
+        prob_threshold,
+        iou_threshold=0.3,
+        top_k=-1,
+    ):
+        boxes = boxes[0]
+        confidences = confidences[0]
+        picked_box_probs = []
+        picked_labels = []
+        for class_index in range(1, confidences.shape[1]):
+            probs = confidences[:, class_index]
+            mask = probs > prob_threshold
+            probs = probs[mask]
+            if probs.shape[0] == 0:
+                continue
+            subset_boxes = boxes[mask, :]
+            box_probs = np.concatenate([subset_boxes, probs.reshape(-1, 1)], axis=1)
+            box_probs = box_utils_numpy.hard_nms(
+                box_probs,
+                iou_threshold=iou_threshold,
+                top_k=top_k,
+            )
+            picked_box_probs.append(box_probs)
+            picked_labels.extend([class_index] * box_probs.shape[0])
+        if not picked_box_probs:
+            return np.array([]), np.array([]), np.array([])
+        picked_box_probs = np.concatenate(picked_box_probs)
+        picked_box_probs[:, 0] *= width
+        picked_box_probs[:, 1] *= height
+        picked_box_probs[:, 2] *= width
+        picked_box_probs[:, 3] *= height
+        return (
+            picked_box_probs[:, :4].astype(np.int32),
+            np.array(picked_labels),
+            picked_box_probs[:, 4],
+        )
+
+    def tta(self, src):
+        horizontal_rot = cv2.rotate(src, cv2.ROTATE_180)
+        grayscale = cv2.cvtColor(src, cv2.COLOR_RGB2GRAY)
+        grayscale = cv2.cvtColor(grayscale, cv2.COLOR_GRAY2RGB)
+        return [src, horizontal_rot, grayscale]
+
+    def find_spoof(self, img):
+        ret = []
+        threshold = 0.6
+        image_mean = np.array([127, 127, 127])
+
+        image = cv2.resize(img, (320, 240))
+        image = (image - image_mean) / 128
+        image = np.transpose(image, [2, 0, 1])
+        image = np.expand_dims(image, axis=0)
+        image = image.astype(np.float32)
+
+        confidences, boxes = self.face_model.run(None, {self.face_inputname: image})
+        boxes, _, _ = self.find_boxes(
+            img.shape[1], img.shape[0], confidences, boxes, threshold
+        )
+
+        classify_mean, classify_std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
+        for i in range(boxes.shape[0]):
+            (startX, startY, endX, endY) = boxes[i, :]
+
+            face = img[startY:endY, startX:endX]
+            if face.size == 0:
+                continue
+            face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)
+
+            # Preprocess
+            face = align_face(face)
+            face = crop_square(face, 256)
+
+            probs_all = []
+            for face in self.tta(face):
+                # Normalize
+                face = face / 255.0
+                face = (face - classify_mean) / classify_std
+                face = np.transpose(face, [2, 0, 1])
+                face = np.expand_dims(face, axis=0)
+                face = face.astype(np.float32)
+
+                predicted = self.classifier.run(None, {"input": face})
+                predicted_id = np.argmax(predicted)
+                probs = softmax(predicted[0][0])
+                probs_all.append(probs)
+
+            final_probs = np.mean(probs_all, axis=0)
+            predicted_id = np.argmax(final_probs)
+            ret.append(
+                {
+                    "coords": (startX, startY, endX, endY),
+                    "is_real": bool(predicted_id),
+                    "probs": final_probs[predicted_id],
+                }
+            )
+        return ret