Added facial expression recognition to opencv spaces

README.md

@@ -7,6 +7,13 @@ sdk: gradio
 sdk_version: 5.34.2
 app_file: app.py
 pinned: false
+short_description: Facial expression recognition using OpenCV
+tags:
+  - opencv
+  - face-detection
+  - facial-expression
+  - onnx
+  - gradio
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,64 @@
+import cv2 as cv
+import numpy as np
+import gradio as gr
+import datetime
+from huggingface_hub import hf_hub_download
+
+from facial_fer_model import FacialExpressionRecog
+from yunet import YuNet
+
+# Download ONNX model from Hugging Face
+FD_MODEL_PATH = hf_hub_download(repo_id="opencv/face_detection_yunet", filename="face_detection_yunet_2023mar.onnx")
+FER_MODEL_PATH = hf_hub_download(repo_id="opencv/facial_expression_recognition", filename="facial_expression_recognition_mobilefacenet_2022july.onnx")
+
+backend_id = cv.dnn.DNN_BACKEND_OPENCV
+target_id = cv.dnn.DNN_TARGET_CPU
+
+fer_model = FacialExpressionRecog(modelPath=FER_MODEL_PATH, backendId=backend_id, targetId=target_id)
+detect_model = YuNet(modelPath=FD_MODEL_PATH)
+
+def visualize(image, det_res, fer_res):
+    output = image.copy()
+    landmark_color = [(255, 0, 0), (0, 0, 255), (0, 255, 0), (255, 0, 255), (0, 255, 255)]
+
+    for det, fer_type in zip(det_res, fer_res):
+        bbox = det[0:4].astype(np.int32)
+        fer_type_str = FacialExpressionRecog.getDesc(fer_type)
+        cv.rectangle(output, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), (0, 255, 0), 2)
+        cv.putText(output, fer_type_str, (bbox[0], bbox[1] - 10), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)
+
+        landmarks = det[4:14].astype(np.int32).reshape((5, 2))
+        for idx, landmark in enumerate(landmarks):
+            cv.circle(output, landmark, 2, landmark_color[idx], 2)
+
+    return output
+
+def detect_expression(input_image):
+    image = cv.cvtColor(input_image, cv.COLOR_RGB2BGR)
+    h, w, _ = image.shape
+    detect_model.setInputSize([w, h])
+
+    dets = detect_model.infer(image)
+    if dets is None:
+        return cv.cvtColor(image, cv.COLOR_BGR2RGB)
+
+    fer_res = []
+    for face_points in dets:
+        result = fer_model.infer(image, face_points[:-1])
+        fer_res.append(result[0])
+
+    output = visualize(image, dets, fer_res)
+    return cv.cvtColor(output, cv.COLOR_BGR2RGB)
+
+# Gradio UI
+demo = gr.Interface(
+    fn=detect_expression,
+    inputs=gr.Image(type="numpy", label="Upload Image"),
+    outputs=gr.Image(type="numpy", label="Facial Expression Result"),
+    title="Facial Expression Recognition (FER) with OpenCV DNN",
+    description="Detects faces and recognizes facial expressions using YuNet + MobileFaceNet ONNX models.",
+    allow_flagging="never"
+)
+
+if __name__ == "__main__":
+    demo.launch()

facial_fer_model.py

@@ -0,0 +1,176 @@
+# This file is part of OpenCV Zoo project.
+# It is subject to the license terms in the LICENSE file found in the same directory.
+#
+# Copyright (C) 2022, Shenzhen Institute of Artificial Intelligence and Robotics for Society, all rights reserved.
+# Third party copyrights are property of their respective owners.
+
+import numpy as np
+import cv2 as cv
+
+class FacialExpressionRecog:
+    def __init__(self, modelPath, backendId=0, targetId=0):
+        self._modelPath = modelPath
+        self._backendId = backendId
+        self._targetId = targetId
+
+        self._model = cv.dnn.readNet(self._modelPath)
+        self._model.setPreferableBackend(self._backendId)
+        self._model.setPreferableTarget(self._targetId)
+
+        self._align_model = FaceAlignment()
+
+        self._inputNames = 'data'
+        self._outputNames = ['label']
+        self._inputSize = [112, 112]
+        self._mean = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
+        self._std = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    def setBackendAndTarget(self, backendId, targetId):
+        self._backendId = backendId
+        self._targetId = targetId
+        self._model.setPreferableBackend(self._backendId)
+        self._model.setPreferableTarget(self._targetId)
+
+    def _preprocess(self, image, bbox):
+        if bbox is not None:
+            image = self._align_model.get_align_image(image, bbox[4:].reshape(-1, 2))
+        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
+        image = image.astype(np.float32, copy=False) / 255.0
+        image -= self._mean
+        image /= self._std
+        return cv.dnn.blobFromImage(image)
+
+    def infer(self, image, bbox=None):
+        # Preprocess
+        inputBlob = self._preprocess(image, bbox)
+
+        # Forward
+        self._model.setInput(inputBlob, self._inputNames)
+        outputBlob = self._model.forward(self._outputNames)
+
+        # Postprocess
+        results = self._postprocess(outputBlob)
+
+        return results
+
+    def _postprocess(self, outputBlob):
+        result = np.argmax(outputBlob[0], axis=1).astype(np.uint8)
+        return result
+
+    @staticmethod
+    def getDesc(ind):
+        _expression_enum = ["angry", "disgust", "fearful", "happy", "neutral", "sad", "surprised"]
+        return _expression_enum[ind]
+
+
+class FaceAlignment():
+    def __init__(self, reflective=False):
+        self._std_points = np.array([[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], [41.5493, 92.3655], [70.7299, 92.2041]])
+        self.reflective = reflective
+
+    def __tformfwd(self, trans, uv):
+        uv = np.hstack((uv, np.ones((uv.shape[0], 1))))
+        xy = np.dot(uv, trans)
+        xy = xy[:, 0:-1]
+        return xy
+
+    def __tforminv(self, trans, uv):
+        Tinv = np.linalg.inv(trans)
+        xy = self.__tformfwd(Tinv, uv)
+        return xy
+
+    def __findNonreflectiveSimilarity(self, uv, xy, options=None):
+        options = {"K": 2}
+
+        K = options["K"]
+        M = xy.shape[0]
+        x = xy[:, 0].reshape((-1, 1))  # use reshape to keep a column vector
+        y = xy[:, 1].reshape((-1, 1))  # use reshape to keep a column vector
+        # print '--->x, y:\n', x, y
+
+        tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1))))
+        tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1))))
+        X = np.vstack((tmp1, tmp2))
+        # print '--->X.shape: ', X.shape
+        # print 'X:\n', X
+
+        u = uv[:, 0].reshape((-1, 1))  # use reshape to keep a column vector
+        v = uv[:, 1].reshape((-1, 1))  # use reshape to keep a column vector
+        U = np.vstack((u, v))
+        # print '--->U.shape: ', U.shape
+        # print 'U:\n', U
+
+        # We know that X * r = U
+        if np.linalg.matrix_rank(X) >= 2 * K:
+            r, _, _, _ = np.linalg.lstsq(X, U, rcond=-1)
+            # print(r, X, U, sep="\n")
+            r = np.squeeze(r)
+        else:
+            raise Exception("cp2tform:twoUniquePointsReq")
+
+        sc = r[0]
+        ss = r[1]
+        tx = r[2]
+        ty = r[3]
+
+        Tinv = np.array([[sc, -ss, 0], [ss, sc, 0], [tx, ty, 1]])
+        T = np.linalg.inv(Tinv)
+        T[:, 2] = np.array([0, 0, 1])
+
+        return T, Tinv
+
+    def __findSimilarity(self, uv, xy, options=None):
+        options = {"K": 2}
+
+        #    uv = np.array(uv)
+        #    xy = np.array(xy)
+
+        # Solve for trans1
+        trans1, trans1_inv = self.__findNonreflectiveSimilarity(uv, xy, options)
+
+        # manually reflect the xy data across the Y-axis
+        xyR = xy
+        xyR[:, 0] = -1 * xyR[:, 0]
+        # Solve for trans2
+        trans2r, trans2r_inv = self.__findNonreflectiveSimilarity(uv, xyR, options)
+
+        # manually reflect the tform to undo the reflection done on xyR
+        TreflectY = np.array([[-1, 0, 0], [0, 1, 0], [0, 0, 1]])
+        trans2 = np.dot(trans2r, TreflectY)
+
+        # Figure out if trans1 or trans2 is better
+        xy1 = self.__tformfwd(trans1, uv)
+        norm1 = np.linalg.norm(xy1 - xy)
+        xy2 = self.__tformfwd(trans2, uv)
+        norm2 = np.linalg.norm(xy2 - xy)
+
+        if norm1 <= norm2:
+            return trans1, trans1_inv
+        else:
+            trans2_inv = np.linalg.inv(trans2)
+            return trans2, trans2_inv
+
+    def __get_similarity_transform(self, src_pts, dst_pts):
+        if self.reflective:
+            trans, trans_inv = self.__findSimilarity(src_pts, dst_pts)
+        else:
+            trans, trans_inv = self.__findNonreflectiveSimilarity(src_pts, dst_pts)
+        return trans, trans_inv
+
+    def __cvt_tform_mat_for_cv2(self, trans):
+        cv2_trans = trans[:, 0:2].T
+        return cv2_trans
+
+    def get_similarity_transform_for_cv2(self, src_pts, dst_pts):
+        trans, trans_inv = self.__get_similarity_transform(src_pts, dst_pts)
+        cv2_trans = self.__cvt_tform_mat_for_cv2(trans)
+        return cv2_trans, trans
+
+    def get_align_image(self, image, lm5_points):
+        assert lm5_points is not None
+        tfm, trans = self.get_similarity_transform_for_cv2(lm5_points, self._std_points)
+        return cv.warpAffine(image, tfm, (112, 112))

requirements.txt

@@ -0,0 +1,4 @@
+opencv-python
+gradio
+numpy
+huggingface_hub

yunet.py

@@ -0,0 +1,55 @@
+# This file is part of OpenCV Zoo project.
+# It is subject to the license terms in the LICENSE file found in the same directory.
+#
+# Copyright (C) 2021, Shenzhen Institute of Artificial Intelligence and Robotics for Society, all rights reserved.
+# Third party copyrights are property of their respective owners.
+
+from itertools import product
+
+import numpy as np
+import cv2 as cv
+
+class YuNet:
+    def __init__(self, modelPath, inputSize=[320, 320], confThreshold=0.6, nmsThreshold=0.3, topK=5000, backendId=0, targetId=0):
+        self._modelPath = modelPath
+        self._inputSize = tuple(inputSize) # [w, h]
+        self._confThreshold = confThreshold
+        self._nmsThreshold = nmsThreshold
+        self._topK = topK
+        self._backendId = backendId
+        self._targetId = targetId
+
+        self._model = cv.FaceDetectorYN.create(
+            model=self._modelPath,
+            config="",
+            input_size=self._inputSize,
+            score_threshold=self._confThreshold,
+            nms_threshold=self._nmsThreshold,
+            top_k=self._topK,
+            backend_id=self._backendId,
+            target_id=self._targetId)
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    def setBackendAndTarget(self, backendId, targetId):
+        self._backendId = backendId
+        self._targetId = targetId
+        self._model = cv.FaceDetectorYN.create(
+            model=self._modelPath,
+            config="",
+            input_size=self._inputSize,
+            score_threshold=self._confThreshold,
+            nms_threshold=self._nmsThreshold,
+            top_k=self._topK,
+            backend_id=self._backendId,
+            target_id=self._targetId)
+
+    def setInputSize(self, input_size):
+        self._model.setInputSize(tuple(input_size))
+
+    def infer(self, image):
+        # Forward
+        faces = self._model.detect(image)
+        return np.empty(shape=(0, 5)) if faces[1] is None else faces[1]