Merge pull request #4 from dimitribarbot/add-retina-face-detector

Add retina face detector

Author: dimitribarbot

README.md

@@ -48,7 +48,7 @@ Pickle files have all been converted to safetensors by Kijai. If necessary, they
 ### Face detectors
 For human mode, you can either use the original default [Insightface](https://github.com/deepinsight/insightface), or [Google's MediaPipe](https://github.com/google-ai-edge/mediapipe), or [Face Alignment](https://github.com/1adrianb/face-alignment) (see [Settings](#settings) section above or [API](#api) section below). 
 
-Biggest difference is the license: Insightface is strictly for NON-COMMERCIAL use. MediaPipe is a bit worse at detection, and can't run on GPU in Windows, though it's much faster on CPU compared to Insightface. Face Alignment can use blazeface back camera model (or SFD), it's far better for smaller faces than MediaPipe, that only can use the blazeface short model. The warmup on the first run when using this can take a long time, but subsequent runs are quick.
+Biggest difference is the license: Insightface is strictly for NON-COMMERCIAL use. MediaPipe is a bit worse at detection, and can't run on GPU in Windows, though it's much faster on CPU compared to Insightface. Face Alignment can use blazeface back camera model (or SFD or RetinaFace), it's far better for smaller faces than MediaPipe, that only can use the blazeface short model. The warmup on the first run when using this can take a long time, but subsequent runs are quick.
 
 Insightface models go here (automatically downloaded if the folder is not present during first run): `stable-diffusion-webui/models/insightface/models/buffalo_l`. If necessary, they can be downloaded from: https://github.com/deepinsight/insightface/releases/download/v0.7/buffalo_l.zip.
 
@@ -73,7 +73,7 @@ Parameters are the same as LivePortrait ones (see output of command `python infe
 - `save_output`: `true` if you want output videos to be saved in `output_dir` (as in LivePortrait), `false` otherwise.
 - `use_model_cache`: `true` if you want live portrait and face detector models to be cached for subsequent calls using same models, `false` otherwise.
 - `human_face_detector`: `insightface`, `mediapipe` or `facealignment`. Face detector to be used by human inference. Default to the `Human face detector` UI setting if defined or `insightface` if not set neither in settings nor in endpoint body.
-- `face_alignment_detector`: `blazeface`, `blazeface_back_camera` or `sfd`. Face detector to be used by human inference when Face Alignment is selected as `human_face_detector`. Default to the `Face alignment detector` UI setting if defined or `blazeface_back_camera` if not set neither in settings nor in endpoint body.
+- `face_alignment_detector`: `blazeface`, `blazeface_back_camera`, `retinaface` or `sfd`. Face detector to be used by human inference when Face Alignment is selected as `human_face_detector`. Default to the `Face alignment detector` UI setting if defined or `blazeface_back_camera` if not set neither in settings nor in endpoint body.
 - `face_alignment_detector_device`: `cuda`, `cpu` or `mps`. Device to be used by face detector when Face Alignment is selected as `human_face_detector`. Default to `cuda`.
 - `face_alignment_detector_dtype`: `fp16`, `bf16` or `fp32`. Device type to be used by face detector when Face Alignment is selected as `human_face_detector`. Default to `fp16`.
 - `source_face_index`: Face index (0-based) to be cropped in the source image or video if `flag_do_crop` is set to `True`.

liveportrait/config/crop_config.py

@@ -44,7 +44,7 @@ class CropConfig(PrintableConfig):
     vy_ratio_crop_driving_video: float = -0.1  # adjust x offset
     direction: str = "large-small"  # direction of cropping
     ########## face alignment option ##########
-    face_alignment_detector: Literal['blazeface', 'blazeface_back_camera', 'sfd'] = 'blazeface_back_camera'
+    face_alignment_detector: Literal['blazeface', 'blazeface_back_camera', 'retinaface', 'sfd'] = 'blazeface_back_camera'
     face_alignment_detector_device: Literal['cuda', 'cpu', 'mps'] = 'cuda'
     face_alignment_detector_dtype: Literal['fp16', 'bf16', 'fp32'] = 'fp16'
     ########## face index ##########

liveportrait/gradio_pipeline.py

@@ -344,7 +344,7 @@ def execute_image_retargeting(
             if input_lip_ratio != self.source_lip_ratio:
                 combined_lip_ratio_tensor = self.live_portrait_wrapper.calc_combined_lip_ratio([[float(input_lip_ratio)]], source_lmk_user)
                 lip_delta = self.live_portrait_wrapper.retarget_lip(x_s_user, combined_lip_ratio_tensor)
-                print(lip_delta)
+                # print(lip_delta)
             x_d_new = x_d_new + \
                     (eyes_delta if eyes_delta is not None else 0) + \
                     (lip_delta if lip_delta is not None else 0)

liveportrait/utils/cropper.py

@@ -50,8 +50,6 @@ def __init__(self, **kwargs) -> None:
         else:
             try:
                 if torch.backends.mps.is_available():
-                    # Shape inference currently fails with CoreMLExecutionProvider
-                    # for the retinaface model
                     device = "mps"
                     face_analysis_wrapper_provider = ["CPUExecutionProvider"]
                 else:

liveportrait/utils/cropper_face_alignment.py

@@ -52,8 +52,6 @@ def __init__(self, **kwargs) -> None:
         else:
             try:
                 if torch.backends.mps.is_available():
-                    # Shape inference currently fails with CoreMLExecutionProvider
-                    # for the retinaface model
                     device = "mps"
                 else:
                     device = "cuda"
@@ -67,11 +65,21 @@ def __init__(self, **kwargs) -> None:
         )
         self.human_landmark_runner.warmup()
 
-        if 'blazeface' in face_detector:
-            face_detector_kwargs = {'back_model': face_detector == 'blazeface_back_camera'}
-            self.fa = FaceAlignment(LandmarksType.TWO_D, flip_input=False, device=face_detector_device, dtype=face_detector_dtype, face_detector='blazeface', face_detector_kwargs=face_detector_kwargs)
+        if face_detector == 'blazeface':
+            face_detector_kwargs = {'back_model': face_detector_dtype == 'blazeface_back_camera'}
+        elif face_detector == 'retinaface':
+            face_detector_kwargs = {'fp16': face_detector == torch.float16}
         else:
-            self.fa = FaceAlignment(LandmarksType.TWO_D, flip_input=False, device=face_detector_device, dtype=face_detector_dtype, face_detector=face_detector)
+            face_detector_kwargs = {}
+
+        self.fa = FaceAlignment(
+            LandmarksType.TWO_D,
+            flip_input=False,
+            device=face_detector_device,
+            dtype=face_detector_dtype,
+            face_detector=face_detector,
+            face_detector_kwargs=face_detector_kwargs
+        )
 
         if self.image_type == "animal_face":
             from .animal_landmark_runner import XPoseRunner as AnimalLandmarkRunner

liveportrait/utils/dependencies/face_alignment/detection/retinaface/__init__.py

@@ -0,0 +1 @@
+from .retinaface_detector import RetinaFaceDetector as FaceDetector