Coverage for src/bob/bio/face/preprocessor/croppers.py: 83%
120 statements
« prev ^ index » next coverage.py v7.6.0, created at 2024-07-13 00:04 +0200
1#!/usr/bin/env python
2# @author: Tiago de Freitas Pereira
4"""
5Implements some face croppers
6"""
9import logging
11from sklearn.base import BaseEstimator, TransformerMixin
13logger = logging.getLogger("bob.bio.face")
15import cv2
16import numpy as np
18from bob.io.image import bob_to_opencvbgr, opencvbgr_to_bob
21class FaceEyesNorm(TransformerMixin, BaseEstimator):
22 """
23 Geometric normalize a face using the eyes positions
24 This function extracts the facial image based on the eye locations (or the location of other fixed point, see note below). "
25 The geometric normalization is applied such that the eyes are placed to **fixed positions** in the normalized image.
26 The image is cropped at the same time, so that no unnecessary operations are executed.
28 There are three types of annotations:
29 - **eyes-center**: The eyes are located at the center of the face. In this case, `reference_eyes_location` expects
30 a dictionary with two keys: `leye` and `reye`.
32 - **left-profile**: The eyes are located at the corner of the face. In this case, `reference_eyes_location` expects
33 a dictionary with two keys: `leye` and `mouth`.
35 - **right-profile**: The eyes are located at the corner of the face. In this case, `reference_eyes_location` expects
36 a dictionary with two keys: `reye` and `mouth`.
39 Parameters
40 ----------
42 reference_eyes_location : dict
43 The reference eyes location. It is a dictionary with two keys.
45 final_image_size : tuple
46 The final size of the image
48 allow_upside_down_normalized_faces: bool
49 If set to True, the normalized face will be flipped if the eyes are placed upside down.
51 annotation_type : str
52 The type of annotation. It can be either 'eyes-center' or 'left-profile' or 'right-profile'
54 opencv_interpolation : int
55 The interpolation method to be used by OpenCV for the function cv2.warpAffine
58 """
60 def __init__(
61 self,
62 reference_eyes_location,
63 final_image_size,
64 allow_upside_down_normalized_faces=False,
65 annotation_type="eyes-center",
66 opencv_interpolation=cv2.INTER_LINEAR,
67 ):
68 self.annotation_type = annotation_type
69 self.reference_eyes_location = reference_eyes_location
70 self._check_annotations(self.reference_eyes_location)
71 self.opencv_interpolation = opencv_interpolation
73 self.allow_upside_down_normalized_faces = (
74 allow_upside_down_normalized_faces
75 )
76 (
77 self.target_eyes_distance,
78 self.target_eyes_center,
79 self.target_eyes_angle,
80 ) = self._get_anthropometric_measurements(reference_eyes_location)
82 self.final_image_size = final_image_size
84 def _check_annotations(self, positions):
85 if self.annotation_type == "eyes-center":
86 assert "leye" in positions
87 assert "reye" in positions
88 elif self.annotation_type == "left-profile":
89 assert "leye" in positions
90 assert "mouth" in positions
91 elif self.annotation_type == "right-profile":
92 assert "reye" in positions
93 assert "mouth" in positions
94 else:
95 raise ValueError(
96 "The annotation type must be either 'eyes-center', 'left-profile' or 'right-profile'"
97 )
99 def _decode_positions(self, positions):
100 """
101 Return the annotation positions, based on the annotation type
102 """
103 if self.annotation_type == "eyes-center":
104 return np.array(positions["leye"]), np.array(positions["reye"])
105 elif self.annotation_type == "left-profile":
106 return np.array(positions["leye"]), np.array(positions["mouth"])
107 elif self.annotation_type == "right-profile":
108 return np.array(positions["reye"]), np.array(positions["mouth"])
109 else:
110 raise ValueError(
111 "The annotation type must be either 'eyes-center', 'left-profile' or 'right-profile'"
112 )
114 def _get_anthropometric_measurements(self, positions):
115 """
116 Given the eyes coordinates, it computes the
117 - The angle between the eyes coordinates
118 - The distance between the eyes coordinates
119 - The center of the eyes coordinates
121 """
123 # double dy = leftEye[0] - rightEye[0], dx = leftEye[1] - rightEye[1];
124 # double angle = std::atan2(dy, dx);
125 coordinate_a, coordinate_b = self._decode_positions(positions)
126 delta = coordinate_a - coordinate_b
127 eyes_angle = np.arctan2(delta[0], delta[1]) * 180 / np.pi # to degrees
129 # Or scaling factor
130 # m_eyesDistance / sqrt(_sqr(leftEye[0]-rightEye[0]) + _sqr(leftEye[1]-rightEye[1]))
131 eyes_distance = np.linalg.norm(delta)
133 eyes_center = 1 / 2 * (coordinate_a + coordinate_b)
135 return eyes_distance, eyes_center, eyes_angle
137 def _more_tags(self):
138 return {"requires_fit": False}
140 def fit(self, X, y=None):
141 return self
143 def _check_upsidedown(self, annotations):
144 coordinate_a, coordinate_b = self._decode_positions(annotations)
145 reference_coordinate_a, reference_coordinate_b = self._decode_positions(
146 self.reference_eyes_location
147 )
149 reye_desired_width = reference_coordinate_a[1]
150 leye_desired_width = reference_coordinate_b[1]
151 right_eye = coordinate_a
152 left_eye = coordinate_b
154 if (
155 reye_desired_width > leye_desired_width
156 and right_eye[1] < left_eye[1]
157 ) or (
158 reye_desired_width < leye_desired_width
159 and right_eye[1] > left_eye[1]
160 ):
161 raise ValueError(
162 "Looks like 'leye' and 'reye' in annotations: {annot} are swapped. "
163 "This will make the normalized face upside down (compared to the original "
164 "image). Most probably your annotations are wrong. Otherwise, you can set "
165 "the ``allow_upside_down_normalized_faces`` parameter to "
166 "True.".format(annot=annotations)
167 )
169 def _rotate_image_center(self, image, angle, reference_point):
170 """
171 Rotate the image around the center by the given angle.
172 """
174 rot_mat = cv2.getRotationMatrix2D(reference_point[::-1], angle, 1.0)
176 return cv2.warpAffine(
177 image, rot_mat, image.shape[1::-1], flags=self.opencv_interpolation
178 )
180 def _translate_image(self, image, x, y):
181 t_mat = np.float32([[1, 0, x], [0, 1, y]])
182 return cv2.warpAffine(
183 image, t_mat, image.shape[1::-1], flags=self.opencv_interpolation
184 )
186 def transform(self, X, annotations=None):
187 """
188 Geometric normalize a face using the eyes positions
190 Parameters
191 ----------
193 X : numpy.ndarray
194 The image to be normalized
196 annotations : dict
197 The annotations of the image. It needs to contain ''reye'' and ''leye'' positions
200 Returns
201 -------
203 cropped_image : numpy.ndarray
204 The normalized image
206 """
208 self._check_annotations(annotations)
210 if not self.allow_upside_down_normalized_faces:
211 self._check_upsidedown(annotations)
213 (
214 source_eyes_distance,
215 source_eyes_center,
216 source_eyes_angle,
217 ) = self._get_anthropometric_measurements(annotations)
219 # m_geomNorm->setRotationAngle(angle * 180. / M_PI - m_eyesAngle);
220 # Computing the rotation angle with respect to the target eyes angle in degrees
221 rotational_angle = source_eyes_angle - self.target_eyes_angle
223 # source_target_ratio = source_eyes_distance / self.target_eyes_distance
224 target_source_ratio = self.target_eyes_distance / source_eyes_distance
226 #
228 # ROTATION WITH OPEN CV
230 cropped_image = bob_to_opencvbgr(X) if X.ndim > 2 else X
231 original_height = cropped_image.shape[0]
232 original_width = cropped_image.shape[1]
234 cropped_image = self._rotate_image_center(
235 cropped_image, rotational_angle, source_eyes_center
236 )
238 # Cropping
240 target_eyes_center_rescaled = np.floor(
241 self.target_eyes_center / target_source_ratio
242 ).astype("int")
244 top = int(source_eyes_center[0] - target_eyes_center_rescaled[0])
245 left = int(source_eyes_center[1] - target_eyes_center_rescaled[1])
247 bottom = max(0, top) + (
248 int(self.final_image_size[0] / target_source_ratio)
249 )
250 right = max(0, left) + (
251 int(self.final_image_size[1] / target_source_ratio)
252 )
254 cropped_image = cropped_image[
255 max(0, top) : bottom, max(0, left) : right, ...
256 ]
258 # Checking if we need to pad the cropped image
259 # This happens when the cropped image extrapolate the original image dimensions
260 expanded_image = cropped_image
262 if original_height < bottom or original_width < right:
263 pad_height = (
264 cropped_image.shape[0] + (bottom - original_height)
265 if original_height < bottom
266 else cropped_image.shape[0]
267 )
269 pad_width = (
270 cropped_image.shape[1] + (right - original_width)
271 if original_width < right
272 else cropped_image.shape[1]
273 )
275 expanded_image = (
276 np.zeros(
277 (pad_height, pad_width, 3),
278 dtype=cropped_image.dtype,
279 )
280 if cropped_image.ndim > 2
281 else np.zeros(
282 (pad_height, pad_width), dtype=cropped_image.dtype
283 )
284 )
286 expanded_image[
287 0 : cropped_image.shape[0], 0 : cropped_image.shape[1], ...
288 ] = cropped_image
290 # Checking if we need to translate the image.
291 # This happens when the top, left coordinates on the source images is negative
292 if top < 0 or left < 0:
293 expanded_image = self._translate_image(
294 expanded_image, -1 * min(0, left), -1 * min(0, top)
295 )
297 # Scaling
299 expanded_image = cv2.resize(
300 expanded_image,
301 self.final_image_size[::-1],
302 interpolation=self.opencv_interpolation,
303 )
305 expanded_image = (
306 opencvbgr_to_bob(expanded_image) if X.ndim > 2 else expanded_image
307 )
309 return expanded_image
312class FaceCropBoundingBox(TransformerMixin, BaseEstimator):
313 """
314 Crop the face based on Bounding box positions
316 Parameters
317 ----------
319 final_image_size : tuple
320 The final size of the image after cropping in case resize=True
322 margin : float
323 The margin to be added to the bounding box
326 """
328 def __init__(
329 self,
330 final_image_size,
331 margin=0.5,
332 opencv_interpolation=cv2.INTER_LINEAR,
333 ):
334 self.margin = margin
335 self.final_image_size = final_image_size
336 self.opencv_interpolation = opencv_interpolation
338 def transform(self, X, annotations, resize=True):
339 """
340 Crop the face based on Bounding box positions
342 Parameters
343 ----------
345 X : numpy.ndarray
346 The image to be normalized
348 annotations : dict
349 The annotations of the image. It needs to contain ''topleft'' and ''bottomright'' positions
351 resize: bool
352 If True, the image will be resized to the final size
353 In this case, margin is not used
355 """
357 assert "topleft" in annotations
358 assert "bottomright" in annotations
360 # If it's grayscaled, expand dims
361 if X.ndim == 2:
362 logger.warning(
363 "Gray-scaled image. Expanding the channels before detection"
364 )
365 X = np.repeat(np.expand_dims(X, 0), 3, axis=0)
367 top = int(annotations["topleft"][0])
368 left = int(annotations["topleft"][1])
370 bottom = int(annotations["bottomright"][0])
371 right = int(annotations["bottomright"][1])
373 width = right - left
374 height = bottom - top
376 if resize:
377 # If resizing, don't use the expanded borders
378 face_crop = X[
379 :,
380 top:bottom,
381 left:right,
382 ]
384 face_crop = (
385 bob_to_opencvbgr(face_crop) if face_crop.ndim > 2 else face_crop
386 )
388 face_crop = cv2.resize(
389 face_crop,
390 self.final_image_size[::-1],
391 interpolation=self.opencv_interpolation,
392 )
394 face_crop = opencvbgr_to_bob(np.array(face_crop))
396 else:
397 # Expanding the borders
398 top_expanded = int(np.maximum(top - self.margin * height, 0))
399 left_expanded = int(np.maximum(left - self.margin * width, 0))
401 bottom_expanded = int(
402 np.minimum(bottom + self.margin * height, X.shape[1])
403 )
404 right_expanded = int(
405 np.minimum(right + self.margin * width, X.shape[2])
406 )
408 face_crop = X[
409 :,
410 top_expanded:bottom_expanded,
411 left_expanded:right_expanded,
412 ]
414 return face_crop
416 def _more_tags(self):
417 return {"requires_fit": False}
419 def fit(self, X, y=None):
420 return self