Coverage for src/bob/bio/face/annotator/tinyface.py: 12%
141 statements
« prev ^ index » next coverage.py v7.6.0, created at 2024-07-13 00:04 +0200
1import logging
2import os
3import pickle
5from collections import namedtuple
7import numpy as np
8import pkg_resources
10from bob.bio.base.database.utils import download_file, md5_hash
11from bob.bio.face.color import gray_to_rgb
12from bob.io.image import to_matplotlib
14from .Base import Base
16logger = logging.getLogger(__name__)
19class TinyFace(Base):
21 """TinyFace face detector. Original Model is ``ResNet101`` from
22 https://github.com/peiyunh/tiny. Please check for details. The
23 model used in this section is the MxNet version from
24 https://github.com/chinakook/hr101_mxnet.
26 Attributes
27 ----------
28 prob_thresh: float
29 Thresholds are a trade-off between false positives and missed detections.
30 """
32 def __init__(self, prob_thresh=0.5, **kwargs):
33 super().__init__(**kwargs)
35 import mxnet as mx
37 urls = [
38 "https://www.idiap.ch/software/bob/data/bob/bob.ip.facedetect/master/tinyface_detector.tar.gz"
39 ]
41 self.checkpoint_path = download_file(
42 urls=urls,
43 destination_sub_directory="data/tinyface_detector",
44 destination_filename="tinyface_detector.tar.gz",
45 checksum="f24e820b47a7440d7cdd7e0c43d4d455",
46 checksum_fct=md5_hash,
47 extract=True,
48 )
50 self.MAX_INPUT_DIM = 5000.0
51 self.prob_thresh = prob_thresh
52 self.nms_thresh = 0.1
53 self.model_root = pkg_resources.resource_filename(
54 __name__, self.checkpoint_path
55 )
57 sym, arg_params, aux_params = mx.model.load_checkpoint(
58 os.path.join(self.checkpoint_path, "hr101"), 0
59 )
60 all_layers = sym.get_internals()
62 meta_file = open(os.path.join(self.checkpoint_path, "meta.pkl"), "rb")
63 self.clusters = pickle.load(meta_file)
64 self.averageImage = pickle.load(meta_file)
65 meta_file.close()
66 self.clusters_h = self.clusters[:, 3] - self.clusters[:, 1] + 1
67 self.clusters_w = self.clusters[:, 2] - self.clusters[:, 0] + 1
68 self.normal_idx = np.where(self.clusters[:, 4] == 1)
70 self.mod = mx.mod.Module(
71 symbol=all_layers["fusex_output"],
72 data_names=["data"],
73 label_names=None,
74 )
75 self.mod.bind(
76 for_training=False,
77 data_shapes=[("data", (1, 3, 224, 224))],
78 label_shapes=None,
79 force_rebind=False,
80 )
81 self.mod.set_params(
82 arg_params=arg_params, aux_params=aux_params, force_init=False
83 )
85 @staticmethod
86 def _nms(dets, prob_thresh):
87 x1 = dets[:, 0]
88 y1 = dets[:, 1]
89 x2 = dets[:, 2]
90 y2 = dets[:, 3]
91 scores = dets[:, 4]
93 areas = (x2 - x1 + 1) * (y2 - y1 + 1)
95 order = scores.argsort()[::-1]
97 keep = []
98 while order.size > 0:
99 i = order[0]
100 keep.append(i)
101 xx1 = np.maximum(x1[i], x1[order[1:]])
102 yy1 = np.maximum(y1[i], y1[order[1:]])
103 xx2 = np.minimum(x2[i], x2[order[1:]])
104 yy2 = np.minimum(y2[i], y2[order[1:]])
105 w = np.maximum(0.0, xx2 - xx1 + 1)
106 h = np.maximum(0.0, yy2 - yy1 + 1)
107 inter = w * h
109 ovr = inter / (areas[i] + areas[order[1:]] - inter)
110 inds = np.where(ovr <= prob_thresh)[0]
112 order = order[inds + 1]
113 return keep
115 def annotations(self, img):
116 """Detects and annotates all faces in the image.
118 Parameters
119 ----------
120 image : numpy.ndarray
121 An RGB image in Bob format.
123 Returns
124 -------
125 list
126 A list of annotations. Annotations are dictionaries that contain the
127 following keys: ``topleft``, ``bottomright``, ``reye``, ``leye``.
128 (``reye`` and ``leye`` are the estimated results, not captured by the
129 model.)
130 """
131 import cv2 as cv
132 import mxnet as mx
134 Batch = namedtuple("Batch", ["data"])
136 raw_img = img
137 if len(raw_img.shape) == 2:
138 raw_img = gray_to_rgb(raw_img)
139 assert img.shape[0] == 3, img.shape
141 raw_img = to_matplotlib(raw_img)
142 raw_img = raw_img[..., ::-1]
144 raw_h = raw_img.shape[0]
145 raw_w = raw_img.shape[1]
147 raw_img = cv.cvtColor(raw_img, cv.COLOR_BGR2RGB)
148 raw_img_f = raw_img.astype(np.float32)
150 min_scale = min(
151 np.floor(np.log2(np.max(self.clusters_w[self.normal_idx] / raw_w))),
152 np.floor(np.log2(np.max(self.clusters_h[self.normal_idx] / raw_h))),
153 )
154 max_scale = min(1.0, -np.log2(max(raw_h, raw_w) / self.MAX_INPUT_DIM))
156 scales_down = np.arange(min_scale, 0 + 0.0001, 1.0)
157 scales_up = np.arange(0.5, max_scale + 0.0001, 0.5)
158 scales_pow = np.hstack((scales_down, scales_up))
159 scales = np.power(2.0, scales_pow)
161 bboxes = np.empty(shape=(0, 5))
162 for s in scales[::-1]:
163 img = cv.resize(raw_img_f, (0, 0), fx=s, fy=s)
164 img = np.transpose(img, (2, 0, 1))
165 img = img - self.averageImage
167 tids = []
168 if s <= 1.0:
169 tids = list(range(4, 12))
170 else:
171 tids = list(range(4, 12)) + list(range(18, 25))
172 ignoredTids = list(
173 set(range(0, self.clusters.shape[0])) - set(tids)
174 )
175 img_h = img.shape[1]
176 img_w = img.shape[2]
177 img = img[np.newaxis, :]
179 self.mod.reshape(data_shapes=[("data", (1, 3, img_h, img_w))])
180 self.mod.forward(Batch([mx.nd.array(img)]))
181 self.mod.get_outputs()[0].wait_to_read()
182 fusex_res = self.mod.get_outputs()[0]
184 score_cls = mx.nd.slice_axis(
185 fusex_res, axis=1, begin=0, end=25, name="score_cls"
186 )
187 score_reg = mx.nd.slice_axis(
188 fusex_res, axis=1, begin=25, end=None, name="score_reg"
189 )
190 prob_cls = mx.nd.sigmoid(score_cls)
192 prob_cls_np = prob_cls.asnumpy()
193 prob_cls_np[0, ignoredTids, :, :] = 0.0
195 _, fc, fy, fx = np.where(prob_cls_np > self.prob_thresh)
197 cy = fy * 8 - 1
198 cx = fx * 8 - 1
199 ch = self.clusters[fc, 3] - self.clusters[fc, 1] + 1
200 cw = self.clusters[fc, 2] - self.clusters[fc, 0] + 1
202 Nt = self.clusters.shape[0]
204 score_reg_np = score_reg.asnumpy()
205 tx = score_reg_np[0, 0:Nt, :, :]
206 ty = score_reg_np[0, Nt : 2 * Nt, :, :]
207 tw = score_reg_np[0, 2 * Nt : 3 * Nt, :, :]
208 th = score_reg_np[0, 3 * Nt : 4 * Nt, :, :]
210 dcx = cw * tx[fc, fy, fx]
211 dcy = ch * ty[fc, fy, fx]
212 rcx = cx + dcx
213 rcy = cy + dcy
214 rcw = cw * np.exp(tw[fc, fy, fx])
215 rch = ch * np.exp(th[fc, fy, fx])
217 score_cls_np = score_cls.asnumpy()
218 scores = score_cls_np[0, fc, fy, fx]
220 tmp_bboxes = np.vstack(
221 (rcx - rcw / 2, rcy - rch / 2, rcx + rcw / 2, rcy + rch / 2)
222 )
223 tmp_bboxes = np.vstack((tmp_bboxes / s, scores))
224 tmp_bboxes = tmp_bboxes.transpose()
225 bboxes = np.vstack((bboxes, tmp_bboxes))
227 refind_idx = self._nms(bboxes, self.nms_thresh)
228 refind_bboxes = bboxes[refind_idx]
229 refind_bboxes = refind_bboxes.astype(np.int32)
231 annotations = refind_bboxes
232 annots = []
233 for i in range(len(refind_bboxes)):
234 topleft = round(float(annotations[i][1])), round(
235 float(annotations[i][0])
236 )
237 bottomright = (
238 round(float(annotations[i][3])),
239 round(float(annotations[i][2])),
240 )
241 width = float(annotations[i][2]) - float(annotations[i][0])
242 length = float(annotations[i][3]) - float(annotations[i][1])
243 right_eye = (
244 round((0.37) * length + float(annotations[i][1])),
245 round((0.3) * width + float(annotations[i][0])),
246 )
247 left_eye = (
248 round((0.37) * length + float(annotations[i][1])),
249 round((0.7) * width + float(annotations[i][0])),
250 )
251 annots.append(
252 {
253 "topleft": topleft,
254 "bottomright": bottomright,
255 "reye": right_eye,
256 "leye": left_eye,
257 }
258 )
260 return annots
262 def annotate(self, image, **kwargs):
263 """Annotates an image using tinyface
265 Parameters
266 ----------
267 image : numpy.array
268 An RGB image in Bob format.
269 **kwargs
270 Ignored.
272 Returns
273 -------
274 dict
275 Annotations with (topleft, bottomright) keys (or None).
276 """
278 # return the annotations for the first/largest face
279 annotations = self.annotations(image)
281 if annotations:
282 return annotations[0]
283 else:
284 return None