Coverage for src/bob/bio/face/utils.py: 26%

1import logging

3from collections.abc import Iterable

5from sklearn.pipeline import Pipeline

7from bob.pipelines import wrap

9from .preprocessor import (

10 BoundingBoxAnnotatorCrop,

11 FaceCrop,

12 MultiFaceCrop,

13 Scale,

14)

15from .preprocessor.croppers import FaceEyesNorm

17logger = logging.getLogger(__name__)

20def lookup_config_from_database(database):

21 """

22 Read configuration values that might be already defined in the database configuration

23 file.

24 """

25 if database is not None:

26 annotation_type = database.annotation_type

27 fixed_positions = database.fixed_positions

28 memory_demanding = (

29 database.memory_demanding

30 if hasattr(database, "memory_demanding")

31 else False

32 )

34 else:

35 annotation_type = None

36 fixed_positions = None

37 memory_demanding = False

39 return annotation_type, fixed_positions, memory_demanding

42def cropped_positions_arcface(annotation_type="eyes-center"):

43 """

44 Returns the 112 x 112 crop used in iResnet based models

45 The crop follows the following rule:

47 - In X --> (112/2)-1

48 - In Y, leye --> 16+(112/2) --> 72

49 - In Y, reye --> (112/2)-16 --> 40

51 This will leave 16 pixels between left eye and left border and right eye and right border

53 For reference, https://github.com/deepinsight/insightface/blob/master/recognition/arcface_mxnet/common/face_align.py

54 contains the cropping code for training the original ArcFace-InsightFace model. Due to this code not being very explicit,

55 we choose to pick our own default cropped positions. They have been tested to provide good evaluation performance

56 on the Mobio dataset.

58 For sensitive applications, you can use custom cropped position that you optimize for your specific dataset,

59 such as is done in https://gitlab.idiap.ch/bob/bob.bio.face/-/blob/master/notebooks/50-shades-of-face.ipynb

61 """

63 if isinstance(annotation_type, str):

64 if annotation_type in ("eyes-center", "bounding-box"):

65 cropped_positions = {

66 "leye": (55, 72),

67 "reye": (55, 40),

68 }

69 elif annotation_type == "left-profile":

70 cropped_positions = {"leye": (52, 56), "mouth": (91, 56)}

71 elif annotation_type == "right-profile":

72 cropped_positions = {"reye": (52, 56), "mouth": (91, 56)}

73 else:

74 raise ValueError(

75 f"Annotations of the type `{annotation_type}` not supported"

76 )

78 return cropped_positions

80 if isinstance(annotation_type, Iterable):

81 return [cropped_positions_arcface(item) for item in annotation_type]

83 raise ValueError(

84 f"Annotations of the type `{annotation_type}` not supported."

85 )

88def dnn_default_cropping(cropped_image_size, annotation_type):

89 """

90 Computes the default cropped positions for the FaceCropper used with Neural-Net based

91 extractors, proportionally to the target image size

94 Parameters

95 ----------

96 cropped_image_size : tuple

97 A tuple (HEIGHT, WIDTH) describing the target size of the cropped image.

99 annotation_type: str or list of str

100 Type of annotations. Possible values are: `bounding-box`, `eyes-center`, 'left-profile',

101 'right-profile' and None, or a combination of those as a list

102

103 Returns

104 -------

105

106 cropped_positions:

107 The dictionary of cropped positions that will be feeded to the FaceCropper, or a list of such dictionaries if

108 ``annotation_type`` is a list

109 """

110

111 if isinstance(annotation_type, str):

112 CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH = cropped_image_size

113

114 cropped_positions = {}

115

116 if annotation_type == "bounding-box":

117 TOP_LEFT_POS = (0, 0)

118 BOTTOM_RIGHT_POS = (CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH)

119 cropped_positions.update(

120 {"topleft": TOP_LEFT_POS, "bottomright": BOTTOM_RIGHT_POS}

121 )

122

123 if annotation_type in ["bounding-box", "eyes-center"]:

124 # We also add cropped eye positions if `bounding-box`, to work with the BoundingBoxCropAnnotator

125 RIGHT_EYE_POS = (

126 round(2 / 7 * CROPPED_IMAGE_HEIGHT),

127 round(1 / 3 * CROPPED_IMAGE_WIDTH),

128 )

129 LEFT_EYE_POS = (

130 round(2 / 7 * CROPPED_IMAGE_HEIGHT),

131 round(2 / 3 * CROPPED_IMAGE_WIDTH),

132 )

133 cropped_positions.update(

134 {"leye": LEFT_EYE_POS, "reye": RIGHT_EYE_POS}

135 )

136

137 elif annotation_type == "left-profile":

138 EYE_POS = (

139 round(2 / 7 * CROPPED_IMAGE_HEIGHT),

140 round(3 / 8 * CROPPED_IMAGE_WIDTH),

141 )

142 MOUTH_POS = (

143 round(5 / 7 * CROPPED_IMAGE_HEIGHT),

144 round(3 / 8 * CROPPED_IMAGE_WIDTH),

145 )

146 cropped_positions.update({"leye": EYE_POS, "mouth": MOUTH_POS})

147

148 elif annotation_type == "right-profile":

149 EYE_POS = (

150 round(2 / 7 * CROPPED_IMAGE_HEIGHT),

151 round(5 / 8 * CROPPED_IMAGE_WIDTH),

152 )

153 MOUTH_POS = (

154 round(5 / 7 * CROPPED_IMAGE_HEIGHT),

155 round(5 / 8 * CROPPED_IMAGE_WIDTH),

156 )

157 cropped_positions.update({"reye": EYE_POS, "mouth": MOUTH_POS})

158

159 else:

160 logger.warning(

161 f"Annotation type {annotation_type} is not supported. Input images will be fully scaled."

162 )

163 cropped_positions = None

164

165 return cropped_positions

166

167 if isinstance(annotation_type, Iterable):

168 return [

169 dnn_default_cropping(cropped_image_size, item)

170 for item in annotation_type

171 ]

172

173 logger.warning(

174 f"Annotation type {annotation_type} is not supported. Input images will be fully scaled."

175 )

176 return None

177

178

179def legacy_default_cropping(cropped_image_size, annotation_type):

180 """

181 Computes the default cropped positions for the FaceCropper used with legacy extractors,

182 proportionally to the target image size

183

184

185 Parameters

186 ----------

187 cropped_image_size : tuple

188 A tuple (HEIGHT, WIDTH) describing the target size of the cropped image.

189

190 annotation_type: str

191 Type of annotations. Possible values are: `bounding-box`, `eyes-center`, 'left-profile',

192 'right-profile' and None, or a combination of those as a list

193

194 Returns

195 -------

196

197 cropped_positions:

198 The dictionary of cropped positions that will be feeded to the FaceCropper, or a list of such dictionaries if

199 ``annotation_type`` is a list

200 """

201

202 if isinstance(annotation_type, str):

203 CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH = cropped_image_size

204

205 cropped_positions = {}

206

207 if annotation_type == "bounding-box":

208 TOP_LEFT_POS = (0, 0)

209 BOTTOM_RIGHT_POS = (CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH)

210 cropped_positions.update(

211 {"topleft": TOP_LEFT_POS, "bottomright": BOTTOM_RIGHT_POS}

212 )

213

214 if annotation_type in ["bounding-box", "eyes-center"]:

215 # We also add cropped eye positions if `bounding-box`, to work with the BoundingBoxCropAnnotator

216

217 RIGHT_EYE_POS = (

218 CROPPED_IMAGE_HEIGHT // 5,

219 CROPPED_IMAGE_WIDTH // 4 - 1,

220 )

221 LEFT_EYE_POS = (

222 CROPPED_IMAGE_HEIGHT // 5,

223 CROPPED_IMAGE_WIDTH // 4 * 3,

224 )

225 cropped_positions.update(

226 {"leye": LEFT_EYE_POS, "reye": RIGHT_EYE_POS}

227 )

228

229 elif annotation_type == "left-profile":

230 # Main reference https://gitlab.idiap.ch/bob/bob.chapter.FRICE/-/blob/master/bob/chapter/FRICE/script/pose.py

231 EYE_POS = (

232 CROPPED_IMAGE_HEIGHT // 5,

233 CROPPED_IMAGE_WIDTH // 7 * 3 - 2,

234 )

235 MOUTH_POS = (

236 CROPPED_IMAGE_HEIGHT // 3 * 2,

237 CROPPED_IMAGE_WIDTH // 7 * 3 - 2,

238 )

239 cropped_positions.update({"leye": EYE_POS, "mouth": MOUTH_POS})

240

241 elif annotation_type == "right-profile":

242 # Main reference https://gitlab.idiap.ch/bob/bob.chapter.FRICE/-/blob/master/bob/chapter/FRICE/script/pose.py

243 EYE_POS = (

244 CROPPED_IMAGE_HEIGHT // 5,

245 CROPPED_IMAGE_WIDTH // 7 * 4 + 2,

246 )

247 MOUTH_POS = (

248 CROPPED_IMAGE_HEIGHT // 3 * 2,

249 CROPPED_IMAGE_WIDTH // 7 * 4 + 2,

250 )

251 cropped_positions.update({"reye": EYE_POS, "mouth": MOUTH_POS})

252

253 else:

254 logger.warning(

255 f"Annotation type {annotation_type} is not supported. Input images will be fully scaled."

256 )

257 cropped_positions = None

258

259 return cropped_positions

260

261 if isinstance(annotation_type, Iterable):

262 return [

263 legacy_default_cropping(cropped_image_size, item)

264 for item in annotation_type

265 ]

266

267 logger.warning(

268 f"Annotation type {annotation_type} is not supported. Input images will be fully scaled."

269 )

270 return None

271

272

273def pad_default_cropping(cropped_image_size, annotation_type):

274 """

275 Computes the default cropped positions for the FaceCropper used in PAD applications,

276 proportionally to the target image size

277

278

279 Parameters

280 ----------

281 cropped_image_size : tuple

282 A tuple (HEIGHT, WIDTH) describing the target size of the cropped image.

283

284 annotation_type: str

285 Type of annotations. Possible values are: `bounding-box`, `eyes-center` and None, or a combination of those as a list

286

287 Returns

288 -------

289

290 cropped_positions:

291 The dictionary of cropped positions that will be feeded to the FaceCropper, or a list of such dictionaries if

292 ``annotation_type`` is a list

293 """

294 if cropped_image_size[0] != cropped_image_size[1]:

295 logger.warning(

296 "PAD cropping is designed for a square cropped image size. Got : {}".format(

297 cropped_image_size

298 )

299 )

300 else:

301 face_size = cropped_image_size[0]

302

303 cropped_positions = {}

304

305 if annotation_type == "bounding-box":

306 cropped_positions.update(

307 {

308 "topleft": (0, 0),

309 "bottomright": cropped_image_size,

310 }

311 )

312

313 if annotation_type in ["bounding-box", "eyes-center"]:

314 # We also add cropped eye positions if `bounding-box`, to work with the BoundingBoxCropAnnotator

315

316 eyes_distance = (face_size + 1) / 2.0

317 eyes_center = (face_size / 4.0, (face_size - 0.5) / 2.0)

318 right_eye = (eyes_center[0], eyes_center[1] - eyes_distance / 2)

319 left_eye = (eyes_center[0], eyes_center[1] + eyes_distance / 2)

320 cropped_positions.update({"reye": right_eye, "leye": left_eye})

321

322 else:

323 logger.warning(

324 f"Annotation type {annotation_type} is not supported. Input images will be fully scaled."

325 )

326 cropped_positions = None

327

328 return cropped_positions

329

330

331def make_cropper(

332 cropped_image_size,

333 cropped_positions,

334 fixed_positions=None,

335 color_channel="rgb",

336 annotator=None,

337 **kwargs,

338):

339 """

340 Solve the face FaceCropper and additionally returns the necessary

341 transform_extra_arguments for wrapping the cropper with a SampleWrapper.

342

343 """

344 face_cropper = face_crop_solver(

345 cropped_image_size=cropped_image_size,

346 cropped_positions=cropped_positions,

347 fixed_positions=fixed_positions,

348 annotator=annotator,

349 color_channel=color_channel,

350 dtype="float64",

351 **kwargs,

352 )

353

354 transform_extra_arguments = (

355 None

356 if (cropped_positions is None or fixed_positions is not None)

357 else (("annotations", "annotations"),)

358 )

359

360 return face_cropper, transform_extra_arguments

361

362

363def embedding_transformer(

364 cropped_image_size,

365 embedding,

366 cropped_positions,

367 fixed_positions=None,

368 color_channel="rgb",

369 annotator=None,

370 **kwargs,

371):

372 """

373 Creates a pipeline composed by a FaceCropper and an Embedding extractor.

374 This transformer is suited for Facenet based architectures

375

376 .. warning::

377 This will resize images to the requested `image_size`

378

379 """

380

381 face_cropper, transform_extra_arguments = make_cropper(

382 cropped_image_size=cropped_image_size,

383 cropped_positions=cropped_positions,

384 fixed_positions=fixed_positions,

385 color_channel=color_channel,

386 annotator=annotator,

387 **kwargs,

388 )

389

390 # Support None and "passthrough" Estimators

391 if embedding is not None and type(embedding) is not str:

392 embedding = wrap(["sample"], embedding)

393

394 transformer = Pipeline(

395 [

396 (

397 "cropper",

398 wrap(

399 ["sample"],

400 face_cropper,

401 transform_extra_arguments=transform_extra_arguments,

402 ),

403 ),

404 ("embedding", embedding),

405 ]

406 )

407

408 return transformer

409

410

411def face_crop_solver(

412 cropped_image_size,

413 cropped_positions=None,

414 color_channel="rgb",

415 fixed_positions=None,

416 annotator=None,

417 dtype="uint8",

418 **kwargs,

419):

420 """

421 Decide which face cropper to use.

422 """

423 # If there's not cropped positions, just resize

424 if cropped_positions is None:

425 return Scale(cropped_image_size)

426 else:

427 # Detects the face and crops it without eye detection

428

429 if isinstance(cropped_positions, list):

430 # TODO: This is a hack to support multiple annotations for left, right and eyes center profile

431 # We need to do a more elegant solution

432

433 croppers = []

434 for positions in cropped_positions:

435 if "leye" in positions and "reye" in positions:

436 cropper = FaceCrop(

437 cropped_image_size=cropped_image_size,

438 cropped_positions=positions,

439 fixed_positions=fixed_positions,

440 color_channel=color_channel,

441 annotator=annotator,

442 dtype=dtype,

443 cropper=FaceEyesNorm(

444 positions,

445 cropped_image_size,

446 annotation_type="eyes-center",

447 ),

448 )

449

450 elif "leye" in positions and "mouth" in positions:

451 cropper = FaceCrop(

452 cropped_image_size=cropped_image_size,

453 cropped_positions=positions,

454 fixed_positions=fixed_positions,

455 color_channel=color_channel,

456 annotator=annotator,

457 dtype=dtype,

458 cropper=FaceEyesNorm(

459 positions,

460 cropped_image_size,

461 annotation_type="left-profile",

462 ),

463 )

464 elif "reye" in positions and "mouth" in positions:

465 cropper = FaceCrop(

466 cropped_image_size=cropped_image_size,

467 cropped_positions=positions,

468 fixed_positions=fixed_positions,

469 color_channel=color_channel,

470 annotator=annotator,

471 dtype=dtype,

472 cropper=FaceEyesNorm(

473 positions,

474 cropped_image_size,

475 annotation_type="right-profile",

476 ),

477 )

478

479 else:

480 raise ValueError(

481 f"Unsupported list of annotations {cropped_positions}"

482 )

483

484 croppers.append(cropper)

485

486 return MultiFaceCrop(croppers)

487 else:

488 # If the eyes annotations are provided

489 if (

490 "topleft" in cropped_positions

491 or "bottomright" in cropped_positions

492 ):

493 eyes_cropper = FaceEyesNorm(

494 cropped_positions, cropped_image_size

495 )

496 return BoundingBoxAnnotatorCrop(

497 eyes_cropper=eyes_cropper,

498 annotator=annotator,

499 )

500

501 else:

502 return FaceCrop(

503 cropped_image_size=cropped_image_size,

504 cropped_positions=cropped_positions,

505 color_channel=color_channel,

506 fixed_positions=fixed_positions,

507 dtype=dtype,

508 annotator=annotator,

509 **kwargs,

510 )

511

512

513def get_default_cropped_positions(mode, cropped_image_size, annotation_type):

514 """

515 Computes the default cropped positions for the FaceCropper,

516 proportionally to the target image size

517

518

519 Parameters

520 ----------

521 mode: str

522 Which default cropping to use. Available modes are : `legacy` (legacy baselines), `facenet`, `arcface`,

523 and `pad`.

524

525 cropped_image_size : tuple

526 A tuple (HEIGHT, WIDTH) describing the target size of the cropped image.

527

528 annotation_type: str

529 Type of annotations. Possible values are: `bounding-box`, `eyes-center` and None, or a combination of those as a list

530

531 Returns

532 -------

533

534 cropped_positions:

535 The dictionary of cropped positions that will be feeded to the FaceCropper, or a list of such dictionaries if

536 ``annotation_type`` is a list

537 """

538 if mode == "legacy":

539 return legacy_default_cropping(cropped_image_size, annotation_type)

540 elif mode in ["dnn", "facenet", "arcface"]:

541 return dnn_default_cropping(cropped_image_size, annotation_type)

542 elif mode == "pad":

543 return pad_default_cropping(cropped_image_size, annotation_type)

544 else:

545 raise ValueError("Unknown default cropping mode `{}`".format(mode))