Coverage for src/bob/bio/face/embeddings/tensorflow.py: 86%

1#!/usr/bin/env python 

2# vim: set fileencoding=utf-8 : 

3# Tiago de Freitas Pereira <tiago.pereira@idiap.ch> 

4 

5# Tranformers based on tensorflow 

6 

7 

8import numpy as np 

9import tensorflow as tf 

10 

11from sklearn.base import BaseEstimator, TransformerMixin 

12from sklearn.utils import check_array 

13 

14from bob.bio.base.algorithm import Distance 

15from bob.bio.base.database.utils import download_file, md5_hash 

16from bob.bio.base.pipelines import PipelineSimple 

17from bob.bio.face.annotator import MTCNN 

18from bob.bio.face.utils import ( 

19 cropped_positions_arcface, 

20 dnn_default_cropping, 

21 embedding_transformer, 

22) 

23 

24 

25def to_channels_last(image): 

26 """Converts the image to channel_last format. This is the same format as in 

27 matplotlib, skimage, and etc. 

28 

29 Parameters 

30 ---------- 

31 image : `tf.Tensor` 

32 At least a 3 dimensional image. If the dimension is more than 3, the 

33 last 3 dimensions are assumed to be [C, H, W]. 

34 

35 Returns 

36 ------- 

37 image : `tf.Tensor` 

38 The image in [..., H, W, C] format. 

39 

40 Raises 

41 ------ 

42 ValueError 

43 If dim of image is less than 3. 

44 """ 

45 ndim = len(image.shape) 

46 if ndim < 3: 

47 raise ValueError( 

48 "The image needs to be at least 3 dimensional but it " 

49 "was {}".format(ndim) 

50 ) 

51 axis_order = [1, 2, 0] 

52 shift = ndim - 3 

53 axis_order = list(range(ndim - 3)) + [n + shift for n in axis_order] 

54 return tf.transpose(a=image, perm=axis_order) 

55 

56 

57def to_channels_first(image): 

58 """Converts the image to channel_first format. This is the same format as 

59 in Bob's image and video. 

60 

61 Parameters 

62 ---------- 

63 image : `tf.Tensor` 

64 At least a 3 dimensional image. If the dimension is more than 3, the 

65 last 3 dimensions are assumed to be [H, W, C]. 

66 

67 Returns 

68 ------- 

69 image : `tf.Tensor` 

70 The image in [..., C, H, W] format. 

71 

72 Raises 

73 ------ 

74 ValueError 

75 If dim of image is less than 3. 

76 """ 

77 ndim = len(image.shape) 

78 if ndim < 3: 

79 raise ValueError( 

80 "The image needs to be at least 3 dimensional but it " 

81 "was {}".format(ndim) 

82 ) 

83 axis_order = [2, 0, 1] 

84 shift = ndim - 3 

85 axis_order = list(range(ndim - 3)) + [n + shift for n in axis_order] 

86 return tf.transpose(a=image, perm=axis_order) 

87 

88 

89def sanderberg_rescaling(): 

90 # FIXED_STANDARDIZATION from https://github.com/davidsandberg/facenet 

91 # [-0.99609375, 0.99609375] 

92 preprocessor = tf.keras.layers.experimental.preprocessing.Rescaling( 

93 scale=1 / 128, offset=-127.5 / 128 

94 ) 

95 return preprocessor 

96 

97 

98class TensorflowTransformer(TransformerMixin, BaseEstimator): 

99 """ 

100 Base Transformer for Tensorflow architectures. 

101 

102 Parameters 

103 ---------- 

104 

105 checkpoint_path: str 

106 Path containing the checkpoint 

107 

108 preprocessor: 

109 A function that will transform the data right before forward 

110 

111 memory_demanding bool 

112 If `True`, the `transform` method will run one sample at the time. 

113 This is useful when there is not enough memory available to forward big chucks of data. 

114 """ 

115 

116 def __init__( 

117 self, 

118 checkpoint_path, 

119 preprocessor=None, 

120 memory_demanding=False, 

121 **kwargs, 

122 ): 

123 super().__init__(**kwargs) 

124 self.checkpoint_path = checkpoint_path 

125 self.model = None 

126 self.preprocessor = preprocessor 

127 self.memory_demanding = memory_demanding 

128 

129 def load_model(self): 

130 self.model = tf.keras.models.load_model( 

131 self.checkpoint_path, compile=False 

132 ) 

133 

134 def transform(self, X): 

135 def _transform(X): 

136 X = tf.convert_to_tensor(X) 

137 X = to_channels_last(X) 

138 

139 if X.shape[-3:] != self.model.input_shape[-3:]: 

140 raise ValueError( 

141 f"Image shape {X.shape} not supported. Expected {self.model.input_shape}" 

142 ) 

143 

144 return self.inference(X).numpy() 

145 

146 if self.model is None: 

147 self.load_model() 

148 

149 X = check_array(X, allow_nd=True) 

150 

151 if self.memory_demanding: 

152 features = np.array([_transform(x[None, ...]) for x in X]) 

153 

154 # If we ndim is > than 3. We should stack them all 

155 # The enroll_features can come from a source where there are `N` samples containing 

156 # nxd samples 

157 if features.ndim >= 3: 

158 features = np.vstack(features) 

159 

160 return features 

161 

162 else: 

163 return _transform(X) 

164 

165 def __getstate__(self): 

166 # Handling unpicklable objects 

167 d = self.__dict__.copy() 

168 d["model"] = None 

169 return d 

170 

171 def inference(self, X): 

172 if self.preprocessor is not None: 

173 X = self.preprocessor(tf.cast(X, "float32")) 

174 

175 prelogits = self.model.predict_on_batch(X) 

176 embeddings = tf.math.l2_normalize(prelogits, axis=-1) 

177 return embeddings 

178 

179 def _more_tags(self): 

180 return {"requires_fit": False} 

181 

182 def __del__(self): 

183 self.model = None 

184 

185 

186class InceptionResnetv2_MsCeleb_CenterLoss_2018(TensorflowTransformer): 

187 """ 

188 InceptionResnet v2 model trained in 2018 using the MSCeleb dataset in the context of the work: 

189 

190 Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816. 

191 

192 """ 

193 

194 def __init__(self, memory_demanding=False, **kwargs): 

195 urls = [ 

196 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_msceleb_centerloss_2018.tar.gz", 

197 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_msceleb_centerloss_2018.tar.gz", 

198 ] 

199 

200 checkpoint_path = download_file( 

201 urls=urls, 

202 destination_sub_directory="data/tensorflow/inceptionresnetv2_msceleb_centerloss_2018", 

203 destination_filename="inceptionresnetv2_msceleb_centerloss_2018.tar.gz", 

204 checksum="7c0aa46bba16c01768a38594a3b4c14d", 

205 checksum_fct=md5_hash, 

206 extract=True, 

207 ) 

208 

209 super(InceptionResnetv2_MsCeleb_CenterLoss_2018, self).__init__( 

210 checkpoint_path, 

211 preprocessor=tf.image.per_image_standardization, 

212 memory_demanding=memory_demanding, 

213 **kwargs, 

214 ) 

215 

216 

217class InceptionResnetv2_Casia_CenterLoss_2018(TensorflowTransformer): 

218 """ 

219 InceptionResnet v2 model trained in 2018 using the CasiaWebFace dataset in the context of the work: 

220 

221 Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816. 

222 

223 """ 

224 

225 def __init__(self, memory_demanding=False, **kwargs): 

226 urls = [ 

227 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_casia_centerloss_2018.tar.gz", 

228 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv2_casia_centerloss_2018.tar.gz", 

229 ] 

230 

231 checkpoint_path = download_file( 

232 urls=urls, 

233 destination_sub_directory="data/tensorflow/inceptionresnetv2_casia_centerloss_2018", 

234 destination_filename="inceptionresnetv2_casia_centerloss_2018.tar.gz", 

235 checksum="1e0b62e45430a8d7516d7a6101a24c40", 

236 checksum_fct=md5_hash, 

237 extract=True, 

238 ) 

239 

240 super(InceptionResnetv2_Casia_CenterLoss_2018, self).__init__( 

241 checkpoint_path, 

242 preprocessor=tf.image.per_image_standardization, 

243 memory_demanding=memory_demanding, 

244 **kwargs, 

245 ) 

246 

247 

248class InceptionResnetv1_Casia_CenterLoss_2018(TensorflowTransformer): 

249 """ 

250 InceptionResnet v1 model trained in 2018 using the CasiaWebFace dataset in the context of the work: 

251 

252 Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816. 

253 

254 """ 

255 

256 def __init__(self, memory_demanding=False, **kwargs): 

257 urls = [ 

258 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_casia_centerloss_2018.tar.gz", 

259 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_casia_centerloss_2018.tar.gz", 

260 ] 

261 

262 checkpoint_path = download_file( 

263 urls=urls, 

264 destination_sub_directory="data/tensorflow/inceptionresnetv1_casia_centerloss_2018", 

265 destination_filename="inceptionresnetv1_casia_centerloss_2018.tar.gz", 

266 checksum="6601e6f6840ae863c7daf31a7c6b9a27", 

267 checksum_fct=md5_hash, 

268 extract=True, 

269 ) 

270 

271 super(InceptionResnetv1_Casia_CenterLoss_2018, self).__init__( 

272 checkpoint_path, 

273 preprocessor=tf.image.per_image_standardization, 

274 memory_demanding=memory_demanding, 

275 **kwargs, 

276 ) 

277 

278 

279class InceptionResnetv1_MsCeleb_CenterLoss_2018(TensorflowTransformer): 

280 """ 

281 InceptionResnet v1 model trained in 2018 using the MsCeleb dataset in the context of the work: 

282 

283 Freitas Pereira, Tiago, André Anjos, and Sébastien Marcel. "Heterogeneous face recognition using domain specific units." IEEE Transactions on Information Forensics and Security 14.7 (2018): 1803-1816. 

284 

285 """ 

286 

287 def __init__(self, memory_demanding=False, **kwargs): 

288 urls = [ 

289 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_msceleb_centerloss_2018.tar.gz", 

290 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/inceptionresnetv1_msceleb_centerloss_2018.tar.gz", 

291 ] 

292 

293 checkpoint_path = download_file( 

294 urls=urls, 

295 destination_sub_directory="data/tensorflow/inceptionresnetv1_msceleb_centerloss_2018", 

296 destination_filename="inceptionresnetv1_msceleb_centerloss_2018.tar.gz", 

297 checksum="1ca0149619e4e9320a927ea65b2b5521", 

298 checksum_fct=md5_hash, 

299 extract=True, 

300 ) 

301 

302 super(InceptionResnetv1_MsCeleb_CenterLoss_2018, self).__init__( 

303 checkpoint_path, 

304 preprocessor=tf.image.per_image_standardization, 

305 memory_demanding=memory_demanding, 

306 **kwargs, 

307 ) 

308 

309 

310class FaceNetSanderberg_20170512_110547(TensorflowTransformer): 

311 """ 

312 Wrapper for the free FaceNet from David Sanderberg model 20170512_110547: 

313 https://github.com/davidsandberg/facenet 

314 

315 And for a preprocessor you can use:: 

316 

317 from bob.bio.face.preprocessor import FaceCrop 

318 # This is the size of the image that this model expects 

319 CROPPED_IMAGE_HEIGHT = 160 

320 CROPPED_IMAGE_WIDTH = 160 

321 # eye positions for frontal images 

322 RIGHT_EYE_POS = (46, 53) 

323 LEFT_EYE_POS = (46, 107) 

324 # Crops the face using eye annotations 

325 preprocessor = FaceCrop( 

326 cropped_image_size=(CROPPED_IMAGE_HEIGHT, CROPPED_IMAGE_WIDTH), 

327 cropped_positions={'leye': LEFT_EYE_POS, 'reye': RIGHT_EYE_POS}, 

328 color_channel='rgb' 

329 ) 

330 """ 

331 

332 def __init__(self, memory_demanding=False, **kwargs): 

333 urls = [ 

334 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/facenet_sanderberg_20170512_110547.tar.gz" 

335 ] 

336 

337 checkpoint_path = download_file( 

338 urls=urls, 

339 destination_sub_directory="data/tensorflow/facenet_sanderberg_20170512_110547", 

340 destination_filename="facenet_sanderberg_20170512_110547.tar.gz", 

341 checksum="734d1c997c10acdcdffc79fb51a2e715", 

342 checksum_fct=md5_hash, 

343 extract=True, 

344 ) 

345 

346 super(FaceNetSanderberg_20170512_110547, self).__init__( 

347 checkpoint_path, 

348 tf.image.per_image_standardization, 

349 memory_demanding=memory_demanding, 

350 **kwargs, 

351 ) 

352 

353 

354class Resnet50_MsCeleb_ArcFace_2021(TensorflowTransformer): 

355 """ 

356 Resnet50 Backbone trained with the MSCeleb 1M database. 

357 

358 The bottleneck layer (a.k.a embedding) has 512d. 

359 

360 The configuration file used to trained is: 

361 

362 .. warning:: 

363 This configuration file might change in future releases 

364 

365 ```yaml 

366 batch-size: 128 

367 face-size: 112 

368 face-output_size: 112 

369 n-classes: 85742 

370 

371 

372 # Backbone 

373 backbone: 'resnet50' 

374 head: 'arcface' 

375 s: 10 

376 bottleneck: 512 

377 m: 0.5 

378 

379 # Training parameters 

380 solver: "sgd" 

381 lr: 0.1 

382 dropout-rate: 0.5 

383 epochs: 500 

384 

385 

386 train-tf-record-path: "<PATH>" 

387 validation-tf-record-path: "<PATH>" 

388 

389 ``` 

390 

391 

392 """ 

393 

394 def __init__(self, memory_demanding=False, **kwargs): 

395 urls = [ 

396 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50-msceleb-arcface_2021-48ec5cb8.tar.gz", 

397 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50-msceleb-arcface_2021-48ec5cb8.tar.gz", 

398 ] 

399 

400 checkpoint_path = download_file( 

401 urls=urls, 

402 destination_sub_directory="data/tensorflow/resnet50-msceleb-arcface_2021-48ec5cb8", 

403 destination_filename="resnet50-msceleb-arcface_2021-48ec5cb8.tar.gz", 

404 checksum="17946f121af5ddd18c637c4620e54da6", 

405 checksum_fct=md5_hash, 

406 extract=True, 

407 ) 

408 

409 super(Resnet50_MsCeleb_ArcFace_2021, self).__init__( 

410 checkpoint_path, 

411 preprocessor=lambda X: X / 255.0, 

412 memory_demanding=memory_demanding, 

413 **kwargs, 

414 ) 

415 

416 

417class Resnet50_MsCeleb_ArcFace_20210521(TensorflowTransformer): 

418 """ 

419 Resnet50 Backbone trained with the MSCeleb 1M database. The bottleneck layer (a.k.a embedding) has 512d. 

420 

421 The difference from this one to :any:`Resnet50_MsCeleb_ArcFace_2021` is the MSCeleb version used to train it. 

422 This one uses 100% of the data pruned from annotators. 

423 

424 

425 The configuration file used to trained is: 

426 

427 .. warning:: 

428 This configuration file might change in future releases 

429 

430 

431 ```yaml 

432 batch-size: 128 

433 face-size: 112 

434 face-output_size: 112 

435 n-classes: 83009 

436 

437 

438 # Backbone 

439 backbone: 'resnet50' 

440 head: 'arcface' 

441 s: 30 

442 bottleneck: 512 

443 m: 0.5 

444 

445 # Training parameters 

446 solver: "sgd" 

447 lr: 0.1 

448 dropout-rate: 0.5 

449 epochs: 300 

450 

451 

452 train-tf-record-path: "<PATH>" 

453 validation-tf-record-path: "<PATH>" 

454 

455 ``` 

456 

457 

458 """ 

459 

460 def __init__(self, memory_demanding=False, **kwargs): 

461 urls = [ 

462 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50-msceleb-arcface_20210521-e9bc085c.tar.gz", 

463 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50-msceleb-arcface_20210521-e9bc085c.tar.gz", 

464 ] 

465 

466 checkpoint_path = download_file( 

467 urls=urls, 

468 destination_sub_directory="data/tensorflow/resnet50-msceleb-arcface_20210521-801991f0", 

469 destination_filename="resnet50-msceleb-arcface_20210521-e9bc085c.tar.gz", 

470 checksum="e33090eea4951ce80be4620a0dac680d", 

471 checksum_fct=md5_hash, 

472 extract=True, 

473 ) 

474 

475 super(Resnet50_MsCeleb_ArcFace_20210521, self).__init__( 

476 checkpoint_path, 

477 preprocessor=lambda X: X / 255.0, 

478 memory_demanding=memory_demanding, 

479 **kwargs, 

480 ) 

481 

482 

483class Resnet101_MsCeleb_ArcFace_20210521(TensorflowTransformer): 

484 """ 

485 Resnet101 Backbone trained with the MSCeleb 1M database. The bottleneck layer (a.k.a embedding) has 512d. 

486 

487 The difference from this one to :any:`Resnet101_MsCeleb_ArcFace_2021` is the MSCeleb version used to train it. 

488 This one uses 100% of the data pruned from annotators. 

489 

490 

491 The configuration file used to trained is: 

492 

493 .. warning:: 

494 This configuration file might change in future releases 

495 

496 

497 ```yaml 

498 batch-size: 128 

499 face-size: 112 

500 face-output_size: 112 

501 n-classes: 83009 

502 

503 

504 # Backbone 

505 backbone: 'resnet50' 

506 head: 'arcface' 

507 s: 30 

508 bottleneck: 512 

509 m: 0.5 

510 

511 # Training parameters 

512 solver: "sgd" 

513 lr: 0.1 

514 dropout-rate: 0.5 

515 epochs: 300 

516 

517 

518 train-tf-record-path: "<PATH>" 

519 validation-tf-record-path: "<PATH>" 

520 

521 ``` 

522 

523 

524 """ 

525 

526 def __init__(self, memory_demanding=False, **kwargs): 

527 urls = [ 

528 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet101-msceleb-arcface_20210521.tar.gz", 

529 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet101-msceleb-arcface_20210521.tar.gz", 

530 ] 

531 

532 checkpoint_path = download_file( 

533 urls=urls, 

534 destination_sub_directory="data/tensorflow/resnet101-msceleb-arcface_20210521", 

535 destination_filename="resnet101-msceleb-arcface_20210521.tar.gz", 

536 checksum="c1b2124cb69186ff965f7e818f9f8641", 

537 checksum_fct=md5_hash, 

538 extract=True, 

539 ) 

540 

541 super(Resnet101_MsCeleb_ArcFace_20210521, self).__init__( 

542 checkpoint_path, 

543 preprocessor=lambda X: X / 255.0, 

544 memory_demanding=memory_demanding, 

545 **kwargs, 

546 ) 

547 

548 

549class IResnet50_MsCeleb_ArcFace_20210623(TensorflowTransformer): 

550 """ 

551 IResnet50 Backbone trained with the MSCeleb 1M database. The bottleneck layer (a.k.a embedding) has 512d. 

552 

553 The complete code to reproduce this model is in the (private) repository: 

554 bob.project.hardening/-/commit/9ac25c0a17c9628b7a99e84217cd7c680f1a3e1e 

555 but you can reproduce it using 

556 https://gitlab.idiap.ch/bob/bob.bio.face/-/blob/eed9276c7c1306c2ccfe290a0149ade3a80d247a/cnn_training/arcface_large_batch.py 

557 script and the following configuration:: 

558 

559 CONFIG = { 

560 "n-workers": 8, 

561 "batch-size": 256, 

562 "n-train-samples-per-epoch": 256_000 * 1, 

563 "real-n-train-samples": 985702, 

564 "shuffle-buffer": int(1e6), 

565 "face-size": 126, 

566 "face-output_size": 112, 

567 "n-classes": 83009, 

568 "backbone": "resnet50_large_batch", 

569 "use-l2-regularizer": False, 

570 "batch-norm-decay": 0.9, 

571 "batch-norm-epsilon": 1e-5, 

572 "head": "arcface", 

573 "s": 30, 

574 "bottleneck": 512, 

575 "m": 0.5, 

576 "dropout-rate": 0.0, 

577 "learning-rate-schedule": "none", 

578 "train-tf-record-path": "/face-tfrecords/126x126/msceleb_facecrop/*.tfrecords", 

579 "validation-tf-record-path": "/face-tfrecords/126x126/lfw_sharded/*.tfrecords", 

580 "checkpoint-path": "/temp/hardening/arcface_sgd_prelu/w8_b1000_fp16_drp0", 

581 "pre-train": False, 

582 "epochs": 6000, 

583 } 

584 strategy_fn = "multi-worker-mirrored-strategy" 

585 mixed_precision_policy = "mixed_float16" 

586 initial_lr = 0.1 / 512 * CONFIG["batch-size"] * CONFIG["n-workers"] 

587 real_n_steps_per_epoch = CONFIG["real-n-train-samples"] / (CONFIG["batch-size"] * CONFIG["n-workers"]) 

588 params = { 

589 "optimizer": { 

590 "type": "sgdw", 

591 "sgdw": { 

592 "momentum": min(0.9 * initial_lr, 0.999), 

593 "nesterov": False, 

594 "weight_decay": 5e-4, 

595 }, 

596 }, 

597 "learning_rate": { 

598 "type": "stepwise", 

599 "stepwise": { 

600 "boundaries": [int(i * real_n_steps_per_epoch) for i in [11, 17, 22]], 

601 "values": [initial_lr / (10 ** i) for i in range(0, 4)], 

602 }, 

603 }, 

604 } 

605 

606 The tensorboard logs can be found in: https://tensorboard.dev/experiment/6bBn0ya3SeilJ2elcZZoSg 

607 The model at epoch 90 is used. 

608 """ 

609 

610 def __init__(self, memory_demanding=False, **kwargs): 

611 urls = [ 

612 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/arcface_iresnet50_msceleb_idiap-089640d2.tar.gz", 

613 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/arcface_iresnet50_msceleb_idiap-089640d2.tar.gz", 

614 ] 

615 

616 checkpoint_path = download_file( 

617 urls=urls, 

618 destination_sub_directory="data/tensorflow/arcface_iresnet50_msceleb_idiap-089640d2", 

619 destination_filename="arcface_iresnet50_msceleb_idiap-089640d2.tar.gz", 

620 checksum="089640d2", 

621 extract=True, 

622 ) 

623 

624 super().__init__( 

625 checkpoint_path, 

626 preprocessor=lambda X: X / 255.0, 

627 memory_demanding=memory_demanding, 

628 **kwargs, 

629 ) 

630 

631 

632class IResnet100_MsCeleb_ArcFace_20210623(TensorflowTransformer): 

633 """ 

634 IResnet100 Backbone trained with the MSCeleb 1M database. The bottleneck layer (a.k.a embedding) has 512d. 

635 

636 The complete code to reproduce this model is in the (private) repository: 

637 bob.project.hardening/-/commit/b162ca60d26fcf8a93f6767f5b5a026a406c1076 

638 but you can reproduce it using 

639 https://gitlab.idiap.ch/bob/bob.bio.face/-/blob/eed9276c7c1306c2ccfe290a0149ade3a80d247a/cnn_training/arcface_large_batch.py 

640 script and the following configuration:: 

641 

642 CONFIG = { 

643 "n-workers": 8, 

644 "batch-size": 128, 

645 "n-train-samples-per-epoch": 256_000 * 1, 

646 "real-n-train-samples": 985702, 

647 "shuffle-buffer": int(1e5), 

648 "face-size": 126, 

649 "face-output_size": 112, 

650 "n-classes": 83009, 

651 "backbone": "iresnet100", 

652 "use-l2-regularizer": False, 

653 "batch-norm-decay": 0.9, 

654 "batch-norm-epsilon": 1e-5, 

655 "head": "arcface", 

656 "s": 30, 

657 "bottleneck": 512, 

658 "m": 0.5, 

659 "dropout-rate": 0.0, 

660 "learning-rate-schedule": "none", 

661 "train-tf-record-path": "/face-tfrecords/126x126/msceleb_facecrop/*.tfrecords", 

662 "validation-tf-record-path": "/face-tfrecords/126x126/lfw_sharded/*.tfrecords", 

663 "checkpoint-path": "/temp/hardening/arcface_sgd_prelu/i100_w8_b128_fp16_drp0", 

664 "pre-train": False, 

665 "epochs": 6000, 

666 } 

667 strategy_fn = "multi-worker-mirrored-strategy" 

668 mixed_precision_policy = "mixed_float16" 

669 initial_lr = 0.1 / 512 * CONFIG["batch-size"] * CONFIG["n-workers"] 

670 real_n_steps_per_epoch = CONFIG["real-n-train-samples"] / (CONFIG["batch-size"] * CONFIG["n-workers"]) 

671 params = { 

672 "optimizer": { 

673 "type": "sgdw", 

674 "sgdw": { 

675 "momentum": min(0.9 * initial_lr, 0.999), 

676 "nesterov": False, 

677 "weight_decay": 5e-4, 

678 }, 

679 }, 

680 "learning_rate": { 

681 # with ReduceLROnPlateau callback 

682 "type": "constant", 

683 "constant": { 

684 "learning_rate": initial_lr, 

685 } 

686 }, 

687 } 

688 

689 The tensorboard logs can be found in: https://tensorboard.dev/experiment/HYJTPiowRMa36VZHDLJqdg/ 

690 The model is saved based on best ``epoch_embeddings_embedding_accuracy``, epoch 51 

691 """ 

692 

693 def __init__(self, memory_demanding=False): 

694 urls = [ 

695 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/arcface_iresnet100_msceleb_idiap-1b22d544.tar.gz", 

696 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/arcface_iresnet100_msceleb_idiap-1b22d544.tar.gz", 

697 ] 

698 

699 checkpoint_path = download_file( 

700 urls=urls, 

701 destination_sub_directory="data/tensorflow/arcface_iresnet100_msceleb_idiap-1b22d544", 

702 destination_filename="arcface_iresnet100_msceleb_idiap-1b22d544.tar.gz", 

703 checksum="1b22d544", 

704 extract=True, 

705 ) 

706 

707 super().__init__( 

708 checkpoint_path, 

709 preprocessor=lambda X: X / 255.0, 

710 memory_demanding=memory_demanding, 

711 ) 

712 

713 

714class Resnet50_VGG2_ArcFace_2021(TensorflowTransformer): 

715 """ 

716 Resnet50 Backbone trained with the VGG2 database. 

717 

718 The bottleneck layer (a.k.a embedding) has 512d. 

719 

720 The configuration file used to trained is: 

721 

722 .. warning:: 

723 This configuration file might change in future releases 

724 

725 ```yaml 

726 batch-size: 128 

727 face-size: 112 

728 face-output_size: 112 

729 n-classes: 8631 

730 

731 

732 # Backbone 

733 backbone: 'resnet50' 

734 head: 'arcface' 

735 s: 64 

736 bottleneck: 512 

737 m: 0.5 

738 

739 # Training parameters 

740 solver: "sgd" 

741 lr: 0.1 

742 dropout-rate: 0.5 

743 epochs: 1047 

744 

745 

746 train-tf-record-path: "<PATH>" 

747 validation-tf-record-path: "<PATH>" 

748 

749 ``` 

750 

751 

752 """ 

753 

754 def __init__(self, memory_demanding=False, **kwargs): 

755 urls = [ 

756 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_vgg2_arcface_2021.tar.gz", 

757 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/resnet50_vgg2_arcface_2021.tar.gz", 

758 ] 

759 

760 checkpoint_path = download_file( 

761 urls=urls, 

762 destination_sub_directory="data/tensorflow/resnet50_vgg2_arcface_2021", 

763 destination_filename="resnet50_vgg2_arcface_2021.tar.gz", 

764 checksum="64f89c8cb55e7a0d9c7e13ff412b6a13", 

765 checksum_fct=md5_hash, 

766 extract=True, 

767 ) 

768 

769 super(Resnet50_VGG2_ArcFace_2021, self).__init__( 

770 checkpoint_path, 

771 preprocessor=lambda X: X / 255.0, 

772 memory_demanding=memory_demanding, 

773 **kwargs, 

774 ) 

775 

776 def inference(self, X): 

777 if self.preprocessor is not None: 

778 X = self.preprocessor(tf.cast(X, "float32")) 

779 

780 prelogits = self.model.predict_on_batch(X) 

781 embeddings = tf.math.l2_normalize(prelogits, axis=-1) 

782 return embeddings 

783 

784 

785class MobileNetv2_MsCeleb_ArcFace_2021(TensorflowTransformer): 

786 """ 

787 MobileNet Backbone trained with the MSCeleb 1M database. 

788 

789 The bottleneck layer (a.k.a embedding) has 512d. 

790 

791 The configuration file used to trained is: 

792 

793 .. warning:: 

794 This configuration file might change in future releases 

795 

796 ```yaml 

797 batch-size: 128 

798 face-size: 112 

799 face-output_size: 112 

800 n-classes: 85742 

801 

802 

803 # Backbone 

804 backbone: 'mobilenet-v2' 

805 head: 'arcface' 

806 s: 10 

807 bottleneck: 512 

808 m: 0.5 

809 

810 # Training parameters 

811 solver: "sgd" 

812 lr: 0.01 

813 dropout-rate: 0.5 

814 epochs: 500 

815 

816 

817 train-tf-record-path: "<PATH>" 

818 validation-tf-record-path: "<PATH>" 

819 

820 ``` 

821 

822 

823 """ 

824 

825 def __init__(self, memory_demanding=False, **kwargs): 

826 urls = [ 

827 "https://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/mobilenet-v2-msceleb-arcface-2021-e012cb66.tar.gz", 

828 "http://www.idiap.ch/software/bob/data/bob/bob.bio.face/master/tf2/mobilenet-v2-msceleb-arcface-2021-e012cb66.tar.gz", 

829 ] 

830 

831 checkpoint_path = download_file( 

832 urls=urls, 

833 destination_sub_directory="data/tensorflow/mobilenet-v2-msceleb-arcface-2021-e012cb66", 

834 destination_filename="mobilenet-v2-msceleb-arcface-2021-e012cb66.tar.gz", 

835 checksum="dd1399b86f01725c7b07b480b703e02a", 

836 checksum_fct=md5_hash, 

837 extract=True, 

838 ) 

839 

840 super(MobileNetv2_MsCeleb_ArcFace_2021, self).__init__( 

841 checkpoint_path, 

842 preprocessor=lambda X: X / 255.0, 

843 memory_demanding=memory_demanding, 

844 **kwargs, 

845 ) 

846 

847 

848def facenet_template(embedding, annotation_type, fixed_positions=None): 

849 """ 

850 Facenet baseline template. 

851 This one will crop the face at :math:`160 \\times 160` 

852 

853 Parameters 

854 ---------- 

855 

856 embedding: obj 

857 Transformer that takes a cropped face and extract the embeddings 

858 

859 annotation_type: str 

860 Type of the annotations (e.g. `eyes-center') 

861 

862 fixed_positions: dict 

863 Set it if in your face images are registered to a fixed position in the image 

864 """ 

865 # DEFINE CROPPING 

866 cropped_image_size = (160, 160) 

867 

868 if annotation_type == "eyes-center" or annotation_type == "bounding-box": 

869 # Hard coding eye positions for backward consistency 

870 # cropped_positions = { 

871 cropped_positions = dnn_default_cropping( 

872 cropped_image_size, annotation_type="eyes-center" 

873 ) 

874 if annotation_type == "bounding-box": 

875 # This will allow us to use `BoundingBoxAnnotatorCrop` 

876 cropped_positions.update( 

877 {"topleft": (0, 0), "bottomright": cropped_image_size} 

878 ) 

879 

880 else: 

881 cropped_positions = dnn_default_cropping( 

882 cropped_image_size, annotation_type 

883 ) 

884 

885 annotator = MTCNN(min_size=40, factor=0.709, thresholds=(0.1, 0.2, 0.2)) 

886 

887 # ASSEMBLE TRANSFORMER 

888 transformer = embedding_transformer( 

889 cropped_image_size=cropped_image_size, 

890 embedding=embedding, 

891 cropped_positions=cropped_positions, 

892 fixed_positions=fixed_positions, 

893 color_channel="rgb", 

894 annotator=annotator, 

895 ) 

896 

897 algorithm = Distance() 

898 

899 return PipelineSimple(transformer, algorithm) 

900 

901 

902def resnet_template(embedding, annotation_type, fixed_positions=None): 

903 # DEFINE CROPPING 

904 # cropped_image_size = (112, 112) 

905 # if annotation_type == "eyes-center": 

906 # # Hard coding eye positions for backward consistency 

907 # cropped_positions = cropped_positions_arcface() 

908 # else: 

909 # cropped_positions = dnn_default_cropping(cropped_image_size, annotation_type) 

910 # DEFINE CROPPING 

911 cropped_image_size = (112, 112) 

912 if annotation_type == "eyes-center" or annotation_type == "bounding-box": 

913 # Hard coding eye positions for backward consistency 

914 # cropped_positions = { 

915 cropped_positions = cropped_positions_arcface() 

916 if annotation_type == "bounding-box": 

917 # This will allow us to use `BoundingBoxAnnotatorCrop` 

918 cropped_positions.update( 

919 {"topleft": (0, 0), "bottomright": cropped_image_size} 

920 ) 

921 

922 else: 

923 cropped_positions = dnn_default_cropping( 

924 cropped_image_size, annotation_type 

925 ) 

926 

927 annotator = MTCNN(min_size=40, factor=0.709, thresholds=(0.1, 0.2, 0.2)) 

928 transformer = embedding_transformer( 

929 cropped_image_size=cropped_image_size, 

930 embedding=embedding, 

931 cropped_positions=cropped_positions, 

932 fixed_positions=fixed_positions, 

933 color_channel="rgb", 

934 annotator=annotator, 

935 ) 

936 

937 algorithm = Distance() 

938 

939 return PipelineSimple(transformer, algorithm) 

940 

941 

942def resnet50_msceleb_arcface_2021( 

943 annotation_type, fixed_positions=None, memory_demanding=False 

944): 

945 """ 

946 Get the Resnet50 pipeline which will crop the face :math:`112 \\times 112` and 

947 use the :py:class:`Resnet50_MsCeleb_ArcFace_2021` to extract the features 

948 

949 Parameters 

950 ---------- 

951 

952 annotation_type: str 

953 Type of the annotations (e.g. `eyes-center') 

954 

955 fixed_positions: dict 

956 Set it if in your face images are registered to a fixed position in the image 

957 

958 memory_demanding: bool 

959 

960 """ 

961 

962 return resnet_template( 

963 embedding=Resnet50_MsCeleb_ArcFace_2021( 

964 memory_demanding=memory_demanding 

965 ), 

966 annotation_type=annotation_type, 

967 fixed_positions=fixed_positions, 

968 ) 

969 

970 

971def resnet50_msceleb_arcface_20210521( 

972 annotation_type, fixed_positions=None, memory_demanding=False 

973): 

974 """ 

975 Get the Resnet50 pipeline which will crop the face :math:`112 \\times 112` and 

976 use the :py:class:`Resnet50_MsCeleb_ArcFace_20210521` to extract the features 

977 

978 Parameters 

979 ---------- 

980 

981 annotation_type: str 

982 Type of the annotations (e.g. `eyes-center') 

983 

984 fixed_positions: dict 

985 Set it if in your face images are registered to a fixed position in the image 

986 

987 memory_demanding: bool 

988 

989 """ 

990 

991 return resnet_template( 

992 embedding=Resnet50_MsCeleb_ArcFace_20210521( 

993 memory_demanding=memory_demanding 

994 ), 

995 annotation_type=annotation_type, 

996 fixed_positions=fixed_positions, 

997 ) 

998 

999 

1000def resnet101_msceleb_arcface_20210521( 

1001 annotation_type, fixed_positions=None, memory_demanding=False 

1002): 

1003 """ 

1004 Get the Resnet50 pipeline which will crop the face :math:`112 \\times 112` and 

1005 use the :py:class:`Resnet50_MsCeleb_ArcFace_20210521` to extract the features 

1006 

1007 Parameters 

1008 ---------- 

1009 

1010 annotation_type: str 

1011 Type of the annotations (e.g. `eyes-center') 

1012 

1013 fixed_positions: dict 

1014 Set it if in your face images are registered to a fixed position in the image 

1015 

1016 memory_demanding: bool 

1017 

1018 """ 

1019 

1020 return resnet_template( 

1021 embedding=Resnet101_MsCeleb_ArcFace_20210521( 

1022 memory_demanding=memory_demanding 

1023 ), 

1024 annotation_type=annotation_type, 

1025 fixed_positions=fixed_positions, 

1026 ) 

1027 

1028 

1029def iresnet50_msceleb_arcface_20210623( 

1030 annotation_type, fixed_positions=None, memory_demanding=False 

1031): 

1032 """ 

1033 Get the iresnet50 pipeline which will crop the face :math:`112 \\times 112` and 

1034 use the :py:class:`IResnet50_MsCeleb_ArcFace_20210623` to extract the features 

1035 

1036 Parameters 

1037 ---------- 

1038 

1039 annotation_type: str 

1040 Type of the annotations (e.g. `eyes-center') 

1041 

1042 fixed_positions: dict 

1043 Set it if in your face images are registered to a fixed position in the image 

1044 

1045 memory_demanding: bool 

1046 

1047 """ 

1048 return resnet_template( 

1049 embedding=IResnet50_MsCeleb_ArcFace_20210623( 

1050 memory_demanding=memory_demanding 

1051 ), 

1052 annotation_type=annotation_type, 

1053 fixed_positions=fixed_positions, 

1054 ) 

1055 

1056 

1057def iresnet100_msceleb_arcface_20210623( 

1058 annotation_type, fixed_positions=None, memory_demanding=False 

1059): 

1060 """ 

1061 Get the iresnet100 pipeline which will crop the face :math:`112 \\times 112` and 

1062 use the :py:class:`IResnet100_MsCeleb_ArcFace_20210623` to extract the features 

1063 

1064 Parameters 

1065 ---------- 

1066 

1067 annotation_type: str 

1068 Type of the annotations (e.g. `eyes-center') 

1069 

1070 fixed_positions: dict 

1071 Set it if in your face images are registered to a fixed position in the image 

1072 

1073 memory_demanding: bool 

1074 

1075 """ 

1076 return resnet_template( 

1077 embedding=IResnet100_MsCeleb_ArcFace_20210623( 

1078 memory_demanding=memory_demanding 

1079 ), 

1080 annotation_type=annotation_type, 

1081 fixed_positions=fixed_positions, 

1082 ) 

1083 

1084 

1085def resnet50_vgg2_arcface_2021( 

1086 annotation_type, fixed_positions=None, memory_demanding=False 

1087): 

1088 """ 

1089 Get the Resnet50 pipeline which will crop the face :math:`112 \\times 112` and 

1090 use the :py:class:`Resnet50_VGG2_ArcFace_2021` to extract the features 

1091 

1092 Parameters 

1093 ---------- 

1094 

1095 annotation_type: str 

1096 Type of the annotations (e.g. `eyes-center') 

1097 

1098 fixed_positions: dict