Coverage for src/bob/bio/face/embeddings/opencv.py: 83%

1#!/usr/bin/env python 

2# vim: set fileencoding=utf-8 : 

3# Yu Linghu & Xinyi Zhang <yu.linghu@uzh.ch, xinyi.zhang@uzh.ch> 

4# Tiago de Freitas Pereira <tiago.pereira@idiap.ch> 

5 

6import os 

7 

8from sklearn.base import BaseEstimator, TransformerMixin 

9from sklearn.utils import check_array 

10 

11from bob.bio.base.algorithm import Distance 

12from bob.bio.base.database.utils import download_file, md5_hash 

13from bob.bio.base.pipelines import PipelineSimple 

14from bob.bio.face.annotator import MTCNN 

15from bob.bio.face.utils import dnn_default_cropping, embedding_transformer 

16 

17 

18class OpenCVTransformer(TransformerMixin, BaseEstimator): 

19 """ 

20 Base Transformer using the OpenCV DNN interface (https://docs.opencv.org/master/d2/d58/tutorial_table_of_content_dnn.html). 

21 

22 

23 .. note:: 

24 This class supports Caffe ``.caffemodel``, Tensorflow ``.pb``, Torch ``.t7`` ``.net``, Darknet ``.weights``, DLDT ``.bin``, and ONNX ``.onnx`` 

25 

26 

27 Parameters 

28 ---------- 

29 

30 checkpoint_path: str 

31 Path containing the checkpoint 

32 

33 config: 

34 Path containing some configuration file (e.g. .json, .prototxt) 

35 

36 preprocessor: 

37 A function that will transform the data right before forward. The default transformation is `X/255` 

38 

39 """ 

40 

41 def __init__( 

42 self, 

43 checkpoint_path=None, 

44 config=None, 

45 preprocessor=lambda x: x / 255, 

46 **kwargs, 

47 ): 

48 super().__init__(**kwargs) 

49 self.checkpoint_path = checkpoint_path 

50 self.config = config 

51 self.model = None 

52 self.preprocessor = preprocessor 

53 

54 def _load_model(self): 

55 import cv2 

56 

57 net = cv2.dnn.readNet(self.checkpoint_path, self.config) 

58 self.model = net 

59 

60 def transform(self, X): 

61 """__call__(image) -> feature 

62 

63 Extracts the features from the given image. 

64 

65 **Parameters:** 

66 

67 X : 2D :py:class:`numpy.ndarray` (floats) 

68 The image to extract the features from. 

69 

70 **Returns:** 

71 

72 feature : 2D or 3D :py:class:`numpy.ndarray` (floats) 

73 The list of features extracted from the image. 

74 """ 

75 

76 if self.model is None: 

77 self._load_model() 

78 

79 X = check_array(X, allow_nd=True) 

80 

81 X = self.preprocessor(X) 

82 

83 self.model.setInput(X) 

84 

85 return self.model.forward() 

86 

87 def __getstate__(self): 

88 # Handling unpicklable objects 

89 

90 d = self.__dict__.copy() 

91 d["model"] = None 

92 return d 

93 

94 def _more_tags(self): 

95 return {"requires_fit": False} 

96 

97 

98class VGG16_Oxford(OpenCVTransformer): 

99 """ 

100 Original VGG16 model from the paper: https://www.robots.ox.ac.uk/~vgg/publications/2015/Parkhi15/parkhi15.pdf 

101 

102 """ 

103 

104 def __init__(self, embedding_layer="fc7"): 

105 urls = [ 

106 "https://www.robots.ox.ac.uk/~vgg/software/vgg_face/src/vgg_face_caffe.tar.gz", 

107 "http://bobconda.lab.idiap.ch/public-upload/data/bob/bob.bio.face/master/caffe/vgg_face_caffe.tar.gz", 

108 ] 

109 

110 path = download_file( 

111 urls=urls, 

112 destination_sub_directory="data/caffe/vgg_face_caffe", 

113 destination_filename="vgg_face_caffe.tar.gz", 

114 checksum="ee707ac6e890bc148cb155adeaad12be", 

115 checksum_fct=md5_hash, 

116 extract=True, 

117 ) 

118 config = os.path.join( 

119 path, "vgg_face_caffe", "VGG_FACE_deploy.prototxt" 

120 ) 

121 checkpoint_path = os.path.join( 

122 path, "vgg_face_caffe", "VGG_FACE.caffemodel" 

123 ) 

124 

125 caffe_average_img = [129.1863, 104.7624, 93.5940] 

126 self.embedding_layer = embedding_layer 

127 

128 def preprocessor(X): 

129 """ 

130 Normalize using data from caffe 

131 

132 Caffe has the shape `C x H x W` and the chanel is BGR and 

133 

134 """ 

135 

136 # Subtracting 

137 X[:, 0, :, :] -= caffe_average_img[0] 

138 X[:, 1, :, :] -= caffe_average_img[1] 

139 X[:, 2, :, :] -= caffe_average_img[2] 

140 

141 # To BGR 

142 X = X[:, ::-1, :, :].astype("float32") 

143 

144 return X 

145 

146 super(VGG16_Oxford, self).__init__( 

147 checkpoint_path, config, preprocessor 

148 ) 

149 

150 def _load_model(self): 

151 import cv2 

152 

153 net = cv2.dnn.readNet(self.checkpoint_path, self.config) 

154 self.model = net 

155 

156 def transform(self, X): 

157 if self.model is None: 

158 self._load_model() 

159 

160 X = check_array(X, allow_nd=True) 

161 

162 X = self.preprocessor(X) 

163 

164 self.model.setInput(X) 

165 

166 return self.model.forward(self.embedding_layer) 

167 

168 

169def vgg16_oxford_baseline(annotation_type, fixed_positions=None): 

170 """ 

171 Get the VGG16 pipeline which will crop the face :math:`224 \\times 224` 

172 use the :py:class:`VGG16_Oxford` 

173 

174 Parameters 

175 ---------- 

176 

177 annotation_type: str 

178 Type of the annotations (e.g. `eyes-center') 

179 

180 fixed_positions: dict 

181 Set it if in your face images are registered to a fixed position in the image 

182 """ 

183 

184 # DEFINE CROPPING 

185 cropped_image_size = (224, 224) 

186 

187 if annotation_type == "eyes-center" or annotation_type == "bounding-box": 

188 # Hard coding eye positions for backward consistency 

189 # cropped_positions = { 

190 cropped_positions = {"reye": (112, 82), "leye": (112, 142)} 

191 if annotation_type == "bounding-box": 

192 # This will allow us to use `BoundingBoxAnnotatorCrop` 

193 cropped_positions.update( 

194 {"topleft": (0, 0), "bottomright": cropped_image_size} 

195 ) 

196 else: 

197 cropped_positions = dnn_default_cropping( 

198 cropped_image_size, annotation_type 

199 ) 

200 

201 annotator = MTCNN(min_size=40, factor=0.709, thresholds=(0.1, 0.2, 0.2)) 

202 transformer = embedding_transformer( 

203 cropped_image_size=cropped_image_size, 

204 embedding=VGG16_Oxford(), 

205 cropped_positions=cropped_positions, 

206 fixed_positions=fixed_positions, 

207 color_channel="rgb", 

208 annotator=annotator, 

209 ) 

210 

211 algorithm = Distance() 

212 

213 return PipelineSimple(transformer, algorithm)