1 /*
2 By downloading, copying, installing or using the software you agree to this
3 license. If you do not agree to this license, do not download, install,
4 copy or use the software.
5 License Agreement
6 For Open Source Computer Vision Library
7 (3-clause BSD License)
8 Copyright (C) 2013, OpenCV Foundation, all rights reserved.
9 Third party copyrights are property of their respective owners.
10 Redistribution and use in source and binary forms, with or without modification,
11 are permitted provided that the following conditions are met:
12 * Redistributions of source code must retain the above copyright notice,
13 this list of conditions and the following disclaimer.
14 * Redistributions in binary form must reproduce the above copyright notice,
15 this list of conditions and the following disclaimer in the documentation
16 and/or other materials provided with the distribution.
17 * Neither the names of the copyright holders nor the names of the contributors
18 may be used to endorse or promote products derived from this software
19 without specific prior written permission.
20 This software is provided by the copyright holders and contributors "as is" and
21 any express or implied warranties, including, but not limited to, the implied
22 warranties of merchantability and fitness for a particular purpose are
23 disclaimed. In no event shall copyright holders or contributors be liable for
24 any direct, indirect, incidental, special, exemplary, or consequential damages
25 (including, but not limited to, procurement of substitute goods or services;
26 loss of use, data, or profits; or business interruption) however caused
27 and on any theory of liability, whether in contract, strict liability,
28 or tort (including negligence or otherwise) arising in any way out of
29 the use of this software, even if advised of the possibility of such damage.
30
31 This file was part of GSoC Project: Facemark API for OpenCV
32 Final report: https://gist.github.com/kurnianggoro/74de9121e122ad0bd825176751d47ecc
33 Student: Laksono Kurnianggoro
34 Mentor: Delia Passalacqua
35 */
36
37 #include "precomp.hpp"
38 #include "opencv2/face.hpp"
39 #include <fstream>
40 #include <cmath>
41 #include <ctime>
42 #include <cstdio>
43 #include <cstdarg>
44
45 namespace cv {
46 namespace face {
47
48 #define TIMER_BEGIN { double __time__ = (double)getTickCount();
49 #define TIMER_NOW ((getTickCount() - __time__) / getTickFrequency())
50 #define TIMER_END }
51
52 #define SIMILARITY_TRANSFORM(x, y, scale, rotate) do { \
53 double x_tmp = scale * (rotate(0, 0)*x + rotate(0, 1)*y); \
54 double y_tmp = scale * (rotate(1, 0)*x + rotate(1, 1)*y); \
55 x = x_tmp; y = y_tmp; \
56 } while(0)
57
Params()58 FacemarkLBF::Params::Params(){
59
60 cascade_face = "";
61 shape_offset = 0.0;
62 n_landmarks = 68;
63 initShape_n = 10;
64 stages_n=5;
65 tree_n=6;
66 tree_depth=5;
67 bagging_overlap = 0.4;
68 model_filename = "";
69 save_model = true;
70 verbose = true;
71 seed = 0;
72
73 int _pupils[][6] = { { 36, 37, 38, 39, 40, 41 }, { 42, 43, 44, 45, 46, 47 } };
74 for (int i = 0; i < 6; i++) {
75 pupils[0].push_back(_pupils[0][i]);
76 pupils[1].push_back(_pupils[1][i]);
77 }
78
79 int _feats_m[] = { 500, 500, 500, 300, 300, 300, 200, 200, 200, 100 };
80 double _radius_m[] = { 0.3, 0.2, 0.15, 0.12, 0.10, 0.10, 0.08, 0.06, 0.06, 0.05 };
81 for (int i = 0; i < 10; i++) {
82 feats_m.push_back(_feats_m[i]);
83 radius_m.push_back(_radius_m[i]);
84 }
85
86 detectROI = Rect(-1,-1,-1,-1);
87 }
88
read(const cv::FileNode & fn)89 void FacemarkLBF::Params::read( const cv::FileNode& fn ){
90 *this = FacemarkLBF::Params();
91
92 if (!fn["verbose"].empty())
93 fn["verbose"] >> verbose;
94
95 }
96
write(cv::FileStorage & fs) const97 void FacemarkLBF::Params::write( cv::FileStorage& fs ) const{
98 fs << "verbose" << verbose;
99 }
100
101 class FacemarkLBFImpl : public FacemarkLBF {
102 public:
103 FacemarkLBFImpl( const FacemarkLBF::Params ¶meters = FacemarkLBF::Params() );
104
105 void read( const FileNode& /*fn*/ ) CV_OVERRIDE;
106 void write( FileStorage& /*fs*/ ) const CV_OVERRIDE;
107
108 void loadModel(String fs) CV_OVERRIDE;
109
110 bool setFaceDetector(bool(*f)(InputArray , OutputArray, void * extra_params ), void* userData) CV_OVERRIDE;
111 bool getFaces(InputArray image, OutputArray faces) CV_OVERRIDE;
112 bool getData(void * items) CV_OVERRIDE;
113
114 Params params;
115
116 protected:
117
118 bool fit(InputArray image, InputArray faces, OutputArrayOfArrays landmarks) CV_OVERRIDE;
119 bool fitImpl( const Mat image, std::vector<Point2f> & landmarks );//!< from a face
120
121 bool addTrainingSample(InputArray image, InputArray landmarks) CV_OVERRIDE;
122 void training(void* parameters) CV_OVERRIDE;
123
124 Rect getBBox(Mat &img, const Mat_<double> shape);
125 void prepareTrainingData(Mat img, std::vector<Point2f> facePoints,
126 std::vector<Mat> & cropped, std::vector<Mat> & shapes, std::vector<BBox> &boxes);
127 void data_augmentation(std::vector<Mat> &imgs, std::vector<Mat> >_shapes, std::vector<BBox> &bboxes);
128 Mat getMeanShape(std::vector<Mat> >_shapes, std::vector<BBox> &bboxes);
129
130 bool defaultFaceDetector(const Mat& image, std::vector<Rect>& faces);
131
132 CascadeClassifier face_cascade;
133 FN_FaceDetector faceDetector;
134 void* faceDetectorData;
135
136 /*training data*/
137 std::vector<std::vector<Point2f> > data_facemarks; //original position
138 std::vector<Mat> data_faces; //face ROI
139 std::vector<BBox> data_boxes;
140 std::vector<Mat> data_shapes; //position in the face ROI
141
142 private:
143 bool isModelTrained;
144
145 /*---------------LBF Class---------------------*/
146 class LBF {
147 public:
148 void calcSimilarityTransform(const Mat &shape1, const Mat &shape2, double &scale, Mat &rotate);
149 std::vector<Mat> getDeltaShapes(std::vector<Mat> >_shapes, std::vector<Mat> ¤t_shapes,
150 std::vector<BBox> &bboxes, Mat &mean_shape);
151 double calcVariance(const Mat &vec);
152 double calcVariance(const std::vector<double> &vec);
153 double calcMeanError(std::vector<Mat> >_shapes, std::vector<Mat> ¤t_shapes, int landmark_n , std::vector<int> &left, std::vector<int> &right );
154
155 };
156
157 /*---------------RandomTree Class---------------------*/
158 class RandomTree : public LBF {
159 public:
RandomTree()160 RandomTree(){};
~RandomTree()161 ~RandomTree(){};
162
163 void initTree(int landmark_id, int depth, std::vector<int>, std::vector<double>);
164 void train(std::vector<Mat> &imgs, std::vector<Mat> ¤t_shapes, std::vector<BBox> &bboxes,
165 std::vector<Mat> &delta_shapes, Mat &mean_shape, std::vector<int> &index, int stage);
166 void splitNode(std::vector<cv::Mat> &imgs, std::vector<cv::Mat> ¤t_shapes, std::vector<BBox> &bboxes,
167 cv::Mat &delta_shapes, cv::Mat &mean_shape, std::vector<int> &root, int idx, int stage);
168
169 void write(FileStorage fs, int forestId, int i, int j);
170 void read(FileStorage fs, int forestId, int i, int j);
171
172 int depth;
173 int nodes_n;
174 int landmark_id;
175 cv::Mat_<double> feats;
176 std::vector<int> thresholds;
177
178 std::vector<int> params_feats_m;
179 std::vector<double> params_radius_m;
180 };
181 /*---------------RandomForest Class---------------------*/
182 class RandomForest : public LBF {
183 public:
RandomForest()184 RandomForest(){};
~RandomForest()185 ~RandomForest(){};
186
187 void initForest(int landmark_n, int trees_n, int tree_depth, double , std::vector<int>, std::vector<double>, bool);
188 void train(std::vector<cv::Mat> &imgs, std::vector<cv::Mat> ¤t_shapes, \
189 std::vector<BBox> &bboxes, std::vector<cv::Mat> &delta_shapes, cv::Mat &mean_shape, int stage);
190 Mat generateLBF(Mat &img, Mat ¤t_shape, BBox &bbox, Mat &mean_shape);
191
192 void write(FileStorage fs, int forestId);
193 void read(FileStorage fs, int forestId);
194
195 bool verbose;
196 int landmark_n;
197 int trees_n, tree_depth;
198 double overlap_ratio;
199 std::vector<std::vector<RandomTree> > random_trees;
200
201 std::vector<int> feats_m;
202 std::vector<double> radius_m;
203 };
204 /*---------------Regressor Class---------------------*/
205 class Regressor : public LBF {
206 protected:
207 struct feature_node{
208 int index;
209 double value;
210 };
211 public:
Regressor()212 Regressor(){};
~Regressor()213 ~Regressor(){};
214
215 void initRegressor(Params);
216 void trainRegressor(std::vector<cv::Mat> &imgs, std::vector<cv::Mat> >_shapes, \
217 std::vector<cv::Mat> ¤t_shapes, std::vector<BBox> &bboxes, \
218 cv::Mat &mean_shape, int start_from, Params );
219 Mat globalRegressionPredict(const Mat &lbf, int stage);
220 Mat predict(Mat &img, BBox &bbox);
221
222 void write(FileStorage fs, Params config);
223 void read(FileStorage fs, Params & config);
224
225 void globalRegressionTrain(
226 std::vector<Mat> &lbfs, std::vector<Mat> &delta_shapes,
227 int stage, Params config
228 );
229
230 Mat supportVectorRegression(
231 feature_node **x, double *y, int nsamples, int feat_size, bool verbose=0
232 );
233
234 int stages_n;
235 int landmark_n;
236 cv::Mat mean_shape;
237 std::vector<RandomForest> random_forests;
238 std::vector<cv::Mat> gl_regression_weights;
239
240 }; // LBF
241
242 Regressor regressor;
243 }; // class
244
245 /*
246 * Constructor
247 */
create(const FacemarkLBF::Params & parameters)248 Ptr<FacemarkLBF> FacemarkLBF::create(const FacemarkLBF::Params ¶meters){
249 return Ptr<FacemarkLBFImpl>(new FacemarkLBFImpl(parameters));
250 }
251 /*
252 * Constructor
253 */
createFacemarkLBF()254 Ptr<Facemark> createFacemarkLBF(){
255 const FacemarkLBF::Params parameters;
256 return Ptr<FacemarkLBFImpl>(new FacemarkLBFImpl(parameters));
257 }
258
FacemarkLBFImpl(const FacemarkLBF::Params & parameters)259 FacemarkLBFImpl::FacemarkLBFImpl( const FacemarkLBF::Params ¶meters ) :
260 faceDetector(NULL), faceDetectorData(NULL)
261 {
262 isModelTrained = false;
263 params = parameters;
264 }
265
setFaceDetector(bool (* f)(InputArray,OutputArray,void * extra_params),void * userData)266 bool FacemarkLBFImpl::setFaceDetector(bool(*f)(InputArray , OutputArray, void * extra_params ), void* userData){
267 faceDetector = f;
268 faceDetectorData = userData;
269 return true;
270 }
271
getFaces(InputArray image,OutputArray faces_)272 bool FacemarkLBFImpl::getFaces(InputArray image, OutputArray faces_)
273 {
274 if (!faceDetector)
275 {
276 std::vector<Rect> faces;
277 defaultFaceDetector(image.getMat(), faces);
278 Mat(faces).copyTo(faces_);
279 return true;
280 }
281 return faceDetector(image, faces_, faceDetectorData);
282 }
283
defaultFaceDetector(const Mat & image,std::vector<Rect> & faces)284 bool FacemarkLBFImpl::defaultFaceDetector(const Mat& image, std::vector<Rect>& faces){
285 Mat gray;
286
287 faces.clear();
288
289 if (image.channels() > 1)
290 {
291 cvtColor(image, gray, COLOR_BGR2GRAY);
292 }
293 else
294 {
295 gray = image;
296 }
297
298 equalizeHist(gray, gray);
299
300 if (face_cascade.empty())
301 {
302 { /* check the cascade classifier file */
303 std::ifstream infile;
304 infile.open(params.cascade_face.c_str(), std::ios::in);
305 if (!infile)
306 CV_Error_(Error::StsBadArg, ("The cascade classifier model is not found: %s", params.cascade_face.c_str()));
307 }
308 face_cascade.load(params.cascade_face.c_str());
309 CV_Assert(!face_cascade.empty());
310 }
311 face_cascade.detectMultiScale(gray, faces, 1.05, 2, CASCADE_SCALE_IMAGE, Size(30, 30) );
312 return true;
313 }
314
getData(void * items)315 bool FacemarkLBFImpl::getData(void * items){
316 CV_UNUSED(items);
317 return false;
318 }
319
addTrainingSample(InputArray image,InputArray landmarks)320 bool FacemarkLBFImpl::addTrainingSample(InputArray image, InputArray landmarks){
321 // FIXIT
322 std::vector<Point2f> & _landmarks = *(std::vector<Point2f>*)landmarks.getObj();
323 prepareTrainingData(image.getMat(), _landmarks, data_faces, data_shapes, data_boxes);
324 return true;
325 }
326
training(void * parameters)327 void FacemarkLBFImpl::training(void* parameters){
328 CV_UNUSED(parameters);
329
330 if (data_faces.empty())
331 {
332 CV_Error(Error::StsBadArg, "Training data is not provided. Consider to add using addTrainingSample() function!");
333 }
334
335 if (params.model_filename.empty() && params.save_model)
336 {
337 CV_Error(Error::StsBadArg, "The parameter model_filename should be set!");
338 }
339
340 // flip the image and swap the landmark position
341 data_augmentation(data_faces, data_shapes, data_boxes);
342
343 Mat mean_shape = getMeanShape(data_shapes, data_boxes);
344
345 int N = (int)data_faces.size();
346 int L = N*params.initShape_n;
347 std::vector<Mat> imgs(L), gt_shapes(L), current_shapes(L);
348 std::vector<BBox> bboxes(L);
349 RNG rng(params.seed);
350 for (int i = 0; i < N; i++) {
351 for (int j = 0; j < params.initShape_n; j++) {
352 int idx = i*params.initShape_n + j;
353 int k = rng.uniform(0, N - 1);
354 k = (k >= i) ? k + 1 : k; // require k != i
355 imgs[idx] = data_faces[i];
356 gt_shapes[idx] = data_shapes[i];
357 bboxes[idx] = data_boxes[i];
358 current_shapes[idx] = data_boxes[i].reproject(data_boxes[k].project(data_shapes[k]));
359 }
360 }
361
362 regressor.initRegressor(params);
363 regressor.trainRegressor(imgs, gt_shapes, current_shapes, bboxes, mean_shape, 0, params);
364
365 if(params.save_model){
366 FileStorage fs(params.model_filename.c_str(),FileStorage::WRITE_BASE64);
367 regressor.write(fs, params);
368 }
369
370 isModelTrained = true;
371 }
372
373 /**
374 * @brief Copy the contents of a corners vector to an OutputArray, settings its size.
375 */
_copyVector2Output(std::vector<std::vector<Point2f>> & vec,OutputArrayOfArrays out)376 static void _copyVector2Output(std::vector< std::vector< Point2f > > &vec, OutputArrayOfArrays out)
377 {
378 out.create((int)vec.size(), 1, CV_32FC2);
379
380 if (out.isMatVector()) {
381 for (unsigned int i = 0; i < vec.size(); i++) {
382 out.create(68, 1, CV_32FC2, i);
383 Mat &m = out.getMatRef(i);
384 Mat(Mat(vec[i]).t()).copyTo(m);
385 }
386 }
387 else if (out.isUMatVector()) {
388 for (unsigned int i = 0; i < vec.size(); i++) {
389 out.create(68, 1, CV_32FC2, i);
390 UMat &m = out.getUMatRef(i);
391 Mat(Mat(vec[i]).t()).copyTo(m);
392 }
393 }
394 else if (out.kind() == _OutputArray::STD_VECTOR_VECTOR) {
395 for (unsigned int i = 0; i < vec.size(); i++) {
396 out.create(68, 1, CV_32FC2, i);
397 Mat m = out.getMat(i);
398 Mat(Mat(vec[i]).t()).copyTo(m);
399 }
400 }
401 else {
402 CV_Error(cv::Error::StsNotImplemented,
403 "Only Mat vector, UMat vector, and vector<vector> OutputArrays are currently supported.");
404 }
405 }
406
fit(InputArray image,InputArray roi,OutputArrayOfArrays _landmarks)407 bool FacemarkLBFImpl::fit(InputArray image, InputArray roi, OutputArrayOfArrays _landmarks)
408 {
409 Mat roimat = roi.getMat();
410 std::vector<Rect> faces = roimat.reshape(4, roimat.rows);
411 if (faces.empty()) return false;
412
413 std::vector<std::vector<Point2f> > landmarks;
414
415 landmarks.resize(faces.size());
416
417 for(unsigned i=0; i<faces.size();i++){
418 params.detectROI = faces[i];
419 fitImpl(image.getMat(), landmarks[i]);
420 }
421 _copyVector2Output(landmarks, _landmarks);
422 return true;
423 }
424
fitImpl(const Mat image,std::vector<Point2f> & landmarks)425 bool FacemarkLBFImpl::fitImpl( const Mat image, std::vector<Point2f>& landmarks){
426 if (landmarks.size()>0)
427 landmarks.clear();
428
429 if (!isModelTrained) {
430 CV_Error(Error::StsBadArg, "The LBF model is not trained yet. Please provide a trained model.");
431 }
432
433 Mat img;
434 if(image.channels()>1){
435 cvtColor(image,img,COLOR_BGR2GRAY);
436 }else{
437 img = image;
438 }
439
440 Rect box;
441 if (params.detectROI.width>0){
442 box = params.detectROI;
443 }else{
444 std::vector<Rect> rects;
445
446 if (!getFaces(img, rects)) return 0;
447 if (rects.empty()) return 0; //failed to get face
448 box = rects[0];
449 }
450
451 double min_x, min_y, max_x, max_y;
452 min_x = std::max(0., (double)box.x - box.width / 2);
453 max_x = std::min(img.cols - 1., (double)box.x+box.width + box.width / 2);
454 min_y = std::max(0., (double)box.y - box.height / 2);
455 max_y = std::min(img.rows - 1., (double)box.y + box.height + box.height / 2);
456
457 double w = max_x - min_x;
458 double h = max_y - min_y;
459
460 BBox bbox(box.x - min_x, box.y - min_y, box.width, box.height);
461 Mat crop = img(Rect((int)min_x, (int)min_y, (int)w, (int)h)).clone();
462 Mat shape = regressor.predict(crop, bbox);
463
464 if(params.detectROI.width>0){
465 landmarks = Mat(shape.reshape(2)+Scalar(min_x, min_y));
466 params.detectROI.width = -1;
467 }else{
468 landmarks = Mat(shape.reshape(2)+Scalar(min_x, min_y));
469 }
470
471 return 1;
472 }
473
read(const cv::FileNode & fn)474 void FacemarkLBFImpl::read( const cv::FileNode& fn ){
475 params.read( fn );
476 }
477
write(cv::FileStorage & fs) const478 void FacemarkLBFImpl::write( cv::FileStorage& fs ) const {
479 params.write( fs );
480 }
481
loadModel(String s)482 void FacemarkLBFImpl::loadModel(String s){
483 if(params.verbose) printf("loading data from : %s\n", s.c_str());
484 std::ifstream infile;
485 infile.open(s.c_str(), std::ios::in);
486 if (!infile) {
487 CV_Error(Error::StsBadArg, "No valid input file was given, please check the given filename.");
488 }
489
490 FileStorage fs(s.c_str(),FileStorage::READ);
491 regressor.read(fs, params);
492
493 isModelTrained = true;
494 }
495
getBBox(Mat & img,const Mat_<double> shape)496 Rect FacemarkLBFImpl::getBBox(Mat &img, const Mat_<double> shape) {
497 std::vector<Rect> rects;
498
499 if(!faceDetector){
500 defaultFaceDetector(img, rects);
501 }else{
502 faceDetector(img, rects, faceDetectorData);
503 }
504
505 if (rects.size() == 0) return Rect(-1, -1, -1, -1);
506 double center_x=0, center_y=0, min_x, max_x, min_y, max_y;
507
508 min_x = shape(0, 0);
509 max_x = shape(0, 0);
510 min_y = shape(0, 1);
511 max_y = shape(0, 1);
512
513 for (int i = 0; i < shape.rows; i++) {
514 center_x += shape(i, 0);
515 center_y += shape(i, 1);
516 min_x = std::min(min_x, shape(i, 0));
517 max_x = std::max(max_x, shape(i, 0));
518 min_y = std::min(min_y, shape(i, 1));
519 max_y = std::max(max_y, shape(i, 1));
520 }
521 center_x /= shape.rows;
522 center_y /= shape.rows;
523
524 for (int i = 0; i < (int)rects.size(); i++) {
525 Rect r = rects[i];
526 if (max_x - min_x > r.width*1.5) continue;
527 if (max_y - min_y > r.height*1.5) continue;
528 if (abs(center_x - (r.x + r.width / 2)) > r.width / 2) continue;
529 if (abs(center_y - (r.y + r.height / 2)) > r.height / 2) continue;
530 return r;
531 }
532 return Rect(-1, -1, -1, -1);
533 }
534
prepareTrainingData(Mat img,std::vector<Point2f> facePoints,std::vector<Mat> & cropped,std::vector<Mat> & shapes,std::vector<BBox> & boxes)535 void FacemarkLBFImpl::prepareTrainingData(Mat img, std::vector<Point2f> facePoints,
536 std::vector<Mat> & cropped, std::vector<Mat> & shapes, std::vector<BBox> &boxes)
537 {
538 Mat shape;
539 Mat _shape = Mat(facePoints).reshape(1);
540 Rect box = getBBox(img, _shape);
541
542 if(img.channels()>1){
543 cvtColor(img,img,COLOR_BGR2GRAY);
544 }
545
546 if(box.x != -1){
547 _shape.convertTo(shape, CV_64FC1);
548 Mat sx = shape.col(0);
549 Mat sy = shape.col(1);
550 double min_x, max_x, min_y, max_y;
551 minMaxIdx(sx, &min_x, &max_x);
552 minMaxIdx(sy, &min_y, &max_y);
553
554 min_x = std::max(0., min_x - (double)box.width / 2.);
555 max_x = std::min(img.cols - 1., max_x + (double)box.width / 2.);
556 min_y = std::max(0., min_y - (double)box.height / 2.);
557 max_y = std::min(img.rows - 1., max_y + (double)box.height / 2.);
558
559 double w = max_x - min_x;
560 double h = max_y - min_y;
561
562 shape = Mat(shape.reshape(2)-Scalar(min_x, min_y)).reshape(1);
563
564 boxes.push_back(BBox(box.x - min_x, box.y - min_y, box.width, box.height));
565 Mat crop = img(Rect((int)min_x, (int)min_y, (int)w, (int)h)).clone();
566 cropped.push_back(crop);
567 shapes.push_back(shape);
568
569 } // if box is valid
570 } // prepareTrainingData
571
data_augmentation(std::vector<Mat> & imgs,std::vector<Mat> & gt_shapes,std::vector<BBox> & bboxes)572 void FacemarkLBFImpl::data_augmentation(std::vector<Mat> &imgs, std::vector<Mat> >_shapes, std::vector<BBox> &bboxes) {
573 int N = (int)imgs.size();
574 imgs.reserve(2 * N);
575 gt_shapes.reserve(2 * N);
576 bboxes.reserve(2 * N);
577 for (int i = 0; i < N; i++) {
578 Mat img_flipped;
579 Mat_<double> gt_shape_flipped(gt_shapes[i].size());
580 flip(imgs[i], img_flipped, 1);
581 int w = img_flipped.cols - 1;
582 // int h = img_flipped.rows - 1;
583 for (int k = 0; k < gt_shapes[i].rows; k++) {
584 gt_shape_flipped(k, 0) = w - gt_shapes[i].at<double>(k, 0);
585 gt_shape_flipped(k, 1) = gt_shapes[i].at<double>(k, 1);
586 }
587 int x_b, y_b, w_b, h_b;
588 x_b = w - (int)bboxes[i].x - (int)bboxes[i].width;
589 y_b = (int)bboxes[i].y;
590 w_b = (int)bboxes[i].width;
591 h_b = (int)bboxes[i].height;
592 BBox bbox_flipped(x_b, y_b, w_b, h_b);
593
594 imgs.push_back(img_flipped);
595 gt_shapes.push_back(gt_shape_flipped);
596 bboxes.push_back(bbox_flipped);
597
598 }
599 #define SWAP(shape, i, j) do { \
600 double tmp = shape.at<double>(i-1, 0); \
601 shape.at<double>(i-1, 0) = shape.at<double>(j-1, 0); \
602 shape.at<double>(j-1, 0) = tmp; \
603 tmp = shape.at<double>(i-1, 1); \
604 shape.at<double>(i-1, 1) = shape.at<double>(j-1, 1); \
605 shape.at<double>(j-1, 1) = tmp; \
606 } while(0)
607
608 if (params.n_landmarks == 29) {
609 for (int i = N; i < (int)gt_shapes.size(); i++) {
610 SWAP(gt_shapes[i], 1, 2);
611 SWAP(gt_shapes[i], 3, 4);
612 SWAP(gt_shapes[i], 5, 7);
613 SWAP(gt_shapes[i], 6, 8);
614 SWAP(gt_shapes[i], 13, 15);
615 SWAP(gt_shapes[i], 9, 10);
616 SWAP(gt_shapes[i], 11, 12);
617 SWAP(gt_shapes[i], 17, 18);
618 SWAP(gt_shapes[i], 14, 16);
619 SWAP(gt_shapes[i], 19, 20);
620 SWAP(gt_shapes[i], 23, 24);
621 }
622 }
623 else if (params.n_landmarks == 68) {
624 for (int i = N; i < (int)gt_shapes.size(); i++) {
625 for (int k = 1; k <= 8; k++) SWAP(gt_shapes[i], k, 18 - k);
626 for (int k = 18; k <= 22; k++) SWAP(gt_shapes[i], k, 45 - k);
627 for (int k = 37; k <= 40; k++) SWAP(gt_shapes[i], k, 83 - k);
628 SWAP(gt_shapes[i], 42, 47);
629 SWAP(gt_shapes[i], 41, 48);
630 SWAP(gt_shapes[i], 32, 36);
631 SWAP(gt_shapes[i], 33, 35);
632 for (int k = 49; k <= 51; k++) SWAP(gt_shapes[i], k, 104 - k);
633 SWAP(gt_shapes[i], 60, 56);
634 SWAP(gt_shapes[i], 59, 57);
635 SWAP(gt_shapes[i], 61, 65);
636 SWAP(gt_shapes[i], 62, 64);
637 SWAP(gt_shapes[i], 68, 66);
638 }
639 }
640 else {
641 printf("Wrong n_landmarks, currently only 29 and 68 landmark points are supported");
642 }
643
644 #undef SWAP
645
646 }
647
BBox()648 FacemarkLBFImpl::BBox::BBox() {}
~BBox()649 FacemarkLBFImpl::BBox::~BBox() {}
650
BBox(double _x,double _y,double w,double h)651 FacemarkLBFImpl::BBox::BBox(double _x, double _y, double w, double h) {
652 x = _x;
653 y = _y;
654 width = w;
655 height = h;
656 x_center = x + w / 2.;
657 y_center = y + h / 2.;
658 x_scale = w / 2.;
659 y_scale = h / 2.;
660 }
661
662 // Project absolute shape to relative shape binding to this bbox
project(const Mat & shape) const663 Mat FacemarkLBFImpl::BBox::project(const Mat &shape) const {
664 Mat_<double> res(shape.rows, shape.cols);
665 const Mat_<double> &shape_ = (Mat_<double>)shape;
666 for (int i = 0; i < shape.rows; i++) {
667 res(i, 0) = (shape_(i, 0) - x_center) / x_scale;
668 res(i, 1) = (shape_(i, 1) - y_center) / y_scale;
669 }
670 return std::move(res);
671 }
672
673 // Project relative shape to absolute shape binding to this bbox
reproject(const Mat & shape) const674 Mat FacemarkLBFImpl::BBox::reproject(const Mat &shape) const {
675 Mat_<double> res(shape.rows, shape.cols);
676 const Mat_<double> &shape_ = (Mat_<double>)shape;
677 for (int i = 0; i < shape.rows; i++) {
678 res(i, 0) = shape_(i, 0)*x_scale + x_center;
679 res(i, 1) = shape_(i, 1)*y_scale + y_center;
680 }
681 return std::move(res);
682 }
683
getMeanShape(std::vector<Mat> & gt_shapes,std::vector<BBox> & bboxes)684 Mat FacemarkLBFImpl::getMeanShape(std::vector<Mat> >_shapes, std::vector<BBox> &bboxes) {
685
686 int N = (int)gt_shapes.size();
687 Mat mean_shape = Mat::zeros(gt_shapes[0].rows, 2, CV_64FC1);
688 for (int i = 0; i < N; i++) {
689 mean_shape += bboxes[i].project(gt_shapes[i]);
690 }
691 mean_shape /= N;
692 return mean_shape;
693 }
694
695 // Similarity Transform, project shape2 to shape1
696 // p1 ~= scale * rotate * p2, p1 and p2 are vector in math
calcSimilarityTransform(const Mat & shape1,const Mat & shape2,double & scale,Mat & rotate)697 void FacemarkLBFImpl::LBF::calcSimilarityTransform(const Mat &shape1, const Mat &shape2, double &scale, Mat &rotate) {
698 Mat_<double> rotate_(2, 2);
699 double x1_center, y1_center, x2_center, y2_center;
700 x1_center = cv::mean(shape1.col(0))[0];
701 y1_center = cv::mean(shape1.col(1))[0];
702 x2_center = cv::mean(shape2.col(0))[0];
703 y2_center = cv::mean(shape2.col(1))[0];
704
705 Mat temp1(shape1.rows, shape1.cols, CV_64FC1);
706 Mat temp2(shape2.rows, shape2.cols, CV_64FC1);
707 temp1.col(0) = shape1.col(0) - x1_center;
708 temp1.col(1) = shape1.col(1) - y1_center;
709 temp2.col(0) = shape2.col(0) - x2_center;
710 temp2.col(1) = shape2.col(1) - y2_center;
711
712 Mat_<double> covar1, covar2;
713 Mat_<double> mean1, mean2;
714 calcCovarMatrix(temp1, covar1, mean1, COVAR_COLS);
715 calcCovarMatrix(temp2, covar2, mean2, COVAR_COLS);
716
717 double s1 = sqrt(cv::norm(covar1));
718 double s2 = sqrt(cv::norm(covar2));
719 scale = s1 / s2;
720 temp1 /= s1;
721 temp2 /= s2;
722
723 double num = temp1.col(1).dot(temp2.col(0)) - temp1.col(0).dot(temp2.col(1));
724 double den = temp1.col(0).dot(temp2.col(0)) + temp1.col(1).dot(temp2.col(1));
725 double normed = sqrt(num*num + den*den);
726 double sin_theta = num / normed;
727 double cos_theta = den / normed;
728 rotate_(0, 0) = cos_theta; rotate_(0, 1) = -sin_theta;
729 rotate_(1, 0) = sin_theta; rotate_(1, 1) = cos_theta;
730 rotate = rotate_;
731 }
732
733 // Get relative delta_shapes for predicting target
getDeltaShapes(std::vector<Mat> & gt_shapes,std::vector<Mat> & current_shapes,std::vector<BBox> & bboxes,Mat & mean_shape)734 std::vector<Mat> FacemarkLBFImpl::LBF::getDeltaShapes(std::vector<Mat> >_shapes, std::vector<Mat> ¤t_shapes,
735 std::vector<BBox> &bboxes, Mat &mean_shape) {
736 std::vector<Mat> delta_shapes;
737 int N = (int)gt_shapes.size();
738 delta_shapes.resize(N);
739 double scale;
740 Mat_<double> rotate;
741 for (int i = 0; i < N; i++) {
742 delta_shapes[i] = bboxes[i].project(gt_shapes[i]) - bboxes[i].project(current_shapes[i]);
743 calcSimilarityTransform(mean_shape, bboxes[i].project(current_shapes[i]), scale, rotate);
744 // delta_shapes[i] = scale * delta_shapes[i] * rotate.t(); // the result is better without this part
745 }
746 return delta_shapes;
747 }
748
calcVariance(const Mat & vec)749 double FacemarkLBFImpl::LBF::calcVariance(const Mat &vec) {
750 double m1 = cv::mean(vec)[0];
751 double m2 = cv::mean(vec.mul(vec))[0];
752 double variance = m2 - m1*m1;
753 return variance;
754 }
755
calcVariance(const std::vector<double> & vec)756 double FacemarkLBFImpl::LBF::calcVariance(const std::vector<double> &vec) {
757 if (vec.size() == 0) return 0.;
758 Mat_<double> vec_(vec);
759 double m1 = cv::mean(vec_)[0];
760 double m2 = cv::mean(vec_.mul(vec_))[0];
761 double variance = m2 - m1*m1;
762 return variance;
763 }
764
calcMeanError(std::vector<Mat> & gt_shapes,std::vector<Mat> & current_shapes,int landmark_n,std::vector<int> & left,std::vector<int> & right)765 double FacemarkLBFImpl::LBF::calcMeanError(std::vector<Mat> >_shapes, std::vector<Mat> ¤t_shapes, int landmark_n , std::vector<int> &left, std::vector<int> &right ) {
766 int N = (int)gt_shapes.size();
767
768 double e = 0;
769 // every train data
770 for (int i = 0; i < N; i++) {
771 const Mat_<double> >_shape = (Mat_<double>)gt_shapes[i];
772 const Mat_<double> ¤t_shape = (Mat_<double>)current_shapes[i];
773 double x1, y1, x2, y2;
774 x1 = x2 = y1 = y2 = 0;
775 for (int j = 0; j < (int)left.size(); j++) {
776 x1 += gt_shape(left[j], 0);
777 y1 += gt_shape(left[j], 1);
778 }
779 for (int j = 0; j < (int)right.size(); j++) {
780 x2 += gt_shape(right[j], 0);
781 y2 += gt_shape(right[j], 1);
782 }
783 x1 /= left.size(); y1 /= left.size();
784 x2 /= right.size(); y2 /= right.size();
785 double pupils_distance = sqrt((x2 - x1)*(x2 - x1) + (y2 - y1)*(y2 - y1));
786 // every landmark
787 double e_ = 0;
788 for (int j = 0; j < landmark_n; j++) {
789 e_ += norm(gt_shape.row(j) - current_shape.row(j));
790 }
791 e += e_ / pupils_distance;
792 }
793 e /= N*landmark_n;
794 return e;
795 }
796
797 /*---------------RandomTree Implementation---------------------*/
initTree(int _landmark_id,int _depth,std::vector<int> feats_m,std::vector<double> radius_m)798 void FacemarkLBFImpl::RandomTree::initTree(int _landmark_id, int _depth, std::vector<int> feats_m, std::vector<double> radius_m) {
799 landmark_id = _landmark_id;
800 depth = _depth;
801 nodes_n = 1 << depth;
802 feats = Mat::zeros(nodes_n, 4, CV_64FC1);
803 thresholds.resize(nodes_n);
804
805 params_feats_m = feats_m;
806 params_radius_m = radius_m;
807 }
808
train(std::vector<Mat> & imgs,std::vector<Mat> & current_shapes,std::vector<BBox> & bboxes,std::vector<Mat> & delta_shapes,Mat & mean_shape,std::vector<int> & index,int stage)809 void FacemarkLBFImpl::RandomTree::train(std::vector<Mat> &imgs, std::vector<Mat> ¤t_shapes, std::vector<BBox> &bboxes,
810 std::vector<Mat> &delta_shapes, Mat &mean_shape, std::vector<int> &index, int stage) {
811 Mat_<double> delta_shapes_((int)delta_shapes.size(), 2);
812 for (int i = 0; i < (int)delta_shapes.size(); i++) {
813 delta_shapes_(i, 0) = delta_shapes[i].at<double>(landmark_id, 0);
814 delta_shapes_(i, 1) = delta_shapes[i].at<double>(landmark_id, 1);
815 }
816 splitNode(imgs, current_shapes, bboxes, delta_shapes_, mean_shape, index, 1, stage);
817 }
818
splitNode(std::vector<Mat> & imgs,std::vector<Mat> & current_shapes,std::vector<BBox> & bboxes,Mat & delta_shapes,Mat & mean_shape,std::vector<int> & root,int idx,int stage)819 void FacemarkLBFImpl::RandomTree::splitNode(std::vector<Mat> &imgs, std::vector<Mat> ¤t_shapes, std::vector<BBox> &bboxes,
820 Mat &delta_shapes, Mat &mean_shape, std::vector<int> &root, int idx, int stage) {
821
822 int N = (int)root.size();
823 if (N == 0) {
824 thresholds[idx] = 0;
825 feats.row(idx).setTo(0);
826 std::vector<int> left, right;
827 // split left and right child in DFS
828 if (2 * idx < feats.rows / 2)
829 splitNode(imgs, current_shapes, bboxes, delta_shapes, mean_shape, left, 2 * idx, stage);
830 if (2 * idx + 1 < feats.rows / 2)
831 splitNode(imgs, current_shapes, bboxes, delta_shapes, mean_shape, right, 2 * idx + 1, stage);
832 return;
833 }
834
835 int feats_m = params_feats_m[stage];
836 double radius_m = params_radius_m[stage];
837 Mat_<double> candidate_feats(feats_m, 4);
838 RNG rng(getTickCount());
839 // generate feature pool
840 for (int i = 0; i < feats_m; i++) {
841 double x1, y1, x2, y2;
842 x1 = rng.uniform(-1., 1.); y1 = rng.uniform(-1., 1.);
843 x2 = rng.uniform(-1., 1.); y2 = rng.uniform(-1., 1.);
844 if (x1*x1 + y1*y1 > 1. || x2*x2 + y2*y2 > 1.) {
845 i--;
846 continue;
847 }
848 candidate_feats[i][0] = x1 * radius_m;
849 candidate_feats[i][1] = y1 * radius_m;
850 candidate_feats[i][2] = x2 * radius_m;
851 candidate_feats[i][3] = y2 * radius_m;
852 }
853 // calc features
854 Mat_<int> densities(feats_m, N);
855 for (int i = 0; i < N; i++) {
856 double scale;
857 Mat_<double> rotate;
858 const Mat_<double> ¤t_shape = (Mat_<double>)current_shapes[root[i]];
859 BBox &bbox = bboxes[root[i]];
860 Mat &img = imgs[root[i]];
861 calcSimilarityTransform(bbox.project(current_shape), mean_shape, scale, rotate);
862 for (int j = 0; j < feats_m; j++) {
863 double x1 = candidate_feats(j, 0);
864 double y1 = candidate_feats(j, 1);
865 double x2 = candidate_feats(j, 2);
866 double y2 = candidate_feats(j, 3);
867 SIMILARITY_TRANSFORM(x1, y1, scale, rotate);
868 SIMILARITY_TRANSFORM(x2, y2, scale, rotate);
869
870 x1 = x1*bbox.x_scale + current_shape(landmark_id, 0);
871 y1 = y1*bbox.y_scale + current_shape(landmark_id, 1);
872 x2 = x2*bbox.x_scale + current_shape(landmark_id, 0);
873 y2 = y2*bbox.y_scale + current_shape(landmark_id, 1);
874 x1 = max(0., min(img.cols - 1., x1)); y1 = max(0., min(img.rows - 1., y1));
875 x2 = max(0., min(img.cols - 1., x2)); y2 = max(0., min(img.rows - 1., y2));
876 densities(j, i) = (int)img.at<uchar>(int(y1), int(x1)) - (int)img.at<uchar>(int(y2), int(x2));
877 }
878 }
879 Mat_<int> densities_sorted;
880 cv::sort(densities, densities_sorted, SORT_EVERY_ROW + SORT_ASCENDING);
881 //select a feat which reduces maximum variance
882 double variance_all = (calcVariance(delta_shapes.col(0)) + calcVariance(delta_shapes.col(1)))*N;
883 double variance_reduce_max = 0;
884 int threshold = 0;
885 int feat_id = 0;
886 std::vector<double> left_x, left_y, right_x, right_y;
887 left_x.reserve(N); left_y.reserve(N);
888 right_x.reserve(N); right_y.reserve(N);
889 for (int j = 0; j < feats_m; j++) {
890 left_x.clear(); left_y.clear();
891 right_x.clear(); right_y.clear();
892 int threshold_ = densities_sorted(j, (int)(N*rng.uniform(0.05, 0.95)));
893 for (int i = 0; i < N; i++) {
894 if (densities(j, i) < threshold_) {
895 left_x.push_back(delta_shapes.at<double>(root[i], 0));
896 left_y.push_back(delta_shapes.at<double>(root[i], 1));
897 }
898 else {
899 right_x.push_back(delta_shapes.at<double>(root[i], 0));
900 right_y.push_back(delta_shapes.at<double>(root[i], 1));
901 }
902 }
903 double variance_ = (calcVariance(left_x) + calcVariance(left_y))*left_x.size() + \
904 (calcVariance(right_x) + calcVariance(right_y))*right_x.size();
905 double variance_reduce = variance_all - variance_;
906 if (variance_reduce > variance_reduce_max) {
907 variance_reduce_max = variance_reduce;
908 threshold = threshold_;
909 feat_id = j;
910 }
911 }
912 thresholds[idx] = threshold;
913 feats(idx, 0) = candidate_feats(feat_id, 0); feats(idx, 1) = candidate_feats(feat_id, 1);
914 feats(idx, 2) = candidate_feats(feat_id, 2); feats(idx, 3) = candidate_feats(feat_id, 3);
915 // generate left and right child
916 std::vector<int> left, right;
917 left.reserve(N);
918 right.reserve(N);
919 for (int i = 0; i < N; i++) {
920 if (densities(feat_id, i) < threshold) left.push_back(root[i]);
921 else right.push_back(root[i]);
922 }
923 // split left and right child in DFS
924 if (2 * idx < feats.rows / 2)
925 splitNode(imgs, current_shapes, bboxes, delta_shapes, mean_shape, left, 2 * idx, stage);
926 if (2 * idx + 1 < feats.rows / 2)
927 splitNode(imgs, current_shapes, bboxes, delta_shapes, mean_shape, right, 2 * idx + 1, stage);
928 }
929
write(FileStorage fs,int k,int i,int j)930 void FacemarkLBFImpl::RandomTree::write(FileStorage fs, int k, int i, int j) {
931
932 String x;
933 x = cv::format("tree_%i_%i_%i",k,i,j);
934 fs << x << feats;
935 x = cv::format("thresholds_%i_%i_%i",k,i,j);
936 fs << x << thresholds;
937 }
938
read(FileStorage fs,int k,int i,int j)939 void FacemarkLBFImpl::RandomTree::read(FileStorage fs, int k, int i, int j) {
940 String x;
941 x = cv::format("tree_%i_%i_%i",k,i,j);
942 fs[x] >> feats;
943 x = cv::format("thresholds_%i_%i_%i",k,i,j);
944 fs[x] >> thresholds;
945 }
946
947
948 /*---------------RandomForest Implementation---------------------*/
initForest(int _landmark_n,int _trees_n,int _tree_depth,double _overlap_ratio,std::vector<int> _feats_m,std::vector<double> _radius_m,bool verbose_mode)949 void FacemarkLBFImpl::RandomForest::initForest(
950 int _landmark_n,
951 int _trees_n,
952 int _tree_depth,
953 double _overlap_ratio,
954 std::vector<int>_feats_m,
955 std::vector<double>_radius_m,
956 bool verbose_mode
957 ) {
958 trees_n = _trees_n;
959 landmark_n = _landmark_n;
960 tree_depth = _tree_depth;
961 overlap_ratio = _overlap_ratio;
962
963 feats_m = _feats_m;
964 radius_m = _radius_m;
965
966 verbose = verbose_mode;
967
968 random_trees.resize(landmark_n);
969 for (int i = 0; i < landmark_n; i++) {
970 random_trees[i].resize(trees_n);
971 for (int j = 0; j < trees_n; j++) random_trees[i][j].initTree(i, tree_depth, feats_m, radius_m);
972 }
973 }
974
train(std::vector<Mat> & imgs,std::vector<Mat> & current_shapes,std::vector<BBox> & bboxes,std::vector<Mat> & delta_shapes,Mat & mean_shape,int stage)975 void FacemarkLBFImpl::RandomForest::train(std::vector<Mat> &imgs, std::vector<Mat> ¤t_shapes, \
976 std::vector<BBox> &bboxes, std::vector<Mat> &delta_shapes, Mat &mean_shape, int stage) {
977 int N = (int)imgs.size();
978 int Q = int(N / ((1. - overlap_ratio) * trees_n));
979
980 #ifdef _OPENMP
981 #pragma omp parallel for
982 #endif
983 for (int i = 0; i < landmark_n; i++) {
984 TIMER_BEGIN
985 std::vector<int> root;
986 for (int j = 0; j < trees_n; j++) {
987 int start = max(0, int(floor(j*Q - j*Q*overlap_ratio)));
988 int end = min(int(start + Q + 1), N);
989 int L = end - start;
990 root.resize(L);
991 for (int k = 0; k < L; k++) root[k] = start + k;
992 random_trees[i][j].train(imgs, current_shapes, bboxes, delta_shapes, mean_shape, root, stage);
993 }
994 if(verbose) printf("Train %2dth of %d landmark Done, it costs %.4lf s\n", i+1, landmark_n, TIMER_NOW);
995 TIMER_END
996 }
997 }
998
generateLBF(Mat & img,Mat & current_shape,BBox & bbox,Mat & mean_shape)999 Mat FacemarkLBFImpl::RandomForest::generateLBF(Mat &img, Mat ¤t_shape, BBox &bbox, Mat &mean_shape) {
1000 Mat_<int> lbf_feat(1, landmark_n*trees_n);
1001 double scale;
1002 Mat_<double> rotate;
1003 calcSimilarityTransform(bbox.project(current_shape), mean_shape, scale, rotate);
1004
1005 int base = 1 << (tree_depth - 1);
1006
1007 #ifdef _OPENMP
1008 #pragma omp parallel for
1009 #endif
1010 for (int i = 0; i < landmark_n; i++) {
1011 for (int j = 0; j < trees_n; j++) {
1012 RandomTree &tree = random_trees[i][j];
1013 int code = 0;
1014 int idx = 1;
1015 for (int k = 1; k < tree.depth; k++) {
1016 double x1 = tree.feats(idx, 0);
1017 double y1 = tree.feats(idx, 1);
1018 double x2 = tree.feats(idx, 2);
1019 double y2 = tree.feats(idx, 3);
1020 SIMILARITY_TRANSFORM(x1, y1, scale, rotate);
1021 SIMILARITY_TRANSFORM(x2, y2, scale, rotate);
1022
1023 x1 = x1*bbox.x_scale + current_shape.at<double>(i, 0);
1024 y1 = y1*bbox.y_scale + current_shape.at<double>(i, 1);
1025 x2 = x2*bbox.x_scale + current_shape.at<double>(i, 0);
1026 y2 = y2*bbox.y_scale + current_shape.at<double>(i, 1);
1027 x1 = max(0., min(img.cols - 1., x1)); y1 = max(0., min(img.rows - 1., y1));
1028 x2 = max(0., min(img.cols - 1., x2)); y2 = max(0., min(img.rows - 1., y2));
1029 int density = img.at<uchar>(int(y1), int(x1)) - img.at<uchar>(int(y2), int(x2));
1030 code <<= 1;
1031 if (density < tree.thresholds[idx]) {
1032 idx = 2 * idx;
1033 }
1034 else {
1035 code += 1;
1036 idx = 2 * idx + 1;
1037 }
1038 }
1039 lbf_feat(i*trees_n + j) = (i*trees_n + j)*base + code;
1040 }
1041 }
1042 return std::move(lbf_feat);
1043 }
1044
write(FileStorage fs,int k)1045 void FacemarkLBFImpl::RandomForest::write(FileStorage fs, int k) {
1046 for (int i = 0; i < landmark_n; i++) {
1047 for (int j = 0; j < trees_n; j++) {
1048 random_trees[i][j].write(fs,k,i,j);
1049 }
1050 }
1051 }
1052
read(FileStorage fs,int k)1053 void FacemarkLBFImpl::RandomForest::read(FileStorage fs,int k)
1054 {
1055 for (int i = 0; i < landmark_n; i++) {
1056 for (int j = 0; j < trees_n; j++) {
1057 random_trees[i][j].initTree(i, tree_depth, feats_m, radius_m);
1058 random_trees[i][j].read(fs,k,i,j);
1059 }
1060 }
1061 }
1062
1063 /*---------------Regressor Implementation---------------------*/
initRegressor(Params config)1064 void FacemarkLBFImpl::Regressor::initRegressor(Params config) {
1065 stages_n = config.stages_n;
1066 landmark_n = config.n_landmarks;
1067
1068 random_forests.resize(stages_n);
1069 for (int i = 0; i < stages_n; i++)
1070 random_forests[i].initForest(
1071 config.n_landmarks,
1072 config.tree_n,
1073 config.tree_depth,
1074 config.bagging_overlap,
1075 config.feats_m,
1076 config.radius_m,
1077 config.verbose
1078 );
1079
1080 mean_shape.create(config.n_landmarks, 2, CV_64FC1);
1081
1082 gl_regression_weights.resize(stages_n);
1083 int F = config.n_landmarks * config.tree_n * (1 << (config.tree_depth - 1));
1084
1085 for (int i = 0; i < stages_n; i++) {
1086 gl_regression_weights[i].create(2 * config.n_landmarks, F, CV_64FC1);
1087 }
1088 }
1089
trainRegressor(std::vector<Mat> & imgs,std::vector<Mat> & gt_shapes,std::vector<Mat> & current_shapes,std::vector<BBox> & bboxes,Mat & mean_shape_,int start_from,Params config)1090 void FacemarkLBFImpl::Regressor::trainRegressor(std::vector<Mat> &imgs, std::vector<Mat> >_shapes, std::vector<Mat> ¤t_shapes,
1091 std::vector<BBox> &bboxes, Mat &mean_shape_, int start_from, Params config) {
1092 CV_Assert(start_from >= 0 && start_from < stages_n);
1093 mean_shape = mean_shape_;
1094 int N = (int)imgs.size();
1095
1096 for (int k = start_from; k < stages_n; k++) {
1097 std::vector<Mat> delta_shapes = getDeltaShapes(gt_shapes, current_shapes, bboxes, mean_shape);
1098
1099 // train random forest
1100 if(config.verbose) printf("training random forest %dth of %d stages, ",k+1, stages_n);
1101 TIMER_BEGIN
1102 random_forests[k].train(imgs, current_shapes, bboxes, delta_shapes, mean_shape, k);
1103 if(config.verbose) printf("costs %.4lf s\n", TIMER_NOW);
1104 TIMER_END
1105
1106 // generate lbf of every train data
1107 std::vector<Mat> lbfs;
1108 lbfs.resize(N);
1109 for (int i = 0; i < N; i++) {
1110 lbfs[i] = random_forests[k].generateLBF(imgs[i], current_shapes[i], bboxes[i], mean_shape);
1111 }
1112
1113 // global regression
1114 if(config.verbose) printf("start train global regression of %dth stage\n", k);
1115 TIMER_BEGIN
1116 globalRegressionTrain(lbfs, delta_shapes, k, config);
1117 if(config.verbose) printf("end of train global regression of %dth stage, costs %.4lf s\n", k, TIMER_NOW);
1118 TIMER_END
1119
1120 // update current_shapes
1121 double scale;
1122 Mat rotate;
1123 for (int i = 0; i < N; i++) {
1124 Mat delta_shape = globalRegressionPredict(lbfs[i], k);
1125 calcSimilarityTransform(bboxes[i].project(current_shapes[i]), mean_shape, scale, rotate);
1126 current_shapes[i] = bboxes[i].reproject(bboxes[i].project(current_shapes[i]) + scale * delta_shape * rotate.t());
1127 }
1128
1129 // calc mean error
1130 double e = calcMeanError(gt_shapes, current_shapes, config.n_landmarks, config.pupils[0],config.pupils[1]);
1131 if(config.verbose) printf("Train %dth stage Done with Error = %lf\n", k, e);
1132
1133 } // for int k
1134 }//Regressor::training
1135
globalRegressionTrain(std::vector<Mat> & lbfs,std::vector<Mat> & delta_shapes,int stage,Params config)1136 void FacemarkLBFImpl::Regressor::globalRegressionTrain(
1137 std::vector<Mat> &lbfs, std::vector<Mat> &delta_shapes,
1138 int stage, Params config
1139 ) {
1140
1141 int N = (int)lbfs.size();
1142 int M = lbfs[0].cols;
1143 int F = config.n_landmarks*config.tree_n*(1 << (config.tree_depth - 1));
1144 int landmark_n_ = delta_shapes[0].rows;
1145 feature_node **X = (feature_node **)malloc(N * sizeof(feature_node *));
1146 double **Y = (double **)malloc(landmark_n_ * 2 * sizeof(double *));
1147 for (int i = 0; i < N; i++) {
1148 X[i] = (feature_node *)malloc((M + 1) * sizeof(feature_node));
1149 for (int j = 0; j < M; j++) {
1150 X[i][j].index = lbfs[i].at<int>(0, j) + 1; // index starts from 1
1151 X[i][j].value = 1;
1152 }
1153 X[i][M].index = -1;
1154 X[i][M].value = -1;
1155 }
1156 for (int i = 0; i < landmark_n_; i++) {
1157 Y[2 * i] = (double *)malloc(N*sizeof(double));
1158 Y[2 * i + 1] = (double *)malloc(N*sizeof(double));
1159 for (int j = 0; j < N; j++) {
1160 Y[2 * i][j] = delta_shapes[j].at<double>(i, 0);
1161 Y[2 * i + 1][j] = delta_shapes[j].at<double>(i, 1);
1162 }
1163 }
1164
1165 double *y;
1166 Mat weights;
1167 for(int i=0; i< landmark_n_; i++){
1168 y = Y[2 * i];
1169 Mat wx = supportVectorRegression(X,y,N,F,config.verbose);
1170 weights.push_back(wx);
1171
1172 y = Y[2 * i + 1];
1173 Mat wy = supportVectorRegression(X,y,N,F,config.verbose);
1174 weights.push_back(wy);
1175 }
1176
1177 gl_regression_weights[stage] = weights;
1178
1179 // free
1180 for (int i = 0; i < N; i++) free(X[i]);
1181 for (int i = 0; i < 2 * landmark_n_; i++) free(Y[i]);
1182 free(X);
1183 free(Y);
1184 } // Regressor:globalRegressionTrain
1185
1186 /*adapted from the liblinear library*/
1187 /* TODO: change feature_node to MAT
1188 * as the index in feature_node is only used for "counter"
1189 */
supportVectorRegression(feature_node ** x,double * y,int nsamples,int feat_size,bool verbose)1190 Mat FacemarkLBFImpl::Regressor::supportVectorRegression(
1191 feature_node **x, double *y, int nsamples, int feat_size, bool verbose
1192 ){
1193 #define GETI(i) ((int) y[i])
1194
1195 std::vector<double> w;
1196 w.resize(feat_size);
1197
1198 RNG rng(0);
1199 int l = nsamples; // n-samples
1200 double C = 1./(double)nsamples;
1201 double p = 0;
1202 int w_size = feat_size; //feat size
1203 double eps = 0.00001;
1204 int i, s, iter = 0;
1205 int max_iter = 1000;
1206 int active_size = l;
1207 std::vector<int> index(l);
1208
1209 double d, G, H;
1210 double Gmax_old = HUGE_VAL;
1211 double Gmax_new, Gnorm1_new;
1212 double Gnorm1_init = -1.0; // Gnorm1_init is initialized at the first iteration
1213 std::vector<double> beta(l);
1214 std::vector<double> QD(l);
1215
1216 double lambda[1], upper_bound[1];
1217 lambda[0] = 0.5/C;
1218 upper_bound[0] = HUGE_VAL;
1219
1220 // Initial beta can be set here. Note that
1221 // -upper_bound <= beta[i] <= upper_bound
1222 for(i=0; i<l; i++)
1223 beta[i] = 0;
1224
1225 for(i=0; i<w_size; i++)
1226 w[i] = 0;
1227
1228 for(i=0; i<l; i++){
1229 QD[i] = 0;
1230 feature_node *xi = x[i];
1231 while(xi->index != -1){
1232 double val = xi->value;
1233 QD[i] += val*val;
1234 w[xi->index-1] += beta[i]*val;
1235 xi++;
1236 }
1237 index[i] = i;
1238 }
1239
1240 while(iter < max_iter){
1241 Gmax_new = 0;
1242 Gnorm1_new = 0;
1243
1244 for(i=0; i<active_size; i++){
1245 int j = i+rng.uniform(0,RAND_MAX)%(active_size-i);
1246 swap(index[i], index[j]);
1247 }
1248
1249 for(s=0; s<active_size; s++){
1250 i = index[s];
1251 G = -y[i] + lambda[GETI(i)]*beta[i];
1252 H = QD[i] + lambda[GETI(i)];
1253
1254 feature_node *xi = x[i];
1255 while(xi->index != -1){
1256 int ind = xi->index-1;
1257 double val = xi->value;
1258 G += val*w[ind];
1259 xi++;
1260 }
1261
1262 double Gp = G+p;
1263 double Gn = G-p;
1264 double violation = 0;
1265 if(beta[i] == 0){
1266 if(Gp < 0)
1267 violation = -Gp;
1268 else if(Gn > 0)
1269 violation = Gn;
1270 else if(Gp>Gmax_old && Gn<-Gmax_old){
1271 active_size--;
1272 swap(index[s], index[active_size]);
1273 s--;
1274 continue;
1275 }
1276 }else if(beta[i] >= upper_bound[GETI(i)]){
1277 if(Gp > 0)
1278 violation = Gp;
1279 else if(Gp < -Gmax_old){
1280 active_size--;
1281 swap(index[s], index[active_size]);
1282 s--;
1283 continue;
1284 }
1285 }else if(beta[i] <= -upper_bound[GETI(i)]){
1286 if(Gn < 0)
1287 violation = -Gn;
1288 else if(Gn > Gmax_old){
1289 active_size--;
1290 swap(index[s], index[active_size]);
1291 s--;
1292 continue;
1293 }
1294 }else if(beta[i] > 0)
1295 violation = fabs(Gp);
1296 else
1297 violation = fabs(Gn);
1298
1299 Gmax_new = max(Gmax_new, violation);
1300 Gnorm1_new += violation;
1301
1302 // obtain Newton direction d
1303 if(Gp < H*beta[i])
1304 d = -Gp/H;
1305 else if(Gn > H*beta[i])
1306 d = -Gn/H;
1307 else
1308 d = -beta[i];
1309
1310 if(fabs(d) < 1.0e-12)
1311 continue;
1312
1313 double beta_old = beta[i];
1314 beta[i] = min(max(beta[i]+d, -upper_bound[GETI(i)]), upper_bound[GETI(i)]);
1315 d = beta[i]-beta_old;
1316
1317 if(d != 0){
1318 xi = x[i];
1319 while(xi->index != -1){
1320 w[xi->index-1] += d*xi->value;
1321 xi++;
1322 }
1323 }
1324 }// for s<active_size
1325
1326 if(iter == 0)
1327 Gnorm1_init = Gnorm1_new;
1328 iter++;
1329
1330 if(Gnorm1_new <= eps*Gnorm1_init){
1331 if(active_size == l)
1332 break;
1333 else{
1334 active_size = l;
1335 Gmax_old = HUGE_VAL;
1336 continue;
1337 }
1338 }
1339
1340 Gmax_old = Gmax_new;
1341 } //while <max_iter
1342
1343 if(verbose) printf("optimization finished, #iter = %d\n", iter);
1344 if(iter >= max_iter && verbose)
1345 printf("WARNING: reaching max number of iterations\n");
1346
1347 // calculate objective value
1348 double v = 0;
1349 int nSV = 0;
1350 for(i=0; i<w_size; i++)
1351 v += w[i]*w[i];
1352 v = 0.5*v;
1353 for(i=0; i<l; i++){
1354 v += p*fabs(beta[i]) - y[i]*beta[i] + 0.5*lambda[GETI(i)]*beta[i]*beta[i];
1355 if(beta[i] != 0)
1356 nSV++;
1357 }
1358
1359 if(verbose) printf("Objective value = %lf\n", v);
1360 if(verbose) printf("nSV = %d\n",nSV);
1361
1362 return Mat(Mat(w).t()).clone();
1363
1364 }//end
1365
globalRegressionPredict(const Mat & lbf,int stage)1366 Mat FacemarkLBFImpl::Regressor::globalRegressionPredict(const Mat &lbf, int stage) {
1367 const Mat_<double> &weight = (Mat_<double>)gl_regression_weights[stage];
1368 Mat_<double> delta_shape(weight.rows / 2, 2);
1369 const double *w_ptr = NULL;
1370 const int *lbf_ptr = lbf.ptr<int>(0);
1371
1372 //#pragma omp parallel for num_threads(2) private(w_ptr)
1373 for (int i = 0; i < delta_shape.rows; i++) {
1374 w_ptr = weight.ptr<double>(2 * i);
1375 double y = 0;
1376 for (int j = 0; j < lbf.cols; j++) y += w_ptr[lbf_ptr[j]];
1377 delta_shape(i, 0) = y;
1378
1379 w_ptr = weight.ptr<double>(2 * i + 1);
1380 y = 0;
1381 for (int j = 0; j < lbf.cols; j++) y += w_ptr[lbf_ptr[j]];
1382 delta_shape(i, 1) = y;
1383 }
1384 return std::move(delta_shape);
1385 } // Regressor::globalRegressionPredict
1386
predict(Mat & img,BBox & bbox)1387 Mat FacemarkLBFImpl::Regressor::predict(Mat &img, BBox &bbox) {
1388 Mat current_shape = bbox.reproject(mean_shape);
1389 double scale;
1390 Mat rotate;
1391 Mat lbf_feat;
1392 for (int k = 0; k < stages_n; k++) {
1393 // generate lbf
1394 lbf_feat = random_forests[k].generateLBF(img, current_shape, bbox, mean_shape);
1395 // update current_shapes
1396 Mat delta_shape = globalRegressionPredict(lbf_feat, k);
1397 delta_shape = delta_shape.reshape(0, landmark_n);
1398 calcSimilarityTransform(bbox.project(current_shape), mean_shape, scale, rotate);
1399 current_shape = bbox.reproject(bbox.project(current_shape) + scale * delta_shape * rotate.t());
1400 }
1401 return current_shape;
1402 } // Regressor::predict
1403
write(FileStorage fs,Params config)1404 void FacemarkLBFImpl::Regressor::write(FileStorage fs, Params config) {
1405
1406 fs << "stages_n" << config.stages_n;
1407 fs << "tree_n" << config.tree_n;
1408 fs << "tree_depth" << config.tree_depth;
1409 fs << "n_landmarks" << config.n_landmarks;
1410
1411 fs << "regressor_meanshape" <<mean_shape;
1412
1413 // every stages
1414 String x;
1415 for (int k = 0; k < config.stages_n; k++) {
1416 if(config.verbose) printf("Write %dth stage\n", k);
1417 random_forests[k].write(fs,k);
1418 x = cv::format("weights_%i",k);
1419 fs << x << gl_regression_weights[k];
1420 }
1421 }
1422
read(FileStorage fs,Params & config)1423 void FacemarkLBFImpl::Regressor::read(FileStorage fs, Params & config){
1424 fs["stages_n"] >> config.stages_n;
1425 fs["tree_n"] >> config.tree_n;
1426 fs["tree_depth"] >> config.tree_depth;
1427 fs["n_landmarks"] >> config.n_landmarks;
1428
1429 stages_n = config.stages_n;
1430 landmark_n = config.n_landmarks;
1431
1432 initRegressor(config);
1433
1434 fs["regressor_meanshape"] >> mean_shape;
1435
1436 // every stages
1437 String x;
1438 for (int k = 0; k < stages_n; k++) {
1439 random_forests[k].initForest(
1440 config.n_landmarks,
1441 config.tree_n,
1442 config.tree_depth,
1443 config.bagging_overlap,
1444 config.feats_m,
1445 config.radius_m,
1446 config.verbose
1447 );
1448 random_forests[k].read(fs,k);
1449
1450 x = cv::format("weights_%i",k);
1451 fs[x] >> gl_regression_weights[k];
1452 }
1453 }
1454
1455 #undef TIMER_BEGIN
1456 #undef TIMER_NOW
1457 #undef TIMER_END
1458 #undef SIMILARITY_TRANSFORM
1459 } /* namespace face */
1460 } /* namespace cv */
1461