1 #include "opencv2/face.hpp"
2 #include "opencv2/imgproc.hpp"
3 #include "opencv2/imgcodecs.hpp"
4 #include "opencv2/highgui.hpp"
5 #include "opencv2/objdetect.hpp"
6 #include "opencv2/photo.hpp" // seamlessClone()
7 #include <iostream>
8 using namespace cv;
9 using namespace cv::face;
10 using namespace std;
11 
myDetector(InputArray image,OutputArray faces,CascadeClassifier * face_cascade)12 static bool myDetector(InputArray image, OutputArray faces, CascadeClassifier *face_cascade)
13 {
14     Mat gray;
15 
16     if (image.channels() > 1)
17         cvtColor(image, gray, COLOR_BGR2GRAY);
18     else
19         gray = image.getMat().clone();
20 
21     equalizeHist(gray, gray);
22 
23     std::vector<Rect> faces_;
24     face_cascade->detectMultiScale(gray, faces_, 1.4, 2, CASCADE_SCALE_IMAGE, Size(30, 30));
25     Mat(faces_).copyTo(faces);
26     return true;
27 }
28 
29 void divideIntoTriangles(Rect rect, vector<Point2f> &points, vector< vector<int> > &delaunayTri);
30 void warpTriangle(Mat &img1, Mat &img2, vector<Point2f> &triangle1, vector<Point2f> &triangle2);
31 
32 //Divide the face into triangles for warping
divideIntoTriangles(Rect rect,vector<Point2f> & points,vector<vector<int>> & Tri)33 void divideIntoTriangles(Rect rect, vector<Point2f> &points, vector< vector<int> > &Tri){
34 
35     // Create an instance of Subdiv2D
36     Subdiv2D subdiv(rect);
37     // Insert points into subdiv
38     for( vector<Point2f>::iterator it = points.begin(); it != points.end(); it++)
39         subdiv.insert(*it);
40     vector<Vec6f> triangleList;
41     subdiv.getTriangleList(triangleList);
42     vector<Point2f> pt(3);
43     vector<int> ind(3);
44     for( size_t i = 0; i < triangleList.size(); i++ )
45     {
46         Vec6f triangle = triangleList[i];
47         pt[0] = Point2f(triangle[0], triangle[1]);
48         pt[1] = Point2f(triangle[2], triangle[3]);
49         pt[2] = Point2f(triangle[4], triangle[5]);
50         if ( rect.contains(pt[0]) && rect.contains(pt[1]) && rect.contains(pt[2])){
51             for(int j = 0; j < 3; j++)
52                 for(size_t k = 0; k < points.size(); k++)
53                     if(abs(pt[j].x - points[k].x) < 1.0 && abs(pt[j].y - points[k].y) < 1)
54                         ind[j] =(int) k;
55             Tri.push_back(ind);
56         }
57     }
58 }
warpTriangle(Mat & img1,Mat & img2,vector<Point2f> & triangle1,vector<Point2f> & triangle2)59 void warpTriangle(Mat &img1, Mat &img2, vector<Point2f> &triangle1, vector<Point2f> &triangle2)
60 {
61     Rect rectangle1 = boundingRect(triangle1);
62     Rect rectangle2 = boundingRect(triangle2);
63     // Offset points by left top corner of the respective rectangles
64     vector<Point2f> triangle1Rect, triangle2Rect;
65     vector<Point> triangle2RectInt;
66     for(int i = 0; i < 3; i++)
67     {
68         triangle1Rect.push_back( Point2f( triangle1[i].x - rectangle1.x, triangle1[i].y - rectangle1.y) );
69         triangle2Rect.push_back( Point2f( triangle2[i].x - rectangle2.x, triangle2[i].y - rectangle2.y) );
70         triangle2RectInt.push_back( Point((int)(triangle2[i].x - rectangle2.x),(int) (triangle2[i].y - rectangle2.y))); // for fillConvexPoly
71     }
72     // Get mask by filling triangle
73     Mat mask = Mat::zeros(rectangle2.height, rectangle2.width, CV_32FC3);
74     fillConvexPoly(mask, triangle2RectInt, Scalar(1.0, 1.0, 1.0), 16, 0);
75     // Apply warpImage to small rectangular patches
76     Mat img1Rect;
77     img1(rectangle1).copyTo(img1Rect);
78     Mat img2Rect = Mat::zeros(rectangle2.height, rectangle2.width, img1Rect.type());
79     Mat warp_mat = getAffineTransform(triangle1Rect, triangle2Rect);
80     warpAffine( img1Rect, img2Rect, warp_mat, img2Rect.size(), INTER_LINEAR, BORDER_REFLECT_101);
81     multiply(img2Rect,mask, img2Rect);
82     multiply(img2(rectangle2), Scalar(1.0,1.0,1.0) - mask, img2(rectangle2));
83     img2(rectangle2) = img2(rectangle2) + img2Rect;
84 }
main(int argc,char ** argv)85 int main( int argc, char** argv)
86 {
87     //Give the path to the directory containing all the files containing data
88     CommandLineParser parser(argc, argv,
89         "{ help h usage ? |      | give the following arguments in following format }"
90         "{ image1 i1      |      | (required) path to the first image file in which you want to apply swapping }"
91         "{ image2 i2      |      | (required) path to the second image file in which you want to apply face swapping }"
92         "{ model m        |      | (required) path to the file containing model to be loaded for face landmark detection}"
93         "{ face_cascade f |      | Path to the face cascade xml file which you want to use as a detector}"
94     );
95     // Read in the input arguments
96     if (parser.has("help")){
97         parser.printMessage();
98         cerr << "TIP: Use absolute paths to avoid any problems with the software!" << endl;
99         return 0;
100     }
101     Mat img1=imread(parser.get<string>("image1"));
102     Mat img2=imread(parser.get<string>("image2"));
103     if (img1.empty()||img2.empty()){
104         if(img1.empty()){
105             parser.printMessage();
106             cerr << parser.get<string>("image1")<<" not found" << endl;
107             return -1;
108         }
109         if (img2.empty()){
110             parser.printMessage();
111             cerr << parser.get<string>("image2")<<" not found" << endl;
112             return -1;
113         }
114     }
115     string modelfile_name(parser.get<string>("model"));
116     if (modelfile_name.empty()){
117         parser.printMessage();
118         cerr << "Model file name not found." << endl;
119         return -1;
120     }
121     string cascade_name(parser.get<string>("face_cascade"));
122     if (cascade_name.empty()){
123         parser.printMessage();
124         cerr << "The name of the cascade classifier to be loaded to detect faces is not found" << endl;
125         return -1;
126     }
127     //create a pointer to call the base class
128     //pass the face cascade xml file which you want to pass as a detector
129     CascadeClassifier face_cascade;
130     face_cascade.load(cascade_name);
131     FacemarkKazemi::Params params;
132     Ptr<FacemarkKazemi> facemark = FacemarkKazemi::create(params);
133     facemark->setFaceDetector((FN_FaceDetector)myDetector, &face_cascade);
134     facemark->loadModel(modelfile_name);
135     cout<<"Loaded model"<<endl;
136     //vector to store the faces detected in the image
137     vector<Rect> faces1,faces2;
138     vector< vector<Point2f> > shape1,shape2;
139     //Detect faces in the current image
140     float ratio1 = (float)img1.cols/(float)img1.rows;
141     float ratio2 = (float)img2.cols/(float)img2.rows;
142     resize(img1,img1,Size((int)(640*ratio1),(int)(640*ratio1)), 0, 0, INTER_LINEAR_EXACT);
143     resize(img2,img2,Size((int)(640*ratio2),(int)(640*ratio2)), 0, 0, INTER_LINEAR_EXACT);
144     Mat img1Warped = img2.clone();
145     facemark->getFaces(img1,faces1);
146     facemark->getFaces(img2,faces2);
147     //Initialise the shape of the faces
148     facemark->fit(img1,faces1,shape1);
149     facemark->fit(img2,faces2,shape2);
150     unsigned long numswaps = (unsigned long)min((unsigned long)shape1.size(),(unsigned long)shape2.size());
151     for(unsigned long z=0;z<numswaps;z++){
152         vector<Point2f> points1 = shape1[z];
153         vector<Point2f> points2 = shape2[z];
154         img1.convertTo(img1, CV_32F);
155         img1Warped.convertTo(img1Warped, CV_32F);
156         // Find convex hull
157         vector<Point2f> boundary_image1;
158         vector<Point2f> boundary_image2;
159         vector<int> index;
160         convexHull(Mat(points2),index, false, false);
161         for(size_t i = 0; i < index.size(); i++)
162         {
163             boundary_image1.push_back(points1[index[i]]);
164             boundary_image2.push_back(points2[index[i]]);
165         }
166         // Triangulation for points on the convex hull
167         vector< vector<int> > triangles;
168         Rect rect(0, 0, img1Warped.cols, img1Warped.rows);
169         divideIntoTriangles(rect, boundary_image2, triangles);
170         // Apply affine transformation to Delaunay triangles
171         for(size_t i = 0; i < triangles.size(); i++)
172         {
173             vector<Point2f> triangle1, triangle2;
174             // Get points for img1, img2 corresponding to the triangles
175             for(int j = 0; j < 3; j++)
176             {
177                 triangle1.push_back(boundary_image1[triangles[i][j]]);
178                 triangle2.push_back(boundary_image2[triangles[i][j]]);
179             }
180             warpTriangle(img1, img1Warped, triangle1, triangle2);
181         }
182         // Calculate mask
183         vector<Point> hull;
184         for(size_t i = 0; i < boundary_image2.size(); i++)
185         {
186             Point pt((int)boundary_image2[i].x,(int)boundary_image2[i].y);
187             hull.push_back(pt);
188         }
189         Mat mask = Mat::zeros(img2.rows, img2.cols, img2.depth());
190         fillConvexPoly(mask,&hull[0],(int)hull.size(), Scalar(255,255,255));
191         // Clone seamlessly.
192         Rect r = boundingRect(boundary_image2);
193         Point center = (r.tl() + r.br()) / 2;
194         Mat output;
195         img1Warped.convertTo(img1Warped, CV_8UC3);
196         seamlessClone(img1Warped,img2, mask, center, output, NORMAL_CLONE);
197         imshow("Face_Swapped", output);
198         waitKey(0);
199         destroyAllWindows();
200     }
201     return 0;
202 }