1 #include "opencv2/calib3d.hpp"
2 #include "opencv2/imgproc.hpp"
3 #include "opencv2/imgcodecs.hpp"
4 #include "opencv2/highgui.hpp"
5 #include "opencv2/core/utility.hpp"
6 #include "opencv2/ximgproc.hpp"
7 #include <iostream>
8 #include <string>
9
10 using namespace cv;
11 using namespace cv::ximgproc;
12 using namespace std;
13
14 Rect computeROI(Size2i src_sz, Ptr<StereoMatcher> matcher_instance);
15
16 const String keys =
17 "{help h usage ? | | print this message }"
18 "{@left |../data/aloeL.jpg | left view of the stereopair }"
19 "{@right |../data/aloeR.jpg | right view of the stereopair }"
20 "{GT |../data/aloeGT.png| optional ground-truth disparity (MPI-Sintel or Middlebury format) }"
21 "{dst_path |None | optional path to save the resulting filtered disparity map }"
22 "{dst_raw_path |None | optional path to save raw disparity map before filtering }"
23 "{algorithm |bm | stereo matching method (bm or sgbm) }"
24 "{filter |wls_conf | used post-filtering (wls_conf or wls_no_conf or fbs_conf) }"
25 "{no-display | | don't display results }"
26 "{no-downscale | | force stereo matching on full-sized views to improve quality }"
27 "{dst_conf_path |None | optional path to save the confidence map used in filtering }"
28 "{vis_mult |1.0 | coefficient used to scale disparity map visualizations }"
29 "{max_disparity |160 | parameter of stereo matching }"
30 "{window_size |-1 | parameter of stereo matching }"
31 "{wls_lambda |8000.0 | parameter of wls post-filtering }"
32 "{wls_sigma |1.5 | parameter of wls post-filtering }"
33 "{fbs_spatial |16.0 | parameter of fbs post-filtering }"
34 "{fbs_luma |8.0 | parameter of fbs post-filtering }"
35 "{fbs_chroma |8.0 | parameter of fbs post-filtering }"
36 "{fbs_lambda |128.0 | parameter of fbs post-filtering }"
37 ;
38
main(int argc,char ** argv)39 int main(int argc, char** argv)
40 {
41 CommandLineParser parser(argc,argv,keys);
42 parser.about("Disparity Filtering Demo");
43 if (parser.has("help"))
44 {
45 parser.printMessage();
46 return 0;
47 }
48
49 String left_im = parser.get<String>(0);
50 String right_im = parser.get<String>(1);
51 String GT_path = parser.get<String>("GT");
52
53 String dst_path = parser.get<String>("dst_path");
54 String dst_raw_path = parser.get<String>("dst_raw_path");
55 String dst_conf_path = parser.get<String>("dst_conf_path");
56 String algo = parser.get<String>("algorithm");
57 String filter = parser.get<String>("filter");
58 bool no_display = parser.has("no-display");
59 bool no_downscale = parser.has("no-downscale");
60 int max_disp = parser.get<int>("max_disparity");
61 double lambda = parser.get<double>("wls_lambda");
62 double sigma = parser.get<double>("wls_sigma");
63 double fbs_spatial = parser.get<double>("fbs_spatial");
64 double fbs_luma = parser.get<double>("fbs_luma");
65 double fbs_chroma = parser.get<double>("fbs_chroma");
66 double fbs_lambda = parser.get<double>("fbs_lambda");
67 double vis_mult = parser.get<double>("vis_mult");
68
69 int wsize;
70 if(parser.get<int>("window_size")>=0) //user provided window_size value
71 wsize = parser.get<int>("window_size");
72 else
73 {
74 if(algo=="sgbm")
75 wsize = 3; //default window size for SGBM
76 else if(!no_downscale && algo=="bm" && filter=="wls_conf")
77 wsize = 7; //default window size for BM on downscaled views (downscaling is performed only for wls_conf)
78 else
79 wsize = 15; //default window size for BM on full-sized views
80 }
81
82 if (!parser.check())
83 {
84 parser.printErrors();
85 return -1;
86 }
87
88 //! [load_views]
89 Mat left = imread(left_im ,IMREAD_COLOR);
90 if ( left.empty() )
91 {
92 cout<<"Cannot read image file: "<<left_im;
93 return -1;
94 }
95
96 Mat right = imread(right_im,IMREAD_COLOR);
97 if ( right.empty() )
98 {
99 cout<<"Cannot read image file: "<<right_im;
100 return -1;
101 }
102 //! [load_views]
103
104 bool noGT;
105 Mat GT_disp;
106 if (GT_path=="../data/aloeGT.png" && left_im!="../data/aloeL.jpg")
107 noGT=true;
108 else
109 {
110 noGT=false;
111 if(readGT(GT_path,GT_disp)!=0)
112 {
113 cout<<"Cannot read ground truth image file: "<<GT_path<<endl;
114 return -1;
115 }
116 }
117
118 Mat left_for_matcher, right_for_matcher;
119 Mat left_disp,right_disp;
120 Mat filtered_disp,solved_disp,solved_filtered_disp;
121 Mat conf_map = Mat(left.rows,left.cols,CV_8U);
122 conf_map = Scalar(255);
123 Rect ROI;
124 Ptr<DisparityWLSFilter> wls_filter;
125 double matching_time, filtering_time;
126 double solving_time = 0;
127 if(max_disp<=0 || max_disp%16!=0)
128 {
129 cout<<"Incorrect max_disparity value: it should be positive and divisible by 16";
130 return -1;
131 }
132 if(wsize<=0 || wsize%2!=1)
133 {
134 cout<<"Incorrect window_size value: it should be positive and odd";
135 return -1;
136 }
137
138 if(filter=="wls_conf") // filtering with confidence (significantly better quality than wls_no_conf)
139 {
140 if(!no_downscale)
141 {
142 // downscale the views to speed-up the matching stage, as we will need to compute both left
143 // and right disparity maps for confidence map computation
144 //! [downscale]
145 max_disp/=2;
146 if(max_disp%16!=0)
147 max_disp += 16-(max_disp%16);
148 resize(left ,left_for_matcher ,Size(),0.5,0.5, INTER_LINEAR_EXACT);
149 resize(right,right_for_matcher,Size(),0.5,0.5, INTER_LINEAR_EXACT);
150 //! [downscale]
151 }
152 else
153 {
154 left_for_matcher = left.clone();
155 right_for_matcher = right.clone();
156 }
157
158 if(algo=="bm")
159 {
160 //! [matching]
161 Ptr<StereoBM> left_matcher = StereoBM::create(max_disp,wsize);
162 wls_filter = createDisparityWLSFilter(left_matcher);
163 Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
164
165 cvtColor(left_for_matcher, left_for_matcher, COLOR_BGR2GRAY);
166 cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
167
168 matching_time = (double)getTickCount();
169 left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
170 right_matcher->compute(right_for_matcher,left_for_matcher, right_disp);
171 matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
172 //! [matching]
173 }
174 else if(algo=="sgbm")
175 {
176 Ptr<StereoSGBM> left_matcher = StereoSGBM::create(0,max_disp,wsize);
177 left_matcher->setP1(24*wsize*wsize);
178 left_matcher->setP2(96*wsize*wsize);
179 left_matcher->setPreFilterCap(63);
180 left_matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
181 wls_filter = createDisparityWLSFilter(left_matcher);
182 Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
183
184 matching_time = (double)getTickCount();
185 left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
186 right_matcher->compute(right_for_matcher,left_for_matcher, right_disp);
187 matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
188 }
189 else
190 {
191 cout<<"Unsupported algorithm";
192 return -1;
193 }
194
195 //! [filtering]
196 wls_filter->setLambda(lambda);
197 wls_filter->setSigmaColor(sigma);
198 filtering_time = (double)getTickCount();
199 wls_filter->filter(left_disp,left,filtered_disp,right_disp);
200 filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
201 //! [filtering]
202 conf_map = wls_filter->getConfidenceMap();
203
204 // Get the ROI that was used in the last filter call:
205 ROI = wls_filter->getROI();
206 if(!no_downscale)
207 {
208 // upscale raw disparity and ROI back for a proper comparison:
209 resize(left_disp,left_disp,Size(),2.0,2.0,INTER_LINEAR_EXACT);
210 left_disp = left_disp*2.0;
211 ROI = Rect(ROI.x*2,ROI.y*2,ROI.width*2,ROI.height*2);
212 }
213 }
214 else if(filter=="fbs_conf") // filtering with fbs and confidence using also wls pre-processing
215 {
216 if(!no_downscale)
217 {
218 // downscale the views to speed-up the matching stage, as we will need to compute both left
219 // and right disparity maps for confidence map computation
220 //! [downscale_wls]
221 max_disp/=2;
222 if(max_disp%16!=0)
223 max_disp += 16-(max_disp%16);
224 resize(left ,left_for_matcher ,Size(),0.5,0.5);
225 resize(right,right_for_matcher,Size(),0.5,0.5);
226 //! [downscale_wls]
227 }
228 else
229 {
230 left_for_matcher = left.clone();
231 right_for_matcher = right.clone();
232 }
233
234 if(algo=="bm")
235 {
236 //! [matching_wls]
237 Ptr<StereoBM> left_matcher = StereoBM::create(max_disp,wsize);
238 wls_filter = createDisparityWLSFilter(left_matcher);
239 Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
240
241 cvtColor(left_for_matcher, left_for_matcher, COLOR_BGR2GRAY);
242 cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
243
244 matching_time = (double)getTickCount();
245 left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
246 right_matcher->compute(right_for_matcher,left_for_matcher, right_disp);
247 matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
248 //! [matching_wls]
249 }
250 else if(algo=="sgbm")
251 {
252 Ptr<StereoSGBM> left_matcher = StereoSGBM::create(0,max_disp,wsize);
253 left_matcher->setP1(24*wsize*wsize);
254 left_matcher->setP2(96*wsize*wsize);
255 left_matcher->setPreFilterCap(63);
256 left_matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
257 wls_filter = createDisparityWLSFilter(left_matcher);
258 Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
259
260 matching_time = (double)getTickCount();
261 left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
262 right_matcher->compute(right_for_matcher,left_for_matcher, right_disp);
263 matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
264 }
265 else
266 {
267 cout<<"Unsupported algorithm";
268 return -1;
269 }
270
271 //! [filtering_wls]
272 wls_filter->setLambda(lambda);
273 wls_filter->setSigmaColor(sigma);
274 filtering_time = (double)getTickCount();
275 wls_filter->filter(left_disp,left,filtered_disp,right_disp);
276 filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
277 //! [filtering_wls]
278
279 conf_map = wls_filter->getConfidenceMap();
280
281 Mat left_disp_resized;
282 resize(left_disp,left_disp_resized,left.size());
283
284 // Get the ROI that was used in the last filter call:
285 ROI = wls_filter->getROI();
286 if(!no_downscale)
287 {
288 // upscale raw disparity and ROI back for a proper comparison:
289 resize(left_disp,left_disp,Size(),2.0,2.0);
290 left_disp = left_disp*2.0;
291 left_disp_resized = left_disp_resized*2.0;
292 ROI = Rect(ROI.x*2,ROI.y*2,ROI.width*2,ROI.height*2);
293 }
294
295 #ifdef HAVE_EIGEN
296 //! [filtering_fbs]
297 solving_time = (double)getTickCount();
298 fastBilateralSolverFilter(left, left_disp_resized, conf_map/255.0f, solved_disp, fbs_spatial, fbs_luma, fbs_chroma, fbs_lambda);
299 solving_time = ((double)getTickCount() - solving_time)/getTickFrequency();
300 //! [filtering_fbs]
301
302 //! [filtering_wls2fbs]
303 fastBilateralSolverFilter(left, filtered_disp, conf_map/255.0f, solved_filtered_disp, fbs_spatial, fbs_luma, fbs_chroma, fbs_lambda);
304 //! [filtering_wls2fbs]
305 #else
306 (void)fbs_spatial;
307 (void)fbs_luma;
308 (void)fbs_chroma;
309 (void)fbs_lambda;
310 #endif
311 }
312 else if(filter=="wls_no_conf")
313 {
314 /* There is no convenience function for the case of filtering with no confidence, so we
315 will need to set the ROI and matcher parameters manually */
316
317 left_for_matcher = left.clone();
318 right_for_matcher = right.clone();
319
320 if(algo=="bm")
321 {
322 Ptr<StereoBM> matcher = StereoBM::create(max_disp,wsize);
323 matcher->setTextureThreshold(0);
324 matcher->setUniquenessRatio(0);
325 cvtColor(left_for_matcher, left_for_matcher, COLOR_BGR2GRAY);
326 cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
327 ROI = computeROI(left_for_matcher.size(),matcher);
328 wls_filter = createDisparityWLSFilterGeneric(false);
329 wls_filter->setDepthDiscontinuityRadius((int)ceil(0.33*wsize));
330
331 matching_time = (double)getTickCount();
332 matcher->compute(left_for_matcher,right_for_matcher,left_disp);
333 matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
334 }
335 else if(algo=="sgbm")
336 {
337 Ptr<StereoSGBM> matcher = StereoSGBM::create(0,max_disp,wsize);
338 matcher->setUniquenessRatio(0);
339 matcher->setDisp12MaxDiff(1000000);
340 matcher->setSpeckleWindowSize(0);
341 matcher->setP1(24*wsize*wsize);
342 matcher->setP2(96*wsize*wsize);
343 matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
344 ROI = computeROI(left_for_matcher.size(),matcher);
345 wls_filter = createDisparityWLSFilterGeneric(false);
346 wls_filter->setDepthDiscontinuityRadius((int)ceil(0.5*wsize));
347
348 matching_time = (double)getTickCount();
349 matcher->compute(left_for_matcher,right_for_matcher,left_disp);
350 matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
351 }
352 else
353 {
354 cout<<"Unsupported algorithm";
355 return -1;
356 }
357
358 wls_filter->setLambda(lambda);
359 wls_filter->setSigmaColor(sigma);
360 filtering_time = (double)getTickCount();
361 wls_filter->filter(left_disp,left,filtered_disp,Mat(),ROI);
362 filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
363 }
364 else
365 {
366 cout<<"Unsupported filter";
367 return -1;
368 }
369
370 //collect and print all the stats:
371 cout.precision(2);
372 cout<<"Matching time: "<<matching_time<<"s"<<endl;
373 cout<<"Filtering time: "<<filtering_time<<"s"<<endl;
374 cout<<"Solving time: "<<solving_time<<"s"<<endl;
375 cout<<endl;
376
377 double MSE_before,percent_bad_before,MSE_after,percent_bad_after;
378 if(!noGT)
379 {
380 MSE_before = computeMSE(GT_disp,left_disp,ROI);
381 percent_bad_before = computeBadPixelPercent(GT_disp,left_disp,ROI);
382 MSE_after = computeMSE(GT_disp,filtered_disp,ROI);
383 percent_bad_after = computeBadPixelPercent(GT_disp,filtered_disp,ROI);
384
385 cout.precision(5);
386 cout<<"MSE before filtering: "<<MSE_before<<endl;
387 cout<<"MSE after filtering: "<<MSE_after<<endl;
388 cout<<endl;
389 cout.precision(3);
390 cout<<"Percent of bad pixels before filtering: "<<percent_bad_before<<endl;
391 cout<<"Percent of bad pixels after filtering: "<<percent_bad_after<<endl;
392 }
393
394 if(dst_path!="None")
395 {
396 Mat filtered_disp_vis;
397 getDisparityVis(filtered_disp,filtered_disp_vis,vis_mult);
398 imwrite(dst_path,filtered_disp_vis);
399 }
400 if(dst_raw_path!="None")
401 {
402 Mat raw_disp_vis;
403 getDisparityVis(left_disp,raw_disp_vis,vis_mult);
404 imwrite(dst_raw_path,raw_disp_vis);
405 }
406 if(dst_conf_path!="None")
407 {
408 imwrite(dst_conf_path,conf_map);
409 }
410
411 if(!no_display)
412 {
413 namedWindow("left", WINDOW_AUTOSIZE);
414 imshow("left", left);
415 namedWindow("right", WINDOW_AUTOSIZE);
416 imshow("right", right);
417
418 if(!noGT)
419 {
420 Mat GT_disp_vis;
421 getDisparityVis(GT_disp,GT_disp_vis,vis_mult);
422 namedWindow("ground-truth disparity", WINDOW_AUTOSIZE);
423 imshow("ground-truth disparity", GT_disp_vis);
424 }
425
426 //! [visualization]
427 Mat raw_disp_vis;
428 getDisparityVis(left_disp,raw_disp_vis,vis_mult);
429 namedWindow("raw disparity", WINDOW_AUTOSIZE);
430 imshow("raw disparity", raw_disp_vis);
431 Mat filtered_disp_vis;
432 getDisparityVis(filtered_disp,filtered_disp_vis,vis_mult);
433 namedWindow("filtered disparity", WINDOW_AUTOSIZE);
434 imshow("filtered disparity", filtered_disp_vis);
435
436 if(!solved_disp.empty())
437 {
438 Mat solved_disp_vis;
439 getDisparityVis(solved_disp,solved_disp_vis,vis_mult);
440 namedWindow("solved disparity", WINDOW_AUTOSIZE);
441 imshow("solved disparity", solved_disp_vis);
442
443 Mat solved_filtered_disp_vis;
444 getDisparityVis(solved_filtered_disp,solved_filtered_disp_vis,vis_mult);
445 namedWindow("solved wls disparity", WINDOW_AUTOSIZE);
446 imshow("solved wls disparity", solved_filtered_disp_vis);
447 }
448
449 while(1)
450 {
451 char key = (char)waitKey();
452 if( key == 27 || key == 'q' || key == 'Q') // 'ESC'
453 break;
454 }
455 //! [visualization]
456 }
457
458 return 0;
459 }
460
computeROI(Size2i src_sz,Ptr<StereoMatcher> matcher_instance)461 Rect computeROI(Size2i src_sz, Ptr<StereoMatcher> matcher_instance)
462 {
463 int min_disparity = matcher_instance->getMinDisparity();
464 int num_disparities = matcher_instance->getNumDisparities();
465 int block_size = matcher_instance->getBlockSize();
466
467 int bs2 = block_size/2;
468 int minD = min_disparity, maxD = min_disparity + num_disparities - 1;
469
470 int xmin = maxD + bs2;
471 int xmax = src_sz.width + minD - bs2;
472 int ymin = bs2;
473 int ymax = src_sz.height - bs2;
474
475 Rect r(xmin, ymin, xmax - xmin, ymax - ymin);
476 return r;
477 }
478
