1 // -*- c-basic-offset: 4 ; tab-width: 4 -*-
2 /*
3 * Copyright (C) 2007-2008 Anael Orlinski
4 *
5 * This file is part of Panomatic.
6 *
7 * Panomatic is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * Panomatic is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with Panomatic; if not, write to the Free Software
19 * <http://www.gnu.org/licenses/>.
20 */
21
22 #ifndef __detectpano_panodetector_h
23 #define __detectpano_panodetector_h
24
25 #include <hugin_config.h>
26
27 #include "PanoDetectorDefs.h"
28 #include <memory>
29 #include <string>
30 #include <map>
31 #include <localfeatures/Image.h>
32 #include <localfeatures/PointMatch.h>
33 #include "TestCode.h"
34
35 #include <localfeatures/KeyPoint.h>
36 #include <localfeatures/KeyPointDetector.h>
37
38 #include <flann/flann.hpp>
39
40 #include <vigra_ext/ROIImage.h>
41
42 #include <panodata/Panorama.h>
43 #include <algorithms/optimizer/PTOptimizer.h>
44 #include <celeste/Celeste.h>
45
46 class PanoDetector
47 {
48 public:
49 typedef std::vector<std::string> FileNameList_t;
50 typedef std::vector<std::string>::iterator FileNameListIt_t;
51 typedef KDTreeSpace::KDTree<KDElemKeyPoint, double> KPKDTree;
52 typedef std::shared_ptr<KPKDTree> KPKDTreePtr;
53
54 typedef lfeat::KeyPointDetector KeyPointDetector;
55
56 /** for selecting matching strategy */
57 enum MatchingStrategy
58 {
59 ALLPAIRS=0,
60 LINEAR,
61 MULTIROW,
62 PREALIGNED
63 };
64
65 PanoDetector();
66 ~PanoDetector();
67
68 bool checkData();
69 void printDetails();
70 void printFilenames();
71 void printHelp();
72 void run();
73 bool match(std::vector<HuginBase::UIntSet> &checkedPairs);
74 bool matchMultiRow();
75 /** does only matches image pairs which overlaps and don't have control points
76 @param aExecutor executor for threading
77 @param pano pano, which should be used for determing of overlap, can contain also less images than _panoramaInfo
78 @param connectedImages contains a list of already connected or tested image pairs, which should be skipped
79 @param imgMap map of image nr in partial pano and full panorama
80 @param exactOverlap if true, only really overlapping image pairs are matched, if false it increases the hfov
81 to take also narrow overlaps better into account
82 @return true, if detection was successful
83 */
84 bool matchPrealigned(HuginBase::Panorama* pano, std::vector<HuginBase::UIntSet> &connectedImages, std::vector<size_t> imgMap, bool exactOverlap=true);
85
86
87 // accessors
88
getPanoramaInfo()89 inline HuginBase::Panorama* getPanoramaInfo() const
90 {
91 return _panoramaInfo;
92 }
93
setKeyPointsIdx(std::vector<int> keyPointsIdx)94 inline void setKeyPointsIdx(std::vector<int> keyPointsIdx)
95 {
96 _keyPointsIdx = keyPointsIdx;
97 }
getKeyPointsIdx()98 inline std::vector<int> getKeyPointsIdx() const
99 {
100 return _keyPointsIdx;
101 }
102 inline void setWriteAllKeyPoints(bool writeAllKeyPoints=true)
103 {
104 _writeAllKeyPoints = writeAllKeyPoints;
105 }
getWriteAllKeyPoints()106 inline bool getWriteAllKeyPoints() const
107 {
108 return _writeAllKeyPoints;
109 }
110
setVerbose(int level)111 inline void setVerbose(int level)
112 {
113 _verbose = level;
114 }
getVerbose()115 inline int getVerbose() const
116 {
117 return _verbose;
118 }
119
setSieve1Width(int iWidth)120 inline void setSieve1Width(int iWidth)
121 {
122 _sieve1Width = iWidth;
123 }
setSieve1Height(int iHeight)124 inline void setSieve1Height(int iHeight)
125 {
126 _sieve1Height = iHeight;
127 }
setSieve1Size(int iSize)128 inline void setSieve1Size(int iSize)
129 {
130 _sieve1Size = iSize;
131 }
getSieve1Width()132 inline int getSieve1Width() const
133 {
134 return _sieve1Width;
135 }
getSieve1Height()136 inline int getSieve1Height() const
137 {
138 return _sieve1Height;
139 }
getSieve1Size()140 inline int getSieve1Size() const
141 {
142 return _sieve1Size;
143 }
144
setKDTreeSearchSteps(int iSteps)145 inline void setKDTreeSearchSteps(int iSteps)
146 {
147 _kdTreeSearchSteps = iSteps;
148 }
setKDTreeSecondDistance(double iDist)149 inline void setKDTreeSecondDistance(double iDist)
150 {
151 _kdTreeSecondDistance = iDist;
152 }
getKDTreeSearchSteps()153 inline int getKDTreeSearchSteps() const
154 {
155 return _kdTreeSearchSteps;
156 }
getKDTreeSecondDistance()157 inline double getKDTreeSecondDistance() const
158 {
159 return _kdTreeSecondDistance;
160 }
161
setMinimumMatches(int iMatches)162 inline void setMinimumMatches(int iMatches)
163 {
164 _minimumMatches = iMatches;
165 }
setRansacIterations(int iIters)166 inline void setRansacIterations(int iIters)
167 {
168 _ransacIters = iIters;
169 }
setRansacDistanceThreshold(int iDT)170 inline void setRansacDistanceThreshold(int iDT)
171 {
172 _ransacDistanceThres = iDT;
173 }
setRansacMode(HuginBase::RANSACOptimizer::Mode mode)174 inline void setRansacMode(HuginBase::RANSACOptimizer::Mode mode)
175 {
176 _ransacMode = mode;
177 }
getMinimumMatches()178 inline int getMinimumMatches() const
179 {
180 return _minimumMatches;
181 }
getRansacIterations()182 inline int getRansacIterations() const
183 {
184 return _ransacIters;
185 }
getRansacDistanceThreshold()186 inline int getRansacDistanceThreshold() const
187 {
188 return _ransacDistanceThres;
189 }
getRansacMode()190 inline HuginBase::RANSACOptimizer::Mode getRansacMode()
191 {
192 return _ransacMode;
193 }
194
setSieve2Width(int iWidth)195 inline void setSieve2Width(int iWidth)
196 {
197 _sieve2Width = iWidth;
198 }
setSieve2Height(int iHeight)199 inline void setSieve2Height(int iHeight)
200 {
201 _sieve2Height = iHeight;
202 }
setSieve2Size(int iSize)203 inline void setSieve2Size(int iSize)
204 {
205 _sieve2Size = iSize;
206 }
getSieve2Width()207 inline int getSieve2Width() const
208 {
209 return _sieve2Width;
210 }
getSieve2Height()211 inline int getSieve2Height() const
212 {
213 return _sieve2Height;
214 }
getSieve2Size()215 inline int getSieve2Size() const
216 {
217 return _sieve2Size;
218 }
219
setLinearMatchLen(int iLen)220 inline void setLinearMatchLen(int iLen)
221 {
222 _linearMatchLen = iLen;
223 }
getLinearMatchLen()224 inline int getLinearMatchLen() const
225 {
226 return _linearMatchLen;
227 }
setMatchingStrategy(MatchingStrategy iMatchStrategy)228 inline void setMatchingStrategy(MatchingStrategy iMatchStrategy)
229 {
230 _matchingStrategy = iMatchStrategy;
231 }
getMatchingStrategy()232 inline MatchingStrategy getMatchingStrategy() const
233 {
234 return _matchingStrategy;
235 }
236
getDownscale()237 inline bool getDownscale() const
238 {
239 return _downscale;
240 }
setDownscale(bool iDown)241 inline void setDownscale(bool iDown)
242 {
243 _downscale = iDown;
244 }
245
246 // inline void setNumberOfKeys(int iNumKeys) { _numKeys = iNumKeys; }
setOutputFile(const std::string & outputFile)247 inline void setOutputFile(const std::string& outputFile)
248 {
249 _outputFile = outputFile;
250 _outputGiven=true;
251 }
setInputFile(const std::string & inputFile)252 inline void setInputFile(const std::string& inputFile)
253 {
254 _inputFile = inputFile;
255 }
setKeyfilesPath(const std::string & keypath)256 inline void setKeyfilesPath(const std::string& keypath)
257 {
258 _keypath = keypath;
259 }
getCached()260 inline bool getCached() const
261 {
262 return _cache;
263 }
setCached(bool iCached)264 inline void setCached(bool iCached)
265 {
266 _cache = iCached;
267 }
getCleanup()268 inline bool getCleanup() const
269 {
270 return _cleanup;
271 }
setCleanup(bool iCleanup)272 inline void setCleanup(bool iCleanup)
273 {
274 _cleanup = iCleanup;
275 }
getCeleste()276 inline bool getCeleste() const
277 {
278 return _celeste;
279 };
setCeleste(bool iCeleste)280 inline void setCeleste(bool iCeleste)
281 {
282 _celeste = iCeleste;
283 };
getCelesteThreshold()284 inline double getCelesteThreshold() const
285 {
286 return _celesteThreshold;
287 };
setCelesteThreshold(double iCelesteThreshold)288 inline void setCelesteThreshold(double iCelesteThreshold)
289 {
290 _celesteThreshold = iCelesteThreshold;
291 };
getCelesteRadius()292 inline int getCelesteRadius() const
293 {
294 return _celesteRadius;
295 };
setCelesteRadius(int iCelesteRadius)296 inline void setCelesteRadius(int iCelesteRadius)
297 {
298 _celesteRadius = iCelesteRadius;
299 };
setTest(bool iTest)300 inline void setTest(bool iTest)
301 {
302 _test = iTest;
303 }
getTest()304 inline bool getTest() const
305 {
306 return _test;
307 }
setCores(int iCores)308 inline void setCores(int iCores)
309 {
310 _cores = iCores;
311 }
312
313 // predeclaration
314 struct ImgData;
315 struct MatchData;
316
317 private:
318 // prevent copying of class
319 PanoDetector(const PanoDetector&);
320 PanoDetector& operator=(const PanoDetector&);
321
322 // options
323
324 bool _writeAllKeyPoints;
325 std::vector<int> _keyPointsIdx;
326
327 int _verbose;
328
329 int _sieve1Width;
330 int _sieve1Height;
331 int _sieve1Size;
332
333 int _kdTreeSearchSteps;
334 double _kdTreeSecondDistance;
335
336 int _minimumMatches;
337 HuginBase::RANSACOptimizer::Mode _ransacMode;
338 int _ransacIters;
339 int _ransacDistanceThres;
340
341 int _sieve2Width;
342 int _sieve2Height;
343 int _sieve2Size;
344
345 MatchingStrategy _matchingStrategy;
346 int _linearMatchLen;
347
348 bool _test;
349 int _cores;
350 bool _downscale;
351 bool _cache;
352 bool _cleanup;
353 bool _celeste;
354 double _celesteThreshold;
355 int _celesteRadius;
356 std::string _keypath;
357 std::string _prefix;
358
359 // bool _stereoRemap;
360
361 // list of files
362 std::string _outputFile;
363 bool _outputGiven;
364 std::string _inputFile;
365
366 // Store panorama information
367 HuginBase::Panorama* _panoramaInfo;
368 HuginBase::Panorama _panoramaInfoCopy;
369
370 /** search for image layer and image stacks for the multirow matching step */
371 void buildMultiRowImageSets();
372
373 /** image set contains only the images with the median exposure of each stack */
374 HuginBase::UIntSet _image_layer;
375 /** vector with image numbers of all stacks, contains only the unlinked stacks */
376 std::vector<HuginBase::UIntVector> _image_stacks;
377
378 bool loadProject();
379 bool checkLoadSuccess();
380 void CleanupKeyfiles();
381
382 void writeOutput();
383 void writeKeyfile(ImgData& imgInfo);
384
385 // internals
386 public:
387 struct ImgData
388 {
389 /** enumeration of different detection modes */
390 enum SizeMode { DOWNSCALED, REMAPPED, FULLSIZE};
391 std::string _name;
392
393 int _number;
394 int _detectWidth;
395 int _detectHeight;
396
397 lfeat::Image _ii;
398 vigra::BImage _distancemap;
399
400 HuginBase::PanoramaOptions _projOpts;
401
402 bool _hasakeyfile;
403 std::string _keyfilename;
404
405 lfeat::KeyPointVect_t _kp;
406 int _descLength;
407 bool _loadFail;
408
409 // kdtree
410 flann::Matrix<double> _flann_descriptors;
411 flann::Index<flann::L2<double> > * _flann_index;
412
ImgDataImgData413 ImgData()
414 {
415 _loadFail = false;
416 _number = 0;
417 _detectWidth = 0;
418 _detectHeight = 0;
419 m_sizeMode = FULLSIZE;
420 _hasakeyfile = false;
421 _descLength = 0;
422 _flann_index = NULL;
423 }
424
~ImgDataImgData425 ~ImgData()
426 {
427 if (_flann_index != NULL)
428 {
429 delete _flann_index;
430 };
431 if (_flann_descriptors.rows + _flann_descriptors.cols > 0)
432 {
433 delete[]_flann_descriptors.ptr();
434 };
435 }
SetSizeModeImgData436 void SetSizeMode(const SizeMode newSizeMode) { m_sizeMode = newSizeMode; };
GetSizeModeImgData437 SizeMode GetSizeMode() const { return m_sizeMode; };
IsDownscaleImgData438 bool IsDownscale() const { return m_sizeMode == DOWNSCALED; };
NeedsRemappingImgData439 bool NeedsRemapping() const { return m_sizeMode == REMAPPED; };
440 private:
441 SizeMode m_sizeMode;
442 };
443
444 typedef std::map<int, ImgData> ImgData_t;
445 typedef std::map<int, ImgData>::iterator ImgDataIt_t;
446
447 struct MatchData
448 {
449 ImgData* _i1;
450 ImgData* _i2;
451 lfeat::PointMatchVector_t _matches;
452 };
453
454 typedef std::vector<MatchData> MatchData_t;
455 typedef std::vector<MatchData>::iterator MatchDataIt_t;
456
457 // actions
458 static bool LoadKeypoints(ImgData& ioImgInfo, const PanoDetector& iPanoDetector);
459
460 static bool AnalyzeImage(ImgData& ioImgInfo, const PanoDetector& iPanoDetector);
461 static bool FindKeyPointsInImage(ImgData& ioImgInfo, const PanoDetector& iPanoDetector);
462 static bool FilterKeyPointsInImage(ImgData& ioImgInfo, const PanoDetector& iPanoDetector);
463 static bool MakeKeyPointDescriptorsInImage(ImgData& ioImgInfo, const PanoDetector& iPanoDetector);
464 static bool RemapBackKeypoints(ImgData& ioImgInfo, const PanoDetector& iPanoDetector);
465 static bool BuildKDTreesInImage(ImgData& ioImgInfo, const PanoDetector& iPanoDetector);
466 static bool FreeMemoryInImage(ImgData& ioImgInfo, const PanoDetector& iPanoDetector);
467
468 static bool FindMatchesInPair(MatchData& ioMatchData, const PanoDetector& iPanoDetector);
469 static bool RansacMatchesInPair(MatchData& ioMatchData, const PanoDetector& iPanoDetector);
470 static bool RansacMatchesInPairCam(MatchData& ioMatchData, const PanoDetector& iPanoDetector);
471 static bool RansacMatchesInPairHomography(MatchData& ioMatchData, const PanoDetector& iPanoDetector);
472 static bool FilterMatchesInPair(MatchData& ioMatchData, const PanoDetector& iPanoDetector);
473
474 private:
475 bool LoadSVMModel();
476 ImgData_t _filesData;
477 struct celeste::svm_model* svmModel;
478 };
479
480 /** returns the filename for the keyfile for a given image */
481 std::string getKeyfilenameFor(std::string keyfilesPath, std::string filename);
482
483 // dummy panotools progress functions
ptProgress(int command,char * argument)484 static int ptProgress( int command, char* argument )
485 {
486 return 1;
487 }
ptinfoDlg(int command,char * argument)488 static int ptinfoDlg( int command, char* argument )
489 {
490 return 1;
491 }
492
493 #endif // __detectpano_panodetector_h
494