1 // Copyright (c) 2018, ETH Zurich and UNC Chapel Hill.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 //     * Redistributions of source code must retain the above copyright
8 //       notice, this list of conditions and the following disclaimer.
9 //
10 //     * Redistributions in binary form must reproduce the above copyright
11 //       notice, this list of conditions and the following disclaimer in the
12 //       documentation and/or other materials provided with the distribution.
13 //
14 //     * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of
15 //       its contributors may be used to endorse or promote products derived
16 //       from this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
22 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 // POSSIBILITY OF SUCH DAMAGE.
29 //
30 // Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de)
31 
32 #ifndef COLMAP_SRC_MVS_FUSION_H_
33 #define COLMAP_SRC_MVS_FUSION_H_
34 
35 #include <unordered_set>
36 #include <vector>
37 
38 #include <Eigen/Core>
39 
40 #include "mvs/depth_map.h"
41 #include "mvs/image.h"
42 #include "mvs/mat.h"
43 #include "mvs/model.h"
44 #include "mvs/normal_map.h"
45 #include "mvs/workspace.h"
46 #include "util/alignment.h"
47 #include "util/cache.h"
48 #include "util/math.h"
49 #include "util/ply.h"
50 #include "util/threading.h"
51 
52 namespace colmap {
53 namespace mvs {
54 
55 struct StereoFusionOptions {
56   // Maximum image size in either dimension.
57   int max_image_size = -1;
58 
59   // Minimum number of fused pixels to produce a point.
60   int min_num_pixels = 5;
61 
62   // Maximum number of pixels to fuse into a single point.
63   int max_num_pixels = 10000;
64 
65   // Maximum depth in consistency graph traversal.
66   int max_traversal_depth = 100;
67 
68   // Maximum relative difference between measured and projected pixel.
69   double max_reproj_error = 2.0f;
70 
71   // Maximum relative difference between measured and projected depth.
72   double max_depth_error = 0.01f;
73 
74   // Maximum angular difference in degrees of normals of pixels to be fused.
75   double max_normal_error = 10.0f;
76 
77   // Number of overlapping images to transitively check for fusing points.
78   int check_num_images = 50;
79 
80   // Cache size in gigabytes for fusion. The fusion keeps the bitmaps, depth
81   // maps, normal maps, and consistency graphs of this number of images in
82   // memory. A higher value leads to less disk access and faster fusion, while
83   // a lower value leads to reduced memory usage. Note that a single image can
84   // consume a lot of memory, if the consistency graph is dense.
85   double cache_size = 32.0;
86 
87   // Check the options for validity.
88   bool Check() const;
89 
90   // Print the options to stdout.
91   void Print() const;
92 };
93 
94 class StereoFusion : public Thread {
95  public:
96   EIGEN_MAKE_ALIGNED_OPERATOR_NEW
97 
98   StereoFusion(const StereoFusionOptions& options,
99                const std::string& workspace_path,
100                const std::string& workspace_format,
101                const std::string& pmvs_option_name,
102                const std::string& input_type);
103 
104   const std::vector<PlyPoint>& GetFusedPoints() const;
105   const std::vector<std::vector<int>>& GetFusedPointsVisibility() const;
106 
107  private:
108   void Run();
109   void Fuse();
110 
111   const StereoFusionOptions options_;
112   const std::string workspace_path_;
113   const std::string workspace_format_;
114   const std::string pmvs_option_name_;
115   const std::string input_type_;
116   const float max_squared_reproj_error_;
117   const float min_cos_normal_error_;
118 
119   std::unique_ptr<Workspace> workspace_;
120   std::vector<char> used_images_;
121   std::vector<char> fused_images_;
122   std::vector<std::vector<int>> overlapping_images_;
123   std::vector<Mat<bool>> fused_pixel_masks_;
124   std::vector<std::pair<int, int>> depth_map_sizes_;
125   std::vector<std::pair<float, float>> bitmap_scales_;
126   std::vector<Eigen::Matrix<float, 3, 4, Eigen::RowMajor>> P_;
127   std::vector<Eigen::Matrix<float, 3, 4, Eigen::RowMajor>> inv_P_;
128   std::vector<Eigen::Matrix<float, 3, 3, Eigen::RowMajor>> inv_R_;
129 
130   struct FusionData {
131     int image_idx = kInvalidImageId;
132     int row = 0;
133     int col = 0;
134     int traversal_depth = -1;
operatorFusionData135     bool operator()(const FusionData& data1, const FusionData& data2) {
136       return data1.image_idx > data2.image_idx;
137     }
138   };
139 
140   // Next points to fuse.
141   std::vector<FusionData> fusion_queue_;
142 
143   // Already fused points.
144   std::vector<PlyPoint> fused_points_;
145   std::vector<std::vector<int>> fused_points_visibility_;
146 
147   // Points of different pixels of the currently point to be fused.
148   std::vector<float> fused_point_x_;
149   std::vector<float> fused_point_y_;
150   std::vector<float> fused_point_z_;
151   std::vector<float> fused_point_nx_;
152   std::vector<float> fused_point_ny_;
153   std::vector<float> fused_point_nz_;
154   std::vector<uint8_t> fused_point_r_;
155   std::vector<uint8_t> fused_point_g_;
156   std::vector<uint8_t> fused_point_b_;
157   std::unordered_set<int> fused_point_visibility_;
158 };
159 
160 // Write the visiblity information into a binary file of the following format:
161 //
162 //    <num_points : uint64_t>
163 //    <num_visible_images_for_point1 : uint32_t>
164 //    <point1_image_idx1 : uint32_t><point1_image_idx2 : uint32_t> ...
165 //    <num_visible_images_for_point2 : uint32_t>
166 //    <point2_image_idx2 : uint32_t><point2_image_idx2 : uint32_t> ...
167 //    ...
168 //
169 // Note that an image_idx in the case of the mvs::StereoFuser does not
170 // correspond to the image_id of a Reconstruction, but the index of the image in
171 // the mvs::Model, which is the location of the image in the images.bin/.txt.
172 void WritePointsVisibility(
173     const std::string& path,
174     const std::vector<std::vector<int>>& points_visibility);
175 
176 }  // namespace mvs
177 }  // namespace colmap
178 
179 #endif  // COLMAP_SRC_MVS_FUSION_H_
180