1 // Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // * Redistributions of source code must retain the above copyright 8 // notice, this list of conditions and the following disclaimer. 9 // 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // 14 // * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 // its contributors may be used to endorse or promote products derived 16 // from this software without specific prior written permission. 17 // 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 // POSSIBILITY OF SUCH DAMAGE. 29 // 30 // Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 32 #ifndef COLMAP_SRC_MVS_FUSION_H_ 33 #define COLMAP_SRC_MVS_FUSION_H_ 34 35 #include <unordered_set> 36 #include <vector> 37 38 #include <Eigen/Core> 39 40 #include "mvs/depth_map.h" 41 #include "mvs/image.h" 42 #include "mvs/mat.h" 43 #include "mvs/model.h" 44 #include "mvs/normal_map.h" 45 #include "mvs/workspace.h" 46 #include "util/alignment.h" 47 #include "util/cache.h" 48 #include "util/math.h" 49 #include "util/ply.h" 50 #include "util/threading.h" 51 52 namespace colmap { 53 namespace mvs { 54 55 struct StereoFusionOptions { 56 // Maximum image size in either dimension. 57 int max_image_size = -1; 58 59 // Minimum number of fused pixels to produce a point. 60 int min_num_pixels = 5; 61 62 // Maximum number of pixels to fuse into a single point. 63 int max_num_pixels = 10000; 64 65 // Maximum depth in consistency graph traversal. 66 int max_traversal_depth = 100; 67 68 // Maximum relative difference between measured and projected pixel. 69 double max_reproj_error = 2.0f; 70 71 // Maximum relative difference between measured and projected depth. 72 double max_depth_error = 0.01f; 73 74 // Maximum angular difference in degrees of normals of pixels to be fused. 75 double max_normal_error = 10.0f; 76 77 // Number of overlapping images to transitively check for fusing points. 78 int check_num_images = 50; 79 80 // Cache size in gigabytes for fusion. The fusion keeps the bitmaps, depth 81 // maps, normal maps, and consistency graphs of this number of images in 82 // memory. A higher value leads to less disk access and faster fusion, while 83 // a lower value leads to reduced memory usage. Note that a single image can 84 // consume a lot of memory, if the consistency graph is dense. 85 double cache_size = 32.0; 86 87 // Check the options for validity. 88 bool Check() const; 89 90 // Print the options to stdout. 91 void Print() const; 92 }; 93 94 class StereoFusion : public Thread { 95 public: 96 EIGEN_MAKE_ALIGNED_OPERATOR_NEW 97 98 StereoFusion(const StereoFusionOptions& options, 99 const std::string& workspace_path, 100 const std::string& workspace_format, 101 const std::string& pmvs_option_name, 102 const std::string& input_type); 103 104 const std::vector<PlyPoint>& GetFusedPoints() const; 105 const std::vector<std::vector<int>>& GetFusedPointsVisibility() const; 106 107 private: 108 void Run(); 109 void Fuse(); 110 111 const StereoFusionOptions options_; 112 const std::string workspace_path_; 113 const std::string workspace_format_; 114 const std::string pmvs_option_name_; 115 const std::string input_type_; 116 const float max_squared_reproj_error_; 117 const float min_cos_normal_error_; 118 119 std::unique_ptr<Workspace> workspace_; 120 std::vector<char> used_images_; 121 std::vector<char> fused_images_; 122 std::vector<std::vector<int>> overlapping_images_; 123 std::vector<Mat<bool>> fused_pixel_masks_; 124 std::vector<std::pair<int, int>> depth_map_sizes_; 125 std::vector<std::pair<float, float>> bitmap_scales_; 126 std::vector<Eigen::Matrix<float, 3, 4, Eigen::RowMajor>> P_; 127 std::vector<Eigen::Matrix<float, 3, 4, Eigen::RowMajor>> inv_P_; 128 std::vector<Eigen::Matrix<float, 3, 3, Eigen::RowMajor>> inv_R_; 129 130 struct FusionData { 131 int image_idx = kInvalidImageId; 132 int row = 0; 133 int col = 0; 134 int traversal_depth = -1; operatorFusionData135 bool operator()(const FusionData& data1, const FusionData& data2) { 136 return data1.image_idx > data2.image_idx; 137 } 138 }; 139 140 // Next points to fuse. 141 std::vector<FusionData> fusion_queue_; 142 143 // Already fused points. 144 std::vector<PlyPoint> fused_points_; 145 std::vector<std::vector<int>> fused_points_visibility_; 146 147 // Points of different pixels of the currently point to be fused. 148 std::vector<float> fused_point_x_; 149 std::vector<float> fused_point_y_; 150 std::vector<float> fused_point_z_; 151 std::vector<float> fused_point_nx_; 152 std::vector<float> fused_point_ny_; 153 std::vector<float> fused_point_nz_; 154 std::vector<uint8_t> fused_point_r_; 155 std::vector<uint8_t> fused_point_g_; 156 std::vector<uint8_t> fused_point_b_; 157 std::unordered_set<int> fused_point_visibility_; 158 }; 159 160 // Write the visiblity information into a binary file of the following format: 161 // 162 // <num_points : uint64_t> 163 // <num_visible_images_for_point1 : uint32_t> 164 // <point1_image_idx1 : uint32_t><point1_image_idx2 : uint32_t> ... 165 // <num_visible_images_for_point2 : uint32_t> 166 // <point2_image_idx2 : uint32_t><point2_image_idx2 : uint32_t> ... 167 // ... 168 // 169 // Note that an image_idx in the case of the mvs::StereoFuser does not 170 // correspond to the image_id of a Reconstruction, but the index of the image in 171 // the mvs::Model, which is the location of the image in the images.bin/.txt. 172 void WritePointsVisibility( 173 const std::string& path, 174 const std::vector<std::vector<int>>& points_visibility); 175 176 } // namespace mvs 177 } // namespace colmap 178 179 #endif // COLMAP_SRC_MVS_FUSION_H_ 180