1 // Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are met: 6 // 7 // * Redistributions of source code must retain the above copyright 8 // notice, this list of conditions and the following disclaimer. 9 // 10 // * Redistributions in binary form must reproduce the above copyright 11 // notice, this list of conditions and the following disclaimer in the 12 // documentation and/or other materials provided with the distribution. 13 // 14 // * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of 15 // its contributors may be used to endorse or promote products derived 16 // from this software without specific prior written permission. 17 // 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE 22 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 // POSSIBILITY OF SUCH DAMAGE. 29 // 30 // Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) 31 32 #ifndef COLMAP_SRC_MVS_PATCH_MATCH_CUDA_H_ 33 #define COLMAP_SRC_MVS_PATCH_MATCH_CUDA_H_ 34 35 #include <iostream> 36 #include <memory> 37 #include <vector> 38 39 #include <cuda_runtime.h> 40 41 #include "mvs/cuda_array_wrapper.h" 42 #include "mvs/depth_map.h" 43 #include "mvs/gpu_mat.h" 44 #include "mvs/gpu_mat_prng.h" 45 #include "mvs/gpu_mat_ref_image.h" 46 #include "mvs/image.h" 47 #include "mvs/normal_map.h" 48 #include "mvs/patch_match.h" 49 50 namespace colmap { 51 namespace mvs { 52 53 class PatchMatchCuda { 54 public: 55 PatchMatchCuda(const PatchMatchOptions& options, 56 const PatchMatch::Problem& problem); 57 ~PatchMatchCuda(); 58 59 void Run(); 60 61 DepthMap GetDepthMap() const; 62 NormalMap GetNormalMap() const; 63 Mat<float> GetSelProbMap() const; 64 std::vector<int> GetConsistentImageIdxs() const; 65 66 private: 67 template <int kWindowSize, int kWindowStep> 68 void RunWithWindowSizeAndStep(); 69 70 void ComputeCudaConfig(); 71 72 void InitRefImage(); 73 void InitSourceImages(); 74 void InitTransforms(); 75 void InitWorkspaceMemory(); 76 77 // Rotate reference image by 90 degrees in counter-clockwise direction. 78 void Rotate(); 79 80 const PatchMatchOptions options_; 81 const PatchMatch::Problem problem_; 82 83 // Dimensions for sweeping from top to bottom, i.e. one thread per column. 84 dim3 sweep_block_size_; 85 dim3 sweep_grid_size_; 86 // Dimensions for element-wise operations, i.e. one thread per pixel. 87 dim3 elem_wise_block_size_; 88 dim3 elem_wise_grid_size_; 89 90 // Original (not rotated) dimension of reference image. 91 size_t ref_width_; 92 size_t ref_height_; 93 94 // Rotation of reference image in pi/2. This is equivalent to the number of 95 // calls to `rotate` mod 4. 96 int rotation_in_half_pi_; 97 98 std::unique_ptr<CudaArrayWrapper<uint8_t>> ref_image_device_; 99 std::unique_ptr<CudaArrayWrapper<uint8_t>> src_images_device_; 100 std::unique_ptr<CudaArrayWrapper<float>> src_depth_maps_device_; 101 102 // Relative poses from rotated versions of reference image to source images 103 // corresponding to _rotationInHalfPi: 104 // 105 // [S(1), S(2), S(3), ..., S(n)] 106 // 107 // where n is the number of source images and: 108 // 109 // S(i) = [K_i(0, 0), K_i(0, 2), K_i(1, 1), K_i(1, 2), R_i(:), T_i(:) 110 // C_i(:), P(:), P^-1(:)] 111 // 112 // where i denotes the index of the source image and K is its calibration. 113 // R, T, C, P, P^-1 denote the relative rotation, translation, camera 114 // center, projection, and inverse projection from there reference to the 115 // i-th source image. 116 std::unique_ptr<CudaArrayWrapper<float>> poses_device_[4]; 117 118 // Calibration matrix for rotated versions of reference image 119 // as {K[0, 0], K[0, 2], K[1, 1], K[1, 2]} corresponding to _rotationInHalfPi. 120 float ref_K_host_[4][4]; 121 float ref_inv_K_host_[4][4]; 122 123 // Data for reference image. 124 std::unique_ptr<GpuMatRefImage> ref_image_; 125 std::unique_ptr<GpuMat<float>> depth_map_; 126 std::unique_ptr<GpuMat<float>> normal_map_; 127 std::unique_ptr<GpuMat<float>> sel_prob_map_; 128 std::unique_ptr<GpuMat<float>> prev_sel_prob_map_; 129 std::unique_ptr<GpuMat<float>> cost_map_; 130 std::unique_ptr<GpuMatPRNG> rand_state_map_; 131 std::unique_ptr<GpuMat<uint8_t>> consistency_mask_; 132 133 // Shared memory is too small to hold local state for each thread, 134 // so this is workspace memory in global memory. 135 std::unique_ptr<GpuMat<float>> global_workspace_; 136 }; 137 138 } // namespace mvs 139 } // namespace colmap 140 141 #endif // COLMAP_SRC_MVS_PATCH_MATCH_CUDA_H_ 142