1 // Copyright (c) 2018, ETH Zurich and UNC Chapel Hill.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright
8 // notice, this list of conditions and the following disclaimer.
9 //
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 //
14 // * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of
15 // its contributors may be used to endorse or promote products derived
16 // from this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
22 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 // POSSIBILITY OF SUCH DAMAGE.
29 //
30 // Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de)
31
32 #ifndef COLMAP_SRC_BASE_IMAGE_H_
33 #define COLMAP_SRC_BASE_IMAGE_H_
34
35 #include <string>
36 #include <vector>
37
38 #include <Eigen/Core>
39
40 #include "base/camera.h"
41 #include "base/point2d.h"
42 #include "base/visibility_pyramid.h"
43 #include "util/alignment.h"
44 #include "util/logging.h"
45 #include "util/math.h"
46 #include "util/types.h"
47
48 namespace colmap {
49
50 // Class that holds information about an image. An image is the product of one
51 // camera shot at a certain location (parameterized as the pose). An image may
52 // share a camera with multiple other images, if its intrinsics are the same.
53 class Image {
54 public:
55 EIGEN_MAKE_ALIGNED_OPERATOR_NEW
56
57 Image();
58
59 // Setup / tear down the image and necessary internal data structures before
60 // and after being used in reconstruction.
61 void SetUp(const Camera& camera);
62 void TearDown();
63
64 // Access the unique identifier of the image.
65 inline image_t ImageId() const;
66 inline void SetImageId(const image_t image_id);
67
68 // Access the name of the image.
69 inline const std::string& Name() const;
70 inline std::string& Name();
71 inline void SetName(const std::string& name);
72
73 // Access the unique identifier of the camera. Note that multiple images
74 // might share the same camera.
75 inline camera_t CameraId() const;
76 inline void SetCameraId(const camera_t camera_id);
77 // Check whether identifier of camera has been set.
78 inline bool HasCamera() const;
79
80 // Check if image is registered.
81 inline bool IsRegistered() const;
82 inline void SetRegistered(const bool registered);
83
84 // Get the number of image points.
85 inline point2D_t NumPoints2D() const;
86
87 // Get the number of triangulations, i.e. the number of points that
88 // are part of a 3D point track.
89 inline point2D_t NumPoints3D() const;
90
91 // Get the number of observations, i.e. the number of image points that
92 // have at least one correspondence to another image.
93 inline point2D_t NumObservations() const;
94 inline void SetNumObservations(const point2D_t num_observations);
95
96 // Get the number of correspondences for all image points.
97 inline point2D_t NumCorrespondences() const;
98 inline void SetNumCorrespondences(const point2D_t num_observations);
99
100 // Get the number of observations that see a triangulated point, i.e. the
101 // number of image points that have at least one correspondence to a
102 // triangulated point in another image.
103 inline point2D_t NumVisiblePoints3D() const;
104
105 // Get the score of triangulated observations. In contrast to
106 // `NumVisiblePoints3D`, this score also captures the distribution
107 // of triangulated observations in the image. This is useful to select
108 // the next best image in incremental reconstruction, because a more
109 // uniform distribution of observations results in more robust registration.
110 inline size_t Point3DVisibilityScore() const;
111
112 // Access quaternion vector as (qw, qx, qy, qz) specifying the rotation of the
113 // pose which is defined as the transformation from world to image space.
114 inline const Eigen::Vector4d& Qvec() const;
115 inline Eigen::Vector4d& Qvec();
116 inline double Qvec(const size_t idx) const;
117 inline double& Qvec(const size_t idx);
118 inline void SetQvec(const Eigen::Vector4d& qvec);
119
120 // Quaternion prior, e.g. given by EXIF gyroscope tag.
121 inline const Eigen::Vector4d& QvecPrior() const;
122 inline Eigen::Vector4d& QvecPrior();
123 inline double QvecPrior(const size_t idx) const;
124 inline double& QvecPrior(const size_t idx);
125 inline bool HasQvecPrior() const;
126 inline void SetQvecPrior(const Eigen::Vector4d& qvec);
127
128 // Access quaternion vector as (tx, ty, tz) specifying the translation of the
129 // pose which is defined as the transformation from world to image space.
130 inline const Eigen::Vector3d& Tvec() const;
131 inline Eigen::Vector3d& Tvec();
132 inline double Tvec(const size_t idx) const;
133 inline double& Tvec(const size_t idx);
134 inline void SetTvec(const Eigen::Vector3d& tvec);
135
136 // Quaternion prior, e.g. given by EXIF GPS tag.
137 inline const Eigen::Vector3d& TvecPrior() const;
138 inline Eigen::Vector3d& TvecPrior();
139 inline double TvecPrior(const size_t idx) const;
140 inline double& TvecPrior(const size_t idx);
141 inline bool HasTvecPrior() const;
142 inline void SetTvecPrior(const Eigen::Vector3d& tvec);
143
144 // Access the coordinates of image points.
145 inline const class Point2D& Point2D(const point2D_t point2D_idx) const;
146 inline class Point2D& Point2D(const point2D_t point2D_idx);
147 inline const std::vector<class Point2D>& Points2D() const;
148 void SetPoints2D(const std::vector<Eigen::Vector2d>& points);
149 void SetPoints2D(const std::vector<class Point2D>& points);
150
151 // Set the point as triangulated, i.e. it is part of a 3D point track.
152 void SetPoint3DForPoint2D(const point2D_t point2D_idx,
153 const point3D_t point3D_id);
154
155 // Set the point as not triangulated, i.e. it is not part of a 3D point track.
156 void ResetPoint3DForPoint2D(const point2D_t point2D_idx);
157
158 // Check whether an image point has a correspondence to an image point in
159 // another image that has a 3D point.
160 inline bool IsPoint3DVisible(const point2D_t point2D_idx) const;
161
162 // Check whether one of the image points is part of the 3D point track.
163 bool HasPoint3D(const point3D_t point3D_id) const;
164
165 // Indicate that another image has a point that is triangulated and has
166 // a correspondence to this image point. Note that this must only be called
167 // after calling `SetUp`.
168 void IncrementCorrespondenceHasPoint3D(const point2D_t point2D_idx);
169
170 // Indicate that another image has a point that is not triangulated any more
171 // and has a correspondence to this image point. This assumes that
172 // `IncrementCorrespondenceHasPoint3D` was called for the same image point
173 // and correspondence before. Note that this must only be called
174 // after calling `SetUp`.
175 void DecrementCorrespondenceHasPoint3D(const point2D_t point2D_idx);
176
177 // Normalize the quaternion vector.
178 void NormalizeQvec();
179
180 // Compose the projection matrix from world to image space.
181 Eigen::Matrix3x4d ProjectionMatrix() const;
182
183 // Compose the inverse projection matrix from image to world space
184 Eigen::Matrix3x4d InverseProjectionMatrix() const;
185
186 // Compose rotation matrix from quaternion vector.
187 Eigen::Matrix3d RotationMatrix() const;
188
189 // Extract the projection center in world space.
190 Eigen::Vector3d ProjectionCenter() const;
191
192 // Extract the viewing direction of the image.
193 Eigen::Vector3d ViewingDirection() const;
194
195 // The number of levels in the 3D point multi-resolution visibility pyramid.
196 static const int kNumPoint3DVisibilityPyramidLevels;
197
198 private:
199 // Identifier of the image, if not specified `kInvalidImageId`.
200 image_t image_id_;
201
202 // The name of the image, i.e. the relative path.
203 std::string name_;
204
205 // The identifier of the associated camera. Note that multiple images might
206 // share the same camera. If not specified `kInvalidCameraId`.
207 camera_t camera_id_;
208
209 // Whether the image is successfully registered in the reconstruction.
210 bool registered_;
211
212 // The number of 3D points the image observes, i.e. the sum of its `points2D`
213 // where `point3D_id != kInvalidPoint3DId`.
214 point2D_t num_points3D_;
215
216 // The number of image points that have at least one correspondence to
217 // another image.
218 point2D_t num_observations_;
219
220 // The sum of correspondences per image point.
221 point2D_t num_correspondences_;
222
223 // The number of 2D points, which have at least one corresponding 2D point in
224 // another image that is part of a 3D point track, i.e. the sum of `points2D`
225 // where `num_tris > 0`.
226 point2D_t num_visible_points3D_;
227
228 // The pose of the image, defined as the transformation from world to image.
229 Eigen::Vector4d qvec_;
230 Eigen::Vector3d tvec_;
231
232 // The pose prior of the image, e.g. extracted from EXIF tags.
233 Eigen::Vector4d qvec_prior_;
234 Eigen::Vector3d tvec_prior_;
235
236 // All image points, including points that are not part of a 3D point track.
237 std::vector<class Point2D> points2D_;
238
239 // Per image point, the number of correspondences that have a 3D point.
240 std::vector<image_t> num_correspondences_have_point3D_;
241
242 // Data structure to compute the distribution of triangulated correspondences
243 // in the image. Note that this structure is only usable after `SetUp`.
244 VisibilityPyramid point3D_visibility_pyramid_;
245 };
246
247 ////////////////////////////////////////////////////////////////////////////////
248 // Implementation
249 ////////////////////////////////////////////////////////////////////////////////
250
ImageId()251 image_t Image::ImageId() const { return image_id_; }
252
SetImageId(const image_t image_id)253 void Image::SetImageId(const image_t image_id) { image_id_ = image_id; }
254
Name()255 const std::string& Image::Name() const { return name_; }
256
Name()257 std::string& Image::Name() { return name_; }
258
SetName(const std::string & name)259 void Image::SetName(const std::string& name) { name_ = name; }
260
CameraId()261 inline camera_t Image::CameraId() const { return camera_id_; }
262
SetCameraId(const camera_t camera_id)263 inline void Image::SetCameraId(const camera_t camera_id) {
264 CHECK_NE(camera_id, kInvalidCameraId);
265 camera_id_ = camera_id;
266 }
267
HasCamera()268 inline bool Image::HasCamera() const { return camera_id_ != kInvalidCameraId; }
269
IsRegistered()270 bool Image::IsRegistered() const { return registered_; }
271
SetRegistered(const bool registered)272 void Image::SetRegistered(const bool registered) { registered_ = registered; }
273
NumPoints2D()274 point2D_t Image::NumPoints2D() const {
275 return static_cast<point2D_t>(points2D_.size());
276 }
277
NumPoints3D()278 point2D_t Image::NumPoints3D() const { return num_points3D_; }
279
NumObservations()280 point2D_t Image::NumObservations() const { return num_observations_; }
281
SetNumObservations(const point2D_t num_observations)282 void Image::SetNumObservations(const point2D_t num_observations) {
283 num_observations_ = num_observations;
284 }
285
NumCorrespondences()286 point2D_t Image::NumCorrespondences() const { return num_correspondences_; }
287
SetNumCorrespondences(const point2D_t num_correspondences)288 void Image::SetNumCorrespondences(const point2D_t num_correspondences) {
289 num_correspondences_ = num_correspondences;
290 }
291
NumVisiblePoints3D()292 point2D_t Image::NumVisiblePoints3D() const { return num_visible_points3D_; }
293
Point3DVisibilityScore()294 size_t Image::Point3DVisibilityScore() const {
295 return point3D_visibility_pyramid_.Score();
296 }
297
Qvec()298 const Eigen::Vector4d& Image::Qvec() const { return qvec_; }
299
Qvec()300 Eigen::Vector4d& Image::Qvec() { return qvec_; }
301
Qvec(const size_t idx)302 inline double Image::Qvec(const size_t idx) const { return qvec_(idx); }
303
Qvec(const size_t idx)304 inline double& Image::Qvec(const size_t idx) { return qvec_(idx); }
305
SetQvec(const Eigen::Vector4d & qvec)306 void Image::SetQvec(const Eigen::Vector4d& qvec) { qvec_ = qvec; }
307
QvecPrior()308 const Eigen::Vector4d& Image::QvecPrior() const { return qvec_prior_; }
309
QvecPrior()310 Eigen::Vector4d& Image::QvecPrior() { return qvec_prior_; }
311
QvecPrior(const size_t idx)312 inline double Image::QvecPrior(const size_t idx) const {
313 return qvec_prior_(idx);
314 }
315
QvecPrior(const size_t idx)316 inline double& Image::QvecPrior(const size_t idx) { return qvec_prior_(idx); }
317
HasQvecPrior()318 inline bool Image::HasQvecPrior() const { return !IsNaN(qvec_prior_.sum()); }
319
SetQvecPrior(const Eigen::Vector4d & qvec)320 void Image::SetQvecPrior(const Eigen::Vector4d& qvec) { qvec_prior_ = qvec; }
321
Tvec()322 const Eigen::Vector3d& Image::Tvec() const { return tvec_; }
323
Tvec()324 Eigen::Vector3d& Image::Tvec() { return tvec_; }
325
Tvec(const size_t idx)326 inline double Image::Tvec(const size_t idx) const { return tvec_(idx); }
327
Tvec(const size_t idx)328 inline double& Image::Tvec(const size_t idx) { return tvec_(idx); }
329
SetTvec(const Eigen::Vector3d & tvec)330 void Image::SetTvec(const Eigen::Vector3d& tvec) { tvec_ = tvec; }
331
TvecPrior()332 const Eigen::Vector3d& Image::TvecPrior() const { return tvec_prior_; }
333
TvecPrior()334 Eigen::Vector3d& Image::TvecPrior() { return tvec_prior_; }
335
TvecPrior(const size_t idx)336 inline double Image::TvecPrior(const size_t idx) const {
337 return tvec_prior_(idx);
338 }
339
TvecPrior(const size_t idx)340 inline double& Image::TvecPrior(const size_t idx) { return tvec_prior_(idx); }
341
HasTvecPrior()342 inline bool Image::HasTvecPrior() const { return !IsNaN(tvec_prior_.sum()); }
343
SetTvecPrior(const Eigen::Vector3d & tvec)344 void Image::SetTvecPrior(const Eigen::Vector3d& tvec) { tvec_prior_ = tvec; }
345
Point2D(const point2D_t point2D_idx)346 const class Point2D& Image::Point2D(const point2D_t point2D_idx) const {
347 return points2D_.at(point2D_idx);
348 }
349
Point2D(const point2D_t point2D_idx)350 class Point2D& Image::Point2D(const point2D_t point2D_idx) {
351 return points2D_.at(point2D_idx);
352 }
353
Points2D()354 const std::vector<class Point2D>& Image::Points2D() const { return points2D_; }
355
IsPoint3DVisible(const point2D_t point2D_idx)356 bool Image::IsPoint3DVisible(const point2D_t point2D_idx) const {
357 return num_correspondences_have_point3D_.at(point2D_idx) > 0;
358 }
359
360 } // namespace colmap
361
362 EIGEN_DEFINE_STL_VECTOR_SPECIALIZATION_CUSTOM(colmap::Image)
363
364 #endif // COLMAP_SRC_BASE_IMAGE_H_
365