1 // Copyright (c) 2018, ETH Zurich and UNC Chapel Hill.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright
8 // notice, this list of conditions and the following disclaimer.
9 //
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 //
14 // * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of
15 // its contributors may be used to endorse or promote products derived
16 // from this software without specific prior written permission.
17 //
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
22 // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26 // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 // POSSIBILITY OF SUCH DAMAGE.
29 //
30 // Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de)
31
32 #include "ui/model_viewer_widget.h"
33
34 #include "ui/main_window.h"
35
36 #define SELECTION_BUFFER_IMAGE_IDX 0
37 #define SELECTION_BUFFER_POINT_IDX 1
38
39 const Eigen::Vector4f kSelectedPointColor(0.0f, 1.0f, 0.0f, 1.0f);
40
41 const Eigen::Vector4f kSelectedImagePlaneColor(1.0f, 0.0f, 1.0f, 0.6f);
42 const Eigen::Vector4f kSelectedImageFrameColor(0.8f, 0.0f, 0.8f, 1.0f);
43
44 const Eigen::Vector4f kMovieGrabberImagePlaneColor(0.0f, 1.0f, 1.0f, 0.6f);
45 const Eigen::Vector4f kMovieGrabberImageFrameColor(0.0f, 0.8f, 0.8f, 1.0f);
46
47 const Eigen::Vector4f kGridColor(0.2f, 0.2f, 0.2f, 0.6f);
48 const Eigen::Vector4f kXAxisColor(0.9f, 0.0f, 0.0f, 0.5f);
49 const Eigen::Vector4f kYAxisColor(0.0f, 0.9f, 0.0f, 0.5f);
50 const Eigen::Vector4f kZAxisColor(0.0f, 0.0f, 0.9f, 0.5f);
51
52 namespace colmap {
53 namespace {
54
55 // Generate unique index from RGB color in the range [0, 256^3].
RGBToIndex(const uint8_t r,const uint8_t g,const uint8_t b)56 inline size_t RGBToIndex(const uint8_t r, const uint8_t g, const uint8_t b) {
57 return static_cast<size_t>(r) + static_cast<size_t>(g) * 256 +
58 static_cast<size_t>(b) * 65536;
59 }
60
61 // Derive color from unique index, generated by `RGBToIndex`.
IndexToRGB(const size_t index)62 inline Eigen::Vector4f IndexToRGB(const size_t index) {
63 Eigen::Vector4f color;
64 color(0) = ((index & 0x000000FF) >> 0) / 255.0f;
65 color(1) = ((index & 0x0000FF00) >> 8) / 255.0f;
66 color(2) = ((index & 0x00FF0000) >> 16) / 255.0f;
67 color(3) = 1.0f;
68 return color;
69 }
70
BuildImageModel(const Image & image,const Camera & camera,const float image_size,const Eigen::Vector4f & plane_color,const Eigen::Vector4f & frame_color,TrianglePainter::Data * triangle1,TrianglePainter::Data * triangle2,LinePainter::Data * line1,LinePainter::Data * line2,LinePainter::Data * line3,LinePainter::Data * line4,LinePainter::Data * line5,LinePainter::Data * line6,LinePainter::Data * line7,LinePainter::Data * line8)71 void BuildImageModel(const Image& image, const Camera& camera,
72 const float image_size, const Eigen::Vector4f& plane_color,
73 const Eigen::Vector4f& frame_color,
74 TrianglePainter::Data* triangle1,
75 TrianglePainter::Data* triangle2, LinePainter::Data* line1,
76 LinePainter::Data* line2, LinePainter::Data* line3,
77 LinePainter::Data* line4, LinePainter::Data* line5,
78 LinePainter::Data* line6, LinePainter::Data* line7,
79 LinePainter::Data* line8) {
80 // Generate camera dimensions in OpenGL (world) coordinate space.
81 const float kBaseCameraWidth = 1024.0f;
82 const float image_width = image_size * camera.Width() / kBaseCameraWidth;
83 const float image_height = image_width * static_cast<float>(camera.Height()) /
84 static_cast<float>(camera.Width());
85 const float image_extent = std::max(image_width, image_height);
86 const float camera_extent = std::max(camera.Width(), camera.Height());
87 const float camera_extent_world =
88 static_cast<float>(camera.ImageToWorldThreshold(camera_extent));
89 const float focal_length = 2.0f * image_extent / camera_extent_world;
90
91 const Eigen::Matrix<float, 3, 4> inv_proj_matrix =
92 image.InverseProjectionMatrix().cast<float>();
93
94 // Projection center, top-left, top-right, bottom-right, bottom-left corners.
95
96 const Eigen::Vector3f pc = inv_proj_matrix.rightCols<1>();
97 const Eigen::Vector3f tl =
98 inv_proj_matrix *
99 Eigen::Vector4f(-image_width, image_height, focal_length, 1);
100 const Eigen::Vector3f tr =
101 inv_proj_matrix *
102 Eigen::Vector4f(image_width, image_height, focal_length, 1);
103 const Eigen::Vector3f br =
104 inv_proj_matrix *
105 Eigen::Vector4f(image_width, -image_height, focal_length, 1);
106 const Eigen::Vector3f bl =
107 inv_proj_matrix *
108 Eigen::Vector4f(-image_width, -image_height, focal_length, 1);
109
110 // Image plane as two triangles.
111
112 triangle1->point1 =
113 PointPainter::Data(tl(0), tl(1), tl(2), plane_color(0), plane_color(1),
114 plane_color(2), plane_color(3));
115 triangle1->point2 =
116 PointPainter::Data(tr(0), tr(1), tr(2), plane_color(0), plane_color(1),
117 plane_color(2), plane_color(3));
118 triangle1->point3 =
119 PointPainter::Data(bl(0), bl(1), bl(2), plane_color(0), plane_color(1),
120 plane_color(2), plane_color(3));
121
122 triangle2->point1 =
123 PointPainter::Data(bl(0), bl(1), bl(2), plane_color(0), plane_color(1),
124 plane_color(2), plane_color(3));
125 triangle2->point2 =
126 PointPainter::Data(tr(0), tr(1), tr(2), plane_color(0), plane_color(1),
127 plane_color(2), plane_color(3));
128 triangle2->point3 =
129 PointPainter::Data(br(0), br(1), br(2), plane_color(0), plane_color(1),
130 plane_color(2), plane_color(3));
131
132 // Frame around image plane and connecting lines to projection center.
133
134 line1->point1 =
135 PointPainter::Data(pc(0), pc(1), pc(2), frame_color(0), frame_color(1),
136 frame_color(2), frame_color(3));
137 line1->point2 =
138 PointPainter::Data(tl(0), tl(1), tl(2), frame_color(0), frame_color(1),
139 frame_color(2), frame_color(3));
140
141 line2->point1 =
142 PointPainter::Data(pc(0), pc(1), pc(2), frame_color(0), frame_color(1),
143 frame_color(2), frame_color(3));
144 line2->point2 =
145 PointPainter::Data(tr(0), tr(1), tr(2), frame_color(0), frame_color(1),
146 frame_color(2), frame_color(3));
147
148 line3->point1 =
149 PointPainter::Data(pc(0), pc(1), pc(2), frame_color(0), frame_color(1),
150 frame_color(2), frame_color(3));
151 line3->point2 =
152 PointPainter::Data(br(0), br(1), br(2), frame_color(0), frame_color(1),
153 frame_color(2), frame_color(3));
154
155 line4->point1 =
156 PointPainter::Data(pc(0), pc(1), pc(2), frame_color(0), frame_color(1),
157 frame_color(2), frame_color(3));
158 line4->point2 =
159 PointPainter::Data(bl(0), bl(1), bl(2), frame_color(0), frame_color(1),
160 frame_color(2), frame_color(3));
161
162 line5->point1 =
163 PointPainter::Data(tl(0), tl(1), tl(2), frame_color(0), frame_color(1),
164 frame_color(2), frame_color(3));
165 line5->point2 =
166 PointPainter::Data(tr(0), tr(1), tr(2), frame_color(0), frame_color(1),
167 frame_color(2), frame_color(3));
168
169 line6->point1 =
170 PointPainter::Data(tr(0), tr(1), tr(2), frame_color(0), frame_color(1),
171 frame_color(2), frame_color(3));
172 line6->point2 =
173 PointPainter::Data(br(0), br(1), br(2), frame_color(0), frame_color(1),
174 frame_color(2), frame_color(3));
175
176 line7->point1 =
177 PointPainter::Data(br(0), br(1), br(2), frame_color(0), frame_color(1),
178 frame_color(2), frame_color(3));
179 line7->point2 =
180 PointPainter::Data(bl(0), bl(1), bl(2), frame_color(0), frame_color(1),
181 frame_color(2), frame_color(3));
182
183 line8->point1 =
184 PointPainter::Data(bl(0), bl(1), bl(2), frame_color(0), frame_color(1),
185 frame_color(2), frame_color(3));
186 line8->point2 =
187 PointPainter::Data(tl(0), tl(1), tl(2), frame_color(0), frame_color(1),
188 frame_color(2), frame_color(3));
189 }
190
191 } // namespace
192
ModelViewerWidget(QWidget * parent,OptionManager * options)193 ModelViewerWidget::ModelViewerWidget(QWidget* parent, OptionManager* options)
194 : QOpenGLWidget(parent),
195 options_(options),
196 point_viewer_widget_(new PointViewerWidget(parent, this, options)),
197 image_viewer_widget_(
198 new DatabaseImageViewerWidget(parent, this, options)),
199 movie_grabber_widget_(new MovieGrabberWidget(parent, this)),
200 mouse_is_pressed_(false),
201 focus_distance_(kInitFocusDistance),
202 selected_image_id_(kInvalidImageId),
203 selected_point3D_id_(kInvalidPoint3DId),
204 coordinate_grid_enabled_(true),
205 near_plane_(kInitNearPlane) {
206 background_color_[0] = 1.0f;
207 background_color_[1] = 1.0f;
208 background_color_[2] = 1.0f;
209
210 QSurfaceFormat format;
211 format.setDepthBufferSize(24);
212 format.setMajorVersion(3);
213 format.setMinorVersion(2);
214 format.setSamples(4);
215 format.setProfile(QSurfaceFormat::CoreProfile);
216 #ifdef DEBUG
217 format.setOption(QSurfaceFormat::DebugContext);
218 #endif
219 setFormat(format);
220 QSurfaceFormat::setDefaultFormat(format);
221
222 SetPointColormap(new PointColormapPhotometric());
223 SetImageColormap(new ImageColormapUniform());
224
225 image_size_ = static_cast<float>(devicePixelRatio() * image_size_);
226 point_size_ = static_cast<float>(devicePixelRatio() * point_size_);
227 }
228
initializeGL()229 void ModelViewerWidget::initializeGL() {
230 initializeOpenGLFunctions();
231 glEnable(GL_DEPTH_TEST);
232 glEnable(GL_BLEND);
233 glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
234 glEnable(GL_VERTEX_PROGRAM_POINT_SIZE);
235 SetupPainters();
236 SetupView();
237 }
238
paintGL()239 void ModelViewerWidget::paintGL() {
240 glClearColor(background_color_[0], background_color_[1], background_color_[2],
241 1.0f);
242 glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
243
244 const QMatrix4x4 pmv_matrix = projection_matrix_ * model_view_matrix_;
245
246 // Model view matrix for center of view
247 QMatrix4x4 model_view_center_matrix = model_view_matrix_;
248 const Eigen::Vector4f rot_center =
249 QMatrixToEigen(model_view_matrix_).inverse() *
250 Eigen::Vector4f(0, 0, -focus_distance_, 1);
251 model_view_center_matrix.translate(rot_center(0), rot_center(1),
252 rot_center(2));
253
254 // Coordinate system
255 if (coordinate_grid_enabled_) {
256 const QMatrix4x4 pmvc_matrix =
257 projection_matrix_ * model_view_center_matrix;
258 coordinate_axes_painter_.Render(pmv_matrix, width(), height(), 2);
259 coordinate_grid_painter_.Render(pmvc_matrix, width(), height(), 1);
260 }
261
262 // Points
263 point_painter_.Render(pmv_matrix, point_size_);
264 point_connection_painter_.Render(pmv_matrix, width(), height(), 1);
265
266 // Images
267 image_line_painter_.Render(pmv_matrix, width(), height(), 1);
268 image_triangle_painter_.Render(pmv_matrix);
269 image_connection_painter_.Render(pmv_matrix, width(), height(), 1);
270
271 // Movie grabber cameras
272 movie_grabber_path_painter_.Render(pmv_matrix, width(), height(), 1.5);
273 movie_grabber_line_painter_.Render(pmv_matrix, width(), height(), 1);
274 movie_grabber_triangle_painter_.Render(pmv_matrix);
275 }
276
resizeGL(int width,int height)277 void ModelViewerWidget::resizeGL(int width, int height) {
278 glViewport(0, 0, width, height);
279 ComposeProjectionMatrix();
280 UploadCoordinateGridData();
281 }
282
ReloadReconstruction()283 void ModelViewerWidget::ReloadReconstruction() {
284 if (reconstruction == nullptr) {
285 return;
286 }
287
288 cameras = reconstruction->Cameras();
289 points3D = reconstruction->Points3D();
290 reg_image_ids = reconstruction->RegImageIds();
291
292 images.clear();
293 for (const image_t image_id : reg_image_ids) {
294 images[image_id] = reconstruction->Image(image_id);
295 }
296
297 statusbar_status_label->setText(QString().sprintf(
298 "%d Images - %d Points", static_cast<int>(reg_image_ids.size()),
299 static_cast<int>(points3D.size())));
300
301 Upload();
302 }
303
ClearReconstruction()304 void ModelViewerWidget::ClearReconstruction() {
305 cameras.clear();
306 images.clear();
307 points3D.clear();
308 reg_image_ids.clear();
309 reconstruction = nullptr;
310 Upload();
311 }
312
GetProjectionType() const313 int ModelViewerWidget::GetProjectionType() const {
314 return options_->render->projection_type;
315 }
316
SetPointColormap(PointColormapBase * colormap)317 void ModelViewerWidget::SetPointColormap(PointColormapBase* colormap) {
318 point_colormap_.reset(colormap);
319 }
320
SetImageColormap(ImageColormapBase * colormap)321 void ModelViewerWidget::SetImageColormap(ImageColormapBase* colormap) {
322 image_colormap_.reset(colormap);
323 }
324
UpdateMovieGrabber()325 void ModelViewerWidget::UpdateMovieGrabber() {
326 UploadMovieGrabberData();
327 update();
328 }
329
EnableCoordinateGrid()330 void ModelViewerWidget::EnableCoordinateGrid() {
331 coordinate_grid_enabled_ = true;
332 update();
333 }
334
DisableCoordinateGrid()335 void ModelViewerWidget::DisableCoordinateGrid() {
336 coordinate_grid_enabled_ = false;
337 update();
338 }
339
ChangeFocusDistance(const float delta)340 void ModelViewerWidget::ChangeFocusDistance(const float delta) {
341 if (delta == 0.0f) {
342 return;
343 }
344 const float prev_focus_distance = focus_distance_;
345 float diff = delta * ZoomScale() * kFocusSpeed;
346 focus_distance_ -= diff;
347 if (focus_distance_ < kMinFocusDistance) {
348 focus_distance_ = kMinFocusDistance;
349 diff = prev_focus_distance - focus_distance_;
350 } else if (focus_distance_ > kMaxFocusDistance) {
351 focus_distance_ = kMaxFocusDistance;
352 diff = prev_focus_distance - focus_distance_;
353 }
354 const Eigen::Matrix4f vm_mat = QMatrixToEigen(model_view_matrix_).inverse();
355 const Eigen::Vector3f tvec(0, 0, diff);
356 const Eigen::Vector3f tvec_rot = vm_mat.block<3, 3>(0, 0) * tvec;
357 model_view_matrix_.translate(tvec_rot(0), tvec_rot(1), tvec_rot(2));
358 ComposeProjectionMatrix();
359 UploadCoordinateGridData();
360 update();
361 }
362
ChangeNearPlane(const float delta)363 void ModelViewerWidget::ChangeNearPlane(const float delta) {
364 if (delta == 0.0f) {
365 return;
366 }
367 near_plane_ *= (1.0f + delta / 100.0f * kNearPlaneScaleSpeed);
368 near_plane_ = std::max(kMinNearPlane, std::min(kMaxNearPlane, near_plane_));
369 ComposeProjectionMatrix();
370 UploadCoordinateGridData();
371 update();
372 }
373
ChangePointSize(const float delta)374 void ModelViewerWidget::ChangePointSize(const float delta) {
375 if (delta == 0.0f) {
376 return;
377 }
378 point_size_ *= (1.0f + delta / 100.0f * kPointScaleSpeed);
379 point_size_ = std::max(kMinPointSize, std::min(kMaxPointSize, point_size_));
380 update();
381 }
382
RotateView(const float x,const float y,const float prev_x,const float prev_y)383 void ModelViewerWidget::RotateView(const float x, const float y,
384 const float prev_x, const float prev_y) {
385 if (x - prev_x == 0 && y - prev_y == 0) {
386 return;
387 }
388
389 // Rotation according to the Arcball method "ARCBALL: A User Interface for
390 // Specifying Three-Dimensional Orientation Using a Mouse", Ken Shoemake,
391 // University of Pennsylvania, 1992.
392
393 // Determine Arcball vector on unit sphere.
394 const Eigen::Vector3f u = PositionToArcballVector(x, y);
395 const Eigen::Vector3f v = PositionToArcballVector(prev_x, prev_y);
396
397 // Angle between vectors.
398 const float angle = 2.0f * std::acos(std::min(1.0f, u.dot(v)));
399
400 const float kMinAngle = 1e-3f;
401 if (angle > kMinAngle) {
402 const Eigen::Matrix4f vm_mat = QMatrixToEigen(model_view_matrix_).inverse();
403
404 // Rotation axis.
405 Eigen::Vector3f axis = vm_mat.block<3, 3>(0, 0) * v.cross(u);
406 axis = axis.normalized();
407 // Center of rotation is current focus.
408 const Eigen::Vector4f rot_center =
409 vm_mat * Eigen::Vector4f(0, 0, -focus_distance_, 1);
410 // First shift to rotation center, then rotate and shift back.
411 model_view_matrix_.translate(rot_center(0), rot_center(1), rot_center(2));
412 model_view_matrix_.rotate(RadToDeg(angle), axis(0), axis(1), axis(2));
413 model_view_matrix_.translate(-rot_center(0), -rot_center(1),
414 -rot_center(2));
415 update();
416 }
417 }
418
TranslateView(const float x,const float y,const float prev_x,const float prev_y)419 void ModelViewerWidget::TranslateView(const float x, const float y,
420 const float prev_x, const float prev_y) {
421 if (x - prev_x == 0 && y - prev_y == 0) {
422 return;
423 }
424
425 Eigen::Vector3f tvec(x - prev_x, prev_y - y, 0.0f);
426
427 if (options_->render->projection_type ==
428 RenderOptions::ProjectionType::PERSPECTIVE) {
429 tvec *= ZoomScale();
430 } else if (options_->render->projection_type ==
431 RenderOptions::ProjectionType::ORTHOGRAPHIC) {
432 tvec *= 2.0f * OrthographicWindowExtent() / height();
433 }
434
435 const Eigen::Matrix4f vm_mat = QMatrixToEigen(model_view_matrix_).inverse();
436
437 const Eigen::Vector3f tvec_rot = vm_mat.block<3, 3>(0, 0) * tvec;
438 model_view_matrix_.translate(tvec_rot(0), tvec_rot(1), tvec_rot(2));
439
440 update();
441 }
442
ChangeCameraSize(const float delta)443 void ModelViewerWidget::ChangeCameraSize(const float delta) {
444 if (delta == 0.0f) {
445 return;
446 }
447 image_size_ *= (1.0f + delta / 100.0f * kImageScaleSpeed);
448 image_size_ = std::max(kMinImageSize, std::min(kMaxImageSize, image_size_));
449 UploadImageData();
450 UploadMovieGrabberData();
451 update();
452 }
453
ResetView()454 void ModelViewerWidget::ResetView() {
455 SetupView();
456 Upload();
457 }
458
ModelViewMatrix() const459 QMatrix4x4 ModelViewerWidget::ModelViewMatrix() const {
460 return model_view_matrix_;
461 }
462
SetModelViewMatrix(const QMatrix4x4 & matrix)463 void ModelViewerWidget::SetModelViewMatrix(const QMatrix4x4& matrix) {
464 model_view_matrix_ = matrix;
465 update();
466 }
467
SelectObject(const int x,const int y)468 void ModelViewerWidget::SelectObject(const int x, const int y) {
469 makeCurrent();
470
471 // Ensure that anti-aliasing does not change the colors of objects.
472 glDisable(GL_MULTISAMPLE);
473
474 glClearColor(1.0f, 1.0f, 1.0f, 1.0f);
475 glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
476
477 // Upload data in selection mode (one color per object).
478 UploadImageData(true);
479 UploadPointData(true);
480
481 // Render in selection mode, with larger points to improve selection accuracy.
482 const QMatrix4x4 pmv_matrix = projection_matrix_ * model_view_matrix_;
483 image_triangle_painter_.Render(pmv_matrix);
484 point_painter_.Render(pmv_matrix, 2 * point_size_);
485
486 const int scaled_x = devicePixelRatio() * x;
487 const int scaled_y = devicePixelRatio() * (height() - y - 1);
488
489 QOpenGLFramebufferObjectFormat fbo_format;
490 fbo_format.setSamples(0);
491 QOpenGLFramebufferObject fbo(1, 1, fbo_format);
492
493 glBindFramebuffer(GL_READ_FRAMEBUFFER, defaultFramebufferObject());
494 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, fbo.handle());
495 glBlitFramebuffer(scaled_x, scaled_y, scaled_x + 1, scaled_y + 1, 0, 0, 1, 1,
496 GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT, GL_NEAREST);
497
498 fbo.bind();
499 std::array<uint8_t, 3> color;
500 glReadPixels(0, 0, 1, 1, GL_RGB, GL_UNSIGNED_BYTE, color.data());
501 fbo.release();
502
503 const size_t index = RGBToIndex(color[0], color[1], color[2]);
504
505 if (index < selection_buffer_.size()) {
506 const char buffer_type = selection_buffer_[index].second;
507 if (buffer_type == SELECTION_BUFFER_IMAGE_IDX) {
508 selected_image_id_ = static_cast<image_t>(selection_buffer_[index].first);
509 selected_point3D_id_ = kInvalidPoint3DId;
510 ShowImageInfo(selected_image_id_);
511 } else if (buffer_type == SELECTION_BUFFER_POINT_IDX) {
512 selected_image_id_ = kInvalidImageId;
513 selected_point3D_id_ = selection_buffer_[index].first;
514 ShowPointInfo(selection_buffer_[index].first);
515 } else {
516 selected_image_id_ = kInvalidImageId;
517 selected_point3D_id_ = kInvalidPoint3DId;
518 image_viewer_widget_->hide();
519 }
520 } else {
521 selected_image_id_ = kInvalidImageId;
522 selected_point3D_id_ = kInvalidPoint3DId;
523 image_viewer_widget_->hide();
524 }
525
526 // Re-enable, since temporarily disabled above.
527 glEnable(GL_MULTISAMPLE);
528
529 selection_buffer_.clear();
530
531 UploadPointData();
532 UploadImageData();
533 UploadPointConnectionData();
534 UploadImageConnectionData();
535
536 update();
537 }
538
SelectMoviewGrabberView(const size_t view_idx)539 void ModelViewerWidget::SelectMoviewGrabberView(const size_t view_idx) {
540 selected_movie_grabber_view_ = view_idx;
541 UploadMovieGrabberData();
542 update();
543 }
544
GrabImage()545 QImage ModelViewerWidget::GrabImage() {
546 makeCurrent();
547
548 DisableCoordinateGrid();
549
550 paintGL();
551
552 const int scaled_width = static_cast<int>(devicePixelRatio() * width());
553 const int scaled_height = static_cast<int>(devicePixelRatio() * height());
554
555 QOpenGLFramebufferObjectFormat fbo_format;
556 fbo_format.setSamples(0);
557 QOpenGLFramebufferObject fbo(scaled_width, scaled_height, fbo_format);
558
559 glBindFramebuffer(GL_READ_FRAMEBUFFER, defaultFramebufferObject());
560 glBindFramebuffer(GL_DRAW_FRAMEBUFFER, fbo.handle());
561 glBlitFramebuffer(0, 0, scaled_width, scaled_height, 0, 0, scaled_width,
562 scaled_height, GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT,
563 GL_NEAREST);
564
565 fbo.bind();
566 QImage image(scaled_width, scaled_height, QImage::Format_RGB888);
567 glReadPixels(0, 0, scaled_width, scaled_height, GL_RGB, GL_UNSIGNED_BYTE,
568 image.bits());
569 fbo.release();
570
571 EnableCoordinateGrid();
572
573 return image.mirrored();
574 }
575
GrabMovie()576 void ModelViewerWidget::GrabMovie() { movie_grabber_widget_->show(); }
577
ShowPointInfo(const point3D_t point3D_id)578 void ModelViewerWidget::ShowPointInfo(const point3D_t point3D_id) {
579 point_viewer_widget_->Show(point3D_id);
580 }
581
ShowImageInfo(const image_t image_id)582 void ModelViewerWidget::ShowImageInfo(const image_t image_id) {
583 image_viewer_widget_->ShowImageWithId(image_id);
584 }
585
PointSize() const586 float ModelViewerWidget::PointSize() const { return point_size_; }
587
ImageSize() const588 float ModelViewerWidget::ImageSize() const { return image_size_; }
589
SetPointSize(const float point_size)590 void ModelViewerWidget::SetPointSize(const float point_size) {
591 point_size_ = point_size;
592 }
593
SetImageSize(const float image_size)594 void ModelViewerWidget::SetImageSize(const float image_size) {
595 image_size_ = image_size;
596 UploadImageData();
597 }
598
SetBackgroundColor(const float r,const float g,const float b)599 void ModelViewerWidget::SetBackgroundColor(const float r, const float g,
600 const float b) {
601 background_color_[0] = r;
602 background_color_[1] = g;
603 background_color_[2] = b;
604 update();
605 }
606
mousePressEvent(QMouseEvent * event)607 void ModelViewerWidget::mousePressEvent(QMouseEvent* event) {
608 if (mouse_press_timer_.isActive()) { // Select objects (2. click)
609 mouse_is_pressed_ = false;
610 mouse_press_timer_.stop();
611 selection_buffer_.clear();
612 SelectObject(event->pos().x(), event->pos().y());
613 } else { // Set timer to remember 1. click
614 mouse_press_timer_.setSingleShot(true);
615 mouse_press_timer_.start(kDoubleClickInterval);
616 mouse_is_pressed_ = true;
617 prev_mouse_pos_ = event->pos();
618 }
619 event->accept();
620 }
621
mouseReleaseEvent(QMouseEvent * event)622 void ModelViewerWidget::mouseReleaseEvent(QMouseEvent* event) {
623 mouse_is_pressed_ = false;
624 event->accept();
625 }
626
mouseMoveEvent(QMouseEvent * event)627 void ModelViewerWidget::mouseMoveEvent(QMouseEvent* event) {
628 if (mouse_is_pressed_) {
629 if (event->buttons() & Qt::RightButton ||
630 (event->buttons() & Qt::LeftButton &&
631 event->modifiers() & Qt::ControlModifier)) {
632 TranslateView(event->pos().x(), event->pos().y(), prev_mouse_pos_.x(),
633 prev_mouse_pos_.y());
634 } else if (event->buttons() & Qt::LeftButton) {
635 RotateView(event->pos().x(), event->pos().y(), prev_mouse_pos_.x(),
636 prev_mouse_pos_.y());
637 }
638 }
639 prev_mouse_pos_ = event->pos();
640 event->accept();
641 }
642
wheelEvent(QWheelEvent * event)643 void ModelViewerWidget::wheelEvent(QWheelEvent* event) {
644 if (event->modifiers() & Qt::ControlModifier) {
645 ChangePointSize(event->delta());
646 } else if (event->modifiers() & Qt::AltModifier) {
647 ChangeCameraSize(event->delta());
648 } else if (event->modifiers() & Qt::ShiftModifier) {
649 ChangeNearPlane(event->delta());
650 } else {
651 ChangeFocusDistance(event->delta());
652 }
653 event->accept();
654 }
655
SetupPainters()656 void ModelViewerWidget::SetupPainters() {
657 makeCurrent();
658
659 coordinate_axes_painter_.Setup();
660 coordinate_grid_painter_.Setup();
661
662 point_painter_.Setup();
663 point_connection_painter_.Setup();
664
665 image_line_painter_.Setup();
666 image_triangle_painter_.Setup();
667 image_connection_painter_.Setup();
668
669 movie_grabber_path_painter_.Setup();
670 movie_grabber_line_painter_.Setup();
671 movie_grabber_triangle_painter_.Setup();
672 }
673
SetupView()674 void ModelViewerWidget::SetupView() {
675 point_size_ = kInitPointSize;
676 image_size_ = kInitImageSize;
677 focus_distance_ = kInitFocusDistance;
678 model_view_matrix_.setToIdentity();
679 model_view_matrix_.translate(0, 0, -focus_distance_);
680 model_view_matrix_.rotate(225, 1, 0, 0);
681 model_view_matrix_.rotate(-45, 0, 1, 0);
682 }
683
Upload()684 void ModelViewerWidget::Upload() {
685 point_colormap_->Prepare(cameras, images, points3D, reg_image_ids);
686 image_colormap_->Prepare(cameras, images, points3D, reg_image_ids);
687
688 ComposeProjectionMatrix();
689
690 UploadPointData();
691 UploadImageData();
692 UploadMovieGrabberData();
693 UploadPointConnectionData();
694 UploadImageConnectionData();
695
696 update();
697 }
698
UploadCoordinateGridData()699 void ModelViewerWidget::UploadCoordinateGridData() {
700 makeCurrent();
701
702 const float scale = ZoomScale();
703
704 // View center grid.
705 std::vector<LinePainter::Data> grid_data(3);
706
707 grid_data[0].point1 =
708 PointPainter::Data(-20 * scale, 0, 0, kGridColor(0), kGridColor(1),
709 kGridColor(2), kGridColor(3));
710 grid_data[0].point2 =
711 PointPainter::Data(20 * scale, 0, 0, kGridColor(0), kGridColor(1),
712 kGridColor(2), kGridColor(3));
713
714 grid_data[1].point1 =
715 PointPainter::Data(0, -20 * scale, 0, kGridColor(0), kGridColor(1),
716 kGridColor(2), kGridColor(3));
717 grid_data[1].point2 =
718 PointPainter::Data(0, 20 * scale, 0, kGridColor(0), kGridColor(1),
719 kGridColor(2), kGridColor(3));
720
721 grid_data[2].point1 =
722 PointPainter::Data(0, 0, -20 * scale, kGridColor(0), kGridColor(1),
723 kGridColor(2), kGridColor(3));
724 grid_data[2].point2 =
725 PointPainter::Data(0, 0, 20 * scale, kGridColor(0), kGridColor(1),
726 kGridColor(2), kGridColor(3));
727
728 coordinate_grid_painter_.Upload(grid_data);
729
730 // Coordinate axes.
731 std::vector<LinePainter::Data> axes_data(3);
732
733 axes_data[0].point1 = PointPainter::Data(
734 0, 0, 0, kXAxisColor(0), kXAxisColor(1), kXAxisColor(2), kXAxisColor(3));
735 axes_data[0].point2 =
736 PointPainter::Data(50 * scale, 0, 0, kXAxisColor(0), kXAxisColor(1),
737 kXAxisColor(2), kXAxisColor(3));
738
739 axes_data[1].point1 = PointPainter::Data(
740 0, 0, 0, kYAxisColor(0), kYAxisColor(1), kYAxisColor(2), kYAxisColor(3));
741 axes_data[1].point2 =
742 PointPainter::Data(0, 50 * scale, 0, kYAxisColor(0), kYAxisColor(1),
743 kYAxisColor(2), kYAxisColor(3));
744
745 axes_data[2].point1 = PointPainter::Data(
746 0, 0, 0, kZAxisColor(0), kZAxisColor(1), kZAxisColor(2), kZAxisColor(3));
747 axes_data[2].point2 =
748 PointPainter::Data(0, 0, 50 * scale, kZAxisColor(0), kZAxisColor(1),
749 kZAxisColor(2), kZAxisColor(3));
750
751 coordinate_axes_painter_.Upload(axes_data);
752 }
753
UploadPointData(const bool selection_mode)754 void ModelViewerWidget::UploadPointData(const bool selection_mode) {
755 makeCurrent();
756
757 std::vector<PointPainter::Data> data;
758
759 // Assume we want to display the majority of points
760 data.reserve(points3D.size());
761
762 const size_t min_track_len =
763 static_cast<size_t>(options_->render->min_track_len);
764
765 if (selected_image_id_ == kInvalidImageId &&
766 images.count(selected_image_id_) == 0) {
767 for (const auto& point3D : points3D) {
768 if (point3D.second.Error() <= options_->render->max_error &&
769 point3D.second.Track().Length() >= min_track_len) {
770 PointPainter::Data painter_point;
771
772 painter_point.x = static_cast<float>(point3D.second.XYZ(0));
773 painter_point.y = static_cast<float>(point3D.second.XYZ(1));
774 painter_point.z = static_cast<float>(point3D.second.XYZ(2));
775
776 Eigen::Vector4f color;
777 if (selection_mode) {
778 const size_t index = selection_buffer_.size();
779 selection_buffer_.push_back(
780 std::make_pair(point3D.first, SELECTION_BUFFER_POINT_IDX));
781 color = IndexToRGB(index);
782
783 } else if (point3D.first == selected_point3D_id_) {
784 color = kSelectedPointColor;
785 } else {
786 color = point_colormap_->ComputeColor(point3D.first, point3D.second);
787 }
788
789 painter_point.r = color(0);
790 painter_point.g = color(1);
791 painter_point.b = color(2);
792 painter_point.a = color(3);
793
794 data.push_back(painter_point);
795 }
796 }
797 } else { // Image selected
798 const auto& selected_image = images[selected_image_id_];
799 for (const auto& point3D : points3D) {
800 if (point3D.second.Error() <= options_->render->max_error &&
801 point3D.second.Track().Length() >= min_track_len) {
802 PointPainter::Data painter_point;
803
804 painter_point.x = static_cast<float>(point3D.second.XYZ(0));
805 painter_point.y = static_cast<float>(point3D.second.XYZ(1));
806 painter_point.z = static_cast<float>(point3D.second.XYZ(2));
807
808 Eigen::Vector4f color;
809 if (selection_mode) {
810 const size_t index = selection_buffer_.size();
811 selection_buffer_.push_back(
812 std::make_pair(point3D.first, SELECTION_BUFFER_POINT_IDX));
813 color = IndexToRGB(index);
814 } else if (selected_image.HasPoint3D(point3D.first)) {
815 color = kSelectedImagePlaneColor;
816 } else if (point3D.first == selected_point3D_id_) {
817 color = kSelectedPointColor;
818 } else {
819 color = point_colormap_->ComputeColor(point3D.first, point3D.second);
820 }
821
822 painter_point.r = color(0);
823 painter_point.g = color(1);
824 painter_point.b = color(2);
825 painter_point.a = color(3);
826
827 data.push_back(painter_point);
828 }
829 }
830 }
831
832 point_painter_.Upload(data);
833 }
834
UploadPointConnectionData()835 void ModelViewerWidget::UploadPointConnectionData() {
836 makeCurrent();
837
838 std::vector<LinePainter::Data> line_data;
839
840 if (selected_point3D_id_ == kInvalidPoint3DId) {
841 // No point selected, so upload empty data
842 point_connection_painter_.Upload(line_data);
843 return;
844 }
845
846 const auto& point3D = points3D[selected_point3D_id_];
847
848 // 3D point position.
849 LinePainter::Data line;
850 line.point1 = PointPainter::Data(
851 static_cast<float>(point3D.XYZ(0)), static_cast<float>(point3D.XYZ(1)),
852 static_cast<float>(point3D.XYZ(2)), kSelectedPointColor(0),
853 kSelectedPointColor(1), kSelectedPointColor(2), 0.8f);
854
855 // All images in which 3D point is observed.
856 for (const auto& track_el : point3D.Track().Elements()) {
857 const Image& conn_image = images[track_el.image_id];
858 const Eigen::Vector3f conn_proj_center =
859 conn_image.ProjectionCenter().cast<float>();
860 line.point2 = PointPainter::Data(
861 conn_proj_center(0), conn_proj_center(1), conn_proj_center(2),
862 kSelectedPointColor(0), kSelectedPointColor(1), kSelectedPointColor(2),
863 0.8f);
864 line_data.push_back(line);
865 }
866
867 point_connection_painter_.Upload(line_data);
868 }
869
UploadImageData(const bool selection_mode)870 void ModelViewerWidget::UploadImageData(const bool selection_mode) {
871 makeCurrent();
872
873 std::vector<LinePainter::Data> line_data;
874 line_data.reserve(8 * reg_image_ids.size());
875
876 std::vector<TrianglePainter::Data> triangle_data;
877 triangle_data.reserve(2 * reg_image_ids.size());
878
879 for (const image_t image_id : reg_image_ids) {
880 const Image& image = images[image_id];
881 const Camera& camera = cameras[image.CameraId()];
882
883 Eigen::Vector4f plane_color;
884 Eigen::Vector4f frame_color;
885 if (selection_mode) {
886 const size_t index = selection_buffer_.size();
887 selection_buffer_.push_back(
888 std::make_pair(image_id, SELECTION_BUFFER_IMAGE_IDX));
889 plane_color = frame_color = IndexToRGB(index);
890 } else {
891 if (image_id == selected_image_id_) {
892 plane_color = kSelectedImagePlaneColor;
893 frame_color = kSelectedImageFrameColor;
894 } else {
895 image_colormap_->ComputeColor(image, &plane_color, &frame_color);
896 }
897 }
898
899 LinePainter::Data line1, line2, line3, line4, line5, line6, line7, line8;
900 TrianglePainter::Data triangle1, triangle2;
901 BuildImageModel(image, camera, image_size_, plane_color, frame_color,
902 &triangle1, &triangle2, &line1, &line2, &line3, &line4,
903 &line5, &line6, &line7, &line8);
904
905 // Lines are not colored with the indexed color in selection mode, so do not
906 // show them, so they do not block the selection process
907 if (!selection_mode) {
908 line_data.push_back(line1);
909 line_data.push_back(line2);
910 line_data.push_back(line3);
911 line_data.push_back(line4);
912 line_data.push_back(line5);
913 line_data.push_back(line6);
914 line_data.push_back(line7);
915 line_data.push_back(line8);
916 }
917
918 triangle_data.push_back(triangle1);
919 triangle_data.push_back(triangle2);
920 }
921
922 image_line_painter_.Upload(line_data);
923 image_triangle_painter_.Upload(triangle_data);
924 }
925
UploadImageConnectionData()926 void ModelViewerWidget::UploadImageConnectionData() {
927 makeCurrent();
928
929 std::vector<LinePainter::Data> line_data;
930 std::vector<image_t> image_ids;
931
932 if (selected_image_id_ != kInvalidImageId) {
933 // Show connections to selected images
934 image_ids.push_back(selected_image_id_);
935 } else if (options_->render->image_connections) {
936 // Show all connections
937 image_ids = reg_image_ids;
938 } else { // Disabled, so upload empty data
939 image_connection_painter_.Upload(line_data);
940 return;
941 }
942
943 for (const image_t image_id : image_ids) {
944 const Image& image = images.at(image_id);
945
946 const Eigen::Vector3f proj_center = image.ProjectionCenter().cast<float>();
947
948 // Collect all connected images
949 std::unordered_set<image_t> conn_image_ids;
950
951 for (const Point2D& point2D : image.Points2D()) {
952 if (point2D.HasPoint3D()) {
953 const Point3D& point3D = points3D[point2D.Point3DId()];
954 for (const auto& track_elem : point3D.Track().Elements()) {
955 conn_image_ids.insert(track_elem.image_id);
956 }
957 }
958 }
959
960 // Selected image in the center.
961 LinePainter::Data line;
962 line.point1 = PointPainter::Data(
963 proj_center(0), proj_center(1), proj_center(2),
964 kSelectedImageFrameColor(0), kSelectedImageFrameColor(1),
965 kSelectedImageFrameColor(2), 0.8f);
966
967 // All connected images to the selected image.
968 for (const image_t conn_image_id : conn_image_ids) {
969 const Image& conn_image = images[conn_image_id];
970 const Eigen::Vector3f conn_proj_center =
971 conn_image.ProjectionCenter().cast<float>();
972 line.point2 = PointPainter::Data(
973 conn_proj_center(0), conn_proj_center(1), conn_proj_center(2),
974 kSelectedImageFrameColor(0), kSelectedImageFrameColor(1),
975 kSelectedImageFrameColor(2), 0.8f);
976 line_data.push_back(line);
977 }
978 }
979
980 image_connection_painter_.Upload(line_data);
981 }
982
UploadMovieGrabberData()983 void ModelViewerWidget::UploadMovieGrabberData() {
984 makeCurrent();
985
986 std::vector<LinePainter::Data> path_data;
987 path_data.reserve(movie_grabber_widget_->views.size());
988
989 std::vector<LinePainter::Data> line_data;
990 line_data.reserve(4 * movie_grabber_widget_->views.size());
991
992 std::vector<TrianglePainter::Data> triangle_data;
993 triangle_data.reserve(2 * movie_grabber_widget_->views.size());
994
995 if (movie_grabber_widget_->views.size() > 0) {
996 const Image& image0 = movie_grabber_widget_->views[0];
997 Eigen::Vector3f prev_proj_center = image0.ProjectionCenter().cast<float>();
998
999 for (size_t i = 1; i < movie_grabber_widget_->views.size(); ++i) {
1000 const Image& image = movie_grabber_widget_->views[i];
1001 const Eigen::Vector3f curr_proj_center =
1002 image.ProjectionCenter().cast<float>();
1003 LinePainter::Data path;
1004 path.point1 = PointPainter::Data(
1005 prev_proj_center(0), prev_proj_center(1), prev_proj_center(2),
1006 kSelectedImagePlaneColor(0), kSelectedImagePlaneColor(1),
1007 kSelectedImagePlaneColor(2), kSelectedImagePlaneColor(3));
1008 path.point2 = PointPainter::Data(
1009 curr_proj_center(0), curr_proj_center(1), curr_proj_center(2),
1010 kSelectedImagePlaneColor(0), kSelectedImagePlaneColor(1),
1011 kSelectedImagePlaneColor(2), kSelectedImagePlaneColor(3));
1012 path_data.push_back(path);
1013 prev_proj_center = curr_proj_center;
1014 }
1015
1016 // Setup dummy camera with same settings as current OpenGL viewpoint.
1017 const float kDefaultImageWdith = 2048.0f;
1018 const float kDefaultImageHeight = 1536.0f;
1019 const float focal_length =
1020 -2.0f * std::tan(DegToRad(kFieldOfView) / 2.0f) * kDefaultImageWdith;
1021 Camera camera;
1022 camera.InitializeWithId(SimplePinholeCameraModel::model_id, focal_length,
1023 kDefaultImageWdith, kDefaultImageHeight);
1024
1025 // Build all camera models
1026 for (size_t i = 0; i < movie_grabber_widget_->views.size(); ++i) {
1027 const Image& image = movie_grabber_widget_->views[i];
1028 Eigen::Vector4f plane_color;
1029 Eigen::Vector4f frame_color;
1030 if (i == selected_movie_grabber_view_) {
1031 plane_color = kSelectedImagePlaneColor;
1032 frame_color = kSelectedImageFrameColor;
1033 } else {
1034 plane_color = kMovieGrabberImagePlaneColor;
1035 frame_color = kMovieGrabberImageFrameColor;
1036 }
1037
1038 LinePainter::Data line1, line2, line3, line4, line5, line6, line7, line8;
1039 TrianglePainter::Data triangle1, triangle2;
1040 BuildImageModel(image, camera, image_size_, plane_color, frame_color,
1041 &triangle1, &triangle2, &line1, &line2, &line3, &line4,
1042 &line5, &line6, &line7, &line8);
1043
1044 line_data.push_back(line1);
1045 line_data.push_back(line2);
1046 line_data.push_back(line3);
1047 line_data.push_back(line4);
1048 line_data.push_back(line5);
1049 line_data.push_back(line6);
1050 line_data.push_back(line7);
1051 line_data.push_back(line8);
1052
1053 triangle_data.push_back(triangle1);
1054 triangle_data.push_back(triangle2);
1055 }
1056 }
1057
1058 movie_grabber_path_painter_.Upload(path_data);
1059 movie_grabber_line_painter_.Upload(line_data);
1060 movie_grabber_triangle_painter_.Upload(triangle_data);
1061 }
1062
ComposeProjectionMatrix()1063 void ModelViewerWidget::ComposeProjectionMatrix() {
1064 projection_matrix_.setToIdentity();
1065 if (options_->render->projection_type ==
1066 RenderOptions::ProjectionType::PERSPECTIVE) {
1067 projection_matrix_.perspective(kFieldOfView, AspectRatio(), near_plane_,
1068 kFarPlane);
1069 } else if (options_->render->projection_type ==
1070 RenderOptions::ProjectionType::ORTHOGRAPHIC) {
1071 const float extent = OrthographicWindowExtent();
1072 projection_matrix_.ortho(-AspectRatio() * extent, AspectRatio() * extent,
1073 -extent, extent, near_plane_, kFarPlane);
1074 }
1075 }
1076
ZoomScale() const1077 float ModelViewerWidget::ZoomScale() const {
1078 // "Constant" scale factor w.r.t. zoom-level.
1079 return 2.0f * std::tan(static_cast<float>(DegToRad(kFieldOfView)) / 2.0f) *
1080 std::abs(focus_distance_) / height();
1081 }
1082
AspectRatio() const1083 float ModelViewerWidget::AspectRatio() const {
1084 return static_cast<float>(width()) / static_cast<float>(height());
1085 }
1086
OrthographicWindowExtent() const1087 float ModelViewerWidget::OrthographicWindowExtent() const {
1088 return std::tan(DegToRad(kFieldOfView) / 2.0f) * focus_distance_;
1089 }
1090
PositionToArcballVector(const float x,const float y) const1091 Eigen::Vector3f ModelViewerWidget::PositionToArcballVector(
1092 const float x, const float y) const {
1093 Eigen::Vector3f vec(2.0f * x / width() - 1, 1 - 2.0f * y / height(), 0.0f);
1094 const float norm2 = vec.squaredNorm();
1095 if (norm2 <= 1.0f) {
1096 vec.z() = std::sqrt(1.0f - norm2);
1097 } else {
1098 vec = vec.normalized();
1099 }
1100 return vec;
1101 }
1102
1103 } // namespace colmap
1104