1 /*
2 * This program is free software; you can redistribute it and/or
3 * modify it under the terms of the GNU General Public License
4 * as published by the Free Software Foundation; either version 2
5 * of the License, or (at your option) any later version.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software Foundation,
14 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15 *
16 * The Original Code is Copyright (C) 2011 Blender Foundation.
17 * All rights reserved.
18 */
19
20 /** \file
21 * \ingroup bke
22 *
23 * This file contains blender-side implementation of camera solver.
24 */
25
26 #include <limits.h>
27
28 #include "MEM_guardedalloc.h"
29
30 #include "DNA_anim_types.h"
31 #include "DNA_movieclip_types.h"
32
33 #include "BLI_listbase.h"
34 #include "BLI_math.h"
35 #include "BLI_string.h"
36 #include "BLI_utildefines.h"
37
38 #include "BLT_translation.h"
39
40 #include "BKE_fcurve.h"
41 #include "BKE_movieclip.h"
42 #include "BKE_tracking.h"
43
44 #include "RNA_access.h"
45
46 #include "libmv-capi.h"
47 #include "tracking_private.h"
48
49 typedef struct MovieReconstructContext {
50 struct libmv_Tracks *tracks;
51 bool select_keyframes;
52 int keyframe1, keyframe2;
53 int refine_flags;
54
55 struct libmv_Reconstruction *reconstruction;
56
57 char object_name[MAX_NAME];
58 bool is_camera;
59 short motion_flag;
60
61 libmv_CameraIntrinsicsOptions camera_intrinsics_options;
62
63 float reprojection_error;
64
65 TracksMap *tracks_map;
66
67 int sfra, efra;
68
69 /* Details about reconstruction error, reported by Libmv. */
70 char error_message[1024];
71 } MovieReconstructContext;
72
73 typedef struct ReconstructProgressData {
74 short *stop;
75 short *do_update;
76 float *progress;
77 char *stats_message;
78 int message_size;
79 } ReconstructProgressData;
80
81 /* Create new libmv Tracks structure from blender's tracks list. */
libmv_tracks_new(MovieClip * clip,ListBase * tracksbase,int width,int height)82 static struct libmv_Tracks *libmv_tracks_new(MovieClip *clip,
83 ListBase *tracksbase,
84 int width,
85 int height)
86 {
87 int tracknr = 0;
88 MovieTrackingTrack *track;
89 struct libmv_Tracks *tracks = libmv_tracksNew();
90
91 track = tracksbase->first;
92 while (track) {
93 FCurve *weight_fcurve = id_data_find_fcurve(
94 &clip->id, track, &RNA_MovieTrackingTrack, "weight", 0, NULL);
95
96 for (int a = 0; a < track->markersnr; a++) {
97 MovieTrackingMarker *marker = &track->markers[a];
98
99 if ((marker->flag & MARKER_DISABLED) == 0) {
100 float weight = track->weight;
101
102 if (weight_fcurve) {
103 int scene_framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
104 weight = evaluate_fcurve(weight_fcurve, scene_framenr);
105 }
106
107 libmv_tracksInsert(tracks,
108 marker->framenr,
109 tracknr,
110 (marker->pos[0] + track->offset[0]) * width,
111 (marker->pos[1] + track->offset[1]) * height,
112 weight);
113 }
114 }
115
116 track = track->next;
117 tracknr++;
118 }
119
120 return tracks;
121 }
122
123 /* Retrieve refined camera intrinsics from libmv to blender. */
reconstruct_retrieve_libmv_intrinsics(MovieReconstructContext * context,MovieTracking * tracking)124 static void reconstruct_retrieve_libmv_intrinsics(MovieReconstructContext *context,
125 MovieTracking *tracking)
126 {
127 struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
128 struct libmv_CameraIntrinsics *libmv_intrinsics = libmv_reconstructionExtractIntrinsics(
129 libmv_reconstruction);
130
131 libmv_CameraIntrinsicsOptions camera_intrinsics_options;
132 libmv_cameraIntrinsicsExtractOptions(libmv_intrinsics, &camera_intrinsics_options);
133
134 tracking_trackingCameraFromIntrinscisOptions(tracking, &camera_intrinsics_options);
135 }
136
137 /* Retrieve reconstructed tracks from libmv to blender.
138 * Actually, this also copies reconstructed cameras
139 * from libmv to movie clip datablock.
140 */
reconstruct_retrieve_libmv_tracks(MovieReconstructContext * context,MovieTracking * tracking)141 static bool reconstruct_retrieve_libmv_tracks(MovieReconstructContext *context,
142 MovieTracking *tracking)
143 {
144 struct libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
145 MovieTrackingReconstruction *reconstruction = NULL;
146 MovieReconstructedCamera *reconstructed;
147 MovieTrackingTrack *track;
148 ListBase *tracksbase = NULL;
149 int tracknr = 0;
150 bool ok = true;
151 bool origin_set = false;
152 int sfra = context->sfra, efra = context->efra;
153 float imat[4][4];
154
155 if (context->is_camera) {
156 tracksbase = &tracking->tracks;
157 reconstruction = &tracking->reconstruction;
158 }
159 else {
160 MovieTrackingObject *object = BKE_tracking_object_get_named(tracking, context->object_name);
161
162 tracksbase = &object->tracks;
163 reconstruction = &object->reconstruction;
164 }
165
166 unit_m4(imat);
167
168 track = tracksbase->first;
169 while (track) {
170 double pos[3];
171
172 if (libmv_reprojectionPointForTrack(libmv_reconstruction, tracknr, pos)) {
173 track->bundle_pos[0] = pos[0];
174 track->bundle_pos[1] = pos[1];
175 track->bundle_pos[2] = pos[2];
176
177 track->flag |= TRACK_HAS_BUNDLE;
178 track->error = libmv_reprojectionErrorForTrack(libmv_reconstruction, tracknr);
179 }
180 else {
181 track->flag &= ~TRACK_HAS_BUNDLE;
182 ok = false;
183
184 printf("Unable to reconstruct position for track #%d '%s'\n", tracknr, track->name);
185 }
186
187 track = track->next;
188 tracknr++;
189 }
190
191 if (reconstruction->cameras) {
192 MEM_freeN(reconstruction->cameras);
193 }
194
195 reconstruction->camnr = 0;
196 reconstruction->cameras = NULL;
197 reconstructed = MEM_callocN((efra - sfra + 1) * sizeof(MovieReconstructedCamera),
198 "temp reconstructed camera");
199
200 for (int a = sfra; a <= efra; a++) {
201 double matd[4][4];
202
203 if (libmv_reprojectionCameraForImage(libmv_reconstruction, a, matd)) {
204 float mat[4][4];
205 float error = libmv_reprojectionErrorForImage(libmv_reconstruction, a);
206
207 /* TODO(sergey): Use transpose utility. */
208 for (int i = 0; i < 4; i++) {
209 for (int j = 0; j < 4; j++) {
210 mat[i][j] = matd[i][j];
211 }
212 }
213
214 /* Ensure first camera has got zero rotation and transform.
215 * This is essential for object tracking to work -- this way
216 * we'll always know object and environment are properly
217 * oriented.
218 *
219 * There's one weak part tho, which is requirement object
220 * motion starts at the same frame as camera motion does,
221 * otherwise that;' be a Russian roulette whether object is
222 * aligned correct or not.
223 */
224 if (!origin_set) {
225 invert_m4_m4(imat, mat);
226 unit_m4(mat);
227 origin_set = true;
228 }
229 else {
230 mul_m4_m4m4(mat, imat, mat);
231 }
232
233 copy_m4_m4(reconstructed[reconstruction->camnr].mat, mat);
234 reconstructed[reconstruction->camnr].framenr = a;
235 reconstructed[reconstruction->camnr].error = error;
236 reconstruction->camnr++;
237 }
238 else {
239 ok = false;
240 printf("No camera for frame %d\n", a);
241 }
242 }
243
244 if (reconstruction->camnr) {
245 int size = reconstruction->camnr * sizeof(MovieReconstructedCamera);
246 reconstruction->cameras = MEM_mallocN(size, "reconstructed camera");
247 memcpy(reconstruction->cameras, reconstructed, size);
248 }
249
250 if (origin_set) {
251 track = tracksbase->first;
252 while (track) {
253 if (track->flag & TRACK_HAS_BUNDLE) {
254 mul_v3_m4v3(track->bundle_pos, imat, track->bundle_pos);
255 }
256
257 track = track->next;
258 }
259 }
260
261 MEM_freeN(reconstructed);
262
263 return ok;
264 }
265
266 /* Retrieve all the libmv data from context to blender's side data blocks. */
reconstruct_retrieve_libmv(MovieReconstructContext * context,MovieTracking * tracking)267 static int reconstruct_retrieve_libmv(MovieReconstructContext *context, MovieTracking *tracking)
268 {
269 /* take the intrinsics back from libmv */
270 reconstruct_retrieve_libmv_intrinsics(context, tracking);
271
272 return reconstruct_retrieve_libmv_tracks(context, tracking);
273 }
274
275 /* Convert blender's refinement flags to libmv's. */
reconstruct_refine_intrinsics_get_flags(MovieTracking * tracking,MovieTrackingObject * object)276 static int reconstruct_refine_intrinsics_get_flags(MovieTracking *tracking,
277 MovieTrackingObject *object)
278 {
279 int refine = tracking->settings.refine_camera_intrinsics;
280 int flags = 0;
281
282 if ((object->flag & TRACKING_OBJECT_CAMERA) == 0) {
283 return 0;
284 }
285
286 if (refine & REFINE_FOCAL_LENGTH) {
287 flags |= LIBMV_REFINE_FOCAL_LENGTH;
288 }
289
290 if (refine & REFINE_PRINCIPAL_POINT) {
291 flags |= LIBMV_REFINE_PRINCIPAL_POINT;
292 }
293
294 if (refine & REFINE_RADIAL_DISTORTION_K1) {
295 flags |= LIBMV_REFINE_RADIAL_DISTORTION_K1;
296 }
297
298 if (refine & REFINE_RADIAL_DISTORTION_K2) {
299 flags |= LIBMV_REFINE_RADIAL_DISTORTION_K2;
300 }
301
302 return flags;
303 }
304
305 /* Count tracks which has markers at both of keyframes. */
reconstruct_count_tracks_on_both_keyframes(MovieTracking * tracking,MovieTrackingObject * object)306 static int reconstruct_count_tracks_on_both_keyframes(MovieTracking *tracking,
307 MovieTrackingObject *object)
308 {
309 ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
310 int tot = 0;
311 int frame1 = object->keyframe1, frame2 = object->keyframe2;
312 MovieTrackingTrack *track;
313
314 track = tracksbase->first;
315 while (track) {
316 if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame1)) {
317 if (BKE_tracking_track_has_enabled_marker_at_frame(track, frame2)) {
318 tot++;
319 }
320 }
321
322 track = track->next;
323 }
324
325 return tot;
326 }
327
328 /* Perform early check on whether everything is fine to start reconstruction. */
BKE_tracking_reconstruction_check(MovieTracking * tracking,MovieTrackingObject * object,char * error_msg,int error_size)329 bool BKE_tracking_reconstruction_check(MovieTracking *tracking,
330 MovieTrackingObject *object,
331 char *error_msg,
332 int error_size)
333 {
334 if (tracking->settings.motion_flag & TRACKING_MOTION_MODAL) {
335 /* TODO: check for number of tracks? */
336 return true;
337 }
338 if ((tracking->settings.reconstruction_flag & TRACKING_USE_KEYFRAME_SELECTION) == 0) {
339 /* automatic keyframe selection does not require any pre-process checks */
340 if (reconstruct_count_tracks_on_both_keyframes(tracking, object) < 8) {
341 BLI_strncpy(error_msg,
342 N_("At least 8 common tracks on both keyframes are needed for reconstruction"),
343 error_size);
344
345 return false;
346 }
347 }
348
349 #ifndef WITH_LIBMV
350 BLI_strncpy(error_msg, N_("Blender is compiled without motion tracking library"), error_size);
351 return false;
352 #endif
353
354 return true;
355 }
356
357 /* Create context for camera/object motion reconstruction.
358 * Copies all data needed for reconstruction from movie
359 * clip datablock, so editing this clip is safe during
360 * reconstruction job is in progress.
361 */
BKE_tracking_reconstruction_context_new(MovieClip * clip,MovieTrackingObject * object,int keyframe1,int keyframe2,int width,int height)362 MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieClip *clip,
363 MovieTrackingObject *object,
364 int keyframe1,
365 int keyframe2,
366 int width,
367 int height)
368 {
369 MovieTracking *tracking = &clip->tracking;
370 MovieReconstructContext *context = MEM_callocN(sizeof(MovieReconstructContext),
371 "MovieReconstructContext data");
372 ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
373 float aspy = 1.0f / tracking->camera.pixel_aspect;
374 int num_tracks = BLI_listbase_count(tracksbase);
375 int sfra = INT_MAX, efra = INT_MIN;
376 MovieTrackingTrack *track;
377
378 BLI_strncpy(context->object_name, object->name, sizeof(context->object_name));
379 context->is_camera = object->flag & TRACKING_OBJECT_CAMERA;
380 context->motion_flag = tracking->settings.motion_flag;
381
382 context->select_keyframes = (tracking->settings.reconstruction_flag &
383 TRACKING_USE_KEYFRAME_SELECTION) != 0;
384
385 tracking_cameraIntrinscisOptionsFromTracking(
386 tracking, width, height, &context->camera_intrinsics_options);
387
388 context->tracks_map = tracks_map_new(context->object_name, context->is_camera, num_tracks, 0);
389
390 track = tracksbase->first;
391 while (track) {
392 int first = 0, last = track->markersnr - 1;
393 MovieTrackingMarker *first_marker = &track->markers[0];
394 MovieTrackingMarker *last_marker = &track->markers[track->markersnr - 1];
395
396 /* find first not-disabled marker */
397 while (first <= track->markersnr - 1 && first_marker->flag & MARKER_DISABLED) {
398 first++;
399 first_marker++;
400 }
401
402 /* find last not-disabled marker */
403 while (last >= 0 && last_marker->flag & MARKER_DISABLED) {
404 last--;
405 last_marker--;
406 }
407
408 if (first <= track->markersnr - 1) {
409 sfra = min_ii(sfra, first_marker->framenr);
410 }
411
412 if (last >= 0) {
413 efra = max_ii(efra, last_marker->framenr);
414 }
415
416 tracks_map_insert(context->tracks_map, track, NULL);
417
418 track = track->next;
419 }
420
421 context->sfra = sfra;
422 context->efra = efra;
423
424 context->tracks = libmv_tracks_new(clip, tracksbase, width, height * aspy);
425 context->keyframe1 = keyframe1;
426 context->keyframe2 = keyframe2;
427 context->refine_flags = reconstruct_refine_intrinsics_get_flags(tracking, object);
428
429 context->error_message[0] = '\0';
430
431 return context;
432 }
433
BKE_tracking_reconstruction_report_error_message(MovieReconstructContext * context,const char * error_message)434 void BKE_tracking_reconstruction_report_error_message(MovieReconstructContext *context,
435 const char *error_message)
436 {
437 if (context->error_message[0]) {
438 /* Only keep initial error message, the rest are inducted ones. */
439 return;
440 }
441 BLI_strncpy(context->error_message, error_message, sizeof(context->error_message));
442 }
443
BKE_tracking_reconstruction_error_message_get(const MovieReconstructContext * context)444 const char *BKE_tracking_reconstruction_error_message_get(const MovieReconstructContext *context)
445 {
446 return context->error_message;
447 }
448
449 /* Free memory used by a reconstruction process. */
BKE_tracking_reconstruction_context_free(MovieReconstructContext * context)450 void BKE_tracking_reconstruction_context_free(MovieReconstructContext *context)
451 {
452 if (context->reconstruction) {
453 libmv_reconstructionDestroy(context->reconstruction);
454 }
455
456 libmv_tracksDestroy(context->tracks);
457
458 tracks_map_free(context->tracks_map, NULL);
459
460 MEM_freeN(context);
461 }
462
463 /* Callback which is called from libmv side to update progress in the interface. */
reconstruct_update_solve_cb(void * customdata,double progress,const char * message)464 static void reconstruct_update_solve_cb(void *customdata, double progress, const char *message)
465 {
466 ReconstructProgressData *progressdata = customdata;
467
468 if (progressdata->progress) {
469 *progressdata->progress = progress;
470 *progressdata->do_update = true;
471 }
472
473 BLI_snprintf(
474 progressdata->stats_message, progressdata->message_size, "Solving camera | %s", message);
475 }
476
477 /* Fill in reconstruction options structure from reconstruction context. */
reconstructionOptionsFromContext(libmv_ReconstructionOptions * reconstruction_options,MovieReconstructContext * context)478 static void reconstructionOptionsFromContext(libmv_ReconstructionOptions *reconstruction_options,
479 MovieReconstructContext *context)
480 {
481 reconstruction_options->select_keyframes = context->select_keyframes;
482
483 reconstruction_options->keyframe1 = context->keyframe1;
484 reconstruction_options->keyframe2 = context->keyframe2;
485
486 reconstruction_options->refine_intrinsics = context->refine_flags;
487 }
488
489 /* Solve camera/object motion and reconstruct 3D markers position
490 * from a prepared reconstruction context.
491 *
492 * stop is not actually used at this moment, so reconstruction
493 * job could not be stopped.
494 *
495 * do_update, progress and stat_message are set by reconstruction
496 * callback in libmv side and passing to an interface.
497 */
BKE_tracking_reconstruction_solve(MovieReconstructContext * context,short * stop,short * do_update,float * progress,char * stats_message,int message_size)498 void BKE_tracking_reconstruction_solve(MovieReconstructContext *context,
499 short *stop,
500 short *do_update,
501 float *progress,
502 char *stats_message,
503 int message_size)
504 {
505 float error;
506
507 ReconstructProgressData progressdata;
508
509 libmv_ReconstructionOptions reconstruction_options;
510
511 progressdata.stop = stop;
512 progressdata.do_update = do_update;
513 progressdata.progress = progress;
514 progressdata.stats_message = stats_message;
515 progressdata.message_size = message_size;
516
517 reconstructionOptionsFromContext(&reconstruction_options, context);
518
519 if (context->motion_flag & TRACKING_MOTION_MODAL) {
520 context->reconstruction = libmv_solveModal(context->tracks,
521 &context->camera_intrinsics_options,
522 &reconstruction_options,
523 reconstruct_update_solve_cb,
524 &progressdata);
525 }
526 else {
527 context->reconstruction = libmv_solveReconstruction(context->tracks,
528 &context->camera_intrinsics_options,
529 &reconstruction_options,
530 reconstruct_update_solve_cb,
531 &progressdata);
532
533 if (context->select_keyframes) {
534 /* store actual keyframes used for reconstruction to update them in the interface later */
535 context->keyframe1 = reconstruction_options.keyframe1;
536 context->keyframe2 = reconstruction_options.keyframe2;
537 }
538 }
539
540 error = libmv_reprojectionError(context->reconstruction);
541
542 context->reprojection_error = error;
543 }
544
545 /* Finish reconstruction process by copying reconstructed data
546 * to an actual movie clip datablock.
547 */
BKE_tracking_reconstruction_finish(MovieReconstructContext * context,MovieTracking * tracking)548 bool BKE_tracking_reconstruction_finish(MovieReconstructContext *context, MovieTracking *tracking)
549 {
550 MovieTrackingReconstruction *reconstruction;
551 MovieTrackingObject *object;
552
553 if (!libmv_reconstructionIsValid(context->reconstruction)) {
554 BKE_tracking_reconstruction_report_error_message(
555 context, "Failed to solve the motion: most likely there are no good keyframes");
556 return false;
557 }
558
559 tracks_map_merge(context->tracks_map, tracking);
560 BKE_tracking_dopesheet_tag_update(tracking);
561
562 object = BKE_tracking_object_get_named(tracking, context->object_name);
563
564 if (context->is_camera) {
565 reconstruction = &tracking->reconstruction;
566 }
567 else {
568 reconstruction = &object->reconstruction;
569 }
570
571 /* update keyframe in the interface */
572 if (context->select_keyframes) {
573 object->keyframe1 = context->keyframe1;
574 object->keyframe2 = context->keyframe2;
575 }
576
577 reconstruction->error = context->reprojection_error;
578 reconstruction->flag |= TRACKING_RECONSTRUCTED;
579
580 if (!reconstruct_retrieve_libmv(context, tracking)) {
581 return false;
582 }
583
584 return true;
585 }
586
tracking_scale_reconstruction(ListBase * tracksbase,MovieTrackingReconstruction * reconstruction,const float scale[3])587 static void tracking_scale_reconstruction(ListBase *tracksbase,
588 MovieTrackingReconstruction *reconstruction,
589 const float scale[3])
590 {
591 float first_camera_delta[3] = {0.0f, 0.0f, 0.0f};
592
593 if (reconstruction->camnr > 0) {
594 mul_v3_v3v3(first_camera_delta, reconstruction->cameras[0].mat[3], scale);
595 }
596
597 for (int i = 0; i < reconstruction->camnr; i++) {
598 MovieReconstructedCamera *camera = &reconstruction->cameras[i];
599 mul_v3_v3(camera->mat[3], scale);
600 sub_v3_v3(camera->mat[3], first_camera_delta);
601 }
602
603 LISTBASE_FOREACH (MovieTrackingTrack *, track, tracksbase) {
604 if (track->flag & TRACK_HAS_BUNDLE) {
605 mul_v3_v3(track->bundle_pos, scale);
606 sub_v3_v3(track->bundle_pos, first_camera_delta);
607 }
608 }
609 }
610
611 /* Apply scale on all reconstructed cameras and bundles,
612 * used by camera scale apply operator.
613 */
BKE_tracking_reconstruction_scale(MovieTracking * tracking,float scale[3])614 void BKE_tracking_reconstruction_scale(MovieTracking *tracking, float scale[3])
615 {
616 LISTBASE_FOREACH (MovieTrackingObject *, object, &tracking->objects) {
617 ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
618 MovieTrackingReconstruction *reconstruction = BKE_tracking_object_get_reconstruction(tracking,
619 object);
620
621 tracking_scale_reconstruction(tracksbase, reconstruction, scale);
622 }
623 }
624