1 /*
2 * producer_slowmotion.c -- create subspeed frames
3 * Author: Zachary Drew
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 */
19
20 #include "filter_motion_est.h"
21 #include <framework/mlt.h>
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <math.h>
26 #include <string.h>
27 #include <sys/time.h>
28 #include <assert.h>
29 #define SHIFT 8
30 #define ABS(a) ((a) >= 0 ? (a) : (-(a)))
31
32 // This is used to constrains pixel operations between two blocks to be within the image boundry
constrain(int * x,int * y,int * w,int * h,const int dx,const int dy,const int left,const int right,const int top,const int bottom)33 inline static int constrain( int *x, int *y, int *w, int *h,
34 const int dx, const int dy,
35 const int left, const int right,
36 const int top, const int bottom)
37 {
38 uint32_t penalty = 1 << SHIFT; // Retain a few extra bits of precision
39 int x2 = *x + dx;
40 int y2 = *y + dy;
41 int w_remains = *w;
42 int h_remains = *h;
43
44 // Origin of macroblock moves left of image boundy
45 if( *x < left || x2 < left ) {
46 w_remains = *w - left + ((*x < x2) ? *x : x2);
47 *x += *w - w_remains;
48 }
49 // Portion of macroblock moves right of image boundry
50 else if( *x + *w > right || x2 + *w > right )
51 w_remains = right - ((*x > x2) ? *x : x2);
52
53 // Origin of macroblock moves above image boundy
54 if( *y < top || y2 < top ) {
55 h_remains = *h - top + ((*y < y2) ? *y : y2);
56 *y += *h - h_remains;
57 }
58 // Portion of macroblock moves below image boundry
59 else if( *y + *h > bottom || y2 + *h > bottom )
60 h_remains = bottom - ((*y > y2) ? *y : y2);
61
62 if( w_remains == *w && h_remains == *h ) return penalty;
63 if( w_remains <= 0 || h_remains <= 0) return 0; // Block is clipped out of existence
64 penalty = (*w * *h * penalty)
65 / ( w_remains * h_remains); // Recipricol of the fraction of the block that remains
66
67 *w = w_remains; // Update the width and height
68 *h = h_remains;
69
70 return penalty;
71 }
72
motion_interpolate(uint8_t * first_image,uint8_t * second_image,uint8_t * output,int top_mb,int bottom_mb,int left_mb,int right_mb,int mb_w,int mb_h,int width,int height,int xstride,int ystride,double scale,motion_vector * vectors)73 static void motion_interpolate( uint8_t *first_image, uint8_t *second_image, uint8_t *output,
74 int top_mb, int bottom_mb, int left_mb, int right_mb,
75 int mb_w, int mb_h,
76 int width, int height,
77 int xstride, int ystride,
78 double scale,
79 motion_vector *vectors )
80 {
81 assert ( scale >= 0.0 && scale <= 1.0 );
82
83 int i, j;
84 int x,y,w,h;
85 int dx, dy;
86 int scaled_dx, scaled_dy;
87 int tx,ty;
88 uint8_t *f,*s,*r;
89 motion_vector *here;
90 int mv_width = width / mb_w;
91
92 for( j = top_mb; j <= bottom_mb; j++ ){
93 for( i = left_mb; i <= right_mb; i++ ){
94
95 here = vectors + j*mv_width + i;
96 scaled_dx = (1.0 - scale) * (double)here->dx;
97 scaled_dy = (1.0 - scale) * (double)here->dy;
98 dx = here->dx;
99 dy = here->dy;
100 w = mb_w; h = mb_h;
101 x = i * w; y = j * h;
102
103 // Denoise function caused some blocks to be completely clipped, ignore them
104 if (constrain( &x, &y, &w, &h, dx, dy, 0, width, 0, height) == 0 )
105 continue;
106
107 for( ty = y; ty < y + h ; ty++ ){
108 for( tx = x; tx < x + w ; tx++ ){
109
110 f = first_image + (tx + dx )*xstride + (ty + dy )*ystride;
111 s = second_image + (tx )*xstride + (ty )*ystride;
112 r = output + (tx+scaled_dx)*xstride + (ty+scaled_dy)*ystride;
113 /*
114 if( ABS(f[0] - s[0]) > 3 * here->msad / (mb_w * mb_h * 2) )
115 {
116 r[0] = f[0];
117 r[1] = f[1];
118 }
119
120 else
121 {
122
123 */
124 r[0] = ( 1.0 - scale ) * (double)f[0] + scale * (double)s[0];
125
126 if( dx % 2 == 0 )
127 {
128 if( scaled_dx % 2 == 0 )
129 r[1] = ( 1.0 - scale ) * (double)f[1] + scale * (double) s[1];
130 else
131 *(r-1) = ( 1.0 - scale ) * (double)f[1] + scale * (double) s[1];
132 }
133 else
134 {
135 if( scaled_dx %2 == 0 )
136 // FIXME: may exceed boundaries
137 r[1] = ( 1.0 - scale ) * ( (double)(*(f-1) + (double)f[3]) / 2.0 ) + scale * (double) s[1];
138 else
139 // FIXME: may exceed boundaries
140 *(r-1) = ( 1.0 - scale ) * ( (double)(*(f-1) + (double)f[3]) / 2.0 ) + scale * (double) s[1];
141 }
142 // }
143 }
144 }
145 }
146 }
147 }
148
149 // Image stack(able) method
slowmotion_get_image(mlt_frame this,uint8_t ** image,mlt_image_format * format,int * width,int * height,int writable)150 static int slowmotion_get_image( mlt_frame this, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable )
151 {
152
153 // Get the filter object and properties
154 mlt_producer producer = mlt_frame_pop_service( this );
155 mlt_frame second_frame = mlt_frame_pop_service( this );
156 mlt_frame first_frame = mlt_frame_pop_service( this );
157
158 mlt_properties producer_properties = MLT_PRODUCER_PROPERTIES( producer );
159
160 // Frame properties objects
161 mlt_properties frame_properties = MLT_FRAME_PROPERTIES( this );
162 mlt_properties first_frame_properties = MLT_FRAME_PROPERTIES( first_frame );
163 mlt_properties second_frame_properties = MLT_FRAME_PROPERTIES( second_frame );
164
165 // image stride
166 *format = mlt_image_yuv422;
167 int size = *width * *height * 2;
168 int xstride = 2;
169 int ystride = 2 * *width;
170
171 uint8_t *output = mlt_properties_get_data( producer_properties, "output_buffer", 0 );
172 if( output == NULL )
173 {
174 output = mlt_pool_alloc( size );
175
176 // Let someone else clean up
177 mlt_properties_set_data( producer_properties, "output_buffer", output, size, mlt_pool_release, NULL );
178 }
179
180 uint8_t *first_image = mlt_properties_get_data( first_frame_properties, "image", NULL );
181 uint8_t *second_image = mlt_properties_get_data( second_frame_properties, "image", NULL );
182
183 // which frames are buffered?
184
185 int error = 0;
186
187 if( first_image == NULL )
188 {
189 error = mlt_frame_get_image( first_frame, &first_image, format, width, height, writable );
190
191 if( error != 0 ) {
192 fprintf(stderr, "first_image == NULL get image died\n");
193 return error;
194 }
195 }
196
197 if( second_image == NULL )
198 {
199 error = mlt_frame_get_image( second_frame, &second_image, format, width, height, writable );
200
201 if( error != 0 ) {
202 fprintf(stderr, "second_image == NULL get image died\n");
203 return error;
204 }
205 }
206
207 // These need to passed onto the frame for other
208 mlt_properties_pass_list( frame_properties, second_frame_properties,
209 "motion_est.left_mb, motion_est.right_mb, \
210 motion_est.top_mb, motion_est.bottom_mb, \
211 motion_est.macroblock_width, motion_est.macroblock_height" );
212
213 // Pass the pointer to the vectors without serializing
214 mlt_properties_set_data( frame_properties, "motion_est.vectors",
215 mlt_properties_get_data( second_frame_properties, "motion_est.vectors", NULL ),
216 0, NULL, NULL );
217
218
219 // Start with a base image
220 memcpy( output, first_image, size );
221
222 if( mlt_properties_get_int( producer_properties, "method" ) == 1 ) {
223
224 mlt_position first_position = mlt_frame_get_position( first_frame );
225 double actual_position = mlt_producer_get_speed( producer ) * (double)mlt_frame_get_position( this );
226 double scale = actual_position - first_position;
227
228 motion_interpolate
229 (
230 first_image, second_image, output,
231 mlt_properties_get_int( second_frame_properties, "motion_est.top_mb" ),
232 mlt_properties_get_int( second_frame_properties, "motion_est.bottom_mb" ),
233 mlt_properties_get_int( second_frame_properties, "motion_est.left_mb" ),
234 mlt_properties_get_int( second_frame_properties, "motion_est.right_mb" ),
235 mlt_properties_get_int( second_frame_properties, "motion_est.macroblock_width" ),
236 mlt_properties_get_int( second_frame_properties, "motion_est.macroblock_height" ),
237 *width, *height,
238 xstride, ystride,
239 scale,
240 mlt_properties_get_data( second_frame_properties, "motion_est.vectors", NULL )
241 );
242
243 if( mlt_properties_get_int( producer_properties, "debug" ) == 1 ) {
244 mlt_filter watermark = mlt_properties_get_data( producer_properties, "watermark", NULL );
245
246 if( watermark == NULL ) {
247 mlt_profile profile = mlt_service_profile( MLT_PRODUCER_SERVICE( producer ) );
248 watermark = mlt_factory_filter( profile, "watermark", NULL );
249 mlt_properties_set_data( producer_properties, "watermark", watermark, 0, (mlt_destructor)mlt_filter_close, NULL );
250 mlt_producer_attach( producer, watermark );
251 }
252
253 mlt_properties wm_properties = MLT_FILTER_PROPERTIES( watermark );
254
255 char disp[30];
256 sprintf(disp, "+%10.2f.txt", actual_position);
257 mlt_properties_set( wm_properties, "resource", disp );
258
259 }
260
261 }
262
263 *image = output;
264 mlt_frame_set_image( this, output, size, NULL );
265
266 // Make sure that no further scaling is done
267 mlt_properties_set( frame_properties, "rescale.interps", "none" );
268 mlt_properties_set( frame_properties, "scale", "off" );
269
270 mlt_frame_close( first_frame );
271 mlt_frame_close( second_frame );
272
273 return 0;
274 }
275
slowmotion_get_frame(mlt_producer this,mlt_frame_ptr frame,int index)276 static int slowmotion_get_frame( mlt_producer this, mlt_frame_ptr frame, int index )
277 {
278 if ( !frame )
279 return 1;
280 // Construct a new frame
281 *frame = mlt_frame_init( MLT_PRODUCER_SERVICE( this ) );
282
283 mlt_properties properties = MLT_PRODUCER_PROPERTIES(this);
284
285
286 if( *frame )
287 {
288
289 mlt_frame first_frame = mlt_properties_get_data( properties, "first_frame", NULL );
290 mlt_frame second_frame = mlt_properties_get_data( properties, "second_frame", NULL );
291
292 mlt_position first_position = (first_frame != NULL) ? mlt_frame_get_position( first_frame ) : -1;
293 mlt_position second_position = (second_frame != NULL) ? mlt_frame_get_position( second_frame ) : -1;
294
295 // Get the real producer
296 mlt_producer real_producer = mlt_properties_get_data( properties, "producer", NULL );
297
298 // Our "in" needs to be the same, keep it so
299 mlt_properties_pass_list( MLT_PRODUCER_PROPERTIES( real_producer ), properties, "in" );
300
301 // Calculate our positions
302 double actual_position = mlt_producer_get_speed( this ) * (double)mlt_producer_position( this );
303 mlt_position need_first = floor( actual_position );
304 mlt_position need_second = need_first + 1;
305
306 if( need_first != first_position )
307 {
308 mlt_frame_close( first_frame );
309 first_position = -1;
310 first_frame = NULL;
311 }
312
313 if( need_second != second_position)
314 {
315 mlt_frame_close( second_frame );
316 second_position = -1;
317 second_frame = NULL;
318 }
319
320 if( first_frame == NULL )
321 {
322 // Seek the producer to the correct place
323 mlt_producer_seek( real_producer, need_first );
324
325 // Get the frame
326 mlt_service_get_frame( MLT_PRODUCER_SERVICE( real_producer ), &first_frame, index );
327 }
328
329 if( second_frame == NULL )
330 {
331 // Seek the producer to the correct place
332 mlt_producer_seek( real_producer, need_second );
333
334 // Get the frame
335 mlt_service_get_frame( MLT_PRODUCER_SERVICE( real_producer ), &second_frame, index );
336 }
337
338 // Make sure things are in their place
339 mlt_properties_set_data( properties, "first_frame", first_frame, 0, NULL, NULL );
340 mlt_properties_set_data( properties, "second_frame", second_frame, 0, NULL, NULL );
341
342 mlt_properties_set_int( MLT_FRAME_PROPERTIES( *frame ), "test_image", 0 );
343
344 // Stack the producer and producer's get image
345 mlt_frame_push_service( *frame, first_frame );
346 mlt_properties_inc_ref( MLT_FRAME_PROPERTIES( first_frame ) );
347
348 mlt_frame_push_service( *frame, second_frame );
349 mlt_properties_inc_ref( MLT_FRAME_PROPERTIES( second_frame ) );
350
351 mlt_frame_push_service( *frame, this );
352 mlt_frame_push_service( *frame, slowmotion_get_image );
353
354 // Give the returned frame temporal identity
355 mlt_frame_set_position( *frame, mlt_producer_position( this ) );
356 }
357
358 return 0;
359 }
360
producer_slowmotion_init(mlt_profile profile,mlt_service_type type,const char * id,char * arg)361 mlt_producer producer_slowmotion_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
362 {
363 mlt_producer this = mlt_producer_new( profile );
364
365 // Wrap the loader
366 mlt_producer real_producer = mlt_factory_producer( profile, NULL, arg );
367
368 // We need to apply the motion estimation filter manually
369 mlt_filter filter = mlt_factory_filter( profile, "motion_est", NULL );
370
371 if ( this != NULL && real_producer != NULL && filter != NULL)
372 {
373 // attach the motion_est filter to the real producer
374 mlt_producer_attach( real_producer, filter );
375
376 // Get the properties of this producer
377 mlt_properties properties = MLT_PRODUCER_PROPERTIES( this );
378
379 // The loader normalised it for us already
380 mlt_properties_set_int( properties, "loader_normalised", 1);
381
382 // Store the producer and filter
383 mlt_properties_set_data( properties, "producer", real_producer, 0, ( mlt_destructor )mlt_producer_close, NULL );
384 mlt_properties_set_data( properties, "motion_est", filter, 0, ( mlt_destructor )mlt_filter_close, NULL );
385 mlt_properties_set_int( MLT_FILTER_PROPERTIES( filter ), "macroblock_width", 16 );
386 mlt_properties_set_int( MLT_FILTER_PROPERTIES( filter ), "macroblock_height", 16 );
387 mlt_properties_set_int( MLT_FILTER_PROPERTIES( filter ), "denoise", 0 );
388
389 // Grap some stuff from the real_producer
390 mlt_properties_pass_list( properties, MLT_PRODUCER_PROPERTIES( real_producer ),
391 "in, out, length, resource" );
392
393 // Since we control the seeking, prevent it from seeking on its own
394 mlt_producer_set_speed( real_producer, 0 );
395
396 //mlt_properties_set( properties, "method", "onefield" );
397
398 // Override the get_frame method
399 this->get_frame = slowmotion_get_frame;
400
401 }
402 else
403 {
404 if ( this )
405 mlt_producer_close( this );
406 if ( real_producer )
407 mlt_producer_close( real_producer );
408 if ( filter )
409 mlt_filter_close( filter );
410
411 this = NULL;
412 }
413 return this;
414 }
415