1 /*
2 * filter_dance.c -- animate images size and position to the audio
3 * Copyright (C) 2015 Meltytech, LLC
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2.1 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19
20 #include <framework/mlt.h>
21 #include <stdlib.h> // calloc(), free()
22 #include <math.h> // sin()
23 #include <string.h> // strdup()
24
25 // Private Constants
26 static const double PI = 3.14159265358979323846;
27
28 // Private Types
29 typedef struct
30 {
31 mlt_filter affine;
32 mlt_filter fft;
33 char* mag_prop_name;
34 int rel_pos;
35 double phase;
36 int preprocess_warned;
37 } private_data;
38
apply(double positive,double negative,double mag,double max_range)39 static double apply( double positive, double negative, double mag, double max_range )
40 {
41 if( mag == 0.0 )
42 {
43 return 0.0;
44 }
45 else if( mag > 0.0 && positive > 0.0 )
46 {
47 return positive * mag * max_range;
48 }
49 else if( mag < 0.0 && negative > 0.0 )
50 {
51 return negative * mag * max_range;
52 }
53 else if( positive )
54 {
55 return positive * fabs( mag ) * max_range;
56 }
57 else if ( negative )
58 {
59 return negative * -fabs( mag ) * max_range;
60 }
61
62 return 0.0;
63 }
64
filter_get_audio(mlt_frame frame,void ** buffer,mlt_audio_format * format,int * frequency,int * channels,int * samples)65 static int filter_get_audio( mlt_frame frame, void** buffer, mlt_audio_format* format, int* frequency, int* channels, int* samples )
66 {
67 mlt_filter filter = (mlt_filter)mlt_frame_pop_audio( frame );
68 mlt_properties filter_properties = MLT_FILTER_PROPERTIES( filter );
69 private_data* pdata = (private_data*)filter->child;
70 mlt_profile profile = mlt_service_profile( MLT_FILTER_SERVICE(filter) );
71
72 // Create the FFT filter the first time.
73 if( !pdata->fft )
74 {
75 pdata->fft = mlt_factory_filter( profile, "fft", NULL );
76 mlt_properties_set_int( MLT_FILTER_PROPERTIES( pdata->fft ), "window_size",
77 mlt_properties_get_int( filter_properties, "window_size" ) );
78 if( !pdata->fft )
79 {
80 mlt_log_warning( MLT_FILTER_SERVICE(filter), "Unable to create FFT.\n" );
81 return 1;
82 }
83 }
84
85 mlt_properties fft_properties = MLT_FILTER_PROPERTIES( pdata->fft );
86 double low_freq = mlt_properties_get_int( filter_properties, "frequency_low" );
87 double hi_freq = mlt_properties_get_int( filter_properties, "frequency_high" );
88 double threshold = mlt_properties_get_int( filter_properties, "threshold" );
89 double osc = mlt_properties_get_int( filter_properties, "osc" );
90 float peak = 0;
91
92 // The service must stay locked while using the private data
93 mlt_service_lock( MLT_FILTER_SERVICE( filter ) );
94
95 // Perform FFT processing on the frame
96 mlt_filter_process( pdata->fft, frame );
97 mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
98
99 float* bins = mlt_properties_get_data( fft_properties, "bins", NULL );
100 double window_level = mlt_properties_get_double( fft_properties, "window_level" );
101
102 if( bins && window_level == 1.0 )
103 {
104 // Find the peak FFT magnitude in the configured range of frequencies
105 int bin_count = mlt_properties_get_int( fft_properties, "bin_count" );
106 double bin_width = mlt_properties_get_double( fft_properties, "bin_width" );
107 int bin = 0;
108 for( bin = 0; bin < bin_count; bin++ )
109 {
110 double F = bin_width * (double)bin;
111 if( F >= low_freq && F <= hi_freq )
112 {
113 if( bins[bin] > peak )
114 {
115 peak = bins[bin];
116 }
117 }
118 }
119 }
120
121 mlt_service_unlock( MLT_FILTER_SERVICE( filter ) );
122
123 // Scale the magnitude to dB and apply oscillation
124 double dB = peak > 0.0 ? 20 * log10( peak ) : -1000.0;
125 double mag = 0.0;
126 if( dB >= threshold )
127 {
128 // Scale to range 0.0-1.0
129 mag = 1 - (dB / threshold);
130 if( osc != 0 )
131 {
132 // Apply the oscillation
133 double fps = mlt_profile_fps( profile );
134 double t = pdata->rel_pos / fps;
135 mag = mag * sin( 2 * PI * osc * t + pdata->phase );
136 }
137 pdata->rel_pos++;
138 } else {
139 pdata->rel_pos = 1;
140 // Alternate the phase so that the dancing alternates directions to the beat.
141 pdata->phase = pdata->phase ? 0 : PI;
142 mag = 0;
143 }
144
145 // Save the magnitude as a property on the frame to be used in get_image()
146 mlt_properties_set_double( MLT_FRAME_PROPERTIES(frame), pdata->mag_prop_name, mag );
147
148 return 0;
149 }
150
151 /** Get the image.
152 */
filter_get_image(mlt_frame frame,uint8_t ** image,mlt_image_format * format,int * width,int * height,int writable)153 static int filter_get_image( mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable )
154 {
155 int error = 0;
156 mlt_filter filter = (mlt_filter)mlt_frame_pop_service( frame );
157 private_data* pdata = (private_data*)filter->child;
158 mlt_properties filter_properties = MLT_FILTER_PROPERTIES( filter );
159 mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
160
161 if( mlt_properties_exists( frame_properties, pdata->mag_prop_name ) )
162 {
163 double mag = mlt_properties_get_double( frame_properties, pdata->mag_prop_name );
164 mlt_profile profile = mlt_service_profile( MLT_FILTER_SERVICE(filter) );
165
166 // scale_x and scale_y are in the range 0.0 to x.0 with:
167 // 0.0 = the largest possible
168 // < 1.0 = increase size (zoom in)
169 // 1.0 = no scaling
170 // > 1.0 = decrease size (zoom out)
171 double initial_zoom = mlt_properties_get_double( filter_properties, "initial_zoom" );
172 double zoom = mlt_properties_get_double( filter_properties, "zoom" );
173 double scale_xy = (100.0 / initial_zoom ) - ( fabs(mag) * (zoom / 100.0) );
174 if( scale_xy < 0.1 ) scale_xy = 0.1;
175
176 // ox is in the range -width to +width with:
177 // > 0 = offset to the left
178 // 0 = no offset
179 // < 0 = offset to the right
180 double left = mlt_properties_get_double( filter_properties, "left" );
181 double right = mlt_properties_get_double( filter_properties, "right" );
182 double ox = apply( left, right, mag, (double)profile->width / 100.0 );
183
184 // oy is in the range -height to +height with:
185 // > 0 = offset up
186 // 0 = no offset
187 // < 0 = offset down
188 double up = mlt_properties_get_double( filter_properties, "up" );
189 double down = mlt_properties_get_double( filter_properties, "down" );
190 double oy = apply( up, down, mag, (double)profile->height / 100.0 );
191
192 // fix_rotate_x is in the range -360 to +360 with:
193 // > 0 = rotate clockwise
194 // 0 = no rotation
195 // < 0 = rotate anticlockwise
196 double counterclockwise = mlt_properties_get_double( filter_properties, "counterclockwise" );
197 double clockwise = mlt_properties_get_double( filter_properties, "clockwise" );
198 double fix_rotate_x = apply( clockwise, counterclockwise, mag, 1.0 );
199
200 // Perform the affine.
201 mlt_service_lock( MLT_FILTER_SERVICE( filter ) );
202 mlt_properties affine_properties = MLT_FILTER_PROPERTIES( pdata->affine );
203 mlt_properties_set_double( affine_properties, "transition.scale_x", scale_xy );
204 mlt_properties_set_double( affine_properties, "transition.scale_y", scale_xy );
205 mlt_properties_set_double( affine_properties, "transition.ox", ox );
206 mlt_properties_set_double( affine_properties, "transition.oy", oy );
207 mlt_properties_set_double( affine_properties, "transition.fix_rotate_x", fix_rotate_x );
208 mlt_filter_process( pdata->affine, frame );
209 error = mlt_frame_get_image( frame, image, format, width, height, 0 );
210 mlt_service_unlock( MLT_FILTER_SERVICE( filter ) );
211 } else {
212 if ( pdata->preprocess_warned++ == 2 )
213 {
214 // This filter depends on the consumer processing the audio before the
215 // video.
216 mlt_log_warning( MLT_FILTER_SERVICE(filter), "Audio not preprocessed. Unable to dance.\n" );
217 }
218 mlt_frame_get_image( frame, image, format, width, height, 0 );
219 }
220
221 return error;
222 }
223
224 /** Filter processing.
225 */
filter_process(mlt_filter filter,mlt_frame frame)226 static mlt_frame filter_process( mlt_filter filter, mlt_frame frame )
227 {
228 mlt_frame_push_audio( frame, filter );
229 mlt_frame_push_audio( frame, filter_get_audio );
230 mlt_frame_push_service( frame, filter );
231 mlt_frame_push_get_image( frame, filter_get_image );
232 return frame;
233 }
234
filter_close(mlt_filter filter)235 static void filter_close( mlt_filter filter )
236 {
237 private_data* pdata = (private_data*)filter->child;
238
239 if ( pdata )
240 {
241 mlt_filter_close( pdata->affine );
242 mlt_filter_close( pdata->fft );
243 free( pdata->mag_prop_name );
244 free( pdata );
245 }
246 filter->child = NULL;
247 filter->close = NULL;
248 filter->parent.close = NULL;
249 mlt_service_close( &filter->parent );
250 }
251
252 /** Constructor for the filter.
253 */
filter_dance_init(mlt_profile profile,mlt_service_type type,const char * id,char * arg)254 mlt_filter filter_dance_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
255 {
256 mlt_filter filter = mlt_filter_new();
257 private_data* pdata = (private_data*)calloc( 1, sizeof(private_data) );
258 mlt_filter affine_filter = mlt_factory_filter( profile, "affine", "colour:0x00000000" );
259
260 if ( filter && pdata && affine_filter )
261 {
262 mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
263 mlt_properties_set_int( properties, "_filter_private", 1 );
264 mlt_properties_set_int( properties, "frequency_low", 20 );
265 mlt_properties_set_int( properties, "frequency_high", 20000 );
266 mlt_properties_set_double( properties, "threshold", -30.0 );
267 mlt_properties_set_double( properties, "osc", 5.0 );
268 mlt_properties_set_double( properties, "initial_zoom", 100.0 );
269 mlt_properties_set_double( properties, "zoom", 0.0 );
270 mlt_properties_set_double( properties, "left", 0.0 );
271 mlt_properties_set_double( properties, "right", 0.0 );
272 mlt_properties_set_double( properties, "up", 0.0 );
273 mlt_properties_set_double( properties, "down", 0.0 );
274 mlt_properties_set_double( properties, "clockwise", 0.0 );
275 mlt_properties_set_double( properties, "counterclockwise", 0.0 );
276 mlt_properties_set_int( properties, "window_size", 2048 );
277
278 // Create a unique ID for storing data on the frame
279 pdata->mag_prop_name = calloc( 1, 20 );
280 snprintf( pdata->mag_prop_name, 20, "fft_mag.%p", filter );
281 pdata->mag_prop_name[20 - 1] = '\0';
282
283 pdata->affine = affine_filter;
284 pdata->fft = 0;
285
286 filter->close = filter_close;
287 filter->process = filter_process;
288 filter->child = pdata;
289 }
290 else
291 {
292 mlt_log_error( MLT_FILTER_SERVICE(filter), "Filter dance failed\n" );
293
294 if( filter )
295 {
296 mlt_filter_close( filter );
297 }
298
299 if( affine_filter )
300 {
301 mlt_filter_close( affine_filter );
302 }
303
304 if( pdata )
305 {
306 free( pdata );
307 }
308
309 filter = NULL;
310 }
311 return filter;
312 }
313