1 /*
2 * filter_lightshow.cpp -- animate color to the audio
3 * Copyright (C) 2015-2020 Meltytech, LLC
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2.1 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19
20 #include "common.h"
21 #include <framework/mlt.h>
22 #include <stdlib.h> // calloc(), free()
23 #include <math.h> // sin()
24 #include <QPainter>
25 #include <QImage>
26
27 // Private Constants
28 static const double PI = 3.14159265358979323846;
29
30 // Private Types
31 typedef struct
32 {
33 mlt_filter fft;
34 char* mag_prop_name;
35 int rel_pos;
36 int preprocess_warned;
37 } private_data;
38
filter_get_audio(mlt_frame frame,void ** buffer,mlt_audio_format * format,int * frequency,int * channels,int * samples)39 static int filter_get_audio( mlt_frame frame, void** buffer, mlt_audio_format* format, int* frequency, int* channels, int* samples )
40 {
41 mlt_filter filter = (mlt_filter)mlt_frame_pop_audio( frame );
42 mlt_properties filter_properties = MLT_FILTER_PROPERTIES( filter );
43 private_data* pdata = (private_data*)filter->child;
44
45 // Create the FFT filter the first time.
46 if( !pdata->fft )
47 {
48 mlt_profile profile = mlt_service_profile( MLT_FILTER_SERVICE(filter) );
49 pdata->fft = mlt_factory_filter( profile, "fft", NULL );
50 mlt_properties_set_int( MLT_FILTER_PROPERTIES( pdata->fft ), "window_size",
51 mlt_properties_get_int( filter_properties, "window_size" ) );
52 if( !pdata->fft )
53 {
54 mlt_log_warning( MLT_FILTER_SERVICE(filter), "Unable to create FFT.\n" );
55 return 1;
56 }
57 }
58
59 mlt_properties fft_properties = MLT_FILTER_PROPERTIES( pdata->fft );
60 double low_freq = mlt_properties_get_int( filter_properties, "frequency_low" );
61 double hi_freq = mlt_properties_get_int( filter_properties, "frequency_high" );
62 double threshold = mlt_properties_get_int( filter_properties, "threshold" );
63 double osc = mlt_properties_get_int( filter_properties, "osc" );
64 float peak = 0;
65
66 // The service must stay locked while using the private data
67 mlt_service_lock( MLT_FILTER_SERVICE( filter ) );
68
69 // Perform FFT processing on the frame
70 mlt_filter_process( pdata->fft, frame );
71 mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
72
73 float* bins = (float*)mlt_properties_get_data( fft_properties, "bins", NULL );
74 double window_level = mlt_properties_get_double( fft_properties, "window_level" );
75
76 if( bins && window_level == 1.0 )
77 {
78 // Find the peak FFT magnitude in the configured range of frequencies
79 int bin_count = mlt_properties_get_int( fft_properties, "bin_count" );
80 double bin_width = mlt_properties_get_double( fft_properties, "bin_width" );
81 int bin = 0;
82 for( bin = 0; bin < bin_count; bin++ )
83 {
84 double F = bin_width * (double)bin;
85 if( F >= low_freq && F <= hi_freq )
86 {
87 if( bins[bin] > peak )
88 {
89 peak = bins[bin];
90 }
91 }
92 }
93 }
94
95 mlt_service_unlock( MLT_FILTER_SERVICE( filter ) );
96
97 // Scale the magnitude to dB and apply oscillation
98 double dB = peak > 0.0 ? 20 * log10( peak ) : -1000.0;
99 double mag = 0.0;
100 if( dB >= threshold )
101 {
102 // Scale to range 0.0-1.0
103 mag = 1 - (dB / threshold);
104 if( osc != 0 )
105 {
106 // Apply the oscillation
107 double fps = mlt_profile_fps( mlt_service_profile( MLT_FILTER_SERVICE(filter) ) );
108 double t = pdata->rel_pos / fps;
109 mag = mag * sin( 2 * PI * osc * t );
110 }
111 pdata->rel_pos++;
112 } else {
113 pdata->rel_pos = 1;
114 mag = 0;
115 }
116
117 // Save the magnitude as a property on the frame to be used in get_image()
118 mlt_properties_set_double( MLT_FRAME_PROPERTIES(frame), pdata->mag_prop_name, mag );
119
120 return 0;
121 }
122
setup_pen(QPainter & p,QRect & rect,mlt_properties filter_properties)123 static void setup_pen( QPainter& p, QRect& rect, mlt_properties filter_properties )
124 {
125 QVector<QColor> colors;
126 bool color_found = true;
127
128 // Find user specified colors for the gradient
129 while( color_found ) {
130 QString prop_name = QString("color.") + QString::number(colors.size() + 1);
131 if( mlt_properties_exists(filter_properties, prop_name.toUtf8().constData() ) ) {
132 mlt_color mcolor = mlt_properties_get_color( filter_properties, prop_name.toUtf8().constData() );
133 colors.append( QColor( mcolor.r, mcolor.g, mcolor.b, mcolor.a ) );
134 } else {
135 color_found = false;
136 }
137 }
138
139 if( !colors.size() ) {
140 // No color specified. Just use white.
141 p.setBrush( Qt::white );
142 } else if( colors.size() == 1 ) {
143 // Only use one color
144 p.setBrush( colors[0] );
145 } else {
146 // Use Gradient
147 qreal sx = 1.0;
148 qreal sy = 1.0;
149 qreal dx = rect.x();
150 qreal dy = rect.y();
151 qreal radius = rect.width() / 2;
152
153 if ( rect.width() > rect.height() )
154 {
155 radius = rect.height() / 2;
156 sx = (qreal)rect.width() / (qreal)rect.height();
157 } else if ( rect.height() > rect.width() ) {
158 radius = rect.width() / 2;
159 sy = (qreal)rect.height() / (qreal)rect.width();
160 }
161
162 QPointF center( radius, radius );
163 QRadialGradient gradient( center, radius );
164
165 qreal step = 1.0 / ( colors.size() - 1 );
166 for( int i = 0; i < colors.size(); i++ )
167 {
168 gradient.setColorAt( (qreal)i * step, colors[i] );
169 }
170
171 QBrush brush( gradient );
172 QTransform transform( sx, 0.0, 0.0, 0.0, sy, 0.0, dx, dy, 1.0 );
173 brush.setTransform( transform );
174 p.setBrush( brush );
175 }
176 p.setPen( QColor(0,0,0,0) ); // Clear pen
177 }
178
draw_light(mlt_properties filter_properties,QImage * qimg,mlt_rect * rect,double mag)179 static void draw_light( mlt_properties filter_properties, QImage* qimg, mlt_rect* rect, double mag )
180 {
181 QPainter p( qimg );
182 QRect r( rect->x, rect->y, rect->w, rect->h );
183 p.setRenderHint( QPainter::Antialiasing );
184 // Output transparency = input transparency
185 p.setCompositionMode(QPainter::CompositionMode_SourceAtop);
186 setup_pen( p, r, filter_properties );
187 p.setOpacity( mag );
188 p.drawRect( r );
189 p.end();
190 }
191
192 /** Get the image.
193 */
filter_get_image(mlt_frame frame,uint8_t ** image,mlt_image_format * format,int * width,int * height,int writable)194 static int filter_get_image( mlt_frame frame, uint8_t **image, mlt_image_format *format, int *width, int *height, int writable )
195 {
196 int error = 0;
197 mlt_filter filter = (mlt_filter)mlt_frame_pop_service( frame );
198
199 private_data* pdata = (private_data*)filter->child;
200 mlt_properties filter_properties = MLT_FILTER_PROPERTIES( filter );
201 mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
202
203 if( mlt_properties_exists( frame_properties, pdata->mag_prop_name ) )
204 {
205 double mag = mlt_properties_get_double( frame_properties, pdata->mag_prop_name );
206 mlt_position position = mlt_filter_get_position( filter, frame );
207 mlt_position length = mlt_filter_get_length2( filter, frame );
208 mlt_rect rect = mlt_properties_anim_get_rect( filter_properties, "rect", position, length );
209
210 // Get the current image
211 *format = mlt_image_rgba;
212 error = mlt_frame_get_image( frame, image, format, width, height, 1 );
213
214 if ( strchr( mlt_properties_get( filter_properties, "rect" ), '%' ) ) {
215 rect.x *= *width;
216 rect.w *= *width;
217 rect.y *= *height;
218 rect.h *= *height;
219 } else {
220 mlt_profile profile = mlt_service_profile(MLT_FILTER_SERVICE(filter));
221 double scale = mlt_profile_scale_width(profile, *width);
222 rect.x *= scale;
223 rect.w *= scale;
224 scale = mlt_profile_scale_height(profile, *height);
225 rect.y *= scale;
226 rect.h *= scale;
227 }
228
229 // Draw the light
230 if( !error ) {
231 QImage qimg( *width, *height, QImage::Format_ARGB32 );
232 convert_mlt_to_qimage_rgba( *image, &qimg, *width, *height );
233 draw_light( filter_properties, &qimg, &rect, mag );
234 convert_qimage_to_mlt_rgba( &qimg, *image, *width, *height );
235 }
236 } else {
237 if ( pdata->preprocess_warned++ == 2 )
238 {
239 // This filter depends on the consumer processing the audio before
240 // the video.
241 mlt_log_warning( MLT_FILTER_SERVICE(filter), "Audio not preprocessed.\n" );
242 }
243 mlt_frame_get_image( frame, image, format, width, height, writable );
244 }
245
246 return error;
247 }
248
249 /** Filter processing.
250 */
filter_process(mlt_filter filter,mlt_frame frame)251 static mlt_frame filter_process( mlt_filter filter, mlt_frame frame )
252 {
253 if( mlt_frame_is_test_card( frame ) ) {
254 // The producer does not generate video. This filter will create an
255 // image on the producer's behalf.
256 mlt_properties frame_properties = MLT_FRAME_PROPERTIES( frame );
257 mlt_profile profile = mlt_service_profile( MLT_FILTER_SERVICE(filter) );
258 mlt_properties_set_int( frame_properties, "progressive", 1 );
259 mlt_properties_set_double( frame_properties, "aspect_ratio", mlt_profile_sar( profile ) );
260 mlt_properties_set_int( frame_properties, "meta.media.width", profile->width );
261 mlt_properties_set_int( frame_properties, "meta.media.height", profile->height );
262 // Tell the framework that there really is an image.
263 mlt_properties_set_int( frame_properties, "test_image", 0 );
264 // Push a callback to create the image.
265 mlt_frame_push_get_image( frame, create_image );
266 }
267
268 mlt_frame_push_audio( frame, filter );
269 mlt_frame_push_audio( frame, (void*)filter_get_audio );
270 mlt_frame_push_service( frame, filter );
271 mlt_frame_push_get_image( frame, filter_get_image );
272 return frame;
273 }
274
filter_close(mlt_filter filter)275 static void filter_close( mlt_filter filter )
276 {
277 private_data* pdata = (private_data*)filter->child;
278
279 if ( pdata )
280 {
281 mlt_filter_close( pdata->fft );
282 free( pdata->mag_prop_name );
283 free( pdata );
284 }
285 filter->child = NULL;
286 filter->close = NULL;
287 filter->parent.close = NULL;
288 mlt_service_close( &filter->parent );
289 }
290
291 /** Constructor for the filter.
292 */
293
294 extern "C" {
295
filter_lightshow_init(mlt_profile profile,mlt_service_type type,const char * id,char * arg)296 mlt_filter filter_lightshow_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
297 {
298 mlt_filter filter = mlt_filter_new();
299 private_data* pdata = (private_data*)calloc( 1, sizeof(private_data) );
300
301 if ( filter && pdata && createQApplicationIfNeeded( MLT_FILTER_SERVICE(filter) ) )
302 {
303 mlt_properties properties = MLT_FILTER_PROPERTIES( filter );
304 mlt_properties_set_int( properties, "_filter_private", 1 );
305 mlt_properties_set_int( properties, "frequency_low", 20 );
306 mlt_properties_set_int( properties, "frequency_high", 20000 );
307 mlt_properties_set_double( properties, "threshold", -30.0 );
308 mlt_properties_set_double( properties, "osc", 5.0 );
309 mlt_properties_set( properties, "color.1", "0xffffffff" );
310 mlt_properties_set( properties, "rect", "0% 0% 100% 100%" );
311 mlt_properties_set_int( properties, "window_size", 2048 );
312
313 // Create a unique ID for storing data on the frame
314 pdata->mag_prop_name = (char*)calloc( 1, 20 );
315 snprintf( pdata->mag_prop_name, 20, "fft_mag.%p", filter );
316 pdata->mag_prop_name[20 - 1] = '\0';
317
318 pdata->fft = 0;
319
320 filter->close = filter_close;
321 filter->process = filter_process;
322 filter->child = pdata;
323 }
324 else
325 {
326 mlt_log_error( MLT_FILTER_SERVICE(filter), "Filter lightshow failed\n" );
327
328 if( filter )
329 {
330 mlt_filter_close( filter );
331 }
332
333 if( pdata )
334 {
335 free( pdata );
336 }
337
338 filter = NULL;
339 }
340 return filter;
341 }
342
343 }
344
345