1 /*
2 * filter_rbpitch.c -- adjust audio pitch
3 * Copyright (C) 2020 Meltytech, LLC
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 */
19
20 #include <framework/mlt.h>
21 #include <framework/mlt_log.h>
22
23 #include <rubberband/RubberBandStretcher.h>
24
25 #include <algorithm>
26 #include <cstring>
27 #include <math.h>
28
29 using namespace RubberBand;
30
31 // Private Types
32 typedef struct
33 {
34 RubberBandStretcher* s;
35 int rubberband_frequency;
36 uint64_t in_samples;
37 uint64_t out_samples;
38 } private_data;
39
40 static const size_t MAX_CHANNELS = 10;
41
rbpitch_get_audio(mlt_frame frame,void ** buffer,mlt_audio_format * format,int * frequency,int * channels,int * samples)42 static int rbpitch_get_audio( mlt_frame frame, void **buffer, mlt_audio_format *format, int *frequency, int *channels, int *samples )
43 {
44 mlt_filter filter = static_cast<mlt_filter>(mlt_frame_pop_audio( frame ));
45 mlt_properties filter_properties = MLT_FILTER_PROPERTIES(filter);
46 private_data* pdata = (private_data*)filter->child;
47 if ( *channels > (int)MAX_CHANNELS )
48 {
49 mlt_log_error( MLT_FILTER_SERVICE(filter), "Too many channels requested: %d > %d\n", *channels, (int)MAX_CHANNELS );
50 return mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
51 }
52
53 mlt_properties unique_properties = mlt_frame_get_unique_properties( frame, MLT_FILTER_SERVICE(filter) );
54 if ( !unique_properties )
55 {
56 mlt_log_error( MLT_FILTER_SERVICE(filter), "Missing unique_properites\n" );
57 return mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
58 }
59
60 // Get the producer's audio
61 int requested_frequency = *frequency;
62 int requested_samples = *samples;
63 *format = mlt_audio_float;
64 int error = mlt_frame_get_audio( frame, buffer, format, frequency, channels, samples );
65 if ( error ) return error;
66
67 // Make sure the audio is in the correct format
68 // This is useful if the filter is encapsulated in a producer and does not
69 // have a normalizing filter before it.
70 if (*format != mlt_audio_float && frame->convert_audio != NULL)
71 {
72 frame->convert_audio( frame, buffer, format, mlt_audio_float );
73 }
74
75 // Sanity check parameters
76 // rubberband library crashes have been seen with a very large scale factor
77 // or very small sampling frequency. Very small scale factor and very high
78 // sampling frequency can result in too much audio lag.
79 // Disallow these extreme scenarios for now. Maybe it will be improved in
80 // the future.
81 double pitchscale = mlt_properties_get_double( unique_properties, "pitchscale" );
82 pitchscale = CLAMP( pitchscale, 0.05, 50.0 );
83 double timeratio = 1.0;
84 int stretch = mlt_properties_get_int( unique_properties, "stretch" );
85 int rubberband_frequency = *frequency;
86 if( stretch )
87 {
88 rubberband_frequency = requested_frequency;
89 timeratio = (double)requested_samples / (double)*samples;
90 }
91 rubberband_frequency = CLAMP( rubberband_frequency, 10000, 300000 );
92
93 // Protect the RubberBandStretcher instance.
94 mlt_service_lock( MLT_FILTER_SERVICE(filter) );
95
96 // Configure the stretcher.
97 RubberBandStretcher* s = pdata->s;
98 if ( !s || s->available() == -1 || (int)s->getChannelCount() != *channels || pdata->rubberband_frequency != rubberband_frequency )
99 {
100 mlt_log_debug( MLT_FILTER_SERVICE(filter), "Create a new stretcher\t%d\t%d\t%f\n", *channels, rubberband_frequency, pitchscale );
101 delete s;
102 // Create a rubberband instance
103 RubberBandStretcher::Options options = RubberBandStretcher::OptionProcessRealTime;
104 s = new RubberBandStretcher(rubberband_frequency, *channels, options, 1.0, pitchscale);
105 pdata->s = s;
106 pdata->rubberband_frequency = rubberband_frequency;
107 pdata->in_samples = 0;
108 pdata->out_samples = 0;
109 }
110 s->setPitchScale(pitchscale);
111 if( pitchscale >= 0.5 && pitchscale <= 2.0 )
112 {
113 // Pitch adjustment < 200%
114 s->setPitchOption(RubberBandStretcher::OptionPitchHighQuality);
115 s->setTransientsOption(RubberBandStretcher::OptionTransientsCrisp);
116 }
117 else
118 {
119 // Pitch adjustment > 200%
120 // "HighConsistency" and "Smooth" options help to avoid large memory
121 // consumption and crashes that can occur for large pitch adjustments.
122 s->setPitchOption(RubberBandStretcher::OptionPitchHighConsistency);
123 s->setTransientsOption(RubberBandStretcher::OptionTransientsSmooth);
124 }
125 s->setTimeRatio( timeratio );
126
127 // Configure input and output buffers and counters.
128 int consumed_samples = 0;
129 int total_consumed_samples = 0;
130 int received_samples = 0;
131 struct mlt_audio_s in;
132 struct mlt_audio_s out;
133 mlt_audio_set_values( &in, *buffer, *frequency, *format, *samples, *channels );
134 if( stretch )
135 {
136 *frequency = requested_frequency;
137 *samples = requested_samples;
138 }
139 mlt_audio_set_values( &out, NULL, *frequency, *format, *samples, *channels );
140 mlt_audio_alloc_data( &out );
141
142 // Process all input samples
143 while ( true )
144 {
145 // Send more samples to the stretcher
146 if ( consumed_samples == in.samples )
147 {
148 // Continue to repeat input samples into the stretcher until it
149 // provides the desired number of samples out.
150 consumed_samples = 0;
151 mlt_log_debug( MLT_FILTER_SERVICE(filter), "Repeat samples\n");
152 }
153 int process_samples = std::min( in.samples - consumed_samples, (int)s->getSamplesRequired() );
154 if ( process_samples == 0 && received_samples == out.samples && total_consumed_samples < in.samples )
155 {
156 // No more out samples are needed, but input samples are still available.
157 // Send the final input samples for processing.
158 process_samples = in.samples - total_consumed_samples;
159 }
160 if ( process_samples > 0 )
161 {
162 float* in_planes[MAX_CHANNELS];
163 for ( int i = 0; i < in.channels; i++ )
164 {
165 in_planes[i] = ((float*)in.data) + (in.samples * i) + consumed_samples;
166 }
167 s->process( in_planes, process_samples, false );
168 consumed_samples += process_samples;
169 total_consumed_samples += process_samples;
170 pdata->in_samples += process_samples;
171 }
172
173 // Receive samples from the stretcher
174 int retrieve_samples = std::min( out.samples - received_samples, s->available() );
175 if ( retrieve_samples > 0 )
176 {
177 float* out_planes[MAX_CHANNELS];
178 for ( int i = 0; i < out.channels; i++ )
179 {
180 out_planes[i] = ((float*)out.data) + (out.samples * i) + received_samples;
181 }
182 retrieve_samples = (int)s->retrieve( out_planes, retrieve_samples );
183 received_samples += retrieve_samples;
184 pdata->out_samples += retrieve_samples;
185 }
186
187 mlt_log_debug( MLT_FILTER_SERVICE(filter), "Process: %d\t Retrieve: %d\n", process_samples, retrieve_samples );
188
189 if ( received_samples == out.samples && total_consumed_samples >= in.samples )
190 {
191 // There is nothing more to do;
192 break;
193 }
194 }
195
196 // Save the processed samples.
197 mlt_audio_shrink( &out, received_samples );
198 mlt_frame_set_audio( frame, out.data, out.format, 0, out.release_data );
199 mlt_audio_get_values( &out, buffer, frequency, format, samples, channels );
200
201 // Report the latency.
202 double latency = (double)(pdata->in_samples - pdata->out_samples) * 1000.0 / (double)*frequency;
203 mlt_properties_set_double( filter_properties, "latency", latency );
204
205 mlt_service_unlock( MLT_FILTER_SERVICE(filter) );
206
207 mlt_log_debug( MLT_FILTER_SERVICE(filter), "Requested: %d\tReceived: %d\tSent: %d\tLatency: %d(%fms)\n", requested_samples, in.samples, out.samples, (int)(pdata->in_samples - pdata->out_samples), latency );
208 return error;
209 }
210
filter_process(mlt_filter filter,mlt_frame frame)211 static mlt_frame filter_process( mlt_filter filter, mlt_frame frame )
212 {
213 mlt_properties filter_properties = MLT_FILTER_PROPERTIES( filter );
214 mlt_position position = mlt_filter_get_position( filter, frame );
215 mlt_position length = mlt_filter_get_length2( filter, frame );
216
217 // Determine the pitchscale
218 double pitchscale = 1.0;
219 if ( mlt_properties_exists( filter_properties, "pitchscale" ) )
220 {
221 pitchscale = mlt_properties_anim_get_double( filter_properties, "pitchscale", position, length );
222 }
223 else
224 {
225 double octaveshift = mlt_properties_anim_get_double( filter_properties, "octaveshift", position, length );
226 pitchscale = pow(2, octaveshift);
227 }
228 if ( pitchscale <= 0.0 || /*check for nan:*/pitchscale != pitchscale )
229 {
230 pitchscale = 1.0;
231 }
232
233 // Save the pitchscale on the frame to be used in rbpitch_get_audio
234 mlt_properties unique_properties = mlt_frame_unique_properties( frame, MLT_FILTER_SERVICE(filter) );
235 mlt_properties_set_double( unique_properties, "pitchscale", pitchscale );
236 mlt_properties_set_int( unique_properties, "stretch", mlt_properties_get_int( filter_properties, "stretch" ) );
237
238 mlt_frame_push_audio( frame, (void*)filter );
239 mlt_frame_push_audio( frame, (void*)rbpitch_get_audio );
240
241 return frame;
242 }
243
close_filter(mlt_filter filter)244 static void close_filter( mlt_filter filter )
245 {
246 private_data* pdata = (private_data*)filter->child;
247 if ( pdata )
248 {
249 RubberBandStretcher* s = static_cast<RubberBandStretcher*>(pdata->s);
250 if ( s )
251 {
252 delete s;
253 }
254 free( pdata );
255 filter->child = NULL;
256 }
257 }
258
259 extern "C" {
260
filter_rbpitch_init(mlt_profile profile,mlt_service_type type,const char * id,char * arg)261 mlt_filter filter_rbpitch_init( mlt_profile profile, mlt_service_type type, const char *id, char *arg )
262 {
263 mlt_filter filter = mlt_filter_new();
264 private_data* pdata = (private_data*)calloc( 1, sizeof(private_data) );
265
266 if( filter && pdata )
267 {
268 pdata->s = NULL;
269 pdata->rubberband_frequency = 0;
270 pdata->in_samples = 0;
271 pdata->out_samples = 0;
272
273 filter->process = filter_process;
274 filter->close = close_filter;
275 filter->child = pdata;
276 }
277 else
278 {
279 mlt_log_error( MLT_FILTER_SERVICE(filter), "Failed to initialize\n" );
280
281 if( filter )
282 {
283 mlt_filter_close( filter );
284 }
285
286 if( pdata )
287 {
288 free( pdata );
289 }
290
291 filter = NULL;
292 }
293 return filter;
294 }
295
296 }
297