1 /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
2
3 /*
4 Rubber Band Library
5 An audio time-stretching and pitch-shifting library.
6 Copyright 2007-2021 Particular Programs Ltd.
7
8 This program is free software; you can redistribute it and/or
9 modify it under the terms of the GNU General Public License as
10 published by the Free Software Foundation; either version 2 of the
11 License, or (at your option) any later version. See the file
12 COPYING included with this distribution for more information.
13
14 Alternatively, if you have a valid commercial licence for the
15 Rubber Band Library obtained by agreement with the copyright
16 holders, you may redistribute and/or modify it under the terms
17 described in that licence.
18
19 If you wish to distribute code using the Rubber Band Library
20 under terms other than those of the GNU General Public License,
21 you must obtain a valid commercial licence before doing so.
22 */
23
24 #include "RubberBandVampPlugin.h"
25
26 #include "StretchCalculator.h"
27 #include "system/sysutils.h"
28
29 #include <cmath>
30 #include <cstdio>
31
32 using std::string;
33 using std::vector;
34 using std::cerr;
35 using std::endl;
36
37 class RubberBandVampPlugin::Impl
38 {
39 public:
40 size_t m_stepSize;
41 size_t m_blockSize;
42 size_t m_sampleRate;
43
44 float m_timeRatio;
45 float m_pitchRatio;
46
47 bool m_realtime;
48 bool m_elasticTiming;
49 int m_transientMode;
50 bool m_phaseIndependent;
51 int m_windowLength;
52
53 RubberBand::RubberBandStretcher *m_stretcher;
54
55 int m_incrementsOutput;
56 int m_aggregateIncrementsOutput;
57 int m_divergenceOutput;
58 int m_phaseResetDfOutput;
59 int m_smoothedPhaseResetDfOutput;
60 int m_phaseResetPointsOutput;
61 int m_timeSyncPointsOutput;
62
63 size_t m_counter;
64 size_t m_accumulatedIncrement;
65
66 float **m_outputDump;
67
68 FeatureSet processOffline(const float *const *inputBuffers,
69 Vamp::RealTime timestamp);
70
71 FeatureSet getRemainingFeaturesOffline();
72
73 FeatureSet processRealTime(const float *const *inputBuffers,
74 Vamp::RealTime timestamp);
75
76 FeatureSet getRemainingFeaturesRealTime();
77
78 FeatureSet createFeatures(size_t inputIncrement,
79 std::vector<int> &outputIncrements,
80 std::vector<float> &phaseResetDf,
81 std::vector<int> &exactPoints,
82 std::vector<float> &smoothedDf,
83 size_t baseCount,
84 bool includeFinal);
85 };
86
87
RubberBandVampPlugin(float inputSampleRate)88 RubberBandVampPlugin::RubberBandVampPlugin(float inputSampleRate) :
89 Plugin(inputSampleRate)
90 {
91 m_d = new Impl();
92 m_d->m_stepSize = 0;
93 m_d->m_timeRatio = 1.f;
94 m_d->m_pitchRatio = 1.f;
95 m_d->m_realtime = false;
96 m_d->m_elasticTiming = true;
97 m_d->m_transientMode = 0;
98 m_d->m_phaseIndependent = false;
99 m_d->m_windowLength = 0;
100 m_d->m_stretcher = 0;
101 m_d->m_sampleRate = lrintf(m_inputSampleRate);
102 }
103
~RubberBandVampPlugin()104 RubberBandVampPlugin::~RubberBandVampPlugin()
105 {
106 if (m_d->m_outputDump) {
107 for (size_t i = 0; i < m_d->m_stretcher->getChannelCount(); ++i) {
108 delete[] m_d->m_outputDump[i];
109 }
110 delete[] m_d->m_outputDump;
111 }
112 delete m_d->m_stretcher;
113 delete m_d;
114 }
115
116 string
getIdentifier() const117 RubberBandVampPlugin::getIdentifier() const
118 {
119 return "rubberband";
120 }
121
122 string
getName() const123 RubberBandVampPlugin::getName() const
124 {
125 return "Rubber Band Timestretch Analysis";
126 }
127
128 string
getDescription() const129 RubberBandVampPlugin::getDescription() const
130 {
131 return "Carry out analysis phases of time stretcher process";
132 }
133
134 string
getMaker() const135 RubberBandVampPlugin::getMaker() const
136 {
137 return "Breakfast Quay";
138 }
139
140 int
getPluginVersion() const141 RubberBandVampPlugin::getPluginVersion() const
142 {
143 return 1;
144 }
145
146 string
getCopyright() const147 RubberBandVampPlugin::getCopyright() const
148 {
149 return "";//!!!
150 }
151
152 RubberBandVampPlugin::OutputList
getOutputDescriptors() const153 RubberBandVampPlugin::getOutputDescriptors() const
154 {
155 OutputList list;
156
157 size_t rate = 0;
158 if (m_d->m_stretcher) {
159 rate = lrintf(m_inputSampleRate / m_d->m_stretcher->getInputIncrement());
160 }
161
162 OutputDescriptor d;
163 d.identifier = "increments";
164 d.name = "Output Increments";
165 d.description = "Output time increment for each input step";
166 d.unit = "samples";
167 d.hasFixedBinCount = true;
168 d.binCount = 1;
169 d.hasKnownExtents = false;
170 d.isQuantized = true;
171 d.quantizeStep = 1.0;
172 d.sampleType = OutputDescriptor::VariableSampleRate;
173 d.sampleRate = float(rate);
174 m_d->m_incrementsOutput = list.size();
175 list.push_back(d);
176
177 d.identifier = "aggregate_increments";
178 d.name = "Accumulated Output Increments";
179 d.description = "Accumulated output time increments";
180 d.sampleRate = 0;
181 m_d->m_aggregateIncrementsOutput = list.size();
182 list.push_back(d);
183
184 d.identifier = "divergence";
185 d.name = "Divergence from Linear";
186 d.description = "Difference between actual output time and the output time for a theoretical linear stretch";
187 d.isQuantized = false;
188 d.sampleRate = 0;
189 m_d->m_divergenceOutput = list.size();
190 list.push_back(d);
191
192 d.identifier = "phaseresetdf";
193 d.name = "Phase Reset Detection Function";
194 d.description = "Curve whose peaks are used to identify transients for phase reset points";
195 d.unit = "";
196 d.sampleRate = float(rate);
197 m_d->m_phaseResetDfOutput = list.size();
198 list.push_back(d);
199
200 d.identifier = "smoothedphaseresetdf";
201 d.name = "Smoothed Phase Reset Detection Function";
202 d.description = "Phase reset curve smoothed for peak picking";
203 d.unit = "";
204 m_d->m_smoothedPhaseResetDfOutput = list.size();
205 list.push_back(d);
206
207 d.identifier = "phaseresetpoints";
208 d.name = "Phase Reset Points";
209 d.description = "Points estimated as transients at which phase reset occurs";
210 d.unit = "";
211 d.hasFixedBinCount = true;
212 d.binCount = 0;
213 d.hasKnownExtents = false;
214 d.isQuantized = false;
215 d.sampleRate = 0;
216 m_d->m_phaseResetPointsOutput = list.size();
217 list.push_back(d);
218
219 d.identifier = "timesyncpoints";
220 d.name = "Time Sync Points";
221 d.description = "Salient points which stretcher aims to place with strictly correct timing";
222 d.unit = "";
223 d.hasFixedBinCount = true;
224 d.binCount = 0;
225 d.hasKnownExtents = false;
226 d.isQuantized = false;
227 d.sampleRate = 0;
228 m_d->m_timeSyncPointsOutput = list.size();
229 list.push_back(d);
230
231 return list;
232 }
233
234 RubberBandVampPlugin::ParameterList
getParameterDescriptors() const235 RubberBandVampPlugin::getParameterDescriptors() const
236 {
237 ParameterList list;
238
239 ParameterDescriptor d;
240 d.identifier = "timeratio";
241 d.name = "Time Ratio";
242 d.description = "Ratio to modify overall duration by";
243 d.unit = "%";
244 d.minValue = 1;
245 d.maxValue = 500;
246 d.defaultValue = 100;
247 d.isQuantized = false;
248 list.push_back(d);
249
250 d.identifier = "pitchratio";
251 d.name = "Pitch Scale Ratio";
252 d.description = "Frequency ratio to modify pitch by";
253 d.unit = "%";
254 d.minValue = 1;
255 d.maxValue = 500;
256 d.defaultValue = 100;
257 d.isQuantized = false;
258 list.push_back(d);
259
260 d.identifier = "mode";
261 d.name = "Processing Mode";
262 d.description = ""; //!!!
263 d.unit = "";
264 d.minValue = 0;
265 d.maxValue = 1;
266 d.defaultValue = 0;
267 d.isQuantized = true;
268 d.quantizeStep = 1;
269 d.valueNames.clear();
270 d.valueNames.push_back("Offline");
271 d.valueNames.push_back("Real Time");
272 list.push_back(d);
273
274 d.identifier = "stretchtype";
275 d.name = "Stretch Flexibility";
276 d.description = ""; //!!!
277 d.unit = "";
278 d.minValue = 0;
279 d.maxValue = 1;
280 d.defaultValue = 0;
281 d.isQuantized = true;
282 d.quantizeStep = 1;
283 d.valueNames.clear();
284 d.valueNames.push_back("Elastic");
285 d.valueNames.push_back("Precise");
286 list.push_back(d);
287
288 d.identifier = "transientmode";
289 d.name = "Transient Handling";
290 d.description = ""; //!!!
291 d.unit = "";
292 d.minValue = 0;
293 d.maxValue = 2;
294 d.defaultValue = 0;
295 d.isQuantized = true;
296 d.quantizeStep = 1;
297 d.valueNames.clear();
298 d.valueNames.push_back("Mixed");
299 d.valueNames.push_back("Smooth");
300 d.valueNames.push_back("Crisp");
301 list.push_back(d);
302
303 d.identifier = "phasemode";
304 d.name = "Phase Handling";
305 d.description = ""; //!!!
306 d.unit = "";
307 d.minValue = 0;
308 d.maxValue = 1;
309 d.defaultValue = 0;
310 d.isQuantized = true;
311 d.quantizeStep = 1;
312 d.valueNames.clear();
313 d.valueNames.push_back("Laminar");
314 d.valueNames.push_back("Independent");
315 list.push_back(d);
316
317 d.identifier = "windowmode";
318 d.name = "Window Length";
319 d.description = ""; //!!!
320 d.unit = "";
321 d.minValue = 0;
322 d.maxValue = 2;
323 d.defaultValue = 0;
324 d.isQuantized = true;
325 d.quantizeStep = 1;
326 d.valueNames.clear();
327 d.valueNames.push_back("Standard");
328 d.valueNames.push_back("Short");
329 d.valueNames.push_back("Long");
330 list.push_back(d);
331
332 return list;
333 }
334
335 float
getParameter(std::string id) const336 RubberBandVampPlugin::getParameter(std::string id) const
337 {
338 if (id == "timeratio") return m_d->m_timeRatio * 100.f;
339 if (id == "pitchratio") return m_d->m_pitchRatio * 100.f;
340 if (id == "mode") return m_d->m_realtime ? 1.f : 0.f;
341 if (id == "stretchtype") return m_d->m_elasticTiming ? 0.f : 1.f;
342 if (id == "transientmode") return float(m_d->m_transientMode);
343 if (id == "phasemode") return m_d->m_phaseIndependent ? 1.f : 0.f;
344 if (id == "windowmode") return float(m_d->m_windowLength);
345 return 0.f;
346 }
347
348 void
setParameter(std::string id,float value)349 RubberBandVampPlugin::setParameter(std::string id, float value)
350 {
351 if (id == "timeratio") {
352 m_d->m_timeRatio = value / 100;
353 } else if (id == "pitchratio") {
354 m_d->m_pitchRatio = value / 100;
355 } else {
356 bool set = (value > 0.5);
357 if (id == "mode") m_d->m_realtime = set;
358 else if (id == "stretchtype") m_d->m_elasticTiming = !set;
359 else if (id == "transientmode") m_d->m_transientMode = int(value + 0.5);
360 else if (id == "phasemode") m_d->m_phaseIndependent = set;
361 else if (id == "windowmode") m_d->m_windowLength = int(value + 0.5);
362 }
363 }
364
365 bool
initialise(size_t channels,size_t stepSize,size_t blockSize)366 RubberBandVampPlugin::initialise(size_t channels, size_t stepSize, size_t blockSize)
367 {
368 if (channels < getMinChannelCount() ||
369 channels > getMaxChannelCount()) return false;
370
371 m_d->m_stepSize = std::min(stepSize, blockSize);
372 m_d->m_blockSize = stepSize;
373
374 RubberBand::RubberBandStretcher::Options options = 0;
375
376 if (m_d->m_realtime)
377 options |= RubberBand::RubberBandStretcher::OptionProcessRealTime;
378 else options |= RubberBand::RubberBandStretcher::OptionProcessOffline;
379
380 if (m_d->m_elasticTiming)
381 options |= RubberBand::RubberBandStretcher::OptionStretchElastic;
382 else options |= RubberBand::RubberBandStretcher::OptionStretchPrecise;
383
384 if (m_d->m_transientMode == 0)
385 options |= RubberBand::RubberBandStretcher::OptionTransientsMixed;
386 else if (m_d->m_transientMode == 1)
387 options |= RubberBand::RubberBandStretcher::OptionTransientsSmooth;
388 else options |= RubberBand::RubberBandStretcher::OptionTransientsCrisp;
389
390 if (m_d->m_phaseIndependent)
391 options |= RubberBand::RubberBandStretcher::OptionPhaseIndependent;
392 else options |= RubberBand::RubberBandStretcher::OptionPhaseLaminar;
393
394 if (m_d->m_windowLength == 0)
395 options |= RubberBand::RubberBandStretcher::OptionWindowStandard;
396 else if (m_d->m_windowLength == 1)
397 options |= RubberBand::RubberBandStretcher::OptionWindowShort;
398 else options |= RubberBand::RubberBandStretcher::OptionWindowLong;
399
400 delete m_d->m_stretcher;
401 m_d->m_stretcher = new RubberBand::RubberBandStretcher
402 (m_d->m_sampleRate, channels, options);
403 m_d->m_stretcher->setDebugLevel(1);
404 m_d->m_stretcher->setTimeRatio(m_d->m_timeRatio);
405 m_d->m_stretcher->setPitchScale(m_d->m_pitchRatio);
406
407 m_d->m_counter = 0;
408 m_d->m_accumulatedIncrement = 0;
409
410 m_d->m_outputDump = 0;
411
412 return true;
413 }
414
415 void
reset()416 RubberBandVampPlugin::reset()
417 {
418 if (m_d->m_stretcher) m_d->m_stretcher->reset();
419 }
420
421 RubberBandVampPlugin::FeatureSet
process(const float * const * inputBuffers,Vamp::RealTime timestamp)422 RubberBandVampPlugin::process(const float *const *inputBuffers,
423 Vamp::RealTime timestamp)
424 {
425 if (m_d->m_realtime) {
426 return m_d->processRealTime(inputBuffers, timestamp);
427 } else {
428 return m_d->processOffline(inputBuffers, timestamp);
429 }
430 }
431
432 RubberBandVampPlugin::FeatureSet
getRemainingFeatures()433 RubberBandVampPlugin::getRemainingFeatures()
434 {
435 if (m_d->m_realtime) {
436 return m_d->getRemainingFeaturesRealTime();
437 } else {
438 return m_d->getRemainingFeaturesOffline();
439 }
440 }
441
442 RubberBandVampPlugin::FeatureSet
processOffline(const float * const * inputBuffers,Vamp::RealTime)443 RubberBandVampPlugin::Impl::processOffline(const float *const *inputBuffers,
444 Vamp::RealTime /* timestamp */)
445 {
446 if (!m_stretcher) {
447 cerr << "ERROR: RubberBandVampPlugin::processOffline: "
448 << "RubberBandVampPlugin has not been initialised"
449 << endl;
450 return FeatureSet();
451 }
452
453 m_stretcher->study(inputBuffers, m_blockSize, false);
454 return FeatureSet();
455 }
456
457 RubberBandVampPlugin::FeatureSet
getRemainingFeaturesOffline()458 RubberBandVampPlugin::Impl::getRemainingFeaturesOffline()
459 {
460 m_stretcher->study(0, 0, true);
461
462 m_stretcher->calculateStretch();
463
464 int rate = m_sampleRate;
465
466 RubberBand::StretchCalculator sc(rate, m_stretcher->getInputIncrement(), true);
467
468 size_t inputIncrement = m_stretcher->getInputIncrement();
469 std::vector<int> outputIncrements = m_stretcher->getOutputIncrements();
470 std::vector<float> phaseResetDf = m_stretcher->getPhaseResetCurve();
471 std::vector<int> peaks = m_stretcher->getExactTimePoints();
472 std::vector<float> smoothedDf = sc.smoothDF(phaseResetDf);
473
474 FeatureSet features = createFeatures
475 (inputIncrement, outputIncrements, phaseResetDf, peaks, smoothedDf,
476 0, true);
477
478 return features;
479 }
480
481 RubberBandVampPlugin::FeatureSet
processRealTime(const float * const * inputBuffers,Vamp::RealTime)482 RubberBandVampPlugin::Impl::processRealTime(const float *const *inputBuffers,
483 Vamp::RealTime /* timestamp */)
484 {
485 // This function is not in any way a real-time function (i.e. it
486 // has no requirement to be RT safe); it simply operates the
487 // stretcher in RT mode.
488
489 if (!m_stretcher) {
490 cerr << "ERROR: RubberBandVampPlugin::processRealTime: "
491 << "RubberBandVampPlugin has not been initialised"
492 << endl;
493 return FeatureSet();
494 }
495
496 m_stretcher->process(inputBuffers, m_blockSize, false);
497
498 size_t inputIncrement = m_stretcher->getInputIncrement();
499 std::vector<int> outputIncrements = m_stretcher->getOutputIncrements();
500 std::vector<float> phaseResetDf = m_stretcher->getPhaseResetCurve();
501 std::vector<float> smoothedDf; // not meaningful in RT mode
502 std::vector<int> dummyPoints;
503 FeatureSet features = createFeatures
504 (inputIncrement, outputIncrements, phaseResetDf, dummyPoints, smoothedDf,
505 m_counter, false);
506 m_counter += outputIncrements.size();
507
508 int available = 0;
509 while ((available = m_stretcher->available()) > 0) {
510 if (!m_outputDump) {
511 m_outputDump = new float *[m_stretcher->getChannelCount()];
512 for (size_t i = 0; i < m_stretcher->getChannelCount(); ++i) {
513 m_outputDump[i] = new float[m_blockSize];
514 }
515 }
516 m_stretcher->retrieve(m_outputDump,
517 std::min(int(m_blockSize), available));
518 }
519
520 return features;
521 }
522
523 RubberBandVampPlugin::FeatureSet
getRemainingFeaturesRealTime()524 RubberBandVampPlugin::Impl::getRemainingFeaturesRealTime()
525 {
526 return FeatureSet();
527 }
528
529 RubberBandVampPlugin::FeatureSet
createFeatures(size_t inputIncrement,std::vector<int> & outputIncrements,std::vector<float> & phaseResetDf,std::vector<int> & exactPoints,std::vector<float> & smoothedDf,size_t baseCount,bool includeFinal)530 RubberBandVampPlugin::Impl::createFeatures(size_t inputIncrement,
531 std::vector<int> &outputIncrements,
532 std::vector<float> &phaseResetDf,
533 std::vector<int> &exactPoints,
534 std::vector<float> &smoothedDf,
535 size_t baseCount,
536 bool includeFinal)
537 {
538 size_t actual = m_accumulatedIncrement;
539
540 double overallRatio = m_timeRatio * m_pitchRatio;
541
542 char label[200];
543
544 FeatureSet features;
545
546 int rate = m_sampleRate;
547
548 size_t epi = 0;
549
550 for (size_t i = 0; i < outputIncrements.size(); ++i) {
551
552 size_t frame = (baseCount + i) * inputIncrement;
553
554 int oi = outputIncrements[i];
555 bool hard = false;
556 bool soft = false;
557
558 if (oi < 0) {
559 oi = -oi;
560 hard = true;
561 }
562
563 if (epi < exactPoints.size() && int(i) == exactPoints[epi]) {
564 soft = true;
565 ++epi;
566 }
567
568 double linear = (frame * overallRatio);
569
570 Vamp::RealTime t = Vamp::RealTime::frame2RealTime(frame, rate);
571
572 Feature feature;
573 feature.hasTimestamp = true;
574 feature.timestamp = t;
575 feature.values.push_back(float(oi));
576 feature.label = Vamp::RealTime::frame2RealTime(oi, rate).toText();
577 features[m_incrementsOutput].push_back(feature);
578
579 feature.values.clear();
580 feature.values.push_back(float(actual));
581 feature.label = Vamp::RealTime::frame2RealTime(actual, rate).toText();
582 features[m_aggregateIncrementsOutput].push_back(feature);
583
584 feature.values.clear();
585 feature.values.push_back(actual - linear);
586
587 sprintf(label, "expected %ld, actual %ld, difference %ld (%s ms)",
588 long(linear), long(actual), long(actual - linear),
589 // frame2RealTime expects an integer frame number,
590 // hence our multiplication factor
591 (Vamp::RealTime::frame2RealTime
592 (lrintf((actual - linear) * 1000), rate) / 1000)
593 .toText().c_str());
594 feature.label = label;
595
596 features[m_divergenceOutput].push_back(feature);
597 actual += oi;
598
599 char buf[30];
600
601 if (i < phaseResetDf.size()) {
602 feature.values.clear();
603 feature.values.push_back(phaseResetDf[i]);
604 sprintf(buf, "%d", int(baseCount + i));
605 feature.label = buf;
606 features[m_phaseResetDfOutput].push_back(feature);
607 }
608
609 if (i < smoothedDf.size()) {
610 feature.values.clear();
611 feature.values.push_back(smoothedDf[i]);
612 features[m_smoothedPhaseResetDfOutput].push_back(feature);
613 }
614
615 if (hard) {
616 feature.values.clear();
617 feature.label = "Phase Reset";
618 features[m_phaseResetPointsOutput].push_back(feature);
619 }
620
621 if (hard || soft) {
622 feature.values.clear();
623 feature.label = "Time Sync";
624 features[m_timeSyncPointsOutput].push_back(feature);
625 }
626 }
627
628 if (includeFinal) {
629 Vamp::RealTime t = Vamp::RealTime::frame2RealTime
630 (inputIncrement * (baseCount + outputIncrements.size()), rate);
631 Feature feature;
632 feature.hasTimestamp = true;
633 feature.timestamp = t;
634 feature.label = Vamp::RealTime::frame2RealTime(actual, rate).toText();
635 feature.values.clear();
636 feature.values.push_back(float(actual));
637 features[m_aggregateIncrementsOutput].push_back(feature);
638
639 float linear = ((baseCount + outputIncrements.size())
640 * inputIncrement * overallRatio);
641 feature.values.clear();
642 feature.values.push_back(actual - linear);
643 feature.label = // see earlier comment
644 (Vamp::RealTime::frame2RealTime //!!! update this as earlier label
645 (lrintf((actual - linear) * 1000), rate) / 1000)
646 .toText();
647 features[m_divergenceOutput].push_back(feature);
648 }
649
650 m_accumulatedIncrement = actual;
651
652 return features;
653 }
654
655