1 /**********************************************************************
2
3 Audacity: A Digital Audio Editor
4
5 SoundTouchEffect.cpp
6
7 Dominic Mazzoni, Vaughan Johnson
8
9 This abstract class contains all of the common code for an
10 effect that uses SoundTouch to do its processing (ChangeTempo
11 and ChangePitch).
12
13 **********************************************************************/
14
15
16
17 #if USE_SOUNDTOUCH
18 #include "SoundTouchEffect.h"
19
20 #include <math.h>
21
22 #include "../LabelTrack.h"
23 #include "../WaveClip.h"
24 #include "../WaveTrack.h"
25 #include "../NoteTrack.h"
26 #include "TimeWarper.h"
27
28 // Soundtouch defines these as well, which are also in generated configmac.h
29 // and configunix.h, so get rid of them before including,
30 // to avoid compiler warnings, and be sure to do this
31 // after all other #includes, to avoid any mischief that might result
32 // from doing the un-definitions before seeing any wx headers.
33 #undef PACKAGE_NAME
34 #undef PACKAGE_STRING
35 #undef PACKAGE_TARNAME
36 #undef PACKAGE_VERSION
37 #undef PACKAGE_BUGREPORT
38 #undef PACKAGE
39 #undef VERSION
40 #include "SoundTouch.h"
41
42 #ifdef USE_MIDI
EffectSoundTouch()43 EffectSoundTouch::EffectSoundTouch()
44 {
45 mSemitones = 0;
46 }
47 #endif
48
~EffectSoundTouch()49 EffectSoundTouch::~EffectSoundTouch()
50 {
51 }
52
ProcessLabelTrack(LabelTrack * lt,const TimeWarper & warper)53 bool EffectSoundTouch::ProcessLabelTrack(
54 LabelTrack *lt, const TimeWarper &warper)
55 {
56 // SetTimeWarper(std::make_unique<RegionTimeWarper>(mCurT0, mCurT1,
57 // std::make_unique<LinearTimeWarper>(mCurT0, mCurT0,
58 // mCurT1, mCurT0 + (mCurT1-mCurT0)*mFactor)));
59 lt->WarpLabels(warper);
60 return true;
61 }
62
63 #ifdef USE_MIDI
ProcessNoteTrack(NoteTrack * nt,const TimeWarper & warper)64 bool EffectSoundTouch::ProcessNoteTrack(NoteTrack *nt, const TimeWarper &warper)
65 {
66 nt->WarpAndTransposeNotes(mCurT0, mCurT1, warper, mSemitones);
67 return true;
68 }
69 #endif
70
ProcessWithTimeWarper(InitFunction initer,const TimeWarper & warper,bool preserveLength)71 bool EffectSoundTouch::ProcessWithTimeWarper(InitFunction initer,
72 const TimeWarper &warper,
73 bool preserveLength)
74 {
75 // Assumes that mSoundTouch has already been initialized
76 // by the subclass for subclass-specific parameters. The
77 // time warper should also be set.
78
79 // Check if this effect will alter the selection length; if so, we need
80 // to operate on sync-lock selected tracks.
81 bool mustSync = true;
82 if (mT1 == warper.Warp(mT1)) {
83 mustSync = false;
84 }
85
86 //Iterate over each track
87 // Needs all for sync-lock grouping.
88 this->CopyInputTracks(true);
89 bool bGoodResult = true;
90
91 mPreserveLength = preserveLength;
92 mCurTrackNum = 0;
93 m_maxNewLength = 0.0;
94
95 mOutputTracks->Leaders().VisitWhile( bGoodResult,
96 [&]( LabelTrack *lt, const Track::Fallthrough &fallthrough ) {
97 if ( !(lt->GetSelected() || (mustSync && lt->IsSyncLockSelected())) )
98 return fallthrough();
99 if (!ProcessLabelTrack(lt, warper))
100 bGoodResult = false;
101 },
102 #ifdef USE_MIDI
103 [&]( NoteTrack *nt, const Track::Fallthrough &fallthrough ) {
104 if ( !(nt->GetSelected() || (mustSync && nt->IsSyncLockSelected())) )
105 return fallthrough();
106 if (!ProcessNoteTrack(nt, warper))
107 bGoodResult = false;
108 },
109 #endif
110 [&]( WaveTrack *leftTrack, const Track::Fallthrough &fallthrough ) {
111 if (!leftTrack->GetSelected())
112 return fallthrough();
113
114 //Get start and end times from selection
115 mCurT0 = mT0;
116 mCurT1 = mT1;
117
118 //Set the current bounds to whichever left marker is
119 //greater and whichever right marker is less
120 mCurT0 = wxMax(mT0, mCurT0);
121 mCurT1 = wxMin(mT1, mCurT1);
122
123 // Process only if the right marker is to the right of the left marker
124 if (mCurT1 > mCurT0) {
125 mSoundTouch = std::make_unique<soundtouch::SoundTouch>();
126 initer(mSoundTouch.get());
127
128 // TODO: more-than-two-channels
129 auto channels = TrackList::Channels(leftTrack);
130 auto rightTrack = (channels.size() > 1)
131 ? * ++ channels.first
132 : nullptr;
133 if ( rightTrack ) {
134 double t;
135
136 //Adjust bounds by the right tracks markers
137 t = rightTrack->GetStartTime();
138 t = wxMax(mT0, t);
139 mCurT0 = wxMin(mCurT0, t);
140 t = rightTrack->GetEndTime();
141 t = wxMin(mT1, t);
142 mCurT1 = wxMax(mCurT1, t);
143
144 //Transform the marker timepoints to samples
145 auto start = leftTrack->TimeToLongSamples(mCurT0);
146 auto end = leftTrack->TimeToLongSamples(mCurT1);
147
148 //Inform soundtouch there's 2 channels
149 mSoundTouch->setChannels(2);
150
151 //ProcessStereo() (implemented below) processes a stereo track
152 if (!ProcessStereo(leftTrack, rightTrack, start, end, warper))
153 bGoodResult = false;
154 mCurTrackNum++; // Increment for rightTrack, too.
155 } else {
156 //Transform the marker timepoints to samples
157 auto start = leftTrack->TimeToLongSamples(mCurT0);
158 auto end = leftTrack->TimeToLongSamples(mCurT1);
159
160 //Inform soundtouch there's a single channel
161 mSoundTouch->setChannels(1);
162
163 //ProcessOne() (implemented below) processes a single track
164 if (!ProcessOne(leftTrack, start, end, warper))
165 bGoodResult = false;
166 }
167
168 mSoundTouch.reset();
169 }
170 mCurTrackNum++;
171 },
172 [&]( Track *t ) {
173 if (mustSync && t->IsSyncLockSelected()) {
174 t->SyncLockAdjust(mT1, warper.Warp(mT1));
175 }
176 }
177 );
178
179 if (bGoodResult) {
180 ReplaceProcessedTracks(bGoodResult);
181 }
182
183 return bGoodResult;
184 }
185
End()186 void EffectSoundTouch::End()
187 {
188 mSoundTouch.reset();
189 }
190
191 //ProcessOne() takes a track, transforms it to bunch of buffer-blocks,
192 //and executes ProcessSoundTouch on these blocks
ProcessOne(WaveTrack * track,sampleCount start,sampleCount end,const TimeWarper & warper)193 bool EffectSoundTouch::ProcessOne(WaveTrack *track,
194 sampleCount start, sampleCount end,
195 const TimeWarper &warper)
196 {
197 mSoundTouch->setSampleRate((unsigned int)(track->GetRate()+0.5));
198
199 auto outputTrack = track->EmptyCopy();
200
201 //Get the length of the buffer (as double). len is
202 //used simple to calculate a progress meter, so it is easier
203 //to make it a double now than it is to do it later
204 auto len = (end - start).as_double();
205
206 {
207 //Initiate a processing buffer. This buffer will (most likely)
208 //be shorter than the length of the track being processed.
209 Floats buffer{ track->GetMaxBlockSize() };
210
211 //Go through the track one buffer at a time. s counts which
212 //sample the current buffer starts at.
213 auto s = start;
214 while (s < end) {
215 //Get a block of samples (smaller than the size of the buffer)
216 const auto block = wxMin(8192,
217 limitSampleBufferSize( track->GetBestBlockSize(s), end - s ));
218
219 //Get the samples from the track and put them in the buffer
220 track->GetFloats(buffer.get(), s, block);
221
222 //Add samples to SoundTouch
223 mSoundTouch->putSamples(buffer.get(), block);
224
225 //Get back samples from SoundTouch
226 unsigned int outputCount = mSoundTouch->numSamples();
227 if (outputCount > 0) {
228 Floats buffer2{ outputCount };
229 mSoundTouch->receiveSamples(buffer2.get(), outputCount);
230 outputTrack->Append((samplePtr)buffer2.get(), floatSample, outputCount);
231 }
232
233 //Increment s one blockfull of samples
234 s += block;
235
236 //Update the Progress meter
237 if (TrackProgress(mCurTrackNum, (s - start).as_double() / len))
238 return false;
239 }
240
241 // Tell SoundTouch to finish processing any remaining samples
242 mSoundTouch->flush(); // this should only be used for changeTempo - it dumps data otherwise with pRateTransposer->clear();
243
244 unsigned int outputCount = mSoundTouch->numSamples();
245 if (outputCount > 0) {
246 Floats buffer2{ outputCount };
247 mSoundTouch->receiveSamples(buffer2.get(), outputCount);
248 outputTrack->Append((samplePtr)buffer2.get(), floatSample, outputCount);
249 }
250
251 // Flush the output WaveTrack (since it's buffered, too)
252 outputTrack->Flush();
253 }
254
255 // Transfer output samples to the original
256 Finalize(track, outputTrack.get(), warper);
257
258 double newLength = outputTrack->GetEndTime();
259 m_maxNewLength = wxMax(m_maxNewLength, newLength);
260
261 //Return true because the effect processing succeeded.
262 return true;
263 }
264
ProcessStereo(WaveTrack * leftTrack,WaveTrack * rightTrack,sampleCount start,sampleCount end,const TimeWarper & warper)265 bool EffectSoundTouch::ProcessStereo(
266 WaveTrack* leftTrack, WaveTrack* rightTrack,
267 sampleCount start, sampleCount end, const TimeWarper &warper)
268 {
269 mSoundTouch->setSampleRate((unsigned int)(leftTrack->GetRate() + 0.5));
270
271 auto outputLeftTrack = leftTrack->EmptyCopy();
272 auto outputRightTrack = rightTrack->EmptyCopy();
273
274 //Get the length of the buffer (as double). len is
275 //used simple to calculate a progress meter, so it is easier
276 //to make it a double now than it is to do it later
277 double len = (end - start).as_double();
278
279 //Initiate a processing buffer. This buffer will (most likely)
280 //be shorter than the length of the track being processed.
281 // Make soundTouchBuffer twice as big as MaxBlockSize for each channel,
282 // because Soundtouch wants them interleaved, i.e., each
283 // Soundtouch sample is left-right pair.
284 auto maxBlockSize = leftTrack->GetMaxBlockSize();
285 {
286 Floats leftBuffer{ maxBlockSize };
287 Floats rightBuffer{ maxBlockSize };
288 Floats soundTouchBuffer{ maxBlockSize * 2 };
289
290 // Go through the track one stereo buffer at a time.
291 // sourceSampleCount counts the sample at which the current buffer starts,
292 // per channel.
293 auto sourceSampleCount = start;
294 while (sourceSampleCount < end) {
295 auto blockSize = limitSampleBufferSize(
296 leftTrack->GetBestBlockSize(sourceSampleCount),
297 end - sourceSampleCount
298 );
299
300 // Get the samples from the tracks and put them in the buffers.
301 leftTrack->GetFloats((leftBuffer.get()), sourceSampleCount, blockSize);
302 rightTrack->GetFloats((rightBuffer.get()), sourceSampleCount, blockSize);
303
304 // Interleave into soundTouchBuffer.
305 for (decltype(blockSize) index = 0; index < blockSize; index++) {
306 soundTouchBuffer[index * 2] = leftBuffer[index];
307 soundTouchBuffer[(index * 2) + 1] = rightBuffer[index];
308 }
309
310 //Add samples to SoundTouch
311 mSoundTouch->putSamples(soundTouchBuffer.get(), blockSize);
312
313 //Get back samples from SoundTouch
314 unsigned int outputCount = mSoundTouch->numSamples();
315 if (outputCount > 0)
316 this->ProcessStereoResults(outputCount, outputLeftTrack.get(), outputRightTrack.get());
317
318 //Increment sourceSampleCount one blockfull of samples
319 sourceSampleCount += blockSize;
320
321 //Update the Progress meter
322 // mCurTrackNum is left track. Include right track.
323 int nWhichTrack = mCurTrackNum;
324 double frac = (sourceSampleCount - start).as_double() / len;
325 if (frac < 0.5)
326 frac *= 2.0; // Show twice as far for each track, because we're doing 2 at once.
327 else
328 {
329 nWhichTrack++;
330 frac -= 0.5;
331 frac *= 2.0; // Show twice as far for each track, because we're doing 2 at once.
332 }
333 if (TrackProgress(nWhichTrack, frac))
334 return false;
335 }
336
337 // Tell SoundTouch to finish processing any remaining samples
338 mSoundTouch->flush();
339
340 unsigned int outputCount = mSoundTouch->numSamples();
341 if (outputCount > 0)
342 this->ProcessStereoResults(outputCount, outputLeftTrack.get(), outputRightTrack.get());
343
344 // Flush the output WaveTracks (since they're buffered, too)
345 outputLeftTrack->Flush();
346 outputRightTrack->Flush();
347 }
348
349 // Transfer output samples to the original
350 Finalize(leftTrack, outputLeftTrack.get(), warper);
351 Finalize(rightTrack, outputRightTrack.get(), warper);
352
353
354 // Track the longest result length
355 double newLength = outputLeftTrack->GetEndTime();
356 m_maxNewLength = wxMax(m_maxNewLength, newLength);
357 newLength = outputRightTrack->GetEndTime();
358 m_maxNewLength = wxMax(m_maxNewLength, newLength);
359
360 //Return true because the effect processing succeeded.
361 return true;
362 }
363
ProcessStereoResults(const size_t outputCount,WaveTrack * outputLeftTrack,WaveTrack * outputRightTrack)364 bool EffectSoundTouch::ProcessStereoResults(const size_t outputCount,
365 WaveTrack* outputLeftTrack,
366 WaveTrack* outputRightTrack)
367 {
368 Floats outputSoundTouchBuffer{ outputCount * 2 };
369 mSoundTouch->receiveSamples(outputSoundTouchBuffer.get(), outputCount);
370
371 // Dis-interleave outputSoundTouchBuffer into separate track buffers.
372 Floats outputLeftBuffer{ outputCount };
373 Floats outputRightBuffer{ outputCount };
374 for (unsigned int index = 0; index < outputCount; index++)
375 {
376 outputLeftBuffer[index] = outputSoundTouchBuffer[index*2];
377 outputRightBuffer[index] = outputSoundTouchBuffer[(index*2)+1];
378 }
379
380 outputLeftTrack->Append((samplePtr)outputLeftBuffer.get(), floatSample, outputCount);
381 outputRightTrack->Append((samplePtr)outputRightBuffer.get(), floatSample, outputCount);
382
383 return true;
384 }
385
Finalize(WaveTrack * orig,WaveTrack * out,const TimeWarper & warper)386 void EffectSoundTouch::Finalize(WaveTrack* orig, WaveTrack* out, const TimeWarper &warper)
387 {
388 if (mPreserveLength) {
389 auto newLen = out->GetPlaySamplesCount();
390 auto oldLen = out->TimeToLongSamples(mCurT1) - out->TimeToLongSamples(mCurT0);
391
392 // Pad output track to original length since SoundTouch may remove samples
393 if (newLen < oldLen) {
394 out->InsertSilence(out->LongSamplesToTime(newLen - 1),
395 out->LongSamplesToTime(oldLen - newLen));
396 }
397 // Trim output track to original length since SoundTouch may add extra samples
398 else if (newLen > oldLen) {
399 out->Trim(0, out->LongSamplesToTime(oldLen));
400 }
401 }
402
403 // Silenced samples will be inserted in gaps between clips, so capture where these
404 // gaps are for later deletion
405 std::vector<std::pair<double, double>> gaps;
406 double last = mCurT0;
407 auto clips = orig->SortedClipArray();
408 auto front = clips.front();
409 auto back = clips.back();
410 for (auto &clip : clips) {
411 auto st = clip->GetPlayStartTime();
412 auto et = clip->GetPlayEndTime();
413
414 if (st >= mCurT0 || et < mCurT1) {
415 if (mCurT0 < st && clip == front) {
416 gaps.push_back(std::make_pair(mCurT0, st));
417 }
418 else if (last < st && mCurT0 <= last ) {
419 gaps.push_back(std::make_pair(last, st));
420 }
421
422 if (et < mCurT1 && clip == back) {
423 gaps.push_back(std::make_pair(et, mCurT1));
424 }
425 }
426 last = et;
427 }
428
429 // Take the output track and insert it in place of the original sample data
430 orig->ClearAndPaste(mCurT0, mCurT1, out, true, true, &warper);
431
432 // Finally, recreate the gaps
433 for (auto gap : gaps) {
434 auto st = orig->LongSamplesToTime(orig->TimeToLongSamples(gap.first));
435 auto et = orig->LongSamplesToTime(orig->TimeToLongSamples(gap.second));
436 if (st >= mCurT0 && et <= mCurT1 && st != et)
437 {
438 orig->SplitDelete(warper.Warp(st), warper.Warp(et));
439 }
440 }
441 }
442
443 #endif // USE_SOUNDTOUCH
444