1
/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */
5
An audio time-stretching and pitch-shifting library.
6
Copyright 2007-2008 Chris Cannam.
8
This program is free software; you can redistribute it and/or
9
modify it under the terms of the GNU General Public License as
10
published by the Free Software Foundation; either version 2 of the
11
License, or (at your option) any later version. See the file
12
COPYING included with this distribution for more information.
15
#include "RubberBandVampPlugin.h"
17
#include "StretchCalculator.h"
27
class RubberBandVampPlugin::Impl
40
bool m_phaseIndependent;
43
RubberBand::RubberBandStretcher *m_stretcher;
45
int m_incrementsOutput;
46
int m_aggregateIncrementsOutput;
47
int m_divergenceOutput;
48
int m_phaseResetDfOutput;
49
int m_smoothedPhaseResetDfOutput;
50
int m_phaseResetPointsOutput;
51
int m_timeSyncPointsOutput;
54
size_t m_accumulatedIncrement;
58
FeatureSet processOffline(const float *const *inputBuffers,
59
Vamp::RealTime timestamp);
61
FeatureSet getRemainingFeaturesOffline();
63
FeatureSet processRealTime(const float *const *inputBuffers,
64
Vamp::RealTime timestamp);
66
FeatureSet getRemainingFeaturesRealTime();
68
FeatureSet createFeatures(size_t inputIncrement,
69
std::vector<int> &outputIncrements,
70
std::vector<float> &phaseResetDf,
71
std::vector<int> &exactPoints,
72
std::vector<float> &smoothedDf,
78
RubberBandVampPlugin::RubberBandVampPlugin(float inputSampleRate) :
79
Plugin(inputSampleRate)
83
m_d->m_timeRatio = 1.f;
84
m_d->m_pitchRatio = 1.f;
85
m_d->m_realtime = false;
86
m_d->m_elasticTiming = true;
87
m_d->m_transientMode = 0;
88
m_d->m_phaseIndependent = false;
89
m_d->m_windowLength = 0;
91
m_d->m_sampleRate = lrintf(m_inputSampleRate);
94
RubberBandVampPlugin::~RubberBandVampPlugin()
96
if (m_d->m_outputDump) {
97
for (size_t i = 0; i < m_d->m_stretcher->getChannelCount(); ++i) {
98
delete[] m_d->m_outputDump[i];
100
delete[] m_d->m_outputDump;
102
delete m_d->m_stretcher;
107
RubberBandVampPlugin::getIdentifier() const
113
RubberBandVampPlugin::getName() const
115
return "Rubber Band Timestretch Analysis";
119
RubberBandVampPlugin::getDescription() const
121
return "Carry out analysis phases of time stretcher process";
125
RubberBandVampPlugin::getMaker() const
127
return "Breakfast Quay";
131
RubberBandVampPlugin::getPluginVersion() const
137
RubberBandVampPlugin::getCopyright() const
142
RubberBandVampPlugin::OutputList
143
RubberBandVampPlugin::getOutputDescriptors() const
148
if (m_d->m_stretcher) {
149
rate = lrintf(m_inputSampleRate / m_d->m_stretcher->getInputIncrement());
153
d.identifier = "increments";
154
d.name = "Output Increments";
155
d.description = "Output time increment for each input step";
157
d.hasFixedBinCount = true;
159
d.hasKnownExtents = false;
160
d.isQuantized = true;
161
d.quantizeStep = 1.0;
162
d.sampleType = OutputDescriptor::VariableSampleRate;
163
d.sampleRate = float(rate);
164
m_d->m_incrementsOutput = list.size();
167
d.identifier = "aggregate_increments";
168
d.name = "Accumulated Output Increments";
169
d.description = "Accumulated output time increments";
171
m_d->m_aggregateIncrementsOutput = list.size();
174
d.identifier = "divergence";
175
d.name = "Divergence from Linear";
176
d.description = "Difference between actual output time and the output time for a theoretical linear stretch";
177
d.isQuantized = false;
179
m_d->m_divergenceOutput = list.size();
182
d.identifier = "phaseresetdf";
183
d.name = "Phase Reset Detection Function";
184
d.description = "Curve whose peaks are used to identify transients for phase reset points";
186
d.sampleRate = float(rate);
187
m_d->m_phaseResetDfOutput = list.size();
190
d.identifier = "smoothedphaseresetdf";
191
d.name = "Smoothed Phase Reset Detection Function";
192
d.description = "Phase reset curve smoothed for peak picking";
194
m_d->m_smoothedPhaseResetDfOutput = list.size();
197
d.identifier = "phaseresetpoints";
198
d.name = "Phase Reset Points";
199
d.description = "Points estimated as transients at which phase reset occurs";
201
d.hasFixedBinCount = true;
203
d.hasKnownExtents = false;
204
d.isQuantized = false;
206
m_d->m_phaseResetPointsOutput = list.size();
209
d.identifier = "timesyncpoints";
210
d.name = "Time Sync Points";
211
d.description = "Salient points which stretcher aims to place with strictly correct timing";
213
d.hasFixedBinCount = true;
215
d.hasKnownExtents = false;
216
d.isQuantized = false;
218
m_d->m_timeSyncPointsOutput = list.size();
224
RubberBandVampPlugin::ParameterList
225
RubberBandVampPlugin::getParameterDescriptors() const
229
ParameterDescriptor d;
230
d.identifier = "timeratio";
231
d.name = "Time Ratio";
232
d.description = "Ratio to modify overall duration by";
236
d.defaultValue = 100;
237
d.isQuantized = false;
240
d.identifier = "pitchratio";
241
d.name = "Pitch Scale Ratio";
242
d.description = "Frequency ratio to modify pitch by";
246
d.defaultValue = 100;
247
d.isQuantized = false;
250
d.identifier = "mode";
251
d.name = "Processing Mode";
252
d.description = ""; //!!!
257
d.isQuantized = true;
259
d.valueNames.clear();
260
d.valueNames.push_back("Offline");
261
d.valueNames.push_back("Real Time");
264
d.identifier = "stretchtype";
265
d.name = "Stretch Flexibility";
266
d.description = ""; //!!!
271
d.isQuantized = true;
273
d.valueNames.clear();
274
d.valueNames.push_back("Elastic");
275
d.valueNames.push_back("Precise");
278
d.identifier = "transientmode";
279
d.name = "Transient Handling";
280
d.description = ""; //!!!
285
d.isQuantized = true;
287
d.valueNames.clear();
288
d.valueNames.push_back("Mixed");
289
d.valueNames.push_back("Smooth");
290
d.valueNames.push_back("Crisp");
293
d.identifier = "phasemode";
294
d.name = "Phase Handling";
295
d.description = ""; //!!!
300
d.isQuantized = true;
302
d.valueNames.clear();
303
d.valueNames.push_back("Peak Locked");
304
d.valueNames.push_back("Independent");
307
d.identifier = "windowmode";
308
d.name = "Window Length";
309
d.description = ""; //!!!
314
d.isQuantized = true;
316
d.valueNames.clear();
317
d.valueNames.push_back("Standard");
318
d.valueNames.push_back("Short");
319
d.valueNames.push_back("Long");
326
RubberBandVampPlugin::getParameter(std::string id) const
328
if (id == "timeratio") return m_d->m_timeRatio * 100.f;
329
if (id == "pitchratio") return m_d->m_pitchRatio * 100.f;
330
if (id == "mode") return m_d->m_realtime ? 1.f : 0.f;
331
if (id == "stretchtype") return m_d->m_elasticTiming ? 0.f : 1.f;
332
if (id == "transientmode") return float(m_d->m_transientMode);
333
if (id == "phasemode") return m_d->m_phaseIndependent ? 1.f : 0.f;
334
if (id == "windowmode") return float(m_d->m_windowLength);
339
RubberBandVampPlugin::setParameter(std::string id, float value)
341
if (id == "timeratio") {
342
m_d->m_timeRatio = value / 100;
343
} else if (id == "pitchratio") {
344
m_d->m_pitchRatio = value / 100;
346
bool set = (value > 0.5);
347
if (id == "mode") m_d->m_realtime = set;
348
else if (id == "stretchtype") m_d->m_elasticTiming = !set;
349
else if (id == "transientmode") m_d->m_transientMode = int(value + 0.5);
350
else if (id == "phasemode") m_d->m_phaseIndependent = set;
351
else if (id == "windowmode") m_d->m_windowLength = int(value + 0.5);
356
RubberBandVampPlugin::initialise(size_t channels, size_t stepSize, size_t blockSize)
358
if (channels < getMinChannelCount() ||
359
channels > getMaxChannelCount()) return false;
361
m_d->m_stepSize = std::min(stepSize, blockSize);
362
m_d->m_blockSize = stepSize;
364
RubberBand::RubberBandStretcher::Options options = 0;
367
options |= RubberBand::RubberBandStretcher::OptionProcessRealTime;
368
else options |= RubberBand::RubberBandStretcher::OptionProcessOffline;
370
if (m_d->m_elasticTiming)
371
options |= RubberBand::RubberBandStretcher::OptionStretchElastic;
372
else options |= RubberBand::RubberBandStretcher::OptionStretchPrecise;
374
if (m_d->m_transientMode == 0)
375
options |= RubberBand::RubberBandStretcher::OptionTransientsMixed;
376
else if (m_d->m_transientMode == 1)
377
options |= RubberBand::RubberBandStretcher::OptionTransientsSmooth;
378
else options |= RubberBand::RubberBandStretcher::OptionTransientsCrisp;
380
if (m_d->m_phaseIndependent)
381
options |= RubberBand::RubberBandStretcher::OptionPhaseIndependent;
382
else options |= RubberBand::RubberBandStretcher::OptionPhaseLaminar;
384
if (m_d->m_windowLength == 0)
385
options |= RubberBand::RubberBandStretcher::OptionWindowStandard;
386
else if (m_d->m_windowLength == 1)
387
options |= RubberBand::RubberBandStretcher::OptionWindowShort;
388
else options |= RubberBand::RubberBandStretcher::OptionWindowLong;
390
delete m_d->m_stretcher;
391
m_d->m_stretcher = new RubberBand::RubberBandStretcher
392
(m_d->m_sampleRate, channels, options);
393
m_d->m_stretcher->setDebugLevel(1);
394
m_d->m_stretcher->setTimeRatio(m_d->m_timeRatio);
395
m_d->m_stretcher->setPitchScale(m_d->m_pitchRatio);
398
m_d->m_accumulatedIncrement = 0;
400
m_d->m_outputDump = 0;
406
RubberBandVampPlugin::reset()
408
// delete m_stretcher; //!!! or just if (m_stretcher) m_stretcher->reset();
409
// m_stretcher = new RubberBand::RubberBandStretcher(lrintf(m_inputSampleRate), channels);
410
if (m_d->m_stretcher) m_d->m_stretcher->reset();
413
RubberBandVampPlugin::FeatureSet
414
RubberBandVampPlugin::process(const float *const *inputBuffers,
415
Vamp::RealTime timestamp)
417
if (m_d->m_realtime) {
418
return m_d->processRealTime(inputBuffers, timestamp);
420
return m_d->processOffline(inputBuffers, timestamp);
424
RubberBandVampPlugin::FeatureSet
425
RubberBandVampPlugin::getRemainingFeatures()
427
if (m_d->m_realtime) {
428
return m_d->getRemainingFeaturesRealTime();
430
return m_d->getRemainingFeaturesOffline();
434
RubberBandVampPlugin::FeatureSet
435
RubberBandVampPlugin::Impl::processOffline(const float *const *inputBuffers,
436
Vamp::RealTime timestamp)
439
cerr << "ERROR: RubberBandVampPlugin::processOffline: "
440
<< "RubberBandVampPlugin has not been initialised"
445
m_stretcher->study(inputBuffers, m_blockSize, false);
449
RubberBandVampPlugin::FeatureSet
450
RubberBandVampPlugin::Impl::getRemainingFeaturesOffline()
452
m_stretcher->study(0, 0, true);
454
m_stretcher->calculateStretch();
456
int rate = m_sampleRate;
458
RubberBand::StretchCalculator sc(rate,
459
m_stretcher->getInputIncrement(),
462
size_t inputIncrement = m_stretcher->getInputIncrement();
463
std::vector<int> outputIncrements = m_stretcher->getOutputIncrements();
464
std::vector<float> phaseResetDf = m_stretcher->getPhaseResetCurve();
465
std::vector<int> peaks = m_stretcher->getExactTimePoints();
466
std::vector<float> smoothedDf = sc.smoothDF(phaseResetDf);
468
FeatureSet features = createFeatures
469
(inputIncrement, outputIncrements, phaseResetDf, peaks, smoothedDf,
475
RubberBandVampPlugin::FeatureSet
476
RubberBandVampPlugin::Impl::processRealTime(const float *const *inputBuffers,
477
Vamp::RealTime timestamp)
479
// This function is not in any way a real-time function (i.e. it
480
// has no requirement to be RT safe); it simply operates the
481
// stretcher in RT mode.
484
cerr << "ERROR: RubberBandVampPlugin::processRealTime: "
485
<< "RubberBandVampPlugin has not been initialised"
490
m_stretcher->process(inputBuffers, m_blockSize, false);
492
size_t inputIncrement = m_stretcher->getInputIncrement();
493
std::vector<int> outputIncrements = m_stretcher->getOutputIncrements();
494
std::vector<float> phaseResetDf = m_stretcher->getPhaseResetCurve();
495
std::vector<float> smoothedDf; // not meaningful in RT mode
496
std::vector<int> dummyPoints;
497
FeatureSet features = createFeatures
498
(inputIncrement, outputIncrements, phaseResetDf, dummyPoints, smoothedDf,
500
m_counter += outputIncrements.size();
503
while ((available = m_stretcher->available()) > 0) {
505
m_outputDump = new float *[m_stretcher->getChannelCount()];
506
for (size_t i = 0; i < m_stretcher->getChannelCount(); ++i) {
507
m_outputDump[i] = new float[m_blockSize];
510
m_stretcher->retrieve(m_outputDump,
511
std::min(int(m_blockSize), available));
517
RubberBandVampPlugin::FeatureSet
518
RubberBandVampPlugin::Impl::getRemainingFeaturesRealTime()
523
RubberBandVampPlugin::FeatureSet
524
RubberBandVampPlugin::Impl::createFeatures(size_t inputIncrement,
525
std::vector<int> &outputIncrements,
526
std::vector<float> &phaseResetDf,
527
std::vector<int> &exactPoints,
528
std::vector<float> &smoothedDf,
532
size_t actual = m_accumulatedIncrement;
534
double overallRatio = m_timeRatio * m_pitchRatio;
540
int rate = m_sampleRate;
544
for (size_t i = 0; i < outputIncrements.size(); ++i) {
546
size_t frame = (baseCount + i) * inputIncrement;
548
int oi = outputIncrements[i];
557
if (epi < exactPoints.size() && int(i) == exactPoints[epi]) {
562
double linear = (frame * overallRatio);
564
Vamp::RealTime t = Vamp::RealTime::frame2RealTime(frame, rate);
567
feature.hasTimestamp = true;
568
feature.timestamp = t;
569
feature.values.push_back(float(oi));
570
feature.label = Vamp::RealTime::frame2RealTime(oi, rate).toText();
571
features[m_incrementsOutput].push_back(feature);
573
feature.values.clear();
574
feature.values.push_back(float(actual));
575
feature.label = Vamp::RealTime::frame2RealTime(actual, rate).toText();
576
features[m_aggregateIncrementsOutput].push_back(feature);
578
feature.values.clear();
579
feature.values.push_back(actual - linear);
581
sprintf(label, "expected %ld, actual %ld, difference %ld (%s ms)",
582
long(linear), long(actual), long(actual - linear),
583
// frame2RealTime expects an integer frame number,
584
// hence our multiplication factor
585
(Vamp::RealTime::frame2RealTime
586
(lrintf((actual - linear) * 1000), rate) / 1000)
588
feature.label = label;
590
features[m_divergenceOutput].push_back(feature);
595
if (i < phaseResetDf.size()) {
596
feature.values.clear();
597
feature.values.push_back(phaseResetDf[i]);
598
sprintf(buf, "%d", int(baseCount + i));
600
features[m_phaseResetDfOutput].push_back(feature);
603
if (i < smoothedDf.size()) {
604
feature.values.clear();
605
feature.values.push_back(smoothedDf[i]);
606
features[m_smoothedPhaseResetDfOutput].push_back(feature);
610
feature.values.clear();
611
feature.label = "Phase Reset";
612
features[m_phaseResetPointsOutput].push_back(feature);
616
feature.values.clear();
617
feature.label = "Time Sync";
618
features[m_timeSyncPointsOutput].push_back(feature);
623
Vamp::RealTime t = Vamp::RealTime::frame2RealTime
624
(inputIncrement * (baseCount + outputIncrements.size()), rate);
626
feature.hasTimestamp = true;
627
feature.timestamp = t;
628
feature.label = Vamp::RealTime::frame2RealTime(actual, rate).toText();
629
feature.values.clear();
630
feature.values.push_back(float(actual));
631
features[m_aggregateIncrementsOutput].push_back(feature);
633
float linear = ((baseCount + outputIncrements.size())
634
* inputIncrement * overallRatio);
635
feature.values.clear();
636
feature.values.push_back(actual - linear);
637
feature.label = // see earlier comment
638
(Vamp::RealTime::frame2RealTime //!!! update this as earlier label
639
(lrintf((actual - linear) * 1000), rate) / 1000)
641
features[m_divergenceOutput].push_back(feature);
644
m_accumulatedIncrement = actual;