feat: add support for STEM file in analyzers

This will allow analyser to make the most of the multiple channels to analyze specific features of the track. It will also generate a multi channel waveform
mixxxdj · Apr 18, 2024 · 97cb238 · 97cb238
1 parent 2f0518d
commit 97cb238
Show file tree

Hide file tree

Showing 24 changed files with 357 additions and 73 deletions.
diff --git a/src/analyzer/analyzer.h b/src/analyzer/analyzer.h
@@ -26,6 +26,7 @@ class Analyzer {
     //  3. If the initialization failed log the internal error and return false.
     virtual bool initialize(const AnalyzerTrack& track,
             mixxx::audio::SampleRate sampleRate,
+            mixxx::audio::ChannelCount channelCount,
             SINT frameLength) = 0;
 
     // Analyze the next chunk of audio samples and return true if successful.
@@ -68,9 +69,10 @@ class AnalyzerWithState final {
 
     bool initialize(const AnalyzerTrack& track,
             mixxx::audio::SampleRate sampleRate,
+            mixxx::audio::ChannelCount channelCount,
             SINT frameLength) {
         DEBUG_ASSERT(!m_active);
-        return m_active = m_analyzer->initialize(track, sampleRate, frameLength);
+        return m_active = m_analyzer->initialize(track, sampleRate, channelCount, frameLength);
     }
 
     void processSamples(const CSAMPLE* pIn, const int count) {

diff --git a/src/analyzer/analyzerbeats.cpp b/src/analyzer/analyzerbeats.cpp
@@ -42,6 +42,7 @@ AnalyzerBeats::AnalyzerBeats(UserSettingsPointer pConfig, bool enforceBpmDetecti
 
 bool AnalyzerBeats::initialize(const AnalyzerTrack& track,
         mixxx::audio::SampleRate sampleRate,
+        mixxx::audio::ChannelCount channelCount,
         SINT frameLength) {
     if (frameLength <= 0) {
         return false;
@@ -86,6 +87,7 @@ bool AnalyzerBeats::initialize(const AnalyzerTrack& track,
              << "\nFast analysis:" << m_bPreferencesFastAnalysis;
 
     m_sampleRate = sampleRate;
+    m_channelCount = channelCount;
     // In fast analysis mode, skip processing after
     // kFastAnalysisSecondsToAnalyze seconds are analyzed.
     if (m_bPreferencesFastAnalysis) {
@@ -199,12 +201,34 @@ bool AnalyzerBeats::processSamples(const CSAMPLE* pIn, SINT count) {
         return false;
     }
 
-    m_currentFrame += count / mixxx::kAnalysisChannels;
+    SINT numFrames = count / m_channelCount;
+    const CSAMPLE* pBeatInput = pIn;
+    CSAMPLE* pDrumChannel = nullptr;
+
+    if (m_channelCount > mixxx::kAnalysisChannels) {
+        // If we have multi channel file (a stem file), we extract the first
+        // stereo channel, as it is the drum stem by convention
+        count = numFrames * mixxx::kAnalysisChannels;
+        pDrumChannel = SampleUtil::alloc(count);
+
+        VERIFY_OR_DEBUG_ASSERT(pDrumChannel) {
+            return false;
+        }
+
+        SampleUtil::copyMultiToStereo(pDrumChannel, pIn, numFrames, m_channelCount, 0);
+        pBeatInput = pDrumChannel;
+    }
+
+    m_currentFrame += numFrames;
     if (m_currentFrame > m_maxFramesToProcess) {
         return true; // silently ignore all remaining samples
     }
 
-    return m_pPlugin->processSamples(pIn, count);
+    bool ret = m_pPlugin->processSamples(pBeatInput, count);
+    if (pDrumChannel) {
+        SampleUtil::free(pDrumChannel);
+    }
+    return ret;
 }
 
 void AnalyzerBeats::cleanup() {

diff --git a/src/analyzer/analyzerbeats.h b/src/analyzer/analyzerbeats.h
@@ -21,6 +21,7 @@ class AnalyzerBeats : public Analyzer {
 
     bool initialize(const AnalyzerTrack& track,
             mixxx::audio::SampleRate sampleRate,
+            mixxx::audio::ChannelCount channelCount,
             SINT frameLength) override;
     bool processSamples(const CSAMPLE* pIn, SINT count) override;
     void storeResults(TrackPointer tio) override;
@@ -41,6 +42,7 @@ class AnalyzerBeats : public Analyzer {
     bool m_bPreferencesFastAnalysis;
 
     mixxx::audio::SampleRate m_sampleRate;
+    mixxx::audio::ChannelCount m_channelCount;
     SINT m_maxFramesToProcess;
     SINT m_currentFrame;
 };
diff --git a/src/analyzer/analyzerebur128.cpp b/src/analyzer/analyzerebur128.cpp
@@ -24,14 +24,15 @@ AnalyzerEbur128::~AnalyzerEbur128() {
 bool AnalyzerEbur128::initialize(
         const AnalyzerTrack& track,
         mixxx::audio::SampleRate sampleRate,
+        mixxx::audio::ChannelCount channelCount,
         SINT frameLength) {
     if (m_rgSettings.isAnalyzerDisabled(2, track.getTrack()) || frameLength <= 0) {
         qDebug() << "Skipping AnalyzerEbur128";
         return false;
     }
     DEBUG_ASSERT(m_pState == nullptr);
     m_pState = ebur128_init(
-            mixxx::kAnalysisChannels,
+            channelCount,
             sampleRate,
             EBUR128_MODE_I);
     return m_pState != nullptr;
@@ -50,7 +51,7 @@ bool AnalyzerEbur128::processSamples(const CSAMPLE* pIn, SINT count) {
         return false;
     }
     ScopedTimer t(u"AnalyzerEbur128::processSamples()");
-    size_t frames = count / mixxx::kAnalysisChannels;
+    size_t frames = count / m_pState->channels;
     int e = ebur128_add_frames_float(m_pState, pIn, frames);
     VERIFY_OR_DEBUG_ASSERT(e == EBUR128_SUCCESS) {
         qWarning() << "AnalyzerEbur128::processSamples() failed with" << e;

diff --git a/src/analyzer/analyzerebur128.h b/src/analyzer/analyzerebur128.h
@@ -16,6 +16,7 @@ class AnalyzerEbur128 : public Analyzer {
 
     bool initialize(const AnalyzerTrack& track,
             mixxx::audio::SampleRate sampleRate,
+            mixxx::audio::ChannelCount channelCount,
             SINT frameLength) override;
     bool processSamples(const CSAMPLE* pIn, SINT count) override;
     void storeResults(TrackPointer pTrack) override;

diff --git a/src/analyzer/analyzergain.cpp b/src/analyzer/analyzergain.cpp
@@ -22,11 +22,13 @@ AnalyzerGain::~AnalyzerGain() {
 
 bool AnalyzerGain::initialize(const AnalyzerTrack& track,
         mixxx::audio::SampleRate sampleRate,
+        mixxx::audio::ChannelCount channelCount,
         SINT frameLength) {
     if (m_rgSettings.isAnalyzerDisabled(1, track.getTrack()) || frameLength <= 0) {
         qDebug() << "Skipping AnalyzerGain";
         return false;
     }
+    m_channelCount = channelCount;
 
     return m_pReplayGain->initialise(
             sampleRate,
@@ -39,18 +41,39 @@ void AnalyzerGain::cleanup() {
 bool AnalyzerGain::processSamples(const CSAMPLE* pIn, SINT count) {
     ScopedTimer t(u"AnalyzerGain::process()");
 
-    SINT numFrames = count / mixxx::kAnalysisChannels;
+    SINT numFrames = count / m_channelCount;
+
+    const CSAMPLE* pGainInput = pIn;
+    CSAMPLE* pMixedChannel = nullptr;
+
+    if (m_channelCount > mixxx::kAnalysisChannels) {
+        // If we have multi channel file (a stem file), we mix all the stems
+        // together in a stereo channel
+        count = numFrames * mixxx::kAnalysisChannels;
+        pMixedChannel = SampleUtil::alloc(count);
+        VERIFY_OR_DEBUG_ASSERT(pMixedChannel) {
+            return false;
+        }
+        SampleUtil::mixMultichannelToStereo(pMixedChannel, pIn, numFrames, m_channelCount);
+        pGainInput = pMixedChannel;
+    }
+
     if (numFrames > static_cast<SINT>(m_pLeftTempBuffer.size())) {
         m_pLeftTempBuffer.resize(numFrames);
         m_pRightTempBuffer.resize(numFrames);
     }
     SampleUtil::deinterleaveBuffer(m_pLeftTempBuffer.data(),
             m_pRightTempBuffer.data(),
-            pIn,
+            pGainInput,
             numFrames);
     SampleUtil::applyGain(m_pLeftTempBuffer.data(), 32767, numFrames);
     SampleUtil::applyGain(m_pRightTempBuffer.data(), 32767, numFrames);
-    return m_pReplayGain->process(m_pLeftTempBuffer.data(), m_pRightTempBuffer.data(), numFrames);
+    bool ret = m_pReplayGain->process(
+            m_pLeftTempBuffer.data(), m_pRightTempBuffer.data(), numFrames);
+    if (pMixedChannel) {
+        SampleUtil::free(pMixedChannel);
+    }
+    return ret;
 }
 
 void AnalyzerGain::storeResults(TrackPointer pTrack) {

diff --git a/src/analyzer/analyzergain.h b/src/analyzer/analyzergain.h
@@ -25,6 +25,7 @@ class AnalyzerGain : public Analyzer {
 
     bool initialize(const AnalyzerTrack& track,
             mixxx::audio::SampleRate sampleRate,
+            mixxx::audio::ChannelCount channelCount,
             SINT frameLength) override;
     bool processSamples(const CSAMPLE* pIn, SINT count) override;
     void storeResults(TrackPointer tio) override;
@@ -34,5 +35,6 @@ class AnalyzerGain : public Analyzer {
     ReplayGainSettings m_rgSettings;
     std::vector<CSAMPLE> m_pLeftTempBuffer;
     std::vector<CSAMPLE> m_pRightTempBuffer;
+    mixxx::audio::ChannelCount m_channelCount;
     ReplayGain* m_pReplayGain;
 };
diff --git a/src/analyzer/analyzerkey.cpp b/src/analyzer/analyzerkey.cpp
@@ -43,6 +43,7 @@ AnalyzerKey::AnalyzerKey(const KeyDetectionSettings& keySettings)
 
 bool AnalyzerKey::initialize(const AnalyzerTrack& track,
         mixxx::audio::SampleRate sampleRate,
+        mixxx::audio::ChannelCount channelCount,
         SINT frameLength) {
     if (frameLength <= 0) {
         return false;
@@ -75,6 +76,7 @@ bool AnalyzerKey::initialize(const AnalyzerTrack& track,
              << "\nFast analysis:" << m_bPreferencesFastAnalysisEnabled;
 
     m_sampleRate = sampleRate;
+    m_channelCount = channelCount;
     m_totalFrames = frameLength;
     // In fast analysis mode, skip processing after
     // kFastAnalysisSecondsToAnalyze seconds are analyzed.
@@ -155,12 +157,39 @@ bool AnalyzerKey::processSamples(const CSAMPLE* pIn, SINT count) {
         return false;
     }
 
-    m_currentFrame += count / mixxx::kAnalysisChannels;
+    SINT numFrames = count / m_channelCount;
+    m_currentFrame += numFrames;
+
     if (m_currentFrame > m_maxFramesToProcess) {
         return true; // silently ignore remaining samples
     }
 
-    return m_pPlugin->processSamples(pIn, count);
+    const CSAMPLE* pKeyInput = pIn;
+    CSAMPLE* pHarmonicMixedChannel = nullptr;
+
+    if (m_channelCount > mixxx::kAnalysisChannels) {
+        // If we have multi channel file (a stem file), we mix all the stems
+        // together except the first one which contains drums or beats by
+        // convention
+        count = numFrames * mixxx::kAnalysisChannels;
+        pHarmonicMixedChannel = SampleUtil::alloc(count);
+        VERIFY_OR_DEBUG_ASSERT(pHarmonicMixedChannel) {
+            return false;
+        }
+
+        SampleUtil::mixMultichannelToStereo(pHarmonicMixedChannel,
+                pIn,
+                numFrames,
+                m_channelCount,
+                1 /*exclude the first stem, 0b0001*/);
+        pKeyInput = pHarmonicMixedChannel;
+    }
+
+    bool ret = m_pPlugin->processSamples(pKeyInput, count);
+    if (pHarmonicMixedChannel) {
+        SampleUtil::free(pHarmonicMixedChannel);
+    }
+    return ret;
 }
 
 void AnalyzerKey::cleanup() {

diff --git a/src/analyzer/analyzerkey.h b/src/analyzer/analyzerkey.h
@@ -20,6 +20,7 @@ class AnalyzerKey : public Analyzer {
 
     bool initialize(const AnalyzerTrack& track,
             mixxx::audio::SampleRate sampleRate,
+            mixxx::audio::ChannelCount channelCount,
             SINT frameLength) override;
     bool processSamples(const CSAMPLE* pIn, SINT count) override;
     void storeResults(TrackPointer tio) override;
@@ -35,6 +36,7 @@ class AnalyzerKey : public Analyzer {
     std::unique_ptr<mixxx::AnalyzerKeyPlugin> m_pPlugin;
     QString m_pluginId;
     mixxx::audio::SampleRate m_sampleRate;
+    mixxx::audio::ChannelCount m_channelCount;
     SINT m_totalFrames;
     SINT m_maxFramesToProcess;
     SINT m_currentFrame;

diff --git a/src/analyzer/analyzersilence.cpp b/src/analyzer/analyzersilence.cpp
@@ -41,6 +41,7 @@ AnalyzerSilence::AnalyzerSilence(UserSettingsPointer pConfig)
 
 bool AnalyzerSilence::initialize(const AnalyzerTrack& track,
         mixxx::audio::SampleRate sampleRate,
+        mixxx::audio::ChannelCount channelCount,
         SINT frameLength) {
     Q_UNUSED(sampleRate);
     Q_UNUSED(frameLength);
@@ -52,6 +53,7 @@ bool AnalyzerSilence::initialize(const AnalyzerTrack& track,
     m_framesProcessed = 0;
     m_signalStart = -1;
     m_signalEnd = -1;
+    m_channelCount = channelCount;
 
     return true;
 }
@@ -85,7 +87,26 @@ bool AnalyzerSilence::verifyFirstSound(
 }
 
 bool AnalyzerSilence::processSamples(const CSAMPLE* pIn, SINT count) {
-    std::span<const CSAMPLE> samples = mixxx::spanutil::spanFromPtrLen(pIn, count);
+    SINT numFrames = count / m_channelCount;
+
+    const CSAMPLE* pSilenceInput = pIn;
+    CSAMPLE* pMixedChannel = nullptr;
+
+    if (m_channelCount > mixxx::kAnalysisChannels) {
+        // If we have multi channel file (a stem file), we mix all the stems
+        // together in a stereo channel
+        count = numFrames * mixxx::kAnalysisChannels;
+        pMixedChannel = SampleUtil::alloc(count);
+        SampleUtil::clear(pMixedChannel, count);
+
+        VERIFY_OR_DEBUG_ASSERT(pMixedChannel) {
+            return false;
+        }
+        SampleUtil::mixMultichannelToStereo(pMixedChannel, pIn, numFrames, m_channelCount);
+        pSilenceInput = pMixedChannel;
+    }
+
+    std::span<const CSAMPLE> samples = mixxx::spanutil::spanFromPtrLen(pSilenceInput, count);
     if (m_signalStart < 0) {
         const SINT firstSoundSample = findFirstSoundInChunk(samples);
         if (firstSoundSample < count) {

diff --git a/src/analyzer/analyzersilence.h b/src/analyzer/analyzersilence.h
@@ -23,6 +23,7 @@ class AnalyzerSilence : public Analyzer {
 
     bool initialize(const AnalyzerTrack& track,
             mixxx::audio::SampleRate sampleRate,
+            mixxx::audio::ChannelCount channelCount,
             SINT frameLength) override;
     bool processSamples(const CSAMPLE* pIn, SINT count) override;
     void storeResults(TrackPointer pTrack) override;
@@ -51,6 +52,7 @@ class AnalyzerSilence : public Analyzer {
 
   private:
     UserSettingsPointer m_pConfig;
+    mixxx::audio::ChannelCount m_channelCount;
     SINT m_framesProcessed;
     SINT m_signalStart;
     SINT m_signalEnd;