Merge pull request mixxxdj#12112 from fwcd/au-effects-backend

Effects: Add backend for Audio Unit (AU) plugins on macOS

CMakeLists.txt

@@ -1595,6 +1595,21 @@ if(APPLE)
       target_compile_definitions(mixxx-lib PUBLIC __MACOS_ITUNES_LIBRARY__)
     endif()
   endif()
+
+  option(AU_EFFECTS "Audio Unit (AU) effects integration" ON)
+  if(AU_EFFECTS)
+    target_sources(mixxx-lib PRIVATE
+      src/effects/backends/audiounit/audiounitbackend.mm
+      src/effects/backends/audiounit/audiounitmanager.mm
+      src/effects/backends/audiounit/audiouniteffectprocessor.mm
+      src/effects/backends/audiounit/audiounitmanifest.mm
+    )
+    target_link_libraries(mixxx-lib PRIVATE
+      "-weak_framework AudioToolbox"
+      "-weak_framework AVFAudio"
+    )
+    target_compile_definitions(mixxx-lib PUBLIC __AU_EFFECTS__)
+  endif()
 endif()
 
 if(EMSCRIPTEN)

src/effects/backends/audiounit/audiounitbackend.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "effects/defs.h"
+
+EffectsBackendPointer createAudioUnitBackend();

src/effects/backends/audiounit/audiounitbackend.mm

@@ -0,0 +1,112 @@
+#include "effects/backends/audiounit/audiounitbackend.h"
+
+#import <AVFAudio/AVFAudio.h>
+#import <AudioToolbox/AudioToolbox.h>
+#import <Foundation/Foundation.h>
+
+#include <QDebug>
+#include <QHash>
+#include <QList>
+#include <QString>
+#include <memory>
+
+#include "effects/backends/audiounit/audiounitbackend.h"
+#include "effects/backends/audiounit/audiouniteffectprocessor.h"
+#include "effects/backends/audiounit/audiounitmanifest.h"
+#include "effects/defs.h"
+
+/// An effects backend for Audio Unit (AU) plugins. macOS-only.
+class AudioUnitBackend : public EffectsBackend {
+  public:
+    AudioUnitBackend() : m_componentsById([[NSDictionary alloc] init]) {
+        loadAudioUnits();
+    }
+
+    ~AudioUnitBackend() override {
+    }
+
+    EffectBackendType getType() const override {
+        return EffectBackendType::AudioUnit;
+    };
+
+    const QList<QString> getEffectIds() const override {
+        QList<QString> effectIds;
+
+        for (NSString* effectId in m_componentsById) {
+            effectIds.append(QString::fromNSString(effectId));
+        }
+
+        return effectIds;
+    }
+
+    EffectManifestPointer getManifest(const QString& effectId) const override {
+        return m_manifestsById[effectId];
+    }
+
+    const QList<EffectManifestPointer> getManifests() const override {
+        return m_manifestsById.values();
+    }
+
+    bool canInstantiateEffect(const QString& effectId) const override {
+        return [m_componentsById objectForKey:effectId.toNSString()] != nil;
+    }
+
+    std::unique_ptr<EffectProcessor> createProcessor(
+            const EffectManifestPointer pManifest) const override {
+        AVAudioUnitComponent* component =
+                m_componentsById[pManifest->id().toNSString()];
+        return std::make_unique<AudioUnitEffectProcessor>(component);
+    }
+
+  private:
+    NSDictionary<NSString*, AVAudioUnitComponent*>* m_componentsById;
+    QHash<QString, EffectManifestPointer> m_manifestsById;
+
+    void loadAudioUnits() {
+        qDebug() << "Loading audio units...";
+
+        // See
+        // https://developer.apple.com/documentation/audiotoolbox/audio_unit_v3_plug-ins/incorporating_audio_effects_and_instruments?language=objc
+
+        // Create a query for audio components
+        AudioComponentDescription description = {
+                .componentType = kAudioUnitType_Effect,
+                .componentSubType = 0,
+                .componentManufacturer = 0,
+                .componentFlags = 0,
+                .componentFlagsMask = 0,
+        };
+
+        // Find the audio units
+        // TODO: Should we perform this asynchronously (e.g. using Qt's
+        // threading or GCD)?
+        auto manager =
+                [AVAudioUnitComponentManager sharedAudioUnitComponentManager];
+        auto components = [manager componentsMatchingDescription:description];
+
+        // Assign ids to the components
+        NSMutableDictionary<NSString*, AVAudioUnitComponent*>* componentsById =
+                [[NSMutableDictionary alloc] init];
+        QHash<QString, EffectManifestPointer> manifestsById;
+
+        for (AVAudioUnitComponent* component in components) {
+            qDebug() << "Found audio unit" << [component name];
+
+            QString effectId = QString::fromNSString(
+                    [NSString stringWithFormat:@"%@~%@~%@",
+                              [component manufacturerName],
+                              [component name],
+                              [component versionString]]);
+            componentsById[effectId.toNSString()] = component;
+            manifestsById[effectId] = EffectManifestPointer(
+                    new AudioUnitManifest(effectId, component));
+        }
+
+        m_componentsById = componentsById;
+        m_manifestsById = manifestsById;
+    }
+};
+
+EffectsBackendPointer createAudioUnitBackend() {
+    return EffectsBackendPointer(new AudioUnitBackend());
+}

src/effects/backends/audiounit/audiouniteffectprocessor.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#import <AVFAudio/AVFAudio.h>
+#import <AudioToolbox/AudioToolbox.h>
+#import <CoreAudioTypes/CoreAudioTypes.h>
+
+#include <QList>
+#include <atomic>
+
+#include "audio/types.h"
+#include "effects/backends/audiounit/audiounitmanager.h"
+#include "effects/backends/effectprocessor.h"
+#include "engine/engine.h"
+
+class AudioUnitEffectGroupState final : public EffectState {
+  public:
+    AudioUnitEffectGroupState(const mixxx::EngineParameters& engineParameters);
+
+    void render(AudioUnit _Nonnull audioUnit,
+            const mixxx::EngineParameters& engineParameters,
+            const CSAMPLE* _Nonnull pInput,
+            CSAMPLE* _Nonnull pOutput);
+
+  private:
+    AudioTimeStamp m_timestamp;
+    AudioBufferList m_inputBuffers;
+    AudioBufferList m_outputBuffers;
+
+    static OSStatus renderCallbackUntyped(void* _Nonnull rawThis,
+            AudioUnitRenderActionFlags* _Nonnull inActionFlags,
+            const AudioTimeStamp* _Nonnull inTimeStamp,
+            UInt32 inBusNumber,
+            UInt32 inNumFrames,
+            AudioBufferList* _Nonnull ioData);
+    OSStatus renderCallback(AudioUnitRenderActionFlags* _Nonnull inActionFlags,
+            const AudioTimeStamp* _Nonnull inTimeStamp,
+            UInt32 inBusNumber,
+            UInt32 inNumFrames,
+            AudioBufferList* _Nonnull ioData) const;
+};
+
+class AudioUnitEffectProcessor final : public EffectProcessorImpl<AudioUnitEffectGroupState> {
+  public:
+    AudioUnitEffectProcessor(AVAudioUnitComponent* _Nullable component = nil);
+
+    void loadEngineEffectParameters(
+            const QMap<QString, EngineEffectParameterPointer>& parameters)
+            override;
+
+    void processChannel(AudioUnitEffectGroupState* _Nonnull channelState,
+            const CSAMPLE* _Nonnull pInput,
+            CSAMPLE* _Nonnull pOutput,
+            const mixxx::EngineParameters& engineParameters,
+            const EffectEnableState enableState,
+            const GroupFeatureState& groupFeatures) override;
+
+  private:
+    AudioUnitManager m_manager;
+
+    QList<EngineEffectParameterPointer> m_parameters;
+    QList<AudioUnitParameterValue> m_lastValues;
+    mixxx::audio::ChannelCount m_lastChannelCount;
+    mixxx::audio::SampleRate m_lastSampleRate;
+
+    void syncParameters();
+    void syncStreamFormat(const mixxx::EngineParameters& engineParameters);
+};