From 2c41dbd529b59ace2b786ab207026e0a5be03afc Mon Sep 17 00:00:00 2001 From: Andy Grundman Date: Mon, 2 Sep 2024 21:09:11 -0400 Subject: [PATCH] CoreAudio: add UI to enable/disable spatial audio and head-tracking --- app/gui/SettingsView.qml | 74 ++++++++++++++++++- app/settings/streamingpreferences.cpp | 7 ++ app/settings/streamingpreferences.h | 13 ++++ .../coreaudio/au_spatial_renderer.mm | 7 +- .../audio/renderers/coreaudio/coreaudio.cpp | 19 +++-- 5 files changed, 111 insertions(+), 9 deletions(-) diff --git a/app/gui/SettingsView.qml b/app/gui/SettingsView.qml index 30b9ed787..974d6bda2 100644 --- a/app/gui/SettingsView.qml +++ b/app/gui/SettingsView.qml @@ -881,6 +881,78 @@ Flickable { } } + Label { + width: parent.width + id: resSpatialAudioTitle + text: qsTr("Spatial audio") + font.pointSize: 12 + wrapMode: Text.Wrap + visible: Qt.platform.os == "osx" + } + + Row { + spacing: 5 + width: parent.width + visible: Qt.platform.os == "osx" + + AutoResizingComboBox { + // ignore setting the index at first, and actually set it when the component is loaded + Component.onCompleted: { + var saved_sac = StreamingPreferences.spatialAudioConfig + currentIndex = 0 + for (var i = 0; i < spatialAudioListModel.count; i++) { + var el_audio = spatialAudioListModel.get(i).val; + if (saved_sac === el_audio) { + currentIndex = i + break + } + } + activated(currentIndex) + } + + id: spatialAudioComboBox + enabled: StreamingPreferences.audioConfig != StreamingPreferences.AC_STEREO + textRole: "text" + model: ListModel { + id: spatialAudioListModel + ListElement { + text: qsTr("Enabled") + val: StreamingPreferences.SAC_AUTO + } + ListElement { + text: qsTr("Disabled") + val: StreamingPreferences.SAC_DISABLED + } + } + + // ::onActivated must be used, as it only listens for when the index is changed by a human + onActivated : { + StreamingPreferences.spatialAudioConfig = spatialAudioListModel.get(currentIndex).val + } + + ToolTip.delay: 1000 + ToolTip.timeout: 5000 + ToolTip.visible: hovered + ToolTip.text: qsTr("Spatial audio will be used when using any type of headphones, built-in Macbook speakers, and 2-channel USB devices.") + } + + CheckBox { + id: spatialHeadTracking + enabled: StreamingPreferences.audioConfig != StreamingPreferences.AC_STEREO && StreamingPreferences.spatialAudioConfig != StreamingPreferences.SAC_DISABLED + width: parent.width + text: qsTr("Enable head-tracking") + font.pointSize: 12 + checked: StreamingPreferences.spatialHeadTracking + onCheckedChanged: { + StreamingPreferences.spatialHeadTracking = checked + } + + ToolTip.delay: 1000 + ToolTip.timeout: 5000 + ToolTip.visible: hovered + ToolTip.text: qsTr("Requires supported Apple or Beats headphones") + } + } CheckBox { id: audioPcCheck @@ -1176,7 +1248,7 @@ Flickable { ListElement { text: qsTr("Maximized") val: StreamingPreferences.UI_MAXIMIZED - } + } ListElement { text: qsTr("Fullscreen") val: StreamingPreferences.UI_FULLSCREEN diff --git a/app/settings/streamingpreferences.cpp b/app/settings/streamingpreferences.cpp index bb34f6269..9655d4799 100644 --- a/app/settings/streamingpreferences.cpp +++ b/app/settings/streamingpreferences.cpp @@ -19,9 +19,11 @@ #define SER_FULLSCREEN "fullscreen" #define SER_VSYNC "vsync" #define SER_GAMEOPTS "gameopts" +#define SER_HEADTRACKING "headtracking" #define SER_HOSTAUDIO "hostaudio" #define SER_MULTICONT "multicontroller" #define SER_AUDIOCFG "audiocfg" +#define SER_SPATIALAUDIOCFG "spatialaudiocfg" #define SER_VIDEOCFG "videocfg" #define SER_HDR "hdr" #define SER_YUV444 "yuv444" @@ -124,6 +126,7 @@ void StreamingPreferences::reload() unlockBitrate = settings.value(SER_UNLOCK_BITRATE, false).toBool(); enableVsync = settings.value(SER_VSYNC, true).toBool(); gameOptimizations = settings.value(SER_GAMEOPTS, true).toBool(); + spatialHeadTracking = settings.value(SER_HEADTRACKING, false).toBool(); playAudioOnHost = settings.value(SER_HOSTAUDIO, false).toBool(); multiController = settings.value(SER_MULTICONT, true).toBool(); enableMdns = settings.value(SER_MDNS, true).toBool(); @@ -148,6 +151,8 @@ void StreamingPreferences::reload() static_cast(CaptureSysKeysMode::CSK_OFF)).toInt()); audioConfig = static_cast(settings.value(SER_AUDIOCFG, static_cast(AudioConfig::AC_STEREO)).toInt()); + spatialAudioConfig = static_cast(settings.value(SER_SPATIALAUDIOCFG, + static_cast(SpatialAudioConfig::SAC_AUTO)).toInt()); videoCodecConfig = static_cast(settings.value(SER_VIDEOCFG, static_cast(VideoCodecConfig::VCC_AUTO)).toInt()); videoDecoderSelection = static_cast(settings.value(SER_VIDEODEC, @@ -314,6 +319,7 @@ void StreamingPreferences::save() settings.setValue(SER_UNLOCK_BITRATE, unlockBitrate); settings.setValue(SER_VSYNC, enableVsync); settings.setValue(SER_GAMEOPTS, gameOptimizations); + settings.setValue(SER_HEADTRACKING, spatialHeadTracking); settings.setValue(SER_HOSTAUDIO, playAudioOnHost); settings.setValue(SER_MULTICONT, multiController); settings.setValue(SER_MDNS, enableMdns); @@ -328,6 +334,7 @@ void StreamingPreferences::save() settings.setValue(SER_DETECTNETBLOCKING, detectNetworkBlocking); settings.setValue(SER_SHOWPERFOVERLAY, showPerformanceOverlay); settings.setValue(SER_AUDIOCFG, static_cast(audioConfig)); + settings.setValue(SER_SPATIALAUDIOCFG, static_cast(spatialAudioConfig)); settings.setValue(SER_HDR, enableHdr); settings.setValue(SER_YUV444, enableYUV444); settings.setValue(SER_VIDEOCFG, static_cast(videoCodecConfig)); diff --git a/app/settings/streamingpreferences.h b/app/settings/streamingpreferences.h index 3ca216fff..b2adf75ad 100644 --- a/app/settings/streamingpreferences.h +++ b/app/settings/streamingpreferences.h @@ -26,6 +26,13 @@ class StreamingPreferences : public QObject }; Q_ENUM(AudioConfig) + enum SpatialAudioConfig + { + SAC_AUTO, + SAC_DISABLED + }; + Q_ENUM(SpatialAudioConfig) + enum VideoCodecConfig { VCC_AUTO, @@ -112,6 +119,7 @@ class StreamingPreferences : public QObject Q_PROPERTY(bool unlockBitrate MEMBER unlockBitrate NOTIFY unlockBitrateChanged) Q_PROPERTY(bool enableVsync MEMBER enableVsync NOTIFY enableVsyncChanged) Q_PROPERTY(bool gameOptimizations MEMBER gameOptimizations NOTIFY gameOptimizationsChanged) + Q_PROPERTY(bool spatialHeadTracking MEMBER spatialHeadTracking NOTIFY spatialHeadTrackingChanged) Q_PROPERTY(bool playAudioOnHost MEMBER playAudioOnHost NOTIFY playAudioOnHostChanged) Q_PROPERTY(bool multiController MEMBER multiController NOTIFY multiControllerChanged) Q_PROPERTY(bool enableMdns MEMBER enableMdns NOTIFY enableMdnsChanged) @@ -125,6 +133,7 @@ class StreamingPreferences : public QObject Q_PROPERTY(bool detectNetworkBlocking MEMBER detectNetworkBlocking NOTIFY detectNetworkBlockingChanged) Q_PROPERTY(bool showPerformanceOverlay MEMBER showPerformanceOverlay NOTIFY showPerformanceOverlayChanged) Q_PROPERTY(AudioConfig audioConfig MEMBER audioConfig NOTIFY audioConfigChanged) + Q_PROPERTY(SpatialAudioConfig spatialAudioConfig MEMBER spatialAudioConfig NOTIFY spatialAudioConfigChanged) Q_PROPERTY(VideoCodecConfig videoCodecConfig MEMBER videoCodecConfig NOTIFY videoCodecConfigChanged) Q_PROPERTY(bool enableHdr MEMBER enableHdr NOTIFY enableHdrChanged) Q_PROPERTY(bool enableYUV444 MEMBER enableYUV444 NOTIFY enableYUV444Changed) @@ -151,6 +160,7 @@ class StreamingPreferences : public QObject bool unlockBitrate; bool enableVsync; bool gameOptimizations; + bool spatialHeadTracking; bool playAudioOnHost; bool multiController; bool enableMdns; @@ -171,6 +181,7 @@ class StreamingPreferences : public QObject bool keepAwake; int packetSize; AudioConfig audioConfig; + SpatialAudioConfig spatialAudioConfig; VideoCodecConfig videoCodecConfig; bool enableHdr; bool enableYUV444; @@ -187,6 +198,7 @@ class StreamingPreferences : public QObject void unlockBitrateChanged(); void enableVsyncChanged(); void gameOptimizationsChanged(); + void spatialHeadTrackingChanged(); void playAudioOnHostChanged(); void multiControllerChanged(); void unsupportedFpsChanged(); @@ -195,6 +207,7 @@ class StreamingPreferences : public QObject void absoluteMouseModeChanged(); void absoluteTouchModeChanged(); void audioConfigChanged(); + void spatialAudioConfigChanged(); void videoCodecConfigChanged(); void enableHdrChanged(); void enableYUV444Changed(); diff --git a/app/streaming/audio/renderers/coreaudio/au_spatial_renderer.mm b/app/streaming/audio/renderers/coreaudio/au_spatial_renderer.mm index 15d58b88a..916e86344 100644 --- a/app/streaming/audio/renderers/coreaudio/au_spatial_renderer.mm +++ b/app/streaming/audio/renderers/coreaudio/au_spatial_renderer.mm @@ -1,6 +1,7 @@ #import "au_spatial_renderer.h" #import "coreaudio_helpers.h" #import "AllocatedAudioBufferList.h" +#include "settings/streamingpreferences.h" #import #import @@ -211,12 +212,14 @@ OSStatus inputCallback(void *inRefCon, if (outputType == kSpatialMixerOutputType_Headphones) { // XXX: Both of these might require the builder to have a paid Apple developer account due to the use of entitlements. - // XXX Head-tracking doesn't work well currently, with audio glitches on my system. It's off by default pending a config option. // For devices that support it, enable head-tracking. // Apps that use low-latency head-tracking in iOS/tvOS need to set // the audio session category to ambient or run in Game Mode. // Head tracking requires the entitlement com.apple.developer.coremotion.head-pose. - if (0) { + + // XXX Head-tracking may cause audio glitches. It's off by default. + StreamingPreferences *prefs = StreamingPreferences::get(); + if (prefs->spatialHeadTracking) { uint32_t ht = 1; status = AudioUnitSetProperty(m_Mixer, kAudioUnitProperty_SpatialMixerEnableHeadTracking, kAudioUnitScope_Global, 0, &ht, sizeof(uint32_t)); if (status != noErr) { diff --git a/app/streaming/audio/renderers/coreaudio/coreaudio.cpp b/app/streaming/audio/renderers/coreaudio/coreaudio.cpp index d24c9c9b3..99079afda 100644 --- a/app/streaming/audio/renderers/coreaudio/coreaudio.cpp +++ b/app/streaming/audio/renderers/coreaudio/coreaudio.cpp @@ -1,5 +1,6 @@ #include "coreaudio.h" #include "coreaudio_helpers.h" +#include "settings/streamingpreferences.h" #if TARGET_OS_OSX #include @@ -151,8 +152,8 @@ void CoreAudioRenderer::stringifyAudioStats(AUDIO_STATS& stats, char *output, in snprintf( output, length, "Audio stream: %s-channel Opus low-delay @ 48 kHz\n" - "Output device: %s @ %.1f kHz, %u-channel\n" - "Render mode: %s, device type: %s %s, %s\n" + "Output device: %s @ %.1f kHz, %u-channel, %s %s\n" + "Render mode: %s %s\n" "Opus config: %s, frame size: %.1fms, bitrate: %dkbps\n" "Buffer: length: %0.1f packets (%.1fms), current %.1f%%\n" "Packet loss from network: %.2f%%, packets dropped due to resync: %.2f%%\n" @@ -163,8 +164,6 @@ void CoreAudioRenderer::stringifyAudioStats(AUDIO_STATS& stats, char *output, in m_OutputDeviceName, m_OutputASBD.mSampleRate / 1000.0, m_OutputASBD.mChannelsPerFrame, - - m_Spatial ? (m_SpatialAU.m_PersonalizedHRTF ? "personalized spatial" : "spatial") : "passthrough", !strcmp(m_OutputTransportType, "blue") ? "Bluetooth" : !strcmp(m_OutputTransportType, "bltn") ? "built-in" : !strcmp(m_OutputTransportType, "usb ") ? "USB" @@ -175,7 +174,9 @@ void CoreAudioRenderer::stringifyAudioStats(AUDIO_STATS& stats, char *output, in : !strcmp(m_OutputDataSource, "ispk") ? "internal speakers" : !strcmp(m_OutputDataSource, "espk") ? "external speakers" : m_OutputDataSource, - m_Spatial && m_SpatialAU.m_HeadTracking ? "head-tracking: yes" : "", + + m_Spatial ? (m_SpatialAU.m_PersonalizedHRTF ? "personalized spatial" : "spatial") : "passthrough", + m_Spatial && m_SpatialAU.m_HeadTracking ? "with head-tracking" : "", // Work out if we're getting high or low quality from Sunshine. coupled surround is designed for physical speakers ((m_opusConfig->channelCount == 2 && stats.opusBitsPerSec > 128) || !m_opusConfig->coupledStreams) @@ -300,13 +301,19 @@ bool CoreAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* DEBUG_TRACE("CoreAudioRenderer getSpatialMixerOutputType = %d", outputType); - // XXX: let the user choose this if they want if (opusConfig->channelCount > 2) { if (outputType != kSpatialMixerOutputType_ExternalSpeakers) { m_Spatial = true; } } + StreamingPreferences *prefs = StreamingPreferences::get(); + if (prefs->spatialAudioConfig == StreamingPreferences::SAC_DISABLED) { + // User has disabled spatial audio + DEBUG_TRACE("CoreAudioRenderer user has disabled spatial audio"); + m_Spatial = false; + } + // indicate the format our callback will provide samples in // If necessary, the OS takes care of resampling (but not downmixing, hmm) AudioStreamBasicDescription streamDesc;