add audio api for sound loading and playback

app/Cargo.toml

@@ -56,6 +56,7 @@ clap = { workspace = true }
 convert_case = { workspace = true }
 env_logger = { workspace = true }
 futures = { workspace = true }
+flume = { workspace = true }
 glam = { workspace = true }
 local-ip-address = { workspace = true }
 log = { workspace = true }

app/src/client/wasm.rs

@@ -2,6 +2,9 @@ use std::sync::Arc;
 
 use ambient_ecs::{EntityId, SystemGroup, World};
 use ambient_wasm::shared::{get_module_name, MessageType};
+use flume::{Receiver, Sender};
+use ambient_world_audio::{AudioMessage, audio_sender};
+use ambient_audio::Source;
 
 pub fn systems() -> SystemGroup {
     ambient_wasm::client::systems()
@@ -21,6 +24,29 @@ pub fn initialize(world: &mut World) -> anyhow::Result<()> {
         log::log!(level, "[{name}] {prefix}: {}", message.strip_suffix('\n').unwrap_or(message));
     });
 
+    let (tx, rx): (Sender<AudioMessage>, Receiver<AudioMessage>) = flume::unbounded();
+
+    std::thread::spawn(move || {
+        let stream = ambient_audio::AudioStream::new().unwrap();
+        while let Ok(message) = rx.recv() {
+            match message {
+                AudioMessage::Track(t, looping, amp) => {
+                    match looping {
+                        true => {
+                            let sound = stream.mixer().play(t.decode().repeat().gain(amp.clamp(0.0, 1.0)));
+                            sound.wait();
+                        }
+                        false => {
+                            let sound = stream.mixer().play(t.decode().gain(amp.clamp(0.0, 1.0)));
+                            sound.wait();
+                        }
+                    }
+                }
+            }
+        }
+    });
+    world.add_resource(audio_sender(), Arc::new(tx));
+
     ambient_wasm::client::initialize(world, messenger)?;
 
     Ok(())

crates/wasm/Cargo.toml

@@ -7,6 +7,8 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+ambient_world_audio = { path = "../world_audio" }
+ambient_audio = { path = "../audio" }
 ambient_sys = { path = "../sys" }
 ambient_animation = { path = "../animation" }
 ambient_app = { path = "../app" }

crates/wasm/src/client/implementation.rs

@@ -72,3 +72,13 @@ impl wit::asset::Host for Bindings {
         asset_url.to_download_url(&assets).map(|url| Some(url.to_string()))
     }
 }
+
+impl wit::audio::Host for Bindings {
+    fn load(&mut self, url: String) -> anyhow::Result<()> {
+        crate::shared::implementation::audio::load(self.world_mut(), url)
+    }
+
+    fn play(&mut self, name: String, looping: bool, amp: f32) -> anyhow::Result<()> {
+        crate::shared::implementation::audio::play(self.world_mut(), name, looping, amp)
+    }
+}

crates/wasm/src/server/implementation.rs

@@ -281,3 +281,13 @@ impl wit::asset::Host for Bindings {
         Ok(Some(AbsAssetUrl::from_asset_key(path).to_string()))
     }
 }
+
+impl wit::audio::Host for Bindings {
+    fn load(&mut self, url: String) -> anyhow::Result<()> {
+        crate::shared::implementation::audio::load(self.world_mut(), url)
+    }
+
+    fn play(&mut self, name: String, looping: bool, amp: f32) -> anyhow::Result<()> {
+        crate::shared::implementation::audio::play(self.world_mut(), name, looping, amp)
+    }
+}

crates/wasm/src/shared/bindings.rs

@@ -16,6 +16,7 @@ pub struct BindingsBase {
 
 pub trait BindingsBound:
     wit::types::Host
+    + wit::audio::Host
     + wit::asset::Host
     + wit::component::Host
     + wit::entity::Host

crates/wasm/src/shared/implementation/audio.rs

@@ -0,0 +1,36 @@
+use ambient_std::asset_url::{AbsAssetUrl}; // AssetUrl
+use ambient_std::asset_cache::{AsyncAssetKeyExt};
+use ambient_audio::{AudioFromUrl}; //  track::TrackDecodeStream, Source
+use ambient_ecs::{World};
+use ambient_core::{asset_cache, async_ecs::async_run, runtime};
+use ambient_world_audio::{audio_sender, AudioMessage}; // audio_tracks,
+use anyhow::Context;
+
+pub(crate) fn load(world: &World, url: String) -> anyhow::Result<()> {
+    let assets = world.resource(asset_cache()).clone();
+    let asset_url = AbsAssetUrl::from_asset_key(url).to_string();
+    let audio_url = AudioFromUrl { url: AbsAssetUrl::parse(asset_url).context("Failed to parse audio url")? };
+    let _track = audio_url.peek(&assets);
+    Ok(())
+}
+
+pub(crate) fn play(world: &World, url: String, looping: bool, amp: f32) -> anyhow::Result<()> {
+    let assets = world.resource(asset_cache()).clone();
+    let asset_url = AbsAssetUrl::from_asset_key(url).to_string();
+    let audio_url = AudioFromUrl { url: AbsAssetUrl::parse(asset_url).context("Failed to parse audio url")? };
+    let runtime = world.resource(runtime()).clone();
+    let async_run = world.resource(async_run()).clone();
+    runtime.spawn(async move {
+        let track = audio_url.get(&assets).await;
+        async_run.run(move |world| {
+            match track {
+                Ok(track) => {
+                    let sender = world.resource(audio_sender());
+                    sender.send(AudioMessage::Track(track, looping, amp)).unwrap();
+                },
+                Err(e) => log::error!("{e:?}")
+            };
+        });
+    });
+    Ok(())
+}

crates/wasm/src/shared/implementation/mod.rs

@@ -1,3 +1,4 @@
+pub mod audio;
 pub mod component;
 pub mod entity;
 pub mod message;

crates/wasm/wit/audio.wit

@@ -0,0 +1,4 @@
+default interface audio {
+    load: func(url: string)
+    play: func(name: string, looping: bool, amp: float32)
+}

crates/wasm/wit/main.wit

@@ -1,6 +1,8 @@
 world bindings {
     import types: pkg.types
 
+    import audio: pkg.audio
+    import asset: pkg.asset
     import component: pkg.component
     import entity: pkg.entity
     import message: pkg.message
@@ -9,7 +11,6 @@ world bindings {
     import client-player: pkg.client-player
 
     import server-physics: pkg.server-physics
-    import asset: pkg.asset
     import server-message: pkg.server-message
 
     export guest: pkg.guest

crates/world_audio/src/sounds.rs

@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use ambient_audio::{hrtf::HrtfLib, Attenuation, AudioEmitter, AudioListener, AudioMixer, Sound, Source};
+use ambient_audio::{hrtf::HrtfLib, Attenuation, AudioEmitter, AudioListener, AudioMixer, Source, Sound};
 use ambient_ecs::{components, query, EntityId, Resource, World};
 use ambient_element::ElementComponentExt;
 use ambient_std::{cb, Cb};
@@ -20,11 +20,16 @@ components!("audio", {
     hrtf_lib: Arc<HrtfLib>,
     audio_emitter: Arc<Mutex<AudioEmitter>>,
     audio_listener: Arc<Mutex<AudioListener>>,
-
+    @[Resource]
+    audio_sender: Arc<flume::Sender<AudioMessage>>,
     @[Resource]
     audio_mixer: AudioMixer,
 });
 
+pub enum AudioMessage {
+    Track(Arc<ambient_audio::track::Track>, bool, f32)
+}
+
 /// TODO: hook this into the Attenuation inside ambient_audio
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, DerefMut, Deref, From, Into)]
 pub struct AttenuationEditorVisual(Attenuation);
@@ -78,4 +83,4 @@ pub fn play_sound_on_entity<S: 'static + Source>(world: &World, id: EntityId, so
     let listener = get_audio_listener(world)?;
 
     Ok(mixer.play(source.spatial(hrtf_lib, listener.clone(), emitter.clone())))
-}
+}

crates/world_audio/src/systems.rs

@@ -13,9 +13,7 @@ use crate::{audio_emitter, audio_listener, audio_mixer, hrtf_lib};
 pub fn setup_audio(world: &mut World, mixer: AudioMixer) -> anyhow::Result<()> {
     let hrtf = Arc::new(HrtfLib::load(Cursor::new(include_bytes!("../IRC_1002_C.bin")))?);
     world.add_resource(hrtf_lib(), hrtf);
-
     world.add_resource(audio_mixer(), mixer);
-
     Ok(())
 }
 

guest/rust/api/src/audio.rs

@@ -0,0 +1,42 @@
+use crate::internal::wit;
+
+/// Add sound to the audio library in the world you call this
+pub fn load(url: String) -> AudioTrack {
+    wit::audio::load(&url);
+    AudioTrack {
+        name: url,
+        looping: false,
+        amp: 1.0,
+    }
+}
+
+/// The audio query, used to play audio
+#[derive(Clone, Debug)]
+pub struct AudioTrack {
+    /// The name of the audio
+    pub name: String,
+    /// Whether or not the audio should loop
+    pub looping: bool,
+    /// The volume of the audio
+    pub amp: f32,
+}
+
+impl AudioTrack {
+
+    /// Set whether or not the audio should loop
+    pub fn looping(&mut self, looping: bool) -> &mut Self {
+        self.looping = looping;
+        self
+    }
+
+    /// Set the volume of the audio
+    pub fn scale(&mut self, amp: f32) -> &mut Self {
+        self.amp = amp;
+        self
+    }
+
+    /// Play the audio
+    pub fn play(&mut self) {
+        wit::audio::play(&self.name, self.looping, self.amp);
+    }
+}