signals are cool

.gitignore

@@ -1,4 +1,5 @@
 */dist/
+*/pkg/
 */target/
 .DS_Store
 /target/

Cargo.toml

@@ -12,4 +12,5 @@ exclude = [
     "musical-leptos-0",
     "musical-stm32",
     "musical-terminal",
+    "musical-wasm",
 ]

musical-leptos/Cargo.toml

@@ -17,13 +17,13 @@ leptos = { version = "0.6", features = ["csr", "nightly"] }
 leptos_meta = { version = "0.6", features = ["csr", "nightly"] }
 leptos_router = { version = "0.6", features = ["csr", "nightly"] }
 log = "0.4"
-serde = { version = "1.0.198", default-features = false, features = ["derive"] }
 terrors = "0.3.0"
 wasm-bindgen = "0.2"
 wasm-bindgen-futures = "0.4.42"
-web-sys = { version = "0.3", features = ["console", "AudioContext", "AudioContextOptions", "AudioDestinationNode", "AudioWorklet", "AudioWorkletNode", "AudioWorkletNodeOptions", "BaseAudioContext", "BlobPropertyBag", "Document", "MediaDevices", "MediaStream", "MediaStreamConstraints", "MediaStreamAudioSourceNode", "Navigator", "TextDecoder", "Window"] }
+web-sys = { version = "0.3", features = ["console", "AudioContext", "AudioContextOptions", "AudioDestinationNode", "AudioWorklet", "AudioWorkletNode", "AudioWorkletNodeOptions", "BaseAudioContext", "BlobPropertyBag", "Document", "MediaDevices", "MediaStream", "MediaStreamConstraints", "MediaStreamAudioSourceNode", "MessagePort", "Navigator", "TextDecoder", "Window"] }
 
 #leptos_workers = "0.2.1"
+#serde = { version = "1.0.198", default-features = false, features = ["derive"] }
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3"

musical-leptos/rust-toolchain.toml

@@ -0,0 +1 @@
+../rust-toolchain.toml

musical-leptos/src/components/dancing_lights.rs

@@ -9,7 +9,7 @@ use musical_lights_core::{
     windows::HanningWindow,
 };
 use wasm_bindgen::{closure::Closure, JsCast, JsValue};
-use web_sys::{AudioContext, HtmlAudioElement, MediaStream, MediaStreamConstraints};
+use web_sys::{AudioContext, HtmlAudioElement, MediaStream, MediaStreamConstraints, MessageEvent};
 
 use crate::wasm_audio::wasm_audio;
 
@@ -44,8 +44,10 @@ pub fn DancingLights() -> impl IntoView {
     // TODO: do this on button click
     let (listen, set_listen) = create_signal(false);
 
+    let (audio, set_audio) = create_signal(0.0);
+
     // TODO: this is wrong. this runs immediatly, not on first click. why?
-    let start_listening = create_resource(listen, |x| async move {
+    let start_listening = create_resource(listen, move |x| async move {
         if !x {
             return Ok(None);
         }
@@ -58,13 +60,20 @@ pub fn DancingLights() -> impl IntoView {
 
         info!("active media stream: {:?}", media_stream_id);
 
-        let audio_ctx = wasm_audio(&media_stream)
+        let onmessage_callback = Closure::new(move |x: MessageEvent| {
+            // TODO: this seems fragile. how do we be sure of the data type
+            // TODO: this will actually be a vec of 120 f32s when we are done
+            let data = x.data().as_f64().unwrap();
+            set_audio(data);
+        });
+
+        let audio_ctx = wasm_audio(&media_stream, onmessage_callback)
             .await
             .map_err(|x| format!("audio_ctx error: {:?}", x))?;
 
         info!("audio context: {:?}", audio_ctx);
 
-        // // TODO: do we need this?
+        // // TODO: do we need this? does it need to be spawned?
         // let promise = audio_ctx.resume().unwrap();
         // let _ = wasm_bindgen_futures::JsFuture::from(promise).await.unwrap();
 
@@ -94,7 +103,10 @@ pub fn DancingLights() -> impl IntoView {
                     }
                 >
                     Now listening to {media_stream_id}
-                </button> }.into_view(),
+                </button>
+
+                <pre>{audio}</pre>
+            }.into_view(),
             Some(Err(err)) => view! { <div>Error: {err}</div> }.into_view(),
         }}
 

musical-leptos/src/my-wasm-processor.js

@@ -1,3 +1,4 @@
+// TODO: should we have a minimal shim here? maybe fetch and instantiate a second wasm app here?
 
 class MyWasmProcessor extends AudioWorkletProcessor {
     constructor(options) {
@@ -9,27 +10,29 @@ class MyWasmProcessor extends AudioWorkletProcessor {
 
         let [module, foobar] = options.processorOptions;
 
-        // TODO: this is wrong. i think we need a dedicated wasm module for the processor
-        WebAssembly.instantiate(module)
-            .then(obj => {
-                this.wasmInstance = obj.instance;
-                console.log('WASM loaded in worklet');
-            })
-            .catch(err => console.error('Error instantiating WASM module in worklet:', err));
+        // // TODO: this is wrong. i think we need a dedicated wasm module for the processor
+        // WebAssembly.instantiate(module)
+        //     .then(obj => {
+        //         this.wasmInstance = obj.instance;
+        //         console.log('WASM loaded in worklet');
+        //     })
+        //     .catch(err => console.error('Error instantiating WASM module in worklet:', err));
     }
 
     process(inputs, outputs, parameters) {
         if (this.wasmInstance) {
             // TODO: Call your WASM functions here to process audio. Then send it over this.port.postMessage()
         } else {
             let sum = inputs[0][0].reduce((acc, val) => acc + val, 0);
-            console.log("sum:", sum);
+            // console.log("sum:", sum);
+
+            this.port.postMessage(sum);
         }
 
         // browsers all handle this differently
         // chrome, return true or it stops immediatly
         // firefox, return true or it stops when there is no more input
-        // false SHOULD be fine, but 
+        // false SHOULD be fine, but no...
         return true;
     }
 }

musical-leptos/src/wasm_audio.rs

@@ -1,17 +1,20 @@
 use crate::dependent_module;
 
-use log::debug;
-use serde::{Deserialize, Serialize};
-use wasm_bindgen::JsValue;
+use log::{debug, info};
+use wasm_bindgen::closure::Closure;
+use wasm_bindgen::{JsCast, JsValue};
 use wasm_bindgen_futures::JsFuture;
-use web_sys::MediaStream;
 use web_sys::{AudioContext, AudioWorkletNode, AudioWorkletNodeOptions};
+use web_sys::{MediaStream, MessageEvent};
 
 // Use wasm_audio if you have a single wasm audio processor in your application
 // whose samples should be played directly. Ideally, call wasm_audio based on
 // user interaction. Otherwise, resume the context on user interaction, so
 // playback starts reliably on all browsers.
-pub async fn wasm_audio(media_stream: &MediaStream) -> Result<AudioContext, JsValue> {
+pub async fn wasm_audio(
+    media_stream: &MediaStream,
+    onmessage_callback: Closure<dyn FnMut(MessageEvent)>,
+) -> Result<AudioContext, JsValue> {
     let ctx = AudioContext::new()?;
 
     prepare_wasm_audio(&ctx).await?;
@@ -27,7 +30,14 @@ pub async fn wasm_audio(media_stream: &MediaStream) -> Result<AudioContext, JsVa
 
     worklet.connect_with_audio_node(&ctx.destination())?;
 
-    // TODO: i think we need to do something with worklet.port here to get the handled audio samples out
+    let port = worklet.port().unwrap();
+
+    port.set_onmessage(Some(onmessage_callback.as_ref().unchecked_ref()));
+
+    Closure::forget(onmessage_callback);
+
+    // TODO: what should we do with errors?
+    // port.set_onmessageerror(onmessageerror_callback);
 
     debug!("audio input: {:?}", input);
     debug!("audio node: {:?}", worklet);