From 1949fbae94d1cc36e309772b56c260417072a0ab Mon Sep 17 00:00:00 2001 From: mii443 Date: Thu, 15 Feb 2024 21:26:09 +0900 Subject: [PATCH] Virtual device works! but why? --- Cargo.lock | 149 ++++++++++++++++++++++++++++ Cargo.toml | 3 +- local_test.yaml | 2 +- src/audio/resampling.rs | 60 +++++++++-- src/commands/run.rs | 186 ++++++++++++++++++++--------------- src/device/virtual_device.rs | 13 ++- src/main.rs | 9 +- 7 files changed, 326 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1654554..6683544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "1.1.2" @@ -100,6 +115,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bindgen" version = "0.69.4" @@ -324,6 +354,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + [[package]] name = "glob" version = "0.3.1" @@ -342,6 +378,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" + [[package]] name = "home" version = "0.5.9" @@ -499,6 +541,7 @@ dependencies = [ "rustfft", "serde", "serde_yaml", + "tokio", ] [[package]] @@ -513,6 +556,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "ndk" version = "0.7.0" @@ -683,6 +746,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "num_enum" version = "0.5.11" @@ -725,6 +798,15 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + [[package]] name = "oboe" version = "0.5.0" @@ -786,6 +868,12 @@ dependencies = [ "nom", ] +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + [[package]] name = "pkg-config" version = "0.3.29" @@ -904,6 +992,12 @@ dependencies = [ "realfft", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -998,12 +1092,31 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "smallvec" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +[[package]] +name = "socket2" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "strength_reduce" version = "0.2.4" @@ -1058,6 +1171,36 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "toml_datetime" version = "0.6.5" @@ -1119,6 +1262,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.91" diff --git a/Cargo.toml b/Cargo.toml index 96db293..b28cb82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,4 +13,5 @@ num-traits = "0.2.18" rustfft = "6.2.0" serde = { version = "1.0.196", features = ["derive"] } serde_yaml = "0.9.31" -rubato = "0.14.1" \ No newline at end of file +rubato = "0.14.1" +tokio = { version = "1", features = ["full"] } \ No newline at end of file diff --git a/local_test.yaml b/local_test.yaml index 843db8a..325ccff 100644 --- a/local_test.yaml +++ b/local_test.yaml @@ -1,7 +1,7 @@ virtual_devices: - name: "mic" channels: 2 - sample_rate: 96000 + sample_rate: 192000 routes: input: - name: "Mic" diff --git a/src/audio/resampling.rs b/src/audio/resampling.rs index be51989..f7312c3 100644 --- a/src/audio/resampling.rs +++ b/src/audio/resampling.rs @@ -1,4 +1,6 @@ +use num::Complex; use rubato::{FastFixedIn, PolynomialDegree, Resampler}; +use rustfft::FftPlanner; #[inline] pub fn resampling( @@ -6,13 +8,53 @@ pub fn resampling( target_sample_rate: u32, data: Vec>, ) -> Vec> { - let mut resampler = FastFixedIn::::new( - current_sample_rate as f64 / target_sample_rate as f64, - 2.0, - PolynomialDegree::Cubic, - data[0].len(), - data.len(), - ) - .unwrap(); - resampler.process(&data, None).unwrap() + let mut resampled_data = Vec::new(); + + let mut planner = FftPlanner::new(); + + for channel_data in data { + let mut complex_data: Vec> = + channel_data.iter().map(|&x| Complex::new(x, 0.0)).collect(); + + let fft = planner.plan_fft_forward(complex_data.len()); + fft.process(&mut complex_data); + + let adjustment_factor = target_sample_rate as f64 / current_sample_rate as f64; + let new_size = (complex_data.len() as f64 * adjustment_factor).round() as usize; + let mut adjusted_complex_data; + + if adjustment_factor == 1.0 { + adjusted_complex_data = complex_data; + } else if adjustment_factor > 1.0 { + adjusted_complex_data = vec![Complex::new(0.0, 0.0); new_size]; + let step = complex_data.len() as f64 / new_size as f64; + for (i, sample) in adjusted_complex_data.iter_mut().enumerate() { + let idx = (i as f64 * step).floor() as usize; + *sample = complex_data + .get(idx) + .cloned() + .unwrap_or(Complex::new(0.0, 0.0)); + } + } else { + adjusted_complex_data = complex_data + .into_iter() + .step_by(adjustment_factor.recip().round() as usize) + .collect(); + adjusted_complex_data.resize(new_size, Complex::new(0.0, 0.0)); + } + + let ifft = planner.plan_fft_inverse(adjusted_complex_data.len()); + ifft.process(&mut adjusted_complex_data); + + let len = adjusted_complex_data.len(); + + let resampled_channel: Vec = adjusted_complex_data + .into_iter() + .map(|x| x.re / len as f32) + .collect(); + + resampled_data.push(resampled_channel); + } + + resampled_data } diff --git a/src/commands/run.rs b/src/commands/run.rs index 6cdaba4..ec79c6e 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -6,7 +6,7 @@ use std::{ use cpal::{ traits::{DeviceTrait, HostTrait, StreamTrait}, - Device, + Device, Stream, StreamConfig, SupportedStreamConfig, }; use crate::{args::Run, device::virtual_device::VirtualDevice}; @@ -239,7 +239,7 @@ pub fn run(run: Run) { } } - std::thread::sleep(std::time::Duration::from_millis(500)); + let mut threads = vec![]; for output_route in &config.routes.output { match &output_route.device { @@ -250,7 +250,6 @@ pub fn run(run: Run) { .unwrap(); let config = device.default_output_config().unwrap(); - let channels = config.channels(); let sample_rate = config.sample_rate().0; let virtual_device = virtual_devices @@ -261,82 +260,23 @@ pub fn run(run: Run) { .cloned() .collect::>()[0] .clone(); - let index = virtual_device.lock().unwrap().add_output(sample_rate); - let stream = match config.sample_format() { - cpal::SampleFormat::I8 => device.build_output_stream( - &config.into(), - { - move |data: &mut [i8], _| { - output_callback( - data, - channels.into(), - sample_rate, - virtual_device.clone(), - index, - ) - } - }, - move |err| eprintln!("An error occurred on the output stream: {}", err), - None, - ), - cpal::SampleFormat::I16 => device.build_output_stream( - &config.into(), - { - move |data: &mut [i16], _| { - output_callback( - data, - channels.into(), - sample_rate, - virtual_device.clone(), - index, - ) - } - }, - move |err| eprintln!("An error occurred on the output stream: {}", err), - None, - ), - cpal::SampleFormat::I32 => device.build_output_stream( - &config.into(), - { - move |data: &mut [i32], _| { - output_callback( - data, - channels.into(), - sample_rate, - virtual_device.clone(), - index, - ) - } - }, - move |err| eprintln!("An error occurred on the output stream: {}", err), - None, - ), - cpal::SampleFormat::F32 => device.build_output_stream( - &config.into(), - { - move |data: &mut [f32], _| { - output_callback( - data, - channels.into(), - sample_rate, - virtual_device.clone(), - index, - ) - } - }, - move |err| eprintln!("An error occurred on the output stream: {}", err), - None, - ), - sample_format => { - eprintln!("Unsupported sample format: {:?}", sample_format); - return; + let thread = std::thread::spawn({ + let device = device.clone(); + move || { + let stream = create_virtual_device_input_stream( + config, + device.clone(), + virtual_device, + ); + stream.play().unwrap(); + loop { + std::thread::sleep(std::time::Duration::from_secs(1000)); + } } - } - .unwrap(); + }); - stream.play().unwrap(); - streams.push(stream); + threads.push(thread); } crate::config::Device::Remote { remote } => { unimplemented!(); @@ -349,6 +289,87 @@ pub fn run(run: Run) { } } +fn create_virtual_device_input_stream( + config: SupportedStreamConfig, + device: Device, + virtual_device: Arc>, +) -> Stream { + let sample_rate = config.sample_rate().0; + let channels = config.channels(); + let index = virtual_device.lock().unwrap().add_output(sample_rate); + + match config.sample_format() { + cpal::SampleFormat::I8 => device.build_output_stream( + &config.into(), + { + move |data: &mut [i8], _| { + output_callback( + data, + channels.into(), + sample_rate, + virtual_device.clone(), + index, + ) + } + }, + move |err| eprintln!("An error occurred on the output stream: {}", err), + None, + ), + cpal::SampleFormat::I16 => device.build_output_stream( + &config.into(), + { + move |data: &mut [i16], _| { + output_callback( + data, + channels.into(), + sample_rate, + virtual_device.clone(), + index, + ) + } + }, + move |err| eprintln!("An error occurred on the output stream: {}", err), + None, + ), + cpal::SampleFormat::I32 => device.build_output_stream( + &config.into(), + { + move |data: &mut [i32], _| { + output_callback( + data, + channels.into(), + sample_rate, + virtual_device.clone(), + index, + ) + } + }, + move |err| eprintln!("An error occurred on the output stream: {}", err), + None, + ), + cpal::SampleFormat::F32 => device.build_output_stream( + &config.into(), + { + move |data: &mut [f32], _| { + output_callback( + data, + channels.into(), + sample_rate, + virtual_device.clone(), + index, + ) + } + }, + move |err| eprintln!("An error occurred on the output stream: {}", err), + None, + ), + sample_format => { + panic!("Unsupported sample format: {:?}", sample_format); + } + } + .unwrap() +} + fn input_callback( data: &[T], channels: usize, @@ -376,12 +397,23 @@ fn output_callback( { let mut virtual_device = virtual_device.lock().unwrap(); let vd_channels = virtual_device.channels; - let audio_data = virtual_device.take_output( + let mut audio_data = virtual_device.take_output( index, min(channels as u8, vd_channels), sample_rate, data.len() / channels, ); + let mut count = 0; + while audio_data.is_none() && count < 1 { + audio_data = virtual_device.take_output( + index, + min(channels as u8, vd_channels), + sample_rate, + data.len() / channels, + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + count += 1; + } if audio_data.is_none() { println!("audio_data is none"); return; diff --git a/src/device/virtual_device.rs b/src/device/virtual_device.rs index 2d76634..12ddb7f 100644 --- a/src/device/virtual_device.rs +++ b/src/device/virtual_device.rs @@ -9,6 +9,7 @@ pub struct VirtualDevice { output_index: HashMap>, output_buffer: HashMap>>, + simple_buffer: Vec>, } impl VirtualDevice { @@ -19,6 +20,7 @@ impl VirtualDevice { sample_rate, output_index: HashMap::new(), output_buffer: HashMap::new(), + simple_buffer: vec![vec![]; channels as usize], } } @@ -53,10 +55,13 @@ impl VirtualDevice { let end = start + take_size; for channel in 0..channels { - if end > self.output_buffer[&sample_rate][channel as usize].len() { - return None; - } - if start >= self.output_buffer[&sample_rate][channel as usize].len() { + if end >= self.output_buffer[&sample_rate][channel as usize].len() { + println!( + "End of buffer: {}, {}[{}]", + end, + self.output_buffer[&sample_rate][channel as usize].len(), + channel as usize + ); return None; } } diff --git a/src/main.rs b/src/main.rs index 62e5e6e..758d708 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,15 @@ mod args; -mod config; -mod commands; -mod device; mod audio; +mod commands; +mod config; +mod device; use clap::Parser; use crate::args::Cli; -fn main() { +#[tokio::main] +async fn main() { let cli = Cli::parse(); match cli.command {