From 2afed487ed29565cf5363dd3f5e2d6e26f2894b8 Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Wed, 10 Nov 2021 10:59:59 +0800 Subject: [PATCH 01/33] slow compile repro --- Cargo.lock | 74 ++++++++++++++++++++++++++++++- Cargo.toml | 1 + examples/compiler_singlepass.rs | 77 +++++++++++++++++++++++---------- 3 files changed, 128 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7abf5a543..d8cadc753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anyhow" version = "1.0.42" @@ -277,6 +286,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "winapi", +] + [[package]] name = "clang-sys" version = "1.2.0" @@ -294,7 +315,7 @@ version = "2.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" dependencies = [ - "ansi_term", + "ansi_term 0.11.0", "atty", "bitflags", "strsim 0.8.0", @@ -1337,6 +1358,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -1541,7 +1572,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" dependencies = [ - "ansi_term", + "ansi_term 0.11.0", "ctor", "difference", "output_vt100", @@ -2385,19 +2416,57 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-log" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-span-tree" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877726b1570d7764022ef20cc61479b5bcfc1118b90521ce61f6cc9e4f5ffbd8" +dependencies = [ + "tracing", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab69019741fca4d98be3c62d2b75254528b5432233fd8a4d2739fec20278de48" dependencies = [ + "ansi_term 0.12.1", + "chrono", "lazy_static", "matchers", "regex", + "serde", + "serde_json", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -3136,6 +3205,7 @@ dependencies = [ "test-env-log", "test-generator", "tracing", + "tracing-span-tree", "tracing-subscriber", "wasi-test-generator", "wasmer", diff --git a/Cargo.toml b/Cargo.toml index 44449c894..5878b2019 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ wasmer-cache = { version = "2.0.0", path = "lib/cache", optional = true } wasmer-types = { version = "2.0.0", path = "lib/types" } wasmer-middlewares = { version = "2.0.0", path = "lib/middlewares", optional = true } cfg-if = "1.0" +tracing-span-tree = "0.1" [workspace] members = [ diff --git a/examples/compiler_singlepass.rs b/examples/compiler_singlepass.rs index 845b9db30..fe93c2c9f 100644 --- a/examples/compiler_singlepass.rs +++ b/examples/compiler_singlepass.rs @@ -10,24 +10,40 @@ //! //! Ready? +use std::fmt::Write; use wasmer::{imports, wat2wasm, Instance, Module, Store, Value}; use wasmer_compiler_singlepass::Singlepass; use wasmer_engine_universal::Universal; +pub fn many_functions_contract(function_count: u32) -> Vec { + let mut functions = String::new(); + for i in 0..function_count { + writeln!( + &mut functions, + "(func + i32.const {} + drop + return)", + i + ) + .unwrap(); + } + + let code = format!( + r#"(module + (export "main" (func 0)) + {})"#, + functions + ); + wat2wasm(code.as_bytes()).unwrap().to_vec() +} + fn main() -> Result<(), Box> { + tracing_span_tree::span_tree().aggregate(true).enable(); + // Let's declare the Wasm module with the text representation. - let wasm_bytes = wat2wasm( - r#" -(module - (type $sum_t (func (param i32 i32) (result i32))) - (func $sum_f (type $sum_t) (param $x i32) (param $y i32) (result i32) - local.get $x - local.get $y - i32.add) - (export "sum" (func $sum_f))) -"# - .as_bytes(), - )?; + let wasm_bytes = many_functions_contract(150_000); + println!("code.len() = {:?}", wasm_bytes.len() / 1024); // Use Singlepass compiler with the default settings let compiler = Singlepass::default(); @@ -37,24 +53,41 @@ fn main() -> Result<(), Box> { println!("Compiling module..."); // Let's compile the Wasm module. - let module = Module::new(&store, wasm_bytes)?; + // let guard = pprof::ProfilerGuard::new(100).unwrap(); + let module = { + let _span = tracing::debug_span!(target: "vm", "Module::new (compile)").entered(); + + Module::new(&store, wasm_bytes)? + }; + // if let Ok(report) = guard.report().build() { + // // println!("report: {:?}", &report); + // let file = std::fs::File::create("flamegraph2.svg").unwrap(); + // report.flamegraph(file).unwrap(); + // }; // Create an empty import object. let import_object = imports! {}; println!("Instantiating module..."); - // Let's instantiate the Wasm module. - let instance = Instance::new(&module, &import_object)?; + let instance = { + // Let's instantiate the Wasm module. + let _span = tracing::debug_span!(target: "vm", "Instance::new").entered(); + Instance::new(&module, &import_object)? + }; - let sum = instance.exports.get_function("sum")?; + println!("Instantiating module... the second time"); + let instance = { + // This one matches NEAR's execution model of initialization + let _span = tracing::debug_span!(target: "vm", "Instance::new").entered(); + Instance::new(&module, &import_object)? + }; + let main = instance.exports.get_function("main")?; - println!("Calling `sum` function..."); - // Let's call the `sum` exported function. The parameters are a - // slice of `Value`s. The results are a boxed slice of `Value`s. - let results = sum.call(&[Value::I32(1), Value::I32(2)])?; + println!("Calling `main` function..."); + let results = main.call(&[])?; println!("Results: {:?}", results); - assert_eq!(results.to_vec(), vec![Value::I32(3)]); + // assert_eq!(results.to_vec(), vec![Value::I32(3)]); Ok(()) } @@ -63,4 +96,4 @@ fn main() -> Result<(), Box> { #[cfg(feature = "singlepass")] fn test_compiler_singlepass() -> Result<(), Box> { main() -} +} \ No newline at end of file From a925e0a67496c5f8fc2ff25f48bb639a334b1b2d Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Wed, 10 Nov 2021 11:00:14 +0800 Subject: [PATCH 02/33] fix slow compilation by using dynasm VecAssembler --- lib/compiler-singlepass/src/codegen_x64.rs | 12 +++++++----- lib/compiler-singlepass/src/emitter_x64.rs | 4 +++- lib/compiler-singlepass/src/machine.rs | 4 +++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/compiler-singlepass/src/codegen_x64.rs b/lib/compiler-singlepass/src/codegen_x64.rs index beaefaaf3..d44f3be56 100644 --- a/lib/compiler-singlepass/src/codegen_x64.rs +++ b/lib/compiler-singlepass/src/codegen_x64.rs @@ -1,6 +1,6 @@ use crate::address_map::get_function_address_map; use crate::{common_decl::*, config::Singlepass, emitter_x64::*, machine::Machine, x64_decl::*}; -use dynasmrt::{x64::Assembler, DynamicLabel}; +use dynasmrt::{x64::X64Relocation, DynamicLabel, VecAssembler}; use smallvec::{smallvec, SmallVec}; use std::collections::BTreeMap; use std::iter; @@ -22,6 +22,8 @@ use wasmer_types::{ }; use wasmer_vm::{MemoryStyle, TableStyle, TrapCode, VMBuiltinFunctionIndex, VMOffsets}; +type Assembler = VecAssembler; + /// The singlepass per-function code generator. pub struct FuncGen<'a> { // Immutable properties assigned at creation time. @@ -1844,7 +1846,7 @@ impl<'a> FuncGen<'a> { .collect(), ); - let mut assembler = Assembler::new().unwrap(); + let mut assembler = Assembler::new(0); let special_labels = SpecialLabelSet { integer_division_by_zero: assembler.get_label(), heap_access_oob: assembler.get_label(), @@ -8811,7 +8813,7 @@ pub fn gen_std_trampoline( sig: &FunctionType, calling_convention: CallingConvention, ) -> FunctionBody { - let mut a = Assembler::new().unwrap(); + let mut a = Assembler::new(0); // Calculate stack offset. let mut stack_offset: u32 = 0; @@ -8921,7 +8923,7 @@ pub fn gen_std_dynamic_import_trampoline( sig: &FunctionType, calling_convention: CallingConvention, ) -> FunctionBody { - let mut a = Assembler::new().unwrap(); + let mut a = Assembler::new(0); // Allocate argument array. let stack_offset: usize = 16 * std::cmp::max(sig.params().len(), sig.results().len()) + 8; // 16 bytes each + 8 bytes sysv call padding @@ -9043,7 +9045,7 @@ pub fn gen_import_call_trampoline( sig: &FunctionType, calling_convention: CallingConvention, ) -> CustomSection { - let mut a = Assembler::new().unwrap(); + let mut a = Assembler::new(0); // TODO: ARM entry trampoline is not emitted. diff --git a/lib/compiler-singlepass/src/emitter_x64.rs b/lib/compiler-singlepass/src/emitter_x64.rs index 7bd3d7792..4a2d16011 100644 --- a/lib/compiler-singlepass/src/emitter_x64.rs +++ b/lib/compiler-singlepass/src/emitter_x64.rs @@ -1,6 +1,8 @@ pub use crate::x64_decl::{GPR, XMM}; use dynasm::dynasm; -use dynasmrt::{x64::Assembler, AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi}; +use dynasmrt::{x64::X64Relocation, AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler}; + +type Assembler = VecAssembler; /// Force `dynasm!` to use the correct arch (x64) when cross-compiling. /// `dynasm!` proc-macro tries to auto-detect it by default by looking at the diff --git a/lib/compiler-singlepass/src/machine.rs b/lib/compiler-singlepass/src/machine.rs index 73f75bc7b..045bb8387 100644 --- a/lib/compiler-singlepass/src/machine.rs +++ b/lib/compiler-singlepass/src/machine.rs @@ -586,7 +586,9 @@ impl Machine { #[cfg(test)] mod test { use super::*; - use dynasmrt::x64::Assembler; + use dynasmrt::x64::X64Relocation; + use dynasmrt::VecAssembler; + type Assembler = VecAssembler; #[test] fn test_release_locations_keep_state_nopanic() { From c37a873b65798f7968e2da03a6f382cb7e1965ef Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Mon, 15 Nov 2021 10:27:33 +0800 Subject: [PATCH 03/33] undo changes for bench --- Cargo.lock | 74 +------------------------------ Cargo.toml | 1 - examples/compiler_singlepass.rs | 77 ++++++++++----------------------- 3 files changed, 24 insertions(+), 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8cadc753..7abf5a543 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,15 +52,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "anyhow" version = "1.0.42" @@ -286,18 +277,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chrono" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" -dependencies = [ - "libc", - "num-integer", - "num-traits", - "winapi", -] - [[package]] name = "clang-sys" version = "1.2.0" @@ -315,7 +294,7 @@ version = "2.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" dependencies = [ - "ansi_term 0.11.0", + "ansi_term", "atty", "bitflags", "strsim 0.8.0", @@ -1358,16 +1337,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "num-integer" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.14" @@ -1572,7 +1541,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427" dependencies = [ - "ansi_term 0.11.0", + "ansi_term", "ctor", "difference", "output_vt100", @@ -2416,57 +2385,19 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "tracing-log" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" -dependencies = [ - "lazy_static", - "log", - "tracing-core", -] - -[[package]] -name = "tracing-serde" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" -dependencies = [ - "serde", - "tracing-core", -] - -[[package]] -name = "tracing-span-tree" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877726b1570d7764022ef20cc61479b5bcfc1118b90521ce61f6cc9e4f5ffbd8" -dependencies = [ - "tracing", - "tracing-subscriber", -] - [[package]] name = "tracing-subscriber" version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab69019741fca4d98be3c62d2b75254528b5432233fd8a4d2739fec20278de48" dependencies = [ - "ansi_term 0.12.1", - "chrono", "lazy_static", "matchers", "regex", - "serde", - "serde_json", "sharded-slab", - "smallvec", "thread_local", "tracing", "tracing-core", - "tracing-log", - "tracing-serde", ] [[package]] @@ -3205,7 +3136,6 @@ dependencies = [ "test-env-log", "test-generator", "tracing", - "tracing-span-tree", "tracing-subscriber", "wasi-test-generator", "wasmer", diff --git a/Cargo.toml b/Cargo.toml index 5878b2019..44449c894 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,6 @@ wasmer-cache = { version = "2.0.0", path = "lib/cache", optional = true } wasmer-types = { version = "2.0.0", path = "lib/types" } wasmer-middlewares = { version = "2.0.0", path = "lib/middlewares", optional = true } cfg-if = "1.0" -tracing-span-tree = "0.1" [workspace] members = [ diff --git a/examples/compiler_singlepass.rs b/examples/compiler_singlepass.rs index fe93c2c9f..845b9db30 100644 --- a/examples/compiler_singlepass.rs +++ b/examples/compiler_singlepass.rs @@ -10,40 +10,24 @@ //! //! Ready? -use std::fmt::Write; use wasmer::{imports, wat2wasm, Instance, Module, Store, Value}; use wasmer_compiler_singlepass::Singlepass; use wasmer_engine_universal::Universal; -pub fn many_functions_contract(function_count: u32) -> Vec { - let mut functions = String::new(); - for i in 0..function_count { - writeln!( - &mut functions, - "(func - i32.const {} - drop - return)", - i - ) - .unwrap(); - } - - let code = format!( - r#"(module - (export "main" (func 0)) - {})"#, - functions - ); - wat2wasm(code.as_bytes()).unwrap().to_vec() -} - fn main() -> Result<(), Box> { - tracing_span_tree::span_tree().aggregate(true).enable(); - // Let's declare the Wasm module with the text representation. - let wasm_bytes = many_functions_contract(150_000); - println!("code.len() = {:?}", wasm_bytes.len() / 1024); + let wasm_bytes = wat2wasm( + r#" +(module + (type $sum_t (func (param i32 i32) (result i32))) + (func $sum_f (type $sum_t) (param $x i32) (param $y i32) (result i32) + local.get $x + local.get $y + i32.add) + (export "sum" (func $sum_f))) +"# + .as_bytes(), + )?; // Use Singlepass compiler with the default settings let compiler = Singlepass::default(); @@ -53,41 +37,24 @@ fn main() -> Result<(), Box> { println!("Compiling module..."); // Let's compile the Wasm module. - // let guard = pprof::ProfilerGuard::new(100).unwrap(); - let module = { - let _span = tracing::debug_span!(target: "vm", "Module::new (compile)").entered(); - - Module::new(&store, wasm_bytes)? - }; - // if let Ok(report) = guard.report().build() { - // // println!("report: {:?}", &report); - // let file = std::fs::File::create("flamegraph2.svg").unwrap(); - // report.flamegraph(file).unwrap(); - // }; + let module = Module::new(&store, wasm_bytes)?; // Create an empty import object. let import_object = imports! {}; println!("Instantiating module..."); - let instance = { - // Let's instantiate the Wasm module. - let _span = tracing::debug_span!(target: "vm", "Instance::new").entered(); - Instance::new(&module, &import_object)? - }; + // Let's instantiate the Wasm module. + let instance = Instance::new(&module, &import_object)?; - println!("Instantiating module... the second time"); - let instance = { - // This one matches NEAR's execution model of initialization - let _span = tracing::debug_span!(target: "vm", "Instance::new").entered(); - Instance::new(&module, &import_object)? - }; - let main = instance.exports.get_function("main")?; + let sum = instance.exports.get_function("sum")?; - println!("Calling `main` function..."); - let results = main.call(&[])?; + println!("Calling `sum` function..."); + // Let's call the `sum` exported function. The parameters are a + // slice of `Value`s. The results are a boxed slice of `Value`s. + let results = sum.call(&[Value::I32(1), Value::I32(2)])?; println!("Results: {:?}", results); - // assert_eq!(results.to_vec(), vec![Value::I32(3)]); + assert_eq!(results.to_vec(), vec![Value::I32(3)]); Ok(()) } @@ -96,4 +63,4 @@ fn main() -> Result<(), Box> { #[cfg(feature = "singlepass")] fn test_compiler_singlepass() -> Result<(), Box> { main() -} \ No newline at end of file +} From ff3b459e5b384b7c32ebd0d3964099dc641f8bb0 Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Mon, 15 Nov 2021 15:33:22 +0800 Subject: [PATCH 04/33] fix --- lib/compiler-singlepass/src/machine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler-singlepass/src/machine.rs b/lib/compiler-singlepass/src/machine.rs index 045bb8387..36f66b60e 100644 --- a/lib/compiler-singlepass/src/machine.rs +++ b/lib/compiler-singlepass/src/machine.rs @@ -593,7 +593,7 @@ mod test { #[test] fn test_release_locations_keep_state_nopanic() { let mut machine = Machine::new(); - let mut assembler = Assembler::new().unwrap(); + let mut assembler = Assembler::new(0); let locs = machine.acquire_locations( &mut assembler, &(0..10) From 88f1b19ff1db1593807b9b24abf5d6e51970bea5 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 15 Nov 2021 13:23:11 +0000 Subject: [PATCH 05/33] Revert "feat(c-api) Create `OrderedResolver` from a parallel iterator." This reverts commit 446a76312b168e240e99610e4e3857db8ba64bf5. --- lib/c-api/Cargo.toml | 1 - lib/c-api/src/ordered_resolver.rs | 15 ++++----------- lib/c-api/src/wasm_c_api/instance.rs | 5 ++--- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index 503b3925d..050f732c3 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -42,7 +42,6 @@ serde = { version = "1", optional = true, features = ["derive"] } thiserror = "1" typetag = { version = "0.1", optional = true } paste = "1.0" -rayon = "1.5" [dev-dependencies] field-offset = "0.3.3" diff --git a/lib/c-api/src/ordered_resolver.rs b/lib/c-api/src/ordered_resolver.rs index 847e3e9e8..adffaa35c 100644 --- a/lib/c-api/src/ordered_resolver.rs +++ b/lib/c-api/src/ordered_resolver.rs @@ -4,7 +4,6 @@ //! This resolver is used in the Wasm-C-API as the imports are provided //! by index and not by module and name. -use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator}; use std::iter::FromIterator; use wasmer_api::{Export, Exportable, Extern, Resolver}; @@ -26,16 +25,10 @@ impl Resolver for OrderedResolver { impl FromIterator for OrderedResolver { fn from_iter>(iter: I) -> Self { - OrderedResolver { - externs: iter.into_iter().collect(), - } - } -} - -impl FromParallelIterator for OrderedResolver { - fn from_par_iter>(iter: I) -> Self { - OrderedResolver { - externs: iter.into_par_iter().collect(), + let mut externs = Vec::new(); + for extern_ in iter { + externs.push(extern_); } + OrderedResolver { externs } } } diff --git a/lib/c-api/src/wasm_c_api/instance.rs b/lib/c-api/src/wasm_c_api/instance.rs index 50d929211..67b11d3e7 100644 --- a/lib/c-api/src/wasm_c_api/instance.rs +++ b/lib/c-api/src/wasm_c_api/instance.rs @@ -3,7 +3,6 @@ use super::module::wasm_module_t; use super::store::wasm_store_t; use super::trap::wasm_trap_t; use crate::ordered_resolver::OrderedResolver; -use rayon::prelude::*; use std::mem; use std::sync::Arc; use wasmer_api::{Extern, Instance, InstantiationError}; @@ -52,8 +51,8 @@ pub unsafe extern "C" fn wasm_instance_new( let module_import_count = module_imports.len(); let resolver: OrderedResolver = imports .into_slice() - .map(|imports| imports.par_iter()) - .unwrap_or_else(|| [].par_iter()) + .map(|imports| imports.iter()) + .unwrap_or_else(|| [].iter()) .map(|imp| Extern::from((&**imp).clone())) .take(module_import_count) .collect(); From cff3e18a34b3e05a8aa5e678e25c77bd993412f8 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 15 Nov 2021 15:58:40 +0000 Subject: [PATCH 06/33] Fix memory leaks in C API examples --- lib/c-api/examples/exports-global.c | 3 +++ lib/c-api/examples/imports-exports.c | 1 + lib/c-api/examples/instance.c | 1 + lib/c-api/examples/wasi.c | 7 +------ 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/c-api/examples/exports-global.c b/lib/c-api/examples/exports-global.c index f9cae061e..ff20dfdef 100644 --- a/lib/c-api/examples/exports-global.c +++ b/lib/c-api/examples/exports-global.c @@ -103,12 +103,15 @@ int main(int argc, const char* argv[]) { wasmer_last_error_message(error_message, error_length); printf("Attempted to set an immutable global: `%s`\n", error_message); + free(error_message); } wasm_val_t some_set_value = WASM_F32_VAL(21); wasm_global_set(some, &some_set_value); printf("`some` value: %.1f\n", some_value.of.f32); + wasm_globaltype_delete(one_type); + wasm_globaltype_delete(some_type); wasm_module_delete(module); wasm_extern_vec_delete(&exports); wasm_instance_delete(instance); diff --git a/lib/c-api/examples/imports-exports.c b/lib/c-api/examples/imports-exports.c index 0e33e560b..aaf4186c3 100644 --- a/lib/c-api/examples/imports-exports.c +++ b/lib/c-api/examples/imports-exports.c @@ -63,6 +63,7 @@ int main(int argc, const char* argv[]) { printf("Instantiating module...\n"); wasm_instance_t* instance = wasm_instance_new(store, module, &import_object, NULL); + wasm_extern_vec_delete(&import_object); if (!instance) { printf("> Error instantiating module!\n"); diff --git a/lib/c-api/examples/instance.c b/lib/c-api/examples/instance.c index 843b94d21..6eb867d4c 100644 --- a/lib/c-api/examples/instance.c +++ b/lib/c-api/examples/instance.c @@ -15,6 +15,7 @@ int main(int argc, const char* argv[]) { wasm_byte_vec_new(&wat, strlen(wat_string), wat_string); wasm_byte_vec_t wasm_bytes; wat2wasm(&wat, &wasm_bytes); + wasm_byte_vec_delete(&wat); printf("Creating the store...\n"); wasm_engine_t* engine = wasm_engine_new(); diff --git a/lib/c-api/examples/wasi.c b/lib/c-api/examples/wasi.c index 1c895e302..e3047912d 100644 --- a/lib/c-api/examples/wasi.c +++ b/lib/c-api/examples/wasi.c @@ -69,13 +69,7 @@ int main(int argc, const char* argv[]) { // Instantiate. printf("Instantiating module...\n"); - wasm_importtype_vec_t import_types; - wasm_module_imports(module, &import_types); - wasm_extern_vec_t imports; - wasm_extern_vec_new_uninitialized(&imports, import_types.size); - wasm_importtype_vec_delete(&import_types); - bool get_imports_result = wasi_get_imports(store, module, wasi_env, &imports); if (!get_imports_result) { @@ -153,6 +147,7 @@ int main(int argc, const char* argv[]) { fclose(memory_stream); printf("WASI Stdout: `%.*s`\n", (int) stdout_size, stdout); + free(stdout); } From f2633d927cd6e6a7784f2599ba12178104fe3293 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 15 Nov 2021 16:32:02 +0000 Subject: [PATCH 07/33] Bump rust-toolchain to 1.56 --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 74df8b169..e01e6c121 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.53 +1.56 From 75cb5ab78862fe6023ca98b63bdc4f2ab85ac82d Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 15 Nov 2021 16:32:17 +0000 Subject: [PATCH 08/33] Refactor the C API to eliminate memory leaks --- Cargo.lock | 1 - .../src/wasm_c_api/externals/function.rs | 77 +- lib/c-api/src/wasm_c_api/externals/global.rs | 1 - lib/c-api/src/wasm_c_api/externals/memory.rs | 1 - lib/c-api/src/wasm_c_api/externals/mod.rs | 26 +- lib/c-api/src/wasm_c_api/instance.rs | 30 +- lib/c-api/src/wasm_c_api/macros.rs | 661 ++++++------------ lib/c-api/src/wasm_c_api/mod.rs | 2 +- lib/c-api/src/wasm_c_api/module.rs | 54 +- lib/c-api/src/wasm_c_api/trap.rs | 17 +- lib/c-api/src/wasm_c_api/types/export.rs | 27 +- lib/c-api/src/wasm_c_api/types/function.rs | 60 +- lib/c-api/src/wasm_c_api/types/import.rs | 28 +- lib/c-api/src/wasm_c_api/types/mod.rs | 58 +- lib/c-api/src/wasm_c_api/unstable/module.rs | 9 +- lib/c-api/src/wasm_c_api/unstable/wasi.rs | 40 +- lib/c-api/src/wasm_c_api/value.rs | 9 + lib/c-api/src/wasm_c_api/wasi/mod.rs | 11 +- lib/c-api/src/wasm_c_api/wat.rs | 14 +- 19 files changed, 403 insertions(+), 723 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7abf5a543..aa790239f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2723,7 +2723,6 @@ dependencies = [ "lazy_static", "libc", "paste", - "rayon", "serde", "thiserror", "typetag", diff --git a/lib/c-api/src/wasm_c_api/externals/function.rs b/lib/c-api/src/wasm_c_api/externals/function.rs index d04e3c9b9..7db2b533d 100644 --- a/lib/c-api/src/wasm_c_api/externals/function.rs +++ b/lib/c-api/src/wasm_c_api/externals/function.rs @@ -5,6 +5,7 @@ use super::super::value::{wasm_val_inner, wasm_val_t, wasm_val_vec_t}; use super::CApiExternTag; use std::convert::TryInto; use std::ffi::c_void; +use std::mem::MaybeUninit; use std::sync::Arc; use wasmer_api::{Function, RuntimeError, Val}; @@ -27,16 +28,16 @@ impl wasm_func_t { #[allow(non_camel_case_types)] pub type wasm_func_callback_t = unsafe extern "C" fn( - args: *const wasm_val_vec_t, - results: *mut wasm_val_vec_t, -) -> *mut wasm_trap_t; + args: &wasm_val_vec_t, + results: &mut wasm_val_vec_t, +) -> Option>; #[allow(non_camel_case_types)] pub type wasm_func_callback_with_env_t = unsafe extern "C" fn( env: *mut c_void, - args: *const wasm_val_vec_t, - results: *mut wasm_val_vec_t, -) -> *mut wasm_trap_t; + args: &wasm_val_vec_t, + results: &mut wasm_val_vec_t, +) -> Option>; #[allow(non_camel_case_types)] pub type wasm_env_finalizer_t = unsafe extern "C" fn(*mut c_void); @@ -72,15 +73,12 @@ pub unsafe extern "C" fn wasm_func_new( let trap = callback(&processed_args, &mut results); - if !trap.is_null() { - let trap: Box = Box::from_raw(trap); - + if let Some(trap) = trap { return Err(trap.inner); } let processed_results = results - .into_slice() - .expect("Failed to convert `results` into a slice") + .take() .into_iter() .map(TryInto::try_into) .collect::, _>>() @@ -124,12 +122,8 @@ pub unsafe extern "C" fn wasm_func_new_with_env( impl Drop for WrapperEnv { fn drop(&mut self) { - if let Some(env_finalizer) = - Arc::get_mut(&mut self.env_finalizer).and_then(Option::take) - { - if !self.env.is_null() { - unsafe { (env_finalizer)(self.env as _) } - } + if let Some(env_finalizer) = *self.env_finalizer { + unsafe { (env_finalizer)(self.env as _) } } } } @@ -153,15 +147,12 @@ pub unsafe extern "C" fn wasm_func_new_with_env( let trap = callback(env.env, &processed_args, &mut results); - if !trap.is_null() { - let trap: Box = Box::from_raw(trap); - + if let Some(trap) = trap { return Err(trap.inner); } let processed_results = results - .into_slice() - .expect("Failed to convert `results` into a slice") + .take() .into_iter() .map(TryInto::try_into) .collect::, _>>() @@ -201,39 +192,21 @@ pub unsafe extern "C" fn wasm_func_call( let args = args?; let params = args - .into_slice() - .map(|slice| { - slice - .into_iter() - .map(TryInto::try_into) - .collect::, _>>() - .expect("Arguments conversion failed") - }) - .unwrap_or_default(); + .as_slice() + .iter() + .cloned() + .map(TryInto::try_into) + .collect::, _>>() + .expect("Arguments conversion failed"); match func.inner.call(¶ms) { Ok(wasm_results) => { - let vals = wasm_results - .into_iter() - .map(TryInto::try_into) - .collect::, _>>() - .expect("Results conversion failed"); - - // `results` is an uninitialized vector. Set a new value. - if results.is_uninitialized() { - *results = vals.into(); - } - // `results` is an initialized but empty vector. Fill it - // item per item. - else { - let slice = results - .into_slice_mut() - .expect("`wasm_func_call`, results' size is greater than 0 but data is NULL"); - - for (result, value) in slice.iter_mut().zip(vals.iter()) { - (*result).kind = value.kind; - (*result).of = value.of; - } + for (slot, val) in results + .as_uninit_slice() + .iter_mut() + .zip(wasm_results.into_iter()) + { + *slot = MaybeUninit::new(val.try_into().expect("Results conversion failed")); } None diff --git a/lib/c-api/src/wasm_c_api/externals/global.rs b/lib/c-api/src/wasm_c_api/externals/global.rs index 4bdbd81dd..02615cd26 100644 --- a/lib/c-api/src/wasm_c_api/externals/global.rs +++ b/lib/c-api/src/wasm_c_api/externals/global.rs @@ -48,7 +48,6 @@ pub unsafe extern "C" fn wasm_global_new( #[no_mangle] pub unsafe extern "C" fn wasm_global_delete(_global: Option>) {} -// TODO: figure out if these should be deep or shallow copies #[no_mangle] pub unsafe extern "C" fn wasm_global_copy(global: &wasm_global_t) -> Box { // do shallow copy diff --git a/lib/c-api/src/wasm_c_api/externals/memory.rs b/lib/c-api/src/wasm_c_api/externals/memory.rs index e483f3389..97694adb2 100644 --- a/lib/c-api/src/wasm_c_api/externals/memory.rs +++ b/lib/c-api/src/wasm_c_api/externals/memory.rs @@ -38,7 +38,6 @@ pub unsafe extern "C" fn wasm_memory_new( #[no_mangle] pub unsafe extern "C" fn wasm_memory_delete(_memory: Option>) {} -// TODO: figure out if these should be deep or shallow copies #[no_mangle] pub unsafe extern "C" fn wasm_memory_copy(memory: &wasm_memory_t) -> Box { // do shallow copy diff --git a/lib/c-api/src/wasm_c_api/externals/mod.rs b/lib/c-api/src/wasm_c_api/externals/mod.rs index 0a701db1c..cd2b88368 100644 --- a/lib/c-api/src/wasm_c_api/externals/mod.rs +++ b/lib/c-api/src/wasm_c_api/externals/mod.rs @@ -6,7 +6,7 @@ mod table; pub use function::*; pub use global::*; pub use memory::*; -use std::mem; +use std::mem::{self, ManuallyDrop}; pub use table::*; use wasmer_api::{Extern, ExternType}; @@ -150,13 +150,23 @@ impl From for wasm_extern_t { } impl From for Extern { - fn from(other: wasm_extern_t) -> Self { - match other.get_tag() { - CApiExternTag::Function => unsafe { (&*other.inner.function.inner).clone().into() }, - CApiExternTag::Memory => unsafe { (&*other.inner.memory.inner).clone().into() }, - CApiExternTag::Table => unsafe { (&*other.inner.table.inner).clone().into() }, - CApiExternTag::Global => unsafe { (&*other.inner.global.inner).clone().into() }, - } + fn from(mut other: wasm_extern_t) -> Self { + let out = match other.get_tag() { + CApiExternTag::Function => unsafe { + (*ManuallyDrop::take(&mut other.inner.function).inner).into() + }, + CApiExternTag::Memory => unsafe { + (*ManuallyDrop::take(&mut other.inner.memory).inner).into() + }, + CApiExternTag::Table => unsafe { + (*ManuallyDrop::take(&mut other.inner.table).inner).into() + }, + CApiExternTag::Global => unsafe { + (*ManuallyDrop::take(&mut other.inner.global).inner).into() + }, + }; + mem::forget(other); + out } } diff --git a/lib/c-api/src/wasm_c_api/instance.rs b/lib/c-api/src/wasm_c_api/instance.rs index 67b11d3e7..047dd18ee 100644 --- a/lib/c-api/src/wasm_c_api/instance.rs +++ b/lib/c-api/src/wasm_c_api/instance.rs @@ -1,9 +1,8 @@ -use super::externals::{wasm_extern_t, wasm_extern_vec_t}; +use super::externals::wasm_extern_vec_t; use super::module::wasm_module_t; use super::store::wasm_store_t; use super::trap::wasm_trap_t; use crate::ordered_resolver::OrderedResolver; -use std::mem; use std::sync::Arc; use wasmer_api::{Extern, Instance, InstantiationError}; @@ -41,7 +40,7 @@ pub unsafe extern "C" fn wasm_instance_new( _store: Option<&wasm_store_t>, module: Option<&wasm_module_t>, imports: Option<&wasm_extern_vec_t>, - trap: *mut *mut wasm_trap_t, + trap: Option<&mut *mut wasm_trap_t>, ) -> Option> { let module = module?; let imports = imports?; @@ -50,10 +49,9 @@ pub unsafe extern "C" fn wasm_instance_new( let module_imports = wasm_module.imports(); let module_import_count = module_imports.len(); let resolver: OrderedResolver = imports - .into_slice() - .map(|imports| imports.iter()) - .unwrap_or_else(|| [].iter()) - .map(|imp| Extern::from((&**imp).clone())) + .as_slice() + .iter() + .map(|imp| Extern::from(imp.as_ref().unwrap().as_ref().clone())) .take(module_import_count) .collect(); @@ -67,8 +65,10 @@ pub unsafe extern "C" fn wasm_instance_new( } Err(InstantiationError::Start(runtime_error)) => { - let this_trap: Box = Box::new(runtime_error.into()); - *trap = Box::into_raw(this_trap); + if let Some(trap) = trap { + let this_trap: Box = Box::new(runtime_error.into()); + *trap = Box::into_raw(this_trap); + } return None; } @@ -181,17 +181,13 @@ pub unsafe extern "C" fn wasm_instance_exports( out: &mut wasm_extern_vec_t, ) { let instance = &instance.inner; - let mut extern_vec = instance + let extern_vec = instance .exports .iter() - .map(|(_name, r#extern)| Box::into_raw(Box::new(r#extern.clone().into()))) - .collect::>(); - extern_vec.shrink_to_fit(); + .map(|(_name, r#extern)| Some(Box::new(r#extern.clone().into()))) + .collect(); - out.size = extern_vec.len(); - out.data = extern_vec.as_mut_ptr(); - - mem::forget(extern_vec); + out.set_buffer(extern_vec); } #[cfg(test)] diff --git a/lib/c-api/src/wasm_c_api/macros.rs b/lib/c-api/src/wasm_c_api/macros.rs index 99909106b..22a251bb2 100644 --- a/lib/c-api/src/wasm_c_api/macros.rs +++ b/lib/c-api/src/wasm_c_api/macros.rs @@ -1,13 +1,17 @@ -#[doc(hidden)] -#[macro_export] macro_rules! wasm_declare_vec_inner { - ($name:ident) => { - wasm_declare_vec_inner!($name, wasm); - }; + ( + name: $name:ident, + ty: $elem_ty:ty, + c_ty: $c_ty:expr, + new: $new:ident, + empty: $empty:ident, + uninit: $uninit:ident, + copy: $copy:ident, + delete: $delete:ident, + ) => { + #[doc = concat!("Represents a vector of `", $c_ty, "`. - ($name:ident, $prefix:ident) => { - paste::paste! { - #[doc = "Creates an empty vector of [`" $prefix "_" $name "_t`]. +Read the documentation of [`", $c_ty, "`] to see more concrete examples. # Example @@ -18,31 +22,188 @@ macro_rules! wasm_declare_vec_inner { # #include \"tests/wasmer.h\" # int main() { - // Creates an empty vector of `" $prefix "_" $name "_t`. - " $prefix "_" $name "_vec_t vector; - " $prefix "_" $name "_vec_new_empty(&vector); + // Create a vector of 2 `", $c_ty, "`. + ", $c_ty, " x; + ", $c_ty, " y; + ", $c_ty, " items[2] = {x, y}; + + ", stringify!($name), " vector; + ", stringify!($new), "(&vector, 2, items); + + // Check that it contains 2 items. + assert(vector.size == 2); + + // Free it. + ", stringify!($delete), "(&vector); +} +# }) +# .success(); +# } +```")] + #[repr(C)] + pub struct $name { + pub size: usize, + pub data: *mut $elem_ty, + } + + impl $name { + // Note that this does not free any existing buffer. + pub fn set_buffer(&mut self, buffer: Vec<$elem_ty>) { + let mut vec = buffer.into_boxed_slice(); + self.size = vec.len(); + self.data = vec.as_mut_ptr(); + std::mem::forget(vec); + } + + pub fn as_slice(&self) -> &[$elem_ty] { + // Note that we're careful to not create a slice with a null + // pointer as the data pointer, since that isn't defined + // behavior in Rust. + if self.size == 0 { + &[] + } else { + assert!(!self.data.is_null()); + unsafe { std::slice::from_raw_parts(self.data, self.size) } + } + } + + pub fn as_uninit_slice(&mut self) -> &mut [std::mem::MaybeUninit<$elem_ty>] { + // Note that we're careful to not create a slice with a null + // pointer as the data pointer, since that isn't defined + // behavior in Rust. + if self.size == 0 { + &mut [] + } else { + assert!(!self.data.is_null()); + unsafe { std::slice::from_raw_parts_mut(self.data as _, self.size) } + } + } + + pub fn take(&mut self) -> Vec<$elem_ty> { + if self.data.is_null() { + return Vec::new(); + } + let vec = unsafe { Vec::from_raw_parts(self.data, self.size, self.size) }; + self.data = std::ptr::null_mut(); + self.size = 0; + return vec; + } + } + + impl From> for $name { + fn from(vec: Vec<$elem_ty>) -> Self { + let mut vec = vec.into_boxed_slice(); + let result = $name { + size: vec.len(), + data: vec.as_mut_ptr(), + }; + std::mem::forget(vec); + result + } + } + + impl Clone for $name { + fn clone(&self) -> Self { + self.as_slice().to_vec().into() + } + } + + impl Drop for $name { + fn drop(&mut self) { + drop(self.take()); + } + } + + #[doc = concat!("Creates an empty vector of [`", $c_ty, "`]. + +# Example + +```rust +# use inline_c::assert_c; +# fn main() { +# (assert_c! { +# #include \"tests/wasmer.h\" +# +int main() { + // Creates an empty vector of `", $c_ty, "`. + ", stringify!($name), " vector; + ", stringify!($empty), "(&vector); // Check that it is empty. assert(vector.size == 0); // Free it. - " $prefix "_" $name "_vec_delete(&vector); + ", stringify!($delete), "(&vector); } # }) # .success(); # } -```"] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_new_empty>](out: *mut [<$prefix _ $name _vec_t>]) { - // TODO: actually implement this - [<$prefix _ $name _vec_new_uninitialized>](out, 0); - } +```")] + #[no_mangle] + pub extern "C" fn $empty(out: &mut $name) { + out.size = 0; + out.data = std::ptr::null_mut(); } - } + + #[doc = concat!("Creates a new uninitialized vector of [`", $c_ty, "`]. + +# Example + +```rust +# use inline_c::assert_c; +# fn main() { +# (assert_c! { +# #include \"tests/wasmer.h\" +# +int main() { + // Creates an empty vector of `", $c_ty, "`. + ", stringify!($name), " vector; + ", stringify!($uninit), "(&vector, 3); + + // Check that it contains 3 items. + assert(vector.size == 3); + + // Free it. + ", stringify!($delete), "(&vector); +} +# }) +# .success(); +# } +```")] + #[no_mangle] + pub extern "C" fn $uninit(out: &mut $name, size: usize) { + out.set_buffer(vec![Default::default(); size]); + } + + #[doc = concat!("Creates a new vector of [`", $c_ty, "`]. + +# Example + +See the [`", stringify!($name), "`] type to get an example.")] + #[no_mangle] + pub unsafe extern "C" fn $new(out: &mut $name, size: usize, ptr: *const $elem_ty) { + let vec = (0..size).map(|i| ptr.add(i).read()).collect(); + out.set_buffer(vec); + } + + #[doc = concat!("Performs a deep copy of a vector of [`", $c_ty, "`].")] + #[no_mangle] + pub extern "C" fn $copy(out: &mut $name, src: &$name) { + out.set_buffer(src.as_slice().to_vec()); + } + + #[doc = concat!("Deletes a vector of [`", $c_ty, "`]. + +# Example + +See the [`", stringify!($name), "`] type to get an example.")] + #[no_mangle] + pub extern "C" fn $delete(out: &mut $name) { + out.take(); + } + }; } -#[doc(hidden)] -#[macro_export] macro_rules! wasm_declare_vec { ($name:ident) => { wasm_declare_vec!($name, wasm); @@ -50,218 +211,20 @@ macro_rules! wasm_declare_vec { ($name:ident, $prefix:ident) => { paste::paste! { - #[doc = "Represents a vector of `" $prefix "_" $name "_t`. - -Read the documentation of [`" $prefix "_" $name "_t`] to see more concrete examples. - -# Example - -```rust -# use inline_c::assert_c; -# fn main() { -# (assert_c! { -# #include \"tests/wasmer.h\" -# -int main() { - // Create a vector of 2 `" $prefix "_" $name "_t`. - " $prefix "_" $name "_t x; - " $prefix "_" $name "_t y; - " $prefix "_" $name "_t* items[2] = {&x, &y}; - - " $prefix "_" $name "_vec_t vector; - " $prefix "_" $name "_vec_new(&vector, 2, (" $prefix "_" $name "_t*) items); - - // Check that it contains 2 items. - assert(vector.size == 2); - - // Free it. - " $prefix "_" $name "_vec_delete(&vector); -} -# }) -# .success(); -# } -```"] - #[derive(Debug)] - #[repr(C)] - pub struct [<$prefix _ $name _vec_t>] { - pub size: usize, - pub data: *mut [<$prefix _ $name _t>], - } - - impl Clone for [<$prefix _ $name _vec_t>] { - fn clone(&self) -> Self { - if self.data.is_null() { - return Self { - size: self.size, - data: ::std::ptr::null_mut(), - }; - } - let data = - unsafe { - let vec = Vec::from_raw_parts(self.data, self.size, self.size); - let mut vec_copy = vec.clone().into_boxed_slice(); - let new_ptr = vec_copy.as_mut_ptr(); - - ::std::mem::forget(vec); - ::std::mem::forget(vec_copy); - - new_ptr - }; - - Self { - size: self.size, - data, - } - } - } - - impl<'a> From]>> for [<$prefix _ $name _vec_t>] { - fn from(mut vec: Vec<[<$prefix _ $name _t>]>) -> Self { - vec.shrink_to_fit(); - - let length = vec.len(); - let pointer = vec.as_mut_ptr(); - - ::std::mem::forget(vec); - - Self { - size: length, - data: pointer, - } - } - } - - impl<'a, T: Into<[<$prefix _ $name _t>]> + Clone> From<&'a [T]> for [<$prefix _ $name _vec_t>] { - fn from(other: &'a [T]) -> Self { - let size = other.len(); - let mut copied_data = other - .iter() - .cloned() - .map(Into::into) - .collect::]>>() - .into_boxed_slice(); - let data = copied_data.as_mut_ptr(); - ::std::mem::forget(copied_data); - - Self { - size, - data, - } - } - } - - impl [<$prefix _ $name _vec_t>] { - pub unsafe fn into_slice(&self) -> Option<&[[<$prefix _ $name _t>]]>{ - if self.is_uninitialized() { - return None; - } - - Some(::std::slice::from_raw_parts(self.data, self.size)) - } - - pub unsafe fn into_slice_mut(&mut self) -> Option<&mut [[<$prefix _ $name _t>]]>{ - if self.is_uninitialized() { - return None; - } - - Some(::std::slice::from_raw_parts_mut(self.data, self.size)) - } - - pub fn is_uninitialized(&self) -> bool { - self.data.is_null() - } - } - - // TODO: investigate possible memory leak on `init` (owned pointer) - #[doc = "Creates a new vector of [`" $prefix "_" $name "_t`]. - -# Example - -See the [`" $prefix "_" $name "_vec_t`] type to get an example."] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_new>](out: *mut [<$prefix _ $name _vec_t>], length: usize, init: *mut [<$prefix _ $name _t>]) { - let mut bytes: Vec<[<$prefix _ $name _t>]> = Vec::with_capacity(length); - - for i in 0..length { - bytes.push(::std::ptr::read(init.add(i))); - } - - let pointer = bytes.as_mut_ptr(); - debug_assert!(bytes.len() == bytes.capacity()); - - (*out).data = pointer; - (*out).size = length; - ::std::mem::forget(bytes); - } - - #[doc = "Creates a new uninitialized vector of [`" $prefix "_" $name "_t`]. - -# Example - -```rust -# use inline_c::assert_c; -# fn main() { -# (assert_c! { -# #include \"tests/wasmer.h\" -# -int main() { - // Creates an empty vector of `" $prefix "_" $name "_t`. - " $prefix "_" $name "_vec_t vector; - " $prefix "_" $name "_vec_new_uninitialized(&vector, 3); - - // Check that it contains 3 items. - assert(vector.size == 3); - - // Free it. - " $prefix "_" $name "_vec_delete(&vector); -} -# }) -# .success(); -# } -```"] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_new_uninitialized>](out: *mut [<$prefix _ $name _vec_t>], length: usize) { - let mut bytes: Vec<[<$prefix _ $name _t>]> = Vec::with_capacity(length); - let pointer = bytes.as_mut_ptr(); - - (*out).data = pointer; - (*out).size = length; - - ::std::mem::forget(bytes); - } - - #[doc = "Performs a deep copy of a vector of [`" $prefix "_" $name "_t`]."] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_copy>]( - out_ptr: &mut [<$prefix _ $name _vec_t>], - in_ptr: & []) - { - *out_ptr = in_ptr.clone(); - } - - #[doc = "Deletes a vector of [`" $prefix "_" $name "_t`]. - -# Example - -See the [`" $prefix "_" $name "_vec_t`] type to get an example."] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_delete>](ptr: Option<&mut [<$prefix _ $name _vec_t>]>) { - if let Some(vec) = ptr { - if !vec.data.is_null() { - Vec::from_raw_parts(vec.data, vec.size, vec.size); - vec.data = ::std::ptr::null_mut(); - vec.size = 0; - } - } - } + wasm_declare_vec_inner!( + name: [<$prefix _ $name _vec_t>], + ty: [<$prefix _ $name _t>], + c_ty: stringify!([<$prefix _ $name _t>]), + new: [<$prefix _ $name _vec_new>], + empty: [<$prefix _ $name _vec_new_empty>], + uninit: [<$prefix _ $name _vec_new_uninitialized>], + copy: [<$prefix _ $name _vec_copy>], + delete: [<$prefix _ $name _vec_delete>], + ); } - - wasm_declare_vec_inner!($name, $prefix); }; } -#[doc(hidden)] -#[macro_export] macro_rules! wasm_declare_boxed_vec { ($name:ident) => { wasm_declare_boxed_vec!($name, wasm); @@ -269,223 +232,59 @@ macro_rules! wasm_declare_boxed_vec { ($name:ident, $prefix:ident) => { paste::paste! { - #[doc = "Represents a vector of `" $prefix "_" $name "_t`. - -Read the documentation of [`" $prefix "_" $name "_t`] to see more concrete examples."] - #[derive(Debug)] - #[repr(C)] - pub struct [<$prefix _ $name _vec_t>] { - pub size: usize, - pub data: *mut *mut [<$prefix _ $name _t>], - } - - impl Clone for [<$prefix _ $name _vec_t>] { - fn clone(&self) -> Self { - if self.data.is_null() { - return Self { - size: self.size, - data: ::std::ptr::null_mut(), - }; - } - let data = - unsafe { - let data: *mut Option]>> = self.data as _; - let vec = Vec::from_raw_parts(data, self.size, self.size); - let mut vec_copy = vec.clone().into_boxed_slice(); - let new_ptr = vec_copy.as_mut_ptr() as *mut *mut [<$prefix _ $name _t>]; - - ::std::mem::forget(vec); - ::std::mem::forget(vec_copy); - - new_ptr - }; - - Self { - size: self.size, - data, - } - } - } - - impl<'a> From]>>> for [<$prefix _ $name _vec_t>] { - fn from(other: Vec]>>) -> Self { - let boxed_slice: Box<[Box<[<$prefix _ $name _t>]>]> = other.into_boxed_slice(); - let mut boxed_slice: Box<[*mut [<$prefix _ $name _t>]]> = unsafe { ::std::mem::transmute(boxed_slice) }; - let size = boxed_slice.len(); - let data = boxed_slice.as_mut_ptr(); - - ::std::mem::forget(boxed_slice); - Self { - size, - data, - } - } - } - - impl<'a, T: Into<[<$prefix _ $name _t>]> + Clone> From<&'a [T]> for [<$prefix _ $name _vec_t>] { - fn from(other: &'a [T]) -> Self { - let size = other.len(); - let mut copied_data = other - .iter() - .cloned() - .map(Into::into) - .map(Box::new) - .map(Box::into_raw) - .collect::]>>() - .into_boxed_slice(); - let data = copied_data.as_mut_ptr(); - ::std::mem::forget(copied_data); - - Self { - size, - data, - } - } - } - - // TODO: do this properly - impl [<$prefix _ $name _vec_t>] { - pub unsafe fn into_slice(&self) -> Option<&[Box<[<$prefix _ $name _t>]>]>{ - if self.data.is_null() { - return None; - } - - let slice: &[*mut [<$prefix _ $name _t>]] = ::std::slice::from_raw_parts(self.data, self.size); - let slice: &[Box<[<$prefix _ $name _t>]>] = ::std::mem::transmute(slice); - Some(slice) - } - } - - // TODO: investigate possible memory leak on `init` (owned pointer) - #[doc = "Creates a new vector of [`" $prefix "_" $name "_t`]."] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_new>](out: *mut [<$prefix _ $name _vec_t>], length: usize, init: *const *mut [<$prefix _ $name _t>]) { - let mut bytes: Vec<*mut [<$prefix _ $name _t>]> = Vec::with_capacity(length); - - for i in 0..length { - bytes.push(*init.add(i)); - } - - let mut boxed_vec = bytes.into_boxed_slice(); - let pointer = boxed_vec.as_mut_ptr(); - - (*out).data = pointer; - (*out).size = length; - - ::std::mem::forget(boxed_vec); - } - - #[doc = "Creates a new uninitialized vector of [`" $prefix "_" $name "_t`]. - -# Example - -```rust -# use inline_c::assert_c; -# fn main() { -# (assert_c! { -# #include \"tests/wasmer.h\" -# -int main() { - // Creates an empty vector of `" $prefix "_" $name "_t`. - " $prefix "_" $name "_vec_t vector; - " $prefix "_" $name "_vec_new_uninitialized(&vector, 3); - - // Check that it contains 3 items. - assert(vector.size == 3); - - // Free it. - " $prefix "_" $name "_vec_delete(&vector); -} -# }) -# .success(); -# } -```"] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_new_uninitialized>](out: *mut [<$prefix _ $name _vec_t>], length: usize) { - let mut bytes: Vec<*mut [<$prefix _ $name _t>]> = vec![::std::ptr::null_mut(); length]; - let pointer = bytes.as_mut_ptr(); - - (*out).data = pointer; - (*out).size = length; - - ::std::mem::forget(bytes); - } - - #[doc = "Performs a deep copy of a vector of [`" $prefix "_" $name "_t`]."] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_copy>]( - out_ptr: &mut [<$prefix _ $name _vec_t>], - in_ptr: & [<$prefix _ $name _vec_t>]) - { - *out_ptr = in_ptr.clone(); - } - - #[doc = "Deletes a vector of [`" $prefix "_" $name "_t`]. - -# Example - -See the [`" $prefix "_" $name "_vec_t`] type to get an example."] - #[no_mangle] - pub unsafe extern "C" fn [<$prefix _ $name _vec_delete>](ptr: Option<&mut [<$prefix _ $name _vec_t>]>) { - if let Some(vec) = ptr { - if !vec.data.is_null() { - let data = vec.data as *mut Option]>>; - let _data: Vec]>>> = Vec::from_raw_parts(data, vec.size, vec.size); - - vec.data = ::std::ptr::null_mut(); - vec.size = 0; - } - } - } - } - - wasm_declare_vec_inner!($name, $prefix); - }; -} - -#[doc(hidden)] -#[macro_export] -macro_rules! wasm_declare_ref_base { - ($name:ident) => { - wasm_declare_ref_base!($name, wasm); - }; - - ($name:ident, $prefix:ident) => { - wasm_declare_own!($name, $prefix); - - paste::paste! { - #[no_mangle] - pub extern "C" fn [<$prefix _ $name _copy>](_arg: *const [<$prefix _ $name _t>]) -> *mut [<$prefix _ $name _t>] { - todo!("in generated declare ref base"); - //ptr::null_mut() - } - - // TODO: finish this... + wasm_declare_vec_inner!( + name: [<$prefix _ $name _vec_t>], + ty: Option]>>, + c_ty: stringify!([<$prefix _ $name _t>] *), + new: [<$prefix _ $name _vec_new>], + empty: [<$prefix _ $name _vec_new_empty>], + uninit: [<$prefix _ $name _vec_new_uninitialized>], + copy: [<$prefix _ $name _vec_copy>], + delete: [<$prefix _ $name _vec_delete>], + ); } }; } -#[doc(hidden)] -#[macro_export] -macro_rules! wasm_declare_own { +macro_rules! wasm_impl_copy { ($name:ident) => { - wasm_declare_own!($name, $prefix); + wasm_impl_copy!($name, wasm); }; ($name:ident, $prefix:ident) => { paste::paste! { - #[repr(C)] - pub struct [<$prefix _ $name _t>] {} - #[no_mangle] - pub extern "C" fn [<$prefix _ $name _delete>](_arg: *mut [<$prefix _ $name _t>]) { - todo!("in generated delete") + pub extern "C" fn [<$prefix _ $name _copy>](src: Option<&[<$prefix _ $name _t>]>) -> Option]>> { + Some(Box::new(src?.clone())) } } }; } -#[macro_export] +macro_rules! wasm_impl_delete { + ($name:ident) => { + wasm_impl_delete!($name, wasm); + }; + + ($name:ident, $prefix:ident) => { + paste::paste! { + #[no_mangle] + pub extern "C" fn [<$prefix _ $name _delete>](_: Option]>>) {} + } + }; +} + +macro_rules! wasm_impl_copy_delete { + ($name:ident) => { + wasm_impl_copy_delete!($name, wasm); + }; + + ($name:ident, $prefix:ident) => { + wasm_impl_copy!($name, $prefix); + wasm_impl_delete!($name, $prefix); + }; +} + macro_rules! c_try { ($expr:expr; otherwise $return:expr) => {{ let res: Result<_, _> = $expr; diff --git a/lib/c-api/src/wasm_c_api/mod.rs b/lib/c-api/src/wasm_c_api/mod.rs index 10bac64c2..1732669db 100644 --- a/lib/c-api/src/wasm_c_api/mod.rs +++ b/lib/c-api/src/wasm_c_api/mod.rs @@ -23,7 +23,7 @@ /// Private Rust macros. #[macro_use] -pub mod macros; +mod macros; /// An engine drives the compilation and the runtime. /// diff --git a/lib/c-api/src/wasm_c_api/module.rs b/lib/c-api/src/wasm_c_api/module.rs index 2a37a96af..5eb0a716d 100644 --- a/lib/c-api/src/wasm_c_api/module.rs +++ b/lib/c-api/src/wasm_c_api/module.rs @@ -1,9 +1,6 @@ use super::store::wasm_store_t; -use super::types::{ - wasm_byte_vec_t, wasm_exporttype_t, wasm_exporttype_vec_t, wasm_importtype_t, - wasm_importtype_vec_t, -}; -use crate::error::{update_last_error, CApiError}; +use super::types::{wasm_byte_vec_t, wasm_exporttype_vec_t, wasm_importtype_vec_t}; +use crate::error::update_last_error; use std::ptr::NonNull; use std::sync::Arc; use wasmer_api::Module; @@ -36,8 +33,7 @@ pub unsafe extern "C" fn wasm_module_new( let store = store?; let bytes = bytes?; - let bytes = bytes.into_slice()?; - let module = c_try!(Module::from_binary(&store.inner, bytes)); + let module = c_try!(Module::from_binary(&store.inner, bytes.as_slice())); Some(Box::new(wasm_module_t { inner: Arc::new(module), @@ -107,12 +103,7 @@ pub unsafe extern "C" fn wasm_module_validate( None => return false, }; - let bytes = match bytes.into_slice() { - Some(bytes) => bytes, - None => return false, - }; - - if let Err(error) = Module::validate(&store.inner, bytes) { + if let Err(error) = Module::validate(&store.inner, bytes.as_slice()) { update_last_error(error); false @@ -238,11 +229,10 @@ pub unsafe extern "C" fn wasm_module_exports( let exports = module .inner .exports() - .map(Into::into) - .map(Box::new) - .collect::>>(); + .map(|export| Some(Box::new(export.into()))) + .collect(); - *out = exports.into(); + out.set_buffer(exports); } /// Returns an array of the imported types in the module. @@ -379,11 +369,10 @@ pub unsafe extern "C" fn wasm_module_imports( let imports = module .inner .imports() - .map(Into::into) - .map(Box::new) - .collect::>>(); + .map(|import| Some(Box::new(import.into()))) + .collect(); - *out = imports.into(); + out.set_buffer(imports); } /// Deserializes a serialized module binary into a `wasm_module_t`. @@ -472,21 +461,11 @@ pub unsafe extern "C" fn wasm_module_imports( #[no_mangle] pub unsafe extern "C" fn wasm_module_deserialize( store: &wasm_store_t, - bytes: *const wasm_byte_vec_t, + bytes: Option<&wasm_byte_vec_t>, ) -> Option> { - // TODO: read config from store and use that to decide which compiler to use + let bytes = bytes?; - let byte_slice = if bytes.is_null() || (&*bytes).into_slice().is_none() { - update_last_error(CApiError { - msg: "`bytes` is null or represents an empty slice".to_string(), - }); - - return None; - } else { - (&*bytes).into_slice().unwrap() - }; - - let module = c_try!(Module::deserialize(&store.inner, byte_slice)); + let module = c_try!(Module::deserialize(&store.inner, bytes.as_slice())); Some(NonNull::new_unchecked(Box::into_raw(Box::new( wasm_module_t { @@ -503,10 +482,7 @@ pub unsafe extern "C" fn wasm_module_deserialize( /// /// See [`wasm_module_deserialize`]. #[no_mangle] -pub unsafe extern "C" fn wasm_module_serialize( - module: &wasm_module_t, - out_ptr: &mut wasm_byte_vec_t, -) { +pub unsafe extern "C" fn wasm_module_serialize(module: &wasm_module_t, out: &mut wasm_byte_vec_t) { let byte_vec = match module.inner.serialize() { Ok(byte_vec) => byte_vec, Err(err) => { @@ -514,7 +490,7 @@ pub unsafe extern "C" fn wasm_module_serialize( return; } }; - *out_ptr = byte_vec.into(); + out.set_buffer(byte_vec); } #[cfg(test)] diff --git a/lib/c-api/src/wasm_c_api/trap.rs b/lib/c-api/src/wasm_c_api/trap.rs index 4d3d20129..af21b00ef 100644 --- a/lib/c-api/src/wasm_c_api/trap.rs +++ b/lib/c-api/src/wasm_c_api/trap.rs @@ -28,7 +28,7 @@ pub unsafe extern "C" fn wasm_trap_new( _store: &mut wasm_store_t, message: &wasm_message_t, ) -> Option> { - let message_bytes = message.into_slice()?; + let message_bytes = message.as_slice(); // The trap message is typed with `wasm_message_t` which is a // typeref to `wasm_name_t` with the exception that it's a @@ -117,10 +117,7 @@ pub unsafe extern "C" fn wasm_trap_message( let mut byte_vec = message.into_bytes(); byte_vec.push(0); - let byte_vec: wasm_byte_vec_t = byte_vec.into(); - - out.size = byte_vec.size; - out.data = byte_vec.data; + out.set_buffer(byte_vec); } /// Gets the origin frame attached to the trap. @@ -137,10 +134,12 @@ pub unsafe extern "C" fn wasm_trap_trace( out: &mut wasm_frame_vec_t, ) { let frames = trap.inner.trace(); - let frame_vec: wasm_frame_vec_t = frames.into(); - - out.size = frame_vec.size; - out.data = frame_vec.data; + out.set_buffer( + frames + .iter() + .map(|frame| Some(Box::new(frame.into()))) + .collect(), + ); } #[cfg(test)] diff --git a/lib/c-api/src/wasm_c_api/types/export.rs b/lib/c-api/src/wasm_c_api/types/export.rs index 7f07dd22f..39d5ab674 100644 --- a/lib/c-api/src/wasm_c_api/types/export.rs +++ b/lib/c-api/src/wasm_c_api/types/export.rs @@ -1,30 +1,30 @@ -use super::{owned_wasm_name_t, wasm_externtype_t, wasm_name_t}; +use super::{wasm_externtype_t, wasm_name_t}; use wasmer_api::ExportType; #[allow(non_camel_case_types)] #[derive(Clone)] pub struct wasm_exporttype_t { - name: owned_wasm_name_t, + name: wasm_name_t, extern_type: Box, } wasm_declare_boxed_vec!(exporttype); +wasm_impl_copy_delete!(exporttype); #[no_mangle] pub extern "C" fn wasm_exporttype_new( - name: Option<&wasm_name_t>, - extern_type: Option>, -) -> Option> { - let name = unsafe { owned_wasm_name_t::new(name?) }; - Some(Box::new(wasm_exporttype_t { - name, - extern_type: extern_type?, - })) + name: &wasm_name_t, + extern_type: Box, +) -> Box { + Box::new(wasm_exporttype_t { + name: name.clone(), + extern_type, + }) } #[no_mangle] pub extern "C" fn wasm_exporttype_name(export_type: &wasm_exporttype_t) -> &wasm_name_t { - export_type.name.as_ref() + &export_type.name } #[no_mangle] @@ -32,9 +32,6 @@ pub extern "C" fn wasm_exporttype_type(export_type: &wasm_exporttype_t) -> &wasm export_type.extern_type.as_ref() } -#[no_mangle] -pub extern "C" fn wasm_exporttype_delete(_export_type: Option>) {} - impl From for wasm_exporttype_t { fn from(other: ExportType) -> Self { (&other).into() @@ -43,7 +40,7 @@ impl From for wasm_exporttype_t { impl From<&ExportType> for wasm_exporttype_t { fn from(other: &ExportType) -> Self { - let name: owned_wasm_name_t = other.name().to_string().into(); + let name: wasm_name_t = other.name().to_string().into(); let extern_type: Box = Box::new(other.ty().into()); wasm_exporttype_t { name, extern_type } diff --git a/lib/c-api/src/wasm_c_api/types/function.rs b/lib/c-api/src/wasm_c_api/types/function.rs index 3aabea03f..d2aea10e0 100644 --- a/lib/c-api/src/wasm_c_api/types/function.rs +++ b/lib/c-api/src/wasm_c_api/types/function.rs @@ -1,22 +1,30 @@ -use super::{wasm_externtype_t, wasm_valtype_vec_delete, wasm_valtype_vec_t, WasmExternType}; +use super::{wasm_externtype_t, wasm_valtype_vec_t, WasmExternType}; +use std::fmt; use wasmer_api::{ExternType, FunctionType, ValType}; -#[derive(Debug)] pub(crate) struct WasmFunctionType { pub(crate) function_type: FunctionType, - params: Box, - results: Box, + params: wasm_valtype_vec_t, + results: wasm_valtype_vec_t, } impl WasmFunctionType { pub(crate) fn new(function_type: FunctionType) -> Self { - let params: Box = Box::new(function_type.params().into()); - let results: Box = Box::new(function_type.results().into()); + let params: Vec<_> = function_type + .params() + .iter() + .map(|&valtype| Some(Box::new(valtype.into()))) + .collect(); + let results: Vec<_> = function_type + .results() + .iter() + .map(|&valtype| Some(Box::new(valtype.into()))) + .collect(); Self { function_type, - params, - results, + params: params.into(), + results: results.into(), } } } @@ -27,6 +35,12 @@ impl Clone for WasmFunctionType { } } +impl fmt::Debug for WasmFunctionType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.function_type.fmt(f) + } +} + #[allow(non_camel_case_types)] #[derive(Debug, Clone)] #[repr(transparent)] @@ -52,6 +66,7 @@ impl wasm_functype_t { } wasm_declare_boxed_vec!(functype); +wasm_impl_copy_delete!(functype); #[no_mangle] pub unsafe extern "C" fn wasm_functype_new( @@ -62,44 +77,29 @@ pub unsafe extern "C" fn wasm_functype_new( let results = results?; let params_as_valtype: Vec = params - .into_slice()? + .take() .into_iter() - .map(|val| val.as_ref().into()) + .map(|val| val.as_ref().unwrap().as_ref().into()) .collect::>(); let results_as_valtype: Vec = results - .into_slice()? - .iter() - .map(|val| val.as_ref().into()) + .take() + .into_iter() + .map(|val| val.as_ref().unwrap().as_ref().into()) .collect::>(); - wasm_valtype_vec_delete(Some(params)); - wasm_valtype_vec_delete(Some(results)); - Some(Box::new(wasm_functype_t::new(FunctionType::new( params_as_valtype, results_as_valtype, )))) } -#[no_mangle] -pub unsafe extern "C" fn wasm_functype_delete(_function_type: Option>) {} - -#[no_mangle] -pub unsafe extern "C" fn wasm_functype_copy( - function_type: Option<&wasm_functype_t>, -) -> Option> { - let function_type = function_type?; - - Some(Box::new(function_type.clone())) -} - #[no_mangle] pub unsafe extern "C" fn wasm_functype_params( function_type: Option<&wasm_functype_t>, ) -> Option<&wasm_valtype_vec_t> { let function_type = function_type?; - Some(function_type.inner().params.as_ref()) + Some(&function_type.inner().params) } #[no_mangle] @@ -108,5 +108,5 @@ pub unsafe extern "C" fn wasm_functype_results( ) -> Option<&wasm_valtype_vec_t> { let function_type = function_type?; - Some(function_type.inner().results.as_ref()) + Some(&function_type.inner().results) } diff --git a/lib/c-api/src/wasm_c_api/types/import.rs b/lib/c-api/src/wasm_c_api/types/import.rs index fcc0cd2a0..c2de86aed 100644 --- a/lib/c-api/src/wasm_c_api/types/import.rs +++ b/lib/c-api/src/wasm_c_api/types/import.rs @@ -1,12 +1,12 @@ -use super::{owned_wasm_name_t, wasm_externtype_t, wasm_name_t}; +use super::{wasm_externtype_t, wasm_name_t}; use wasmer_api::ImportType; #[allow(non_camel_case_types)] #[derive(Clone)] #[repr(C)] pub struct wasm_importtype_t { - module: owned_wasm_name_t, - name: owned_wasm_name_t, + module: wasm_name_t, + name: wasm_name_t, extern_type: Box, } @@ -14,31 +14,25 @@ wasm_declare_boxed_vec!(importtype); #[no_mangle] pub extern "C" fn wasm_importtype_new( - module: Option<&wasm_name_t>, - name: Option<&wasm_name_t>, + module: Option>, + name: Option>, extern_type: Option>, ) -> Option> { - let (module, name) = unsafe { - ( - owned_wasm_name_t::new(module?), - owned_wasm_name_t::new(name?), - ) - }; Some(Box::new(wasm_importtype_t { - name, - module, + name: *name?, + module: *module?, extern_type: extern_type?, })) } #[no_mangle] pub extern "C" fn wasm_importtype_module(import_type: &wasm_importtype_t) -> &wasm_name_t { - import_type.module.as_ref() + &import_type.module } #[no_mangle] pub extern "C" fn wasm_importtype_name(import_type: &wasm_importtype_t) -> &wasm_name_t { - import_type.name.as_ref() + &import_type.name } #[no_mangle] @@ -57,8 +51,8 @@ impl From for wasm_importtype_t { impl From<&ImportType> for wasm_importtype_t { fn from(other: &ImportType) -> Self { - let module: owned_wasm_name_t = other.module().to_string().into(); - let name: owned_wasm_name_t = other.name().to_string().into(); + let module: wasm_name_t = other.module().to_string().into(); + let name: wasm_name_t = other.name().to_string().into(); let extern_type: Box = Box::new(other.ty().into()); wasm_importtype_t { diff --git a/lib/c-api/src/wasm_c_api/types/mod.rs b/lib/c-api/src/wasm_c_api/types/mod.rs index 541747045..c5b7ed03f 100644 --- a/lib/c-api/src/wasm_c_api/types/mod.rs +++ b/lib/c-api/src/wasm_c_api/types/mod.rs @@ -28,63 +28,9 @@ wasm_declare_vec!(byte); #[allow(non_camel_case_types)] pub type wasm_name_t = wasm_byte_vec_t; -impl AsRef for wasm_name_t { - fn as_ref(&self) -> &wasm_name_t { - &self - } -} - -/// An owned version of `wasm_name_t`. -/// -/// Assumes that data is either valid host-owned or null. -// NOTE: `wasm_name_t` already does a deep copy, so we just derive `Clone` here. -#[derive(Debug, Clone)] -#[repr(transparent)] -#[allow(non_camel_case_types)] -pub struct owned_wasm_name_t(wasm_name_t); - -impl owned_wasm_name_t { - /// Take ownership of some `wasm_name_t` - /// - /// # Safety - /// You must ensure that the data pointed to by `wasm_name_t` is valid and - /// that it is not owned by anyone else. - pub unsafe fn new(name: &wasm_name_t) -> Self { - Self(wasm_name_t { - size: name.size, - data: name.data, - }) - } -} - -impl Drop for owned_wasm_name_t { - fn drop(&mut self) { - if !self.0.data.is_null() { - let _v = unsafe { Vec::from_raw_parts(self.0.data, self.0.size, self.0.size) }; - self.0.data = std::ptr::null_mut(); - self.0.size = 0; - } - // why can't we call this function? - //unsafe { crate::wasm_c_api::macros::wasm_byte_vec_delete(Some(self.0)) } - } -} - -impl AsRef for owned_wasm_name_t { - fn as_ref(&self) -> &wasm_name_t { - &self.0 - } -} - -impl From for owned_wasm_name_t { +impl From for wasm_name_t { fn from(string: String) -> Self { - let mut boxed_str: Box = string.into_boxed_str(); - let data = boxed_str.as_mut_ptr(); - let size = boxed_str.bytes().len(); - let wasm_name = wasm_name_t { data, size }; - - Box::leak(boxed_str); - - Self(wasm_name) + string.into_bytes().into() } } diff --git a/lib/c-api/src/wasm_c_api/unstable/module.rs b/lib/c-api/src/wasm_c_api/unstable/module.rs index dd041401c..9e85b2f85 100644 --- a/lib/c-api/src/wasm_c_api/unstable/module.rs +++ b/lib/c-api/src/wasm_c_api/unstable/module.rs @@ -144,12 +144,9 @@ pub unsafe extern "C" fn wasmer_module_set_name( // own name: &wasm_name_t, ) -> bool { - let name = match name.into_slice() { - Some(name) => match str::from_utf8(name) { - Ok(name) => name, - Err(_) => return false, // not ideal! - }, - None => return false, + let name = match str::from_utf8(name.as_slice()) { + Ok(name) => name, + Err(_) => return false, // not ideal! }; match Arc::get_mut(&mut module.inner) { diff --git a/lib/c-api/src/wasm_c_api/unstable/wasi.rs b/lib/c-api/src/wasm_c_api/unstable/wasi.rs index 4e3f2235f..9a860af04 100644 --- a/lib/c-api/src/wasm_c_api/unstable/wasi.rs +++ b/lib/c-api/src/wasm_c_api/unstable/wasi.rs @@ -2,10 +2,7 @@ //! API. use super::super::{ - externals::wasm_extern_t, - module::wasm_module_t, - store::wasm_store_t, - types::{owned_wasm_name_t, wasm_name_t}, + externals::wasm_extern_t, module::wasm_module_t, store::wasm_store_t, types::wasm_name_t, wasi::wasi_env_t, }; use crate::error::CApiError; @@ -22,8 +19,8 @@ use wasmer_wasi::{generate_import_object_from_env, get_wasi_version}; #[allow(non_camel_case_types)] #[derive(Clone)] pub struct wasmer_named_extern_t { - module: owned_wasm_name_t, - name: owned_wasm_name_t, + module: wasm_name_t, + name: wasm_name_t, r#extern: Box, } @@ -121,7 +118,7 @@ mod __cbindgen_hack__ { pub extern "C" fn wasmer_named_extern_module( named_extern: Option<&wasmer_named_extern_t>, ) -> Option<&wasm_name_t> { - Some(named_extern?.module.as_ref()) + Some(&named_extern?.module) } /// Non-standard function to get the name of a `wasmer_named_extern_t`. @@ -131,7 +128,7 @@ pub extern "C" fn wasmer_named_extern_module( pub extern "C" fn wasmer_named_extern_name( named_extern: Option<&wasmer_named_extern_t>, ) -> Option<&wasm_name_t> { - Some(named_extern?.name.as_ref()) + Some(&named_extern?.name) } /// Non-standard function to get the wrapped extern of a @@ -179,21 +176,22 @@ fn wasi_get_unordered_imports_inner( let import_object = generate_import_object_from_env(store, wasi_env.inner.clone(), version); - *imports = import_object - .into_iter() - .map(|((module, name), export)| { - let module = module.into(); - let name = name.into(); - let extern_inner = Extern::from_vm_export(store, export); + imports.set_buffer( + import_object + .into_iter() + .map(|((module, name), export)| { + let module = module.into(); + let name = name.into(); + let extern_inner = Extern::from_vm_export(store, export); - Box::new(wasmer_named_extern_t { - module, - name, - r#extern: Box::new(extern_inner.into()), + Some(Box::new(wasmer_named_extern_t { + module, + name, + r#extern: Box::new(extern_inner.into()), + })) }) - }) - .collect::>() - .into(); + .collect::>(), + ); Some(()) } diff --git a/lib/c-api/src/wasm_c_api/value.rs b/lib/c-api/src/wasm_c_api/value.rs index ab6891370..da6b508af 100644 --- a/lib/c-api/src/wasm_c_api/value.rs +++ b/lib/c-api/src/wasm_c_api/value.rs @@ -121,6 +121,15 @@ impl Clone for wasm_val_t { } } +impl Default for wasm_val_t { + fn default() -> Self { + Self { + kind: wasm_valkind_enum::WASM_I64 as _, + of: wasm_val_inner { int64_t: 0 }, + } + } +} + #[no_mangle] pub unsafe extern "C" fn wasm_val_copy( // own diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index 1dc461b18..a2a890608 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -356,11 +356,11 @@ fn wasi_get_imports_inner( let import_object = generate_import_object_from_env(store, wasi_env.inner.clone(), version); - *imports = module + imports.set_buffer(c_try!(module .inner .imports() .map(|import_type| { - let export = c_try!(import_object + let export = import_object .resolve_by_name(import_type.module(), import_type.name()) .ok_or_else(|| CApiError { msg: format!( @@ -368,13 +368,12 @@ fn wasi_get_imports_inner( import_type.module(), import_type.name() ), - })); + })?; let inner = Extern::from_vm_export(store, export); - Some(Box::new(inner.into())) + Ok(Some(Box::new(inner.into()))) }) - .collect::>>()? - .into(); + .collect::, CApiError>>())); Some(()) } diff --git a/lib/c-api/src/wasm_c_api/wat.rs b/lib/c-api/src/wasm_c_api/wat.rs index ed47b996a..2831f7b69 100644 --- a/lib/c-api/src/wasm_c_api/wat.rs +++ b/lib/c-api/src/wasm_c_api/wat.rs @@ -11,16 +11,8 @@ use super::types::wasm_byte_vec_t; #[cfg(feature = "wat")] #[no_mangle] pub unsafe extern "C" fn wat2wasm(wat: &wasm_byte_vec_t, out: &mut wasm_byte_vec_t) { - let wat: &[u8] = match wat.into_slice() { - Some(v) => v, - _ => { - out.data = std::ptr::null_mut(); - out.size = 0; - return; - } - }; - let result: wasm_byte_vec_t = match wasmer_api::wat2wasm(wat) { - Ok(val) => val.into_owned().into(), + match wasmer_api::wat2wasm(wat.as_slice()) { + Ok(val) => out.set_buffer(val.into_owned()), Err(err) => { crate::error::update_last_error(err); out.data = std::ptr::null_mut(); @@ -28,8 +20,6 @@ pub unsafe extern "C" fn wat2wasm(wat: &wasm_byte_vec_t, out: &mut wasm_byte_vec return; } }; - - *out = result; } #[cfg(test)] From cca6a6b6cc627c3192e9f6d17ab502cbd13d0e12 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 16 Nov 2021 03:21:06 +0000 Subject: [PATCH 09/33] Bump minimum required LLVM version to 12 --- .github/workflows/documentation.yaml | 6 +++--- .github/workflows/lint.yaml | 10 +++++----- .github/workflows/test-sys.yaml | 12 ++++++------ Cargo.lock | 4 ++-- Makefile | 17 ++++++----------- lib/compiler-llvm/Cargo.toml | 2 +- lib/compiler-llvm/README.md | 4 ++-- lib/compiler-llvm/src/abi/aarch64_systemv.rs | 13 ++++++------- lib/compiler-llvm/src/abi/x86_64_systemv.rs | 15 +++++++++------ 9 files changed, 40 insertions(+), 43 deletions(-) diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index 31996b46d..3e6d58084 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -26,10 +26,10 @@ jobs: mkdir ${{ env.LLVM_DIR }} tar xf llvm.tar.gz --strip-components=1 -C ${{ env.LLVM_DIR }} echo "${{ env.LLVM_DIR }}/bin" >> $GITHUB_PATH - echo "LLVM_SYS_110_PREFIX=${{ env.LLVM_DIR }}" >> $GITHUB_ENV + echo "LLVM_SYS_120_PREFIX=${{ env.LLVM_DIR }}" >> $GITHUB_ENV env: - LLVM_DIR: ${{ github.workspace }}/llvm-11 - LLVM_URL: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/11.x/linux-amd64.tar.gz' + LLVM_DIR: ${{ github.workspace }}/llvm-12 + LLVM_URL: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/linux-amd64.tar.gz' - name: Build & package documentation run: make package-docs - name: Publish documentation diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 8421665f2..b739b354c 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -23,11 +23,11 @@ jobs: components: rustfmt, clippy - name: Install LLVM (Linux) run: | - curl --proto '=https' --tlsv1.2 -sSf https://github.com/wasmerio/llvm-custom-builds/releases/download/11.x/linux-amd64.tar.gz -L -o /opt/llvm.tar.gz - mkdir -p /opt/llvm-11 - tar xf /opt/llvm.tar.gz --strip-components=1 -C /opt/llvm-11 - echo '/opt/llvm-11/bin' >> $GITHUB_PATH - echo 'LLVM_SYS_110_PREFIX=/opt/llvm-11' >> $GITHUB_ENV + curl --proto '=https' --tlsv1.2 -sSf https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/linux-amd64.tar.gz -L -o /opt/llvm.tar.gz + mkdir -p /opt/llvm-12 + tar xf /opt/llvm.tar.gz --strip-components=1 -C /opt/llvm-12 + echo '/opt/llvm-12/bin' >> $GITHUB_PATH + echo 'LLVM_SYS_120_PREFIX=/opt/llvm-12' >> $GITHUB_ENV - run: make lint env: ENABLE_CRANELIFT: "1" diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index d9d84552d..600d8087d 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -42,7 +42,7 @@ jobs: include: - build: linux-x64 os: ubuntu-18.04 - llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/11.x/linux-amd64.tar.gz' + llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/linux-amd64.tar.gz' artifact_name: 'wasmer-linux-amd64' cross_compilation_artifact_name: 'cross_compiled_from_linux' run_test: true @@ -51,7 +51,7 @@ jobs: use_sccache: true - build: macos-x64 os: macos-11 - llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/11.x/darwin-amd64.tar.gz' + llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/darwin-amd64.tar.gz' artifact_name: 'wasmer-darwin-amd64' cross_compilation_artifact_name: 'cross_compiled_from_mac' run_test: true @@ -68,7 +68,7 @@ jobs: run_test_capi: false - build: windows-x64 os: windows-latest - # llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/11.x/windows-amd64.tar.gz' + # llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/windows-amd64.tar.gz' llvm_choco_version: 12.0.1 artifact_name: 'wasmer-windows-amd64' cross_compilation_artifact_name: 'cross_compiled_from_win' @@ -80,7 +80,7 @@ jobs: # os: [self-hosted, linux, ARM64] # random_sccache_port: true # use_sccache: true - # llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/11.x/linux-aarch64.tar.gz' + # llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/linux-aarch64.tar.gz' # artifact_name: 'wasmer-linux-aarch64' # run_integration_tests: false - build: linux-musl-x64 @@ -122,7 +122,7 @@ jobs: choco install llvm --version ${{ matrix.llvm_choco_version }} --allow-downgrade cd 'C:\Program Files\LLVM\' LLVM_DIR=$(pwd) - echo "LLVM_SYS_110_PREFIX=${LLVM_DIR}" >> $GITHUB_ENV + echo "LLVM_SYS_120_PREFIX=${LLVM_DIR}" >> $GITHUB_ENV - name: Install LLVM (macOS Apple Silicon) if: matrix.os == 'macos-11.0' && !matrix.llvm_url run: | @@ -136,7 +136,7 @@ jobs: mkdir ${LLVM_DIR} tar xf llvm.tar.gz --strip-components=1 -C ${LLVM_DIR} echo "${LLVM_DIR}/bin" >> $GITHUB_PATH - echo "LLVM_SYS_110_PREFIX=${LLVM_DIR}" >> $GITHUB_ENV + echo "LLVM_SYS_120_PREFIX=${LLVM_DIR}" >> $GITHUB_ENV env: LLVM_DIR: .llvm - name: Set up dependencies for Mac OS diff --git a/Cargo.lock b/Cargo.lock index 7abf5a543..420d20412 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1156,9 +1156,9 @@ dependencies = [ [[package]] name = "llvm-sys" -version = "110.0.2" +version = "120.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7cc88ba864d592f52132ed3a19a97118fe16c92a63961f54b0ab7279c5407f" +checksum = "b4a810627ac62b396f5fd2214ba9bbd8748d4d6efdc4d2c1c1303ea7a75763ce" dependencies = [ "cc", "lazy_static", diff --git a/Makefile b/Makefile index ff8043cc6..e4b4e0e2b 100644 --- a/Makefile +++ b/Makefile @@ -134,24 +134,19 @@ ifneq ($(ENABLE_LLVM), 0) LLVM_VERSION := $(shell llvm-config --version) # If findstring is not empty, then it have found the value - ifneq (, $(findstring 12,$(LLVM_VERSION))) + ifneq (, $(findstring 13,$(LLVM_VERSION))) compilers += llvm - else ifneq (, $(findstring 11,$(LLVM_VERSION))) - compilers += llvm - else ifneq (, $(findstring 10,$(LLVM_VERSION))) + else ifneq (, $(findstring 12,$(LLVM_VERSION))) compilers += llvm endif # … or try to autodetect LLVM from `llvm-config-`. else - ifneq (, $(shell which llvm-config-12 2>/dev/null)) + ifneq (, $(shell which llvm-config-13 2>/dev/null)) + LLVM_VERSION := $(shell llvm-config-13 --version) + compilers += llvm + else ifneq (, $(shell which llvm-config-12 2>/dev/null)) LLVM_VERSION := $(shell llvm-config-12 --version) compilers += llvm - else ifneq (, $(shell which llvm-config-11 2>/dev/null)) - LLVM_VERSION := $(shell llvm-config-11 --version) - compilers += llvm - else ifneq (, $(shell which llvm-config-10 2>/dev/null)) - LLVM_VERSION := $(shell llvm-config-10 --version) - compilers += llvm endif endif endif diff --git a/lib/compiler-llvm/Cargo.toml b/lib/compiler-llvm/Cargo.toml index cd287e8e9..c4977476b 100644 --- a/lib/compiler-llvm/Cargo.toml +++ b/lib/compiler-llvm/Cargo.toml @@ -28,7 +28,7 @@ loupe = "0.1" package = "inkwell" version = "0.1.0-beta.4" default-features = false -features = ["llvm11-0", "target-x86", "target-aarch64"] +features = ["llvm12-0", "target-x86", "target-aarch64"] [build-dependencies] cc = "1.0" diff --git a/lib/compiler-llvm/README.md b/lib/compiler-llvm/README.md index 7edc76c6c..51368ebe2 100644 --- a/lib/compiler-llvm/README.md +++ b/lib/compiler-llvm/README.md @@ -24,14 +24,14 @@ to native speeds. ## Requirements The LLVM compiler requires a valid installation of LLVM in your system. -It currently requires **LLVM 11**. +It currently requires **LLVM 12**. You can install LLVM easily on your Debian-like system via this command: ```bash wget https://apt.llvm.org/llvm.sh -O /tmp/llvm.sh -sudo bash /tmp/llvm.sh 11 +sudo bash /tmp/llvm.sh 12 ``` Or in macOS: diff --git a/lib/compiler-llvm/src/abi/aarch64_systemv.rs b/lib/compiler-llvm/src/abi/aarch64_systemv.rs index cd4f1c3ab..cee33c606 100644 --- a/lib/compiler-llvm/src/abi/aarch64_systemv.rs +++ b/lib/compiler-llvm/src/abi/aarch64_systemv.rs @@ -4,7 +4,7 @@ use inkwell::{ attributes::{Attribute, AttributeLoc}, builder::Builder, context::Context, - types::{BasicMetadataTypeEnum, BasicType, FunctionType, StructType}, + types::{AnyType, BasicMetadataTypeEnum, BasicType, FunctionType, StructType}, values::{BasicValue, BasicValueEnum, CallSiteValue, FunctionValue, IntValue, PointerValue}, AddressSpace, }; @@ -206,17 +206,16 @@ impl Abi for Aarch64SystemV { .map(|&ty| type_to_llvm(intrinsics, ty)) .collect::>()?; - let sret = context - .struct_type(&basic_types, false) - .ptr_type(AddressSpace::Generic); + let sret = context.struct_type(&basic_types, false); + let sret_ptr = sret.ptr_type(AddressSpace::Generic); let param_types = - std::iter::once(Ok(sret.as_basic_type_enum())).chain(param_types); + std::iter::once(Ok(sret_ptr.as_basic_type_enum())).chain(param_types); let mut attributes = vec![( - context.create_enum_attribute( + context.create_type_attribute( Attribute::get_named_enum_kind_id("sret"), - 0, + sret.as_any_type_enum(), ), AttributeLoc::Param(0), )]; diff --git a/lib/compiler-llvm/src/abi/x86_64_systemv.rs b/lib/compiler-llvm/src/abi/x86_64_systemv.rs index 695cdf3bd..d9513d07a 100644 --- a/lib/compiler-llvm/src/abi/x86_64_systemv.rs +++ b/lib/compiler-llvm/src/abi/x86_64_systemv.rs @@ -4,7 +4,7 @@ use inkwell::{ attributes::{Attribute, AttributeLoc}, builder::Builder, context::Context, - types::{BasicMetadataTypeEnum, BasicType, FunctionType, StructType}, + types::{AnyType, BasicMetadataTypeEnum, BasicType, FunctionType, StructType}, values::{ BasicValue, BasicValueEnum, CallSiteValue, FloatValue, FunctionValue, IntValue, PointerValue, VectorValue, @@ -263,14 +263,17 @@ impl Abi for X86_64SystemV { .map(|&ty| type_to_llvm(intrinsics, ty)) .collect::>()?; - let sret = context - .struct_type(&basic_types, false) - .ptr_type(AddressSpace::Generic); + let sret = context.struct_type(&basic_types, false); + let sret_ptr = sret.ptr_type(AddressSpace::Generic); - let param_types = std::iter::once(Ok(sret.as_basic_type_enum())).chain(param_types); + let param_types = + std::iter::once(Ok(sret_ptr.as_basic_type_enum())).chain(param_types); let mut attributes = vec![( - context.create_enum_attribute(Attribute::get_named_enum_kind_id("sret"), 0), + context.create_type_attribute( + Attribute::get_named_enum_kind_id("sret"), + sret.as_any_type_enum(), + ), AttributeLoc::Param(0), )]; attributes.append(&mut vmctx_attributes(1)); From efb1f745bd29e6f1b6fc52baf3e7f63dfe3e69f6 Mon Sep 17 00:00:00 2001 From: Bo Yao Date: Tue, 16 Nov 2021 12:42:46 +0800 Subject: [PATCH 10/33] cargo fmt --all --- lib/compiler-singlepass/src/emitter_x64.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/compiler-singlepass/src/emitter_x64.rs b/lib/compiler-singlepass/src/emitter_x64.rs index 4a2d16011..6ed4a7412 100644 --- a/lib/compiler-singlepass/src/emitter_x64.rs +++ b/lib/compiler-singlepass/src/emitter_x64.rs @@ -1,6 +1,8 @@ pub use crate::x64_decl::{GPR, XMM}; use dynasm::dynasm; -use dynasmrt::{x64::X64Relocation, AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler}; +use dynasmrt::{ + x64::X64Relocation, AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi, VecAssembler, +}; type Assembler = VecAssembler; From 724c59c751985cc27a3cc569ec28d8d75385b945 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 17 Nov 2021 00:03:28 +0000 Subject: [PATCH 11/33] More bug fixes --- lib/c-api/examples/imports-exports.c | 3 ++- lib/c-api/src/wasm_c_api/macros.rs | 17 ++++++++++++++--- lib/c-api/src/wasm_c_api/types/export.rs | 8 ++++---- lib/c-api/src/wasm_c_api/types/global.rs | 6 +++--- lib/c-api/src/wasm_c_api/types/import.rs | 8 ++++---- lib/c-api/src/wasm_c_api/types/memory.rs | 8 ++++---- lib/c-api/src/wasm_c_api/types/table.rs | 18 +++++++++--------- lib/c-api/src/wasm_c_api/unstable/module.rs | 2 +- 8 files changed, 41 insertions(+), 29 deletions(-) diff --git a/lib/c-api/examples/imports-exports.c b/lib/c-api/examples/imports-exports.c index aaf4186c3..e56a72327 100644 --- a/lib/c-api/examples/imports-exports.c +++ b/lib/c-api/examples/imports-exports.c @@ -63,7 +63,8 @@ int main(int argc, const char* argv[]) { printf("Instantiating module...\n"); wasm_instance_t* instance = wasm_instance_new(store, module, &import_object, NULL); - wasm_extern_vec_delete(&import_object); + wasm_func_delete(host_func); + wasm_global_delete(host_global); if (!instance) { printf("> Error instantiating module!\n"); diff --git a/lib/c-api/src/wasm_c_api/macros.rs b/lib/c-api/src/wasm_c_api/macros.rs index 22a251bb2..04aa90f3d 100644 --- a/lib/c-api/src/wasm_c_api/macros.rs +++ b/lib/c-api/src/wasm_c_api/macros.rs @@ -3,6 +3,7 @@ macro_rules! wasm_declare_vec_inner { name: $name:ident, ty: $elem_ty:ty, c_ty: $c_ty:expr, + c_val: $c_val:expr, new: $new:ident, empty: $empty:ident, uninit: $uninit:ident, @@ -21,10 +22,8 @@ Read the documentation of [`", $c_ty, "`] to see more concrete examples. # (assert_c! { # #include \"tests/wasmer.h\" # -int main() { +void example(", $c_ty, " x, ", $c_ty, " y) { // Create a vector of 2 `", $c_ty, "`. - ", $c_ty, " x; - ", $c_ty, " y; ", $c_ty, " items[2] = {x, y}; ", stringify!($name), " vector; @@ -36,6 +35,8 @@ int main() { // Free it. ", stringify!($delete), "(&vector); } +# +# int main() { example(", $c_val, ", ", $c_val, "); return 0; } # }) # .success(); # } @@ -134,6 +135,8 @@ int main() { // Free it. ", stringify!($delete), "(&vector); + + return 0; } # }) # .success(); @@ -165,6 +168,8 @@ int main() { // Free it. ", stringify!($delete), "(&vector); + + return 0; } # }) # .success(); @@ -215,6 +220,11 @@ macro_rules! wasm_declare_vec { name: [<$prefix _ $name _vec_t>], ty: [<$prefix _ $name _t>], c_ty: stringify!([<$prefix _ $name _t>]), + c_val: concat!("({ ", + stringify!([<$prefix _ $name _t>]), " foo;\n", + "memset(&foo, 0, sizeof(foo));\n", + "foo;\n", + "})"), new: [<$prefix _ $name _vec_new>], empty: [<$prefix _ $name _vec_new_empty>], uninit: [<$prefix _ $name _vec_new_uninitialized>], @@ -236,6 +246,7 @@ macro_rules! wasm_declare_boxed_vec { name: [<$prefix _ $name _vec_t>], ty: Option]>>, c_ty: stringify!([<$prefix _ $name _t>] *), + c_val: "NULL", new: [<$prefix _ $name _vec_new>], empty: [<$prefix _ $name _vec_new_empty>], uninit: [<$prefix _ $name _vec_new_uninitialized>], diff --git a/lib/c-api/src/wasm_c_api/types/export.rs b/lib/c-api/src/wasm_c_api/types/export.rs index 39d5ab674..a60ccf48b 100644 --- a/lib/c-api/src/wasm_c_api/types/export.rs +++ b/lib/c-api/src/wasm_c_api/types/export.rs @@ -5,7 +5,7 @@ use wasmer_api::ExportType; #[derive(Clone)] pub struct wasm_exporttype_t { name: wasm_name_t, - extern_type: Box, + extern_type: wasm_externtype_t, } wasm_declare_boxed_vec!(exporttype); @@ -18,7 +18,7 @@ pub extern "C" fn wasm_exporttype_new( ) -> Box { Box::new(wasm_exporttype_t { name: name.clone(), - extern_type, + extern_type: *extern_type, }) } @@ -29,7 +29,7 @@ pub extern "C" fn wasm_exporttype_name(export_type: &wasm_exporttype_t) -> &wasm #[no_mangle] pub extern "C" fn wasm_exporttype_type(export_type: &wasm_exporttype_t) -> &wasm_externtype_t { - export_type.extern_type.as_ref() + &export_type.extern_type } impl From for wasm_exporttype_t { @@ -41,7 +41,7 @@ impl From for wasm_exporttype_t { impl From<&ExportType> for wasm_exporttype_t { fn from(other: &ExportType) -> Self { let name: wasm_name_t = other.name().to_string().into(); - let extern_type: Box = Box::new(other.ty().into()); + let extern_type: wasm_externtype_t = other.ty().into(); wasm_exporttype_t { name, extern_type } } diff --git a/lib/c-api/src/wasm_c_api/types/global.rs b/lib/c-api/src/wasm_c_api/types/global.rs index bdb9b7eb9..26342e428 100644 --- a/lib/c-api/src/wasm_c_api/types/global.rs +++ b/lib/c-api/src/wasm_c_api/types/global.rs @@ -8,12 +8,12 @@ use wasmer_api::{ExternType, GlobalType}; #[derive(Debug, Clone)] pub(crate) struct WasmGlobalType { pub(crate) global_type: GlobalType, - content: Box, + content: wasm_valtype_t, } impl WasmGlobalType { pub(crate) fn new(global_type: GlobalType) -> Self { - let content = Box::new(global_type.ty.into()); + let content = global_type.ty.into(); Self { global_type, @@ -79,5 +79,5 @@ pub unsafe extern "C" fn wasm_globaltype_mutability( pub unsafe extern "C" fn wasm_globaltype_content( global_type: &wasm_globaltype_t, ) -> &wasm_valtype_t { - global_type.inner().content.as_ref() + &global_type.inner().content } diff --git a/lib/c-api/src/wasm_c_api/types/import.rs b/lib/c-api/src/wasm_c_api/types/import.rs index c2de86aed..5e075ece7 100644 --- a/lib/c-api/src/wasm_c_api/types/import.rs +++ b/lib/c-api/src/wasm_c_api/types/import.rs @@ -7,7 +7,7 @@ use wasmer_api::ImportType; pub struct wasm_importtype_t { module: wasm_name_t, name: wasm_name_t, - extern_type: Box, + extern_type: wasm_externtype_t, } wasm_declare_boxed_vec!(importtype); @@ -21,7 +21,7 @@ pub extern "C" fn wasm_importtype_new( Some(Box::new(wasm_importtype_t { name: *name?, module: *module?, - extern_type: extern_type?, + extern_type: *extern_type?, })) } @@ -37,7 +37,7 @@ pub extern "C" fn wasm_importtype_name(import_type: &wasm_importtype_t) -> &wasm #[no_mangle] pub extern "C" fn wasm_importtype_type(import_type: &wasm_importtype_t) -> &wasm_externtype_t { - import_type.extern_type.as_ref() + &import_type.extern_type } #[no_mangle] @@ -53,7 +53,7 @@ impl From<&ImportType> for wasm_importtype_t { fn from(other: &ImportType) -> Self { let module: wasm_name_t = other.module().to_string().into(); let name: wasm_name_t = other.name().to_string().into(); - let extern_type: Box = Box::new(other.ty().into()); + let extern_type: wasm_externtype_t = other.ty().into(); wasm_importtype_t { module, diff --git a/lib/c-api/src/wasm_c_api/types/memory.rs b/lib/c-api/src/wasm_c_api/types/memory.rs index b98c16494..e60eb8459 100644 --- a/lib/c-api/src/wasm_c_api/types/memory.rs +++ b/lib/c-api/src/wasm_c_api/types/memory.rs @@ -4,18 +4,18 @@ use wasmer_api::{ExternType, MemoryType, Pages}; #[derive(Debug, Clone)] pub(crate) struct WasmMemoryType { pub(crate) memory_type: MemoryType, - limits: Box, + limits: wasm_limits_t, } impl WasmMemoryType { pub(crate) fn new(memory_type: MemoryType) -> Self { - let limits = Box::new(wasm_limits_t { + let limits = wasm_limits_t { min: memory_type.minimum.0 as _, max: memory_type .maximum .map(|max| max.0 as _) .unwrap_or(LIMITS_MAX_SENTINEL), - }); + }; Self { memory_type, @@ -79,5 +79,5 @@ const LIMITS_MAX_SENTINEL: u32 = u32::max_value(); #[no_mangle] pub unsafe extern "C" fn wasm_memorytype_limits(memory_type: &wasm_memorytype_t) -> &wasm_limits_t { - memory_type.inner().limits.as_ref() + &memory_type.inner().limits } diff --git a/lib/c-api/src/wasm_c_api/types/table.rs b/lib/c-api/src/wasm_c_api/types/table.rs index a17c3aef3..a3d74ee79 100644 --- a/lib/c-api/src/wasm_c_api/types/table.rs +++ b/lib/c-api/src/wasm_c_api/types/table.rs @@ -10,21 +10,21 @@ const LIMITS_MAX_SENTINEL: u32 = u32::max_value(); #[derive(Debug, Clone)] pub(crate) struct WasmTableType { - pub(crate) table_type: TableType, - limits: Box, - content: Box, + pub(crate) _table_type: TableType, + limits: wasm_limits_t, + content: wasm_valtype_t, } impl WasmTableType { pub(crate) fn new(table_type: TableType) -> Self { - let limits = Box::new(wasm_limits_t { + let limits = wasm_limits_t { min: table_type.minimum as _, max: table_type.maximum.unwrap_or(LIMITS_MAX_SENTINEL), - }); - let content = Box::new(table_type.ty.into()); + }; + let content = table_type.ty.into(); Self { - table_type, + _table_type: table_type, limits, content, } @@ -79,12 +79,12 @@ pub unsafe extern "C" fn wasm_tabletype_new( #[no_mangle] pub unsafe extern "C" fn wasm_tabletype_limits(table_type: &wasm_tabletype_t) -> &wasm_limits_t { - table_type.inner().limits.as_ref() + &table_type.inner().limits } #[no_mangle] pub unsafe extern "C" fn wasm_tabletype_element(table_type: &wasm_tabletype_t) -> &wasm_valtype_t { - table_type.inner().content.as_ref() + &table_type.inner().content } #[no_mangle] diff --git a/lib/c-api/src/wasm_c_api/unstable/module.rs b/lib/c-api/src/wasm_c_api/unstable/module.rs index 9e85b2f85..81528369b 100644 --- a/lib/c-api/src/wasm_c_api/unstable/module.rs +++ b/lib/c-api/src/wasm_c_api/unstable/module.rs @@ -70,7 +70,7 @@ pub unsafe extern "C" fn wasmer_module_name( } }; - *out = name.as_bytes().to_vec().into(); + out.set_buffer(name.as_bytes().to_vec()); } /// Unstable non-standard Wasmer-specific API to set the module's From 4692b0e231abd932880fcf4b40a460ee6a2a5111 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 17 Nov 2021 15:43:10 +0000 Subject: [PATCH 12/33] Don't emit dynamic symbols for executables on musl --- .cargo/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index d83dcde74..f9b89e8f7 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,4 @@ -[target.'cfg(target_os = "linux")'] +[target.'cfg(all(target_os = "linux", target_env = "gnu"))'] rustflags = [ # Put the VM functions in the dynamic symbol table. "-C", "link-arg=-Wl,-E", From c9e46c47ebada1d83d25cbffab4badac35c157e0 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 17 Nov 2021 16:14:53 +0000 Subject: [PATCH 13/33] Use LLVM 13 in CI --- .github/workflows/documentation.yaml | 8 ++++---- .github/workflows/lint.yaml | 4 ++-- .github/workflows/test-sys.yaml | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index 3e6d58084..bc2fd7d43 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -22,14 +22,14 @@ jobs: - name: Install LLVM shell: bash run: | - curl --proto '=https' --tlsv1.2 -sSf ${{ env.LLVM_URL }} -L -o llvm.tar.gz + curl --proto '=https' --tlsv1.2 -sSf ${{ env.LLVM_URL }} -L -o llvm.tar.xz mkdir ${{ env.LLVM_DIR }} - tar xf llvm.tar.gz --strip-components=1 -C ${{ env.LLVM_DIR }} + tar xf llvm.tar.xz --strip-components=1 -C ${{ env.LLVM_DIR }} echo "${{ env.LLVM_DIR }}/bin" >> $GITHUB_PATH echo "LLVM_SYS_120_PREFIX=${{ env.LLVM_DIR }}" >> $GITHUB_ENV env: - LLVM_DIR: ${{ github.workspace }}/llvm-12 - LLVM_URL: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/linux-amd64.tar.gz' + LLVM_DIR: ${{ github.workspace }}/llvm-13 + LLVM_URL: 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz' - name: Build & package documentation run: make package-docs - name: Publish documentation diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index b739b354c..a0a66703b 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -23,9 +23,9 @@ jobs: components: rustfmt, clippy - name: Install LLVM (Linux) run: | - curl --proto '=https' --tlsv1.2 -sSf https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/linux-amd64.tar.gz -L -o /opt/llvm.tar.gz + curl --proto '=https' --tlsv1.2 -sSf https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz -L -o /opt/llvm.tar.xz mkdir -p /opt/llvm-12 - tar xf /opt/llvm.tar.gz --strip-components=1 -C /opt/llvm-12 + tar xf /opt/llvm.tar.xz --strip-components=1 -C /opt/llvm-12 echo '/opt/llvm-12/bin' >> $GITHUB_PATH echo 'LLVM_SYS_120_PREFIX=/opt/llvm-12' >> $GITHUB_ENV - run: make lint diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index 600d8087d..70b9a7438 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -42,7 +42,7 @@ jobs: include: - build: linux-x64 os: ubuntu-18.04 - llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/linux-amd64.tar.gz' + llvm_url: 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz' artifact_name: 'wasmer-linux-amd64' cross_compilation_artifact_name: 'cross_compiled_from_linux' run_test: true @@ -51,7 +51,7 @@ jobs: use_sccache: true - build: macos-x64 os: macos-11 - llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/darwin-amd64.tar.gz' + llvm_url: 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-x86_64-apple-darwin.tar.xz' artifact_name: 'wasmer-darwin-amd64' cross_compilation_artifact_name: 'cross_compiled_from_mac' run_test: true @@ -69,7 +69,7 @@ jobs: - build: windows-x64 os: windows-latest # llvm_url: 'https://github.com/wasmerio/llvm-custom-builds/releases/download/12.x/windows-amd64.tar.gz' - llvm_choco_version: 12.0.1 + llvm_choco_version: 13.0.0 artifact_name: 'wasmer-windows-amd64' cross_compilation_artifact_name: 'cross_compiled_from_win' run_integration_tests: true @@ -131,10 +131,10 @@ jobs: if: matrix.llvm_url shell: bash run: | - curl --proto '=https' --tlsv1.2 -sSf ${{ matrix.llvm_url }} -L -o llvm.tar.gz + curl --proto '=https' --tlsv1.2 -sSf ${{ matrix.llvm_url }} -L -o llvm.tar.xz LLVM_DIR=$(pwd)/${{ env.LLVM_DIR }} mkdir ${LLVM_DIR} - tar xf llvm.tar.gz --strip-components=1 -C ${LLVM_DIR} + tar xf llvm.tar.xz --strip-components=1 -C ${LLVM_DIR} echo "${LLVM_DIR}/bin" >> $GITHUB_PATH echo "LLVM_SYS_120_PREFIX=${LLVM_DIR}" >> $GITHUB_ENV env: From 10927138976e27ae2e41c32b0fbb379396e46c97 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 18 Nov 2021 02:34:30 +0000 Subject: [PATCH 14/33] Fix memory leak when obtaining the stack bounds of a thread --- lib/vm/src/trap/traphandlers.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/vm/src/trap/traphandlers.rs b/lib/vm/src/trap/traphandlers.rs index c39b2f7ad..79660134f 100644 --- a/lib/vm/src/trap/traphandlers.rs +++ b/lib/vm/src/trap/traphandlers.rs @@ -99,13 +99,16 @@ cfg_if::cfg_if! { unsafe fn thread_stack() -> (usize, usize) { let this_thread = libc::pthread_self(); let mut thread_attrs: libc::pthread_attr_t = mem::zeroed(); - #[cfg(not(target_os = "freebsd"))] - libc::pthread_getattr_np(this_thread, &mut thread_attrs); - #[cfg(target_os = "freebsd")] - libc::pthread_attr_get_np(this_thread, &mut thread_attrs); let mut stackaddr: *mut libc::c_void = ptr::null_mut(); let mut stacksize: libc::size_t = 0; - libc::pthread_attr_getstack(&thread_attrs, &mut stackaddr, &mut stacksize); + #[cfg(not(target_os = "freebsd"))] + let ok = libc::pthread_getattr_np(this_thread, &mut thread_attrs); + #[cfg(target_os = "freebsd")] + let ok = libc::pthread_attr_get_np(this_thread, &mut thread_attrs); + if ok == 0 { + libc::pthread_attr_getstack(&thread_attrs, &mut stackaddr, &mut stacksize); + libc::pthread_attr_destroy(&mut thread_attrs); + } (stackaddr as usize, stacksize) } From cbd169add25fbe226b6d61e4f59919f996a0e3a3 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Fri, 19 Nov 2021 10:32:34 +0000 Subject: [PATCH 15/33] Upgrade rkyv to 0.7 --- Cargo.lock | 44 +++++++++++++++++++++++---- lib/compiler/Cargo.toml | 4 +-- lib/compiler/src/section.rs | 2 +- lib/engine-dylib/Cargo.toml | 2 +- lib/engine-dylib/src/serialize.rs | 16 +++++----- lib/engine-universal/Cargo.toml | 2 +- lib/engine-universal/src/serialize.rs | 16 +++++----- lib/types/Cargo.toml | 2 +- lib/types/src/indexes.rs | 24 +++++++-------- lib/types/src/module.rs | 15 ++++----- lib/vm/Cargo.toml | 2 +- 11 files changed, 79 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 815f04419..23a613570 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,6 +210,27 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +[[package]] +name = "bytecheck" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb738a1e65989ecdcd5bba16079641bd7209688fa546e1064832fd6e012fd32a" +dependencies = [ + "bytecheck_derive", + "ptr_meta", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b4dff26fdc9f847dab475c9fec16f2cba82d5aa1f09981b87c44520721e10a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "byteorder" version = "1.4.3" @@ -1773,22 +1794,33 @@ dependencies = [ ] [[package]] -name = "rkyv" -version = "0.6.7" +name = "rend" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb135b3e5e3311f0a254bfb00333f4bac9ef1d89888b84242a89eb8722b09a07" +checksum = "6d0351a2e529ee30d571ef31faa5a4e0b9addaad087697b77efb20d2809e41c7" dependencies = [ - "memoffset", + "bytecheck", +] + +[[package]] +name = "rkyv" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e419b2e30d088b21c4bf3072561535305df8066e89937ad05fc205b99874c23c" +dependencies = [ + "bytecheck", + "hashbrown 0.11.2", "ptr_meta", + "rend", "rkyv_derive", "seahash", ] [[package]] name = "rkyv_derive" -version = "0.6.7" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba8f489f6b6d8551bb15904293c1ad58a6abafa7d8390d15f7ed05a2afcd87d5" +checksum = "2ae58c4ba80f15f2f0842f4c61729e92c4e33a09bd78196c2b1ab9b0771a3ddf" dependencies = [ "proc-macro2", "quote", diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml index 215c49bdc..ce07cc764 100644 --- a/lib/compiler/Cargo.toml +++ b/lib/compiler/Cargo.toml @@ -20,8 +20,8 @@ hashbrown = { version = "0.11", optional = true } serde = { version = "1.0", features = ["derive"], optional = true } thiserror = "1.0" serde_bytes = { version = "0.11", optional = true } -smallvec = "1.6" -rkyv = { version = "0.6.1", optional = true } +smallvec = "1.6" +rkyv = { version = "0.7.20", optional = true } loupe = "0.1" [features] diff --git a/lib/compiler/src/section.rs b/lib/compiler/src/section.rs index 98063705d..1ac45eeda 100644 --- a/lib/compiler/src/section.rs +++ b/lib/compiler/src/section.rs @@ -22,7 +22,7 @@ use wasmer_types::entity::entity_impl; )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, MemoryUsage)] pub struct SectionIndex(u32); diff --git a/lib/engine-dylib/Cargo.toml b/lib/engine-dylib/Cargo.toml index 1b356e070..aa9e1a356 100644 --- a/lib/engine-dylib/Cargo.toml +++ b/lib/engine-dylib/Cargo.toml @@ -23,7 +23,7 @@ leb128 = "0.2" libloading = "0.7" tempfile = "3.1" which = "4.0" -rkyv = "0.6.1" +rkyv = "0.7.20" loupe = "0.1" [features] diff --git a/lib/engine-dylib/src/serialize.rs b/lib/engine-dylib/src/serialize.rs index 7021834a8..b08731857 100644 --- a/lib/engine-dylib/src/serialize.rs +++ b/lib/engine-dylib/src/serialize.rs @@ -1,10 +1,8 @@ use loupe::MemoryUsage; use rkyv::{ - archived_value, - de::{adapters::SharedDeserializerAdapter, deserializers::AllocDeserializer}, - ser::adapters::SharedSerializerAdapter, - ser::{serializers::WriteSerializer, Serializer as RkyvSerializer}, - Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, + archived_value, de::deserializers::SharedDeserializeMap, ser::serializers::AllocSerializer, + ser::Serializer as RkyvSerializer, Archive, Deserialize as RkyvDeserialize, + Serialize as RkyvSerialize, }; use serde::{Deserialize, Serialize}; use std::error::Error; @@ -59,11 +57,11 @@ impl ModuleMetadata { } pub fn serialize(&mut self) -> Result, CompileError> { - let mut serializer = SharedSerializerAdapter::new(WriteSerializer::new(vec![])); + let mut serializer = AllocSerializer::<4096>::default(); let pos = serializer.serialize_value(self).map_err(to_compile_error)? as u64; - let mut serialized_data = serializer.into_inner().into_inner(); + let mut serialized_data = serializer.into_serializer().into_inner(); serialized_data.extend_from_slice(&pos.to_le_bytes()); - Ok(serialized_data) + Ok(serialized_data.to_vec()) } pub unsafe fn deserialize(metadata_slice: &[u8]) -> Result { @@ -86,7 +84,7 @@ impl ModuleMetadata { pub fn deserialize_from_archive( archived: &ArchivedModuleMetadata, ) -> Result { - let mut deserializer = SharedDeserializerAdapter::new(AllocDeserializer); + let mut deserializer = SharedDeserializeMap::new(); RkyvDeserialize::deserialize(archived, &mut deserializer) .map_err(|e| DeserializeError::CorruptedBinary(format!("{:?}", e))) } diff --git a/lib/engine-universal/Cargo.toml b/lib/engine-universal/Cargo.toml index 683acc8fb..9bee0203f 100644 --- a/lib/engine-universal/Cargo.toml +++ b/lib/engine-universal/Cargo.toml @@ -19,7 +19,7 @@ wasmer-engine = { path = "../engine", version = "2.0.0" } region = "3.0" cfg-if = "1.0" leb128 = "0.2" -rkyv = "0.6.1" +rkyv = "0.7.20" loupe = "0.1" [target.'cfg(target_os = "windows")'.dependencies] diff --git a/lib/engine-universal/src/serialize.rs b/lib/engine-universal/src/serialize.rs index 6e3c768d4..4a8d62c13 100644 --- a/lib/engine-universal/src/serialize.rs +++ b/lib/engine-universal/src/serialize.rs @@ -1,10 +1,8 @@ use loupe::MemoryUsage; use rkyv::{ - archived_value, - de::{adapters::SharedDeserializerAdapter, deserializers::AllocDeserializer}, - ser::adapters::SharedSerializerAdapter, - ser::{serializers::WriteSerializer, Serializer as RkyvSerializer}, - Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, + archived_value, de::deserializers::SharedDeserializeMap, ser::serializers::AllocSerializer, + ser::Serializer as RkyvSerializer, Archive, Deserialize as RkyvDeserialize, + Serialize as RkyvSerialize, }; use wasmer_compiler::{ CompileModuleInfo, CompiledFunctionFrameInfo, CustomSection, Dwarf, FunctionBody, @@ -49,13 +47,13 @@ impl SerializableModule { /// The bytes will have the following format: /// RKYV serialization (any length) + POS (8 bytes) pub fn serialize(&self) -> Result, SerializeError> { - let mut serializer = SharedSerializerAdapter::new(WriteSerializer::new(vec![])); + let mut serializer = AllocSerializer::<4096>::default(); let pos = serializer .serialize_value(self) .map_err(to_serialize_error)? as u64; - let mut serialized_data = serializer.into_inner().into_inner(); + let mut serialized_data = serializer.into_serializer().into_inner(); serialized_data.extend_from_slice(&pos.to_le_bytes()); - Ok(serialized_data) + Ok(serialized_data.to_vec()) } /// Deserialize a Module from a slice. @@ -98,7 +96,7 @@ impl SerializableModule { pub fn deserialize_from_archive( archived: &ArchivedSerializableModule, ) -> Result { - let mut deserializer = SharedDeserializerAdapter::new(AllocDeserializer); + let mut deserializer = SharedDeserializeMap::new(); RkyvDeserialize::deserialize(archived, &mut deserializer) .map_err(|e| DeserializeError::CorruptedBinary(format!("{:?}", e))) } diff --git a/lib/types/Cargo.toml b/lib/types/Cargo.toml index 11c5d3b54..3c3ea3b9b 100644 --- a/lib/types/Cargo.toml +++ b/lib/types/Cargo.toml @@ -14,7 +14,7 @@ edition = "2018" serde = { version = "1.0", features = ["derive", "rc"], optional = true, default-features = false } thiserror = "1.0" indexmap = { version = "1.6", features = ["serde-1"] } -rkyv = { version = "0.6.1", optional = true } +rkyv = { version = "0.7.20", optional = true } loupe = { version = "0.1", features = ["enable-indexmap"] } [features] diff --git a/lib/types/src/indexes.rs b/lib/types/src/indexes.rs index af1975ecc..b7fd2008b 100644 --- a/lib/types/src/indexes.rs +++ b/lib/types/src/indexes.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct LocalFunctionIndex(u32); entity_impl!(LocalFunctionIndex); @@ -44,7 +44,7 @@ entity_impl!(LocalMemoryIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct LocalGlobalIndex(u32); entity_impl!(LocalGlobalIndex); @@ -60,7 +60,7 @@ entity_impl!(ArchivedLocalGlobalIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct FunctionIndex(u32); entity_impl!(FunctionIndex); @@ -76,7 +76,7 @@ entity_impl!(ArchivedFunctionIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct TableIndex(u32); entity_impl!(TableIndex); @@ -92,7 +92,7 @@ entity_impl!(ArchivedTableIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct GlobalIndex(u32); entity_impl!(GlobalIndex); @@ -108,7 +108,7 @@ entity_impl!(ArchivedGlobalIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct MemoryIndex(u32); entity_impl!(MemoryIndex); @@ -124,7 +124,7 @@ entity_impl!(ArchivedMemoryIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct SignatureIndex(u32); entity_impl!(SignatureIndex); @@ -140,7 +140,7 @@ entity_impl!(ArchivedSignatureIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct DataIndex(u32); entity_impl!(DataIndex); @@ -156,7 +156,7 @@ entity_impl!(ArchivedDataIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct ElemIndex(u32); entity_impl!(ElemIndex); @@ -172,7 +172,7 @@ entity_impl!(ArchivedElemIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub struct CustomSectionIndex(u32); entity_impl!(CustomSectionIndex); @@ -188,7 +188,7 @@ entity_impl!(ArchivedCustomSectionIndex); )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub enum ExportIndex { /// Function export. @@ -210,7 +210,7 @@ pub enum ExportIndex { )] #[cfg_attr( feature = "enable-rkyv", - archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) + archive_attr(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)) )] pub enum ImportIndex { /// Function import. diff --git a/lib/types/src/module.rs b/lib/types/src/module.rs index a4a02ed7d..0c1d0c758 100644 --- a/lib/types/src/module.rs +++ b/lib/types/src/module.rs @@ -15,18 +15,17 @@ use crate::{ }; use indexmap::IndexMap; use loupe::MemoryUsage; +use rkyv::ser::ScratchSpace; #[cfg(feature = "enable-rkyv")] use rkyv::{ - de::SharedDeserializer, ser::Serializer, ser::SharedSerializer, Archive, Archived, - Deserialize as RkyvDeserialize, Fallible, Serialize as RkyvSerialize, + de::SharedDeserializeRegistry, ser::Serializer, ser::SharedSerializeRegistry, Archive, + Archived, Deserialize as RkyvDeserialize, Fallible, Serialize as RkyvSerialize, }; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt; use std::iter::ExactSizeIterator; -#[cfg(feature = "enable-rkyv")] -use std::mem::MaybeUninit; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::sync::Arc; @@ -228,20 +227,22 @@ impl Archive for ModuleInfo { type Archived = ::Archived; type Resolver = ::Resolver; - fn resolve(&self, pos: usize, resolver: Self::Resolver, out: &mut MaybeUninit) { + unsafe fn resolve(&self, pos: usize, resolver: Self::Resolver, out: *mut Self::Archived) { ArchivableModuleInfo::from(self).resolve(pos, resolver, out) } } #[cfg(feature = "enable-rkyv")] -impl RkyvSerialize for ModuleInfo { +impl RkyvSerialize + for ModuleInfo +{ fn serialize(&self, serializer: &mut S) -> Result { ArchivableModuleInfo::from(self).serialize(serializer) } } #[cfg(feature = "enable-rkyv")] -impl RkyvDeserialize +impl RkyvDeserialize for Archived { fn deserialize(&self, deserializer: &mut D) -> Result { diff --git a/lib/vm/Cargo.toml b/lib/vm/Cargo.toml index b817d3642..4f1219ef5 100644 --- a/lib/vm/Cargo.toml +++ b/lib/vm/Cargo.toml @@ -21,7 +21,7 @@ more-asserts = "0.2" cfg-if = "1.0" backtrace = "0.3" serde = { version = "1.0", features = ["derive", "rc"] } -rkyv = { version = "0.6.1", optional = true} +rkyv = { version = "0.7.20", optional = true } loupe = { version = "0.1", features = ["enable-indexmap"] } [target.'cfg(target_os = "windows")'.dependencies] From 9e0fe7d6950439739232c0feefbf3a64d24dda86 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Fri, 19 Nov 2021 14:32:52 +0000 Subject: [PATCH 16/33] Use BTreeMap in ModuleInfo to avoid non-deterministic serialization --- lib/types/src/module.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/types/src/module.rs b/lib/types/src/module.rs index 0c1d0c758..135e17ce1 100644 --- a/lib/types/src/module.rs +++ b/lib/types/src/module.rs @@ -23,7 +23,7 @@ use rkyv::{ }; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::iter::ExactSizeIterator; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; @@ -141,10 +141,10 @@ pub struct ArchivableModuleInfo { exports: ArchivableIndexMap, start_function: Option, table_initializers: Vec, - passive_elements: HashMap>, - passive_data: HashMap>, + passive_elements: BTreeMap>, + passive_data: BTreeMap>, global_initializers: PrimaryMap, - function_names: HashMap, + function_names: BTreeMap, signatures: PrimaryMap, functions: PrimaryMap, tables: PrimaryMap, @@ -167,10 +167,10 @@ impl From for ArchivableModuleInfo { exports: ArchivableIndexMap::from(it.exports), start_function: it.start_function, table_initializers: it.table_initializers, - passive_elements: it.passive_elements, - passive_data: it.passive_data, + passive_elements: it.passive_elements.into_iter().collect(), + passive_data: it.passive_data.into_iter().collect(), global_initializers: it.global_initializers, - function_names: it.function_names, + function_names: it.function_names.into_iter().collect(), signatures: it.signatures, functions: it.functions, tables: it.tables, @@ -196,10 +196,10 @@ impl From for ModuleInfo { exports: it.exports.into(), start_function: it.start_function, table_initializers: it.table_initializers, - passive_elements: it.passive_elements, - passive_data: it.passive_data, + passive_elements: it.passive_elements.into_iter().collect(), + passive_data: it.passive_data.into_iter().collect(), global_initializers: it.global_initializers, - function_names: it.function_names, + function_names: it.function_names.into_iter().collect(), signatures: it.signatures, functions: it.functions, tables: it.tables, From 57f876b92295fca5152cfbe3c3ab05cfc1ccb68a Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 15:30:15 +0000 Subject: [PATCH 17/33] Make DWARF frametable generation deterministic in Cranelift --- lib/compiler-cranelift/src/compiler.rs | 94 ++++++++++++-------------- 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index c836b9ce1..6a887752f 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -92,14 +92,13 @@ impl Compiler for CraneliftCompiler { // FDEs will cause some issues in Linux. None } else { - use std::sync::Mutex; match target.triple().default_calling_convention() { Ok(CallingConvention::SystemV) => { match isa.create_systemv_cie() { Some(cie) => { let mut dwarf_frametable = FrameTable::default(); let cie_id = dwarf_frametable.add_cie(cie); - Some((Arc::new(Mutex::new(dwarf_frametable)), cie_id)) + Some((dwarf_frametable, cie_id)) } // Even though we are in a SystemV system, Cranelift doesn't support it None => None, @@ -134,7 +133,7 @@ impl Compiler for CraneliftCompiler { #[cfg(all(target_arch = "x86_64", target_os = "linux"))] let probestack_trampoline_relocation_target = SectionIndex::new(custom_sections.len() - 1); - let functions = function_body_inputs + let (functions, fdes): (Vec, Vec<_>) = function_body_inputs .iter() .collect::)>>() .par_iter() @@ -190,31 +189,25 @@ impl Compiler for CraneliftCompiler { CompileError::Codegen(pretty_error(&context.func, Some(&*isa), error)) })?; - let unwind_info = match compiled_function_unwind_info(&*isa, &context)? { + let (unwind_info, fde) = match compiled_function_unwind_info(&*isa, &context)? { #[cfg(feature = "unwind")] CraneliftUnwindInfo::FDE(fde) => { - if let Some((dwarf_frametable, cie_id)) = &dwarf_frametable { - dwarf_frametable - .lock() - .expect("Can't write into DWARF frametable") - .add_fde( - *cie_id, - fde.to_fde(Address::Symbol { - // The symbol is the kind of relocation. - // "0" is used for functions - symbol: WriterRelocate::FUNCTION_SYMBOL, - // We use the addend as a way to specify the - // function index - addend: i.index() as _, - }), - ); + if dwarf_frametable.is_some() { + let fde = fde.to_fde(Address::Symbol { + // The symbol is the kind of relocation. + // "0" is used for functions + symbol: WriterRelocate::FUNCTION_SYMBOL, + // We use the addend as a way to specify the + // function index + addend: i.index() as _, + }); // The unwind information is inserted into the dwarf section - Some(CompiledFunctionUnwindInfo::Dwarf) + (Some(CompiledFunctionUnwindInfo::Dwarf), Some(fde)) } else { - None + (None, None) } } - other => other.maybe_into_to_windows_unwind(), + other => (other.maybe_into_to_windows_unwind(), None), }; let range = reader.range(); @@ -223,40 +216,41 @@ impl Compiler for CraneliftCompiler { // We transform the Cranelift JumpTable's into compiler JumpTables let func_jt_offsets = transform_jump_table(context.func.jt_offsets); - Ok(CompiledFunction { - body: FunctionBody { - body: code_buf, - unwind_info, + Ok(( + CompiledFunction { + body: FunctionBody { + body: code_buf, + unwind_info, + }, + jt_offsets: func_jt_offsets, + relocations: reloc_sink.func_relocs, + frame_info: CompiledFunctionFrameInfo { + address_map, + traps: trap_sink.traps, + }, }, - jt_offsets: func_jt_offsets, - relocations: reloc_sink.func_relocs, - frame_info: CompiledFunctionFrameInfo { - address_map, - traps: trap_sink.traps, - }, - }) + fde, + )) }) .collect::, CompileError>>()? .into_iter() - .collect::>(); + .unzip(); #[cfg(feature = "unwind")] - let dwarf = { - let dwarf = if let Some((dwarf_frametable, _cie_id)) = dwarf_frametable { - let mut eh_frame = EhFrame(WriterRelocate::new(target.triple().endianness().ok())); - dwarf_frametable - .lock() - .unwrap() - .write_eh_frame(&mut eh_frame) - .unwrap(); + let dwarf = if let Some((mut dwarf_frametable, cie_id)) = dwarf_frametable { + for fde in fdes { + if let Some(fde) = fde { + dwarf_frametable.add_fde(cie_id, fde); + } + } + let mut eh_frame = EhFrame(WriterRelocate::new(target.triple().endianness().ok())); + dwarf_frametable.write_eh_frame(&mut eh_frame).unwrap(); - let eh_frame_section = eh_frame.0.into_section(); - custom_sections.push(eh_frame_section); - Some(Dwarf::new(SectionIndex::new(custom_sections.len() - 1))) - } else { - None - }; - dwarf + let eh_frame_section = eh_frame.0.into_section(); + custom_sections.push(eh_frame_section); + Some(Dwarf::new(SectionIndex::new(custom_sections.len() - 1))) + } else { + None }; #[cfg(not(feature = "unwind"))] let dwarf = None; @@ -289,7 +283,7 @@ impl Compiler for CraneliftCompiler { .collect::>(); Ok(Compilation::new( - functions, + functions.into_iter().collect(), custom_sections, function_call_trampolines, dynamic_function_trampolines, From a61893bbb320454087c287fb84acf436fa1db3f6 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 15:30:38 +0000 Subject: [PATCH 18/33] Use BTreeMap when serializing an IndexMap --- lib/types/src/archives.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/types/src/archives.rs b/lib/types/src/archives.rs index 0a7d84a58..48e30272e 100644 --- a/lib/types/src/archives.rs +++ b/lib/types/src/archives.rs @@ -3,19 +3,21 @@ use core::hash::Hash; use indexmap::IndexMap; use rkyv::{Archive, Deserialize, Serialize}; #[cfg(feature = "std")] -use std::{collections::HashMap, hash::Hash}; +use std::{collections::BTreeMap, hash::Hash}; #[derive(Serialize, Deserialize, Archive)] /// Rkyv Archivable IndexMap -pub struct ArchivableIndexMap { - indices: HashMap, +pub struct ArchivableIndexMap { + indices: BTreeMap, entries: Vec<(K, V)>, } -impl From> for ArchivableIndexMap { +impl From> + for ArchivableIndexMap +{ fn from(it: IndexMap) -> ArchivableIndexMap { let mut r = ArchivableIndexMap { - indices: HashMap::new(), + indices: BTreeMap::new(), entries: Vec::new(), }; let mut i: u64 = 0; @@ -28,7 +30,9 @@ impl From> for Archiv } } -impl Into> for ArchivableIndexMap { +impl Into> + for ArchivableIndexMap +{ fn into(self) -> IndexMap { let mut r = IndexMap::new(); for (k, v) in self.entries.into_iter() { From b32e8d92c1242aa5b35d954928ff08a2294fe1dd Mon Sep 17 00:00:00 2001 From: Anbang Wen Date: Fri, 12 Mar 2021 11:40:10 -0800 Subject: [PATCH 19/33] add a compiler test to check for deterministic This is to test #2173, the empty test always pass while the table one fails sometimes. --- tests/compilers/deterministic.rs | 44 ++++++++++++++++++++++++++++++++ tests/compilers/main.rs | 1 + 2 files changed, 45 insertions(+) create mode 100644 tests/compilers/deterministic.rs diff --git a/tests/compilers/deterministic.rs b/tests/compilers/deterministic.rs new file mode 100644 index 000000000..9c9070ce5 --- /dev/null +++ b/tests/compilers/deterministic.rs @@ -0,0 +1,44 @@ +use anyhow::Result; +use wasmer::{wat2wasm, Module}; + +fn compile_and_compare(wasm: &[u8]) -> Result<()> { + let store = Default::default(); + + // compile for first time + let module = Module::new(&store, wasm)?; + let first = module.serialize()?; + + // compile for second time + let module = Module::new(&store, wasm)?; + let second = module.serialize()?; + + assert!(first == second); + + Ok(()) +} + +#[test] +fn deterministic_empty() -> Result<()> { + let wasm_bytes = wat2wasm( + br#" + (module) + "#, + )?; + + compile_and_compare(&wasm_bytes) +} + +#[test] +fn deterministic_table() -> Result<()> { + let wasm_bytes = wat2wasm( + br#" +(module + (table 2 funcref) + (func $f1) + (func $f2) + (elem (i32.const 0) $f1 $f2)) +"#, + )?; + + compile_and_compare(&wasm_bytes) +} diff --git a/tests/compilers/main.rs b/tests/compilers/main.rs index 290d31c8f..9ba00794f 100644 --- a/tests/compilers/main.rs +++ b/tests/compilers/main.rs @@ -6,6 +6,7 @@ extern crate compiler_test_derive; mod config; +mod deterministic; mod imports; mod issues; mod metering; From 4e46c80c48991aa5d0b4da9863efcb0469750c62 Mon Sep 17 00:00:00 2001 From: Anbang Wen Date: Wed, 17 Mar 2021 18:00:31 -0700 Subject: [PATCH 20/33] fuzz: add a test for for deterministic compilation --- fuzz/Cargo.toml | 5 ++ fuzz/fuzz_targets/deterministic.rs | 77 ++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 fuzz/fuzz_targets/deterministic.rs diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 891de3552..4b52f18a3 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -57,3 +57,8 @@ required-features = ["universal", "cranelift"] name = "dylib_cranelift" path = "fuzz_targets/dylib_cranelift.rs" required-features = ["dylib", "cranelift"] + +[[bin]] +name = "deterministic" +path = "fuzz_targets/deterministic.rs" +required-features = ["dylib", "cranelift"] \ No newline at end of file diff --git a/fuzz/fuzz_targets/deterministic.rs b/fuzz/fuzz_targets/deterministic.rs new file mode 100644 index 000000000..b06615ef5 --- /dev/null +++ b/fuzz/fuzz_targets/deterministic.rs @@ -0,0 +1,77 @@ +#![no_main] + +use libfuzzer_sys::{arbitrary, arbitrary::Arbitrary, fuzz_target}; +use wasm_smith::{Config, ConfiguredModule}; +use wasmer::{CompilerConfig, Engine, Module, Store}; +use wasmer_compiler_cranelift::Cranelift; +use wasmer_compiler_llvm::LLVM; +use wasmer_compiler_singlepass::Singlepass; +use wasmer_engine_jit::JIT; +use wasmer_engine_native::Native; + +#[derive(Arbitrary, Debug, Default, Copy, Clone)] +struct NoImportsConfig; +impl Config for NoImportsConfig { + fn max_imports(&self) -> usize { + 0 + } + fn max_memory_pages(&self) -> u32 { + // https://github.com/wasmerio/wasmer/issues/2187 + 65535 + } + fn allow_start_export(&self) -> bool { + false + } +} + +fn compile_and_compare(name: &str, engine: impl Engine, wasm: &[u8]) { + let store = Store::new(&engine); + + // compile for first time + let module = Module::new(&store, wasm).unwrap(); + let first = module.serialize().unwrap(); + + // compile for second time + let module = Module::new(&store, wasm).unwrap(); + let second = module.serialize().unwrap(); + + if first != second { + panic!("non-deterministic compilation from {}", name); + } +} + +fuzz_target!(|module: ConfiguredModule| { + let wasm_bytes = module.to_bytes(); + + let mut compiler = Cranelift::default(); + compiler.canonicalize_nans(true); + compiler.enable_verifier(); + compile_and_compare( + "jit-cranelift", + JIT::new(compiler.clone()).engine(), + &wasm_bytes, + ); + compile_and_compare( + "native-cranelift", + Native::new(compiler).engine(), + &wasm_bytes, + ); + + // let mut compiler = LLVM::default(); + // compiler.canonicalize_nans(true); + // compiler.enable_verifier(); + // compile_and_compare("jit-llvm", JIT::new(compiler.clone()).engine(), &wasm_bytes); + // compile_and_compare("native-llvm", Native::new(compiler).engine(), &wasm_bytes); + + // let compiler = Singlepass::default(); + // compile_and_compare( + // "jit-singlepass", + // JIT::new(compiler.clone()).engine(), + // &wasm_bytes, + // ); + // compile_and_compare( + // "native-singlepass", + // Native::new(compiler).engine(), + // &wasm_bytes, + // ); +}); From dd8fd6db9fb67df732a5256eb4a3edd1c7e33fc6 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 17:45:50 +0000 Subject: [PATCH 21/33] Fix fuzz target --- fuzz/Cargo.toml | 2 +- fuzz/fuzz_targets/deterministic.rs | 50 ++++++++++++++++-------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 4b52f18a3..e28680e2a 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -61,4 +61,4 @@ required-features = ["dylib", "cranelift"] [[bin]] name = "deterministic" path = "fuzz_targets/deterministic.rs" -required-features = ["dylib", "cranelift"] \ No newline at end of file +required-features = ["universal", "dylib", "cranelift", "llvm", "singlepass"] \ No newline at end of file diff --git a/fuzz/fuzz_targets/deterministic.rs b/fuzz/fuzz_targets/deterministic.rs index b06615ef5..b613d5449 100644 --- a/fuzz/fuzz_targets/deterministic.rs +++ b/fuzz/fuzz_targets/deterministic.rs @@ -6,8 +6,8 @@ use wasmer::{CompilerConfig, Engine, Module, Store}; use wasmer_compiler_cranelift::Cranelift; use wasmer_compiler_llvm::LLVM; use wasmer_compiler_singlepass::Singlepass; -use wasmer_engine_jit::JIT; -use wasmer_engine_native::Native; +use wasmer_engine_dylib::Dylib; +use wasmer_engine_universal::Universal; #[derive(Arbitrary, Debug, Default, Copy, Clone)] struct NoImportsConfig; @@ -47,31 +47,35 @@ fuzz_target!(|module: ConfiguredModule| { compiler.canonicalize_nans(true); compiler.enable_verifier(); compile_and_compare( - "jit-cranelift", - JIT::new(compiler.clone()).engine(), + "universal-cranelift", + Universal::new(compiler.clone()).engine(), &wasm_bytes, ); + //compile_and_compare( + // "dylib-cranelift", + // Dylib::new(compiler).engine(), + // &wasm_bytes, + //); + + let mut compiler = LLVM::default(); + compiler.canonicalize_nans(true); + compiler.enable_verifier(); compile_and_compare( - "native-cranelift", - Native::new(compiler).engine(), + "universal-llvm", + Universal::new(compiler.clone()).engine(), &wasm_bytes, ); + //compile_and_compare("dylib-llvm", Dylib::new(compiler).engine(), &wasm_bytes); - // let mut compiler = LLVM::default(); - // compiler.canonicalize_nans(true); - // compiler.enable_verifier(); - // compile_and_compare("jit-llvm", JIT::new(compiler.clone()).engine(), &wasm_bytes); - // compile_and_compare("native-llvm", Native::new(compiler).engine(), &wasm_bytes); - - // let compiler = Singlepass::default(); - // compile_and_compare( - // "jit-singlepass", - // JIT::new(compiler.clone()).engine(), - // &wasm_bytes, - // ); - // compile_and_compare( - // "native-singlepass", - // Native::new(compiler).engine(), - // &wasm_bytes, - // ); + let compiler = Singlepass::default(); + compile_and_compare( + "universal-singlepass", + Universal::new(compiler.clone()).engine(), + &wasm_bytes, + ); + //compile_and_compare( + // "dylib-singlepass", + // Dylib::new(compiler).engine(), + // &wasm_bytes, + //); }); From c7e04f142066634ab75879fbb213fd44fb15ef6a Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 17:50:24 +0000 Subject: [PATCH 22/33] Fix build with rkyv disabled --- lib/types/src/module.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/types/src/module.rs b/lib/types/src/module.rs index 135e17ce1..b9fb247c8 100644 --- a/lib/types/src/module.rs +++ b/lib/types/src/module.rs @@ -15,15 +15,17 @@ use crate::{ }; use indexmap::IndexMap; use loupe::MemoryUsage; -use rkyv::ser::ScratchSpace; #[cfg(feature = "enable-rkyv")] use rkyv::{ - de::SharedDeserializeRegistry, ser::Serializer, ser::SharedSerializeRegistry, Archive, - Archived, Deserialize as RkyvDeserialize, Fallible, Serialize as RkyvSerialize, + de::SharedDeserializeRegistry, ser::ScratchSpace, ser::Serializer, + ser::SharedSerializeRegistry, Archive, Archived, Deserialize as RkyvDeserialize, Fallible, + Serialize as RkyvSerialize, }; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashMap}; +#[cfg(feature = "enable-rkyv")] +use std::collections::BTreeMap; +use std::collections::HashMap; use std::fmt; use std::iter::ExactSizeIterator; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; From 60db3b8931e5adaac215837762936776f0766828 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 17:52:25 +0000 Subject: [PATCH 23/33] Remove unneeded BTreeMap from ArchivableIndexMap --- lib/types/src/archives.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/types/src/archives.rs b/lib/types/src/archives.rs index 48e30272e..8ff259716 100644 --- a/lib/types/src/archives.rs +++ b/lib/types/src/archives.rs @@ -3,12 +3,11 @@ use core::hash::Hash; use indexmap::IndexMap; use rkyv::{Archive, Deserialize, Serialize}; #[cfg(feature = "std")] -use std::{collections::BTreeMap, hash::Hash}; +use std::hash::Hash; #[derive(Serialize, Deserialize, Archive)] /// Rkyv Archivable IndexMap pub struct ArchivableIndexMap { - indices: BTreeMap, entries: Vec<(K, V)>, } @@ -17,12 +16,10 @@ impl From> { fn from(it: IndexMap) -> ArchivableIndexMap { let mut r = ArchivableIndexMap { - indices: BTreeMap::new(), entries: Vec::new(), }; let mut i: u64 = 0; for (k, v) in it.into_iter() { - r.indices.insert(k.clone(), i); r.entries.push((k, v)); i += 1; } From 54b4495b3f9a9e3dd60cc1bf00d20d07eb777bab Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 19:33:02 +0000 Subject: [PATCH 24/33] Fix build --- lib/api/src/js/externals/function.rs | 2 +- lib/api/src/sys/externals/function.rs | 2 +- lib/types/src/archives.rs | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/api/src/js/externals/function.rs b/lib/api/src/js/externals/function.rs index d9ccea88b..c9bf58535 100644 --- a/lib/api/src/js/externals/function.rs +++ b/lib/api/src/js/externals/function.rs @@ -1288,7 +1288,7 @@ mod inner { #[test] fn test_into_array() { - assert_eq!(().into_array(), []); + assert_eq!(().into_array(), [0i128; 0]); assert_eq!((1).into_array(), [1]); assert_eq!((1i32, 2i64).into_array(), [1, 2]); assert_eq!( diff --git a/lib/api/src/sys/externals/function.rs b/lib/api/src/sys/externals/function.rs index 773faacd4..3146c8238 100644 --- a/lib/api/src/sys/externals/function.rs +++ b/lib/api/src/sys/externals/function.rs @@ -1472,7 +1472,7 @@ mod inner { #[test] fn test_into_array() { - assert_eq!(().into_array(), []); + assert_eq!(().into_array(), [0i128; 0]); assert_eq!((1).into_array(), [1]); assert_eq!((1i32, 2i64).into_array(), [1, 2]); assert_eq!( diff --git a/lib/types/src/archives.rs b/lib/types/src/archives.rs index 8ff259716..f620520b5 100644 --- a/lib/types/src/archives.rs +++ b/lib/types/src/archives.rs @@ -18,10 +18,8 @@ impl From> let mut r = ArchivableIndexMap { entries: Vec::new(), }; - let mut i: u64 = 0; for (k, v) in it.into_iter() { r.entries.push((k, v)); - i += 1; } r } From 4944ddf75464a534080b1c2f6a7e5bce018def14 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 21:46:39 +0000 Subject: [PATCH 25/33] Fix cranelift build without unwind --- lib/compiler-cranelift/src/compiler.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index 6a887752f..3fbc99809 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -207,7 +207,13 @@ impl Compiler for CraneliftCompiler { (None, None) } } + #[cfg(feature = "unwind")] other => (other.maybe_into_to_windows_unwind(), None), + + // This is a bit hacky, but necessary since gimli is not + // available when the "unwind" feature is disabled. + #[cfg(not(feature = "unwind"))] + other => (other.maybe_into_to_windows_unwind(), None::<()>), }; let range = reader.range(); From 4732b171126ec1b0e397208145cd0c888a2166bf Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 22 Nov 2021 23:12:27 +0000 Subject: [PATCH 26/33] Fix ignore for windows test --- tests/ignores.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ignores.txt b/tests/ignores.txt index 4661fe049..fd9bebcb7 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -70,7 +70,7 @@ cranelift spec::simd::simd_i8x16_arith2 cranelift spec::simd::simd_int_to_int_extend # Windows doesn't overcommit and fails to allocate 4GB of memory -windows wast::wasmer::max_size_of_memory +windows wasmer::max_size_of_memory # Frontends From cb88b6db62ba8e74fad189ab6dfa5202c0d681c5 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 23 Nov 2021 15:19:32 +0000 Subject: [PATCH 27/33] Avoid depending on the host target in compilers --- lib/compiler-cranelift/Cargo.toml | 2 +- lib/compiler-cranelift/src/compiler.rs | 61 ++++++++++++---------- lib/compiler-cranelift/src/sink.rs | 16 ++---- lib/compiler-singlepass/src/codegen_x64.rs | 13 +++-- lib/compiler-singlepass/src/compiler.rs | 14 +++-- lib/compiler-singlepass/src/config.rs | 12 +---- 6 files changed, 59 insertions(+), 59 deletions(-) diff --git a/lib/compiler-cranelift/Cargo.toml b/lib/compiler-cranelift/Cargo.toml index 0405665c2..d687c1230 100644 --- a/lib/compiler-cranelift/Cargo.toml +++ b/lib/compiler-cranelift/Cargo.toml @@ -25,9 +25,9 @@ more-asserts = "0.2" gimli = { version = "0.25", optional = true } smallvec = "1.6" loupe = "0.1" +target-lexicon = { version = "0.12.2", default-features = false } [dev-dependencies] -target-lexicon = { version = "0.12.2", default-features = false } cranelift-codegen = { version = "0.76", features = ["all-arch"] } lazy_static = "1.4" diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index c836b9ce1..ee23136a3 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -21,6 +21,7 @@ use gimli::write::{Address, EhFrame, FrameTable}; use loupe::MemoryUsage; use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use std::sync::Arc; +use target_lexicon::{Architecture, OperatingSystem}; use wasmer_compiler::CompileError; use wasmer_compiler::{CallingConvention, ModuleTranslationState, Target}; use wasmer_compiler::{ @@ -29,14 +30,12 @@ use wasmer_compiler::{ FunctionBodyData, MiddlewareBinaryReader, ModuleMiddleware, ModuleMiddlewareChain, SectionIndex, }; -#[cfg(all(target_arch = "x86_64", target_os = "linux"))] use wasmer_compiler::{ CustomSection, CustomSectionProtection, Relocation, RelocationKind, RelocationTarget, SectionBody, }; use wasmer_types::entity::{EntityRef, PrimaryMap}; use wasmer_types::{FunctionIndex, LocalFunctionIndex, SignatureIndex}; -#[cfg(all(target_arch = "x86_64", target_os = "linux"))] use wasmer_vm::libcalls::LibCall; /// A compiler that compiles a WebAssembly module with Cranelift, translating the Wasm to Cranelift IR, @@ -111,28 +110,36 @@ impl Compiler for CraneliftCompiler { let mut custom_sections = PrimaryMap::new(); - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] - let probestack_trampoline = CustomSection { - protection: CustomSectionProtection::ReadExecute, - // We create a jump to an absolute 64bits address - // with an indrect jump immediatly followed but the absolute address - // JMP [IP+0] FF 25 00 00 00 00 - // 64bits ADDR 00 00 00 00 00 00 00 00 preset to 0 until the relocation takes place - bytes: SectionBody::new_with_vec(vec![ - 0xff, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - ]), - relocations: vec![Relocation { - kind: RelocationKind::Abs8, - reloc_target: RelocationTarget::LibCall(LibCall::Probestack), - // 6 is the size of the jmp instruction. The relocated address must follow - offset: 6, - addend: 0, - }], + let probestack_trampoline_relocation_target = if target.triple().operating_system + == OperatingSystem::Linux + && matches!( + target.triple().architecture, + Architecture::X86_32(_) | Architecture::X86_64 + ) { + let probestack_trampoline = CustomSection { + protection: CustomSectionProtection::ReadExecute, + // We create a jump to an absolute 64bits address + // with an indrect jump immediatly followed but the absolute address + // JMP [IP+0] FF 25 00 00 00 00 + // 64bits ADDR 00 00 00 00 00 00 00 00 preset to 0 until the relocation takes place + bytes: SectionBody::new_with_vec(vec![ + 0xff, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]), + relocations: vec![Relocation { + kind: RelocationKind::Abs8, + reloc_target: RelocationTarget::LibCall(LibCall::Probestack), + // 6 is the size of the jmp instruction. The relocated address must follow + offset: 6, + addend: 0, + }], + }; + custom_sections.push(probestack_trampoline); + + Some(SectionIndex::new(custom_sections.len() - 1)) + } else { + None }; - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] - custom_sections.push(probestack_trampoline); - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] - let probestack_trampoline_relocation_target = SectionIndex::new(custom_sections.len() - 1); let functions = function_body_inputs .iter() @@ -170,12 +177,8 @@ impl Compiler for CraneliftCompiler { )?; let mut code_buf: Vec = Vec::new(); - let mut reloc_sink = RelocSink::new( - &module, - func_index, - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] - probestack_trampoline_relocation_target, - ); + let mut reloc_sink = + RelocSink::new(&module, func_index, probestack_trampoline_relocation_target); let mut trap_sink = TrapSink::new(); let mut stackmap_sink = binemit::NullStackMapSink {}; context diff --git a/lib/compiler-cranelift/src/sink.rs b/lib/compiler-cranelift/src/sink.rs index cf9f6f89e..9640e2614 100644 --- a/lib/compiler-cranelift/src/sink.rs +++ b/lib/compiler-cranelift/src/sink.rs @@ -2,12 +2,10 @@ use crate::translator::{irlibcall_to_libcall, irreloc_to_relocationkind}; use cranelift_codegen::binemit; -#[cfg(target_arch = "x86_64")] use cranelift_codegen::ir::LibCall; use cranelift_codegen::ir::{self, ExternalName}; use cranelift_entity::EntityRef as CraneliftEntityRef; use wasmer_compiler::{JumpTable, Relocation, RelocationTarget, TrapInformation}; -#[cfg(all(target_arch = "x86_64", target_os = "linux"))] use wasmer_compiler::{RelocationKind, SectionIndex}; use wasmer_types::entity::EntityRef; use wasmer_types::{FunctionIndex, LocalFunctionIndex, ModuleInfo}; @@ -24,8 +22,7 @@ pub(crate) struct RelocSink<'a> { pub func_relocs: Vec, /// The section where the probestack trampoline call is located - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] - pub probestack_trampoline_relocation_target: SectionIndex, + pub probestack_trampoline_relocation_target: Option, } impl<'a> binemit::RelocSink for RelocSink<'a> { @@ -45,13 +42,12 @@ impl<'a> binemit::RelocSink for RelocSink<'a> { .expect("The provided function should be local"), ) } else if let ExternalName::LibCall(libcall) = *name { - match libcall { - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] - LibCall::Probestack => { + match (libcall, self.probestack_trampoline_relocation_target) { + (LibCall::Probestack, Some(probestack_trampoline_relocation_target)) => { self.func_relocs.push(Relocation { kind: RelocationKind::X86CallPCRel4, reloc_target: RelocationTarget::CustomSection( - self.probestack_trampoline_relocation_target, + probestack_trampoline_relocation_target, ), offset: offset, addend: addend, @@ -100,8 +96,7 @@ impl<'a> RelocSink<'a> { pub fn new( module: &'a ModuleInfo, func_index: FunctionIndex, - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] - probestack_trampoline_relocation_target: SectionIndex, + probestack_trampoline_relocation_target: Option, ) -> Self { let local_func_index = module .local_func_index(func_index) @@ -110,7 +105,6 @@ impl<'a> RelocSink<'a> { module, local_func_index, func_relocs: Vec::new(), - #[cfg(all(target_arch = "x86_64", target_os = "linux"))] probestack_trampoline_relocation_target, } } diff --git a/lib/compiler-singlepass/src/codegen_x64.rs b/lib/compiler-singlepass/src/codegen_x64.rs index d44f3be56..59b44ec9f 100644 --- a/lib/compiler-singlepass/src/codegen_x64.rs +++ b/lib/compiler-singlepass/src/codegen_x64.rs @@ -91,6 +91,9 @@ pub struct FuncGen<'a> { /// // Ordered by increasing InstructionAddressMap::srcloc. instructions_address_map: Vec, + + /// Calling convention to use. + calling_convention: CallingConvention, } struct SpecialLabelSet { @@ -1012,7 +1015,7 @@ impl<'a> FuncGen<'a> { self.machine.state.stack_values.push(content); } } - let calling_convention = self.config.calling_convention; + let calling_convention = self.calling_convention; let stack_padding: usize = match calling_convention { CallingConvention::WindowsFastcall => 32, @@ -1758,7 +1761,7 @@ impl<'a> FuncGen<'a> { &mut self.assembler, self.local_types.len(), self.signature.params().len(), - self.config.calling_convention, + self.calling_convention, ); // Mark vmctx register. The actual loading of the vmctx value is handled by init_local. @@ -1825,6 +1828,7 @@ impl<'a> FuncGen<'a> { _table_styles: &'a PrimaryMap, local_func_index: LocalFunctionIndex, local_types_excluding_arguments: &[WpType], + calling_convention: CallingConvention, ) -> Result, CodegenError> { let func_index = module.func_index(local_func_index); let sig_index = module.functions[func_index]; @@ -1876,6 +1880,7 @@ impl<'a> FuncGen<'a> { special_labels, src_loc: 0, instructions_address_map: vec![], + calling_convention, }; fg.emit_head()?; Ok(fg) @@ -5406,7 +5411,7 @@ impl<'a> FuncGen<'a> { self.vmoffsets.vmcaller_checked_anyfunc_func_ptr() as usize; let vmcaller_checked_anyfunc_vmctx = self.vmoffsets.vmcaller_checked_anyfunc_vmctx() as usize; - let calling_convention = self.config.calling_convention; + let calling_convention = self.calling_convention; self.emit_call_native( |this| { @@ -6700,7 +6705,7 @@ impl<'a> FuncGen<'a> { self.machine.finalize_locals( &mut self.assembler, &self.locals, - self.config.calling_convention, + self.calling_convention, ); self.assembler.emit_mov( Size::S64, diff --git a/lib/compiler-singlepass/src/compiler.rs b/lib/compiler-singlepass/src/compiler.rs index 37feb7c8e..ba48a6655 100644 --- a/lib/compiler-singlepass/src/compiler.rs +++ b/lib/compiler-singlepass/src/compiler.rs @@ -13,7 +13,7 @@ use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use std::sync::Arc; use wasmer_compiler::{ Architecture, CallingConvention, Compilation, CompileError, CompileModuleInfo, - CompiledFunction, Compiler, CompilerConfig, FunctionBinaryReader, FunctionBody, + CompiledFunction, Compiler, CompilerConfig, CpuFeature, FunctionBinaryReader, FunctionBody, FunctionBodyData, MiddlewareBinaryReader, ModuleMiddleware, ModuleMiddlewareChain, ModuleTranslationState, OperatingSystem, SectionIndex, Target, TrapInformation, }; @@ -62,8 +62,15 @@ impl Compiler for SinglepassCompiler { OperatingSystem::Windows.to_string(), )); }*/ - if let Architecture::X86_32(arch) = target.triple().architecture { - return Err(CompileError::UnsupportedTarget(arch.to_string())); + if target.triple().architecture != Architecture::X86_64 { + return Err(CompileError::UnsupportedTarget( + target.triple().architecture.to_string(), + )); + } + if !target.cpu_features().contains(CpuFeature::AVX) { + return Err(CompileError::UnsupportedTarget( + "x86_64 without AVX".to_string(), + )); } if compile_info.features.multi_value { return Err(CompileError::UnsupportedFeature("multivalue".to_string())); @@ -125,6 +132,7 @@ impl Compiler for SinglepassCompiler { &table_styles, i, &locals, + calling_convention, ) .map_err(to_compile_error)?; diff --git a/lib/compiler-singlepass/src/config.rs b/lib/compiler-singlepass/src/config.rs index c3fa980db..78496e83d 100644 --- a/lib/compiler-singlepass/src/config.rs +++ b/lib/compiler-singlepass/src/config.rs @@ -4,9 +4,7 @@ use crate::compiler::SinglepassCompiler; use loupe::MemoryUsage; use std::sync::Arc; -use wasmer_compiler::{ - CallingConvention, Compiler, CompilerConfig, CpuFeature, ModuleMiddleware, Target, -}; +use wasmer_compiler::{Compiler, CompilerConfig, CpuFeature, ModuleMiddleware, Target}; use wasmer_types::Features; #[derive(Debug, Clone, MemoryUsage)] @@ -15,8 +13,6 @@ pub struct Singlepass { pub(crate) enable_stack_check: bool, /// The middleware chain. pub(crate) middlewares: Vec>, - #[loupe(skip)] - pub(crate) calling_convention: CallingConvention, } impl Singlepass { @@ -27,12 +23,6 @@ impl Singlepass { enable_nan_canonicalization: true, enable_stack_check: false, middlewares: vec![], - calling_convention: match Target::default().triple().default_calling_convention() { - Ok(CallingConvention::WindowsFastcall) => CallingConvention::WindowsFastcall, - Ok(CallingConvention::SystemV) => CallingConvention::SystemV, - //Ok(CallingConvention::AppleAarch64) => AppleAarch64, - _ => panic!("Unsupported Calling convention for Singlepass"), - }, } } From 4f65a561eed7716d1800c5986be67c4921692824 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 23 Nov 2021 15:20:50 +0000 Subject: [PATCH 28/33] Only allow instantiating from a deserialized file for staticlib engine --- lib/engine-staticlib/src/artifact.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/engine-staticlib/src/artifact.rs b/lib/engine-staticlib/src/artifact.rs index 3615dac79..15b393c55 100644 --- a/lib/engine-staticlib/src/artifact.rs +++ b/lib/engine-staticlib/src/artifact.rs @@ -46,6 +46,7 @@ pub struct StaticlibArtifact { /// Length of the serialized metadata metadata_length: usize, symbol_registry: ModuleMetadataSymbolRegistry, + is_compiled: bool, } #[allow(dead_code)] @@ -295,6 +296,7 @@ impl StaticlibArtifact { func_data_registry: engine_inner.func_data().clone(), metadata_length, symbol_registry, + is_compiled: true, }) } @@ -415,6 +417,7 @@ impl StaticlibArtifact { func_data_registry, metadata_length: 0, symbol_registry, + is_compiled: false, }) } @@ -483,6 +486,12 @@ impl Artifact for StaticlibArtifact { } fn preinstantiate(&self) -> Result<(), InstantiationError> { + if self.is_compiled { + panic!( + "a module built with the staticlib engine must be linked \ + into the current executable" + ); + } Ok(()) } From a603c33def41cb3816aea89334cd53d2d43f82f0 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 23 Nov 2021 15:21:34 +0000 Subject: [PATCH 29/33] Validate that CPU features are supported when instantiating a module --- Cargo.lock | 5 +++++ lib/api/src/sys/instance.rs | 6 ++++++ lib/c-api/src/wasm_c_api/instance.rs | 6 ++++++ lib/engine-dylib/Cargo.toml | 1 + lib/engine-dylib/src/artifact.rs | 10 ++++++++-- lib/engine-dylib/src/serialize.rs | 1 + lib/engine-staticlib/Cargo.toml | 1 + lib/engine-staticlib/src/artifact.rs | 10 +++++++++- lib/engine-staticlib/src/serialize.rs | 1 + lib/engine-universal/Cargo.toml | 10 ++++++++-- lib/engine-universal/src/artifact.rs | 8 +++++++- lib/engine-universal/src/serialize.rs | 1 + lib/engine/Cargo.toml | 1 + lib/engine/src/artifact.rs | 16 +++++++++++++++- lib/engine/src/error.rs | 5 +++++ tests/compilers/traps.rs | 4 +++- tests/lib/engine-dummy/Cargo.toml | 11 +++-------- tests/lib/engine-dummy/src/artifact.rs | 9 ++++++++- 18 files changed, 89 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 815f04419..f2ae14ceb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2900,6 +2900,7 @@ name = "wasmer-engine" version = "2.0.0" dependencies = [ "backtrace", + "enumset", "lazy_static", "loupe", "memmap2", @@ -2919,6 +2920,7 @@ name = "wasmer-engine-dummy" version = "2.0.0" dependencies = [ "bincode", + "enumset", "loupe", "serde", "serde_bytes", @@ -2933,6 +2935,7 @@ name = "wasmer-engine-dylib" version = "2.0.0" dependencies = [ "cfg-if 1.0.0", + "enumset", "leb128", "libloading", "loupe", @@ -2954,6 +2957,7 @@ version = "2.0.0" dependencies = [ "bincode", "cfg-if 1.0.0", + "enumset", "leb128", "libloading", "loupe", @@ -2972,6 +2976,7 @@ name = "wasmer-engine-universal" version = "2.0.0" dependencies = [ "cfg-if 1.0.0", + "enumset", "leb128", "loupe", "region", diff --git a/lib/api/src/sys/instance.rs b/lib/api/src/sys/instance.rs index c67518aef..7368c5549 100644 --- a/lib/api/src/sys/instance.rs +++ b/lib/api/src/sys/instance.rs @@ -58,6 +58,11 @@ pub enum InstantiationError { #[error(transparent)] Start(RuntimeError), + /// The module was compiled with a CPU feature that is not available on + /// the current host. + #[error("missing requires CPU features: {0:?}")] + CpuFeature(String), + /// Error occurred when initializing the host environment. #[error(transparent)] HostEnvInitialization(HostEnvInitError), @@ -68,6 +73,7 @@ impl From for InstantiationError { match other { wasmer_engine::InstantiationError::Link(e) => Self::Link(e), wasmer_engine::InstantiationError::Start(e) => Self::Start(e), + wasmer_engine::InstantiationError::CpuFeature(e) => Self::CpuFeature(e), } } } diff --git a/lib/c-api/src/wasm_c_api/instance.rs b/lib/c-api/src/wasm_c_api/instance.rs index 047dd18ee..1b95cbd12 100644 --- a/lib/c-api/src/wasm_c_api/instance.rs +++ b/lib/c-api/src/wasm_c_api/instance.rs @@ -73,6 +73,12 @@ pub unsafe extern "C" fn wasm_instance_new( return None; } + Err(e @ InstantiationError::CpuFeature(_)) => { + crate::error::update_last_error(e.to_string()); + + return None; + } + Err(InstantiationError::HostEnvInitialization(error)) => { crate::error::update_last_error(error); diff --git a/lib/engine-dylib/Cargo.toml b/lib/engine-dylib/Cargo.toml index 1b356e070..6df13e491 100644 --- a/lib/engine-dylib/Cargo.toml +++ b/lib/engine-dylib/Cargo.toml @@ -25,6 +25,7 @@ tempfile = "3.1" which = "4.0" rkyv = "0.6.1" loupe = "0.1" +enumset = "1.0" [features] # Enable the `compiler` feature if you want the engine to compile diff --git a/lib/engine-dylib/src/artifact.rs b/lib/engine-dylib/src/artifact.rs index 1469a3733..3a8d9a7f9 100644 --- a/lib/engine-dylib/src/artifact.rs +++ b/lib/engine-dylib/src/artifact.rs @@ -3,6 +3,7 @@ use crate::engine::{DylibEngine, DylibEngineInner}; use crate::serialize::{ArchivedModuleMetadata, ModuleMetadata}; +use enumset::EnumSet; use libloading::{Library, Symbol as LibrarySymbol}; use loupe::MemoryUsage; use std::error::Error; @@ -17,8 +18,8 @@ use tracing::log::error; #[cfg(feature = "compiler")] use tracing::trace; use wasmer_compiler::{ - Architecture, CompileError, CompiledFunctionFrameInfo, Features, FunctionAddressMap, - OperatingSystem, Symbol, SymbolRegistry, Triple, + Architecture, CompileError, CompiledFunctionFrameInfo, CpuFeature, Features, + FunctionAddressMap, OperatingSystem, Symbol, SymbolRegistry, Triple, }; #[cfg(feature = "compiler")] use wasmer_compiler::{ @@ -211,6 +212,7 @@ impl DylibArtifact { prefix: engine_inner.get_prefix(&data), data_initializers, function_body_lengths, + cpu_features: target.cpu_features().as_u64(), }; let serialized_data = metadata.serialize()?; @@ -800,6 +802,10 @@ impl Artifact for DylibArtifact { &self.metadata.compile_info.features } + fn cpu_features(&self) -> enumset::EnumSet { + EnumSet::from_u64(self.metadata.cpu_features) + } + fn data_initializers(&self) -> &[OwnedDataInitializer] { &*self.metadata.data_initializers } diff --git a/lib/engine-dylib/src/serialize.rs b/lib/engine-dylib/src/serialize.rs index 7021834a8..33df08a6a 100644 --- a/lib/engine-dylib/src/serialize.rs +++ b/lib/engine-dylib/src/serialize.rs @@ -35,6 +35,7 @@ pub struct ModuleMetadata { pub data_initializers: Box<[OwnedDataInitializer]>, // The function body lengths (used to find function by address) pub function_body_lengths: PrimaryMap, + pub cpu_features: u64, } pub struct ModuleMetadataSymbolRegistry<'a> { diff --git a/lib/engine-staticlib/Cargo.toml b/lib/engine-staticlib/Cargo.toml index 860b8637a..cac5c2415 100644 --- a/lib/engine-staticlib/Cargo.toml +++ b/lib/engine-staticlib/Cargo.toml @@ -24,6 +24,7 @@ leb128 = "0.2" libloading = "0.7" tempfile = "3.1" loupe = "0.1" +enumset = "1.0" [features] # Enable the `compiler` feature if you want the engine to compile diff --git a/lib/engine-staticlib/src/artifact.rs b/lib/engine-staticlib/src/artifact.rs index 15b393c55..cf25fb480 100644 --- a/lib/engine-staticlib/src/artifact.rs +++ b/lib/engine-staticlib/src/artifact.rs @@ -3,12 +3,15 @@ use crate::engine::{StaticlibEngine, StaticlibEngineInner}; use crate::serialize::{ModuleMetadata, ModuleMetadataSymbolRegistry}; +use enumset::EnumSet; use loupe::MemoryUsage; use std::collections::BTreeMap; use std::error::Error; use std::mem; use std::sync::Arc; -use wasmer_compiler::{CompileError, Features, OperatingSystem, SymbolRegistry, Triple}; +use wasmer_compiler::{ + CompileError, CpuFeature, Features, OperatingSystem, SymbolRegistry, Triple, +}; #[cfg(feature = "compiler")] use wasmer_compiler::{ CompileModuleInfo, Compiler, FunctionBodyData, ModuleEnvironment, ModuleMiddlewareChain, @@ -182,6 +185,7 @@ impl StaticlibArtifact { prefix: engine_inner.get_prefix(&data), data_initializers, function_body_lengths, + cpu_features: target.cpu_features().as_u64(), }; /* @@ -453,6 +457,10 @@ impl Artifact for StaticlibArtifact { &self.metadata.compile_info.features } + fn cpu_features(&self) -> EnumSet { + EnumSet::from_u64(self.metadata.cpu_features) + } + fn data_initializers(&self) -> &[OwnedDataInitializer] { &*self.metadata.data_initializers } diff --git a/lib/engine-staticlib/src/serialize.rs b/lib/engine-staticlib/src/serialize.rs index cee443a45..3bd2fd799 100644 --- a/lib/engine-staticlib/src/serialize.rs +++ b/lib/engine-staticlib/src/serialize.rs @@ -12,6 +12,7 @@ pub struct ModuleMetadata { pub data_initializers: Box<[OwnedDataInitializer]>, // The function body lengths (used to find function by address) pub function_body_lengths: PrimaryMap, + pub cpu_features: u64, } #[derive(MemoryUsage)] diff --git a/lib/engine-universal/Cargo.toml b/lib/engine-universal/Cargo.toml index 683acc8fb..c76c4154c 100644 --- a/lib/engine-universal/Cargo.toml +++ b/lib/engine-universal/Cargo.toml @@ -11,8 +11,13 @@ readme = "README.md" edition = "2018" [dependencies] -wasmer-types = { path = "../types", version = "2.0.0", features = ["enable-rkyv"] } -wasmer-compiler = { path = "../compiler", version = "2.0.0", features = ["translator", "enable-rkyv"] } +wasmer-types = { path = "../types", version = "2.0.0", features = [ + "enable-rkyv", +] } +wasmer-compiler = { path = "../compiler", version = "2.0.0", features = [ + "translator", + "enable-rkyv", +] } wasmer-vm = { path = "../vm", version = "2.0.0", features = ["enable-rkyv"] } wasmer-engine = { path = "../engine", version = "2.0.0" } # flexbuffers = { path = "../../../flatbuffers/rust/flexbuffers", version = "0.1.0" } @@ -21,6 +26,7 @@ cfg-if = "1.0" leb128 = "0.2" rkyv = "0.6.1" loupe = "0.1" +enumset = "1.0" [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3", features = ["winnt", "impl-default"] } diff --git a/lib/engine-universal/src/artifact.rs b/lib/engine-universal/src/artifact.rs index 6e0d4bac9..61a7aeabe 100644 --- a/lib/engine-universal/src/artifact.rs +++ b/lib/engine-universal/src/artifact.rs @@ -6,9 +6,10 @@ use crate::link::link_module; #[cfg(feature = "compiler")] use crate::serialize::SerializableCompilation; use crate::serialize::SerializableModule; +use enumset::EnumSet; use loupe::MemoryUsage; use std::sync::{Arc, Mutex}; -use wasmer_compiler::{CompileError, Features, Triple}; +use wasmer_compiler::{CompileError, CpuFeature, Features, Triple}; #[cfg(feature = "compiler")] use wasmer_compiler::{CompileModuleInfo, ModuleEnvironment, ModuleMiddlewareChain}; use wasmer_engine::{ @@ -128,6 +129,7 @@ impl UniversalArtifact { compilation: serializable_compilation, compile_info, data_initializers, + cpu_features: engine.target().cpu_features().as_u64(), }; Self::from_parts(&mut inner_engine, serializable) } @@ -307,6 +309,10 @@ impl Artifact for UniversalArtifact { &self.serializable.compile_info.features } + fn cpu_features(&self) -> EnumSet { + EnumSet::from_u64(self.serializable.cpu_features) + } + fn data_initializers(&self) -> &[OwnedDataInitializer] { &*self.serializable.data_initializers } diff --git a/lib/engine-universal/src/serialize.rs b/lib/engine-universal/src/serialize.rs index 6e3c768d4..fa0cfc79f 100644 --- a/lib/engine-universal/src/serialize.rs +++ b/lib/engine-universal/src/serialize.rs @@ -38,6 +38,7 @@ pub struct SerializableModule { pub compilation: SerializableCompilation, pub compile_info: CompileModuleInfo, pub data_initializers: Box<[OwnedDataInitializer]>, + pub cpu_features: u64, } fn to_serialize_error(err: impl std::error::Error) -> SerializeError { diff --git a/lib/engine/Cargo.toml b/lib/engine/Cargo.toml index 39d058e5d..0e19974c4 100644 --- a/lib/engine/Cargo.toml +++ b/lib/engine/Cargo.toml @@ -25,6 +25,7 @@ serde = { version = "1.0", features = ["derive", "rc"] } serde_bytes = { version = "0.11" } lazy_static = "1.4" loupe = "0.1" +enumset = "1.0" [badges] maintenance = { status = "actively-developed" } diff --git a/lib/engine/src/artifact.rs b/lib/engine/src/artifact.rs index e8185d14a..21e53c90a 100644 --- a/lib/engine/src/artifact.rs +++ b/lib/engine/src/artifact.rs @@ -1,12 +1,13 @@ use crate::{ resolve_imports, InstantiationError, Resolver, RuntimeError, SerializeError, Tunables, }; +use enumset::EnumSet; use loupe::MemoryUsage; use std::any::Any; use std::fs; use std::path::Path; use std::sync::Arc; -use wasmer_compiler::Features; +use wasmer_compiler::{CpuFeature, Features}; use wasmer_types::entity::{BoxedSlice, PrimaryMap}; use wasmer_types::{ DataInitializer, FunctionIndex, LocalFunctionIndex, MemoryIndex, ModuleInfo, @@ -43,6 +44,9 @@ pub trait Artifact: Send + Sync + Upcastable + MemoryUsage { /// Returns the features for this Artifact fn features(&self) -> &Features; + /// Returns the CPU features for this Artifact + fn cpu_features(&self) -> EnumSet; + /// Returns the memory styles associated with this `Artifact`. fn memory_styles(&self) -> &PrimaryMap; @@ -96,6 +100,16 @@ pub trait Artifact: Send + Sync + Upcastable + MemoryUsage { resolver: &dyn Resolver, host_state: Box, ) -> Result { + // Validate the CPU features this module was compiled with against the + // host CPU features. + let host_cpu_features = CpuFeature::for_host(); + if !host_cpu_features.is_superset(self.cpu_features()) { + Err(InstantiationError::CpuFeature(format!( + "{:?}", + self.cpu_features().difference(host_cpu_features) + )))?; + } + self.preinstantiate()?; let module = self.module(); diff --git a/lib/engine/src/error.rs b/lib/engine/src/error.rs index 87f771ebe..5874e5d8b 100644 --- a/lib/engine/src/error.rs +++ b/lib/engine/src/error.rs @@ -91,6 +91,11 @@ pub enum InstantiationError { #[error(transparent)] Link(LinkError), + /// The module was compiled with a CPU feature that is not available on + /// the current host. + #[error("module compiled with CPU feature that is missing from host")] + CpuFeature(String), + /// A runtime error occured while invoking the start function #[error(transparent)] Start(RuntimeError), diff --git a/tests/compilers/traps.rs b/tests/compilers/traps.rs index 145bc2529..493090c7e 100644 --- a/tests/compilers/traps.rs +++ b/tests/compilers/traps.rs @@ -257,7 +257,9 @@ fn trap_start_function_import(config: crate::Config) -> Result<()> { .err() .unwrap(); match err { - InstantiationError::Link(_) | InstantiationError::HostEnvInitialization(_) => { + InstantiationError::Link(_) + | InstantiationError::HostEnvInitialization(_) + | InstantiationError::CpuFeature(_) => { panic!("It should be a start error") } InstantiationError::Start(err) => { diff --git a/tests/lib/engine-dummy/Cargo.toml b/tests/lib/engine-dummy/Cargo.toml index ae30b2aef..5bb1369b2 100644 --- a/tests/lib/engine-dummy/Cargo.toml +++ b/tests/lib/engine-dummy/Cargo.toml @@ -16,19 +16,14 @@ serde = { version = "1.0", features = ["derive", "rc"], optional = true } serde_bytes = { version = "0.11", optional = true } bincode = { version = "1.2", optional = true } loupe = "0.1" +enumset = "1.0" [features] # Enable the `compiler` feature if you want the engine to compile # and not be only on headless mode. default = ["serialize", "compiler"] -compiler = [ - "wasmer-compiler/translator" -] -serialize = [ - "serde", - "serde_bytes", - "bincode" -] +compiler = ["wasmer-compiler/translator"] +serialize = ["serde", "serde_bytes", "bincode"] [badges] # TODO: publish this crate again and deprecate it diff --git a/tests/lib/engine-dummy/src/artifact.rs b/tests/lib/engine-dummy/src/artifact.rs index 72fc3d455..9b7523afb 100644 --- a/tests/lib/engine-dummy/src/artifact.rs +++ b/tests/lib/engine-dummy/src/artifact.rs @@ -2,13 +2,14 @@ //! done as separate steps. use crate::engine::DummyEngine; +use enumset::EnumSet; use loupe::MemoryUsage; #[cfg(feature = "serialize")] use serde::{Deserialize, Serialize}; use std::sync::Arc; -use wasmer_compiler::CompileError; #[cfg(feature = "compiler")] use wasmer_compiler::ModuleEnvironment; +use wasmer_compiler::{CompileError, CpuFeature}; use wasmer_engine::{Artifact, DeserializeError, Engine as _, SerializeError, Tunables}; use wasmer_types::entity::{BoxedSlice, PrimaryMap}; use wasmer_types::{ @@ -30,6 +31,7 @@ pub struct DummyArtifactMetadata { // Plans for that module pub memory_styles: PrimaryMap, pub table_styles: PrimaryMap, + pub cpu_features: u64, } /// A Dummy artifact. @@ -104,6 +106,7 @@ impl DummyArtifact { data_initializers, memory_styles, table_styles, + cpu_features: engine.target().cpu_features().as_u64(), }; Self::from_parts(&engine, metadata) } @@ -211,6 +214,10 @@ impl Artifact for DummyArtifact { &self.metadata.features } + fn cpu_features(&self) -> EnumSet { + EnumSet::from_u64(self.metadata.cpu_features) + } + fn data_initializers(&self) -> &[OwnedDataInitializer] { &*self.metadata.data_initializers } From e2c6a39d37e8e4e7532884059d5c26211b198afb Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 23 Nov 2021 16:11:26 +0000 Subject: [PATCH 30/33] Clean up error handling in the C API --- lib/c-api/src/error.rs | 49 ++++++------------- lib/c-api/src/wasm_c_api/engine.rs | 11 ++--- lib/c-api/src/wasm_c_api/instance.rs | 2 +- .../src/wasm_c_api/unstable/target_lexicon.rs | 3 +- lib/c-api/src/wasm_c_api/unstable/wasi.rs | 8 +-- lib/c-api/src/wasm_c_api/value.rs | 4 +- lib/c-api/src/wasm_c_api/wasi/mod.rs | 29 ++++------- 7 files changed, 34 insertions(+), 72 deletions(-) diff --git a/lib/c-api/src/error.rs b/lib/c-api/src/error.rs index acf921a10..2a4066503 100644 --- a/lib/c-api/src/error.rs +++ b/lib/c-api/src/error.rs @@ -47,15 +47,14 @@ //! # } //! ``` -use libc::{c_char, c_int}; +use libc::c_char; use std::cell::RefCell; -use std::error::Error; -use std::fmt::{self, Display, Formatter}; +use std::fmt::Display; use std::ptr::{self, NonNull}; use std::slice; thread_local! { - static LAST_ERROR: RefCell>> = RefCell::new(None); + static LAST_ERROR: RefCell> = RefCell::new(None); } /// Rust function to register a new error. @@ -63,24 +62,23 @@ thread_local! { /// # Example /// /// ```rust,no_run -/// # use wasmer::error::{update_last_error, CApiError}; +/// # use wasmer::error::update_last_error; /// -/// update_last_error(CApiError { -/// msg: "Hello, World!".to_string(), -/// }); +/// update_last_error("Hello, World!"); /// ``` -pub fn update_last_error(err: E) { +pub fn update_last_error(err: E) { LAST_ERROR.with(|prev| { - *prev.borrow_mut() = Some(Box::new(err)); + *prev.borrow_mut() = Some(err.to_string()); }); } /// Retrieve the most recent error, clearing it in the process. -pub(crate) fn take_last_error() -> Option> { +pub(crate) fn take_last_error() -> Option { LAST_ERROR.with(|prev| prev.borrow_mut().take()) } -/// Gets the length in bytes of the last error if any, zero otherwise. +/// Gets the length in bytes of the last error if any, zero otherwise. This +/// includes th NUL terminator byte. /// /// This can be used to dynamically allocate a buffer with the correct number of /// bytes needed to store a message. @@ -89,9 +87,9 @@ pub(crate) fn take_last_error() -> Option> { /// /// See this module's documentation to get a complete example. #[no_mangle] -pub extern "C" fn wasmer_last_error_length() -> c_int { +pub extern "C" fn wasmer_last_error_length() -> usize { LAST_ERROR.with(|prev| match *prev.borrow() { - Some(ref err) => err.to_string().len() as c_int + 1, + Some(ref err) => err.len() + 1, None => 0, }) } @@ -121,8 +119,8 @@ pub extern "C" fn wasmer_last_error_length() -> c_int { #[no_mangle] pub unsafe extern "C" fn wasmer_last_error_message( buffer: Option>, - length: c_int, -) -> c_int { + length: usize, +) -> isize { let buffer = if let Some(buffer_inner) = buffer { buffer_inner } else { @@ -135,8 +133,6 @@ pub unsafe extern "C" fn wasmer_last_error_message( None => return 0, }; - let length = length as usize; - if error_message.len() >= length { // buffer is too small to hold the error message return -1; @@ -154,20 +150,5 @@ pub unsafe extern "C" fn wasmer_last_error_message( // accidentally read into garbage. buffer[error_message.len()] = 0; - error_message.len() as c_int + 1 + error_message.len() as isize + 1 } - -/// Rust type to represent a C API error. -#[derive(Debug)] -pub struct CApiError { - /// The error message. - pub msg: String, -} - -impl Display for CApiError { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "{}", &self.msg) - } -} - -impl Error for CApiError {} diff --git a/lib/c-api/src/wasm_c_api/engine.rs b/lib/c-api/src/wasm_c_api/engine.rs index 4fed7897b..bd77346a5 100644 --- a/lib/c-api/src/wasm_c_api/engine.rs +++ b/lib/c-api/src/wasm_c_api/engine.rs @@ -9,7 +9,7 @@ pub use super::unstable::middlewares::wasm_config_push_middleware; #[cfg(feature = "middlewares")] use super::unstable::middlewares::wasmer_middleware_t; use super::unstable::target_lexicon::wasmer_target_t; -use crate::error::{update_last_error, CApiError}; +use crate::error::update_last_error; use cfg_if::cfg_if; use std::sync::Arc; use wasmer_api::Engine; @@ -433,13 +433,8 @@ pub extern "C" fn wasm_engine_new_with_config( config: Option>, ) -> Option> { #[allow(dead_code)] - fn return_with_error(msg: M) -> Option> - where - M: ToString, - { - update_last_error(CApiError { - msg: msg.to_string(), - }); + fn return_with_error(msg: &str) -> Option> { + update_last_error(msg); return None; } diff --git a/lib/c-api/src/wasm_c_api/instance.rs b/lib/c-api/src/wasm_c_api/instance.rs index 1b95cbd12..c75746d4d 100644 --- a/lib/c-api/src/wasm_c_api/instance.rs +++ b/lib/c-api/src/wasm_c_api/instance.rs @@ -74,7 +74,7 @@ pub unsafe extern "C" fn wasm_instance_new( } Err(e @ InstantiationError::CpuFeature(_)) => { - crate::error::update_last_error(e.to_string()); + crate::error::update_last_error(e); return None; } diff --git a/lib/c-api/src/wasm_c_api/unstable/target_lexicon.rs b/lib/c-api/src/wasm_c_api/unstable/target_lexicon.rs index 269ec426e..ed8e9b2a5 100644 --- a/lib/c-api/src/wasm_c_api/unstable/target_lexicon.rs +++ b/lib/c-api/src/wasm_c_api/unstable/target_lexicon.rs @@ -54,7 +54,6 @@ //! ``` use super::super::types::wasm_name_t; -use crate::error::CApiError; use enumset::EnumSet; use std::slice; use std::str::{self, FromStr}; @@ -153,7 +152,7 @@ pub unsafe extern "C" fn wasmer_triple_new( ))); Some(Box::new(wasmer_triple_t { - inner: c_try!(Triple::from_str(triple).map_err(|e| CApiError { msg: e.to_string() })), + inner: c_try!(Triple::from_str(triple)), })) } diff --git a/lib/c-api/src/wasm_c_api/unstable/wasi.rs b/lib/c-api/src/wasm_c_api/unstable/wasi.rs index 9a860af04..5a1085016 100644 --- a/lib/c-api/src/wasm_c_api/unstable/wasi.rs +++ b/lib/c-api/src/wasm_c_api/unstable/wasi.rs @@ -5,7 +5,6 @@ use super::super::{ externals::wasm_extern_t, module::wasm_module_t, store::wasm_store_t, types::wasm_name_t, wasi::wasi_env_t, }; -use crate::error::CApiError; use wasmer_api::Extern; use wasmer_wasi::{generate_import_object_from_env, get_wasi_version}; @@ -168,11 +167,8 @@ fn wasi_get_unordered_imports_inner( let store = &store.inner; - let version = c_try!( - get_wasi_version(&module.inner, false).ok_or_else(|| CApiError { - msg: "could not detect a WASI version on the given module".to_string(), - }) - ); + let version = c_try!(get_wasi_version(&module.inner, false) + .ok_or("could not detect a WASI version on the given module")); let import_object = generate_import_object_from_env(store, wasi_env.inner.clone(), version); diff --git a/lib/c-api/src/wasm_c_api/value.rs b/lib/c-api/src/wasm_c_api/value.rs index da6b508af..3ead5c0cc 100644 --- a/lib/c-api/src/wasm_c_api/value.rs +++ b/lib/c-api/src/wasm_c_api/value.rs @@ -1,5 +1,5 @@ use super::types::{wasm_ref_t, wasm_valkind_enum}; -use crate::error::{update_last_error, CApiError}; +use crate::error::update_last_error; use std::convert::{TryFrom, TryInto}; use wasmer_api::Val; @@ -156,7 +156,7 @@ pub unsafe extern "C" fn wasm_val_copy( }, Err(e) => { - update_last_error(CApiError { msg: e.to_string() }); + update_last_error(e); return; } diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index a2a890608..8d53ac79a 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -11,7 +11,7 @@ use super::{ module::wasm_module_t, store::wasm_store_t, }; -use crate::error::{update_last_error, CApiError}; +use crate::error::update_last_error; use std::cmp::min; use std::convert::TryFrom; use std::ffi::CStr; @@ -212,9 +212,7 @@ pub unsafe extern "C" fn wasi_env_read_stdout( if let Some(stdout) = stdout.as_mut() { stdout } else { - update_last_error(CApiError { - msg: "could not find a file handle for `stdout`".to_string(), - }); + update_last_error("could not find a file handle for `stdout`"); return -1; } } else { @@ -235,15 +233,11 @@ pub unsafe extern "C" fn wasi_env_read_stderr( if let Some(stderr) = stderr.as_mut() { stderr } else { - update_last_error(CApiError { - msg: "could not find a file handle for `stderr`".to_string(), - }); + update_last_error("could not find a file handle for `stderr`"); return -1; } } else { - update_last_error(CApiError { - msg: "could not find a file handle for `stderr`".to_string(), - }); + update_last_error("could not find a file handle for `stderr`"); return -1; }; read_inner(stderr, inner_buffer) @@ -348,11 +342,8 @@ fn wasi_get_imports_inner( let store = &store.inner; - let version = c_try!( - get_wasi_version(&module.inner, false).ok_or_else(|| CApiError { - msg: "could not detect a WASI version on the given module".to_string(), - }) - ); + let version = c_try!(get_wasi_version(&module.inner, false) + .ok_or("could not detect a WASI version on the given module")); let import_object = generate_import_object_from_env(store, wasi_env.inner.clone(), version); @@ -362,18 +353,18 @@ fn wasi_get_imports_inner( .map(|import_type| { let export = import_object .resolve_by_name(import_type.module(), import_type.name()) - .ok_or_else(|| CApiError { - msg: format!( + .ok_or_else(|| { + format!( "Failed to resolve import \"{}\" \"{}\"", import_type.module(), import_type.name() - ), + ) })?; let inner = Extern::from_vm_export(store, export); Ok(Some(Box::new(inner.into()))) }) - .collect::, CApiError>>())); + .collect::, String>>())); Some(()) } From a1c998c8951fcb4bdfae5b603ca95db260a68ddb Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 23 Nov 2021 16:13:09 +0000 Subject: [PATCH 31/33] Fix compiler warning in singlepass --- lib/compiler-singlepass/src/emitter_x64.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler-singlepass/src/emitter_x64.rs b/lib/compiler-singlepass/src/emitter_x64.rs index 6ed4a7412..0d75b6120 100644 --- a/lib/compiler-singlepass/src/emitter_x64.rs +++ b/lib/compiler-singlepass/src/emitter_x64.rs @@ -483,7 +483,7 @@ macro_rules! binop_shift { macro_rules! jmp_op { ($ins:ident, $assembler:tt, $label:ident) => { - dynasm!($assembler ; $ins =>$label); + dynasm!($assembler ; $ins =>$label) } } From 5312521688059d8a181fe9583a3e2a7c996f0c29 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 23 Nov 2021 16:29:36 +0000 Subject: [PATCH 32/33] Apply review feedback --- lib/compiler-cranelift/src/compiler.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index ee23136a3..8187dbab2 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -112,10 +112,8 @@ impl Compiler for CraneliftCompiler { let probestack_trampoline_relocation_target = if target.triple().operating_system == OperatingSystem::Linux - && matches!( - target.triple().architecture, - Architecture::X86_32(_) | Architecture::X86_64 - ) { + && target.triple().architecture == Architecture::X86_64 + { let probestack_trampoline = CustomSection { protection: CustomSectionProtection::ReadExecute, // We create a jump to an absolute 64bits address From af29488bf935b014d7b81d5cdd458870345884c1 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 23 Nov 2021 17:59:31 +0000 Subject: [PATCH 33/33] Undo breaking change in C API --- lib/c-api/src/error.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/c-api/src/error.rs b/lib/c-api/src/error.rs index 2a4066503..cdf3e155c 100644 --- a/lib/c-api/src/error.rs +++ b/lib/c-api/src/error.rs @@ -47,7 +47,7 @@ //! # } //! ``` -use libc::c_char; +use libc::{c_char, c_int}; use std::cell::RefCell; use std::fmt::Display; use std::ptr::{self, NonNull}; @@ -86,10 +86,11 @@ pub(crate) fn take_last_error() -> Option { /// # Example /// /// See this module's documentation to get a complete example. +// TODO(Amanieu): This should use size_t #[no_mangle] -pub extern "C" fn wasmer_last_error_length() -> usize { +pub extern "C" fn wasmer_last_error_length() -> c_int { LAST_ERROR.with(|prev| match *prev.borrow() { - Some(ref err) => err.len() + 1, + Some(ref err) => err.len() as c_int + 1, None => 0, }) } @@ -116,11 +117,12 @@ pub extern "C" fn wasmer_last_error_length() -> usize { /// # Example /// /// See this module's documentation to get a complete example. +// TODO(Amanieu): This should use size_t #[no_mangle] pub unsafe extern "C" fn wasmer_last_error_message( buffer: Option>, - length: usize, -) -> isize { + length: c_int, +) -> c_int { let buffer = if let Some(buffer_inner) = buffer { buffer_inner } else { @@ -133,6 +135,8 @@ pub unsafe extern "C" fn wasmer_last_error_message( None => return 0, }; + let length = length as usize; + if error_message.len() >= length { // buffer is too small to hold the error message return -1; @@ -150,5 +154,5 @@ pub unsafe extern "C" fn wasmer_last_error_message( // accidentally read into garbage. buffer[error_message.len()] = 0; - error_message.len() as isize + 1 + error_message.len() as c_int + 1 }