diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index c6577b392..5e1221a43 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -2,19 +2,23 @@ on: [push] name: build +env: + RUST_BACKTRACE: 1 + jobs: lint: name: Code lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Install Rust nightly + - name: Install Rust stable uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2019-12-19 + profile: minimal + toolchain: 1.42.0 override: true components: rustfmt, clippy - - run: cargo fmt --all -- --check + - run: make lint test: name: Test in ${{ matrix.build }} @@ -41,8 +45,6 @@ jobs: toolchain: ${{ matrix.rust }} override: true - run: cargo test --release - env: - RUST_BACKTRACE: 1 - name: Build and Test C API run: | make capi @@ -51,5 +53,3 @@ jobs: - name: Build C API on Windows run: make capi if: matrix.os == 'windows-latest' - env: - RUST_BACKTRACE: 1 diff --git a/Cargo.lock b/Cargo.lock index a72252694..378dc5e3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -399,6 +399,31 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ba6eb47c2131e784a38b726eb54c1e1484904f013e576a25354d0124161af6" +[[package]] +name = "dynasm" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42a814e1edeb85dd2a3c6fc0d6bf76d02ca5695d438c70ecee3d90774f3259c5" +dependencies = [ + "bitflags", + "byteorder", + "lazy_static", + "owning_ref", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dynasmrt" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a393aaeb4441a48bcf47b5b6155971f82cc1eb77e22855403ccc0415ac8328d" +dependencies = [ + "byteorder", + "memmap", +] + [[package]] name = "either" version = "1.5.3" @@ -595,7 +620,7 @@ dependencies = [ [[package]] name = "inkwell" version = "0.1.0" -source = "git+https://github.com/theDan64/inkwell?rev=af1846fd8aea530cef9a59170715e5c6c64346f6#af1846fd8aea530cef9a59170715e5c6c64346f6" +source = "git+https://github.com/theDan64/inkwell?rev=1bfecc0a095d7ffdfa20a64630864f0297349508#1bfecc0a095d7ffdfa20a64630864f0297349508" dependencies = [ "either", "inkwell_internals", @@ -609,7 +634,7 @@ dependencies = [ [[package]] name = "inkwell_internals" version = "0.1.0" -source = "git+https://github.com/theDan64/inkwell?rev=af1846fd8aea530cef9a59170715e5c6c64346f6#af1846fd8aea530cef9a59170715e5c6c64346f6" +source = "git+https://github.com/theDan64/inkwell?rev=1bfecc0a095d7ffdfa20a64630864f0297349508#1bfecc0a095d7ffdfa20a64630864f0297349508" dependencies = [ "proc-macro2", "quote", @@ -667,9 +692,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" +checksum = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f" [[package]] name = "libloading" @@ -838,6 +863,15 @@ dependencies = [ "sdl2", ] +[[package]] +name = "owning_ref" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "parking_lot" version = "0.10.2" @@ -1298,6 +1332,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +[[package]] +name = "stable_deref_trait" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" + [[package]] name = "string-interner" version = "0.7.1" @@ -1403,10 +1443,12 @@ dependencies = [ name = "test-utils" version = "0.16.2" dependencies = [ + "wasmer", "wasmer-compiler", "wasmer-compiler-cranelift", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", + "wasmer-engine-jit", ] [[package]] @@ -1694,10 +1736,15 @@ dependencies = [ name = "wasmer-compiler-singlepass" version = "0.16.2" dependencies = [ + "byteorder", + "dynasm", + "dynasmrt", "hashbrown", + "lazy_static", "more-asserts", "rayon", "serde", + "smallvec", "wasm-common", "wasmer-compiler", "wasmer-runtime", diff --git a/Makefile b/Makefile index d502f47a1..9a73d4eea 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,11 @@ test: doc: cargo doc --all-features --document-private-items +RUSTFLAGS := "-D dead-code -D nonstandard-style -D unused-imports -D unused-mut -D unused-variables -D unused-unsafe -D unreachable-patterns -D bad-style -D improper-ctypes -D unused-allocation -D unused-comparisons -D while-true -D unconditional-recursion -D bare-trait-objects -D mutable-borrow-reservation-conflict" # TODO: add `-D missing-docs` +lint: + cargo fmt --all -- --check + RUSTFLAGS=${RUSTFLAGS} cargo clippy + capi-singlepass: cargo build --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features singlepass-backend,wasi diff --git a/bors.toml b/bors.toml new file mode 100644 index 000000000..1003d9678 --- /dev/null +++ b/bors.toml @@ -0,0 +1,6 @@ +status = [ + "wasmerio.wasmer" +] +required_approvals = 1 +timeout_sec = 7200 +delete_merged_branches = true diff --git a/build.rs b/build.rs index e05c17299..f9a72836f 100644 --- a/build.rs +++ b/build.rs @@ -13,10 +13,6 @@ use test_generator::{ with_features, with_test_module, Testsuite, }; -fn is_truthy_env(name: &str) -> bool { - env::var(name).map(|n| n == "1").unwrap_or_default() -} - fn main() -> anyhow::Result<()> { println!("cargo:rerun-if-changed=build.rs"); println!("cargo:rerun-if-changed=tests/ignores.txt"); @@ -30,7 +26,7 @@ fn main() -> anyhow::Result<()> { let mut spectests = Testsuite { buffer: String::new(), path: vec![], - ignores: ignores.clone(), + ignores, }; let backends = vec!["singlepass", "cranelift", "llvm"]; diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index f3377ac33..5f0189e12 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -29,7 +29,7 @@ winapi = "0.3.8" [dev-dependencies] # for the binary wasmer.rs -libc = "0.2" +libc = { version = "0.2.70", default-features = false } wat = "1.0.15" tempfile = "3.1" anyhow = "1.0.28" @@ -41,7 +41,7 @@ maintenance = { status = "actively-developed" } default = ["wat", "cranelift", "jit"] compiler = ["wasmer-engine-jit/compiler"] engine = [] -jit = ["wasmer-engine-jit"] +jit = ["wasmer-engine-jit", "engine"] singlepass = [ "wasmer-compiler-singlepass", "compiler", diff --git a/lib/api/src/exports.rs b/lib/api/src/exports.rs index e618fee32..5271e83d8 100644 --- a/lib/api/src/exports.rs +++ b/lib/api/src/exports.rs @@ -60,6 +60,11 @@ impl Exports { self.map.len() } + /// Return whether or not there are no exports + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Insert a new export into this `Exports` map. pub fn insert(&mut self, name: S, value: E) where @@ -84,7 +89,7 @@ impl Exports { /// type checking manually, please use `get_extern`. pub fn get<'a, T: Exportable<'a>>(&'a self, name: &str) -> Result<&T, ExportError> { match self.map.get(name) { - None => return Err(ExportError::Missing(name.to_string())), + None => Err(ExportError::Missing(name.to_string())), Some(extern_) => T::get_self_from_extern(extern_), } } @@ -114,14 +119,12 @@ impl Exports { self.map.get(name) } - /// Returns true if the `Exports` contains the given name. - pub fn contains(&mut self, name: S) -> bool + /// Returns true if the `Exports` contains the given export name. + pub fn contains(&self, name: S) -> bool where S: Into, { - Arc::get_mut(&mut self.map) - .unwrap() - .contains_key(&name.into()) + self.map.contains_key(&name.into()) } } diff --git a/lib/api/src/externals.rs b/lib/api/src/externals.rs index 48c18e123..0bdcad4d3 100644 --- a/lib/api/src/externals.rs +++ b/lib/api/src/externals.rs @@ -7,12 +7,14 @@ use crate::RuntimeError; use crate::{ExternType, FunctionType, GlobalType, MemoryType, TableType, ValType}; use std::cmp::max; use std::slice; -use wasm_common::{Bytes, HostFunction, Pages, ValueType, WasmTypeList, WithEnv, WithoutEnv}; -use wasmer_engine::Engine as _; +use wasm_common::{ + HostFunction, Pages, SignatureIndex, ValueType, WasmTypeList, WithEnv, WithoutEnv, +}; use wasmer_runtime::{ wasmer_call_trampoline, Export, ExportFunction, ExportGlobal, ExportMemory, ExportTable, - LinearMemory, Table as RuntimeTable, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, - VMGlobalDefinition, VMMemoryDefinition, VMTrampoline, + InstanceHandle, LinearMemory, MemoryError, Table as RuntimeTable, VMCallerCheckedAnyfunc, + VMContext, VMDynamicFunctionImportContext, VMFunctionBody, VMFunctionKind, VMGlobalDefinition, + VMMemoryDefinition, VMTrampoline, }; #[derive(Clone)] @@ -26,10 +28,10 @@ pub enum Extern { impl Extern { pub fn ty(&self) -> ExternType { match self { - Extern::Function(ft) => ExternType::Function(ft.ty().clone()), - Extern::Memory(ft) => ExternType::Memory(ft.ty().clone()), - Extern::Table(tt) => ExternType::Table(tt.ty().clone()), - Extern::Global(gt) => ExternType::Global(gt.ty().clone()), + Extern::Function(ft) => ExternType::Function(ft.ty()), + Extern::Memory(ft) => ExternType::Memory(*ft.ty()), + Extern::Table(tt) => ExternType::Table(*tt.ty()), + Extern::Global(gt) => ExternType::Global(*gt.ty()), } } @@ -162,7 +164,9 @@ impl Global { pub fn set(&self, val: Val) -> Result<(), RuntimeError> { if self.ty().mutability != Mutability::Var { - return Err(RuntimeError::new(format!("immutable global cannot be set"))); + return Err(RuntimeError::new( + "immutable global cannot be set".to_string(), + )); } if val.ty() != self.ty().ty { return Err(RuntimeError::new(format!( @@ -310,7 +314,7 @@ impl Table { src_index, len, ) - .map_err(|e| RuntimeError::from_trap(e))?; + .map_err(RuntimeError::from_trap)?; Ok(()) } @@ -344,25 +348,25 @@ pub struct Memory { } impl Memory { - pub fn new(store: &Store, ty: MemoryType) -> Memory { + pub fn new(store: &Store, ty: MemoryType) -> Result { let tunables = store.engine().tunables(); let memory_plan = tunables.memory_plan(ty); - let memory = tunables.create_memory(memory_plan).unwrap(); + let memory = tunables.create_memory(memory_plan)?; let definition = memory.vmmemory(); - Memory { + Ok(Memory { store: store.clone(), owned_by_store: true, exported: ExportMemory { from: Box::leak(Box::new(memory)), definition: Box::leak(Box::new(definition)), }, - } + }) } - fn definition(&self) -> &VMMemoryDefinition { - unsafe { &*self.exported.definition } + fn definition(&self) -> VMMemoryDefinition { + self.memory().vmmemory() } pub fn ty(&self) -> &MemoryType { @@ -377,6 +381,9 @@ impl Memory { self.data_unchecked_mut() } + /// TODO: document this function, it's trivial to cause UB/break soundness with this + /// method. + #[allow(clippy::mut_from_ref)] pub unsafe fn data_unchecked_mut(&self) -> &mut [u8] { let definition = self.definition(); slice::from_raw_parts_mut(definition.base, definition.current_length) @@ -391,14 +398,14 @@ impl Memory { } pub fn size(&self) -> Pages { - Bytes(self.data_size()).into() + self.memory().size() } fn memory(&self) -> &LinearMemory { unsafe { &*self.exported.from } } - pub fn grow(&self, delta: Pages) -> Option { + pub fn grow(&self, delta: Pages) -> Result { self.memory().grow(delta) } @@ -445,7 +452,7 @@ impl Memory { Memory { store: store.clone(), owned_by_store: false, - exported: wasmer_export.clone(), + exported: wasmer_export, } } } @@ -473,32 +480,26 @@ impl Drop for Memory { /// A function defined in the Wasm module #[derive(Clone, PartialEq)] -pub struct WasmFunc { +pub struct WasmFunctionDefinition { // The trampoline to do the call trampoline: VMTrampoline, } -/// A function defined in the Host -#[derive(Clone, PartialEq)] -pub struct HostFunc { - // func: wasm_common::Func, -} - /// The inner helper #[derive(Clone, PartialEq)] -pub enum InnerFunc { +pub enum FunctionDefinition { /// A function defined in the Wasm side - Wasm(WasmFunc), + Wasm(WasmFunctionDefinition), /// A function defined in the Host side - Host(HostFunc), + Host, } /// A WebAssembly `function`. #[derive(Clone, PartialEq)] pub struct Function { store: Store, + definition: FunctionDefinition, // If the Function is owned by the Store, not the instance - inner: InnerFunc, owned_by_store: bool, exported: ExportFunction, } @@ -515,21 +516,71 @@ impl Function { Rets: WasmTypeList, Env: Sized, { - let func: wasm_common::Func = wasm_common::Func::new(func); + let func: wasm_common::Func = wasm_common::Func::new(func); let address = func.address() as *const VMFunctionBody; - let vmctx = (func.env().unwrap_or(std::ptr::null_mut()) as *mut _) as *mut VMContext; + let vmctx = std::ptr::null_mut() as *mut _ as *mut VMContext; let func_type = func.ty(); let signature = store.engine().register_signature(&func_type); Self { store: store.clone(), owned_by_store: true, - inner: InnerFunc::Host(HostFunc { - // func - }), + definition: FunctionDefinition::Host, exported: ExportFunction { address, vmctx, signature, + kind: VMFunctionKind::Static, + }, + } + } + + #[allow(clippy::cast_ptr_alignment)] + pub fn new_dynamic(store: &Store, ty: &FunctionType, func: F) -> Self + where + F: Fn(&[Val]) -> Result, RuntimeError> + 'static, + { + let dynamic_ctx = + VMDynamicFunctionImportContext::from_context(VMDynamicFunctionWithoutEnv { + func: Box::new(func), + }); + let address = std::ptr::null() as *const () as *const VMFunctionBody; + let vmctx = Box::leak(Box::new(dynamic_ctx)) as *mut _ as *mut VMContext; + let signature = store.engine().register_signature(&ty); + Self { + store: store.clone(), + owned_by_store: true, + definition: FunctionDefinition::Host, + exported: ExportFunction { + address, + kind: VMFunctionKind::Dynamic, + vmctx, + signature, + }, + } + } + + #[allow(clippy::cast_ptr_alignment)] + pub fn new_dynamic_env(store: &Store, ty: &FunctionType, env: &mut Env, func: F) -> Self + where + F: Fn(&mut Env, &[Val]) -> Result, RuntimeError> + 'static, + Env: Sized, + { + let dynamic_ctx = VMDynamicFunctionImportContext::from_context(VMDynamicFunctionWithEnv { + env, + func: Box::new(func), + }); + let address = std::ptr::null() as *const () as *const VMFunctionBody; + let vmctx = Box::leak(Box::new(dynamic_ctx)) as *mut _ as *mut VMContext; + let signature = store.engine().register_signature(&ty); + Self { + store: store.clone(), + owned_by_store: true, + definition: FunctionDefinition::Host, + exported: ExportFunction { + address, + kind: VMFunctionKind::Dynamic, + vmctx, + signature, }, } } @@ -546,19 +597,23 @@ impl Function { Rets: WasmTypeList, Env: Sized, { - let func: wasm_common::Func = wasm_common::Func::new_env(env, func); + let func: wasm_common::Func = wasm_common::Func::new(func); let address = func.address() as *const VMFunctionBody; - let vmctx = (func.env().unwrap_or(std::ptr::null_mut()) as *mut _) as *mut VMContext; + // TODO: We need to refactor the Function context. + // Right now is structured as it's always a `VMContext`. However, only + // Wasm-defined functions have a `VMContext`. + // In the case of Host-defined functions `VMContext` is whatever environment + // the user want to attach to the function. + let vmctx = env as *mut _ as *mut VMContext; let func_type = func.ty(); let signature = store.engine().register_signature(&func_type); Self { store: store.clone(), owned_by_store: true, - inner: InnerFunc::Host(HostFunc { - // func - }), + definition: FunctionDefinition::Host, exported: ExportFunction { address, + kind: VMFunctionKind::Static, vmctx, signature, }, @@ -580,7 +635,7 @@ impl Function { fn call_wasm( &self, - func: &WasmFunc, + func: &WasmFunctionDefinition, params: &[Val], results: &mut [Val], ) -> Result<(), RuntimeError> { @@ -642,10 +697,10 @@ impl Function { } // Load the return values out of `values_vec`. - for (index, value_type) in signature.results().iter().enumerate() { + for (index, &value_type) in signature.results().iter().enumerate() { unsafe { let ptr = values_vec.as_ptr().add(index); - results[index] = Val::read_value_from(ptr, value_type.clone()); + results[index] = Val::read_value_from(ptr, value_type); } } @@ -671,8 +726,8 @@ impl Function { /// call the trampoline. pub fn call(&self, params: &[Val]) -> Result, RuntimeError> { let mut results = vec![Val::null(); self.result_arity()]; - match &self.inner { - InnerFunc::Wasm(wasm) => { + match &self.definition { + FunctionDefinition::Wasm(wasm) => { self.call_wasm(&wasm, params, &mut results)?; } _ => {} // _ => unimplemented!("The host is unimplemented"), @@ -683,12 +738,12 @@ impl Function { pub(crate) fn from_export(store: &Store, wasmer_export: ExportFunction) -> Self { let trampoline = store .engine() - .trampoline(wasmer_export.signature) - .expect("Can't get trampoline for the function"); + .function_call_trampoline(wasmer_export.signature) + .expect("Can't get call trampoline for the function"); Self { store: store.clone(), owned_by_store: false, - inner: InnerFunc::Wasm(WasmFunc { trampoline }), + definition: FunctionDefinition::Wasm(WasmFunctionDefinition { trampoline }), exported: wasmer_export, } } @@ -719,3 +774,113 @@ impl std::fmt::Debug for Function { Ok(()) } } + +/// This trait is one that all dynamic funcitons must fulfill. +trait VMDynamicFunction { + fn call(&self, args: &[Val]) -> Result, RuntimeError>; +} + +struct VMDynamicFunctionWithoutEnv { + func: Box Result, RuntimeError> + 'static>, +} + +impl VMDynamicFunction for VMDynamicFunctionWithoutEnv { + fn call(&self, args: &[Val]) -> Result, RuntimeError> { + (*self.func)(&args) + } +} + +struct VMDynamicFunctionWithEnv +where + Env: Sized, +{ + func: Box Result, RuntimeError> + 'static>, + env: *mut Env, +} + +impl VMDynamicFunction for VMDynamicFunctionWithEnv +where + Env: Sized, +{ + fn call(&self, args: &[Val]) -> Result, RuntimeError> { + unsafe { (*self.func)(&mut *self.env, &args) } + } +} + +trait VMDynamicFunctionImportCall { + fn from_context(ctx: T) -> Self; + fn address_ptr() -> *const VMFunctionBody; + unsafe fn func_wrapper( + &self, + caller_vmctx: *mut VMContext, + sig_index: SignatureIndex, + values_vec: *mut i128, + ); +} + +impl VMDynamicFunctionImportCall for VMDynamicFunctionImportContext { + fn from_context(ctx: T) -> Self { + Self { + address: Self::address_ptr(), + ctx, + } + } + + fn address_ptr() -> *const VMFunctionBody { + Self::func_wrapper as *const () as *const VMFunctionBody + } + + // This function wraps our func, to make it compatible with the + // reverse trampoline signature + unsafe fn func_wrapper( + // Note: we use the trick that the first param to this function is the `VMDynamicFunctionImportContext` + // itself, so rather than doing `dynamic_ctx: &VMDynamicFunctionImportContext`, we simplify it a bit + &self, + caller_vmctx: *mut VMContext, + sig_index: SignatureIndex, + values_vec: *mut i128, + ) { + use std::panic::{self, AssertUnwindSafe}; + let result = panic::catch_unwind(AssertUnwindSafe(|| { + // This is actually safe, since right now the function signature + // receives two contexts: + // 1. `vmctx`: the context associated to where the function is defined. + // It will be `VMContext` in case is defined in Wasm, and a custom + // `Env` in case is host defined. + // 2. `caller_vmctx`: the context associated to whoever is calling that function. + // + // Because this code will only be reached when calling from wasm to host, we + // can assure the callee_vmctx is indeed a VMContext, and hence is completely + // safe to get a handle from it. + let handle = InstanceHandle::from_vmctx(caller_vmctx); + let module = handle.module_ref(); + let func_ty = &module.signatures[sig_index]; + let mut args = Vec::with_capacity(func_ty.params().len()); + for (i, ty) in func_ty.params().iter().enumerate() { + args.push(Val::read_value_from(values_vec.add(i), *ty)); + } + let returns = self.ctx.call(&args)?; + + // We need to dynamically check that the returns + // match the expected types, as well as expected length. + let return_types = returns.iter().map(|ret| ret.ty()).collect::>(); + if return_types != func_ty.results() { + return Err(RuntimeError::new(format!( + "Dynamic function returned wrong signature. Expected {:?} but got {:?}", + func_ty.results(), + return_types + ))); + } + for (i, ret) in returns.iter().enumerate() { + ret.write_value_to(values_vec.add(i)); + } + Ok(()) + })); + + match result { + Ok(Ok(())) => {} + Ok(Err(trap)) => wasmer_runtime::raise_user_trap(Box::new(trap)), + Err(panic) => wasmer_runtime::resume_panic(panic), + } + } +} diff --git a/lib/api/src/instance.rs b/lib/api/src/instance.rs index 2f5ad1e7a..03750d9ed 100644 --- a/lib/api/src/instance.rs +++ b/lib/api/src/instance.rs @@ -61,8 +61,8 @@ impl Instance { .map(|export| { let name = export.name().to_string(); let export = handle.lookup(&name).expect("export"); - let extern_ = Extern::from_export(store, export.clone()); - (name.to_string(), extern_) + let extern_ = Extern::from_export(store, export); + (name, extern_) }) .collect::(); diff --git a/lib/api/src/lib.rs b/lib/api/src/lib.rs index 390c97485..b83cf432d 100644 --- a/lib/api/src/lib.rs +++ b/lib/api/src/lib.rs @@ -34,6 +34,7 @@ pub use wasmer_compiler::{Features, Target}; pub use wasmer_engine::{ DeserializeError, Engine, InstantiationError, LinkError, RuntimeError, SerializeError, }; +pub use wasmer_runtime::MemoryError; // The compilers are mutually exclusive #[cfg(any( diff --git a/lib/api/src/module.rs b/lib/api/src/module.rs index 22bba2378..791005ad0 100644 --- a/lib/api/src/module.rs +++ b/lib/api/src/module.rs @@ -6,8 +6,10 @@ use std::io; use std::path::Path; use std::sync::Arc; use thiserror::Error; -use wasmer_compiler::{CompileError, WasmError}; -use wasmer_engine::{CompiledModule, DeserializeError, Engine, Resolver, SerializeError}; +use wasmer_compiler::CompileError; +#[cfg(feature = "wat")] +use wasmer_compiler::WasmError; +use wasmer_engine::{CompiledModule, DeserializeError, Resolver, SerializeError}; use wasmer_runtime::{ExportsIterator, ImportsIterator, InstanceHandle, Module as ModuleInfo}; #[derive(Error, Debug)] @@ -74,6 +76,7 @@ impl Module { /// let bytes: Vec = vec![]; /// let module = Module::new(&store, bytes)?; /// ``` + #[allow(unreachable_code)] pub fn new(store: &Store, bytes: impl AsRef<[u8]>) -> Result { #[cfg(feature = "wat")] { @@ -206,7 +209,7 @@ impl Module { Ok(Self::from_compiled_module(store, compiled)) } - fn from_compiled_module(store: &Store, compiled: Arc) -> Self { + fn from_compiled_module(store: &Store, compiled: Arc) -> Self { Module { store: store.clone(), compiled, diff --git a/lib/api/src/ptr.rs b/lib/api/src/ptr.rs index fe4f5f75a..e48feda9c 100644 --- a/lib/api/src/ptr.rs +++ b/lib/api/src/ptr.rs @@ -272,7 +272,7 @@ mod test { // create a memory let store = Store::default(); let memory_descriptor = MemoryType::new(1, Some(1), false); - let memory = Memory::new(&store, memory_descriptor); + let memory = Memory::new(&store, memory_descriptor).unwrap(); // test that basic access works and that len = 0 works, but oob does not let start_wasm_ptr: WasmPtr = WasmPtr::new(0); diff --git a/lib/api/src/store.rs b/lib/api/src/store.rs index e4eb6af87..088ba2a16 100644 --- a/lib/api/src/store.rs +++ b/lib/api/src/store.rs @@ -1,7 +1,9 @@ +#[cfg(all(feature = "compiler", feature = "engine"))] use crate::tunables::Tunables; -use std::sync::Arc; -#[cfg(feature = "compiler")] +#[cfg(all(feature = "compiler", feature = "engine"))] use wasmer_compiler::CompilerConfig; + +use std::sync::Arc; use wasmer_engine::Engine; #[derive(Clone)] diff --git a/lib/api/src/tunables.rs b/lib/api/src/tunables.rs index 624237cfb..8eb097535 100644 --- a/lib/api/src/tunables.rs +++ b/lib/api/src/tunables.rs @@ -3,6 +3,7 @@ use std::cmp::min; use target_lexicon::{OperatingSystem, PointerWidth, Triple, HOST}; use wasm_common::{MemoryType, Pages, TableType}; use wasmer_engine::Tunables as BaseTunables; +use wasmer_runtime::MemoryError; use wasmer_runtime::{LinearMemory, Table}; use wasmer_runtime::{MemoryPlan, MemoryStyle, TablePlan, TableStyle}; @@ -93,7 +94,7 @@ impl BaseTunables for Tunables { } /// Create a memory given a memory type - fn create_memory(&self, plan: MemoryPlan) -> Result { + fn create_memory(&self, plan: MemoryPlan) -> Result { LinearMemory::new(&plan) } diff --git a/lib/api/src/types.rs b/lib/api/src/types.rs index 6b130efae..f477362a6 100644 --- a/lib/api/src/types.rs +++ b/lib/api/src/types.rs @@ -64,6 +64,9 @@ impl ValAnyFunc for Val { let export = wasmer_runtime::ExportFunction { address: item.func_ptr, signature: item.type_index, + // All functions in tables are already Static (as dynamic functions + // are converted to use the trampolines with static signatures). + kind: wasmer_runtime::VMFunctionKind::Static, vmctx: item.vmctx, }; let f = Function::from_export(store, export); diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index ffbcbaf4b..37844932e 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib", "staticlib"] [dependencies] lazy_static = "1" -libc = "0.2.60" +libc = { version = "0.2.70", default-features = false } # for generating code in the same way thot the wasm-c-api does # Commented out for now until we can find a solution to the exported function problem # wasmer-wasm-c-api = { version = "0.16.2", path = "crates/wasm-c-api" } diff --git a/lib/c-api/src/error.rs b/lib/c-api/src/error.rs index 12e301979..ee713aebb 100644 --- a/lib/c-api/src/error.rs +++ b/lib/c-api/src/error.rs @@ -1,12 +1,11 @@ //! Read runtime errors. use libc::{c_char, c_int}; -use std::{ - cell::RefCell, - error::Error, - fmt::{self, Display, Formatter}, - ptr, slice, -}; +use std::cell::RefCell; +use std::error::Error; +use std::fmt::{self, Display, Formatter}; +use std::ptr::{self, NonNull}; +use std::slice; thread_local! { static LAST_ERROR: RefCell>> = RefCell::new(None); @@ -66,11 +65,16 @@ pub extern "C" fn wasmer_last_error_length() -> c_int { /// } /// ``` #[no_mangle] -pub unsafe extern "C" fn wasmer_last_error_message(buffer: *mut c_char, length: c_int) -> c_int { - if buffer.is_null() { +pub unsafe extern "C" fn wasmer_last_error_message( + buffer: Option>, + length: c_int, +) -> c_int { + let buffer = if let Some(buffer_inner) = buffer { + buffer_inner + } else { // buffer pointer is null return -1; - } + }; let error_message = match take_last_error() { Some(err) => err.to_string(), @@ -84,7 +88,7 @@ pub unsafe extern "C" fn wasmer_last_error_message(buffer: *mut c_char, length: return -1; } - let buffer = slice::from_raw_parts_mut(buffer as *mut u8, length); + let buffer = slice::from_raw_parts_mut(buffer.cast::().as_ptr(), length); ptr::copy_nonoverlapping( error_message.as_ptr(), diff --git a/lib/c-api/src/export.rs b/lib/c-api/src/export.rs index 59190e4a3..86c9095a1 100644 --- a/lib/c-api/src/export.rs +++ b/lib/c-api/src/export.rs @@ -23,7 +23,7 @@ pub(crate) struct NamedExport { pub(crate) export_type: ExportType, /// The instance that holds the export. - pub(crate) instance: *mut Instance, + pub(crate) instance: NonNull, } /// Opaque pointer to `ImportType`. @@ -400,7 +400,7 @@ pub unsafe extern "C" fn wasmer_export_to_memory( memory: *mut *mut wasmer_memory_t, ) -> wasmer_result_t { let named_export = &*(export as *const NamedExport); - let instance = &*named_export.instance; + let instance = named_export.instance.as_ref(); if let Ok(exported_memory) = instance .exports @@ -476,7 +476,7 @@ pub unsafe extern "C" fn wasmer_export_func_call( let results: &mut [wasmer_value_t] = slice::from_raw_parts_mut(results, results_len as usize); - let instance = &*named_export.instance; + let instance = named_export.instance.as_ref(); let f: &Function = match instance.exports.get(&named_export.export_type.name()) { Ok(f) => f, Err(err) => { diff --git a/lib/c-api/src/global.rs b/lib/c-api/src/global.rs index ebdcfd195..694bce40b 100644 --- a/lib/c-api/src/global.rs +++ b/lib/c-api/src/global.rs @@ -1,6 +1,8 @@ //! Create, set, get and destroy global variables of an instance. +use crate::error::update_last_error; use crate::value::{wasmer_value_t, wasmer_value_tag}; +use std::ptr::NonNull; use wasmer::Global; #[repr(C)] @@ -17,10 +19,7 @@ pub struct wasmer_global_t; /// Creates a new Global and returns a pointer to it. /// The caller owns the object and should call `wasmer_global_destroy` to free it. #[no_mangle] -pub unsafe extern "C" fn wasmer_global_new( - value: wasmer_value_t, - mutable: bool, -) -> *mut wasmer_global_t { +pub extern "C" fn wasmer_global_new(value: wasmer_value_t, mutable: bool) -> *mut wasmer_global_t { let store = crate::get_global_store(); let global = if mutable { Global::new_mut(store, value.into()) @@ -33,8 +32,8 @@ pub unsafe extern "C" fn wasmer_global_new( /// Gets the value stored by the given Global #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_global_get(global: *mut wasmer_global_t) -> wasmer_value_t { - let global = unsafe { &*(global as *mut Global) }; +pub unsafe extern "C" fn wasmer_global_get(global: *mut wasmer_global_t) -> wasmer_value_t { + let global = &*(global as *mut Global); let value: wasmer_value_t = global.get().into(); value } @@ -42,18 +41,22 @@ pub extern "C" fn wasmer_global_get(global: *mut wasmer_global_t) -> wasmer_valu /// Sets the value stored by the given Global #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_global_set(global: *mut wasmer_global_t, value: wasmer_value_t) { - let global = unsafe { &*(global as *mut Global) }; - global.set(value.into()); +pub unsafe extern "C" fn wasmer_global_set(global: *mut wasmer_global_t, value: wasmer_value_t) { + let global = &*(global as *mut Global); + if let Err(err) = global.set(value.into()) { + update_last_error(err); + // can't return an error without breaking the API, probaly a safe change + // return wasmer_result_t::WASMER_ERROR; + } } /// Returns a descriptor (type, mutability) of the given Global #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_global_get_descriptor( +pub unsafe extern "C" fn wasmer_global_get_descriptor( global: *mut wasmer_global_t, ) -> wasmer_global_descriptor_t { - let global = unsafe { &*(global as *mut Global) }; + let global = &*(global as *mut Global); let descriptor = global.ty(); wasmer_global_descriptor_t { mutable: descriptor.mutability.into(), @@ -64,8 +67,8 @@ pub extern "C" fn wasmer_global_get_descriptor( /// Frees memory for the given Global #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_global_destroy(global: *mut wasmer_global_t) { - if !global.is_null() { - unsafe { Box::from_raw(global as *mut Global) }; +pub unsafe extern "C" fn wasmer_global_destroy(global: Option>) { + if let Some(global_inner) = global { + Box::from_raw(global_inner.cast::().as_ptr()); } } diff --git a/lib/c-api/src/import/mod.rs b/lib/c-api/src/import/mod.rs index c6aa9229d..1682a1e8e 100644 --- a/lib/c-api/src/import/mod.rs +++ b/lib/c-api/src/import/mod.rs @@ -10,11 +10,11 @@ use crate::{ wasmer_byte_array, wasmer_result_t, }; use libc::c_uint; +use std::ptr::NonNull; use std::{ //convert::TryFrom, ffi::c_void, os::raw::c_char, - ptr, slice, //sync::Arc, }; @@ -58,10 +58,11 @@ pub struct wasmer_import_object_iter_t; /// See also `wasmer_import_object_append` #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub unsafe extern "C" fn wasmer_import_object_new() -> *mut wasmer_import_object_t { +pub extern "C" fn wasmer_import_object_new() -> NonNull { let import_object = Box::new(ImportObject::new()); - Box::into_raw(import_object) as *mut wasmer_import_object_t + // TODO: use `Box::into_raw_non_null` when it becomes stable + unsafe { NonNull::new_unchecked(Box::into_raw(import_object) as *mut wasmer_import_object_t) } } #[cfg(feature = "wasi")] @@ -319,15 +320,17 @@ pub unsafe extern "C" fn wasmer_import_object_iter_next( /// not return any new data #[no_mangle] pub unsafe extern "C" fn wasmer_import_object_iter_at_end( - import_object_iter: *mut wasmer_import_object_iter_t, + import_object_iter: Option>, ) -> bool { - if import_object_iter.is_null() { + let mut import_object_iter = if let Some(import_object_iter) = import_object_iter { + import_object_iter.cast::() + } else { update_last_error(CApiError { msg: "import_object_iter must not be null".to_owned(), }); return true; - } - let iter = &mut *(import_object_iter as *mut WasmerImportObjectIterator); + }; + let iter = import_object_iter.as_mut(); iter.0.peek().is_none() } @@ -335,10 +338,14 @@ pub unsafe extern "C" fn wasmer_import_object_iter_at_end( /// Frees the memory allocated by `wasmer_import_object_iterate_functions` #[no_mangle] pub unsafe extern "C" fn wasmer_import_object_iter_destroy( - import_object_iter: *mut wasmer_import_object_iter_t, + import_object_iter: Option>, ) { - if !import_object_iter.is_null() { - let _ = Box::from_raw(import_object_iter as *mut WasmerImportObjectIterator); + if let Some(import_object_iter) = import_object_iter { + let _ = Box::from_raw( + import_object_iter + .cast::() + .as_ptr(), + ); } } @@ -348,13 +355,16 @@ pub unsafe extern "C" fn wasmer_import_object_iter_destroy( /// it only frees memory allocated while querying a `wasmer_import_object_t`. #[no_mangle] pub unsafe extern "C" fn wasmer_import_object_imports_destroy( - imports: *mut wasmer_import_t, + imports: Option>, imports_len: u32, ) { - if imports.is_null() { + let imports = if let Some(imp) = imports { + imp + } else { return; - } - let imports: &[wasmer_import_t] = &*slice::from_raw_parts_mut(imports, imports_len as usize); + }; + let imports: &[wasmer_import_t] = + &*slice::from_raw_parts_mut(imports.as_ptr(), imports_len as usize); for import in imports { let _namespace: Vec = Vec::from_raw_parts( import.module_name.bytes as *mut u8, @@ -459,13 +469,14 @@ pub unsafe extern "C" fn wasmer_import_object_extend( #[allow(clippy::cast_ptr_alignment)] #[no_mangle] pub unsafe extern "C" fn wasmer_import_descriptors( - module: *const wasmer_module_t, + module: Option<&wasmer_module_t>, import_descriptors: *mut *mut wasmer_import_descriptors_t, ) { - if module.is_null() { + let module = if let Some(module) = module { + &*(module as *const wasmer_module_t as *const Module) + } else { return; - } - let module = &*(module as *const Module); + }; let descriptors = module.imports().collect::>(); let named_import_descriptors: Box = @@ -480,10 +491,10 @@ pub struct NamedImportDescriptors(Vec); #[allow(clippy::cast_ptr_alignment)] #[no_mangle] pub extern "C" fn wasmer_import_descriptors_destroy( - import_descriptors: *mut wasmer_import_descriptors_t, + import_descriptors: Option>, ) { - if !import_descriptors.is_null() { - unsafe { Box::from_raw(import_descriptors as *mut NamedImportDescriptors) }; + if let Some(id) = import_descriptors { + unsafe { Box::from_raw(id.cast::().as_ptr()) }; } } @@ -491,27 +502,29 @@ pub extern "C" fn wasmer_import_descriptors_destroy( #[allow(clippy::cast_ptr_alignment)] #[no_mangle] pub unsafe extern "C" fn wasmer_import_descriptors_len( - exports: *mut wasmer_import_descriptors_t, + exports: Option>, ) -> c_uint { - if exports.is_null() { + let exports = if let Some(exports) = exports { + exports.cast::() + } else { return 0; - } - (*(exports as *mut NamedImportDescriptors)).0.len() as c_uint + }; + exports.as_ref().0.len() as c_uint } /// Gets import descriptor by index #[allow(clippy::cast_ptr_alignment)] #[no_mangle] pub unsafe extern "C" fn wasmer_import_descriptors_get( - import_descriptors: *mut wasmer_import_descriptors_t, + import_descriptors: Option>, idx: c_uint, -) -> *mut wasmer_import_descriptor_t { - if import_descriptors.is_null() { - return ptr::null_mut(); - } - let named_import_descriptors = &mut *(import_descriptors as *mut NamedImportDescriptors); - &mut (*named_import_descriptors).0[idx as usize] as *mut ImportType - as *mut wasmer_import_descriptor_t +) -> Option> { + let mut nid = import_descriptors?.cast::(); + let named_import_descriptors = nid.as_mut(); + Some( + NonNull::from(&mut named_import_descriptors.0[idx as usize]) + .cast::(), + ) } /// Gets name for the import descriptor @@ -754,16 +767,18 @@ pub unsafe extern "C" fn wasmer_import_func_returns_arity( /// Frees memory for the given Func #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_import_func_destroy(func: *mut wasmer_import_func_t) { - if !func.is_null() { - unsafe { Box::from_raw(func as *mut Function) }; +pub unsafe extern "C" fn wasmer_import_func_destroy(func: Option>) { + if let Some(func) = func { + Box::from_raw(func.cast::().as_ptr()); } } /// Frees memory of the given ImportObject #[no_mangle] -pub extern "C" fn wasmer_import_object_destroy(import_object: *mut wasmer_import_object_t) { - if !import_object.is_null() { - unsafe { Box::from_raw(import_object as *mut ImportObject) }; +pub unsafe extern "C" fn wasmer_import_object_destroy( + import_object: Option>, +) { + if let Some(import_object) = import_object { + Box::from_raw(import_object.cast::().as_ptr()); } } diff --git a/lib/c-api/src/import/wasi.rs b/lib/c-api/src/import/wasi.rs index 5aca465d4..e63f949f7 100644 --- a/lib/c-api/src/import/wasi.rs +++ b/lib/c-api/src/import/wasi.rs @@ -213,7 +213,7 @@ pub unsafe extern "C" fn wasmer_wasi_generate_default_import_object() -> *mut wa let mut wasi_env = wasi::WasiEnv::new(wasi_state); // this API will now leak a `Memory` let memory_type = MemoryType::new(0, None, false); - let memory = Memory::new(store, memory_type); + let memory = Memory::new(store, memory_type).expect("create memory"); wasi_env.set_memory(&memory); // TODO(mark): review lifetime of `Memory` here let import_object = Box::new(wasi::generate_import_object_from_env( diff --git a/lib/c-api/src/instance.rs b/lib/c-api/src/instance.rs index 948717004..a13a86bae 100644 --- a/lib/c-api/src/instance.rs +++ b/lib/c-api/src/instance.rs @@ -11,7 +11,7 @@ use crate::{ use libc::{c_char, c_int, c_void}; use std::collections::HashMap; use std::ffi::CStr; -use std::ptr; +use std::ptr::NonNull; use std::slice; use wasmer::{ Exports, Extern, Function, Global, ImportObject, Instance, Memory, Module, Table, Val, @@ -99,17 +99,19 @@ pub struct wasmer_instance_context_t; #[no_mangle] pub unsafe extern "C" fn wasmer_instantiate( instance: *mut *mut wasmer_instance_t, - wasm_bytes: *mut u8, + wasm_bytes: Option>, wasm_bytes_len: u32, imports: *mut wasmer_import_t, imports_len: c_int, ) -> wasmer_result_t { - if wasm_bytes.is_null() { + let wasm_bytes = if let Some(wasm_bytes_inner) = wasm_bytes { + wasm_bytes_inner + } else { update_last_error(CApiError { msg: "wasm bytes ptr is null".to_string(), }); return wasmer_result_t::WASMER_ERROR; - } + }; let imports: &[wasmer_import_t] = slice::from_raw_parts(imports, imports_len as usize); let mut import_object = ImportObject::new(); let mut namespaces = HashMap::new(); @@ -166,7 +168,7 @@ pub unsafe extern "C" fn wasmer_instantiate( import_object.register(module_name, namespace); } - let bytes: &[u8] = slice::from_raw_parts_mut(wasm_bytes, wasm_bytes_len as usize); + let bytes: &[u8] = slice::from_raw_parts_mut(wasm_bytes.as_ptr(), wasm_bytes_len as usize); let store = crate::get_global_store(); let module_result = Module::from_binary(store, bytes); @@ -206,15 +208,11 @@ pub unsafe extern "C" fn wasmer_instantiate( #[allow(clippy::cast_ptr_alignment)] #[no_mangle] pub extern "C" fn wasmer_instance_context_get( - instance: *mut wasmer_instance_t, -) -> *const wasmer_instance_context_t { - if instance.is_null() { - return ptr::null() as _; - } - + instance: Option>, +) -> Option<&'static wasmer_instance_context_t> { unimplemented!("wasmer_instance_context_get: API changed") /* - let instance = unsafe { &*(instance as *const Instance) }; + let instance = instance?.as_ref(); let context: *const Ctx = instance.context() as *const _; context as *const wasmer_instance_context_t @@ -403,20 +401,24 @@ pub unsafe extern "C" fn wasmer_instance_call( #[allow(clippy::cast_ptr_alignment)] #[no_mangle] pub unsafe extern "C" fn wasmer_instance_exports( - instance: *mut wasmer_instance_t, + instance: Option>, exports: *mut *mut wasmer_exports_t, ) { - if instance.is_null() { + let instance = if let Some(instance) = instance { + instance.cast::() + } else { return; - } + }; - let instance_ref = &mut *(instance as *mut Instance); - let mut exports_vec: Vec = instance_ref + let mut instance_ref_copy = instance.clone(); + let instance_ref = instance_ref_copy.as_mut(); + + let exports_vec: Vec = instance_ref .module() .exports() .map(|export_type| NamedExport { export_type, - instance: instance as *mut Instance, + instance, }) .collect(); @@ -551,8 +553,8 @@ pub extern "C" fn wasmer_instance_context_data_get( /// ``` #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_instance_destroy(instance: *mut wasmer_instance_t) { - if !instance.is_null() { - unsafe { Box::from_raw(instance as *mut Instance) }; +pub unsafe extern "C" fn wasmer_instance_destroy(instance: Option>) { + if let Some(instance_inner) = instance { + Box::from_raw(instance_inner.cast::().as_ptr()); } } diff --git a/lib/c-api/src/lib.rs b/lib/c-api/src/lib.rs index 9b172f602..f959cbc94 100644 --- a/lib/c-api/src/lib.rs +++ b/lib/c-api/src/lib.rs @@ -1,5 +1,7 @@ #![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] #![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] +// temporary while in transition +#![allow(unused_variables)] //! # Wasmer Runtime C API //! diff --git a/lib/c-api/src/memory.rs b/lib/c-api/src/memory.rs index f73e7f9ee..10a98a90d 100644 --- a/lib/c-api/src/memory.rs +++ b/lib/c-api/src/memory.rs @@ -69,9 +69,18 @@ pub unsafe extern "C" fn wasmer_memory_new( }; let store = crate::get_global_store(); let desc = MemoryType::new(Pages(limits.min), max, false); - let new_memory = Memory::new(store, desc); - *memory = Box::into_raw(Box::new(new_memory)) as *mut wasmer_memory_t; - wasmer_result_t::WASMER_OK + match Memory::new(store, desc) { + Ok(new_memory) => { + *memory = Box::into_raw(Box::new(new_memory)) as *mut wasmer_memory_t; + wasmer_result_t::WASMER_OK + } + Err(err) => { + update_last_error(CApiError { + msg: err.to_string(), + }); + wasmer_result_t::WASMER_ERROR + } + } } /// Grows a memory by the given number of pages (of 65Kb each). @@ -105,8 +114,13 @@ pub extern "C" fn wasmer_memory_grow(memory: *mut wasmer_memory_t, delta: u32) - let grow_result = memory.grow(Pages(delta)); match grow_result { - Some(_) => wasmer_result_t::WASMER_OK, - _ => wasmer_result_t::WASMER_ERROR, + Ok(_) => wasmer_result_t::WASMER_OK, + Err(err) => { + update_last_error(CApiError { + msg: err.to_string(), + }); + wasmer_result_t::WASMER_ERROR + } } } diff --git a/lib/c-api/src/module.rs b/lib/c-api/src/module.rs index 8bab7ee16..217beae36 100644 --- a/lib/c-api/src/module.rs +++ b/lib/c-api/src/module.rs @@ -196,7 +196,7 @@ pub unsafe extern "C" fn wasmer_module_serialize( let module = &*(module as *const Module); match module.serialize() { - Ok(mut serialized_module) => { + Ok(serialized_module) => { let boxed_slice = serialized_module.into_boxed_slice(); *serialized_module_out = Box::into_raw(Box::new(boxed_slice)) as _; @@ -267,16 +267,17 @@ pub unsafe extern "C" fn wasmer_serialized_module_from_bytes( #[no_mangle] pub unsafe extern "C" fn wasmer_module_deserialize( module: *mut *mut wasmer_module_t, - serialized_module: *const wasmer_serialized_module_t, + serialized_module: Option<&wasmer_serialized_module_t>, ) -> wasmer_result_t { - if serialized_module.is_null() { + let serialized_module: &[u8] = if let Some(sm) = serialized_module { + &*(sm as *const wasmer_serialized_module_t as *const &[u8]) + } else { update_last_error(CApiError { msg: "`serialized_module` pointer is null".to_string(), }); return wasmer_result_t::WASMER_ERROR; - } + }; - let serialized_module: &[u8] = &*(serialized_module as *const &[u8]); let store = crate::get_global_store(); match Module::deserialize(store, serialized_module) { diff --git a/lib/c-api/src/table.rs b/lib/c-api/src/table.rs index 909fdf041..d1f545b15 100644 --- a/lib/c-api/src/table.rs +++ b/lib/c-api/src/table.rs @@ -1,6 +1,7 @@ //! Create, grow, destroy tables of an instance. use crate::{error::update_last_error, wasmer_limits_t, wasmer_result_t}; +use std::ptr::NonNull; use wasmer::{AnyRef, Table, TableType, Val, ValType}; #[repr(C)] @@ -15,8 +16,8 @@ fn get_default_table_value(table_type: ValType) -> Val { ValType::F32 => Val::F32(0.), ValType::F64 => Val::F64(0.), ValType::V128 => Val::V128(0), - ValType::AnyRef => Val::AnyRef(AnyRef::null()), // todo!("Figure out what the default AnyRef value is"), - ValType::FuncRef => Val::AnyRef(AnyRef::null()), //todo!("Figure out what the default FuncRef value is"), + ValType::AnyRef => Val::AnyRef(AnyRef::null()), + ValType::FuncRef => Val::AnyRef(AnyRef::null()), } } @@ -65,8 +66,11 @@ pub unsafe extern "C" fn wasmer_table_new( /// and `wasmer_last_error_message` to get an error message. #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_table_grow(table: *mut wasmer_table_t, delta: u32) -> wasmer_result_t { - let table = unsafe { &*(table as *mut Table) }; +pub unsafe extern "C" fn wasmer_table_grow( + table: *mut wasmer_table_t, + delta: u32, +) -> wasmer_result_t { + let table = &*(table as *mut Table); let table_type = table.ty().ty; let table_default_value = get_default_table_value(table_type); let delta_result = table.grow(delta, table_default_value); @@ -82,16 +86,16 @@ pub extern "C" fn wasmer_table_grow(table: *mut wasmer_table_t, delta: u32) -> w /// Returns the current length of the given Table #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_table_length(table: *mut wasmer_table_t) -> u32 { - let table = unsafe { &*(table as *mut Table) }; +pub unsafe extern "C" fn wasmer_table_length(table: *mut wasmer_table_t) -> u32 { + let table = &*(table as *mut Table); table.size() } /// Frees memory for the given Table #[allow(clippy::cast_ptr_alignment)] #[no_mangle] -pub extern "C" fn wasmer_table_destroy(table: *mut wasmer_table_t) { - if !table.is_null() { - unsafe { Box::from_raw(table as *mut Table) }; +pub unsafe extern "C" fn wasmer_table_destroy(table: Option>) { + if let Some(table_inner) = table { + Box::from_raw(table_inner.cast::().as_ptr()); } } diff --git a/lib/c-api/tests/CMakeLists.txt b/lib/c-api/tests/CMakeLists.txt index 6393cbe1f..7891a8a65 100644 --- a/lib/c-api/tests/CMakeLists.txt +++ b/lib/c-api/tests/CMakeLists.txt @@ -111,8 +111,7 @@ add_test(test-instantiate test-instantiate) target_link_libraries(test-memory general ${WASMER_LIB}) target_compile_options(test-memory PRIVATE ${COMPILER_OPTIONS}) -# TODO: reenable this test -#add_test(test-memory test-memory) +add_test(test-memory test-memory) target_link_libraries(test-module general ${WASMER_LIB}) target_compile_options(test-module PRIVATE ${COMPILER_OPTIONS}) diff --git a/lib/c-api/tests/test-memory.c b/lib/c-api/tests/test-memory.c index a2aca65a8..25737561a 100644 --- a/lib/c-api/tests/test-memory.c +++ b/lib/c-api/tests/test-memory.c @@ -40,7 +40,7 @@ int main() char *error_str = malloc(error_len); wasmer_last_error_message(error_str, error_len); printf("Error str: `%s`\n", error_str); - assert(0 == strcmp(error_str, "Failed to add pages because would exceed maximum number of pages for the memory. Left: 22, Added: 15")); + assert(0 == strcmp(error_str, "The memory could not grow: current size 12 pages, requested increase: 10 pages")); free(error_str); wasmer_memory_t *bad_memory = NULL; @@ -58,7 +58,7 @@ int main() char *error_str2 = malloc(error_len2); wasmer_last_error_message(error_str2, error_len2); printf("Error str 2: `%s`\n", error_str2); - assert(0 == strcmp(error_str2, "Unable to create because the supplied descriptor is invalid: \"Max number of memory pages is less than the minimum number of pages\"")); + assert(0 == strcmp(error_str2, "The memory plan is invalid because the maximum (10 pages) is less than the minimum (15 pages)")); free(error_str2); printf("Destroy memory\n"); diff --git a/lib/cache/src/hash.rs b/lib/cache/src/hash.rs index a149b1ea0..664dc2740 100644 --- a/lib/cache/src/hash.rs +++ b/lib/cache/src/hash.rs @@ -22,7 +22,7 @@ impl WasmHash { /// is, in fact, a wasm module. pub fn generate(wasm: &[u8]) -> Self { let hash = blake3::hash(wasm); - WasmHash::new(hash.into()) + Self::new(hash.into()) } pub(crate) fn into_array(self) -> [u8; 32] { @@ -56,7 +56,7 @@ impl FromStr for WasmHash { )); } use std::convert::TryInto; - Ok(WasmHash(bytes[0..32].try_into().map_err(|e| { + Ok(Self(bytes[0..32].try_into().map_err(|e| { DeserializeError::Generic(format!("Could not get first 32 bytes: {}", e)) })?)) } diff --git a/lib/compiler-cranelift/build.rs b/lib/compiler-cranelift/build.rs index 1db75f1c1..5e03bc2e0 100644 --- a/lib/compiler-cranelift/build.rs +++ b/lib/compiler-cranelift/build.rs @@ -1,3 +1,8 @@ +//! Wasmer Cranelift compiler build script. +//! +//! Sets the git revsion? for $PURPOSE +//! TODO(syrus): explain what's happening here + use std::process::Command; use std::str; diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index 28f8b1acf..625161649 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -4,16 +4,17 @@ use crate::address_map::get_function_address_map; use crate::config::CraneliftConfig; use crate::func_environ::{get_func_name, FuncEnvironment}; use crate::sink::{RelocSink, TrapSink}; -use crate::trampoline::{make_wasm_trampoline, FunctionBuilderContext}; -use crate::translator::{ - compiled_function_unwind_info, irlibcall_to_libcall, irreloc_to_relocationkind, - signature_to_cranelift_ir, transform_jump_table, FuncTranslator, +use crate::trampoline::{ + make_trampoline_dynamic_function, make_trampoline_function_call, FunctionBuilderContext, }; -use cranelift_codegen::ir::{self, ExternalName}; +use crate::translator::{ + compiled_function_unwind_info, signature_to_cranelift_ir, transform_jump_table, FuncTranslator, +}; +use cranelift_codegen::ir; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::{binemit, isa, Context}; use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; -use wasm_common::entity::{EntityRef, PrimaryMap}; +use wasm_common::entity::PrimaryMap; use wasm_common::{ Features, FunctionIndex, FunctionType, LocalFunctionIndex, MemoryIndex, SignatureIndex, TableIndex, @@ -21,11 +22,9 @@ use wasm_common::{ use wasmer_compiler::CompileError; use wasmer_compiler::{ Compilation, CompiledFunction, CompiledFunctionFrameInfo, Compiler, FunctionBody, - FunctionBodyData, SourceLoc, TrapInformation, + FunctionBodyData, }; use wasmer_compiler::{CompilerConfig, ModuleTranslationState, Target}; -use wasmer_compiler::{Relocation, RelocationTarget}; -use wasmer_runtime::TrapCode; use wasmer_runtime::{MemoryPlan, Module, TablePlan}; /// A compiler that compiles a WebAssembly module with Cranelift, translating the Wasm to Cranelift IR, @@ -159,15 +158,37 @@ impl Compiler for CraneliftCompiler { Ok(Compilation::new(functions, custom_sections)) } - fn compile_wasm_trampolines( + fn compile_function_call_trampolines( &self, signatures: &[FunctionType], ) -> Result, CompileError> { signatures .par_iter() .map_init(FunctionBuilderContext::new, |mut cx, sig| { - make_wasm_trampoline(&*self.isa, &mut cx, sig, std::mem::size_of::()) + make_trampoline_function_call(&*self.isa, &mut cx, sig) }) .collect::, CompileError>>() } + + fn compile_dynamic_function_trampolines( + &self, + module: &Module, + ) -> Result, CompileError> { + use wasmer_runtime::VMOffsets; + let isa = self.isa(); + let frontend_config = isa.frontend_config(); + let offsets = VMOffsets::new(frontend_config.pointer_bytes(), module); + Ok(module + .functions + .values() + .take(module.num_imported_funcs) + .collect::>() + .par_iter() + .map_init(FunctionBuilderContext::new, |mut cx, sig_index| { + make_trampoline_dynamic_function(&*self.isa, &module, &offsets, &mut cx, &sig_index) + }) + .collect::, CompileError>>()? + .into_iter() + .collect::>()) + } } diff --git a/lib/compiler-cranelift/src/lib.rs b/lib/compiler-cranelift/src/lib.rs index 3147e0b88..87f35bafd 100644 --- a/lib/compiler-cranelift/src/lib.rs +++ b/lib/compiler-cranelift/src/lib.rs @@ -58,7 +58,7 @@ pub use crate::compiler::CraneliftCompiler; pub use crate::config::CraneliftConfig; pub use crate::debug::{FrameLayout, FrameLayoutChange, FrameLayouts}; pub use crate::debug::{ModuleMemoryOffset, ModuleVmctxInfo, ValueLabelsRanges}; -pub use crate::trampoline::make_wasm_trampoline; +pub use crate::trampoline::make_trampoline_function_call; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/lib/compiler-cranelift/src/sink.rs b/lib/compiler-cranelift/src/sink.rs index f0ba13412..9316851f1 100644 --- a/lib/compiler-cranelift/src/sink.rs +++ b/lib/compiler-cranelift/src/sink.rs @@ -1,9 +1,6 @@ //! Support for compiling with Cranelift. -use crate::translator::{ - irlibcall_to_libcall, irreloc_to_relocationkind, signature_to_cranelift_ir, - transform_jump_table, FuncTranslator, -}; +use crate::translator::{irlibcall_to_libcall, irreloc_to_relocationkind}; use cranelift_codegen::binemit; use cranelift_codegen::ir::{self, ExternalName}; use wasm_common::entity::EntityRef; @@ -136,7 +133,7 @@ fn translate_ir_trapcode(trap: ir::TrapCode) -> TrapCode { ir::TrapCode::BadConversionToInteger => TrapCode::BadConversionToInteger, ir::TrapCode::UnreachableCodeReached => TrapCode::UnreachableCodeReached, ir::TrapCode::Interrupt => TrapCode::Interrupt, - ir::TrapCode::User(user_code) => unimplemented!("User trap code not supported"), + ir::TrapCode::User(_user_code) => unimplemented!("User trap code not supported"), // ir::TrapCode::User(user_code) => TrapCode::User(user_code), } } diff --git a/lib/compiler-cranelift/src/trampoline/dynamic_function.rs b/lib/compiler-cranelift/src/trampoline/dynamic_function.rs new file mode 100644 index 000000000..6959c93df --- /dev/null +++ b/lib/compiler-cranelift/src/trampoline/dynamic_function.rs @@ -0,0 +1,146 @@ +//! A trampoline generator for calling dynamic host functions from Wasm. + +use super::binemit::TrampolineRelocSink; +use crate::translator::{compiled_function_unwind_info, signature_to_cranelift_ir}; +use cranelift_codegen::ir::{ + types, ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, +}; +use cranelift_codegen::isa::TargetIsa; +use cranelift_codegen::print_errors::pretty_error; +use cranelift_codegen::Context; +use cranelift_codegen::{binemit, ir}; +use std::cmp; +use std::mem; + +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use wasm_common::entity::EntityRef; +use wasm_common::SignatureIndex; +use wasmer_compiler::{CompileError, FunctionBody}; +use wasmer_runtime::{Module, VMOffsets}; + +/// Create a trampoline for invoking a WebAssembly function. +pub fn make_trampoline_dynamic_function( + isa: &dyn TargetIsa, + module: &Module, + offsets: &VMOffsets, + fn_builder_ctx: &mut FunctionBuilderContext, + sig_index: &SignatureIndex, +) -> Result { + let func_type = &module.signatures[*sig_index]; + let pointer_type = isa.pointer_type(); + let frontend_config = isa.frontend_config(); + let signature = signature_to_cranelift_ir(func_type, &frontend_config); + let mut stub_sig = ir::Signature::new(frontend_config.default_call_conv); + // Add the caller `vmctx` parameter. + stub_sig.params.push(ir::AbiParam::special( + pointer_type, + ir::ArgumentPurpose::VMContext, + )); + + // Add the caller/callee `vmctx` parameter. + stub_sig.params.push(ir::AbiParam::new(pointer_type)); + + // Add the `sig_index` parameter. + stub_sig.params.push(ir::AbiParam::new(types::I32)); + + // Add the `values_vec` parameter. + stub_sig.params.push(ir::AbiParam::new(pointer_type)); + + // Compute the size of the values vector. The vmctx and caller vmctx are passed separately. + let value_size = mem::size_of::(); + let values_vec_len = + (value_size * cmp::max(signature.params.len() - 2, signature.returns.len())) as u32; + + let mut context = Context::new(); + context.func = Function::with_name_signature(ExternalName::user(0, 0), signature.clone()); + + let ss = context.func.create_stack_slot(StackSlotData::new( + StackSlotKind::ExplicitSlot, + values_vec_len, + )); + + { + let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx); + let block0 = builder.create_block(); + + builder.append_block_params_for_function_params(block0); + builder.switch_to_block(block0); + builder.seal_block(block0); + + let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0); + let mflags = MemFlags::trusted(); + // We only get the non-vmctx arguments + for i in 2..signature.params.len() { + let val = builder.func.dfg.block_params(block0)[i]; + builder.ins().store( + mflags, + val, + values_vec_ptr_val, + ((i - 2) * value_size) as i32, + ); + } + + let block_params = builder.func.dfg.block_params(block0); + let vmctx_ptr_val = block_params[0]; + let caller_vmctx_ptr_val = block_params[1]; + + // Get the signature index + let caller_sig_id = builder.ins().iconst(types::I32, sig_index.index() as i64); + + let callee_args = vec![ + vmctx_ptr_val, + caller_vmctx_ptr_val, + caller_sig_id, + values_vec_ptr_val, + ]; + + let new_sig = builder.import_signature(stub_sig); + + let mem_flags = ir::MemFlags::trusted(); + let callee_value = builder.ins().load( + pointer_type, + mem_flags, + vmctx_ptr_val, + offsets.vmdynamicfunction_import_context_address() as i32, + ); + + builder + .ins() + .call_indirect(new_sig, callee_value, &callee_args); + + let mflags = MemFlags::trusted(); + let mut results = Vec::new(); + for (i, r) in signature.returns.iter().enumerate() { + let load = builder.ins().load( + r.value_type, + mflags, + values_vec_ptr_val, + (i * value_size) as i32, + ); + results.push(load); + } + builder.ins().return_(&results); + builder.finalize() + } + + let mut code_buf = Vec::new(); + let mut reloc_sink = TrampolineRelocSink {}; + let mut trap_sink = binemit::NullTrapSink {}; + let mut stackmap_sink = binemit::NullStackmapSink {}; + context + .compile_and_emit( + isa, + &mut code_buf, + &mut reloc_sink, + &mut trap_sink, + &mut stackmap_sink, + ) + .map_err(|error| CompileError::Codegen(pretty_error(&context.func, Some(isa), error)))?; + + let unwind_info = compiled_function_unwind_info(isa, &context); + + Ok(FunctionBody { + body: code_buf, + unwind_info, + }) +} diff --git a/lib/compiler-cranelift/src/trampoline/wasm.rs b/lib/compiler-cranelift/src/trampoline/function_call.rs similarity index 93% rename from lib/compiler-cranelift/src/trampoline/wasm.rs rename to lib/compiler-cranelift/src/trampoline/function_call.rs index 3f4368ab0..4c62a5833 100644 --- a/lib/compiler-cranelift/src/trampoline/wasm.rs +++ b/lib/compiler-cranelift/src/trampoline/function_call.rs @@ -7,7 +7,7 @@ //! ``` use super::binemit::TrampolineRelocSink; use crate::translator::{ - compiled_function_unwind_info, signature_to_cranelift_ir, transform_jump_table, + compiled_function_unwind_info, signature_to_cranelift_ir, /*transform_jump_table, */ }; use cranelift_codegen::ir::InstBuilder; use cranelift_codegen::isa::TargetIsa; @@ -15,15 +15,15 @@ use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::Context; use cranelift_codegen::{binemit, ir}; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use std::mem; use wasm_common::FunctionType; -use wasmer_compiler::{CompileError, CompiledFunction, CompiledFunctionFrameInfo, FunctionBody}; +use wasmer_compiler::{CompileError, FunctionBody}; /// Create a trampoline for invoking a WebAssembly function. -pub fn make_wasm_trampoline( +pub fn make_trampoline_function_call( isa: &dyn TargetIsa, fn_builder_ctx: &mut FunctionBuilderContext, func_type: &FunctionType, - value_size: usize, ) -> Result { let pointer_type = isa.pointer_type(); let frontend_config = isa.frontend_config(); @@ -49,6 +49,7 @@ pub fn make_wasm_trampoline( context.func = ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wrapper_sig); context.func.collect_frame_layout_info(); + let value_size = mem::size_of::(); { let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx); let block0 = builder.create_block(); @@ -86,7 +87,7 @@ pub fn make_wasm_trampoline( }) .collect::>(); - let new_sig = builder.import_signature(signature.clone()); + let new_sig = builder.import_signature(signature); let call = builder .ins() diff --git a/lib/compiler-cranelift/src/trampoline/mod.rs b/lib/compiler-cranelift/src/trampoline/mod.rs index 5658c5962..9fe8e5b5f 100644 --- a/lib/compiler-cranelift/src/trampoline/mod.rs +++ b/lib/compiler-cranelift/src/trampoline/mod.rs @@ -1,10 +1,10 @@ #![allow(missing_docs)] -// mod host; -mod wasm; +mod dynamic_function; +mod function_call; -// pub use host::make_host_trampoline; -pub use self::wasm::make_wasm_trampoline; +pub use self::dynamic_function::make_trampoline_dynamic_function; +pub use self::function_call::make_trampoline_function_call; // TODO: Delete pub mod ir { diff --git a/lib/compiler-cranelift/src/translator/func_environ.rs b/lib/compiler-cranelift/src/translator/func_environ.rs index 58dd59433..dc82a13fe 100644 --- a/lib/compiler-cranelift/src/translator/func_environ.rs +++ b/lib/compiler-cranelift/src/translator/func_environ.rs @@ -16,6 +16,7 @@ use wasmer_compiler::WasmResult; /// The value of a WebAssembly global variable. #[derive(Clone, Copy)] pub enum GlobalVariable { + #[allow(dead_code)] /// This is a constant global with a value known at compile time. Const(ir::Value), @@ -29,10 +30,12 @@ pub enum GlobalVariable { ty: ir::Type, }, + #[allow(dead_code)] /// This is a global variable that needs to be handled by the environment. Custom, } +#[allow(dead_code)] /// How to return from functions. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ReturnMode { diff --git a/lib/compiler-cranelift/src/translator/func_state.rs b/lib/compiler-cranelift/src/translator/func_state.rs index 2f17bac30..f02a64124 100644 --- a/lib/compiler-cranelift/src/translator/func_state.rs +++ b/lib/compiler-cranelift/src/translator/func_state.rs @@ -216,15 +216,6 @@ pub struct FuncTranslationState { functions: HashMap, } -// Public methods that are exposed to non-`wasmer_compiler` API consumers. -impl FuncTranslationState { - /// True if the current translation state expresses reachable code, false if it is unreachable. - #[inline] - pub fn reachable(&self) -> bool { - self.reachable - } -} - impl FuncTranslationState { /// Construct a new, empty, `FuncTranslationState` pub(crate) fn new() -> Self { diff --git a/lib/compiler-cranelift/src/translator/translation_utils.rs b/lib/compiler-cranelift/src/translator/translation_utils.rs index e8a56550a..d89955eea 100644 --- a/lib/compiler-cranelift/src/translator/translation_utils.rs +++ b/lib/compiler-cranelift/src/translator/translation_utils.rs @@ -21,13 +21,13 @@ pub fn signature_to_cranelift_ir( target_config: &TargetFrontendConfig, ) -> ir::Signature { let mut sig = ir::Signature::new(target_config.default_call_conv); - sig.params.extend(signature.params().iter().map(|ty| { - let cret_arg: ir::Type = type_to_irtype(ty.clone(), target_config) + sig.params.extend(signature.params().iter().map(|&ty| { + let cret_arg: ir::Type = type_to_irtype(ty, target_config) .expect("only numeric types are supported in function signatures"); AbiParam::new(cret_arg) })); - sig.returns.extend(signature.results().iter().map(|ty| { - let cret_arg: ir::Type = type_to_irtype(ty.clone(), target_config) + sig.returns.extend(signature.results().iter().map(|&ty| { + let cret_arg: ir::Type = type_to_irtype(ty, target_config) .expect("only numeric types are supported in function signatures"); AbiParam::new(cret_arg) })); diff --git a/lib/compiler-llvm/Cargo.toml b/lib/compiler-llvm/Cargo.toml index bb886713b..4e0b1bda6 100644 --- a/lib/compiler-llvm/Cargo.toml +++ b/lib/compiler-llvm/Cargo.toml @@ -16,7 +16,7 @@ wasm-common = { path = "../wasm-common", version = "0.16.2" } target-lexicon = { version = "0.10.0", default-features = false } smallvec = "1" goblin = "0.2" -libc = "0.2.69" +libc = { version = "0.2.70", default-features = false } byteorder = "1" itertools = "0.9.0" rayon = "1.3.0" @@ -24,14 +24,14 @@ rayon = "1.3.0" [target.'cfg(target_arch = "x86_64")'.dependencies.inkwell] #version = "0.1.0-llvm8sample" git = "https://github.com/theDan64/inkwell" -rev = "af1846fd8aea530cef9a59170715e5c6c64346f6" +rev = "1bfecc0a095d7ffdfa20a64630864f0297349508" default-features = false features = ["llvm10-0", "target-x86"] [target.'cfg(target_arch = "aarch64")'.dependencies.inkwell] #version = "0.1.0-llvm8sample" git = "https://github.com/theDan64/inkwell" -rev = "af1846fd8aea530cef9a59170715e5c6c64346f6" +rev = "1bfecc0a095d7ffdfa20a64630864f0297349508" default-features = false features = ["llvm10-0", "target-aarch64"] diff --git a/lib/compiler-llvm/build.rs b/lib/compiler-llvm/build.rs index 45268a39f..3805b00f7 100644 --- a/lib/compiler-llvm/build.rs +++ b/lib/compiler-llvm/build.rs @@ -11,7 +11,7 @@ use std::path::PathBuf; use std::process::Command; // Version of the llvm-sys crate that we (through inkwell) depend on. -const LLVM_SYS_MAJOR_VERSION: &str = "80"; +const LLVM_SYS_MAJOR_VERSION: &str = "100"; const LLVM_SYS_MINOR_VERSION: &str = "0"; // Environment variables that can guide compilation diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index d4a1880ef..b305fa3d5 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -72,6 +72,7 @@ impl Compiler for LLVMCompiler { let mut readonly_section = CustomSection { protection: CustomSectionProtection::Read, bytes: SectionBody::default(), + relocations: vec![], }; for (func_index, _) in &module.functions { @@ -81,6 +82,7 @@ impl Compiler for LLVMCompiler { .cloned() .unwrap_or_else(|| format!("fn{}", func_index.index())); } + let mut module_custom_sections = PrimaryMap::new(); let mut functions = function_body_inputs .into_iter() .collect::)>>() @@ -100,46 +102,41 @@ impl Compiler for LLVMCompiler { }) .collect::, CompileError>>()? .into_iter() - .map(|(mut function, local_relocations, custom_sections)| { - /// We collect the sections data - for (local_idx, custom_section) in custom_sections.iter().enumerate() { - let local_idx = local_idx as u32; - // TODO: these section numbers are potentially wrong, if there's - // no Read and only a ReadExecute then ReadExecute is 0. - let (ref mut section, section_num) = match &custom_section.protection { - CustomSectionProtection::Read => { - (&mut readonly_section, SectionIndex::from_u32(0)) - } - }; - let offset = section.bytes.len() as i64; - section.bytes.append(&custom_section.bytes); - // TODO: we're needlessly rescanning the whole list. - for local_relocation in &local_relocations { - if local_relocation.local_section_index == local_idx { - used_readonly_section = true; - function.relocations.push(Relocation { - kind: local_relocation.kind, - reloc_target: RelocationTarget::CustomSection(section_num), - offset: local_relocation.offset, - addend: local_relocation.addend + offset, - }); + .map(|(mut compiled_function, mut function_custom_sections)| { + let first_section = module_custom_sections.len() as u32; + for (_, custom_section) in function_custom_sections.iter() { + // TODO: remove this call to clone() + let mut custom_section = custom_section.clone(); + for mut reloc in &mut custom_section.relocations { + match reloc.reloc_target { + RelocationTarget::CustomSection(index) => { + reloc.reloc_target = RelocationTarget::CustomSection( + SectionIndex::from_u32(first_section + index.as_u32()), + ) + } + _ => {} } } + module_custom_sections.push(custom_section); } - Ok(function) + for mut reloc in &mut compiled_function.relocations { + match reloc.reloc_target { + RelocationTarget::CustomSection(index) => { + reloc.reloc_target = RelocationTarget::CustomSection( + SectionIndex::from_u32(first_section + index.as_u32()), + ) + } + _ => {} + } + } + compiled_function }) - .collect::, CompileError>>()? - .into_iter() .collect::>(); - let mut custom_sections = PrimaryMap::new(); - if used_readonly_section { - custom_sections.push(readonly_section); - } - Ok(Compilation::new(functions, custom_sections)) + Ok(Compilation::new(functions, module_custom_sections)) } - fn compile_wasm_trampolines( + fn compile_function_call_trampolines( &self, signatures: &[FunctionType], ) -> Result, CompileError> { @@ -150,4 +147,12 @@ impl Compiler for LLVMCompiler { }) .collect::, CompileError>>() } + + fn compile_dynamic_function_trampolines( + &self, + module: &Module, + ) -> Result, CompileError> { + Ok(PrimaryMap::new()) + // unimplemented!("Dynamic funciton trampolines not yet implemented"); + } } diff --git a/lib/compiler-llvm/src/config.rs b/lib/compiler-llvm/src/config.rs index 87a886527..581902aba 100644 --- a/lib/compiler-llvm/src/config.rs +++ b/lib/compiler-llvm/src/config.rs @@ -8,7 +8,7 @@ use inkwell::targets::{ use inkwell::OptimizationLevel; use itertools::Itertools; use target_lexicon::Architecture; -use wasmer_compiler::{Compiler, CompilerConfig, CpuFeature, Features, Target}; +use wasmer_compiler::{Compiler, CompilerConfig, CpuFeature, Features, Target, Triple}; /// The InkWell Module type pub type InkwellModule<'ctx> = inkwell::module::Module<'ctx>; @@ -46,7 +46,30 @@ pub struct LLVMConfig { impl LLVMConfig { /// Creates a new configuration object with the default configuration /// specified. - pub fn new(features: Features, target: Target) -> Self { + pub fn new(mut features: Features, target: Target) -> Self { + // Override the default multi-value switch + features.multi_value = false; + + let operating_system = + if target.triple().operating_system == wasmer_compiler::OperatingSystem::Darwin { + // LLVM detects static relocation + darwin + 64-bit and + // force-enables PIC because MachO doesn't support that + // combination. They don't check whether they're targeting + // MachO, they check whether the OS is set to Darwin. + // + // Since both linux and darwin use SysV ABI, this should work. + wasmer_compiler::OperatingSystem::Linux + } else { + target.triple().operating_system + }; + let triple = Triple { + architecture: target.triple().architecture, + vendor: target.triple().vendor.clone(), + operating_system, + environment: target.triple().environment, + binary_format: target_lexicon::BinaryFormat::Elf, + }; + let target = Target::new(triple, *target.cpu_features()); Self { enable_nan_canonicalization: true, enable_verifier: false, diff --git a/lib/compiler-llvm/src/trampoline/wasm.rs b/lib/compiler-llvm/src/trampoline/wasm.rs index 655e52aa4..28c3213be 100644 --- a/lib/compiler-llvm/src/trampoline/wasm.rs +++ b/lib/compiler-llvm/src/trampoline/wasm.rs @@ -145,8 +145,9 @@ fn generate_trampoline<'ctx>( Type::FuncRef => unimplemented!("funcref unimplemented in trampoline"), }; - let mut args_vec = Vec::with_capacity(func_sig.params().len() + 1); + let mut args_vec = Vec::with_capacity(func_sig.params().len() + 2); args_vec.push(callee_vmctx_ptr); + args_vec.push(caller_vmctx_ptr); let mut i = 0; for param_ty in func_sig.params().iter() { diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index ea4a0b17d..887f495eb 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -29,20 +29,22 @@ use inkwell::{ }; use smallvec::SmallVec; use std::any::Any; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::convert::TryFrom; +use std::num::TryFromIntError; use crate::config::LLVMConfig; -use wasm_common::entity::{EntityRef, PrimaryMap, SecondaryMap}; +use wasm_common::entity::{PrimaryMap, SecondaryMap}; use wasm_common::{ - FunctionIndex, FunctionType, GlobalIndex, LocalFunctionIndex, MemoryIndex, Mutability, - SignatureIndex, TableIndex, Type, + FunctionIndex, FunctionType, GlobalIndex, LocalFunctionIndex, MemoryIndex, MemoryType, + Mutability, SignatureIndex, TableIndex, Type, }; use wasmer_compiler::wasmparser::{self, BinaryReader, MemoryImmediate, Operator}; use wasmer_compiler::{ to_wasm_error, wasm_unsupported, Addend, CodeOffset, CompileError, CompiledFunction, - CompiledFunctionFrameInfo, CustomSection, CustomSectionProtection, FunctionAddressMap, - FunctionBody, FunctionBodyData, InstructionAddressMap, Relocation, RelocationKind, - RelocationTarget, SectionBody, SourceLoc, WasmResult, + CompiledFunctionFrameInfo, CustomSection, CustomSectionProtection, CustomSections, + FunctionAddressMap, FunctionBody, FunctionBodyData, InstructionAddressMap, Relocation, + RelocationKind, RelocationTarget, SectionBody, SectionIndex, SourceLoc, WasmResult, }; use wasmer_runtime::libcalls::LibCall; use wasmer_runtime::Module as WasmerCompilerModule; @@ -52,6 +54,30 @@ use wasmer_runtime::{MemoryPlan, MemoryStyle, TablePlan, VMBuiltinFunctionIndex, use std::fs; use std::io::Write; +use wasm_common::entity::entity_impl; +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +pub struct ElfSectionIndex(u32); +entity_impl!(ElfSectionIndex); +impl ElfSectionIndex { + pub fn is_undef(&self) -> bool { + self.as_u32() == goblin::elf::section_header::SHN_UNDEF + } + + pub fn from_usize(value: usize) -> Result { + match u32::try_from(value) { + Err(_) => Err(CompileError::Codegen(format!( + "elf section index {} does not fit in 32 bits", + value + ))), + Ok(value) => Ok(ElfSectionIndex::from_u32(value)), + } + } + + pub fn as_usize(&self) -> usize { + self.as_u32() as usize + } +} + // TODO fn wptype_to_type(ty: wasmparser::Type) -> WasmResult { match ty { @@ -84,15 +110,6 @@ fn const_zero<'ctx>(ty: BasicTypeEnum<'ctx>) -> BasicValueEnum<'ctx> { } } -// Relocation against a per-function section. -#[derive(Debug)] -pub struct LocalRelocation { - pub kind: RelocationKind, - pub local_section_index: u32, - pub offset: CodeOffset, - pub addend: Addend, -} - impl FuncTranslator { pub fn new() -> Self { Self { @@ -109,9 +126,9 @@ impl FuncTranslator { memory_plans: &PrimaryMap, table_plans: &PrimaryMap, func_names: &SecondaryMap, - ) -> Result<(CompiledFunction, Vec, Vec), CompileError> { + ) -> Result<(CompiledFunction, CustomSections), CompileError> { let func_index = wasm_module.func_index(*local_func_index); - let func_name = func_names.get(func_index).unwrap(); + let func_name = &func_names[func_index]; let module_name = match wasm_module.name.as_ref() { None => format!(" function {}", func_name), Some(module_name) => format!("module {} function {}", module_name, func_name), @@ -124,7 +141,7 @@ impl FuncTranslator { module.set_data_layout(&target_machine.get_target_data().get_data_layout()); let wasm_fn_type = wasm_module .signatures - .get(*wasm_module.functions.get(func_index).unwrap()) + .get(wasm_module.functions[func_index]) .unwrap(); let intrinsics = Intrinsics::declare(&module, &self.ctx); @@ -135,7 +152,7 @@ impl FuncTranslator { // TODO: figure out how many bytes long vmctx is, and mark it dereferenceable. (no need to mark it nonnull once we do this.) // TODO: mark vmctx nofree func.set_personality_function(intrinsics.personality); - func.as_global_value().set_section("wasmer_function"); + func.as_global_value().set_section(".wasmer_function"); let entry = self.ctx.append_basic_block(func, "entry"); let start_of_code = self.ctx.append_basic_block(func, "start_of_code"); @@ -165,7 +182,9 @@ impl FuncTranslator { for idx in 0..wasm_fn_type.params().len() { let ty = wasm_fn_type.params()[idx]; let ty = type_to_llvm(&intrinsics, ty); - let value = func.get_nth_param((idx + 1) as u32).unwrap(); + let value = func + .get_nth_param((idx as u32).checked_add(2).unwrap()) + .unwrap(); // TODO: don't interleave allocas and stores. let alloca = cache_builder.build_alloca(ty, "param"); cache_builder.build_store(alloca, value); @@ -207,6 +226,7 @@ impl FuncTranslator { // TODO: pointer width vmoffsets: VMOffsets::new(8, &wasm_module), wasm_module, + func_names, }; while fcg.state.has_control_frames() { @@ -303,12 +323,14 @@ impl FuncTranslator { .unwrap(); // TODO: remove debugging. + /* let mem_buf_slice = memory_buffer.as_slice(); let mut file = fs::File::create(format!("/home/nicholas/code{}.o", func_name)).unwrap(); let mut pos = 0; while pos < mem_buf_slice.len() { pos += file.write(&mem_buf_slice[pos..]).unwrap(); } + */ let mem_buf_slice = memory_buffer.as_slice(); let object = goblin::Object::parse(&mem_buf_slice).unwrap(); @@ -332,69 +354,118 @@ impl FuncTranslator { Some(name.unwrap()) }; - let wasmer_function_idx = elf + // Build up a mapping from a section to its relocation sections. + let reloc_sections = elf.shdr_relocs.iter().fold( + HashMap::new(), + |mut map: HashMap<_, Vec<_>>, (section_index, reloc_section)| { + let target_section = elf.section_headers[*section_index].sh_info as usize; + let target_section = ElfSectionIndex::from_usize(target_section).unwrap(); + map.entry(target_section).or_default().push(reloc_section); + map + }, + ); + + let mut visited: HashSet = HashSet::new(); + let mut worklist: Vec = Vec::new(); + let mut section_targets: HashMap = HashMap::new(); + + let wasmer_function_index = elf .section_headers .iter() .enumerate() - .filter(|(_, section)| get_section_name(section) == Some("wasmer_function")) - .map(|(idx, _)| idx) - .take(1) + .filter(|(_, section)| get_section_name(section) == Some(".wasmer_function")) + .map(|(index, _)| index) .collect::>(); - // TODO: handle errors here instead of asserting. - assert!(wasmer_function_idx.len() == 1); - let wasmer_function_idx = wasmer_function_idx[0]; + if wasmer_function_index.len() != 1 { + return Err(CompileError::Codegen(format!( + "found {} sections named .wasmer_function", + wasmer_function_index.len() + ))); + } + let wasmer_function_index = wasmer_function_index[0]; + let wasmer_function_index = ElfSectionIndex::from_usize(wasmer_function_index)?; - let bytes = elf.section_headers[wasmer_function_idx].file_range(); - let bytes = mem_buf_slice[bytes.start..bytes.end].to_vec(); + let mut section_to_custom_section = HashMap::new(); - let mut relocations = vec![]; - let mut local_relocations = vec![]; - let mut required_custom_sections = HashMap::new(); + section_targets.insert( + wasmer_function_index, + RelocationTarget::LocalFunc(*local_func_index), + ); - for (section_index, reloc_section) in &elf.shdr_relocs { - let section_name = get_section_name(&elf.section_headers[*section_index]); - if section_name != Some(".relawasmer_function") - && section_name != Some(".relwasmer_function") + let mut next_custom_section: u32 = 0; + let mut elf_section_to_target = |elf_section_index: ElfSectionIndex| { + *section_targets.entry(elf_section_index).or_insert_with(|| { + let next = SectionIndex::from_u32(next_custom_section); + section_to_custom_section.insert(elf_section_index, next); + let target = RelocationTarget::CustomSection(next); + next_custom_section += 1; + target + }) + }; + + let section_bytes = |elf_section_index: ElfSectionIndex| { + let elf_section_index = elf_section_index.as_usize(); + let byte_range = elf.section_headers[elf_section_index].file_range(); + mem_buf_slice[byte_range.start..byte_range.end].to_vec() + }; + + // From elf section index to list of Relocations. Although we use a Vec, + // the order of relocations is not important. + let mut relocations: HashMap> = HashMap::new(); + + // Each iteration of this loop pulls a section and the relocations + // relocations that apply to it. We begin with the ".wasmer_function" + // section, and then parse all relocation sections that apply to that + // section. Those relocations may refer to additional sections which we + // then add to the worklist until we've visited the closure of + // everything needed to run the code in ".wasmer_function". + // + // `worklist` is the list of sections we have yet to visit. It never + // contains any duplicates or sections we've already visited. `visited` + // contains all the sections we've ever added to the worklist in a set + // so that we can quickly check whether a section is new before adding + // it to worklist. `section_to_custom_section` is filled in with all + // the sections we want to include. + worklist.push(wasmer_function_index); + visited.insert(wasmer_function_index); + while let Some(section_index) = worklist.pop() { + for reloc in reloc_sections + .get(§ion_index) + .iter() + .flat_map(|inner| inner.iter().flat_map(|inner2| inner2.iter())) { - continue; - } - for reloc in reloc_section.iter() { let kind = match reloc.r_type { // TODO: these constants are not per-arch, we'll need to // make the whole match per-arch. goblin::elf::reloc::R_X86_64_64 => RelocationKind::Abs8, - _ => unimplemented!("unknown relocation {}", reloc.r_type), + _ => { + return Err(CompileError::Codegen(format!( + "unknown ELF relocation {}", + reloc.r_type + ))); + } }; let offset = reloc.r_offset as u32; let addend = reloc.r_addend.unwrap_or(0); let target = reloc.r_sym; // TODO: error handling - let target = elf.syms.get(target).unwrap(); - if target.st_type() == goblin::elf::sym::STT_SECTION { - let len = required_custom_sections.len(); - let entry = required_custom_sections.entry(target.st_shndx); - let local_section_index = *entry.or_insert(len) as _; - local_relocations.push(LocalRelocation { - kind, - local_section_index, - offset, - addend, - }); - } else if target.st_type() == goblin::elf::sym::STT_FUNC - && target.st_shndx == wasmer_function_idx + let elf_target = elf.syms.get(target).unwrap(); + let elf_target_section = ElfSectionIndex::from_usize(elf_target.st_shndx)?; + let reloc_target = if elf_target.st_type() == goblin::elf::sym::STT_SECTION { + if visited.insert(elf_target_section) { + worklist.push(elf_target_section); + } + elf_section_to_target(elf_target_section) + } else if elf_target.st_type() == goblin::elf::sym::STT_FUNC + && elf_target_section == wasmer_function_index { // This is a function referencing its own byte stream. - relocations.push(Relocation { - kind, - reloc_target: RelocationTarget::LocalFunc(*local_func_index), - offset, - addend, - }); - } else if target.st_type() == goblin::elf::sym::STT_NOTYPE - && target.st_shndx == goblin::elf::section_header::SHN_UNDEF as _ + RelocationTarget::LocalFunc(*local_func_index) + } else if elf_target.st_type() == goblin::elf::sym::STT_NOTYPE + && elf_target_section.is_undef() { // Not defined in this .o file. Maybe another local function? - let name = target.st_name; + let name = elf_target.st_name; let name = elf.strtab.get(name).unwrap().unwrap(); if let Some((index, _)) = func_names.iter().find(|(_, func_name)| *func_name == name) @@ -402,69 +473,78 @@ impl FuncTranslator { let local_index = wasm_module .local_func_index(index) .expect("Relocation to non-local function"); - relocations.push(Relocation { - kind, - reloc_target: RelocationTarget::LocalFunc(local_index), - offset, - addend, - }); + RelocationTarget::LocalFunc(local_index) // Maybe a libcall then? } else if let Some(libcall) = libcalls.get(name) { - relocations.push(Relocation { - kind, - reloc_target: RelocationTarget::LibCall(*libcall), - offset, - addend, - }); + RelocationTarget::LibCall(*libcall) } else { unimplemented!("reference to unknown symbol {}", name); } } else { unimplemented!("unknown relocation {:?} with target {:?}", reloc, target); - } + }; + relocations + .entry(section_index) + .or_default() + .push(Relocation { + kind, + reloc_target, + offset, + addend, + }); } } - let mut custom_sections = vec![]; - custom_sections.resize( - required_custom_sections.len(), - CustomSection { - protection: CustomSectionProtection::Read, - bytes: SectionBody::default(), - }, - ); - for (section_idx, local_section_idx) in required_custom_sections { - let bytes = elf.section_headers[section_idx as usize].file_range(); - let bytes = &mem_buf_slice[bytes.start..bytes.end]; - custom_sections[local_section_idx].bytes.extend(bytes); - } + let mut custom_sections = section_to_custom_section + .iter() + .map(|(elf_section_index, custom_section_index)| { + ( + custom_section_index, + CustomSection { + protection: CustomSectionProtection::Read, + bytes: SectionBody::new_with_vec(section_bytes(*elf_section_index)), + relocations: relocations + .remove_entry(elf_section_index) + .map_or(vec![], |(_, v)| v), + }, + ) + }) + .collect::>(); + custom_sections.sort_unstable_by_key(|a| a.0); + let custom_sections = custom_sections + .into_iter() + .map(|(_, v)| v) + .collect::>(); + + let function_body = FunctionBody { + body: section_bytes(wasmer_function_index), + unwind_info: None, + }; let address_map = FunctionAddressMap { instructions: vec![InstructionAddressMap { srcloc: SourceLoc::default(), code_offset: 0, - code_len: bytes.len(), + code_len: function_body.body.len(), }], start_srcloc: SourceLoc::default(), end_srcloc: SourceLoc::default(), body_offset: 0, - body_len: bytes.len(), + body_len: function_body.body.len(), }; Ok(( CompiledFunction { - body: FunctionBody { - body: bytes, - unwind_info: None, - }, + body: function_body, jt_offsets: SecondaryMap::new(), - relocations, + relocations: relocations + .remove_entry(&wasmer_function_index) + .map_or(vec![], |(_, v)| v), frame_info: CompiledFunctionFrameInfo { address_map, traps: vec![], }, }, - local_relocations, custom_sections, )) } @@ -741,11 +821,14 @@ fn trap_if_not_representable_as_int<'ctx>( builder.build_conditional_branch(out_of_bounds, failure_block, continue_block); builder.position_at_end(failure_block); - builder.build_call( - intrinsics.throw_trap, - &[intrinsics.trap_illegal_arithmetic], - "throw", + let is_nan = builder.build_float_compare(FloatPredicate::UNO, value, value, "is_nan"); + let trap_code = builder.build_select( + is_nan, + intrinsics.trap_bad_conversion_to_integer, + intrinsics.trap_illegal_arithmetic, + "", ); + builder.build_call(intrinsics.throw_trap, &[trap_code], "throw"); builder.build_unreachable(); builder.position_at_end(continue_block); } @@ -772,13 +855,14 @@ fn trap_if_zero_or_overflow<'ctx>( unreachable!() }; + let divisor_is_zero = builder.build_int_compare( + IntPredicate::EQ, + right, + int_type.const_int(0, false), + "divisor_is_zero", + ); let should_trap = builder.build_or( - builder.build_int_compare( - IntPredicate::EQ, - right, - int_type.const_int(0, false), - "divisor_is_zero", - ), + divisor_is_zero, builder.build_and( builder.build_int_compare(IntPredicate::EQ, left, min_value, "left_is_min"), builder.build_int_compare(IntPredicate::EQ, right, neg_one_value, "right_is_neg_one"), @@ -805,11 +889,13 @@ fn trap_if_zero_or_overflow<'ctx>( let should_trap_block = context.append_basic_block(*function, "should_trap_block"); builder.build_conditional_branch(should_trap, should_trap_block, shouldnt_trap_block); builder.position_at_end(should_trap_block); - builder.build_call( - intrinsics.throw_trap, - &[intrinsics.trap_illegal_arithmetic], - "throw", + let trap_code = builder.build_select( + divisor_is_zero, + intrinsics.trap_integer_division_by_zero, + intrinsics.trap_illegal_arithmetic, + "", ); + builder.build_call(intrinsics.throw_trap, &[trap_code], "throw"); builder.build_unreachable(); builder.position_at_end(shouldnt_trap_block); } @@ -849,7 +935,7 @@ fn trap_if_zero<'ctx>( builder.position_at_end(should_trap_block); builder.build_call( intrinsics.throw_trap, - &[intrinsics.trap_illegal_arithmetic], + &[intrinsics.trap_integer_division_by_zero], "throw", ); builder.build_unreachable(); @@ -1073,6 +1159,13 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let offset = self.vmoffsets.vmctx_vmmemory_import(memory_index); let offset = intrinsics.i32_ty.const_int(offset.into(), false); let memory_definition_ptr_ptr = unsafe { builder.build_gep(*vmctx, &[offset], "") }; + let memory_definition_ptr_ptr = builder + .build_bitcast( + memory_definition_ptr_ptr, + intrinsics.i8_ptr_ty.ptr_type(AddressSpace::Generic), + "", + ) + .into_pointer_value(); builder .build_load(memory_definition_ptr_ptr, "") .into_pointer_value() @@ -1102,71 +1195,28 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { // Compute the offset over the memory_base. let imm_offset = intrinsics.i64_ty.const_int(memarg.offset as u64, false); - //let var_offset_i32 = self.state.pop1()?.into_int_value(); let var_offset = builder.build_int_z_extend(var_offset, intrinsics.i64_ty, ""); let offset = builder.build_int_add(var_offset, imm_offset, ""); - // TODO: must bounds check here or before this point (if applicable) let value_ptr = unsafe { builder.build_gep(base, &[offset], "") }; - Ok(builder - .build_bitcast(value_ptr, ptr_ty, "") - .into_pointer_value()) - /* - let memory_cache = ctx.memory(MemoryIndex::from_u32(0), intrinsics, module, &memory_plans); - let (mem_base, mem_bound, minimum, _maximum) = match memory_cache { - MemoryCache::Dynamic { - ptr_to_base_ptr, - ptr_to_bounds, - minimum, - maximum, - } => { - let base = builder - .build_load(ptr_to_base_ptr, "base") - .into_pointer_value(); - let bounds = builder.build_load(ptr_to_bounds, "bounds").into_int_value(); - tbaa_label( - &module, - intrinsics, - "dynamic_memory_base", - base.as_instruction_value().unwrap(), - Some(0), - ); - tbaa_label( - &module, - intrinsics, - "dynamic_memory_bounds", - bounds.as_instruction_value().unwrap(), - Some(0), - ); - (base, bounds, minimum, maximum) - } - MemoryCache::Static { - base_ptr, - bounds, - minimum, - maximum, - } => (base_ptr, bounds, minimum, maximum), - }; - let mem_base = builder - .build_bitcast(mem_base, intrinsics.i8_ptr_ty, &state.var_name()) - .into_pointer_value(); - - // Compute the offset over the memory_base. - let imm_offset = intrinsics.i64_ty.const_int(memarg.offset as u64, false); - let var_offset_i32 = state.pop1()?.into_int_value(); - let var_offset = - builder.build_int_z_extend(var_offset_i32, intrinsics.i64_ty, &state.var_name()); - let effective_offset = builder.build_int_add(var_offset, imm_offset, &state.var_name()); - - if let MemoryCache::Dynamic { .. } = memory_cache { + match memory_plans[memory_index] { + MemoryPlan { + style: MemoryStyle::Dynamic, + offset_guard_size: _, + memory: + MemoryType { + minimum, + maximum, + shared: _, + }, + } => { // If the memory is dynamic, do a bounds check. For static we rely on // the size being a multiple of the page size and hitting a guard page. let value_size_v = intrinsics.i64_ty.const_int(value_size as u64, false); - let ptr_in_bounds = if effective_offset.is_const() { - let load_offset_end = effective_offset.const_add(value_size_v); + let ptr_in_bounds = if offset.is_const() { + let load_offset_end = offset.const_add(value_size_v); let ptr_in_bounds = load_offset_end.const_int_compare( IntPredicate::ULE, - // TODO: Pages to bytes conversion here intrinsics.i64_ty.const_int(minimum.bytes().0 as u64, false), ); if ptr_in_bounds.get_zero_extended_constant() == Some(1) { @@ -1178,14 +1228,16 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { None } .unwrap_or_else(|| { - let load_offset_end = - builder.build_int_add(effective_offset, value_size_v, &state.var_name()); + let load_offset_end = builder.build_int_add(offset, value_size_v, ""); + + let current_length = + builder.build_load(current_length_ptr, "").into_int_value(); builder.build_int_compare( IntPredicate::ULE, load_offset_end, - mem_bound, - &state.var_name(), + current_length, + "", ) }); if !ptr_in_bounds.is_constant_int() @@ -1212,7 +1264,8 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let in_bounds_continue_block = context.append_basic_block(*function, "in_bounds_continue_block"); - let not_in_bounds_block = context.append_basic_block(*function, "not_in_bounds_block"); + let not_in_bounds_block = + context.append_basic_block(*function, "not_in_bounds_block"); builder.build_conditional_branch( ptr_in_bounds, in_bounds_continue_block, @@ -1228,12 +1281,19 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { builder.position_at_end(in_bounds_continue_block); } } + MemoryPlan { + style: MemoryStyle::Static { bound: _ }, + offset_guard_size: _, + memory: _, + } => { + // No bounds checks of any kind! Out of bounds memory accesses + // will hit the guard pages. + } + }; - let ptr = unsafe { builder.build_gep(mem_base, &[effective_offset], &state.var_name()) }; - Ok(builder - .build_bitcast(ptr, ptr_ty, &state.var_name()) - .into_pointer_value()) - */ + Ok(builder + .build_bitcast(value_ptr, ptr_ty, "") + .into_pointer_value()) } } @@ -1440,6 +1500,7 @@ pub struct LLVMFunctionCodeGenerator<'ctx, 'a> { module: &'a Module<'ctx>, vmoffsets: VMOffsets, wasm_module: &'a WasmerCompilerModule, + func_names: &'a SecondaryMap, } impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { @@ -2166,93 +2227,34 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { Operator::GlobalGet { global_index } => { let global_index = GlobalIndex::from_u32(global_index); - let global_type = module.globals[global_index]; - let global_value_type = global_type.ty; - - // TODO: cache loads of const globals. - let _global_mutability = global_type.mutability; - - let global_ptr = - if let Some(local_global_index) = module.local_global_index(global_index) { - let offset = self.vmoffsets.vmctx_vmglobal_definition(local_global_index); - let offset = intrinsics.i32_ty.const_int(offset.into(), false); - unsafe { builder.build_gep(*vmctx, &[offset], "") } - } else { - let offset = self.vmoffsets.vmctx_vmglobal_import(global_index); - let offset = intrinsics.i32_ty.const_int(offset.into(), false); - let global_ptr_ptr = unsafe { builder.build_gep(*vmctx, &[offset], "") }; - let global_ptr_ptr = builder - .build_bitcast(global_ptr_ptr, intrinsics.i8_ptr_ty, "") - .into_pointer_value(); - builder.build_load(global_ptr_ptr, "").into_pointer_value() - }; - let global_ptr = builder - .build_bitcast( - global_ptr, - type_to_llvm_ptr(&intrinsics, global_value_type), - "", - ) - .into_pointer_value(); - let value = builder.build_load(global_ptr, ""); - // TODO: add TBAA info. - self.state.push1(value); + match ctx.global(global_index, intrinsics) { + GlobalCache::Const { value } => { + self.state.push1(value); + } + GlobalCache::Mut { ptr_to_value } => { + let value = builder.build_load(ptr_to_value, ""); + // TODO: tbaa + self.state.push1(value); + } + } } Operator::GlobalSet { global_index } => { let global_index = GlobalIndex::from_u32(global_index); - let global_type = module.globals[global_index]; - let global_value_type = global_type.ty; - - // Note that we don't check mutability, assuming that's already - // been checked by some other verifier. - - let global_ptr = - if let Some(local_global_index) = module.local_global_index(global_index) { - let offset = self.vmoffsets.vmctx_vmglobal_definition(local_global_index); - let offset = intrinsics.i32_ty.const_int(offset.into(), false); - unsafe { builder.build_gep(*vmctx, &[offset], "") } - } else { - let offset = self.vmoffsets.vmctx_vmglobal_import(global_index); - let offset = intrinsics.i32_ty.const_int(offset.into(), false); - let global_ptr_ptr = unsafe { builder.build_gep(*vmctx, &[offset], "") }; - let global_ptr_ptr = builder - .build_bitcast(global_ptr_ptr, intrinsics.i8_ptr_ty, "") - .into_pointer_value(); - builder.build_load(global_ptr_ptr, "").into_pointer_value() - }; - let global_ptr = builder - .build_bitcast( - global_ptr, - type_to_llvm_ptr(&intrinsics, global_value_type), - "", - ) - .into_pointer_value(); - - let (value, info) = self.state.pop1_extra()?; - let value = apply_pending_canonicalization(builder, intrinsics, value, info); - builder.build_store(global_ptr, value); - // TODO: add TBAA info - - /* - let (value, info) = self.state.pop1_extra()?; - let value = apply_pending_canonicalization(builder, intrinsics, value, info); - let index = GlobalIndex::from_u32(global_index); - let global_cache = ctx.global_cache(index, intrinsics, self.module); - match global_cache { - GlobalCache::Mut { ptr_to_value } => { - let store = builder.build_store(ptr_to_value, value); - tbaa_label( - &self.module, - intrinsics, - "global", - store, - Some(global_index), - ); - } - GlobalCache::Const { value: _ } => { - return Err(CompileError::Codegen("global is immutable".to_string())); - } - } - */ + match ctx.global(global_index, intrinsics) { + GlobalCache::Const { value } => { + return Err(CompileError::Codegen(format!( + "global.set on immutable global index {}", + global_index.as_u32() + ))) + } + GlobalCache::Mut { ptr_to_value } => { + let (value, info) = self.state.pop1_extra()?; + let value = + apply_pending_canonicalization(builder, intrinsics, value, info); + builder.build_store(ptr_to_value, value); + // TODO: tbaa + } + } } Operator::Select => { @@ -2299,22 +2301,60 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { } Operator::Call { function_index } => { let func_index = FunctionIndex::from_u32(function_index); - let sigindex = module.functions.get(func_index).unwrap(); - let func_type = module.signatures.get(*sigindex).unwrap(); - let func_name = module.func_names.get(&func_index).unwrap(); + let sigindex = &module.functions[func_index]; + let func_type = &module.signatures[*sigindex]; + let func_name = &self.func_names[func_index]; let llvm_func_type = func_type_to_llvm(&self.context, &intrinsics, func_type); - let func = self.module.get_function(func_name); - // TODO: we could do this by comparing function indices instead - // of going through LLVM APIs and string comparisons. - let func = if func.is_none() { - self.module - .add_function(func_name, llvm_func_type, Some(Linkage::External)) + let (func, callee_vmctx) = if let Some(local_func_index) = + module.local_func_index(func_index) + { + // TODO: we could do this by comparing function indices instead + // of going through LLVM APIs and string comparisons. + let func = self.module.get_function(func_name); + let func = if func.is_none() { + self.module + .add_function(func_name, llvm_func_type, Some(Linkage::External)) + } else { + func.unwrap() + }; + (func.as_global_value().as_pointer_value(), ctx.basic()) } else { - func.unwrap() + let offset = self.vmoffsets.vmctx_vmfunction_import(func_index); + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + let vmfunction_import_ptr = unsafe { builder.build_gep(*vmctx, &[offset], "") }; + let vmfunction_import_ptr = builder + .build_bitcast( + vmfunction_import_ptr, + intrinsics.vmfunction_import_ptr_ty, + "", + ) + .into_pointer_value(); + + let body_ptr_ptr = builder + .build_struct_gep( + vmfunction_import_ptr, + intrinsics.vmfunction_import_body_element, + "", + ) + .unwrap(); + let body_ptr = builder.build_load(body_ptr_ptr, ""); + let body_ptr = builder + .build_bitcast(body_ptr, llvm_func_type.ptr_type(AddressSpace::Generic), "") + .into_pointer_value(); + let vmctx_ptr_ptr = builder + .build_struct_gep( + vmfunction_import_ptr, + intrinsics.vmfunction_import_vmctx_element, + "", + ) + .unwrap(); + let vmctx_ptr = builder.build_load(vmctx_ptr_ptr, ""); + (body_ptr, vmctx_ptr) }; - let params: Vec<_> = std::iter::once(ctx.basic()) + let params: Vec<_> = std::iter::repeat(callee_vmctx) + .take(2) .chain( self.state .peekn_extra(func_type.params().len())? @@ -2399,7 +2439,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { } Operator::CallIndirect { index, table_index } => { let sigindex = SignatureIndex::from_u32(index); - let func_type = module.signatures.get(sigindex).unwrap(); + let func_type = &module.signatures[sigindex]; let expected_dynamic_sigindex = ctx.dynamic_sigindex(sigindex, intrinsics); let (table_base, table_bound) = ctx.table( TableIndex::from_u32(table_index), @@ -2425,7 +2465,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { }; // Load things from the anyfunc data structure. - let (func_ptr, ctx_ptr, found_dynamic_sigindex) = unsafe { + let (func_ptr, found_dynamic_sigindex, ctx_ptr) = unsafe { ( builder .build_load( @@ -2435,20 +2475,20 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { "func_ptr", ) .into_pointer_value(), - builder.build_load( - builder - .build_struct_gep(anyfunc_struct_ptr, 1, "ctx_ptr_ptr") - .unwrap(), - "ctx_ptr", - ), builder .build_load( builder - .build_struct_gep(anyfunc_struct_ptr, 2, "sigindex_ptr") + .build_struct_gep(anyfunc_struct_ptr, 1, "sigindex_ptr") .unwrap(), "sigindex", ) .into_int_value(), + builder.build_load( + builder + .build_struct_gep(anyfunc_struct_ptr, 2, "ctx_ptr_ptr") + .unwrap(), + "ctx_ptr", + ), ) }; @@ -2492,12 +2532,16 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { builder.position_at_end(not_in_bounds_block); builder.build_call( intrinsics.throw_trap, - &[intrinsics.trap_call_indirect_oob], + &[intrinsics.trap_table_access_oob], "throw", ); builder.build_unreachable(); builder.position_at_end(in_bounds_continue_block); + // Next, check if the table element is initialized. + + let elem_initialized = builder.build_is_not_null(func_ptr, ""); + // Next, check if the signature id is correct. let sigindices_equal = builder.build_int_compare( @@ -2507,15 +2551,18 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { "sigindices_equal", ); + let initialized_and_sigindices_match = + builder.build_and(elem_initialized, sigindices_equal, ""); + // Tell llvm that `expected_dynamic_sigindex` should equal `found_dynamic_sigindex`. - let sigindices_equal = builder + let initialized_and_sigindices_match = builder .build_call( intrinsics.expect_i1, &[ - sigindices_equal.as_basic_value_enum(), + initialized_and_sigindices_match.as_basic_value_enum(), intrinsics.i1_ty.const_int(1, false).as_basic_value_enum(), ], - "sigindices_equal_expect", + "initialized_and_sigindices_match_expect", ) .try_as_basic_value() .left() @@ -2526,17 +2573,19 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let sigindices_notequal_block = context.append_basic_block(function, "sigindices_notequal_block"); builder.build_conditional_branch( - sigindices_equal, + initialized_and_sigindices_match, continue_block, sigindices_notequal_block, ); builder.position_at_end(sigindices_notequal_block); - builder.build_call( - intrinsics.throw_trap, - &[intrinsics.trap_call_indirect_sig], - "throw", + let trap_code = builder.build_select( + elem_initialized, + intrinsics.trap_call_indirect_sig, + intrinsics.trap_call_indirect_null, + "", ); + builder.build_call(intrinsics.throw_trap, &[trap_code], "throw"); builder.build_unreachable(); builder.position_at_end(continue_block); @@ -2544,7 +2593,8 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let pushed_args = self.state.popn_save_extra(func_type.params().len())?; - let args: Vec<_> = std::iter::once(ctx_ptr) + let args: Vec<_> = std::iter::repeat(ctx_ptr) + .take(2) .chain(pushed_args.into_iter().enumerate().map(|(i, (v, info))| { match func_type.params()[i] { Type::F32 => builder.build_bitcast( diff --git a/lib/compiler-llvm/src/translator/intrinsics.rs b/lib/compiler-llvm/src/translator/intrinsics.rs index 8aebd3b29..bbe95e0cc 100644 --- a/lib/compiler-llvm/src/translator/intrinsics.rs +++ b/lib/compiler-llvm/src/translator/intrinsics.rs @@ -143,11 +143,14 @@ pub struct Intrinsics<'ctx> { pub f64x2_zero: VectorValue<'ctx>, pub trap_unreachable: BasicValueEnum<'ctx>, + pub trap_call_indirect_null: BasicValueEnum<'ctx>, pub trap_call_indirect_sig: BasicValueEnum<'ctx>, - pub trap_call_indirect_oob: BasicValueEnum<'ctx>, pub trap_memory_oob: BasicValueEnum<'ctx>, pub trap_illegal_arithmetic: BasicValueEnum<'ctx>, + pub trap_integer_division_by_zero: BasicValueEnum<'ctx>, + pub trap_bad_conversion_to_integer: BasicValueEnum<'ctx>, pub trap_misaligned_atomic: BasicValueEnum<'ctx>, + pub trap_table_access_oob: BasicValueEnum<'ctx>, // VM intrinsics. pub memory_grow_dynamic_local: FunctionValue<'ctx>, @@ -169,6 +172,10 @@ pub struct Intrinsics<'ctx> { pub experimental_stackmap: FunctionValue<'ctx>, + pub vmfunction_import_ptr_ty: PointerType<'ctx>, + pub vmfunction_import_body_element: u32, + pub vmfunction_import_vmctx_element: u32, + pub vmmemory_definition_ptr_ty: PointerType<'ctx>, pub vmmemory_definition_base_element: u32, pub vmmemory_definition_current_length_element: u32, @@ -258,8 +265,8 @@ impl<'ctx> Intrinsics<'ctx> { let anyfunc_ty = context.struct_type( &[ i8_ptr_ty_basic, - ctx_ptr_ty.as_basic_type_enum(), sigindex_ty.as_basic_type_enum(), + ctx_ptr_ty.as_basic_type_enum(), ], false, ); @@ -475,12 +482,12 @@ impl<'ctx> Intrinsics<'ctx> { trap_unreachable: i32_ty .const_int(TrapCode::UnreachableCodeReached as _, false) .as_basic_value_enum(), + trap_call_indirect_null: i32_ty + .const_int(TrapCode::IndirectCallToNull as _, false) + .as_basic_value_enum(), trap_call_indirect_sig: i32_ty .const_int(TrapCode::BadSignature as _, false) .as_basic_value_enum(), - trap_call_indirect_oob: i32_ty - .const_int(TrapCode::OutOfBounds as _, false) - .as_basic_value_enum(), trap_memory_oob: i32_ty .const_int(TrapCode::OutOfBounds as _, false) .as_basic_value_enum(), @@ -488,10 +495,19 @@ impl<'ctx> Intrinsics<'ctx> { trap_illegal_arithmetic: i32_ty .const_int(TrapCode::IntegerOverflow as _, false) .as_basic_value_enum(), + trap_integer_division_by_zero: i32_ty + .const_int(TrapCode::IntegerDivisionByZero as _, false) + .as_basic_value_enum(), + trap_bad_conversion_to_integer: i32_ty + .const_int(TrapCode::BadConversionToInteger as _, false) + .as_basic_value_enum(), // TODO: add misaligned atomic traps to wasmer runtime trap_misaligned_atomic: i32_ty .const_int(TrapCode::Interrupt as _, false) .as_basic_value_enum(), + trap_table_access_oob: i32_ty + .const_int(TrapCode::TableAccessOutOfBounds as _, false) + .as_basic_value_enum(), // VM intrinsics. memory_grow_dynamic_local: module.add_function( @@ -577,6 +593,12 @@ impl<'ctx> Intrinsics<'ctx> { None, ), + vmfunction_import_ptr_ty: context + .struct_type(&[i8_ptr_ty_basic, i8_ptr_ty_basic], false) + .ptr_type(AddressSpace::Generic), + vmfunction_import_body_element: 0, + vmfunction_import_vmctx_element: 1, + // TODO: this i64 is actually a rust usize vmmemory_definition_ptr_ty: context .struct_type(&[i8_ptr_ty_basic, i64_ptr_ty_basic], false) @@ -888,9 +910,9 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { }) } - pub fn table_prepare( + fn table_prepare( &mut self, - index: TableIndex, + table_index: TableIndex, intrinsics: &Intrinsics<'ctx>, module: &Module<'ctx>, ) -> (PointerValue<'ctx>, PointerValue<'ctx>) { @@ -904,74 +926,77 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { let TableCache { ptr_to_base_ptr, ptr_to_bounds, - } = *cached_tables.entry(index).or_insert_with(|| { - let (table_array_ptr_ptr, index, field_name) = - if let Some(local_table_index) = wasm_module.local_table_index(index) { - ( - unsafe { - cache_builder - .build_struct_gep( - ctx_ptr_value, - offset_to_index(offsets.vmctx_tables_begin()), - "table_array_ptr_ptr", - ) - .unwrap() - }, - local_table_index.index() as u64, - "context_field_ptr_to_local_table", - ) + } = *cached_tables.entry(table_index).or_insert_with(|| { + let (ptr_to_base_ptr, ptr_to_bounds) = + if let Some(local_table_index) = wasm_module.local_table_index(table_index) { + let offset = intrinsics.i64_ty.const_int( + offsets + .vmctx_vmtable_definition_base(local_table_index) + .into(), + false, + ); + let ptr_to_base_ptr = + unsafe { cache_builder.build_gep(ctx_ptr_value, &[offset], "") }; + let ptr_to_base_ptr = cache_builder + .build_bitcast( + ptr_to_base_ptr, + intrinsics.i8_ptr_ty.ptr_type(AddressSpace::Generic), + "", + ) + .into_pointer_value(); + let offset = intrinsics.i64_ty.const_int( + offsets + .vmctx_vmtable_definition_current_elements(local_table_index) + .into(), + false, + ); + let ptr_to_bounds = + unsafe { cache_builder.build_gep(ctx_ptr_value, &[offset], "") }; + let ptr_to_bounds = cache_builder + .build_bitcast(ptr_to_bounds, intrinsics.i32_ptr_ty, "") + .into_pointer_value(); + (ptr_to_base_ptr, ptr_to_bounds) } else { - ( - unsafe { - cache_builder - .build_struct_gep( - ctx_ptr_value, - offset_to_index(offsets.vmctx_imported_tables_begin()), - "table_array_ptr_ptr", - ) - .unwrap() - }, - index.index() as u64, - "context_field_ptr_to_import_table", - ) + let offset = intrinsics.i64_ty.const_int( + offsets.vmctx_vmtable_import_definition(table_index).into(), + false, + ); + let definition_ptr_ptr = + unsafe { cache_builder.build_gep(ctx_ptr_value, &[offset], "") }; + let definition_ptr_ptr = cache_builder + .build_bitcast( + definition_ptr_ptr, + intrinsics.i8_ptr_ty.ptr_type(AddressSpace::Generic), + "", + ) + .into_pointer_value(); + let definition_ptr = cache_builder + .build_load(definition_ptr_ptr, "") + .into_pointer_value(); + // TODO: TBAA label + + let offset = intrinsics + .i64_ty + .const_int(offsets.vmtable_definition_base().into(), false); + let ptr_to_base_ptr = + unsafe { cache_builder.build_gep(definition_ptr, &[offset], "") }; + let ptr_to_base_ptr = cache_builder + .build_bitcast( + ptr_to_base_ptr, + intrinsics.i8_ptr_ty.ptr_type(AddressSpace::Generic), + "", + ) + .into_pointer_value(); + let offset = intrinsics + .i64_ty + .const_int(offsets.vmtable_definition_current_elements().into(), false); + let ptr_to_bounds = + unsafe { cache_builder.build_gep(definition_ptr, &[offset], "") }; + let ptr_to_bounds = cache_builder + .build_bitcast(ptr_to_bounds, intrinsics.i32_ptr_ty, "") + .into_pointer_value(); + (ptr_to_base_ptr, ptr_to_bounds) }; - - let table_array_ptr = cache_builder - .build_load(table_array_ptr_ptr, "table_array_ptr") - .into_pointer_value(); - tbaa_label( - module, - intrinsics, - field_name, - table_array_ptr.as_instruction_value().unwrap(), - None, - ); - let const_index = intrinsics.i32_ty.const_int(index, false); - let table_ptr_ptr = unsafe { - cache_builder.build_in_bounds_gep(table_array_ptr, &[const_index], "table_ptr_ptr") - }; - let table_ptr = cache_builder - .build_load(table_ptr_ptr, "table_ptr") - .into_pointer_value(); - tbaa_label( - module, - intrinsics, - "table_ptr", - table_array_ptr.as_instruction_value().unwrap(), - Some(index as u32), - ); - - let (ptr_to_base_ptr, ptr_to_bounds) = unsafe { - ( - cache_builder - .build_struct_gep(table_ptr, 0, "base_ptr") - .unwrap(), - cache_builder - .build_struct_gep(table_ptr, 1, "bounds_ptr") - .unwrap(), - ) - }; - TableCache { ptr_to_base_ptr, ptr_to_bounds, @@ -989,10 +1014,14 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { builder: &Builder<'ctx>, ) -> (PointerValue<'ctx>, IntValue<'ctx>) { let (ptr_to_base_ptr, ptr_to_bounds) = self.table_prepare(index, intrinsics, module); - let base_ptr = builder + let base_ptr = self + .cache_builder .build_load(ptr_to_base_ptr, "base_ptr") .into_pointer_value(); - let bounds = builder.build_load(ptr_to_bounds, "bounds").into_int_value(); + let bounds = self + .cache_builder + .build_load(ptr_to_bounds, "bounds") + .into_int_value(); tbaa_label( module, intrinsics, @@ -1059,106 +1088,59 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { }) } - pub fn global_cache( + pub fn global( &mut self, index: GlobalIndex, intrinsics: &Intrinsics<'ctx>, - module: &Module<'ctx>, ) -> GlobalCache<'ctx> { - let (cached_globals, ctx_ptr_value, wasm_module, cache_builder, offsets) = ( + let (cached_globals, wasm_module, ctx_ptr_value, cache_builder, offsets) = ( &mut self.cached_globals, - self.ctx_ptr_value, self.wasm_module, + self.ctx_ptr_value, &self.cache_builder, &self.offsets, ); *cached_globals.entry(index).or_insert_with(|| { - let (globals_array_ptr_ptr, index, mutable, wasmer_ty, field_name) = { - let desc = wasm_module.globals.get(index).unwrap(); - if let Some(_local_global_index) = wasm_module.local_global_index(index) { - ( - unsafe { - cache_builder - .build_struct_gep( - ctx_ptr_value, - offset_to_index(offsets.vmctx_globals_begin()), - "globals_array_ptr_ptr", - ) - .unwrap() - }, - index.index() as u64, - desc.mutability, - desc.ty, - "context_field_ptr_to_local_globals", - ) - } else { - ( - unsafe { - cache_builder - .build_struct_gep( - ctx_ptr_value, - offset_to_index(offsets.vmctx_imported_globals_begin()), - "globals_array_ptr_ptr", - ) - .unwrap() - }, - index.index() as u64, - desc.mutability, - desc.ty, - "context_field_ptr_to_imported_globals", - ) - } - }; + let global_type = wasm_module.globals[index]; + let global_value_type = global_type.ty; - let llvm_ptr_ty = type_to_llvm_ptr(intrinsics, wasmer_ty); - - let global_array_ptr = cache_builder - .build_load(globals_array_ptr_ptr, "global_array_ptr") - .into_pointer_value(); - tbaa_label( - module, - intrinsics, - field_name, - global_array_ptr.as_instruction_value().unwrap(), - None, - ); - let const_index = intrinsics.i32_ty.const_int(index, false); - let global_ptr_ptr = unsafe { - cache_builder.build_in_bounds_gep( - global_array_ptr, - &[const_index], - "global_ptr_ptr", - ) + let global_mutability = global_type.mutability; + let global_ptr = if let Some(local_global_index) = wasm_module.local_global_index(index) + { + let offset = offsets.vmctx_vmglobal_definition(local_global_index); + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + unsafe { cache_builder.build_gep(ctx_ptr_value, &[offset], "") } + } else { + let offset = offsets.vmctx_vmglobal_import(index); + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + let global_ptr_ptr = + unsafe { cache_builder.build_gep(ctx_ptr_value, &[offset], "") }; + let global_ptr_ptr = cache_builder + .build_bitcast( + global_ptr_ptr, + intrinsics.i32_ptr_ty.ptr_type(AddressSpace::Generic), + "", + ) + .into_pointer_value(); + cache_builder + .build_load(global_ptr_ptr, "") + .into_pointer_value() }; let global_ptr = cache_builder - .build_load(global_ptr_ptr, "global_ptr") + .build_bitcast( + global_ptr, + type_to_llvm_ptr(&intrinsics, global_value_type), + "", + ) .into_pointer_value(); - tbaa_label( - module, - intrinsics, - "global_ptr", - global_ptr.as_instruction_value().unwrap(), - Some(index as u32), - ); - let global_ptr_typed = - cache_builder.build_pointer_cast(global_ptr, llvm_ptr_ty, "global_ptr_typed"); - - let mutable = mutable == Mutability::Var; - if mutable { - GlobalCache::Mut { - ptr_to_value: global_ptr_typed, - } - } else { - let value = cache_builder.build_load(global_ptr_typed, "global_value"); - tbaa_label( - module, - intrinsics, - "global", - value.as_instruction_value().unwrap(), - Some(index as u32), - ); - GlobalCache::Const { value } + match global_mutability { + Mutability::Const => GlobalCache::Const { + value: cache_builder.build_load(global_ptr, ""), + }, + Mutability::Var => GlobalCache::Mut { + ptr_to_value: global_ptr, + }, } }) } @@ -1345,7 +1327,8 @@ pub fn func_type_to_llvm<'ctx>( .params() .iter() .map(|&ty| type_to_llvm(intrinsics, ty)); - let param_types: Vec<_> = std::iter::once(intrinsics.ctx_ptr_ty.as_basic_type_enum()) + let param_types: Vec<_> = std::iter::repeat(intrinsics.ctx_ptr_ty.as_basic_type_enum()) + .take(2) .chain(user_param_types) .collect(); diff --git a/lib/compiler-singlepass/Cargo.toml b/lib/compiler-singlepass/Cargo.toml index b04f114cd..8cc641eb9 100644 --- a/lib/compiler-singlepass/Cargo.toml +++ b/lib/compiler-singlepass/Cargo.toml @@ -19,6 +19,11 @@ wasm-common = { path = "../wasm-common", version = "0.16.2", default-features = rayon = "1.3.0" serde = { version = "1.0.106", features = ["derive"] } more-asserts = "0.2.1" +dynasm = "0.5" +dynasmrt = "0.5" +lazy_static = "1.4" +byteorder = "1.3" +smallvec = "1" [badges] maintenance = { status = "actively-developed" } diff --git a/lib/compiler-singlepass/src/codegen_x64.rs b/lib/compiler-singlepass/src/codegen_x64.rs new file mode 100644 index 000000000..e51db5b7c --- /dev/null +++ b/lib/compiler-singlepass/src/codegen_x64.rs @@ -0,0 +1,10931 @@ +#![allow(clippy::forget_copy)] // Used by dynasm. +#![warn(unused_imports)] + +use crate::emitter_x64::*; +use crate::machine::*; +#[cfg(target_arch = "aarch64")] +use dynasmrt::aarch64::Assembler; +#[cfg(target_arch = "x86_64")] +use dynasmrt::x64::Assembler; +use dynasmrt::{AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi}; +use smallvec::SmallVec; +use std::{ + any::Any, + collections::{BTreeMap, HashMap}, + ffi::c_void, + iter, mem, + ptr::NonNull, + slice, + sync::{Arc, RwLock}, + usize, +}; +use wasmer_runtime_core::{ + backend::{ + sys::{Memory, Protect}, + Architecture, CacheGen, CompilerConfig, ExceptionCode, ExceptionTable, InlineBreakpoint, + InlineBreakpointType, MemoryBoundCheckMode, RunnableModule, Token, + }, + cache::{Artifact, Error as CacheError}, + codegen::*, + fault::{self, raw::register_preservation_trampoline}, + loader::CodeMemory, + memory::MemoryType, + module::{ModuleInfo, ModuleInner}, + state::{ + x64::new_machine_state, x64::X64Register, x64_decl::ArgumentRegisterAllocator, + FunctionStateMap, MachineState, MachineValue, ModuleStateMap, OffsetInfo, SuspendOffset, + WasmAbstractValue, + }, + structures::{Map, TypedIndex}, + typed_func::{Trampoline, Wasm}, + types::{ + FuncIndex, FuncSig, GlobalIndex, ImportedGlobalIndex, LocalFuncIndex, LocalGlobalIndex, + LocalOrImport, MemoryIndex, SigIndex, TableIndex, Type, + }, + vm::{self, LocalGlobal, LocalTable, INTERNALS_SIZE}, + wasmparser::{MemoryImmediate, Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType}, +}; + +#[cfg(target_arch = "aarch64")] +#[allow(dead_code)] +static ARCH: Architecture = Architecture::Aarch64; +#[cfg(target_arch = "x86_64")] +#[allow(dead_code)] +static ARCH: Architecture = Architecture::X64; + +/// Inline breakpoint size for x86-64. +pub const INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS: usize = 7; + +/// Inline breakpoint size for aarch64. +pub const INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS: usize = 12; + +static BACKEND_ID: &str = "singlepass"; + +#[cfg(target_arch = "x86_64")] +lazy_static! { + /// Performs a System V call to `target` with [stack_top..stack_base] as the argument list, from right to left. + static ref CONSTRUCT_STACK_AND_CALL_WASM: unsafe extern "C" fn (stack_top: *const u64, stack_base: *const u64, ctx: *mut vm::Ctx, target: *const vm::Func) -> u64 = { + let mut assembler = Assembler::new().unwrap(); + let offset = assembler.offset(); + dynasm!( + assembler + ; push r15 + ; push r14 + ; push r13 + ; push r12 + ; push r11 + ; push rbp + ; mov rbp, rsp + + ; mov r15, rdi + ; mov r14, rsi + ; mov r13, rdx + ; mov r12, rcx + + ; mov rdi, r13 // ctx + + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov rsi, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov rdx, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov rcx, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov r8, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov r9, [r14] + ; sub r14, 8 + ; cmp r14, r15 + ; jb >stack_ready + + ; mov rax, r14 + ; sub rax, r15 + ; sub rsp, rax + ; sub rsp, 8 + ; mov rax, QWORD 0xfffffffffffffff0u64 as i64 + ; and rsp, rax + ; mov rax, rsp + ; loop_begin: + ; mov r11, [r14] + ; mov [rax], r11 + ; sub r14, 8 + ; add rax, 8 + ; cmp r14, r15 + ; jb >stack_ready + ; jmp u64, userdata: *mut u8) -> u64 = { + let mut assembler = Assembler::new().unwrap(); + let offset = assembler.offset(); + dynasm!( + assembler + ; .arch aarch64 + ; sub x0, x0, 16 + ; mov x8, sp + ; str x8, [x0, 0] + ; str x30, [x0, 8] + ; adr x30, >done + ; mov sp, x0 + ; mov x0, x2 + ; br x1 + ; done: + ; ldr x30, [sp, 8] + ; ldr x8, [sp, 0] + ; mov sp, x8 + ; br x30 + ); + let buf = assembler.finalize().unwrap(); + let ret = unsafe { mem::transmute(buf.ptr(offset)) }; + mem::forget(buf); + ret + }; +} + +pub struct X64ModuleCodeGenerator { + functions: Vec, + signatures: Option>>, + function_signatures: Option>>, + function_labels: Option)>>, + assembler: Option, + func_import_count: usize, + + config: Option>, +} + +pub struct X64FunctionCode { + local_function_id: usize, + + signatures: Arc>, + function_signatures: Arc>, + signature: FuncSig, + fsm: FunctionStateMap, + offset: usize, + + assembler: Option, + function_labels: Option)>>, + breakpoints: Option< + HashMap< + AssemblyOffset, + Box Result<(), Box> + Send + Sync + 'static>, + >, + >, + returns: SmallVec<[WpType; 1]>, + locals: Vec, + num_params: usize, + local_types: Vec, + value_stack: Vec, + + /// Metadata about floating point values on the stack. + fp_stack: Vec, + + control_stack: Vec, + machine: Machine, + unreachable_depth: usize, + + config: Arc, + + exception_table: Option, +} + +/// Metadata about a floating-point value. +#[derive(Copy, Clone, Debug)] +struct FloatValue { + /// Do we need to canonicalize the value before its bit pattern is next observed? If so, how? + canonicalization: Option, + + /// Corresponding depth in the main value stack. + depth: usize, +} + +impl FloatValue { + fn new(depth: usize) -> Self { + FloatValue { + canonicalization: None, + depth, + } + } + + fn cncl_f32(depth: usize) -> Self { + FloatValue { + canonicalization: Some(CanonicalizeType::F32), + depth, + } + } + + fn cncl_f64(depth: usize) -> Self { + FloatValue { + canonicalization: Some(CanonicalizeType::F64), + depth, + } + } + + fn promote(self, depth: usize) -> FloatValue { + FloatValue { + canonicalization: match self.canonicalization { + Some(CanonicalizeType::F32) => Some(CanonicalizeType::F64), + Some(CanonicalizeType::F64) => panic!("cannot promote F64"), + None => None, + }, + depth, + } + } + + fn demote(self, depth: usize) -> FloatValue { + FloatValue { + canonicalization: match self.canonicalization { + Some(CanonicalizeType::F64) => Some(CanonicalizeType::F32), + Some(CanonicalizeType::F32) => panic!("cannot demote F32"), + None => None, + }, + depth, + } + } +} + +/// Type of a pending canonicalization floating point value. +/// Sometimes we don't have the type information elsewhere and therefore we need to track it here. +#[derive(Copy, Clone, Debug)] +enum CanonicalizeType { + F32, + F64, +} + +impl CanonicalizeType { + fn to_size(&self) -> Size { + match self { + CanonicalizeType::F32 => Size::S32, + CanonicalizeType::F64 => Size::S64, + } + } +} + +trait PopMany { + fn peek1(&self) -> Result<&T, CodegenError>; + fn pop1(&mut self) -> Result; + fn pop2(&mut self) -> Result<(T, T), CodegenError>; +} + +impl PopMany for Vec { + fn peek1(&self) -> Result<&T, CodegenError> { + match self.last() { + Some(x) => Ok(x), + None => Err(CodegenError { + message: "peek1() expects at least 1 element".into(), + }), + } + } + fn pop1(&mut self) -> Result { + match self.pop() { + Some(x) => Ok(x), + None => Err(CodegenError { + message: "pop1() expects at least 1 element".into(), + }), + } + } + fn pop2(&mut self) -> Result<(T, T), CodegenError> { + if self.len() < 2 { + return Err(CodegenError { + message: "pop2() expects at least 2 elements".into(), + }); + } + + let right = self.pop().unwrap(); + let left = self.pop().unwrap(); + Ok((left, right)) + } +} + +trait WpTypeExt { + fn is_float(&self) -> bool; +} + +impl WpTypeExt for WpType { + fn is_float(&self) -> bool { + match self { + WpType::F32 | WpType::F64 => true, + _ => false, + } + } +} + +enum FuncPtrInner {} +#[repr(transparent)] +#[derive(Copy, Clone, Debug)] +struct FuncPtr(*const FuncPtrInner); +unsafe impl Send for FuncPtr {} +unsafe impl Sync for FuncPtr {} + +pub struct X64ExecutionContext { + #[allow(dead_code)] + code: CodeMemory, + function_pointers: Vec, + function_offsets: Vec, + signatures: Arc>, + breakpoints: BreakpointMap, + func_import_count: usize, + msm: ModuleStateMap, + exception_table: ExceptionTable, +} + +/// On-disk cache format. +/// Offsets are relative to the start of the executable image. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CacheImage { + /// The executable image. + code: Vec, + + /// Offsets to the start of each function. Including trampoline, if any. + /// Trampolines are only present on AArch64. + /// On x86-64, `function_pointers` are identical to `function_offsets`. + function_pointers: Vec, + + /// Offsets to the start of each function after trampoline. + function_offsets: Vec, + + /// Number of imported functions. + func_import_count: usize, + + /// Module state map. + msm: ModuleStateMap, + + /// An exception table that maps instruction offsets to exception codes. + exception_table: ExceptionTable, +} + +#[derive(Debug)] +pub struct ControlFrame { + pub label: DynamicLabel, + pub loop_like: bool, + pub if_else: IfElseState, + pub returns: SmallVec<[WpType; 1]>, + pub value_stack_depth: usize, + pub fp_stack_depth: usize, + pub state: MachineState, + pub state_diff_id: usize, +} + +#[derive(Debug, Copy, Clone)] +pub enum IfElseState { + None, + If(DynamicLabel), + Else, +} + +pub struct SinglepassCache { + buffer: Arc<[u8]>, +} + +impl CacheGen for SinglepassCache { + fn generate_cache(&self) -> Result<(Box<[u8]>, Memory), CacheError> { + let mut memory = Memory::with_size_protect(self.buffer.len(), Protect::ReadWrite) + .map_err(CacheError::SerializeError)?; + + let buffer = &*self.buffer; + + unsafe { + memory.as_slice_mut()[..buffer.len()].copy_from_slice(buffer); + } + + Ok(([].as_ref().into(), memory)) + } +} + +impl RunnableModule for X64ExecutionContext { + fn get_func( + &self, + _: &ModuleInfo, + local_func_index: LocalFuncIndex, + ) -> Option> { + self.function_pointers[self.func_import_count..] + .get(local_func_index.index()) + .and_then(|ptr| NonNull::new(ptr.0 as *mut vm::Func)) + } + + fn get_module_state_map(&self) -> Option { + Some(self.msm.clone()) + } + + fn get_breakpoints(&self) -> Option { + Some(self.breakpoints.clone()) + } + + fn get_exception_table(&self) -> Option<&ExceptionTable> { + Some(&self.exception_table) + } + + unsafe fn patch_local_function(&self, idx: usize, target_address: usize) -> bool { + /* + 0: 48 b8 42 42 42 42 42 42 42 42 movabsq $4774451407313060418, %rax + a: 49 bb 43 43 43 43 43 43 43 43 movabsq $4846791580151137091, %r11 + 14: 41 ff e3 jmpq *%r11 + */ + #[repr(packed)] + struct LocalTrampoline { + movabsq_rax: [u8; 2], + addr_rax: u64, + movabsq_r11: [u8; 2], + addr_r11: u64, + jmpq_r11: [u8; 3], + } + + self.code.make_writable(); + + let trampoline = &mut *(self.function_pointers[self.func_import_count + idx].0 + as *const LocalTrampoline as *mut LocalTrampoline); + trampoline.movabsq_rax[0] = 0x48; + trampoline.movabsq_rax[1] = 0xb8; + trampoline.addr_rax = target_address as u64; + trampoline.movabsq_r11[0] = 0x49; + trampoline.movabsq_r11[1] = 0xbb; + trampoline.addr_r11 = + register_preservation_trampoline as unsafe extern "C" fn() as usize as u64; + trampoline.jmpq_r11[0] = 0x41; + trampoline.jmpq_r11[1] = 0xff; + trampoline.jmpq_r11[2] = 0xe3; + + self.code.make_executable(); + true + } + + fn get_trampoline(&self, _: &ModuleInfo, sig_index: SigIndex) -> Option { + // Correctly unwinding from `catch_unsafe_unwind` on hardware exceptions depends + // on the signal handlers being installed. Here we call `ensure_sighandler` "statically" + // outside `invoke()`. + fault::ensure_sighandler(); + + unsafe extern "C" fn invoke( + _trampoline: Trampoline, + ctx: *mut vm::Ctx, + func: NonNull, + args: *const u64, + rets: *mut u64, + error_out: *mut Option>, + num_params_plus_one: Option>, + ) -> bool { + let rm: &Box = &(&*(*ctx).module).runnable_module; + + let args = + slice::from_raw_parts(args, num_params_plus_one.unwrap().as_ptr() as usize - 1); + + let ret = match fault::catch_unsafe_unwind( + || { + // Puts the arguments onto the stack and calls Wasm entry. + #[cfg(target_arch = "x86_64")] + { + let args_reverse: SmallVec<[u64; 8]> = args.iter().cloned().rev().collect(); + CONSTRUCT_STACK_AND_CALL_WASM( + args_reverse.as_ptr(), + args_reverse.as_ptr().offset(args_reverse.len() as isize), + ctx, + func.as_ptr(), + ) + } + + // FIXME: Currently we are doing a hack here to convert between native aarch64 and + // "emulated" x86 ABIs. Ideally, this should be done using handwritten assembly. + #[cfg(target_arch = "aarch64")] + { + struct CallCtx<'a> { + args: &'a [u64], + ctx: *mut vm::Ctx, + callable: NonNull, + } + extern "C" fn call_fn(f: *mut u8) -> u64 { + unsafe { + let f = &*(f as *const CallCtx); + let callable: extern "C" fn( + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + u64, + ) + -> u64 = std::mem::transmute(f.callable); + let mut args = f.args.iter(); + callable( + f.ctx as u64, + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + args.next().cloned().unwrap_or(0), + ) + } + } + let mut cctx = CallCtx { + args: &args, + ctx: ctx, + callable: func, + }; + use libc::{ + mmap, munmap, MAP_ANON, MAP_NORESERVE, MAP_PRIVATE, PROT_READ, + PROT_WRITE, + }; + const STACK_SIZE: usize = 1048576 * 1024; // 1GB of virtual address space for stack. + let stack_ptr = mmap( + ::std::ptr::null_mut(), + STACK_SIZE, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, + -1, + 0, + ); + if stack_ptr as isize == -1 { + panic!("unable to allocate stack"); + } + // TODO: Mark specific regions in the stack as PROT_NONE. + let ret = SWITCH_STACK( + (stack_ptr as *mut u8).offset(STACK_SIZE as isize) as *mut u64, + call_fn, + &mut cctx as *mut CallCtx as *mut u8, + ); + munmap(stack_ptr, STACK_SIZE); + ret + } + }, + rm.get_breakpoints(), + ) { + Ok(x) => { + if !rets.is_null() { + *rets = x; + } + true + } + Err(err) => { + *error_out = Some(err); + false + } + }; + ret + } + + unsafe extern "C" fn dummy_trampoline( + _: *mut vm::Ctx, + _: NonNull, + _: *const u64, + _: *mut u64, + ) { + unreachable!() + } + + Some(unsafe { + Wasm::from_raw_parts( + dummy_trampoline, + invoke, + NonNull::new((self.signatures.get(sig_index).unwrap().params().len() + 1) as _), // +1 to keep it non-zero + ) + }) + } + + unsafe fn do_early_trap(&self, data: Box) -> ! { + fault::begin_unsafe_unwind(data); + } + + fn get_code(&self) -> Option<&[u8]> { + Some(&self.code) + } + + fn get_offsets(&self) -> Option> { + Some(self.function_offsets.iter().map(|x| x.0).collect()) + } + + fn get_local_function_offsets(&self) -> Option> { + Some( + self.function_offsets[self.func_import_count..] + .iter() + .map(|x| x.0) + .collect(), + ) + } + + /// Returns the inline breakpoint size corresponding to an Architecture. + fn get_inline_breakpoint_size(&self, arch: Architecture) -> Option { + match arch { + Architecture::X64 => Some(INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS), + Architecture::Aarch64 => Some(INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS), + } + } + + /// Attempts to read an inline breakpoint from the code. + /// + /// Inline breakpoints are detected by special instruction sequences that never + /// appear in valid code. + fn read_inline_breakpoint(&self, arch: Architecture, code: &[u8]) -> Option { + match arch { + Architecture::X64 => { + if code.len() < INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS { + None + } else if &code[..INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS - 1] + == &[ + 0x0f, 0x0b, // ud2 + 0x0f, 0xb9, // ud + 0xcd, 0xff, // int 0xff + ] + { + Some(InlineBreakpoint { + size: INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS, + ty: match code[INLINE_BREAKPOINT_SIZE_X86_SINGLEPASS - 1] { + 0 => InlineBreakpointType::Middleware, + _ => return None, + }, + }) + } else { + None + } + } + Architecture::Aarch64 => { + if code.len() < INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS { + None + } else if &code[..INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS - 4] + == &[ + 0, 0, 0, 0, // udf #0 + 0xff, 0xff, 0x00, 0x00, // udf #65535 + ] + { + Some(InlineBreakpoint { + size: INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS, + ty: match code[INLINE_BREAKPOINT_SIZE_AARCH64_SINGLEPASS - 4] { + 0 => InlineBreakpointType::Middleware, + _ => return None, + }, + }) + } else { + None + } + } + } + } +} + +#[derive(Debug)] +pub struct CodegenError { + pub message: String, +} + +#[derive(Copy, Clone, Debug)] +struct CodegenConfig { + memory_bound_check_mode: MemoryBoundCheckMode, + enforce_stack_check: bool, + track_state: bool, + full_preemption: bool, + nan_canonicalization: bool, +} + +impl ModuleCodeGenerator + for X64ModuleCodeGenerator +{ + fn new() -> X64ModuleCodeGenerator { + let a = Assembler::new().unwrap(); + + X64ModuleCodeGenerator { + functions: vec![], + signatures: None, + function_signatures: None, + function_labels: Some(HashMap::new()), + assembler: Some(a), + func_import_count: 0, + config: None, + } + } + + /// Singlepass does validation as it compiles + fn requires_pre_validation() -> bool { + false + } + + fn backend_id() -> &'static str { + BACKEND_ID + } + + fn new_with_target(_: Option, _: Option, _: Option) -> Self { + unimplemented!("cross compilation is not available for singlepass backend") + } + + fn check_precondition(&mut self, _module_info: &ModuleInfo) -> Result<(), CodegenError> { + Ok(()) + } + + fn next_function( + &mut self, + _module_info: Arc>, + _loc: WasmSpan, + ) -> Result<&mut X64FunctionCode, CodegenError> { + let (mut assembler, mut function_labels, breakpoints, exception_table) = + match self.functions.last_mut() { + Some(x) => ( + x.assembler.take().unwrap(), + x.function_labels.take().unwrap(), + x.breakpoints.take().unwrap(), + x.exception_table.take().unwrap(), + ), + None => ( + self.assembler.take().unwrap(), + self.function_labels.take().unwrap(), + HashMap::new(), + ExceptionTable::new(), + ), + }; + + let begin_offset = assembler.offset(); + let begin_label_info = function_labels + .entry(self.functions.len() + self.func_import_count) + .or_insert_with(|| (assembler.new_dynamic_label(), None)); + + begin_label_info.1 = Some(begin_offset); + assembler.arch_emit_entry_trampoline(); + let begin_label = begin_label_info.0; + let mut machine = Machine::new(); + machine.track_state = self.config.as_ref().unwrap().track_state; + + assembler.emit_label(begin_label); + + let signatures = self.signatures.as_ref().unwrap(); + let function_signatures = self.function_signatures.as_ref().unwrap(); + let sig_index = function_signatures + .get(FuncIndex::new( + self.functions.len() + self.func_import_count, + )) + .unwrap() + .clone(); + let sig = signatures.get(sig_index).unwrap().clone(); + let code = X64FunctionCode { + local_function_id: self.functions.len(), + + signatures: signatures.clone(), + function_signatures: function_signatures.clone(), + signature: sig, + fsm: FunctionStateMap::new(new_machine_state(), self.functions.len(), 32, vec![]), // only a placeholder; this is initialized later in `begin_body` + offset: begin_offset.0, + + assembler: Some(assembler), + function_labels: Some(function_labels), + breakpoints: Some(breakpoints), + returns: smallvec![], + locals: vec![], + local_types: vec![], + num_params: 0, + value_stack: vec![], + fp_stack: vec![], + control_stack: vec![], + machine, + unreachable_depth: 0, + config: self.config.as_ref().unwrap().clone(), + exception_table: Some(exception_table), + }; + self.functions.push(code); + Ok(self.functions.last_mut().unwrap()) + } + + fn finalize( + mut self, + _: &ModuleInfo, + ) -> Result< + ( + X64ExecutionContext, + Option, + Box, + ), + CodegenError, + > { + let (assembler, function_labels, breakpoints, exception_table) = + match self.functions.last_mut() { + Some(x) => ( + x.assembler.take().unwrap(), + x.function_labels.take().unwrap(), + x.breakpoints.take().unwrap(), + x.exception_table.take().unwrap(), + ), + None => ( + self.assembler.take().unwrap(), + self.function_labels.take().unwrap(), + HashMap::new(), + ExceptionTable::new(), + ), + }; + + let total_size = assembler.get_offset().0; + let _output = assembler.finalize().unwrap(); + let mut output = CodeMemory::new(_output.len()); + output[0.._output.len()].copy_from_slice(&_output); + output.make_executable(); + + let mut out_labels: Vec = vec![]; + let mut out_offsets: Vec = vec![]; + + for i in 0..function_labels.len() { + let (_, offset) = match function_labels.get(&i) { + Some(x) => x, + None => { + return Err(CodegenError { + message: format!("label not found"), + }); + } + }; + let offset = match offset { + Some(x) => x, + None => { + return Err(CodegenError { + message: format!("offset is none"), + }); + } + }; + out_labels.push(FuncPtr( + unsafe { output.as_ptr().offset(offset.0 as isize) } as _, + )); + out_offsets.push(*offset); + } + + let breakpoints: Arc> = Arc::new( + breakpoints + .into_iter() + .map(|(offset, f)| { + ( + unsafe { output.as_ptr().offset(offset.0 as isize) } as usize, + f, + ) + }) + .collect(), + ); + + let local_function_maps: BTreeMap = self + .functions + .iter() + .map(|x| (x.offset, x.fsm.clone())) + .collect(); + + let msm = ModuleStateMap { + local_functions: local_function_maps, + total_size, + }; + + let cache_image = CacheImage { + code: output.to_vec(), + function_pointers: out_labels + .iter() + .map(|x| { + (x.0 as usize) + .checked_sub(output.as_ptr() as usize) + .unwrap() + }) + .collect(), + function_offsets: out_offsets.iter().map(|x| x.0 as usize).collect(), + func_import_count: self.func_import_count, + msm: msm.clone(), + exception_table: exception_table.clone(), + }; + + let cache = SinglepassCache { + buffer: Arc::from(bincode::serialize(&cache_image).unwrap().into_boxed_slice()), + }; + + Ok(( + X64ExecutionContext { + code: output, + signatures: self.signatures.as_ref().unwrap().clone(), + breakpoints: breakpoints, + func_import_count: self.func_import_count, + function_pointers: out_labels, + function_offsets: out_offsets, + msm: msm, + exception_table, + }, + None, + Box::new(cache), + )) + } + + fn feed_signatures(&mut self, signatures: Map) -> Result<(), CodegenError> { + self.signatures = Some(Arc::new(signatures)); + Ok(()) + } + + fn feed_function_signatures( + &mut self, + assoc: Map, + ) -> Result<(), CodegenError> { + self.function_signatures = Some(Arc::new(assoc)); + Ok(()) + } + + fn feed_import_function(&mut self, sigindex: SigIndex) -> Result<(), CodegenError> { + let labels = self.function_labels.as_mut().unwrap(); + let id = labels.len(); + + let a = self.assembler.as_mut().unwrap(); + let offset = a.offset(); + a.arch_emit_entry_trampoline(); + let label = a.get_label(); + a.emit_label(label); + labels.insert(id, (label, Some(offset))); + + // Singlepass internally treats all arguments as integers, but the standard System V calling convention requires + // floating point arguments to be passed in XMM registers. + // + // FIXME: This is only a workaround. We should fix singlepass to use the standard CC. + let sig = self + .signatures + .as_ref() + .expect("signatures itself") + .get(sigindex) + .expect("signatures"); + // Translation is expensive, so only do it if needed. + if sig + .params() + .iter() + .find(|&&x| x == Type::F32 || x == Type::F64) + .is_some() + { + let mut param_locations: Vec = vec![]; + + // Allocate stack space for arguments. + let stack_offset: i32 = if sig.params().len() > 5 { + 5 * 8 + } else { + (sig.params().len() as i32) * 8 + }; + if stack_offset > 0 { + a.emit_sub( + Size::S64, + Location::Imm32(stack_offset as u32), + Location::GPR(GPR::RSP), + ); + } + + // Store all arguments to the stack to prevent overwrite. + for i in 0..sig.params().len() { + let loc = match i { + 0..=4 => { + static PARAM_REGS: &'static [GPR] = + &[GPR::RSI, GPR::RDX, GPR::RCX, GPR::R8, GPR::R9]; + let loc = Location::Memory(GPR::RSP, (i * 8) as i32); + a.emit_mov(Size::S64, Location::GPR(PARAM_REGS[i]), loc); + loc + } + _ => Location::Memory(GPR::RSP, stack_offset + 8 + ((i - 5) * 8) as i32), + }; + param_locations.push(loc); + } + + // Copy arguments. + let mut argalloc = ArgumentRegisterAllocator::default(); + argalloc.next(Type::I32).unwrap(); // skip vm::Ctx + let mut caller_stack_offset: i32 = 0; + for (i, ty) in sig.params().iter().enumerate() { + let prev_loc = param_locations[i]; + let target = match argalloc.next(*ty) { + Some(X64Register::GPR(gpr)) => Location::GPR(gpr), + Some(X64Register::XMM(xmm)) => Location::XMM(xmm), + None => { + // No register can be allocated. Put this argument on the stack. + // + // Since here we never use fewer registers than by the original call, on the caller's frame + // we always have enough space to store the rearranged arguments, and the copy "backward" between different + // slots in the caller argument region will always work. + a.emit_mov(Size::S64, prev_loc, Location::GPR(GPR::RAX)); + a.emit_mov( + Size::S64, + Location::GPR(GPR::RAX), + Location::Memory(GPR::RSP, stack_offset + 8 + caller_stack_offset), + ); + caller_stack_offset += 8; + continue; + } + }; + a.emit_mov(Size::S64, prev_loc, target); + } + + // Restore stack pointer. + if stack_offset > 0 { + a.emit_add( + Size::S64, + Location::Imm32(stack_offset as u32), + Location::GPR(GPR::RSP), + ); + } + } + + // Emits a tail call trampoline that loads the address of the target import function + // from Ctx and jumps to it. + + let imported_funcs_addr = vm::Ctx::offset_imported_funcs(); + let imported_func = vm::ImportedFunc::size() as usize * id; + let imported_func_addr = imported_func + vm::ImportedFunc::offset_func() as usize; + let imported_func_ctx_addr = imported_func + vm::ImportedFunc::offset_func_ctx() as usize; + let imported_func_ctx_vmctx_addr = vm::FuncCtx::offset_vmctx() as usize; + + a.emit_mov( + Size::S64, + Location::Memory(GPR::RDI, imported_funcs_addr as i32), + Location::GPR(GPR::RAX), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, imported_func_ctx_addr as i32), + Location::GPR(GPR::RDI), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RDI, imported_func_ctx_vmctx_addr as i32), + Location::GPR(GPR::RDI), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, imported_func_addr as i32), + Location::GPR(GPR::RAX), + ); + a.emit_host_redirection(GPR::RAX); + + self.func_import_count += 1; + + Ok(()) + } + + fn feed_compiler_config(&mut self, config: &CompilerConfig) -> Result<(), CodegenError> { + self.config = Some(Arc::new(CodegenConfig { + memory_bound_check_mode: config.memory_bound_check_mode, + enforce_stack_check: config.enforce_stack_check, + track_state: config.track_state, + full_preemption: config.full_preemption, + nan_canonicalization: config.nan_canonicalization, + })); + Ok(()) + } + unsafe fn from_cache(artifact: Artifact, _: Token) -> Result { + let (info, _, memory) = artifact.consume(); + + let cache_image: CacheImage = bincode::deserialize(memory.as_slice()) + .map_err(|x| CacheError::DeserializeError(format!("{:?}", x)))?; + + let mut code_mem = CodeMemory::new(cache_image.code.len()); + code_mem[0..cache_image.code.len()].copy_from_slice(&cache_image.code); + code_mem.make_executable(); + + let function_pointers: Vec = cache_image + .function_pointers + .iter() + .map(|&x| FuncPtr(code_mem.as_ptr().offset(x as isize) as *const FuncPtrInner)) + .collect(); + let function_offsets: Vec = cache_image + .function_offsets + .iter() + .cloned() + .map(AssemblyOffset) + .collect(); + + let ec = X64ExecutionContext { + code: code_mem, + function_pointers, + function_offsets, + signatures: Arc::new(info.signatures.clone()), + breakpoints: Arc::new(HashMap::new()), + func_import_count: cache_image.func_import_count, + msm: cache_image.msm, + exception_table: cache_image.exception_table, + }; + Ok(ModuleInner { + runnable_module: Arc::new(Box::new(ec)), + cache_gen: Box::new(SinglepassCache { + buffer: Arc::from(memory.as_slice().to_vec().into_boxed_slice()), + }), + info, + }) + } +} + +impl X64FunctionCode { + fn mark_trappable( + a: &mut Assembler, + m: &Machine, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], + ) { + let state_diff_id = Self::get_state_diff(m, fsm, control_stack); + let offset = a.get_offset().0; + fsm.trappable_offsets.insert( + offset, + OffsetInfo { + end_offset: offset + 1, + activate_offset: offset, + diff_id: state_diff_id, + }, + ); + fsm.wasm_offset_to_target_offset + .insert(m.state.wasm_inst_offset, SuspendOffset::Trappable(offset)); + } + + /// Marks each address in the code range emitted by `f` with the exception code `code`. + fn mark_range_with_exception_code R, R>( + a: &mut Assembler, + etable: &mut ExceptionTable, + code: ExceptionCode, + f: F, + ) -> R { + let begin = a.get_offset().0; + let ret = f(a); + let end = a.get_offset().0; + for i in begin..end { + etable.offset_to_code.insert(i, code); + } + ret + } + + /// Canonicalizes the floating point value at `input` into `output`. + fn canonicalize_nan( + a: &mut Assembler, + m: &mut Machine, + sz: Size, + input: Location, + output: Location, + ) { + let tmp1 = m.acquire_temp_xmm().unwrap(); + let tmp2 = m.acquire_temp_xmm().unwrap(); + let tmp3 = m.acquire_temp_xmm().unwrap(); + let tmpg1 = m.acquire_temp_gpr().unwrap(); + + Self::emit_relaxed_binop(a, m, Assembler::emit_mov, sz, input, Location::XMM(tmp1)); + + match sz { + Size::S32 => { + a.emit_vcmpunordss(tmp1, XMMOrMemory::XMM(tmp1), tmp2); + a.emit_mov( + Size::S32, + Location::Imm32(0x7FC0_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp3)); + a.emit_vblendvps(tmp2, XMMOrMemory::XMM(tmp3), tmp1, tmp1); + } + Size::S64 => { + a.emit_vcmpunordsd(tmp1, XMMOrMemory::XMM(tmp1), tmp2); + a.emit_mov( + Size::S64, + Location::Imm64(0x7FF8_0000_0000_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp3)); + a.emit_vblendvpd(tmp2, XMMOrMemory::XMM(tmp3), tmp1, tmp1); + } + _ => unreachable!(), + } + + Self::emit_relaxed_binop(a, m, Assembler::emit_mov, sz, Location::XMM(tmp1), output); + + m.release_temp_gpr(tmpg1); + m.release_temp_xmm(tmp3); + m.release_temp_xmm(tmp2); + m.release_temp_xmm(tmp1); + } + + /// Moves `loc` to a valid location for `div`/`idiv`. + fn emit_relaxed_xdiv( + a: &mut Assembler, + m: &mut Machine, + etable: &mut ExceptionTable, + op: fn(&mut Assembler, Size, Location), + sz: Size, + loc: Location, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], + ) { + m.state.wasm_stack_private_depth += 1; + match loc { + Location::Imm64(_) | Location::Imm32(_) => { + a.emit_mov(sz, loc, Location::GPR(GPR::RCX)); // must not be used during div (rax, rdx) + Self::mark_trappable(a, m, fsm, control_stack); + etable + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::IllegalArithmetic); + op(a, sz, Location::GPR(GPR::RCX)); + } + _ => { + Self::mark_trappable(a, m, fsm, control_stack); + etable + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::IllegalArithmetic); + op(a, sz, loc); + } + } + m.state.wasm_stack_private_depth -= 1; + } + + /// Moves `src` and `dst` to valid locations for `movzx`/`movsx`. + fn emit_relaxed_zx_sx( + a: &mut Assembler, + m: &mut Machine, + op: fn(&mut Assembler, Size, Location, Size, Location), + sz_src: Size, + mut src: Location, + sz_dst: Size, + dst: Location, + ) -> Result<(), CodegenError> { + let inner = |m: &mut Machine, a: &mut Assembler, src: Location| match dst { + Location::Imm32(_) | Location::Imm64(_) => { + return Err(CodegenError { + message: format!("emit_relaxed_zx_sx dst Imm: unreachable code"), + }) + } + Location::Memory(_, _) => { + let tmp_dst = m.acquire_temp_gpr().unwrap(); + op(a, sz_src, src, sz_dst, Location::GPR(tmp_dst)); + a.emit_mov(Size::S64, Location::GPR(tmp_dst), dst); + + m.release_temp_gpr(tmp_dst); + Ok(()) + } + Location::GPR(_) => { + op(a, sz_src, src, sz_dst, dst); + Ok(()) + } + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_zx_sx dst: unreachable code"), + }) + } + }; + + match src { + Location::Imm32(_) | Location::Imm64(_) => { + let tmp_src = m.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, src, Location::GPR(tmp_src)); + src = Location::GPR(tmp_src); + + inner(m, a, src)?; + + m.release_temp_gpr(tmp_src); + } + Location::GPR(_) | Location::Memory(_, _) => inner(m, a, src)?, + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_zx_sx src: unreachable code"), + }) + } + } + Ok(()) + } + + /// Moves `src` and `dst` to valid locations for generic instructions. + fn emit_relaxed_binop( + a: &mut Assembler, + m: &mut Machine, + op: fn(&mut Assembler, Size, Location, Location), + sz: Size, + src: Location, + dst: Location, + ) { + enum RelaxMode { + Direct, + SrcToGPR, + DstToGPR, + BothToGPR, + } + let mode = match (src, dst) { + (Location::GPR(_), Location::GPR(_)) + if (op as *const u8 == Assembler::emit_imul as *const u8) => + { + RelaxMode::Direct + } + _ if (op as *const u8 == Assembler::emit_imul as *const u8) => RelaxMode::BothToGPR, + + (Location::Memory(_, _), Location::Memory(_, _)) => RelaxMode::SrcToGPR, + (Location::Imm64(_), Location::Imm64(_)) | (Location::Imm64(_), Location::Imm32(_)) => { + RelaxMode::BothToGPR + } + (_, Location::Imm32(_)) | (_, Location::Imm64(_)) => RelaxMode::DstToGPR, + (Location::Imm64(_), Location::Memory(_, _)) => RelaxMode::SrcToGPR, + (Location::Imm64(_), Location::GPR(_)) + if (op as *const u8 != Assembler::emit_mov as *const u8) => + { + RelaxMode::SrcToGPR + } + (_, Location::XMM(_)) => RelaxMode::SrcToGPR, + _ => RelaxMode::Direct, + }; + + match mode { + RelaxMode::SrcToGPR => { + let temp = m.acquire_temp_gpr().unwrap(); + a.emit_mov(sz, src, Location::GPR(temp)); + op(a, sz, Location::GPR(temp), dst); + m.release_temp_gpr(temp); + } + RelaxMode::DstToGPR => { + let temp = m.acquire_temp_gpr().unwrap(); + a.emit_mov(sz, dst, Location::GPR(temp)); + op(a, sz, src, Location::GPR(temp)); + m.release_temp_gpr(temp); + } + RelaxMode::BothToGPR => { + let temp_src = m.acquire_temp_gpr().unwrap(); + let temp_dst = m.acquire_temp_gpr().unwrap(); + a.emit_mov(sz, src, Location::GPR(temp_src)); + a.emit_mov(sz, dst, Location::GPR(temp_dst)); + op(a, sz, Location::GPR(temp_src), Location::GPR(temp_dst)); + match dst { + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(sz, Location::GPR(temp_dst), dst); + } + _ => {} + } + m.release_temp_gpr(temp_dst); + m.release_temp_gpr(temp_src); + } + RelaxMode::Direct => { + op(a, sz, src, dst); + } + } + } + + /// Moves `src1` and `src2` to valid locations and possibly adds a layer of indirection for `dst` for AVX instructions. + fn emit_relaxed_avx( + a: &mut Assembler, + m: &mut Machine, + op: fn(&mut Assembler, XMM, XMMOrMemory, XMM), + src1: Location, + src2: Location, + dst: Location, + ) -> Result<(), CodegenError> { + Self::emit_relaxed_avx_base( + a, + m, + |a, _, src1, src2, dst| op(a, src1, src2, dst), + src1, + src2, + dst, + )?; + Ok(()) + } + + /// Moves `src1` and `src2` to valid locations and possibly adds a layer of indirection for `dst` for AVX instructions. + fn emit_relaxed_avx_base( + a: &mut Assembler, + m: &mut Machine, + op: F, + src1: Location, + src2: Location, + dst: Location, + ) -> Result<(), CodegenError> { + let tmp1 = m.acquire_temp_xmm().unwrap(); + let tmp2 = m.acquire_temp_xmm().unwrap(); + let tmp3 = m.acquire_temp_xmm().unwrap(); + let tmpg = m.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg)); + a.emit_mov(Size::S32, Location::GPR(tmpg), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg)); + a.emit_mov(Size::S64, Location::GPR(tmpg), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_avx_base src1: unreachable code"), + }) + } + }; + + let src2 = match src2 { + Location::XMM(x) => XMMOrMemory::XMM(x), + Location::Memory(base, disp) => XMMOrMemory::Memory(base, disp), + Location::GPR(_) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + XMMOrMemory::XMM(tmp2) + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg)); + a.emit_mov(Size::S32, Location::GPR(tmpg), Location::XMM(tmp2)); + XMMOrMemory::XMM(tmp2) + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg)); + a.emit_mov(Size::S64, Location::GPR(tmpg), Location::XMM(tmp2)); + XMMOrMemory::XMM(tmp2) + } + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_avx_base src2: unreachable code"), + }) + } + }; + + match dst { + Location::XMM(x) => { + op(a, m, src1, src2, x); + } + Location::Memory(_, _) | Location::GPR(_) => { + op(a, m, src1, src2, tmp3); + a.emit_mov(Size::S64, Location::XMM(tmp3), dst); + } + _ => { + return Err(CodegenError { + message: format!("emit_relaxed_avx_base dst: unreachable code"), + }) + } + } + + m.release_temp_gpr(tmpg); + m.release_temp_xmm(tmp3); + m.release_temp_xmm(tmp2); + m.release_temp_xmm(tmp1); + Ok(()) + } + + /// I32 binary operation with both operands popped from the virtual stack. + fn emit_binop_i32( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) { + // Using Red Zone here. + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + if loc_a != ret { + let tmp = m.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + loc_a, + Location::GPR(tmp), + ); + Self::emit_relaxed_binop(a, m, f, Size::S32, loc_b, Location::GPR(tmp)); + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp), + ret, + ); + m.release_temp_gpr(tmp); + } else { + Self::emit_relaxed_binop(a, m, f, Size::S32, loc_b, ret); + } + + value_stack.push(ret); + } + + /// I64 binary operation with both operands popped from the virtual stack. + fn emit_binop_i64( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) { + // Using Red Zone here. + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + if loc_a != ret { + let tmp = m.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + loc_a, + Location::GPR(tmp), + ); + Self::emit_relaxed_binop(a, m, f, Size::S64, loc_b, Location::GPR(tmp)); + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp), + ret, + ); + m.release_temp_gpr(tmp); + } else { + Self::emit_relaxed_binop(a, m, f, Size::S64, loc_b, ret); + } + + value_stack.push(ret); + } + + /// I32 comparison with `loc_b` from input. + fn emit_cmpop_i32_dynamic_b( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + c: Condition, + loc_b: Location, + ) -> Result<(), CodegenError> { + // Using Red Zone here. + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + match ret { + Location::GPR(x) => { + Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S32, loc_b, loc_a); + a.emit_set(c, x); + a.emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(x)); + } + Location::Memory(_, _) => { + let tmp = m.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S32, loc_b, loc_a); + a.emit_set(c, tmp); + a.emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(tmp)); + a.emit_mov(Size::S32, Location::GPR(tmp), ret); + m.release_temp_gpr(tmp); + } + _ => { + return Err(CodegenError { + message: format!("emit_cmpop_i32_dynamic_b ret: unreachable code"), + }) + } + } + value_stack.push(ret); + Ok(()) + } + + /// I32 comparison with both operands popped from the virtual stack. + fn emit_cmpop_i32( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + c: Condition, + ) -> Result<(), CodegenError> { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + Self::emit_cmpop_i32_dynamic_b(a, m, value_stack, c, loc_b)?; + Ok(()) + } + + /// I64 comparison with `loc_b` from input. + fn emit_cmpop_i64_dynamic_b( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + c: Condition, + loc_b: Location, + ) -> Result<(), CodegenError> { + // Using Red Zone here. + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + match ret { + Location::GPR(x) => { + Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S64, loc_b, loc_a); + a.emit_set(c, x); + a.emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(x)); + } + Location::Memory(_, _) => { + let tmp = m.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S64, loc_b, loc_a); + a.emit_set(c, tmp); + a.emit_and(Size::S32, Location::Imm32(0xff), Location::GPR(tmp)); + a.emit_mov(Size::S32, Location::GPR(tmp), ret); + m.release_temp_gpr(tmp); + } + _ => { + return Err(CodegenError { + message: format!("emit_cmpop_i64_dynamic_b ret: unreachable code"), + }) + } + } + value_stack.push(ret); + Ok(()) + } + + /// I64 comparison with both operands popped from the virtual stack. + fn emit_cmpop_i64( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + c: Condition, + ) -> Result<(), CodegenError> { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + Self::emit_cmpop_i64_dynamic_b(a, m, value_stack, c, loc_b)?; + Ok(()) + } + + /// I32 `lzcnt`/`tzcnt`/`popcnt` with operand popped from the virtual stack. + fn emit_xcnt_i32( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) -> Result<(), CodegenError> { + let loc = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + match loc { + Location::Imm32(_) => { + let tmp = m.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + if let Location::Memory(_, _) = ret { + let out_tmp = m.acquire_temp_gpr().unwrap(); + f(a, Size::S32, Location::GPR(tmp), Location::GPR(out_tmp)); + a.emit_mov(Size::S32, Location::GPR(out_tmp), ret); + m.release_temp_gpr(out_tmp); + } else { + f(a, Size::S32, Location::GPR(tmp), ret); + } + m.release_temp_gpr(tmp); + } + Location::Memory(_, _) | Location::GPR(_) => { + if let Location::Memory(_, _) = ret { + let out_tmp = m.acquire_temp_gpr().unwrap(); + f(a, Size::S32, loc, Location::GPR(out_tmp)); + a.emit_mov(Size::S32, Location::GPR(out_tmp), ret); + m.release_temp_gpr(out_tmp); + } else { + f(a, Size::S32, loc, ret); + } + } + _ => { + return Err(CodegenError { + message: format!("emit_xcnt_i32 loc: unreachable code"), + }) + } + } + value_stack.push(ret); + Ok(()) + } + + /// I64 `lzcnt`/`tzcnt`/`popcnt` with operand popped from the virtual stack. + fn emit_xcnt_i64( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) -> Result<(), CodegenError> { + let loc = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + match loc { + Location::Imm64(_) | Location::Imm32(_) => { + let tmp = m.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + if let Location::Memory(_, _) = ret { + let out_tmp = m.acquire_temp_gpr().unwrap(); + f(a, Size::S64, Location::GPR(tmp), Location::GPR(out_tmp)); + a.emit_mov(Size::S64, Location::GPR(out_tmp), ret); + m.release_temp_gpr(out_tmp); + } else { + f(a, Size::S64, Location::GPR(tmp), ret); + } + m.release_temp_gpr(tmp); + } + Location::Memory(_, _) | Location::GPR(_) => { + if let Location::Memory(_, _) = ret { + let out_tmp = m.acquire_temp_gpr().unwrap(); + f(a, Size::S64, loc, Location::GPR(out_tmp)); + a.emit_mov(Size::S64, Location::GPR(out_tmp), ret); + m.release_temp_gpr(out_tmp); + } else { + f(a, Size::S64, loc, ret); + } + } + _ => { + return Err(CodegenError { + message: format!("emit_xcnt_i64 loc: unreachable code"), + }) + } + } + value_stack.push(ret); + Ok(()) + } + + /// I32 shift with both operands popped from the virtual stack. + fn emit_shift_i32( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + a.emit_mov(Size::S32, loc_b, Location::GPR(GPR::RCX)); + + if loc_a != ret { + Self::emit_relaxed_binop(a, m, Assembler::emit_mov, Size::S32, loc_a, ret); + } + + f(a, Size::S32, Location::GPR(GPR::RCX), ret); + value_stack.push(ret); + } + + /// I64 shift with both operands popped from the virtual stack. + fn emit_shift_i64( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, Size, Location, Location), + ) { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + + a.emit_mov(Size::S64, loc_b, Location::GPR(GPR::RCX)); + + if loc_a != ret { + Self::emit_relaxed_binop(a, m, Assembler::emit_mov, Size::S64, loc_a, ret); + } + + f(a, Size::S64, Location::GPR(GPR::RCX), ret); + value_stack.push(ret); + } + + /// Floating point (AVX) binary operation with both operands popped from the virtual stack. + fn emit_fp_binop_avx( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), + ) -> Result<(), CodegenError> { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); + + Self::emit_relaxed_avx(a, m, f, loc_a, loc_b, ret)?; + Ok(()) + } + + /// Floating point (AVX) comparison with both operands popped from the virtual stack. + fn emit_fp_cmpop_avx( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), + ) -> Result<(), CodegenError> { + let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); + let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); + + Self::emit_relaxed_avx(a, m, f, loc_a, loc_b, ret)?; + + // Workaround for behavior inconsistency among different backing implementations. + // (all bits or only the least significant bit are set to one?) + a.emit_and(Size::S32, Location::Imm32(1), ret); + Ok(()) + } + + /// Floating point (AVX) binary operation with both operands popped from the virtual stack. + fn emit_fp_unop_avx( + a: &mut Assembler, + m: &mut Machine, + value_stack: &mut Vec, + f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), + ) -> Result<(), CodegenError> { + let loc = get_location_released(a, m, value_stack.pop().unwrap()); + let ret = m.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); + + Self::emit_relaxed_avx(a, m, f, loc, loc, ret)?; + Ok(()) + } + + /// Emits a System V call sequence. + /// + /// This function must not use RAX before `cb` is called. + fn emit_call_sysv, F: FnOnce(&mut Assembler)>( + a: &mut Assembler, + m: &mut Machine, + cb: F, + params: I, + state_context: Option<(&mut FunctionStateMap, &mut [ControlFrame])>, + ) -> Result<(), CodegenError> { + // Values pushed in this function are above the shadow region. + m.state.stack_values.push(MachineValue::ExplicitShadow); + + let params: Vec<_> = params.collect(); + + // Save used GPRs. + let used_gprs = m.get_used_gprs(); + for r in used_gprs.iter() { + a.emit_push(Size::S64, Location::GPR(*r)); + let content = m.state.register_values[X64Register::GPR(*r).to_index().0].clone(); + if content == MachineValue::Undefined { + return Err(CodegenError { + message: format!("emit_call_sysv: Undefined used_gprs content"), + }); + } + m.state.stack_values.push(content); + } + + // Save used XMM registers. + let used_xmms = m.get_used_xmms(); + if used_xmms.len() > 0 { + a.emit_sub( + Size::S64, + Location::Imm32((used_xmms.len() * 8) as u32), + Location::GPR(GPR::RSP), + ); + + for (i, r) in used_xmms.iter().enumerate() { + a.emit_mov( + Size::S64, + Location::XMM(*r), + Location::Memory(GPR::RSP, (i * 8) as i32), + ); + } + for r in used_xmms.iter().rev() { + let content = m.state.register_values[X64Register::XMM(*r).to_index().0].clone(); + if content == MachineValue::Undefined { + return Err(CodegenError { + message: format!("emit_call_sysv: Undefined used_xmms content"), + }); + } + m.state.stack_values.push(content); + } + } + + let mut stack_offset: usize = 0; + + // Calculate stack offset. + for (i, _param) in params.iter().enumerate() { + let loc = Machine::get_param_location(1 + i); + match loc { + Location::Memory(_, _) => { + stack_offset += 8; + } + _ => {} + } + } + + // Align stack to 16 bytes. + if (m.get_stack_offset() + used_gprs.len() * 8 + used_xmms.len() * 8 + stack_offset) % 16 + != 0 + { + a.emit_sub(Size::S64, Location::Imm32(8), Location::GPR(GPR::RSP)); + stack_offset += 8; + m.state.stack_values.push(MachineValue::Undefined); + } + + let mut call_movs: Vec<(Location, GPR)> = vec![]; + + // Prepare register & stack parameters. + for (i, param) in params.iter().enumerate().rev() { + let loc = Machine::get_param_location(1 + i); + match loc { + Location::GPR(x) => { + call_movs.push((*param, x)); + } + Location::Memory(_, _) => { + match *param { + Location::GPR(x) => { + let content = + m.state.register_values[X64Register::GPR(x).to_index().0].clone(); + // FIXME: There might be some corner cases (release -> emit_call_sysv -> acquire?) that cause this assertion to fail. + // Hopefully nothing would be incorrect at runtime. + + //assert!(content != MachineValue::Undefined); + m.state.stack_values.push(content); + } + Location::XMM(x) => { + let content = + m.state.register_values[X64Register::XMM(x).to_index().0].clone(); + //assert!(content != MachineValue::Undefined); + m.state.stack_values.push(content); + } + Location::Memory(reg, offset) => { + if reg != GPR::RBP { + return Err(CodegenError { + message: format!("emit_call_sysv loc param: unreachable code"), + }); + } + m.state + .stack_values + .push(MachineValue::CopyStackBPRelative(offset)); + // TODO: Read value at this offset + } + _ => { + m.state.stack_values.push(MachineValue::Undefined); + } + } + match *param { + Location::Imm64(_) => { + // Dummy value slot to be filled with `mov`. + a.emit_push(Size::S64, Location::GPR(GPR::RAX)); + + // Use R10 as the temporary register here, since it is callee-saved and not + // used by the callback `cb`. + a.emit_mov(Size::S64, *param, Location::GPR(GPR::R10)); + a.emit_mov( + Size::S64, + Location::GPR(GPR::R10), + Location::Memory(GPR::RSP, 0), + ); + } + Location::XMM(_) => { + // Dummy value slot to be filled with `mov`. + a.emit_push(Size::S64, Location::GPR(GPR::RAX)); + + // XMM registers can be directly stored to memory. + a.emit_mov(Size::S64, *param, Location::Memory(GPR::RSP, 0)); + } + _ => a.emit_push(Size::S64, *param), + } + } + _ => { + return Err(CodegenError { + message: format!("emit_call_sysv loc: unreachable code"), + }) + } + } + } + + // Sort register moves so that register are not overwritten before read. + sort_call_movs(&mut call_movs); + + // Emit register moves. + for (loc, gpr) in call_movs { + if loc != Location::GPR(gpr) { + a.emit_mov(Size::S64, loc, Location::GPR(gpr)); + } + } + + // Put vmctx as the first parameter. + a.emit_mov( + Size::S64, + Location::GPR(Machine::get_vmctx_reg()), + Machine::get_param_location(0), + ); // vmctx + + if (m.state.stack_values.len() % 2) != 1 { + return Err(CodegenError { + message: format!("emit_call_sysv: explicit shadow takes one slot"), + }); + } + + cb(a); + + // Offset needs to be after the 'call' instruction. + if let Some((fsm, control_stack)) = state_context { + let state_diff_id = Self::get_state_diff(m, fsm, control_stack); + let offset = a.get_offset().0; + fsm.call_offsets.insert( + offset, + OffsetInfo { + end_offset: offset + 1, + activate_offset: offset, + diff_id: state_diff_id, + }, + ); + fsm.wasm_offset_to_target_offset + .insert(m.state.wasm_inst_offset, SuspendOffset::Call(offset)); + } + + // Restore stack. + if stack_offset > 0 { + a.emit_add( + Size::S64, + Location::Imm32(stack_offset as u32), + Location::GPR(GPR::RSP), + ); + if (stack_offset % 8) != 0 { + return Err(CodegenError { + message: format!("emit_call_sysv: Bad restoring stack alignement"), + }); + } + for _ in 0..stack_offset / 8 { + m.state.stack_values.pop().unwrap(); + } + } + + // Restore XMMs. + if used_xmms.len() > 0 { + for (i, r) in used_xmms.iter().enumerate() { + a.emit_mov( + Size::S64, + Location::Memory(GPR::RSP, (i * 8) as i32), + Location::XMM(*r), + ); + } + a.emit_add( + Size::S64, + Location::Imm32((used_xmms.len() * 8) as u32), + Location::GPR(GPR::RSP), + ); + for _ in 0..used_xmms.len() { + m.state.stack_values.pop().unwrap(); + } + } + + // Restore GPRs. + for r in used_gprs.iter().rev() { + a.emit_pop(Size::S64, Location::GPR(*r)); + m.state.stack_values.pop().unwrap(); + } + + if m.state.stack_values.pop().unwrap() != MachineValue::ExplicitShadow { + return Err(CodegenError { + message: format!("emit_call_sysv: Popped value is not ExplicitShadow"), + }); + } + Ok(()) + } + + /// Emits a System V call sequence, specialized for labels as the call target. + fn emit_call_sysv_label>( + a: &mut Assembler, + m: &mut Machine, + label: DynamicLabel, + params: I, + state_context: Option<(&mut FunctionStateMap, &mut [ControlFrame])>, + ) -> Result<(), CodegenError> { + Self::emit_call_sysv(a, m, |a| a.emit_call_label(label), params, state_context)?; + Ok(()) + } + + /// Emits a memory operation. + fn emit_memory_op Result<(), CodegenError>>( + module_info: &ModuleInfo, + config: &CodegenConfig, + a: &mut Assembler, + m: &mut Machine, + etable: &mut ExceptionTable, + addr: Location, + memarg: &MemoryImmediate, + check_alignment: bool, + value_size: usize, + cb: F, + ) -> Result<(), CodegenError> { + // If the memory is dynamic, we need to do bound checking at runtime. + let mem_desc = match MemoryIndex::new(0).local_or_import(module_info) { + LocalOrImport::Local(local_mem_index) => &module_info.memories[local_mem_index], + LocalOrImport::Import(import_mem_index) => { + &module_info.imported_memories[import_mem_index].1 + } + }; + let need_check = match config.memory_bound_check_mode { + MemoryBoundCheckMode::Default => match mem_desc.memory_type() { + MemoryType::Dynamic => true, + MemoryType::Static | MemoryType::SharedStatic => false, + }, + MemoryBoundCheckMode::Enable => true, + MemoryBoundCheckMode::Disable => false, + }; + + let tmp_addr = m.acquire_temp_gpr().unwrap(); + let tmp_base = m.acquire_temp_gpr().unwrap(); + + // Load base into temporary register. + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_memory_base() as i32, + ), + Location::GPR(tmp_base), + ); + + if need_check { + let tmp_bound = m.acquire_temp_gpr().unwrap(); + + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_memory_bound() as i32, + ), + Location::GPR(tmp_bound), + ); + // Adds base to bound so `tmp_bound` now holds the end of linear memory. + a.emit_add(Size::S64, Location::GPR(tmp_base), Location::GPR(tmp_bound)); + a.emit_mov(Size::S32, addr, Location::GPR(tmp_addr)); + + // This branch is used for emitting "faster" code for the special case of (offset + value_size) not exceeding u32 range. + match (memarg.offset as u32).checked_add(value_size as u32) { + Some(0) => {} + Some(x) => { + a.emit_add(Size::S64, Location::Imm32(x), Location::GPR(tmp_addr)); + } + None => { + a.emit_add( + Size::S64, + Location::Imm32(memarg.offset as u32), + Location::GPR(tmp_addr), + ); + a.emit_add( + Size::S64, + Location::Imm32(value_size as u32), + Location::GPR(tmp_addr), + ); + } + } + + // Trap if the end address of the requested area is above that of the linear memory. + a.emit_add(Size::S64, Location::GPR(tmp_base), Location::GPR(tmp_addr)); + a.emit_cmp(Size::S64, Location::GPR(tmp_bound), Location::GPR(tmp_addr)); + + Self::mark_range_with_exception_code( + a, + etable, + ExceptionCode::MemoryOutOfBounds, + |a| a.emit_conditional_trap(Condition::Above), + ); + + m.release_temp_gpr(tmp_bound); + } + + // Calculates the real address, and loads from it. + a.emit_mov(Size::S32, addr, Location::GPR(tmp_addr)); + if memarg.offset != 0 { + a.emit_add( + Size::S64, + Location::Imm32(memarg.offset as u32), + Location::GPR(tmp_addr), + ); + } + a.emit_add(Size::S64, Location::GPR(tmp_base), Location::GPR(tmp_addr)); + m.release_temp_gpr(tmp_base); + + let align = match memarg.flags & 3 { + 0 => 1, + 1 => 2, + 2 => 4, + 3 => 8, + _ => { + return Err(CodegenError { + message: format!("emit_memory_op align: unreachable value"), + }) + } + }; + if check_alignment && align != 1 { + let tmp_aligncheck = m.acquire_temp_gpr().unwrap(); + a.emit_mov( + Size::S32, + Location::GPR(tmp_addr), + Location::GPR(tmp_aligncheck), + ); + a.emit_and( + Size::S64, + Location::Imm32(align - 1), + Location::GPR(tmp_aligncheck), + ); + Self::mark_range_with_exception_code( + a, + etable, + ExceptionCode::MemoryOutOfBounds, + |a| a.emit_conditional_trap(Condition::NotEqual), + ); + m.release_temp_gpr(tmp_aligncheck); + } + + Self::mark_range_with_exception_code(a, etable, ExceptionCode::MemoryOutOfBounds, |a| { + cb(a, m, tmp_addr) + })?; + + m.release_temp_gpr(tmp_addr); + Ok(()) + } + + /// Emits a memory operation. + fn emit_compare_and_swap( + module_info: &ModuleInfo, + config: &CodegenConfig, + a: &mut Assembler, + m: &mut Machine, + etable: &mut ExceptionTable, + loc: Location, + target: Location, + ret: Location, + memarg: &MemoryImmediate, + value_size: usize, + memory_sz: Size, + stack_sz: Size, + cb: F, + ) -> Result<(), CodegenError> { + if memory_sz > stack_sz { + return Err(CodegenError { + message: format!("emit_compare_and_swap: memory size > stac size"), + }); + } + + let compare = m.reserve_unused_temp_gpr(GPR::RAX); + let value = if loc == Location::GPR(GPR::R14) { + GPR::R13 + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + + a.emit_mov(stack_sz, loc, Location::GPR(value)); + + let retry = a.get_label(); + a.emit_label(retry); + + Self::emit_memory_op( + module_info, + config, + a, + m, + etable, + target, + memarg, + true, + value_size, + |a, m, addr| { + // Memory moves with size < 32b do not zero upper bits. + if memory_sz < Size::S32 { + a.emit_xor(Size::S32, Location::GPR(compare), Location::GPR(compare)); + } + a.emit_mov(memory_sz, Location::Memory(addr, 0), Location::GPR(compare)); + a.emit_mov(stack_sz, Location::GPR(compare), ret); + cb(a, m, compare, value); + a.emit_lock_cmpxchg(memory_sz, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + + a.emit_jmp(Condition::NotEqual, retry); + + a.emit_pop(Size::S64, Location::GPR(value)); + m.release_temp_gpr(compare); + Ok(()) + } + + // Checks for underflow/overflow/nan. + fn emit_f32_int_conv_check( + a: &mut Assembler, + m: &mut Machine, + reg: XMM, + lower_bound: f32, + upper_bound: f32, + underflow_label: ::Label, + overflow_label: ::Label, + nan_label: ::Label, + succeed_label: ::Label, + ) { + let lower_bound = f32::to_bits(lower_bound); + let upper_bound = f32::to_bits(upper_bound); + + let tmp = m.acquire_temp_gpr().unwrap(); + let tmp_x = m.acquire_temp_xmm().unwrap(); + + // Underflow. + a.emit_mov(Size::S32, Location::Imm32(lower_bound), Location::GPR(tmp)); + a.emit_mov(Size::S32, Location::GPR(tmp), Location::XMM(tmp_x)); + a.emit_vcmpless(reg, XMMOrMemory::XMM(tmp_x), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::NotEqual, underflow_label); + + // Overflow. + a.emit_mov(Size::S32, Location::Imm32(upper_bound), Location::GPR(tmp)); + a.emit_mov(Size::S32, Location::GPR(tmp), Location::XMM(tmp_x)); + a.emit_vcmpgess(reg, XMMOrMemory::XMM(tmp_x), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::NotEqual, overflow_label); + + // NaN. + a.emit_vcmpeqss(reg, XMMOrMemory::XMM(reg), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::Equal, nan_label); + + a.emit_jmp(Condition::None, succeed_label); + + m.release_temp_xmm(tmp_x); + m.release_temp_gpr(tmp); + } + + // Checks for underflow/overflow/nan before IxxTrunc{U/S}F32. + fn emit_f32_int_conv_check_trap( + a: &mut Assembler, + m: &mut Machine, + etable: &mut ExceptionTable, + reg: XMM, + lower_bound: f32, + upper_bound: f32, + ) { + let trap = a.get_label(); + let end = a.get_label(); + + Self::emit_f32_int_conv_check(a, m, reg, lower_bound, upper_bound, trap, trap, trap, end); + a.emit_label(trap); + etable + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::IllegalArithmetic); + a.emit_ud2(); + a.emit_label(end); + } + + fn emit_f32_int_conv_check_sat< + F1: FnOnce(&mut Assembler, &mut Machine), + F2: FnOnce(&mut Assembler, &mut Machine), + F3: FnOnce(&mut Assembler, &mut Machine), + F4: FnOnce(&mut Assembler, &mut Machine), + >( + a: &mut Assembler, + m: &mut Machine, + reg: XMM, + lower_bound: f32, + upper_bound: f32, + underflow_cb: F1, + overflow_cb: F2, + nan_cb: Option, + convert_cb: F4, + ) { + // As an optimization nan_cb is optional, and when set to None we turn + // use 'underflow' as the 'nan' label. This is useful for callers who + // set the return value to zero for both underflow and nan. + + let underflow = a.get_label(); + let overflow = a.get_label(); + let nan = if nan_cb.is_some() { + a.get_label() + } else { + underflow + }; + let convert = a.get_label(); + let end = a.get_label(); + + Self::emit_f32_int_conv_check( + a, + m, + reg, + lower_bound, + upper_bound, + underflow, + overflow, + nan, + convert, + ); + + a.emit_label(underflow); + underflow_cb(a, m); + a.emit_jmp(Condition::None, end); + + a.emit_label(overflow); + overflow_cb(a, m); + a.emit_jmp(Condition::None, end); + + if let Some(cb) = nan_cb { + a.emit_label(nan); + cb(a, m); + a.emit_jmp(Condition::None, end); + } + + a.emit_label(convert); + convert_cb(a, m); + a.emit_label(end); + } + + // Checks for underflow/overflow/nan. + fn emit_f64_int_conv_check( + a: &mut Assembler, + m: &mut Machine, + reg: XMM, + lower_bound: f64, + upper_bound: f64, + underflow_label: ::Label, + overflow_label: ::Label, + nan_label: ::Label, + succeed_label: ::Label, + ) { + let lower_bound = f64::to_bits(lower_bound); + let upper_bound = f64::to_bits(upper_bound); + + let tmp = m.acquire_temp_gpr().unwrap(); + let tmp_x = m.acquire_temp_xmm().unwrap(); + + // Underflow. + a.emit_mov(Size::S64, Location::Imm64(lower_bound), Location::GPR(tmp)); + a.emit_mov(Size::S64, Location::GPR(tmp), Location::XMM(tmp_x)); + a.emit_vcmplesd(reg, XMMOrMemory::XMM(tmp_x), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::NotEqual, underflow_label); + + // Overflow. + a.emit_mov(Size::S64, Location::Imm64(upper_bound), Location::GPR(tmp)); + a.emit_mov(Size::S64, Location::GPR(tmp), Location::XMM(tmp_x)); + a.emit_vcmpgesd(reg, XMMOrMemory::XMM(tmp_x), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::NotEqual, overflow_label); + + // NaN. + a.emit_vcmpeqsd(reg, XMMOrMemory::XMM(reg), tmp_x); + a.emit_mov(Size::S32, Location::XMM(tmp_x), Location::GPR(tmp)); + a.emit_cmp(Size::S32, Location::Imm32(0), Location::GPR(tmp)); + a.emit_jmp(Condition::Equal, nan_label); + + a.emit_jmp(Condition::None, succeed_label); + + m.release_temp_xmm(tmp_x); + m.release_temp_gpr(tmp); + } + + // Checks for underflow/overflow/nan before IxxTrunc{U/S}F64. + fn emit_f64_int_conv_check_trap( + a: &mut Assembler, + m: &mut Machine, + etable: &mut ExceptionTable, + reg: XMM, + lower_bound: f64, + upper_bound: f64, + ) { + let trap = a.get_label(); + let end = a.get_label(); + + Self::emit_f64_int_conv_check(a, m, reg, lower_bound, upper_bound, trap, trap, trap, end); + a.emit_label(trap); + etable + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::IllegalArithmetic); + a.emit_ud2(); + a.emit_label(end); + } + + fn emit_f64_int_conv_check_sat< + F1: FnOnce(&mut Assembler, &mut Machine), + F2: FnOnce(&mut Assembler, &mut Machine), + F3: FnOnce(&mut Assembler, &mut Machine), + F4: FnOnce(&mut Assembler, &mut Machine), + >( + a: &mut Assembler, + m: &mut Machine, + reg: XMM, + lower_bound: f64, + upper_bound: f64, + underflow_cb: F1, + overflow_cb: F2, + nan_cb: Option, + convert_cb: F4, + ) { + // As an optimization nan_cb is optional, and when set to None we turn + // use 'underflow' as the 'nan' label. This is useful for callers who + // set the return value to zero for both underflow and nan. + + let underflow = a.get_label(); + let overflow = a.get_label(); + let nan = if nan_cb.is_some() { + a.get_label() + } else { + underflow + }; + let convert = a.get_label(); + let end = a.get_label(); + + Self::emit_f64_int_conv_check( + a, + m, + reg, + lower_bound, + upper_bound, + underflow, + overflow, + nan, + convert, + ); + + a.emit_label(underflow); + underflow_cb(a, m); + a.emit_jmp(Condition::None, end); + + a.emit_label(overflow); + overflow_cb(a, m); + a.emit_jmp(Condition::None, end); + + if let Some(cb) = nan_cb { + a.emit_label(nan); + cb(a, m); + a.emit_jmp(Condition::None, end); + } + + a.emit_label(convert); + convert_cb(a, m); + a.emit_label(end); + } + + pub fn get_state_diff( + m: &Machine, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], + ) -> usize { + if !m.track_state { + return usize::MAX; + } + let last_frame = control_stack.last_mut().unwrap(); + let mut diff = m.state.diff(&last_frame.state); + diff.last = Some(last_frame.state_diff_id); + let id = fsm.diffs.len(); + last_frame.state = m.state.clone(); + last_frame.state_diff_id = id; + fsm.diffs.push(diff); + id + } +} + +impl FunctionCodeGenerator for X64FunctionCode { + fn feed_return(&mut self, ty: WpType) -> Result<(), CodegenError> { + self.returns.push(ty); + Ok(()) + } + + fn feed_param(&mut self, ty: WpType) -> Result<(), CodegenError> { + self.num_params += 1; + self.local_types.push(ty); + Ok(()) + } + + fn feed_local(&mut self, ty: WpType, n: usize, _loc: u32) -> Result<(), CodegenError> { + self.local_types.extend(iter::repeat(ty).take(n)); + Ok(()) + } + + fn begin_body(&mut self, _module_info: &ModuleInfo) -> Result<(), CodegenError> { + let a = self.assembler.as_mut().unwrap(); + let start_label = a.get_label(); + // skip the patchpoint during normal execution + a.emit_jmp(Condition::None, start_label); + // patchpoint of 32 1-byte nops + for _ in 0..32 { + a.emit_nop(); + } + a.emit_label(start_label); + a.emit_push(Size::S64, Location::GPR(GPR::RBP)); + a.emit_mov(Size::S64, Location::GPR(GPR::RSP), Location::GPR(GPR::RBP)); + + // Stack check. + if self.config.enforce_stack_check { + a.emit_cmp( + Size::S64, + Location::Memory( + GPR::RDI, // first parameter is vmctx + vm::Ctx::offset_stack_lower_bound() as i32, + ), + Location::GPR(GPR::RSP), + ); + Self::mark_range_with_exception_code( + a, + self.exception_table.as_mut().unwrap(), + ExceptionCode::MemoryOutOfBounds, + |a| a.emit_conditional_trap(Condition::Below), + ); + } + + self.locals = self + .machine + .init_locals(a, self.local_types.len(), self.num_params); + + self.machine.state.register_values + [X64Register::GPR(Machine::get_vmctx_reg()).to_index().0] = MachineValue::Vmctx; + + self.fsm = FunctionStateMap::new( + new_machine_state(), + self.local_function_id, + 32, + (0..self.locals.len()) + .map(|_| WasmAbstractValue::Runtime) + .collect(), + ); + + let diff = self.machine.state.diff(&new_machine_state()); + let state_diff_id = self.fsm.diffs.len(); + self.fsm.diffs.push(diff); + + //println!("initial state = {:?}", self.machine.state); + + a.emit_sub(Size::S64, Location::Imm32(32), Location::GPR(GPR::RSP)); // simulate "red zone" if not supported by the platform + + self.control_stack.push(ControlFrame { + label: a.get_label(), + loop_like: false, + if_else: IfElseState::None, + returns: self.returns.clone(), + value_stack_depth: 0, + fp_stack_depth: 0, + state: self.machine.state.clone(), + state_diff_id, + }); + + // Check interrupt signal without branching + let activate_offset = a.get_offset().0; + + if self.config.full_preemption { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_interrupt_signal_mem() as i32, + ), + Location::GPR(GPR::RAX), + ); + self.fsm.loop_offsets.insert( + a.get_offset().0, + OffsetInfo { + end_offset: a.get_offset().0 + 1, + activate_offset, + diff_id: state_diff_id, + }, + ); + self.fsm.wasm_function_header_target_offset = + Some(SuspendOffset::Loop(a.get_offset().0)); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, 0), + Location::GPR(GPR::RAX), + ); + } + + if self.machine.state.wasm_inst_offset != usize::MAX { + return Err(CodegenError { + message: format!("begin_body: wasm_inst_offset not usize::MAX"), + }); + } + Ok(()) + } + + fn finalize(&mut self) -> Result<(), CodegenError> { + let a = self.assembler.as_mut().unwrap(); + a.emit_ud2(); + Ok(()) + } + + fn feed_event( + &mut self, + ev: Event, + module_info: &ModuleInfo, + _source_loc: u32, + ) -> Result<(), CodegenError> { + assert!(self.fp_stack.len() <= self.value_stack.len()); + + let a = self.assembler.as_mut().unwrap(); + + match ev { + Event::Internal(InternalEvent::FunctionBegin(_)) + | Event::Internal(InternalEvent::FunctionEnd) => { + return Ok(()); + } + _ => {} + } + + self.machine.state.wasm_inst_offset = self.machine.state.wasm_inst_offset.wrapping_add(1); + + //println!("{:?} {}", op, self.value_stack.len()); + let was_unreachable; + + if self.unreachable_depth > 0 { + was_unreachable = true; + + if let Event::Wasm(op) = ev { + match *op { + Operator::Block { .. } | Operator::Loop { .. } | Operator::If { .. } => { + self.unreachable_depth += 1; + } + Operator::End => { + self.unreachable_depth -= 1; + } + Operator::Else => { + // We are in a reachable true branch + if self.unreachable_depth == 1 { + if let Some(IfElseState::If(_)) = + self.control_stack.last().map(|x| x.if_else) + { + self.unreachable_depth -= 1; + } + } + } + _ => {} + } + } + if self.unreachable_depth > 0 { + return Ok(()); + } + } else { + was_unreachable = false; + } + + let op = match ev { + Event::Wasm(x) => x, + Event::WasmOwned(ref x) => x, + Event::Internal(x) => { + match x { + InternalEvent::Breakpoint(callback) => { + self.breakpoints + .as_mut() + .unwrap() + .insert(a.get_offset(), callback); + Self::mark_trappable( + a, + &self.machine, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_inline_breakpoint(InlineBreakpointType::Middleware); + } + InternalEvent::FunctionBegin(_) | InternalEvent::FunctionEnd => {} + InternalEvent::GetInternal(idx) => { + let idx = idx as usize; + if idx >= INTERNALS_SIZE { + return Err(CodegenError { + message: format!("GetInternal: incorrect index value"), + }); + } + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + // Load `internals` pointer. + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_internals() as i32, + ), + Location::GPR(tmp), + ); + + let loc = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(loc); + + // Move internal into the result location. + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::Memory(tmp, (idx * 8) as i32), + loc, + ); + + self.machine.release_temp_gpr(tmp); + } + InternalEvent::SetInternal(idx) => { + let idx = idx as usize; + if idx >= INTERNALS_SIZE { + return Err(CodegenError { + message: format!("SetInternal: incorrect index value"), + }); + } + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + // Load `internals` pointer. + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_internals() as i32, + ), + Location::GPR(tmp), + ); + let loc = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + + // Move internal into storage. + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::Memory(tmp, (idx * 8) as i32), + ); + self.machine.release_temp_gpr(tmp); + } //_ => unimplemented!(), + } + return Ok(()); + } + }; + + match *op { + Operator::GlobalGet { global_index } => { + let global_index = global_index as usize; + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + let loc = match GlobalIndex::new(global_index).local_or_import(module_info) { + LocalOrImport::Local(local_index) => { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_globals() as i32, + ), + Location::GPR(tmp), + ); + a.emit_mov( + Size::S64, + Location::Memory(tmp, (local_index.index() as i32) * 8), + Location::GPR(tmp), + ); + let ty = type_to_wp_type(module_info.globals[local_index].desc.ty); + if ty.is_float() { + self.fp_stack.push(FloatValue::new(self.value_stack.len())); + } + self.machine.acquire_locations( + a, + &[(ty, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0] + } + LocalOrImport::Import(import_index) => { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_imported_globals() as i32, + ), + Location::GPR(tmp), + ); + a.emit_mov( + Size::S64, + Location::Memory(tmp, (import_index.index() as i32) * 8), + Location::GPR(tmp), + ); + let ty = type_to_wp_type(module_info.imported_globals[import_index].1.ty); + if ty.is_float() { + self.fp_stack.push(FloatValue::new(self.value_stack.len())); + } + self.machine.acquire_locations( + a, + &[(ty, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0] + } + }; + self.value_stack.push(loc); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::Memory(tmp, LocalGlobal::offset_data() as i32), + loc, + ); + + self.machine.release_temp_gpr(tmp); + } + Operator::GlobalSet { global_index } => { + let mut global_index = global_index as usize; + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + let ty = if global_index < module_info.imported_globals.len() { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_imported_globals() as i32, + ), + Location::GPR(tmp), + ); + type_to_wp_type( + module_info.imported_globals[ImportedGlobalIndex::new(global_index)] + .1 + .ty, + ) + } else { + global_index -= module_info.imported_globals.len(); + if global_index >= module_info.globals.len() { + return Err(CodegenError { + message: format!("SetGlobal: incorrect global_index value"), + }); + } + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_globals() as i32, + ), + Location::GPR(tmp), + ); + type_to_wp_type( + module_info.globals[LocalGlobalIndex::new(global_index)] + .desc + .ty, + ) + }; + a.emit_mov( + Size::S64, + Location::Memory(tmp, (global_index as i32) * 8), + Location::GPR(tmp), + ); + if ty.is_float() { + let fp = self.fp_stack.pop1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match ty { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + Location::Memory(tmp, LocalGlobal::offset_data() as i32), + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::Memory(tmp, LocalGlobal::offset_data() as i32), + ); + } + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::Memory(tmp, LocalGlobal::offset_data() as i32), + ); + } + self.machine.release_temp_gpr(tmp); + } + Operator::LocalGet { local_index } => { + let local_index = local_index as usize; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + self.locals[local_index], + ret, + ); + self.value_stack.push(ret); + if self.local_types[local_index].is_float() { + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + } + } + Operator::LocalSet { local_index } => { + let local_index = local_index as usize; + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + if self.local_types[local_index].is_float() { + let fp = self.fp_stack.pop1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match self.local_types[local_index] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + self.locals[local_index], + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + self.locals[local_index], + ); + } + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + self.locals[local_index], + ); + } + } + Operator::LocalTee { local_index } => { + let local_index = local_index as usize; + let loc = *self.value_stack.last().unwrap(); + + if self.local_types[local_index].is_float() { + let fp = self.fp_stack.peek1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match self.local_types[local_index] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + self.locals[local_index], + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + self.locals[local_index], + ); + } + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + self.locals[local_index], + ); + } + } + Operator::I32Const { value } => { + self.value_stack.push(Location::Imm32(value as u32)); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value as u32 as u64)); + } + Operator::I32Add => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_add, + ), + Operator::I32Sub => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_sub, + ), + Operator::I32Mul => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_imul, + ), + Operator::I32DivU => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); + a.emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + Assembler::emit_div, + Size::S32, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S32, Location::GPR(GPR::RAX), ret); + self.value_stack.push(ret); + } + Operator::I32DivS => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); + a.emit_cdq(); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + Assembler::emit_idiv, + Size::S32, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S32, Location::GPR(GPR::RAX), ret); + self.value_stack.push(ret); + } + Operator::I32RemU => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); + a.emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + Assembler::emit_div, + Size::S32, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S32, Location::GPR(GPR::RDX), ret); + self.value_stack.push(ret); + } + Operator::I32RemS => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + + let normal_path = a.get_label(); + let end = a.get_label(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0x80000000), + loc_a, + ); + a.emit_jmp(Condition::NotEqual, normal_path); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0xffffffff), + loc_b, + ); + a.emit_jmp(Condition::NotEqual, normal_path); + a.emit_mov(Size::S32, Location::Imm32(0), ret); + a.emit_jmp(Condition::None, end); + + a.emit_label(normal_path); + a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); + a.emit_cdq(); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + Assembler::emit_idiv, + Size::S32, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S32, Location::GPR(GPR::RDX), ret); + self.value_stack.push(ret); + + a.emit_label(end); + } + Operator::I32And => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_and, + ), + Operator::I32Or => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_or, + ), + Operator::I32Xor => Self::emit_binop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_xor, + ), + Operator::I32Eq => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Equal, + )?, + Operator::I32Ne => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::NotEqual, + )?, + Operator::I32Eqz => Self::emit_cmpop_i32_dynamic_b( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Equal, + Location::Imm32(0), + )?, + Operator::I32Clz => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let src = match loc { + Location::Imm32(_) | Location::Memory(_, _) => { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + tmp + } + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I32Clz src: unreachable code"), + }) + } + }; + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let dst = match ret { + Location::Memory(_, _) => self.machine.acquire_temp_gpr().unwrap(), + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I32Clz dst: unreachable code"), + }) + } + }; + + if a.arch_has_xzcnt() { + a.arch_emit_lzcnt(Size::S32, Location::GPR(src), Location::GPR(dst)); + } else { + let zero_path = a.get_label(); + let end = a.get_label(); + + a.emit_test_gpr_64(src); + a.emit_jmp(Condition::Equal, zero_path); + a.emit_bsr(Size::S32, Location::GPR(src), Location::GPR(dst)); + a.emit_xor(Size::S32, Location::Imm32(31), Location::GPR(dst)); + a.emit_jmp(Condition::None, end); + a.emit_label(zero_path); + a.emit_mov(Size::S32, Location::Imm32(32), Location::GPR(dst)); + a.emit_label(end); + } + + match loc { + Location::Imm32(_) | Location::Memory(_, _) => { + self.machine.release_temp_gpr(src); + } + _ => {} + }; + match ret { + Location::Memory(_, _) => { + a.emit_mov(Size::S32, Location::GPR(dst), ret); + self.machine.release_temp_gpr(dst); + } + _ => {} + }; + } + Operator::I32Ctz => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let src = match loc { + Location::Imm32(_) | Location::Memory(_, _) => { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + tmp + } + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I32Ctz src: unreachable code"), + }) + } + }; + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let dst = match ret { + Location::Memory(_, _) => self.machine.acquire_temp_gpr().unwrap(), + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I32Ctz dst: unreachable code"), + }) + } + }; + + if a.arch_has_xzcnt() { + a.arch_emit_tzcnt(Size::S32, Location::GPR(src), Location::GPR(dst)); + } else { + let zero_path = a.get_label(); + let end = a.get_label(); + + a.emit_test_gpr_64(src); + a.emit_jmp(Condition::Equal, zero_path); + a.emit_bsf(Size::S32, Location::GPR(src), Location::GPR(dst)); + a.emit_jmp(Condition::None, end); + a.emit_label(zero_path); + a.emit_mov(Size::S32, Location::Imm32(32), Location::GPR(dst)); + a.emit_label(end); + } + + match loc { + Location::Imm32(_) | Location::Memory(_, _) => { + self.machine.release_temp_gpr(src); + } + _ => {} + }; + match ret { + Location::Memory(_, _) => { + a.emit_mov(Size::S32, Location::GPR(dst), ret); + self.machine.release_temp_gpr(dst); + } + _ => {} + }; + } + Operator::I32Popcnt => Self::emit_xcnt_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_popcnt, + )?, + Operator::I32Shl => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_shl, + ), + Operator::I32ShrU => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_shr, + ), + Operator::I32ShrS => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_sar, + ), + Operator::I32Rotl => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_rol, + ), + Operator::I32Rotr => Self::emit_shift_i32( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_ror, + ), + Operator::I32LtU => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Below, + )?, + Operator::I32LeU => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::BelowEqual, + )?, + Operator::I32GtU => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Above, + )?, + Operator::I32GeU => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::AboveEqual, + )?, + Operator::I32LtS => { + Self::emit_cmpop_i32(a, &mut self.machine, &mut self.value_stack, Condition::Less)?; + } + Operator::I32LeS => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::LessEqual, + )?, + Operator::I32GtS => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Greater, + )?, + Operator::I32GeS => Self::emit_cmpop_i32( + a, + &mut self.machine, + &mut self.value_stack, + Condition::GreaterEqual, + )?, + Operator::I64Const { value } => { + let value = value as u64; + self.value_stack.push(Location::Imm64(value)); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value)); + } + Operator::I64Add => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_add, + ), + Operator::I64Sub => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_sub, + ), + Operator::I64Mul => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_imul, + ), + Operator::I64DivU => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + a.emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + Assembler::emit_div, + Size::S64, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + self.value_stack.push(ret); + } + Operator::I64DivS => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + a.emit_cqo(); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + Assembler::emit_idiv, + Size::S64, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + self.value_stack.push(ret); + } + Operator::I64RemU => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + a.emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + Assembler::emit_div, + Size::S64, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); + self.value_stack.push(ret); + } + Operator::I64RemS => { + // We assume that RAX and RDX are temporary registers here. + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + + let normal_path = a.get_label(); + let end = a.get_label(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S64, + Location::Imm64(0x8000000000000000u64), + loc_a, + ); + a.emit_jmp(Condition::NotEqual, normal_path); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S64, + Location::Imm64(0xffffffffffffffffu64), + loc_b, + ); + a.emit_jmp(Condition::NotEqual, normal_path); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::Imm64(0), + ret, + ); + a.emit_jmp(Condition::None, end); + + a.emit_label(normal_path); + + a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + a.emit_cqo(); + Self::emit_relaxed_xdiv( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + Assembler::emit_idiv, + Size::S64, + loc_b, + &mut self.fsm, + &mut self.control_stack, + ); + a.emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); + self.value_stack.push(ret); + a.emit_label(end); + } + Operator::I64And => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_and, + ), + Operator::I64Or => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_or, + ), + Operator::I64Xor => Self::emit_binop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_xor, + ), + Operator::I64Eq => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Equal, + )?, + Operator::I64Ne => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::NotEqual, + )?, + Operator::I64Eqz => Self::emit_cmpop_i64_dynamic_b( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Equal, + Location::Imm64(0), + )?, + Operator::I64Clz => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let src = match loc { + Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + tmp + } + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I64Clz src: unreachable code"), + }) + } + }; + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let dst = match ret { + Location::Memory(_, _) => self.machine.acquire_temp_gpr().unwrap(), + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I64Clz dst: unreachable code"), + }) + } + }; + + if a.arch_has_xzcnt() { + a.arch_emit_lzcnt(Size::S64, Location::GPR(src), Location::GPR(dst)); + } else { + let zero_path = a.get_label(); + let end = a.get_label(); + + a.emit_test_gpr_64(src); + a.emit_jmp(Condition::Equal, zero_path); + a.emit_bsr(Size::S64, Location::GPR(src), Location::GPR(dst)); + a.emit_xor(Size::S64, Location::Imm32(63), Location::GPR(dst)); + a.emit_jmp(Condition::None, end); + a.emit_label(zero_path); + a.emit_mov(Size::S64, Location::Imm32(64), Location::GPR(dst)); + a.emit_label(end); + } + + match loc { + Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => { + self.machine.release_temp_gpr(src); + } + _ => {} + }; + match ret { + Location::Memory(_, _) => { + a.emit_mov(Size::S64, Location::GPR(dst), ret); + self.machine.release_temp_gpr(dst); + } + _ => {} + }; + } + Operator::I64Ctz => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let src = match loc { + Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + tmp + } + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I64Ctz src: unreachable code"), + }) + } + }; + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let dst = match ret { + Location::Memory(_, _) => self.machine.acquire_temp_gpr().unwrap(), + Location::GPR(reg) => reg, + _ => { + return Err(CodegenError { + message: format!("I64Ctz dst: unreachable code"), + }) + } + }; + + if a.arch_has_xzcnt() { + a.arch_emit_tzcnt(Size::S64, Location::GPR(src), Location::GPR(dst)); + } else { + let zero_path = a.get_label(); + let end = a.get_label(); + + a.emit_test_gpr_64(src); + a.emit_jmp(Condition::Equal, zero_path); + a.emit_bsf(Size::S64, Location::GPR(src), Location::GPR(dst)); + a.emit_jmp(Condition::None, end); + a.emit_label(zero_path); + a.emit_mov(Size::S64, Location::Imm32(64), Location::GPR(dst)); + a.emit_label(end); + } + + match loc { + Location::Imm64(_) | Location::Imm32(_) | Location::Memory(_, _) => { + self.machine.release_temp_gpr(src); + } + _ => {} + }; + match ret { + Location::Memory(_, _) => { + a.emit_mov(Size::S64, Location::GPR(dst), ret); + self.machine.release_temp_gpr(dst); + } + _ => {} + }; + } + Operator::I64Popcnt => Self::emit_xcnt_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_popcnt, + )?, + Operator::I64Shl => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_shl, + ), + Operator::I64ShrU => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_shr, + ), + Operator::I64ShrS => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_sar, + ), + Operator::I64Rotl => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_rol, + ), + Operator::I64Rotr => Self::emit_shift_i64( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_ror, + ), + Operator::I64LtU => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Below, + )?, + Operator::I64LeU => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::BelowEqual, + )?, + Operator::I64GtU => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Above, + )?, + Operator::I64GeU => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::AboveEqual, + )?, + Operator::I64LtS => { + Self::emit_cmpop_i64(a, &mut self.machine, &mut self.value_stack, Condition::Less)?; + } + Operator::I64LeS => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::LessEqual, + )?, + Operator::I64GtS => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::Greater, + )?, + Operator::I64GeS => Self::emit_cmpop_i64( + a, + &mut self.machine, + &mut self.value_stack, + Condition::GreaterEqual, + )?, + Operator::I64ExtendI32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + ret, + ); + } + Operator::I64ExtendI32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S32, + loc, + Size::S64, + ret, + )?; + } + Operator::I32Extend8S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S8, + loc, + Size::S32, + ret, + )?; + } + Operator::I32Extend16S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S16, + loc, + Size::S32, + ret, + )?; + } + Operator::I64Extend8S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S8, + loc, + Size::S64, + ret, + )?; + } + Operator::I64Extend16S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S16, + loc, + Size::S64, + ret, + )?; + } + Operator::I64Extend32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_relaxed_zx_sx( + a, + &mut self.machine, + Assembler::emit_movsx, + Size::S32, + loc, + Size::S64, + ret, + )?; + } + Operator::I32WrapI64 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + ret, + ); + } + + Operator::F32Const { value } => { + self.value_stack.push(Location::Imm32(value.bits())); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value.bits() as u64)); + } + Operator::F32Add => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 2)); + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vaddss, + )?; + } + Operator::F32Sub => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 2)); + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vsubss, + )? + } + Operator::F32Mul => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 2)); + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vmulss, + )? + } + Operator::F32Div => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 2)); + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vdivss, + )? + } + Operator::F32Max => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 2)); + if !a.arch_supports_canonicalize_nan() { + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vmaxss, + )?; + } else { + let src2 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let src1 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_xmm().unwrap(); + let tmp2 = self.machine.acquire_temp_xmm().unwrap(); + let tmpg1 = self.machine.acquire_temp_gpr().unwrap(); + let tmpg2 = self.machine.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("F32Max src1: unreachable code"), + }) + } + }; + let src2 = match src2 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + tmp2 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + _ => { + return Err(CodegenError { + message: format!("F32Max src2: unreachable code"), + }) + } + }; + + let tmp_xmm1 = XMM::XMM8; + let tmp_xmm2 = XMM::XMM9; + let tmp_xmm3 = XMM::XMM10; + + a.emit_mov(Size::S32, Location::XMM(src1), Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::XMM(src2), Location::GPR(tmpg2)); + a.emit_cmp(Size::S32, Location::GPR(tmpg2), Location::GPR(tmpg1)); + a.emit_vmaxss(src1, XMMOrMemory::XMM(src2), tmp_xmm1); + let label1 = a.get_label(); + let label2 = a.get_label(); + a.emit_jmp(Condition::NotEqual, label1); + a.emit_vmovaps(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2)); + a.emit_jmp(Condition::None, label2); + a.emit_label(label1); + a.emit_vxorps(tmp_xmm2, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm2); + a.emit_label(label2); + a.emit_vcmpeqss(src1, XMMOrMemory::XMM(src2), tmp_xmm3); + a.emit_vblendvps(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1); + a.emit_vcmpunordss(src1, XMMOrMemory::XMM(src2), src1); + // load float canonical nan + a.emit_mov( + Size::S64, + Location::Imm32(0x7FC0_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(src2)); + a.emit_vblendvps(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1); + match ret { + Location::XMM(x) => { + a.emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x)); + } + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(Size::S64, Location::XMM(src1), ret); + } + _ => { + return Err(CodegenError { + message: format!("F32Max ret: unreachable code"), + }) + } + } + + self.machine.release_temp_gpr(tmpg2); + self.machine.release_temp_gpr(tmpg1); + self.machine.release_temp_xmm(tmp2); + self.machine.release_temp_xmm(tmp1); + } + } + Operator::F32Min => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 2)); + if !a.arch_supports_canonicalize_nan() { + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vminss, + )?; + } else { + let src2 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let src1 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_xmm().unwrap(); + let tmp2 = self.machine.acquire_temp_xmm().unwrap(); + let tmpg1 = self.machine.acquire_temp_gpr().unwrap(); + let tmpg2 = self.machine.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("F32Min src1: unreachable code"), + }) + } + }; + let src2 = match src2 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + tmp2 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + _ => { + return Err(CodegenError { + message: format!("F32Min src2: unreachable code"), + }) + } + }; + + let tmp_xmm1 = XMM::XMM8; + let tmp_xmm2 = XMM::XMM9; + let tmp_xmm3 = XMM::XMM10; + + a.emit_mov(Size::S32, Location::XMM(src1), Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::XMM(src2), Location::GPR(tmpg2)); + a.emit_cmp(Size::S32, Location::GPR(tmpg2), Location::GPR(tmpg1)); + a.emit_vminss(src1, XMMOrMemory::XMM(src2), tmp_xmm1); + let label1 = a.get_label(); + let label2 = a.get_label(); + a.emit_jmp(Condition::NotEqual, label1); + a.emit_vmovaps(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2)); + a.emit_jmp(Condition::None, label2); + a.emit_label(label1); + // load float -0.0 + a.emit_mov( + Size::S64, + Location::Imm32(0x8000_0000), // Negative zero + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp_xmm2)); + a.emit_label(label2); + a.emit_vcmpeqss(src1, XMMOrMemory::XMM(src2), tmp_xmm3); + a.emit_vblendvps(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1); + a.emit_vcmpunordss(src1, XMMOrMemory::XMM(src2), src1); + // load float canonical nan + a.emit_mov( + Size::S64, + Location::Imm32(0x7FC0_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(src2)); + a.emit_vblendvps(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1); + match ret { + Location::XMM(x) => { + a.emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x)); + } + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(Size::S64, Location::XMM(src1), ret); + } + _ => { + return Err(CodegenError { + message: format!("F32Min ret: unreachable code"), + }) + } + } + + self.machine.release_temp_gpr(tmpg2); + self.machine.release_temp_gpr(tmpg1); + self.machine.release_temp_xmm(tmp2); + self.machine.release_temp_xmm(tmp1); + } + } + Operator::F32Eq => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpeqss, + )? + } + Operator::F32Ne => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpneqss, + )? + } + Operator::F32Lt => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpltss, + )? + } + Operator::F32Le => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpless, + )? + } + Operator::F32Gt => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpgtss, + )? + } + Operator::F32Ge => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpgess, + )? + } + Operator::F32Nearest => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundss_nearest, + )? + } + Operator::F32Floor => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundss_floor, + )? + } + Operator::F32Ceil => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundss_ceil, + )? + } + Operator::F32Trunc => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundss_trunc, + )? + } + Operator::F32Sqrt => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f32(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vsqrtss, + )? + } + + Operator::F32Copysign => { + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let (fp_src1, fp_src2) = self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + + let tmp1 = self.machine.acquire_temp_gpr().unwrap(); + let tmp2 = self.machine.acquire_temp_gpr().unwrap(); + + if a.arch_supports_canonicalize_nan() && self.config.nan_canonicalization { + for (fp, loc, tmp) in [(fp_src1, loc_a, tmp1), (fp_src2, loc_b, tmp2)].iter() { + match fp.canonicalization { + Some(_) => { + Self::canonicalize_nan( + a, + &mut self.machine, + Size::S32, + *loc, + Location::GPR(*tmp), + ); + } + None => { + a.emit_mov(Size::S32, *loc, Location::GPR(*tmp)); + } + } + } + } else { + a.emit_mov(Size::S32, loc_a, Location::GPR(tmp1)); + a.emit_mov(Size::S32, loc_b, Location::GPR(tmp2)); + } + a.emit_and( + Size::S32, + Location::Imm32(0x7fffffffu32), + Location::GPR(tmp1), + ); + a.emit_and( + Size::S32, + Location::Imm32(0x80000000u32), + Location::GPR(tmp2), + ); + a.emit_or(Size::S32, Location::GPR(tmp2), Location::GPR(tmp1)); + a.emit_mov(Size::S32, Location::GPR(tmp1), ret); + self.machine.release_temp_gpr(tmp2); + self.machine.release_temp_gpr(tmp1); + } + + Operator::F32Abs => { + // Preserve canonicalization state. + + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + a.emit_and( + Size::S32, + Location::Imm32(0x7fffffffu32), + Location::GPR(tmp), + ); + a.emit_mov(Size::S32, Location::GPR(tmp), ret); + self.machine.release_temp_gpr(tmp); + } + + Operator::F32Neg => { + // Preserve canonicalization state. + + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + if a.arch_has_fneg() { + let tmp = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp), + ); + a.arch_emit_f32_neg(tmp, tmp); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp), + ret, + ); + self.machine.release_temp_xmm(tmp); + } else { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(tmp)); + a.emit_btc_gpr_imm8_32(31, tmp); + a.emit_mov(Size::S32, Location::GPR(tmp), ret); + self.machine.release_temp_gpr(tmp); + } + } + + Operator::F64Const { value } => { + self.value_stack.push(Location::Imm64(value.bits())); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value.bits())); + } + Operator::F64Add => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 2)); + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vaddsd, + )? + } + Operator::F64Sub => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 2)); + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vsubsd, + )? + } + Operator::F64Mul => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 2)); + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vmulsd, + )? + } + Operator::F64Div => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 2)); + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vdivsd, + )? + } + Operator::F64Max => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 2)); + + if !a.arch_supports_canonicalize_nan() { + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vmaxsd, + )?; + } else { + let src2 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let src1 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_xmm().unwrap(); + let tmp2 = self.machine.acquire_temp_xmm().unwrap(); + let tmpg1 = self.machine.acquire_temp_gpr().unwrap(); + let tmpg2 = self.machine.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("F64Max src1: unreachable code"), + }) + } + }; + let src2 = match src2 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + tmp2 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + _ => { + return Err(CodegenError { + message: format!("F64Max src2: unreachable code"), + }) + } + }; + + let tmp_xmm1 = XMM::XMM8; + let tmp_xmm2 = XMM::XMM9; + let tmp_xmm3 = XMM::XMM10; + + a.emit_mov(Size::S64, Location::XMM(src1), Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::XMM(src2), Location::GPR(tmpg2)); + a.emit_cmp(Size::S64, Location::GPR(tmpg2), Location::GPR(tmpg1)); + a.emit_vmaxsd(src1, XMMOrMemory::XMM(src2), tmp_xmm1); + let label1 = a.get_label(); + let label2 = a.get_label(); + a.emit_jmp(Condition::NotEqual, label1); + a.emit_vmovapd(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2)); + a.emit_jmp(Condition::None, label2); + a.emit_label(label1); + a.emit_vxorpd(tmp_xmm2, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm2); + a.emit_label(label2); + a.emit_vcmpeqsd(src1, XMMOrMemory::XMM(src2), tmp_xmm3); + a.emit_vblendvpd(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1); + a.emit_vcmpunordsd(src1, XMMOrMemory::XMM(src2), src1); + // load float canonical nan + a.emit_mov( + Size::S64, + Location::Imm64(0x7FF8_0000_0000_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(src2)); + a.emit_vblendvpd(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1); + match ret { + Location::XMM(x) => { + a.emit_vmovapd(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x)); + } + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(Size::S64, Location::XMM(src1), ret); + } + _ => { + return Err(CodegenError { + message: format!("F64Max ret: unreachable code"), + }) + } + } + + self.machine.release_temp_gpr(tmpg2); + self.machine.release_temp_gpr(tmpg1); + self.machine.release_temp_xmm(tmp2); + self.machine.release_temp_xmm(tmp1); + } + } + Operator::F64Min => { + self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 2)); + + if !a.arch_supports_canonicalize_nan() { + Self::emit_fp_binop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vminsd, + )?; + } else { + let src2 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let src1 = get_location_released( + a, + &mut self.machine, + self.value_stack.pop().unwrap(), + ); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp1 = self.machine.acquire_temp_xmm().unwrap(); + let tmp2 = self.machine.acquire_temp_xmm().unwrap(); + let tmpg1 = self.machine.acquire_temp_gpr().unwrap(); + let tmpg2 = self.machine.acquire_temp_gpr().unwrap(); + + let src1 = match src1 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src1, Location::XMM(tmp1)); + tmp1 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src1, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp1)); + tmp1 + } + _ => { + return Err(CodegenError { + message: format!("F64Min src1: unreachable code"), + }) + } + }; + let src2 = match src2 { + Location::XMM(x) => x, + Location::GPR(_) | Location::Memory(_, _) => { + a.emit_mov(Size::S64, src2, Location::XMM(tmp2)); + tmp2 + } + Location::Imm32(_) => { + a.emit_mov(Size::S32, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S32, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + Location::Imm64(_) => { + a.emit_mov(Size::S64, src2, Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp2)); + tmp2 + } + _ => { + return Err(CodegenError { + message: format!("F64Min src2: unreachable code"), + }) + } + }; + + let tmp_xmm1 = XMM::XMM8; + let tmp_xmm2 = XMM::XMM9; + let tmp_xmm3 = XMM::XMM10; + + a.emit_mov(Size::S64, Location::XMM(src1), Location::GPR(tmpg1)); + a.emit_mov(Size::S64, Location::XMM(src2), Location::GPR(tmpg2)); + a.emit_cmp(Size::S64, Location::GPR(tmpg2), Location::GPR(tmpg1)); + a.emit_vminsd(src1, XMMOrMemory::XMM(src2), tmp_xmm1); + let label1 = a.get_label(); + let label2 = a.get_label(); + a.emit_jmp(Condition::NotEqual, label1); + a.emit_vmovapd(XMMOrMemory::XMM(tmp_xmm1), XMMOrMemory::XMM(tmp_xmm2)); + a.emit_jmp(Condition::None, label2); + a.emit_label(label1); + // load float -0.0 + a.emit_mov( + Size::S64, + Location::Imm64(0x8000_0000_0000_0000), // Negative zero + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(tmp_xmm2)); + a.emit_label(label2); + a.emit_vcmpeqsd(src1, XMMOrMemory::XMM(src2), tmp_xmm3); + a.emit_vblendvpd(tmp_xmm3, XMMOrMemory::XMM(tmp_xmm2), tmp_xmm1, tmp_xmm1); + a.emit_vcmpunordsd(src1, XMMOrMemory::XMM(src2), src1); + // load float canonical nan + a.emit_mov( + Size::S64, + Location::Imm64(0x7FF8_0000_0000_0000), // Canonical NaN + Location::GPR(tmpg1), + ); + a.emit_mov(Size::S64, Location::GPR(tmpg1), Location::XMM(src2)); + a.emit_vblendvpd(src1, XMMOrMemory::XMM(src2), tmp_xmm1, src1); + match ret { + Location::XMM(x) => { + a.emit_vmovaps(XMMOrMemory::XMM(src1), XMMOrMemory::XMM(x)); + } + Location::Memory(_, _) | Location::GPR(_) => { + a.emit_mov(Size::S64, Location::XMM(src1), ret); + } + _ => { + return Err(CodegenError { + message: format!("F64Min ret: unreachable code"), + }) + } + } + + self.machine.release_temp_gpr(tmpg2); + self.machine.release_temp_gpr(tmpg1); + self.machine.release_temp_xmm(tmp2); + self.machine.release_temp_xmm(tmp1); + } + } + Operator::F64Eq => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpeqsd, + )? + } + Operator::F64Ne => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpneqsd, + )? + } + Operator::F64Lt => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpltsd, + )? + } + Operator::F64Le => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmplesd, + )? + } + Operator::F64Gt => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpgtsd, + )? + } + Operator::F64Ge => { + self.fp_stack.pop2()?; + Self::emit_fp_cmpop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcmpgesd, + )? + } + Operator::F64Nearest => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundsd_nearest, + )? + } + Operator::F64Floor => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundsd_floor, + )? + } + Operator::F64Ceil => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundsd_ceil, + )? + } + Operator::F64Trunc => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vroundsd_trunc, + )? + } + Operator::F64Sqrt => { + self.fp_stack.pop1()?; + self.fp_stack + .push(FloatValue::cncl_f64(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vsqrtsd, + )? + } + + Operator::F64Copysign => { + let loc_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let (fp_src1, fp_src2) = self.fp_stack.pop2()?; + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + + let tmp1 = self.machine.acquire_temp_gpr().unwrap(); + let tmp2 = self.machine.acquire_temp_gpr().unwrap(); + + if a.arch_supports_canonicalize_nan() && self.config.nan_canonicalization { + for (fp, loc, tmp) in [(fp_src1, loc_a, tmp1), (fp_src2, loc_b, tmp2)].iter() { + match fp.canonicalization { + Some(_) => { + Self::canonicalize_nan( + a, + &mut self.machine, + Size::S64, + *loc, + Location::GPR(*tmp), + ); + } + None => { + a.emit_mov(Size::S64, *loc, Location::GPR(*tmp)); + } + } + } + } else { + a.emit_mov(Size::S64, loc_a, Location::GPR(tmp1)); + a.emit_mov(Size::S64, loc_b, Location::GPR(tmp2)); + } + + let c = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov( + Size::S64, + Location::Imm64(0x7fffffffffffffffu64), + Location::GPR(c), + ); + a.emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp1)); + + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(c), + ); + a.emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp2)); + + a.emit_or(Size::S64, Location::GPR(tmp2), Location::GPR(tmp1)); + a.emit_mov(Size::S64, Location::GPR(tmp1), ret); + + self.machine.release_temp_gpr(c); + self.machine.release_temp_gpr(tmp2); + self.machine.release_temp_gpr(tmp1); + } + + Operator::F64Abs => { + // Preserve canonicalization state. + + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let tmp = self.machine.acquire_temp_gpr().unwrap(); + let c = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + a.emit_mov( + Size::S64, + Location::Imm64(0x7fffffffffffffffu64), + Location::GPR(c), + ); + a.emit_and(Size::S64, Location::GPR(c), Location::GPR(tmp)); + a.emit_mov(Size::S64, Location::GPR(tmp), ret); + + self.machine.release_temp_gpr(c); + self.machine.release_temp_gpr(tmp); + } + + Operator::F64Neg => { + // Preserve canonicalization state. + + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + if a.arch_has_fneg() { + let tmp = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp), + ); + a.arch_emit_f64_neg(tmp, tmp); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp), + ret, + ); + self.machine.release_temp_xmm(tmp); + } else { + let tmp = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(tmp)); + a.emit_btc_gpr_imm8_64(63, tmp); + a.emit_mov(Size::S64, Location::GPR(tmp), ret); + self.machine.release_temp_gpr(tmp); + } + } + + Operator::F64PromoteF32 => { + let fp = self.fp_stack.pop1()?; + self.fp_stack.push(fp.promote(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcvtss2sd, + )? + } + Operator::F32DemoteF64 => { + let fp = self.fp_stack.pop1()?; + self.fp_stack.push(fp.demote(self.value_stack.len() - 1)); + Self::emit_fp_unop_avx( + a, + &mut self.machine, + &mut self.value_stack, + Assembler::emit_vcvtsd2ss, + )? + } + + Operator::I32ReinterpretF32 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + let fp = self.fp_stack.pop1()?; + + if !a.arch_supports_canonicalize_nan() + || !self.config.nan_canonicalization + || fp.canonicalization.is_none() + { + if loc != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + ret, + ); + } + } else { + Self::canonicalize_nan(a, &mut self.machine, Size::S32, loc, ret); + } + } + Operator::F32ReinterpretI32 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + + if loc != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + ret, + ); + } + } + + Operator::I64ReinterpretF64 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + let fp = self.fp_stack.pop1()?; + + if !a.arch_supports_canonicalize_nan() + || !self.config.nan_canonicalization + || fp.canonicalization.is_none() + { + if loc != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + ret, + ); + } + } else { + Self::canonicalize_nan(a, &mut self.machine, Size::S64, loc, ret); + } + } + Operator::F64ReinterpretI64 => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + + if loc != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + ret, + ); + } + } + + Operator::I32TruncF32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i32_trunc_uf32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_trap( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + tmp_in, + GEF32_LT_U32_MIN, + LEF32_GT_U32_MAX, + ); + + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I32TruncSatF32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF32_LT_U32_MIN, + LEF32_GT_U32_MAX, + |a, _m| { + a.emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out)); + }, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::u32::MAX), + Location::GPR(tmp_out), + ); + }, + None::, + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i32_trunc_uf32(tmp_in, tmp_out); + } else { + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I32TruncF32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i32_trunc_sf32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_trap( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + tmp_in, + GEF32_LT_I32_MIN, + LEF32_GT_I32_MAX, + ); + + a.emit_cvttss2si_32(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + Operator::I32TruncSatF32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF32_LT_I32_MIN, + LEF32_GT_I32_MAX, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::i32::MIN as u32), + Location::GPR(tmp_out), + ); + }, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::i32::MAX as u32), + Location::GPR(tmp_out), + ); + }, + Some(|a: &mut Assembler, _m: &mut Machine| { + a.emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out)); + }), + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i32_trunc_sf32(tmp_in, tmp_out); + } else { + a.emit_cvttss2si_32(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I64TruncF32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i64_trunc_sf32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_trap( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + tmp_in, + GEF32_LT_I64_MIN, + LEF32_GT_I64_MAX, + ); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I64TruncSatF32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF32_LT_I64_MIN, + LEF32_GT_I64_MAX, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::i64::MIN as u64), + Location::GPR(tmp_out), + ); + }, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::i64::MAX as u64), + Location::GPR(tmp_out), + ); + }, + Some(|a: &mut Assembler, _m: &mut Machine| { + a.emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out)); + }), + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i64_trunc_sf32(tmp_in, tmp_out); + } else { + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I64TruncF32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i64_trunc_uf32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); // xmm2 + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_trap( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + tmp_in, + GEF32_LT_U64_MIN, + LEF32_GT_U64_MAX, + ); + + let tmp = self.machine.acquire_temp_gpr().unwrap(); // r15 + let tmp_x1 = self.machine.acquire_temp_xmm().unwrap(); // xmm1 + let tmp_x2 = self.machine.acquire_temp_xmm().unwrap(); // xmm3 + + a.emit_mov( + Size::S32, + Location::Imm32(1593835520u32), + Location::GPR(tmp), + ); //float 9.22337203E+18 + a.emit_mov(Size::S32, Location::GPR(tmp), Location::XMM(tmp_x1)); + a.emit_mov(Size::S32, Location::XMM(tmp_in), Location::XMM(tmp_x2)); + a.emit_vsubss(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(tmp), + ); + a.emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp)); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out); + a.emit_ucomiss(XMMOrMemory::XMM(tmp_x1), tmp_x2); + a.emit_cmovae_gpr_64(tmp, tmp_out); + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_x2); + self.machine.release_temp_xmm(tmp_x1); + self.machine.release_temp_gpr(tmp); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + Operator::I64TruncSatF32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f32_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF32_LT_U64_MIN, + LEF32_GT_U64_MAX, + |a, _m| { + a.emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out)); + }, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::u64::MAX), + Location::GPR(tmp_out), + ); + }, + None::, + |a, m| { + if a.arch_has_itruncf() { + a.arch_emit_i64_trunc_uf32(tmp_in, tmp_out); + } else { + let tmp = m.acquire_temp_gpr().unwrap(); + let tmp_x1 = m.acquire_temp_xmm().unwrap(); + let tmp_x2 = m.acquire_temp_xmm().unwrap(); + + a.emit_mov( + Size::S32, + Location::Imm32(1593835520u32), + Location::GPR(tmp), + ); //float 9.22337203E+18 + a.emit_mov(Size::S32, Location::GPR(tmp), Location::XMM(tmp_x1)); + a.emit_mov(Size::S32, Location::XMM(tmp_in), Location::XMM(tmp_x2)); + a.emit_vsubss(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(tmp), + ); + a.emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp)); + a.emit_cvttss2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out); + a.emit_ucomiss(XMMOrMemory::XMM(tmp_x1), tmp_x2); + a.emit_cmovae_gpr_64(tmp, tmp_out); + + m.release_temp_xmm(tmp_x2); + m.release_temp_xmm(tmp_x1); + m.release_temp_gpr(tmp); + } + }, + ); + + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I32TruncF64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i32_trunc_uf64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_trap( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + tmp_in, + GEF64_LT_U32_MIN, + LEF64_GT_U32_MAX, + ); + + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I32TruncSatF64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF64_LT_U32_MIN, + LEF64_GT_U32_MAX, + |a, _m| { + a.emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out)); + }, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::u32::MAX), + Location::GPR(tmp_out), + ); + }, + None::, + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i32_trunc_uf64(tmp_in, tmp_out); + } else { + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I32TruncF64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i32_trunc_sf64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + let real_in = match loc { + Location::Imm32(_) | Location::Imm64(_) => { + a.emit_mov(Size::S64, loc, Location::GPR(tmp_out)); + a.emit_mov(Size::S64, Location::GPR(tmp_out), Location::XMM(tmp_in)); + tmp_in + } + Location::XMM(x) => x, + _ => { + a.emit_mov(Size::S64, loc, Location::XMM(tmp_in)); + tmp_in + } + }; + + Self::emit_f64_int_conv_check_trap( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + real_in, + GEF64_LT_I32_MIN, + LEF64_GT_I32_MAX, + ); + + a.emit_cvttsd2si_32(XMMOrMemory::XMM(real_in), tmp_out); + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I32TruncSatF64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + let real_in = match loc { + Location::Imm32(_) | Location::Imm64(_) => { + a.emit_mov(Size::S64, loc, Location::GPR(tmp_out)); + a.emit_mov(Size::S64, Location::GPR(tmp_out), Location::XMM(tmp_in)); + tmp_in + } + Location::XMM(x) => x, + _ => { + a.emit_mov(Size::S64, loc, Location::XMM(tmp_in)); + tmp_in + } + }; + + Self::emit_f64_int_conv_check_sat( + a, + &mut self.machine, + real_in, + GEF64_LT_I32_MIN, + LEF64_GT_I32_MAX, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::i32::MIN as u32), + Location::GPR(tmp_out), + ); + }, + |a, _m| { + a.emit_mov( + Size::S32, + Location::Imm32(std::i32::MAX as u32), + Location::GPR(tmp_out), + ); + }, + Some(|a: &mut Assembler, _m: &mut Machine| { + a.emit_mov(Size::S32, Location::Imm32(0), Location::GPR(tmp_out)); + }), + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i32_trunc_sf64(tmp_in, tmp_out); + } else { + a.emit_cvttsd2si_32(XMMOrMemory::XMM(real_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S32, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I64TruncF64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i64_trunc_sf64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_trap( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + tmp_in, + GEF64_LT_I64_MIN, + LEF64_GT_I64_MAX, + ); + + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I64TruncSatF64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF64_LT_I64_MIN, + LEF64_GT_I64_MAX, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::i64::MIN as u64), + Location::GPR(tmp_out), + ); + }, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::i64::MAX as u64), + Location::GPR(tmp_out), + ); + }, + Some(|a: &mut Assembler, _m: &mut Machine| { + a.emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out)); + }), + |a, _m| { + if a.arch_has_itruncf() { + a.arch_emit_i64_trunc_sf64(tmp_in, tmp_out); + } else { + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + } + }, + ); + + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::I64TruncF64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + if a.arch_has_itruncf() { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + a.arch_emit_i64_trunc_uf64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::GPR(tmp_out), + ret, + ); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); // xmm2 + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_trap( + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + tmp_in, + GEF64_LT_U64_MIN, + LEF64_GT_U64_MAX, + ); + + let tmp = self.machine.acquire_temp_gpr().unwrap(); // r15 + let tmp_x1 = self.machine.acquire_temp_xmm().unwrap(); // xmm1 + let tmp_x2 = self.machine.acquire_temp_xmm().unwrap(); // xmm3 + + a.emit_mov( + Size::S64, + Location::Imm64(4890909195324358656u64), + Location::GPR(tmp), + ); //double 9.2233720368547758E+18 + a.emit_mov(Size::S64, Location::GPR(tmp), Location::XMM(tmp_x1)); + a.emit_mov(Size::S64, Location::XMM(tmp_in), Location::XMM(tmp_x2)); + a.emit_vsubsd(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in); + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(tmp), + ); + a.emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp)); + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out); + a.emit_ucomisd(XMMOrMemory::XMM(tmp_x1), tmp_x2); + a.emit_cmovae_gpr_64(tmp, tmp_out); + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + + self.machine.release_temp_xmm(tmp_x2); + self.machine.release_temp_xmm(tmp_x1); + self.machine.release_temp_gpr(tmp); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + } + + Operator::I64TruncSatF64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack.pop1()?; + + let tmp_out = self.machine.acquire_temp_gpr().unwrap(); + let tmp_in = self.machine.acquire_temp_xmm().unwrap(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::XMM(tmp_in), + ); + Self::emit_f64_int_conv_check_sat( + a, + &mut self.machine, + tmp_in, + GEF64_LT_U64_MIN, + LEF64_GT_U64_MAX, + |a, _m| { + a.emit_mov(Size::S64, Location::Imm64(0), Location::GPR(tmp_out)); + }, + |a, _m| { + a.emit_mov( + Size::S64, + Location::Imm64(std::u64::MAX), + Location::GPR(tmp_out), + ); + }, + None::, + |a, m| { + if a.arch_has_itruncf() { + a.arch_emit_i64_trunc_uf64(tmp_in, tmp_out); + } else { + let tmp = m.acquire_temp_gpr().unwrap(); + let tmp_x1 = m.acquire_temp_xmm().unwrap(); + let tmp_x2 = m.acquire_temp_xmm().unwrap(); + + a.emit_mov( + Size::S64, + Location::Imm64(4890909195324358656u64), + Location::GPR(tmp), + ); //double 9.2233720368547758E+18 + a.emit_mov(Size::S64, Location::GPR(tmp), Location::XMM(tmp_x1)); + a.emit_mov(Size::S64, Location::XMM(tmp_in), Location::XMM(tmp_x2)); + a.emit_vsubsd(tmp_in, XMMOrMemory::XMM(tmp_x1), tmp_in); + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_in), tmp_out); + a.emit_mov( + Size::S64, + Location::Imm64(0x8000000000000000u64), + Location::GPR(tmp), + ); + a.emit_xor(Size::S64, Location::GPR(tmp_out), Location::GPR(tmp)); + a.emit_cvttsd2si_64(XMMOrMemory::XMM(tmp_x2), tmp_out); + a.emit_ucomisd(XMMOrMemory::XMM(tmp_x1), tmp_x2); + a.emit_cmovae_gpr_64(tmp, tmp_out); + + m.release_temp_xmm(tmp_x2); + m.release_temp_xmm(tmp_x1); + m.release_temp_gpr(tmp); + } + }, + ); + + a.emit_mov(Size::S64, Location::GPR(tmp_out), ret); + self.machine.release_temp_xmm(tmp_in); + self.machine.release_temp_gpr(tmp_out); + } + + Operator::F32ConvertI32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i32 to f32 never results in NaN. + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f32_convert_si32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S32, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2ss_32(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F32ConvertI32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i32 to f32 never results in NaN. + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f32_convert_ui32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S32, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F32ConvertI64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i64 to f32 never results in NaN. + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f32_convert_si64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S32, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F32ConvertI64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i64 to f32 never results in NaN. + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f32_convert_ui64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + let do_convert = a.get_label(); + let end_convert = a.get_label(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp_in)); + a.emit_test_gpr_64(tmp_in); + a.emit_jmp(Condition::Signed, do_convert); + a.emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_jmp(Condition::None, end_convert); + a.emit_label(do_convert); + a.emit_mov(Size::S64, Location::GPR(tmp_in), Location::GPR(tmp)); + a.emit_and(Size::S64, Location::Imm32(1), Location::GPR(tmp)); + a.emit_shr(Size::S64, Location::Imm8(1), Location::GPR(tmp_in)); + a.emit_or(Size::S64, Location::GPR(tmp), Location::GPR(tmp_in)); + a.emit_vcvtsi2ss_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_vaddss(tmp_out, XMMOrMemory::XMM(tmp_out), tmp_out); + a.emit_label(end_convert); + a.emit_mov(Size::S32, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + + Operator::F64ConvertI32S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i32 to f64 never results in NaN. + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f64_convert_si32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S32, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2sd_32(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F64ConvertI32U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i32 to f64 never results in NaN. + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S32, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f64_convert_ui32(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S32, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F64ConvertI64S => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i64 to f64 never results in NaN. + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f64_convert_si64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp_in)); + a.emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_mov(Size::S64, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + Operator::F64ConvertI64U => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); // Converting i64 to f64 never results in NaN. + + if a.arch_has_fconverti() { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(tmp_in), + ); + a.arch_emit_f64_convert_ui64(tmp_in, tmp_out); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + Location::XMM(tmp_out), + ret, + ); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } else { + let tmp_out = self.machine.acquire_temp_xmm().unwrap(); + let tmp_in = self.machine.acquire_temp_gpr().unwrap(); + let tmp = self.machine.acquire_temp_gpr().unwrap(); + + let do_convert = a.get_label(); + let end_convert = a.get_label(); + + a.emit_mov(Size::S64, loc, Location::GPR(tmp_in)); + a.emit_test_gpr_64(tmp_in); + a.emit_jmp(Condition::Signed, do_convert); + a.emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_jmp(Condition::None, end_convert); + a.emit_label(do_convert); + a.emit_mov(Size::S64, Location::GPR(tmp_in), Location::GPR(tmp)); + a.emit_and(Size::S64, Location::Imm32(1), Location::GPR(tmp)); + a.emit_shr(Size::S64, Location::Imm8(1), Location::GPR(tmp_in)); + a.emit_or(Size::S64, Location::GPR(tmp), Location::GPR(tmp_in)); + a.emit_vcvtsi2sd_64(tmp_out, GPROrMemory::GPR(tmp_in), tmp_out); + a.emit_vaddsd(tmp_out, XMMOrMemory::XMM(tmp_out), tmp_out); + a.emit_label(end_convert); + a.emit_mov(Size::S64, Location::XMM(tmp_out), ret); + + self.machine.release_temp_gpr(tmp); + self.machine.release_temp_gpr(tmp_in); + self.machine.release_temp_xmm(tmp_out); + } + } + + Operator::Call { function_index } => { + let function_index = function_index as usize; + let label = self + .function_labels + .as_mut() + .unwrap() + .entry(function_index) + .or_insert_with(|| (a.get_label(), None)) + .0; + let sig_index = *self + .function_signatures + .get(FuncIndex::new(function_index)) + .unwrap(); + let sig = self.signatures.get(sig_index).unwrap(); + let param_types: SmallVec<[WpType; 8]> = + sig.params().iter().cloned().map(type_to_wp_type).collect(); + let return_types: SmallVec<[WpType; 1]> = + sig.returns().iter().cloned().map(type_to_wp_type).collect(); + + let params: SmallVec<[_; 8]> = self + .value_stack + .drain(self.value_stack.len() - param_types.len()..) + .collect(); + self.machine.release_locations_only_regs(¶ms); + + self.machine.release_locations_only_osr_state(params.len()); + + // Pop arguments off the FP stack and canonicalize them if needed. + // + // Canonicalization state will be lost across function calls, so early canonicalization + // is necessary here. + while let Some(fp) = self.fp_stack.last() { + if fp.depth >= self.value_stack.len() { + let index = fp.depth - self.value_stack.len(); + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + fp.canonicalization.unwrap().to_size(), + params[index], + params[index], + ); + } + self.fp_stack.pop().unwrap(); + } else { + break; + } + } + + Self::emit_call_sysv_label( + a, + &mut self.machine, + label, + params.iter().map(|x| *x), + Some((&mut self.fsm, &mut self.control_stack)), + )?; + + self.machine.release_locations_only_stack(a, ¶ms); + + if return_types.len() > 0 { + let ret = self.machine.acquire_locations( + a, + &[( + return_types[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; + self.value_stack.push(ret); + if return_types[0].is_float() { + a.emit_mov(Size::S64, Location::XMM(XMM::XMM0), ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + } else { + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + } + } + } + Operator::CallIndirect { index, table_index } => { + if table_index != 0 { + return Err(CodegenError { + message: format!("CallIndirect: table_index is not 0"), + }); + } + let sig = self.signatures.get(SigIndex::new(index as usize)).unwrap(); + let param_types: SmallVec<[WpType; 8]> = + sig.params().iter().cloned().map(type_to_wp_type).collect(); + let return_types: SmallVec<[WpType; 1]> = + sig.returns().iter().cloned().map(type_to_wp_type).collect(); + + let func_index = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + let params: SmallVec<[_; 8]> = self + .value_stack + .drain(self.value_stack.len() - param_types.len()..) + .collect(); + self.machine.release_locations_only_regs(¶ms); + + // Pop arguments off the FP stack and canonicalize them if needed. + // + // Canonicalization state will be lost across function calls, so early canonicalization + // is necessary here. + while let Some(fp) = self.fp_stack.last() { + if fp.depth >= self.value_stack.len() { + let index = fp.depth - self.value_stack.len(); + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + fp.canonicalization.unwrap().to_size(), + params[index], + params[index], + ); + } + self.fp_stack.pop().unwrap(); + } else { + break; + } + } + + let table_base = self.machine.acquire_temp_gpr().unwrap(); + let table_count = self.machine.acquire_temp_gpr().unwrap(); + let sigidx = self.machine.acquire_temp_gpr().unwrap(); + + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + match TableIndex::new(0).local_or_import(module_info) { + LocalOrImport::Local(_) => vm::Ctx::offset_tables(), + LocalOrImport::Import(_) => vm::Ctx::offset_imported_tables(), + } as i32, + ), + Location::GPR(table_base), + ); + a.emit_mov( + Size::S64, + Location::Memory(table_base, 0), + Location::GPR(table_base), + ); + a.emit_mov( + Size::S32, + Location::Memory(table_base, LocalTable::offset_count() as i32), + Location::GPR(table_count), + ); + a.emit_mov( + Size::S64, + Location::Memory(table_base, LocalTable::offset_base() as i32), + Location::GPR(table_base), + ); + a.emit_cmp(Size::S32, func_index, Location::GPR(table_count)); + Self::mark_range_with_exception_code( + a, + self.exception_table.as_mut().unwrap(), + ExceptionCode::CallIndirectOOB, + |a| a.emit_conditional_trap(Condition::BelowEqual), + ); + a.emit_mov(Size::S32, func_index, Location::GPR(table_count)); + a.emit_imul_imm32_gpr64(vm::Anyfunc::size() as u32, table_count); + a.emit_add( + Size::S64, + Location::GPR(table_base), + Location::GPR(table_count), + ); + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_signatures() as i32, + ), + Location::GPR(sigidx), + ); + a.emit_mov( + Size::S32, + Location::Memory(sigidx, (index * 4) as i32), + Location::GPR(sigidx), + ); + a.emit_cmp( + Size::S32, + Location::GPR(sigidx), + Location::Memory(table_count, (vm::Anyfunc::offset_sig_id() as usize) as i32), + ); + Self::mark_range_with_exception_code( + a, + self.exception_table.as_mut().unwrap(), + ExceptionCode::IncorrectCallIndirectSignature, + |a| a.emit_conditional_trap(Condition::NotEqual), + ); + + self.machine.release_temp_gpr(sigidx); + self.machine.release_temp_gpr(table_count); + self.machine.release_temp_gpr(table_base); + + if table_count != GPR::RAX { + a.emit_mov( + Size::S64, + Location::GPR(table_count), + Location::GPR(GPR::RAX), + ); + } + + self.machine.release_locations_only_osr_state(params.len()); + + Self::emit_call_sysv( + a, + &mut self.machine, + |a| { + if a.arch_requires_indirect_call_trampoline() { + a.arch_emit_indirect_call_with_trampoline(Location::Memory( + GPR::RAX, + (vm::Anyfunc::offset_func() as usize) as i32, + )); + } else { + a.emit_call_location(Location::Memory( + GPR::RAX, + (vm::Anyfunc::offset_func() as usize) as i32, + )); + } + }, + params.iter().map(|x| *x), + Some((&mut self.fsm, &mut self.control_stack)), + )?; + + self.machine.release_locations_only_stack(a, ¶ms); + + if return_types.len() > 0 { + let ret = self.machine.acquire_locations( + a, + &[( + return_types[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; + self.value_stack.push(ret); + if return_types[0].is_float() { + a.emit_mov(Size::S64, Location::XMM(XMM::XMM0), ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + } else { + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + } + } + } + Operator::If { ty } => { + let label_end = a.get_label(); + let label_else = a.get_label(); + + let cond = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + let frame = ControlFrame { + label: label_end, + loop_like: false, + if_else: IfElseState::If(label_else), + returns: match ty { + WpTypeOrFuncType::Type(WpType::EmptyBlockType) => smallvec![], + WpTypeOrFuncType::Type(inner_ty) => smallvec![inner_ty], + _ => { + return Err(CodegenError { + message: format!("If: multi-value returns not yet implemented"), + }) + } + }, + value_stack_depth: self.value_stack.len(), + fp_stack_depth: self.fp_stack.len(), + state: self.machine.state.clone(), + state_diff_id: Self::get_state_diff( + &self.machine, + &mut self.fsm, + &mut self.control_stack, + ), + }; + self.control_stack.push(frame); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0), + cond, + ); + a.emit_jmp(Condition::Equal, label_else); + } + Operator::Else => { + let mut frame = self.control_stack.last_mut().unwrap(); + + if !was_unreachable && frame.returns.len() > 0 { + let loc = *self.value_stack.last().unwrap(); + if frame.returns[0].is_float() { + let fp = self.fp_stack.peek1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match frame.returns[0] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + Location::GPR(GPR::RAX), + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } + + let released: &[Location] = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations(a, released); + self.value_stack.truncate(frame.value_stack_depth); + self.fp_stack.truncate(frame.fp_stack_depth); + + match frame.if_else { + IfElseState::If(label) => { + a.emit_jmp(Condition::None, frame.label); + a.emit_label(label); + frame.if_else = IfElseState::Else; + } + _ => { + return Err(CodegenError { + message: format!("Else: frame.if_else unreachable code"), + }) + } + } + } + Operator::Select => { + let cond = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let v_b = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let v_a = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cncl: Option<(Option, Option)> = + if self.fp_stack.len() >= 2 + && self.fp_stack[self.fp_stack.len() - 2].depth == self.value_stack.len() + && self.fp_stack[self.fp_stack.len() - 1].depth + == self.value_stack.len() + 1 + { + let (left, right) = self.fp_stack.pop2()?; + self.fp_stack.push(FloatValue::new(self.value_stack.len())); + Some((left.canonicalization, right.canonicalization)) + } else { + None + }; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let end_label = a.get_label(); + let zero_label = a.get_label(); + + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0), + cond, + ); + a.emit_jmp(Condition::Equal, zero_label); + match cncl { + Some((Some(fp), _)) + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization => + { + Self::canonicalize_nan(a, &mut self.machine, fp.to_size(), v_a, ret); + } + _ => { + if v_a != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + v_a, + ret, + ); + } + } + } + a.emit_jmp(Condition::None, end_label); + a.emit_label(zero_label); + match cncl { + Some((_, Some(fp))) + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization => + { + Self::canonicalize_nan(a, &mut self.machine, fp.to_size(), v_b, ret); + } + _ => { + if v_b != ret { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + v_b, + ret, + ); + } + } + } + a.emit_label(end_label); + } + Operator::Block { ty } => { + let frame = ControlFrame { + label: a.get_label(), + loop_like: false, + if_else: IfElseState::None, + returns: match ty { + WpTypeOrFuncType::Type(WpType::EmptyBlockType) => smallvec![], + WpTypeOrFuncType::Type(inner_ty) => smallvec![inner_ty], + _ => { + return Err(CodegenError { + message: format!("Block: multi-value returns not yet implemented"), + }) + } + }, + value_stack_depth: self.value_stack.len(), + fp_stack_depth: self.fp_stack.len(), + state: self.machine.state.clone(), + state_diff_id: Self::get_state_diff( + &self.machine, + &mut self.fsm, + &mut self.control_stack, + ), + }; + self.control_stack.push(frame); + } + Operator::Loop { ty } => { + let label = a.get_label(); + let state_diff_id = + Self::get_state_diff(&self.machine, &mut self.fsm, &mut self.control_stack); + let activate_offset = a.get_offset().0; + + self.control_stack.push(ControlFrame { + label: label, + loop_like: true, + if_else: IfElseState::None, + returns: match ty { + WpTypeOrFuncType::Type(WpType::EmptyBlockType) => smallvec![], + WpTypeOrFuncType::Type(inner_ty) => smallvec![inner_ty], + _ => { + return Err(CodegenError { + message: format!("Loop: multi-value returns not yet implemented"), + }) + } + }, + value_stack_depth: self.value_stack.len(), + fp_stack_depth: self.fp_stack.len(), + state: self.machine.state.clone(), + state_diff_id, + }); + a.emit_label(label); + + // Check interrupt signal without branching + if self.config.full_preemption { + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_interrupt_signal_mem() as i32, + ), + Location::GPR(GPR::RAX), + ); + self.fsm.loop_offsets.insert( + a.get_offset().0, + OffsetInfo { + end_offset: a.get_offset().0 + 1, + activate_offset, + diff_id: state_diff_id, + }, + ); + self.fsm.wasm_offset_to_target_offset.insert( + self.machine.state.wasm_inst_offset, + SuspendOffset::Loop(a.get_offset().0), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, 0), + Location::GPR(GPR::RAX), + ); + } + } + Operator::Nop => {} + Operator::MemorySize { reserved } => { + let memory_index = MemoryIndex::new(reserved as usize); + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_intrinsics() as i32, + ), + Location::GPR(GPR::RAX), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, vm::Intrinsics::offset_memory_size() as i32), + Location::GPR(GPR::RAX), + ); + Self::emit_call_sysv( + a, + &mut self.machine, + |a| { + let label = a.get_label(); + let after = a.get_label(); + a.emit_jmp(Condition::None, after); + a.emit_label(label); + a.emit_host_redirection(GPR::RAX); + a.emit_label(after); + a.emit_call_label(label); + }, + iter::once(Location::Imm32(memory_index.index() as u32)), + None, + )?; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + } + Operator::MemoryGrow { reserved } => { + let memory_index = MemoryIndex::new(reserved as usize); + let param_pages = self.value_stack.pop().unwrap(); + + self.machine.release_locations_only_regs(&[param_pages]); + + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_intrinsics() as i32, + ), + Location::GPR(GPR::RAX), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, vm::Intrinsics::offset_memory_grow() as i32), + Location::GPR(GPR::RAX), + ); + + self.machine.release_locations_only_osr_state(1); + + Self::emit_call_sysv( + a, + &mut self.machine, + |a| { + let label = a.get_label(); + let after = a.get_label(); + a.emit_jmp(Condition::None, after); + a.emit_label(label); + a.emit_host_redirection(GPR::RAX); + a.emit_label(after); + a.emit_call_label(label); + }, + iter::once(Location::Imm32(memory_index.index() as u32)) + .chain(iter::once(param_pages)), + None, + )?; + + self.machine.release_locations_only_stack(a, &[param_pages]); + + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + } + Operator::I32Load { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::F32Load { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I32Load8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32Load8S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S8, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32Load16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32Load16S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32Store { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::F32Store { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let fp = self.fp_stack.pop1()?; + let config_nan_canonicalization = self.config.nan_canonicalization; + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 4, + |a, m, addr| { + if !a.arch_supports_canonicalize_nan() + || !config_nan_canonicalization + || fp.canonicalization.is_none() + { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + } else { + Self::canonicalize_nan( + a, + m, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + } + + Ok(()) + }, + )?; + } + Operator::I32Store8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32Store16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64Load { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::F64Load { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I64Load8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Load8S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Load16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Load16S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Load32U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 4, + |a, m, addr| { + match ret { + Location::GPR(_) => {} + Location::Memory(base, offset) => { + a.emit_mov( + Size::S32, + Location::Imm32(0), + Location::Memory(base, offset + 4), + ); // clear upper bits + } + _ => { + return Err(CodegenError { + message: format!("I64Load32U ret: unreachable code"), + }) + } + } + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I64Load32S { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movsx, + Size::S32, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64Store { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::F64Store { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let fp = self.fp_stack.pop1()?; + let config_nan_canonicalization = self.config.nan_canonicalization; + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 8, + |a, m, addr| { + if !a.arch_supports_canonicalize_nan() + || !config_nan_canonicalization + || fp.canonicalization.is_none() + { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + } else { + Self::canonicalize_nan( + a, + m, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + } + Ok(()) + }, + )?; + } + Operator::I64Store8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64Store16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64Store32 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + false, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::Unreachable => { + Self::mark_trappable(a, &self.machine, &mut self.fsm, &mut self.control_stack); + self.exception_table + .as_mut() + .unwrap() + .offset_to_code + .insert(a.get_offset().0, ExceptionCode::Unreachable); + a.emit_ud2(); + self.unreachable_depth = 1; + } + Operator::Return => { + let frame = &self.control_stack[0]; + if frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("Return: incorrect frame.returns"), + }); + } + let loc = *self.value_stack.last().unwrap(); + if frame.returns[0].is_float() { + let fp = self.fp_stack.peek1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match frame.returns[0] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + Location::GPR(GPR::RAX), + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + self.unreachable_depth = 1; + } + Operator::Br { relative_depth } => { + let frame = + &self.control_stack[self.control_stack.len() - 1 - (relative_depth as usize)]; + if !frame.loop_like && frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("Br: incorrect frame.returns"), + }); + } + let loc = *self.value_stack.last().unwrap(); + + if frame.returns[0].is_float() { + let fp = self.fp_stack.peek1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match frame.returns[0] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + Location::GPR(GPR::RAX), + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } else { + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + } + } + + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + self.unreachable_depth = 1; + } + Operator::BrIf { relative_depth } => { + let after = a.get_label(); + let cond = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(0), + cond, + ); + a.emit_jmp(Condition::Equal, after); + + let frame = + &self.control_stack[self.control_stack.len() - 1 - (relative_depth as usize)]; + if !frame.loop_like && frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("BrIf: incorrect frame.returns"), + }); + } + let loc = *self.value_stack.last().unwrap(); + if frame.returns[0].is_float() { + let fp = self.fp_stack.peek1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match frame.returns[0] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + Location::GPR(GPR::RAX), + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } else { + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + } + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + + a.emit_label(after); + } + Operator::BrTable { ref table } => { + let (targets, default_target) = table.read_table().map_err(|e| CodegenError { + message: format!("BrTable read_table: {:?}", e), + })?; + let cond = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let table_label = a.get_label(); + let mut table: Vec = vec![]; + let default_br = a.get_label(); + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_cmp, + Size::S32, + Location::Imm32(targets.len() as u32), + cond, + ); + a.emit_jmp(Condition::AboveEqual, default_br); + + a.emit_lea_label(table_label, Location::GPR(GPR::RCX)); + a.emit_mov(Size::S32, cond, Location::GPR(GPR::RDX)); + + let instr_size = a.get_jmp_instr_size(); + a.emit_imul_imm32_gpr64(instr_size as _, GPR::RDX); + a.emit_add(Size::S64, Location::GPR(GPR::RCX), Location::GPR(GPR::RDX)); + a.emit_jmp_location(Location::GPR(GPR::RDX)); + + for target in targets.iter() { + let label = a.get_label(); + a.emit_label(label); + table.push(label); + let frame = + &self.control_stack[self.control_stack.len() - 1 - (*target as usize)]; + if !frame.loop_like && frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!( + "BrTable: incorrect frame.returns for {:?}", + target + ), + }); + } + let loc = *self.value_stack.last().unwrap(); + if frame.returns[0].is_float() { + let fp = self.fp_stack.peek1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match frame.returns[0] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + Location::GPR(GPR::RAX), + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } else { + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + } + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + } + a.emit_label(default_br); + + { + let frame = &self.control_stack + [self.control_stack.len() - 1 - (default_target as usize)]; + if !frame.loop_like && frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("BrTable: incorrect frame.returns"), + }); + } + let loc = *self.value_stack.last().unwrap(); + if frame.returns[0].is_float() { + let fp = self.fp_stack.peek1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match frame.returns[0] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + Location::GPR(GPR::RAX), + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } else { + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + } + } + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); + a.emit_jmp(Condition::None, frame.label); + } + + a.emit_label(table_label); + for x in table { + a.emit_jmp(Condition::None, x); + } + self.unreachable_depth = 1; + } + Operator::Drop => { + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + if let Some(x) = self.fp_stack.last() { + if x.depth == self.value_stack.len() { + self.fp_stack.pop1()?; + } + } + } + Operator::End => { + let frame = self.control_stack.pop().unwrap(); + + if !was_unreachable && frame.returns.len() > 0 { + let loc = *self.value_stack.last().unwrap(); + if frame.returns[0].is_float() { + let fp = self.fp_stack.peek1()?; + if a.arch_supports_canonicalize_nan() + && self.config.nan_canonicalization + && fp.canonicalization.is_some() + { + Self::canonicalize_nan( + a, + &mut self.machine, + match frame.returns[0] { + WpType::F32 => Size::S32, + WpType::F64 => Size::S64, + _ => unreachable!(), + }, + loc, + Location::GPR(GPR::RAX), + ); + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } else { + Self::emit_relaxed_binop( + a, + &mut self.machine, + Assembler::emit_mov, + Size::S64, + loc, + Location::GPR(GPR::RAX), + ); + } + } + + if self.control_stack.len() == 0 { + a.emit_label(frame.label); + self.machine.finalize_locals(a, &self.locals); + a.emit_mov(Size::S64, Location::GPR(GPR::RBP), Location::GPR(GPR::RSP)); + a.emit_pop(Size::S64, Location::GPR(GPR::RBP)); + + // Make a copy of the return value in XMM0, as required by the SysV CC. + match self.signature.returns() { + [x] if *x == Type::F32 || *x == Type::F64 => { + a.emit_mov( + Size::S64, + Location::GPR(GPR::RAX), + Location::XMM(XMM::XMM0), + ); + } + _ => {} + } + a.emit_ret(); + } else { + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations(a, released); + self.value_stack.truncate(frame.value_stack_depth); + self.fp_stack.truncate(frame.fp_stack_depth); + + if !frame.loop_like { + a.emit_label(frame.label); + } + + if let IfElseState::If(label) = frame.if_else { + a.emit_label(label); + } + + if frame.returns.len() > 0 { + if frame.returns.len() != 1 { + return Err(CodegenError { + message: format!("End: incorrect frame.returns"), + }); + } + let loc = self.machine.acquire_locations( + a, + &[( + frame.returns[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), loc); + self.value_stack.push(loc); + if frame.returns[0].is_float() { + self.fp_stack + .push(FloatValue::new(self.value_stack.len() - 1)); + // we already canonicalized at the `Br*` instruction or here previously. + } + } + } + } + Operator::AtomicFence { flags: _ } => { + // Fence is a nop. + // + // Fence was added to preserve information about fences from + // source languages. If in the future Wasm extends the memory + // model, and if we hadn't recorded what fences used to be there, + // it would lead to data races that weren't present in the + // original source language. + } + Operator::I32AtomicLoad { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I32AtomicLoad8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32AtomicLoad16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I32AtomicStore { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32AtomicStore8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32AtomicStore16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicLoad { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicLoad8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64AtomicLoad16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + )?; + Ok(()) + }, + )?; + } + Operator::I64AtomicLoad32U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 4, + |a, m, addr| { + match ret { + Location::GPR(_) => {} + Location::Memory(base, offset) => { + a.emit_mov( + Size::S32, + Location::Imm32(0), + Location::Memory(base, offset + 4), + ); // clear upper bits + } + _ => { + return Err(CodegenError { + message: format!("I64AtomicLoad32U ret: unreachable code"), + }) + } + } + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicStore { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + true, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicStore8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicStore16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I64AtomicStore32 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target_addr, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + } + Operator::I32AtomicRmwAdd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmwAdd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S64, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw8AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw8AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32AddU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwSub { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + a.emit_neg(Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmwSub { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + a.emit_neg(Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S64, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw8SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); + a.emit_neg(Size::S8, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); + a.emit_neg(Size::S16, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw8SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + a.emit_neg(Size::S8, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + a.emit_neg(Size::S16, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32SubU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + a.emit_neg(Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmwAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw8AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw16AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw8AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw16AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw32AndU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmwOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmwOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw8OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw16OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw8OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw16OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw32OrU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmwXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmwXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw8XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmw16XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw8XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw16XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I64AtomicRmw32XorU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + )?; + } + Operator::I32AtomicRmwXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmwXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_xchg(Size::S64, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw8XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw8XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32XchgU { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0)); + Ok(()) + }, + )?; + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S32, Location::GPR(compare), ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmwCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S64, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S64, Location::GPR(compare), ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I32AtomicRmw8CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S8, Location::GPR(compare), Size::S32, ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I32AtomicRmw16CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S16, Location::GPR(compare), Size::S32, ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw8CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S8, Location::GPR(compare), Size::S64, ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw16CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S16, Location::GPR(compare), Size::S64, ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw32CmpxchgU { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + self.exception_table.as_mut().unwrap(), + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S32, Location::GPR(compare), ret); + Ok(()) + }, + )?; + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + _ => { + return Err(CodegenError { + message: format!("not yet implemented: {:?}", op), + }); + } + } + + Ok(()) + } +} + +fn type_to_wp_type(ty: Type) -> WpType { + match ty { + Type::I32 => WpType::I32, + Type::I64 => WpType::I64, + Type::F32 => WpType::F32, + Type::F64 => WpType::F64, + Type::V128 => WpType::V128, + } +} + +fn get_location_released(a: &mut Assembler, m: &mut Machine, loc: Location) -> Location { + m.release_locations(a, &[loc]); + loc +} + +fn sort_call_movs(movs: &mut [(Location, GPR)]) { + for i in 0..movs.len() { + for j in (i + 1)..movs.len() { + if let Location::GPR(src_gpr) = movs[j].0 { + if src_gpr == movs[i].1 { + movs.swap(i, j); + } + } + } + } + + /* + { + use std::collections::{HashMap, HashSet, VecDeque}; + let mut mov_map: HashMap> = HashMap::new(); + for mov in movs.iter() { + if let Location::GPR(src_gpr) = mov.0 { + if src_gpr != mov.1 { + mov_map.entry(src_gpr).or_insert_with(|| HashSet::new()).insert(mov.1); + } + } + } + + for (start, _) in mov_map.iter() { + let mut q: VecDeque = VecDeque::new(); + let mut black: HashSet = HashSet::new(); + + q.push_back(*start); + black.insert(*start); + + while q.len() > 0 { + let reg = q.pop_front().unwrap(); + let empty_set = HashSet::new(); + for x in mov_map.get(®).unwrap_or(&empty_set).iter() { + if black.contains(x) { + panic!("cycle detected"); + } + q.push_back(*x); + black.insert(*x); + } + } + } + } + */ +} + +// Constants for the bounds of truncation operations. These are the least or +// greatest exact floats in either f32 or f64 representation less-than (for +// least) or greater-than (for greatest) the i32 or i64 or u32 or u64 +// min (for least) or max (for greatest), when rounding towards zero. + +/// Greatest Exact Float (32 bits) less-than i32::MIN when rounding towards zero. +const GEF32_LT_I32_MIN: f32 = -2147483904.0; +/// Least Exact Float (32 bits) greater-than i32::MAX when rounding towards zero. +const LEF32_GT_I32_MAX: f32 = 2147483648.0; +/// Greatest Exact Float (32 bits) less-than i64::MIN when rounding towards zero. +const GEF32_LT_I64_MIN: f32 = -9223373136366403584.0; +/// Least Exact Float (32 bits) greater-than i64::MAX when rounding towards zero. +const LEF32_GT_I64_MAX: f32 = 9223372036854775808.0; +/// Greatest Exact Float (32 bits) less-than u32::MIN when rounding towards zero. +const GEF32_LT_U32_MIN: f32 = -1.0; +/// Least Exact Float (32 bits) greater-than u32::MAX when rounding towards zero. +const LEF32_GT_U32_MAX: f32 = 4294967296.0; +/// Greatest Exact Float (32 bits) less-than u64::MIN when rounding towards zero. +const GEF32_LT_U64_MIN: f32 = -1.0; +/// Least Exact Float (32 bits) greater-than u64::MAX when rounding towards zero. +const LEF32_GT_U64_MAX: f32 = 18446744073709551616.0; + +/// Greatest Exact Float (64 bits) less-than i32::MIN when rounding towards zero. +const GEF64_LT_I32_MIN: f64 = -2147483649.0; +/// Least Exact Float (64 bits) greater-than i32::MAX when rounding towards zero. +const LEF64_GT_I32_MAX: f64 = 2147483648.0; +/// Greatest Exact Float (64 bits) less-than i64::MIN when rounding towards zero. +const GEF64_LT_I64_MIN: f64 = -9223372036854777856.0; +/// Least Exact Float (64 bits) greater-than i64::MAX when rounding towards zero. +const LEF64_GT_I64_MAX: f64 = 9223372036854775808.0; +/// Greatest Exact Float (64 bits) less-than u32::MIN when rounding towards zero. +const GEF64_LT_U32_MIN: f64 = -1.0; +/// Least Exact Float (64 bits) greater-than u32::MAX when rounding towards zero. +const LEF64_GT_U32_MAX: f64 = 4294967296.0; +/// Greatest Exact Float (64 bits) less-than u64::MIN when rounding towards zero. +const GEF64_LT_U64_MIN: f64 = -1.0; +/// Least Exact Float (64 bits) greater-than u64::MAX when rounding towards zero. +const LEF64_GT_U64_MAX: f64 = 18446744073709551616.0; diff --git a/lib/compiler-singlepass/src/common_decl.rs b/lib/compiler-singlepass/src/common_decl.rs new file mode 100644 index 000000000..89ae92770 --- /dev/null +++ b/lib/compiler-singlepass/src/common_decl.rs @@ -0,0 +1,155 @@ +use std::collections::BTreeMap; + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub struct RegisterIndex(pub usize); + +/// Information of an inline breakpoint. +/// +/// TODO: Move this into runtime. +#[derive(Clone, Debug)] +pub struct InlineBreakpoint { + /// Size in bytes taken by this breakpoint's instruction sequence. + pub size: usize, + + /// Type of the inline breakpoint. + pub ty: InlineBreakpointType, +} + +/// The type of an inline breakpoint. +#[repr(u8)] +#[derive(Copy, Clone, Debug)] +pub enum InlineBreakpointType { + /// A middleware invocation breakpoint. + Middleware, +} + +/// A kind of wasm or constant value +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub enum WasmAbstractValue { + /// A wasm runtime value + Runtime, + /// A wasm constant value + Const(u64), +} + +/// A container for the state of a running wasm instance. +#[derive(Clone, Debug)] +pub struct MachineState { + /// Stack values. + pub stack_values: Vec, + /// Register values. + pub register_values: Vec, + /// Previous frame. + pub prev_frame: BTreeMap, + /// Wasm stack. + pub wasm_stack: Vec, + /// Private depth of the wasm stack. + pub wasm_stack_private_depth: usize, + /// Wasm instruction offset. + pub wasm_inst_offset: usize, +} + +/// A diff of two `MachineState`s. +#[derive(Clone, Debug, Default)] +pub struct MachineStateDiff { + /// Last. + pub last: Option, + /// Stack push. + pub stack_push: Vec, + /// Stack pop. + pub stack_pop: usize, + + /// Register diff. + pub reg_diff: Vec<(RegisterIndex, MachineValue)>, + + /// Previous frame diff. + pub prev_frame_diff: BTreeMap>, // None for removal + + /// Wasm stack push. + pub wasm_stack_push: Vec, + /// Wasm stack pop. + pub wasm_stack_pop: usize, + /// Private depth of the wasm stack. + pub wasm_stack_private_depth: usize, // absolute value; not a diff. + /// Wasm instruction offset. + pub wasm_inst_offset: usize, // absolute value; not a diff. +} + +/// A kind of machine value. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub enum MachineValue { + /// Undefined. + Undefined, + /// Vmctx. + Vmctx, + /// Vmctx Deref. + VmctxDeref(Vec), + /// Preserve Register. + PreserveRegister(RegisterIndex), + /// Copy Stack BP Relative. + CopyStackBPRelative(i32), // relative to Base Pointer, in byte offset + /// Explicit Shadow. + ExplicitShadow, // indicates that all values above this are above the shadow region + /// Wasm Stack. + WasmStack(usize), + /// Wasm Local. + WasmLocal(usize), + /// Two Halves. + TwoHalves(Box<(MachineValue, MachineValue)>), // 32-bit values. TODO: optimize: add another type for inner "half" value to avoid boxing? +} + +/// A map of function states. +#[derive(Clone, Debug)] +pub struct FunctionStateMap { + /// Initial. + pub initial: MachineState, + /// Local Function Id. + pub local_function_id: usize, + /// Locals. + pub locals: Vec, + /// Shadow size. + pub shadow_size: usize, // for single-pass backend, 32 bytes on x86-64 + /// Diffs. + pub diffs: Vec, + /// Wasm Function Header target offset. + pub wasm_function_header_target_offset: Option, + /// Wasm offset to target offset + pub wasm_offset_to_target_offset: BTreeMap, + /// Loop offsets. + pub loop_offsets: BTreeMap, /* suspend_offset -> info */ + /// Call offsets. + pub call_offsets: BTreeMap, /* suspend_offset -> info */ + /// Trappable offsets. + pub trappable_offsets: BTreeMap, /* suspend_offset -> info */ +} + +/// A kind of suspend offset. +#[derive(Clone, Copy, Debug)] +pub enum SuspendOffset { + /// A loop. + Loop(usize), + /// A call. + Call(usize), + /// A trappable. + Trappable(usize), +} + +/// Info for an offset. +#[derive(Clone, Debug)] +pub struct OffsetInfo { + /// End offset. + pub end_offset: usize, // excluded bound + /// Diff Id. + pub diff_id: usize, + /// Activate offset. + pub activate_offset: usize, +} + +/// A map of module state. +#[derive(Clone, Debug)] +pub struct ModuleStateMap { + /// Local functions. + pub local_functions: BTreeMap, + /// Total size. + pub total_size: usize, +} diff --git a/lib/compiler-singlepass/src/compiler.rs b/lib/compiler-singlepass/src/compiler.rs index 8b9140159..f51d8bbbc 100644 --- a/lib/compiler-singlepass/src/compiler.rs +++ b/lib/compiler-singlepass/src/compiler.rs @@ -65,7 +65,7 @@ impl Compiler for SinglepassCompiler { )) } - fn compile_wasm_trampolines( + fn compile_function_call_trampolines( &self, _signatures: &[FunctionType], ) -> Result, CompileError> { diff --git a/lib/compiler-singlepass/src/config.rs b/lib/compiler-singlepass/src/config.rs index 25b5b8020..e0ff6c6e7 100644 --- a/lib/compiler-singlepass/src/config.rs +++ b/lib/compiler-singlepass/src/config.rs @@ -41,7 +41,7 @@ impl CompilerConfig for SinglepassConfig { } /// Transform it into the compiler - fn compiler(&self) -> Box { + fn compiler(&self) -> Box { Box::new(SinglepassCompiler::new(&self)) } } diff --git a/lib/compiler-singlepass/src/emitter_x64.rs b/lib/compiler-singlepass/src/emitter_x64.rs new file mode 100644 index 000000000..ed0efb8ae --- /dev/null +++ b/lib/compiler-singlepass/src/emitter_x64.rs @@ -0,0 +1,1327 @@ +use crate::common_decl::InlineBreakpointType; +pub use crate::x64_decl::{GPR, XMM}; +use dynasm::dynasm; +use dynasmrt::{x64::Assembler, AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi}; + +fn _dummy(_a: &Assembler) { + dynasm!( + _a + ; .arch x64 + ); +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum Location { + Imm8(u8), + Imm32(u32), + Imm64(u64), + Imm128(u128), + GPR(GPR), + XMM(XMM), + Memory(GPR, i32), +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Condition { + None, + Above, + AboveEqual, + Below, + BelowEqual, + Greater, + GreaterEqual, + Less, + LessEqual, + Equal, + NotEqual, + Signed, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub enum Size { + S8, + S16, + S32, + S64, +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[allow(dead_code)] +pub enum XMMOrMemory { + XMM(XMM), + Memory(GPR, i32), +} + +#[derive(Copy, Clone, Debug)] +#[allow(dead_code)] +pub enum GPROrMemory { + GPR(GPR), + Memory(GPR, i32), +} + +pub trait Emitter { + type Label; + type Offset; + + fn get_label(&mut self) -> Self::Label; + fn get_offset(&self) -> Self::Offset; + fn get_jmp_instr_size(&self) -> u8; + + fn emit_u64(&mut self, x: u64); + + fn emit_label(&mut self, label: Self::Label); + + fn emit_nop(&mut self); + + fn emit_mov(&mut self, sz: Size, src: Location, dst: Location); + fn emit_lea(&mut self, sz: Size, src: Location, dst: Location); + fn emit_lea_label(&mut self, label: Self::Label, dst: Location); + fn emit_cdq(&mut self); + fn emit_cqo(&mut self); + fn emit_xor(&mut self, sz: Size, src: Location, dst: Location); + fn emit_jmp(&mut self, condition: Condition, label: Self::Label); + fn emit_jmp_location(&mut self, loc: Location); + fn emit_conditional_trap(&mut self, condition: Condition); + fn emit_set(&mut self, condition: Condition, dst: GPR); + fn emit_push(&mut self, sz: Size, src: Location); + fn emit_pop(&mut self, sz: Size, dst: Location); + fn emit_cmp(&mut self, sz: Size, left: Location, right: Location); + fn emit_add(&mut self, sz: Size, src: Location, dst: Location); + fn emit_sub(&mut self, sz: Size, src: Location, dst: Location); + fn emit_neg(&mut self, sz: Size, value: Location); + fn emit_imul(&mut self, sz: Size, src: Location, dst: Location); + fn emit_imul_imm32_gpr64(&mut self, src: u32, dst: GPR); + fn emit_div(&mut self, sz: Size, divisor: Location); + fn emit_idiv(&mut self, sz: Size, divisor: Location); + fn emit_shl(&mut self, sz: Size, src: Location, dst: Location); + fn emit_shr(&mut self, sz: Size, src: Location, dst: Location); + fn emit_sar(&mut self, sz: Size, src: Location, dst: Location); + fn emit_rol(&mut self, sz: Size, src: Location, dst: Location); + fn emit_ror(&mut self, sz: Size, src: Location, dst: Location); + fn emit_and(&mut self, sz: Size, src: Location, dst: Location); + fn emit_or(&mut self, sz: Size, src: Location, dst: Location); + fn emit_bsr(&mut self, sz: Size, src: Location, dst: Location); + fn emit_bsf(&mut self, sz: Size, src: Location, dst: Location); + fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location); + fn emit_movzx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location); + fn emit_movsx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location); + fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location); + fn emit_lock_xadd(&mut self, sz: Size, src: Location, dst: Location); + fn emit_lock_cmpxchg(&mut self, sz: Size, src: Location, dst: Location); + + fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR); + fn emit_btc_gpr_imm8_64(&mut self, src: u8, dst: GPR); + + fn emit_cmovae_gpr_32(&mut self, src: GPR, dst: GPR); + fn emit_cmovae_gpr_64(&mut self, src: GPR, dst: GPR); + + fn emit_vmovaps(&mut self, src: XMMOrMemory, dst: XMMOrMemory); + fn emit_vmovapd(&mut self, src: XMMOrMemory, dst: XMMOrMemory); + fn emit_vxorps(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vxorpd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vaddss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vaddsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vsubss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vsubsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vmulss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vmulsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vdivss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vdivsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vmaxss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vmaxsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vminss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vminsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcmpeqss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcmpeqsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcmpneqss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcmpneqsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcmpltss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcmpltsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcmpless(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcmplesd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcmpgtss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcmpgtsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcmpgess(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcmpgesd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcmpunordss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcmpunordsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcmpordss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcmpordsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vsqrtss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vsqrtsd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vroundss_nearest(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vroundss_floor(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vroundss_ceil(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vroundss_trunc(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vroundsd_nearest(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vroundsd_floor(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vroundsd_ceil(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vroundsd_trunc(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_vcvtss2sd(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + fn emit_vcvtsd2ss(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM); + + fn emit_ucomiss(&mut self, src: XMMOrMemory, dst: XMM); + fn emit_ucomisd(&mut self, src: XMMOrMemory, dst: XMM); + + fn emit_cvttss2si_32(&mut self, src: XMMOrMemory, dst: GPR); + fn emit_cvttss2si_64(&mut self, src: XMMOrMemory, dst: GPR); + fn emit_cvttsd2si_32(&mut self, src: XMMOrMemory, dst: GPR); + fn emit_cvttsd2si_64(&mut self, src: XMMOrMemory, dst: GPR); + + fn emit_vcvtsi2ss_32(&mut self, src1: XMM, src2: GPROrMemory, dst: XMM); + fn emit_vcvtsi2ss_64(&mut self, src1: XMM, src2: GPROrMemory, dst: XMM); + fn emit_vcvtsi2sd_32(&mut self, src1: XMM, src2: GPROrMemory, dst: XMM); + fn emit_vcvtsi2sd_64(&mut self, src1: XMM, src2: GPROrMemory, dst: XMM); + + fn emit_vblendvps(&mut self, src1: XMM, src2: XMMOrMemory, mask: XMM, dst: XMM); + fn emit_vblendvpd(&mut self, src1: XMM, src2: XMMOrMemory, mask: XMM, dst: XMM); + + fn emit_test_gpr_64(&mut self, reg: GPR); + + fn emit_ud2(&mut self); + fn emit_ret(&mut self); + fn emit_call_label(&mut self, label: Self::Label); + fn emit_call_location(&mut self, loc: Location); + + fn emit_bkpt(&mut self); + + fn emit_host_redirection(&mut self, target: GPR); + fn emit_inline_breakpoint(&mut self, ty: InlineBreakpointType); + + fn arch_has_itruncf(&self) -> bool { + false + } + fn arch_emit_i32_trunc_sf32(&mut self, _src: XMM, _dst: GPR) { + unimplemented!() + } + fn arch_emit_i32_trunc_sf64(&mut self, _src: XMM, _dst: GPR) { + unimplemented!() + } + fn arch_emit_i32_trunc_uf32(&mut self, _src: XMM, _dst: GPR) { + unimplemented!() + } + fn arch_emit_i32_trunc_uf64(&mut self, _src: XMM, _dst: GPR) { + unimplemented!() + } + fn arch_emit_i64_trunc_sf32(&mut self, _src: XMM, _dst: GPR) { + unimplemented!() + } + fn arch_emit_i64_trunc_sf64(&mut self, _src: XMM, _dst: GPR) { + unimplemented!() + } + fn arch_emit_i64_trunc_uf32(&mut self, _src: XMM, _dst: GPR) { + unimplemented!() + } + fn arch_emit_i64_trunc_uf64(&mut self, _src: XMM, _dst: GPR) { + unimplemented!() + } + + fn arch_has_fconverti(&self) -> bool { + false + } + fn arch_emit_f32_convert_si32(&mut self, _src: GPR, _dst: XMM) { + unimplemented!() + } + fn arch_emit_f32_convert_si64(&mut self, _src: GPR, _dst: XMM) { + unimplemented!() + } + fn arch_emit_f32_convert_ui32(&mut self, _src: GPR, _dst: XMM) { + unimplemented!() + } + fn arch_emit_f32_convert_ui64(&mut self, _src: GPR, _dst: XMM) { + unimplemented!() + } + fn arch_emit_f64_convert_si32(&mut self, _src: GPR, _dst: XMM) { + unimplemented!() + } + fn arch_emit_f64_convert_si64(&mut self, _src: GPR, _dst: XMM) { + unimplemented!() + } + fn arch_emit_f64_convert_ui32(&mut self, _src: GPR, _dst: XMM) { + unimplemented!() + } + fn arch_emit_f64_convert_ui64(&mut self, _src: GPR, _dst: XMM) { + unimplemented!() + } + + fn arch_has_fneg(&self) -> bool { + false + } + fn arch_emit_f32_neg(&mut self, _src: XMM, _dst: XMM) { + unimplemented!() + } + fn arch_emit_f64_neg(&mut self, _src: XMM, _dst: XMM) { + unimplemented!() + } + + fn arch_has_xzcnt(&self) -> bool { + false + } + fn arch_emit_lzcnt(&mut self, _sz: Size, _src: Location, _dst: Location) { + unimplemented!() + } + fn arch_emit_tzcnt(&mut self, _sz: Size, _src: Location, _dst: Location) { + unimplemented!() + } + + fn arch_supports_canonicalize_nan(&self) -> bool { + true + } + + fn arch_requires_indirect_call_trampoline(&self) -> bool { + false + } + + fn arch_emit_indirect_call_with_trampoline(&mut self, _loc: Location) { + unimplemented!() + } + + // Emits entry trampoline just before the real function. + fn arch_emit_entry_trampoline(&mut self) {} +} + +macro_rules! unop_gpr { + ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => { + match ($sz, $loc) { + (Size::S32, Location::GPR(loc)) => { + dynasm!($assembler ; $ins Rd(loc as u8)); + }, + (Size::S64, Location::GPR(loc)) => { + dynasm!($assembler ; $ins Rq(loc as u8)); + }, + _ => $otherwise + } + }; +} + +macro_rules! unop_mem { + ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => { + match ($sz, $loc) { + (Size::S32, Location::Memory(loc, disp)) => { + dynasm!($assembler ; $ins DWORD [Rq(loc as u8) + disp] ); + }, + (Size::S64, Location::Memory(loc, disp)) => { + dynasm!($assembler ; $ins QWORD [Rq(loc as u8) + disp] ); + }, + _ => $otherwise + } + }; +} + +macro_rules! unop_gpr_or_mem { + ($ins:ident, $assembler:tt, $sz:expr, $loc:expr, $otherwise:block) => { + unop_gpr!($ins, $assembler, $sz, $loc, { + unop_mem!($ins, $assembler, $sz, $loc, $otherwise) + }) + }; +} + +macro_rules! binop_imm32_gpr { + ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => { + match ($sz, $src, $dst) { + (Size::S32, Location::Imm32(src), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rd(dst as u8), src as i32); // IMM32_2GPR + }, + (Size::S64, Location::Imm32(src), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rq(dst as u8), src as i32); // IMM32_2GPR + }, + _ => $otherwise + } + }; +} + +macro_rules! binop_imm32_mem { + ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => { + match ($sz, $src, $dst) { + (Size::S32, Location::Imm32(src), Location::Memory(dst, disp)) => { + dynasm!($assembler ; $ins DWORD [Rq(dst as u8) + disp], src as i32); + }, + (Size::S64, Location::Imm32(src), Location::Memory(dst, disp)) => { + dynasm!($assembler ; $ins QWORD [Rq(dst as u8) + disp], src as i32); + }, + _ => $otherwise + } + }; +} + +macro_rules! binop_imm64_gpr { + ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => { + match ($sz, $src, $dst) { + (Size::S64, Location::Imm64(src), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rq(dst as u8), QWORD src as i64); // IMM32_2GPR + }, + _ => $otherwise + } + }; +} + +macro_rules! binop_gpr_gpr { + ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => { + match ($sz, $src, $dst) { + (Size::S32, Location::GPR(src), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rd(dst as u8), Rd(src as u8)); // GPR2GPR + }, + (Size::S64, Location::GPR(src), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rq(dst as u8), Rq(src as u8)); // GPR2GPR + }, + _ => $otherwise + } + }; +} + +macro_rules! binop_gpr_mem { + ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => { + match ($sz, $src, $dst) { + (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!($assembler ; $ins [Rq(dst as u8) + disp], Rd(src as u8)); // GPR2MEM + }, + (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!($assembler ; $ins [Rq(dst as u8) + disp], Rq(src as u8)); // GPR2MEM + }, + _ => $otherwise + } + }; +} + +macro_rules! binop_mem_gpr { + ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => { + match ($sz, $src, $dst) { + (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rd(dst as u8), [Rq(src as u8) + disp]); // MEM2GPR + }, + (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rq(dst as u8), [Rq(src as u8) + disp]); // MEM2GPR + }, + _ => $otherwise + } + }; +} + +macro_rules! binop_all_nofp { + ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => { + binop_imm32_gpr!($ins, $assembler, $sz, $src, $dst, { + binop_imm32_mem!($ins, $assembler, $sz, $src, $dst, { + binop_gpr_gpr!($ins, $assembler, $sz, $src, $dst, { + binop_gpr_mem!($ins, $assembler, $sz, $src, $dst, { + binop_mem_gpr!($ins, $assembler, $sz, $src, $dst, $otherwise) + }) + }) + }) + }) + }; +} + +macro_rules! binop_shift { + ($ins:ident, $assembler:tt, $sz:expr, $src:expr, $dst:expr, $otherwise:block) => { + match ($sz, $src, $dst) { + (Size::S32, Location::GPR(GPR::RCX), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rd(dst as u8), cl); + }, + (Size::S32, Location::GPR(GPR::RCX), Location::Memory(dst, disp)) => { + dynasm!($assembler ; $ins DWORD [Rq(dst as u8) + disp], cl); + }, + (Size::S32, Location::Imm8(imm), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rd(dst as u8), imm as i8); + }, + (Size::S32, Location::Imm8(imm), Location::Memory(dst, disp)) => { + dynasm!($assembler ; $ins DWORD [Rq(dst as u8) + disp], imm as i8); + }, + (Size::S64, Location::GPR(GPR::RCX), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rq(dst as u8), cl); + }, + (Size::S64, Location::GPR(GPR::RCX), Location::Memory(dst, disp)) => { + dynasm!($assembler ; $ins QWORD [Rq(dst as u8) + disp], cl); + }, + (Size::S64, Location::Imm8(imm), Location::GPR(dst)) => { + dynasm!($assembler ; $ins Rq(dst as u8), imm as i8); + }, + (Size::S64, Location::Imm8(imm), Location::Memory(dst, disp)) => { + dynasm!($assembler ; $ins QWORD [Rq(dst as u8) + disp], imm as i8); + }, + _ => $otherwise + } + } +} + +macro_rules! jmp_op { + ($ins:ident, $assembler:tt, $label:ident) => { + dynasm!($assembler ; $ins =>$label); + } +} + +macro_rules! trap_op { + ($ins:ident, $assembler:tt) => { + dynasm!($assembler + ; $ins >trap + ; jmp >after + ; trap: + ; ud2 + ; after: + ); + } +} + +macro_rules! avx_fn { + ($ins:ident, $name:ident) => { + fn $name(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) { + // Dynasm bug: AVX instructions are not encoded correctly. + match src2 { + XMMOrMemory::XMM(x) => match src1 { + XMM::XMM0 => dynasm!(self ; $ins Rx((dst as u8)), xmm0, Rx((x as u8))), + XMM::XMM1 => dynasm!(self ; $ins Rx((dst as u8)), xmm1, Rx((x as u8))), + XMM::XMM2 => dynasm!(self ; $ins Rx((dst as u8)), xmm2, Rx((x as u8))), + XMM::XMM3 => dynasm!(self ; $ins Rx((dst as u8)), xmm3, Rx((x as u8))), + XMM::XMM4 => dynasm!(self ; $ins Rx((dst as u8)), xmm4, Rx((x as u8))), + XMM::XMM5 => dynasm!(self ; $ins Rx((dst as u8)), xmm5, Rx((x as u8))), + XMM::XMM6 => dynasm!(self ; $ins Rx((dst as u8)), xmm6, Rx((x as u8))), + XMM::XMM7 => dynasm!(self ; $ins Rx((dst as u8)), xmm7, Rx((x as u8))), + XMM::XMM8 => dynasm!(self ; $ins Rx((dst as u8)), xmm8, Rx((x as u8))), + XMM::XMM9 => dynasm!(self ; $ins Rx((dst as u8)), xmm9, Rx((x as u8))), + XMM::XMM10 => dynasm!(self ; $ins Rx((dst as u8)), xmm10, Rx((x as u8))), + XMM::XMM11 => dynasm!(self ; $ins Rx((dst as u8)), xmm11, Rx((x as u8))), + XMM::XMM12 => dynasm!(self ; $ins Rx((dst as u8)), xmm12, Rx((x as u8))), + XMM::XMM13 => dynasm!(self ; $ins Rx((dst as u8)), xmm13, Rx((x as u8))), + XMM::XMM14 => dynasm!(self ; $ins Rx((dst as u8)), xmm14, Rx((x as u8))), + XMM::XMM15 => dynasm!(self ; $ins Rx((dst as u8)), xmm15, Rx((x as u8))), + }, + XMMOrMemory::Memory(base, disp) => match src1 { + XMM::XMM0 => dynasm!(self ; $ins Rx((dst as u8)), xmm0, [Rq((base as u8)) + disp]), + XMM::XMM1 => dynasm!(self ; $ins Rx((dst as u8)), xmm1, [Rq((base as u8)) + disp]), + XMM::XMM2 => dynasm!(self ; $ins Rx((dst as u8)), xmm2, [Rq((base as u8)) + disp]), + XMM::XMM3 => dynasm!(self ; $ins Rx((dst as u8)), xmm3, [Rq((base as u8)) + disp]), + XMM::XMM4 => dynasm!(self ; $ins Rx((dst as u8)), xmm4, [Rq((base as u8)) + disp]), + XMM::XMM5 => dynasm!(self ; $ins Rx((dst as u8)), xmm5, [Rq((base as u8)) + disp]), + XMM::XMM6 => dynasm!(self ; $ins Rx((dst as u8)), xmm6, [Rq((base as u8)) + disp]), + XMM::XMM7 => dynasm!(self ; $ins Rx((dst as u8)), xmm7, [Rq((base as u8)) + disp]), + XMM::XMM8 => dynasm!(self ; $ins Rx((dst as u8)), xmm8, [Rq((base as u8)) + disp]), + XMM::XMM9 => dynasm!(self ; $ins Rx((dst as u8)), xmm9, [Rq((base as u8)) + disp]), + XMM::XMM10 => dynasm!(self ; $ins Rx((dst as u8)), xmm10, [Rq((base as u8)) + disp]), + XMM::XMM11 => dynasm!(self ; $ins Rx((dst as u8)), xmm11, [Rq((base as u8)) + disp]), + XMM::XMM12 => dynasm!(self ; $ins Rx((dst as u8)), xmm12, [Rq((base as u8)) + disp]), + XMM::XMM13 => dynasm!(self ; $ins Rx((dst as u8)), xmm13, [Rq((base as u8)) + disp]), + XMM::XMM14 => dynasm!(self ; $ins Rx((dst as u8)), xmm14, [Rq((base as u8)) + disp]), + XMM::XMM15 => dynasm!(self ; $ins Rx((dst as u8)), xmm15, [Rq((base as u8)) + disp]), + }, + } + } + } +} + +macro_rules! avx_i2f_64_fn { + ($ins:ident, $name:ident) => { + fn $name(&mut self, src1: XMM, src2: GPROrMemory, dst: XMM) { + match src2 { + GPROrMemory::GPR(x) => match src1 { + XMM::XMM0 => dynasm!(self ; $ins Rx((dst as u8)), xmm0, Rq((x as u8))), + XMM::XMM1 => dynasm!(self ; $ins Rx((dst as u8)), xmm1, Rq((x as u8))), + XMM::XMM2 => dynasm!(self ; $ins Rx((dst as u8)), xmm2, Rq((x as u8))), + XMM::XMM3 => dynasm!(self ; $ins Rx((dst as u8)), xmm3, Rq((x as u8))), + XMM::XMM4 => dynasm!(self ; $ins Rx((dst as u8)), xmm4, Rq((x as u8))), + XMM::XMM5 => dynasm!(self ; $ins Rx((dst as u8)), xmm5, Rq((x as u8))), + XMM::XMM6 => dynasm!(self ; $ins Rx((dst as u8)), xmm6, Rq((x as u8))), + XMM::XMM7 => dynasm!(self ; $ins Rx((dst as u8)), xmm7, Rq((x as u8))), + XMM::XMM8 => dynasm!(self ; $ins Rx((dst as u8)), xmm8, Rq((x as u8))), + XMM::XMM9 => dynasm!(self ; $ins Rx((dst as u8)), xmm9, Rq((x as u8))), + XMM::XMM10 => dynasm!(self ; $ins Rx((dst as u8)), xmm10, Rq((x as u8))), + XMM::XMM11 => dynasm!(self ; $ins Rx((dst as u8)), xmm11, Rq((x as u8))), + XMM::XMM12 => dynasm!(self ; $ins Rx((dst as u8)), xmm12, Rq((x as u8))), + XMM::XMM13 => dynasm!(self ; $ins Rx((dst as u8)), xmm13, Rq((x as u8))), + XMM::XMM14 => dynasm!(self ; $ins Rx((dst as u8)), xmm14, Rq((x as u8))), + XMM::XMM15 => dynasm!(self ; $ins Rx((dst as u8)), xmm15, Rq((x as u8))), + }, + GPROrMemory::Memory(base, disp) => match src1 { + XMM::XMM0 => dynasm!(self ; $ins Rx((dst as u8)), xmm0, QWORD [Rq((base as u8)) + disp]), + XMM::XMM1 => dynasm!(self ; $ins Rx((dst as u8)), xmm1, QWORD [Rq((base as u8)) + disp]), + XMM::XMM2 => dynasm!(self ; $ins Rx((dst as u8)), xmm2, QWORD [Rq((base as u8)) + disp]), + XMM::XMM3 => dynasm!(self ; $ins Rx((dst as u8)), xmm3, QWORD [Rq((base as u8)) + disp]), + XMM::XMM4 => dynasm!(self ; $ins Rx((dst as u8)), xmm4, QWORD [Rq((base as u8)) + disp]), + XMM::XMM5 => dynasm!(self ; $ins Rx((dst as u8)), xmm5, QWORD [Rq((base as u8)) + disp]), + XMM::XMM6 => dynasm!(self ; $ins Rx((dst as u8)), xmm6, QWORD [Rq((base as u8)) + disp]), + XMM::XMM7 => dynasm!(self ; $ins Rx((dst as u8)), xmm7, QWORD [Rq((base as u8)) + disp]), + XMM::XMM8 => dynasm!(self ; $ins Rx((dst as u8)), xmm8, QWORD [Rq((base as u8)) + disp]), + XMM::XMM9 => dynasm!(self ; $ins Rx((dst as u8)), xmm9, QWORD [Rq((base as u8)) + disp]), + XMM::XMM10 => dynasm!(self ; $ins Rx((dst as u8)), xmm10, QWORD [Rq((base as u8)) + disp]), + XMM::XMM11 => dynasm!(self ; $ins Rx((dst as u8)), xmm11, QWORD [Rq((base as u8)) + disp]), + XMM::XMM12 => dynasm!(self ; $ins Rx((dst as u8)), xmm12, QWORD [Rq((base as u8)) + disp]), + XMM::XMM13 => dynasm!(self ; $ins Rx((dst as u8)), xmm13, QWORD [Rq((base as u8)) + disp]), + XMM::XMM14 => dynasm!(self ; $ins Rx((dst as u8)), xmm14, QWORD [Rq((base as u8)) + disp]), + XMM::XMM15 => dynasm!(self ; $ins Rx((dst as u8)), xmm15, QWORD [Rq((base as u8)) + disp]), + }, + } + } + } +} + +macro_rules! avx_i2f_32_fn { + ($ins:ident, $name:ident) => { + fn $name(&mut self, src1: XMM, src2: GPROrMemory, dst: XMM) { + match src2 { + GPROrMemory::GPR(x) => match src1 { + XMM::XMM0 => dynasm!(self ; $ins Rx((dst as u8)), xmm0, Rd((x as u8))), + XMM::XMM1 => dynasm!(self ; $ins Rx((dst as u8)), xmm1, Rd((x as u8))), + XMM::XMM2 => dynasm!(self ; $ins Rx((dst as u8)), xmm2, Rd((x as u8))), + XMM::XMM3 => dynasm!(self ; $ins Rx((dst as u8)), xmm3, Rd((x as u8))), + XMM::XMM4 => dynasm!(self ; $ins Rx((dst as u8)), xmm4, Rd((x as u8))), + XMM::XMM5 => dynasm!(self ; $ins Rx((dst as u8)), xmm5, Rd((x as u8))), + XMM::XMM6 => dynasm!(self ; $ins Rx((dst as u8)), xmm6, Rd((x as u8))), + XMM::XMM7 => dynasm!(self ; $ins Rx((dst as u8)), xmm7, Rd((x as u8))), + XMM::XMM8 => dynasm!(self ; $ins Rx((dst as u8)), xmm8, Rd((x as u8))), + XMM::XMM9 => dynasm!(self ; $ins Rx((dst as u8)), xmm9, Rd((x as u8))), + XMM::XMM10 => dynasm!(self ; $ins Rx((dst as u8)), xmm10, Rd((x as u8))), + XMM::XMM11 => dynasm!(self ; $ins Rx((dst as u8)), xmm11, Rd((x as u8))), + XMM::XMM12 => dynasm!(self ; $ins Rx((dst as u8)), xmm12, Rd((x as u8))), + XMM::XMM13 => dynasm!(self ; $ins Rx((dst as u8)), xmm13, Rd((x as u8))), + XMM::XMM14 => dynasm!(self ; $ins Rx((dst as u8)), xmm14, Rd((x as u8))), + XMM::XMM15 => dynasm!(self ; $ins Rx((dst as u8)), xmm15, Rd((x as u8))), + }, + GPROrMemory::Memory(base, disp) => match src1 { + XMM::XMM0 => dynasm!(self ; $ins Rx((dst as u8)), xmm0, DWORD [Rq((base as u8)) + disp]), + XMM::XMM1 => dynasm!(self ; $ins Rx((dst as u8)), xmm1, DWORD [Rq((base as u8)) + disp]), + XMM::XMM2 => dynasm!(self ; $ins Rx((dst as u8)), xmm2, DWORD [Rq((base as u8)) + disp]), + XMM::XMM3 => dynasm!(self ; $ins Rx((dst as u8)), xmm3, DWORD [Rq((base as u8)) + disp]), + XMM::XMM4 => dynasm!(self ; $ins Rx((dst as u8)), xmm4, DWORD [Rq((base as u8)) + disp]), + XMM::XMM5 => dynasm!(self ; $ins Rx((dst as u8)), xmm5, DWORD [Rq((base as u8)) + disp]), + XMM::XMM6 => dynasm!(self ; $ins Rx((dst as u8)), xmm6, DWORD [Rq((base as u8)) + disp]), + XMM::XMM7 => dynasm!(self ; $ins Rx((dst as u8)), xmm7, DWORD [Rq((base as u8)) + disp]), + XMM::XMM8 => dynasm!(self ; $ins Rx((dst as u8)), xmm8, DWORD [Rq((base as u8)) + disp]), + XMM::XMM9 => dynasm!(self ; $ins Rx((dst as u8)), xmm9, DWORD [Rq((base as u8)) + disp]), + XMM::XMM10 => dynasm!(self ; $ins Rx((dst as u8)), xmm10, DWORD [Rq((base as u8)) + disp]), + XMM::XMM11 => dynasm!(self ; $ins Rx((dst as u8)), xmm11, DWORD [Rq((base as u8)) + disp]), + XMM::XMM12 => dynasm!(self ; $ins Rx((dst as u8)), xmm12, DWORD [Rq((base as u8)) + disp]), + XMM::XMM13 => dynasm!(self ; $ins Rx((dst as u8)), xmm13, DWORD [Rq((base as u8)) + disp]), + XMM::XMM14 => dynasm!(self ; $ins Rx((dst as u8)), xmm14, DWORD [Rq((base as u8)) + disp]), + XMM::XMM15 => dynasm!(self ; $ins Rx((dst as u8)), xmm15, DWORD [Rq((base as u8)) + disp]), + }, + } + } + } +} + +macro_rules! avx_round_fn { + ($ins:ident, $name:ident, $mode:expr) => { + fn $name(&mut self, src1: XMM, src2: XMMOrMemory, dst: XMM) { + match src2 { + XMMOrMemory::XMM(x) => dynasm!(self ; $ins Rx((dst as u8)), Rx((src1 as u8)), Rx((x as u8)), $mode), + XMMOrMemory::Memory(base, disp) => dynasm!(self ; $ins Rx((dst as u8)), Rx((src1 as u8)), [Rq((base as u8)) + disp], $mode), + } + } + } +} + +impl Emitter for Assembler { + type Label = DynamicLabel; + type Offset = AssemblyOffset; + + fn get_label(&mut self) -> DynamicLabel { + self.new_dynamic_label() + } + + fn get_offset(&self) -> AssemblyOffset { + self.offset() + } + + fn get_jmp_instr_size(&self) -> u8 { + 5 + } + + fn emit_u64(&mut self, x: u64) { + self.push_u64(x); + } + + fn emit_label(&mut self, label: Self::Label) { + dynasm!(self ; => label); + } + + fn emit_nop(&mut self) { + dynasm!(self ; nop); + } + + fn emit_mov(&mut self, sz: Size, src: Location, dst: Location) { + binop_all_nofp!(mov, self, sz, src, dst, { + binop_imm64_gpr!(mov, self, sz, src, dst, { + match (sz, src, dst) { + (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; mov [Rq(dst as u8) + disp], Rb(src as u8)); + } + (Size::S8, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; mov Rb(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S8, Location::Imm32(src), Location::GPR(dst)) => { + dynasm!(self ; mov Rb(dst as u8), src as i8); + } + (Size::S8, Location::Imm64(src), Location::GPR(dst)) => { + dynasm!(self ; mov Rb(dst as u8), src as i8); + } + (Size::S8, Location::Imm32(src), Location::Memory(dst, disp)) => { + dynasm!(self ; mov BYTE [Rq(dst as u8) + disp], src as i8); + } + (Size::S8, Location::Imm64(src), Location::Memory(dst, disp)) => { + dynasm!(self ; mov BYTE [Rq(dst as u8) + disp], src as i8); + } + (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; mov [Rq(dst as u8) + disp], Rw(src as u8)); + } + (Size::S16, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; mov Rw(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S16, Location::Imm32(src), Location::GPR(dst)) => { + dynasm!(self ; mov Rw(dst as u8), src as i16); + } + (Size::S16, Location::Imm64(src), Location::GPR(dst)) => { + dynasm!(self ; mov Rw(dst as u8), src as i16); + } + (Size::S16, Location::Imm32(src), Location::Memory(dst, disp)) => { + dynasm!(self ; mov WORD [Rq(dst as u8) + disp], src as i16); + } + (Size::S16, Location::Imm64(src), Location::Memory(dst, disp)) => { + dynasm!(self ; mov WORD [Rq(dst as u8) + disp], src as i16); + } + (Size::S32, Location::Imm64(src), Location::GPR(dst)) => { + dynasm!(self ; mov Rd(dst as u8), src as i32); + } + (Size::S32, Location::Imm64(src), Location::Memory(dst, disp)) => { + dynasm!(self ; mov DWORD [Rq(dst as u8) + disp], src as i32); + } + (Size::S32, Location::GPR(src), Location::XMM(dst)) => { + dynasm!(self ; movd Rx(dst as u8), Rd(src as u8)); + } + (Size::S32, Location::XMM(src), Location::GPR(dst)) => { + dynasm!(self ; movd Rd(dst as u8), Rx(src as u8)); + } + (Size::S32, Location::Memory(src, disp), Location::XMM(dst)) => { + dynasm!(self ; movd Rx(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S32, Location::XMM(src), Location::Memory(dst, disp)) => { + dynasm!(self ; movd [Rq(dst as u8) + disp], Rx(src as u8)); + } + + (Size::S64, Location::GPR(src), Location::XMM(dst)) => { + dynasm!(self ; movq Rx(dst as u8), Rq(src as u8)); + } + (Size::S64, Location::XMM(src), Location::GPR(dst)) => { + dynasm!(self ; movq Rq(dst as u8), Rx(src as u8)); + } + (Size::S64, Location::Memory(src, disp), Location::XMM(dst)) => { + dynasm!(self ; movq Rx(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S64, Location::XMM(src), Location::Memory(dst, disp)) => { + dynasm!(self ; movq [Rq(dst as u8) + disp], Rx(src as u8)); + } + (_, Location::XMM(src), Location::XMM(dst)) => { + dynasm!(self ; movq Rx(dst as u8), Rx(src as u8)); + } + + _ => panic!("singlepass can't emit MOV {:?} {:?} {:?}", sz, src, dst), + } + }) + }); + } + fn emit_lea(&mut self, sz: Size, src: Location, dst: Location) { + match (sz, src, dst) { + (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; lea Rd(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; lea Rq(dst as u8), [Rq(src as u8) + disp]); + } + _ => panic!("singlepass can't emit LEA {:?} {:?} {:?}", sz, src, dst), + } + } + fn emit_lea_label(&mut self, label: Self::Label, dst: Location) { + match dst { + Location::GPR(x) => { + dynasm!(self ; lea Rq(x as u8), [=>label]); + } + _ => panic!("singlepass can't emit LEA label={:?} {:?}", label, dst), + } + } + fn emit_cdq(&mut self) { + dynasm!(self ; cdq); + } + fn emit_cqo(&mut self) { + dynasm!(self ; cqo); + } + fn emit_xor(&mut self, sz: Size, src: Location, dst: Location) { + binop_all_nofp!(xor, self, sz, src, dst, { + panic!("singlepass can't emit XOR {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_jmp(&mut self, condition: Condition, label: Self::Label) { + match condition { + Condition::None => jmp_op!(jmp, self, label), + Condition::Above => jmp_op!(ja, self, label), + Condition::AboveEqual => jmp_op!(jae, self, label), + Condition::Below => jmp_op!(jb, self, label), + Condition::BelowEqual => jmp_op!(jbe, self, label), + Condition::Greater => jmp_op!(jg, self, label), + Condition::GreaterEqual => jmp_op!(jge, self, label), + Condition::Less => jmp_op!(jl, self, label), + Condition::LessEqual => jmp_op!(jle, self, label), + Condition::Equal => jmp_op!(je, self, label), + Condition::NotEqual => jmp_op!(jne, self, label), + Condition::Signed => jmp_op!(js, self, label), + } + } + fn emit_jmp_location(&mut self, loc: Location) { + match loc { + Location::GPR(x) => dynasm!(self ; jmp Rq(x as u8)), + Location::Memory(base, disp) => dynasm!(self ; jmp QWORD [Rq(base as u8) + disp]), + _ => panic!("singlepass can't emit JMP {:?}", loc), + } + } + fn emit_conditional_trap(&mut self, condition: Condition) { + match condition { + Condition::None => trap_op!(jmp, self), + Condition::Above => trap_op!(ja, self), + Condition::AboveEqual => trap_op!(jae, self), + Condition::Below => trap_op!(jb, self), + Condition::BelowEqual => trap_op!(jbe, self), + Condition::Greater => trap_op!(jg, self), + Condition::GreaterEqual => trap_op!(jge, self), + Condition::Less => trap_op!(jl, self), + Condition::LessEqual => trap_op!(jle, self), + Condition::Equal => trap_op!(je, self), + Condition::NotEqual => trap_op!(jne, self), + Condition::Signed => trap_op!(js, self), + } + } + fn emit_set(&mut self, condition: Condition, dst: GPR) { + match condition { + Condition::Above => dynasm!(self ; seta Rb(dst as u8)), + Condition::AboveEqual => dynasm!(self ; setae Rb(dst as u8)), + Condition::Below => dynasm!(self ; setb Rb(dst as u8)), + Condition::BelowEqual => dynasm!(self ; setbe Rb(dst as u8)), + Condition::Greater => dynasm!(self ; setg Rb(dst as u8)), + Condition::GreaterEqual => dynasm!(self ; setge Rb(dst as u8)), + Condition::Less => dynasm!(self ; setl Rb(dst as u8)), + Condition::LessEqual => dynasm!(self ; setle Rb(dst as u8)), + Condition::Equal => dynasm!(self ; sete Rb(dst as u8)), + Condition::NotEqual => dynasm!(self ; setne Rb(dst as u8)), + Condition::Signed => dynasm!(self ; sets Rb(dst as u8)), + _ => panic!("singlepass can't emit SET {:?} {:?}", condition, dst), + } + } + fn emit_push(&mut self, sz: Size, src: Location) { + match (sz, src) { + (Size::S64, Location::Imm32(src)) => dynasm!(self ; push src as i32), + (Size::S64, Location::GPR(src)) => dynasm!(self ; push Rq(src as u8)), + (Size::S64, Location::Memory(src, disp)) => { + dynasm!(self ; push QWORD [Rq(src as u8) + disp]) + } + _ => panic!("singlepass can't emit PUSH {:?} {:?}", sz, src), + } + } + fn emit_pop(&mut self, sz: Size, dst: Location) { + match (sz, dst) { + (Size::S64, Location::GPR(dst)) => dynasm!(self ; pop Rq(dst as u8)), + (Size::S64, Location::Memory(dst, disp)) => { + dynasm!(self ; pop QWORD [Rq(dst as u8) + disp]) + } + _ => panic!("singlepass can't emit POP {:?} {:?}", sz, dst), + } + } + fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) { + binop_all_nofp!(cmp, self, sz, left, right, { + panic!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right); + }); + } + fn emit_add(&mut self, sz: Size, src: Location, dst: Location) { + binop_all_nofp!(add, self, sz, src, dst, { + panic!("singlepass can't emit ADD {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_sub(&mut self, sz: Size, src: Location, dst: Location) { + binop_all_nofp!(sub, self, sz, src, dst, { + panic!("singlepass can't emit SUB {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_neg(&mut self, sz: Size, value: Location) { + match (sz, value) { + (Size::S8, Location::GPR(value)) => dynasm!(self ; neg Rb(value as u8)), + (Size::S8, Location::Memory(value, disp)) => { + dynasm!(self ; neg [Rq(value as u8) + disp]) + } + (Size::S16, Location::GPR(value)) => dynasm!(self ; neg Rw(value as u8)), + (Size::S16, Location::Memory(value, disp)) => { + dynasm!(self ; neg [Rq(value as u8) + disp]) + } + (Size::S32, Location::GPR(value)) => dynasm!(self ; neg Rd(value as u8)), + (Size::S32, Location::Memory(value, disp)) => { + dynasm!(self ; neg [Rq(value as u8) + disp]) + } + (Size::S64, Location::GPR(value)) => dynasm!(self ; neg Rq(value as u8)), + (Size::S64, Location::Memory(value, disp)) => { + dynasm!(self ; neg [Rq(value as u8) + disp]) + } + _ => panic!("singlepass can't emit NEG {:?} {:?}", sz, value), + } + } + fn emit_imul(&mut self, sz: Size, src: Location, dst: Location) { + binop_gpr_gpr!(imul, self, sz, src, dst, { + binop_mem_gpr!(imul, self, sz, src, dst, { + panic!("singlepass can't emit IMUL {:?} {:?} {:?}", sz, src, dst) + }) + }); + } + fn emit_imul_imm32_gpr64(&mut self, src: u32, dst: GPR) { + dynasm!(self ; imul Rq(dst as u8), Rq(dst as u8), src as i32); + } + fn emit_div(&mut self, sz: Size, divisor: Location) { + unop_gpr_or_mem!(div, self, sz, divisor, { + panic!("singlepass can't emit DIV {:?} {:?}", sz, divisor) + }); + } + fn emit_idiv(&mut self, sz: Size, divisor: Location) { + unop_gpr_or_mem!(idiv, self, sz, divisor, { + panic!("singlepass can't emit IDIV {:?} {:?}", sz, divisor) + }); + } + fn emit_shl(&mut self, sz: Size, src: Location, dst: Location) { + binop_shift!(shl, self, sz, src, dst, { + panic!("singlepass can't emit SHL {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_shr(&mut self, sz: Size, src: Location, dst: Location) { + binop_shift!(shr, self, sz, src, dst, { + panic!("singlepass can't emit SHR {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_sar(&mut self, sz: Size, src: Location, dst: Location) { + binop_shift!(sar, self, sz, src, dst, { + panic!("singlepass can't emit SAR {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_rol(&mut self, sz: Size, src: Location, dst: Location) { + binop_shift!(rol, self, sz, src, dst, { + panic!("singlepass can't emit ROL {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_ror(&mut self, sz: Size, src: Location, dst: Location) { + binop_shift!(ror, self, sz, src, dst, { + panic!("singlepass can't emit ROR {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_and(&mut self, sz: Size, src: Location, dst: Location) { + binop_all_nofp!(and, self, sz, src, dst, { + panic!("singlepass can't emit AND {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_or(&mut self, sz: Size, src: Location, dst: Location) { + binop_all_nofp!(or, self, sz, src, dst, { + panic!("singlepass can't emit OR {:?} {:?} {:?}", sz, src, dst) + }); + } + fn emit_bsr(&mut self, sz: Size, src: Location, dst: Location) { + binop_gpr_gpr!(bsr, self, sz, src, dst, { + binop_mem_gpr!(bsr, self, sz, src, dst, { + panic!("singlepass can't emit BSR {:?} {:?} {:?}", sz, src, dst) + }) + }); + } + fn emit_bsf(&mut self, sz: Size, src: Location, dst: Location) { + binop_gpr_gpr!(bsf, self, sz, src, dst, { + binop_mem_gpr!(bsf, self, sz, src, dst, { + panic!("singlepass can't emit BSF {:?} {:?} {:?}", sz, src, dst) + }) + }); + } + fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) { + binop_gpr_gpr!(popcnt, self, sz, src, dst, { + binop_mem_gpr!(popcnt, self, sz, src, dst, { + panic!("singlepass can't emit POPCNT {:?} {:?} {:?}", sz, src, dst) + }) + }); + } + fn emit_movzx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location) { + match (sz_src, src, sz_dst, dst) { + (Size::S8, Location::GPR(src), Size::S32, Location::GPR(dst)) => { + dynasm!(self ; movzx Rd(dst as u8), Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Size::S32, Location::GPR(dst)) => { + dynasm!(self ; movzx Rd(dst as u8), Rw(src as u8)); + } + (Size::S8, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => { + dynasm!(self ; movzx Rd(dst as u8), BYTE [Rq(src as u8) + disp]); + } + (Size::S16, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => { + dynasm!(self ; movzx Rd(dst as u8), WORD [Rq(src as u8) + disp]); + } + (Size::S8, Location::GPR(src), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movzx Rq(dst as u8), Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movzx Rq(dst as u8), Rw(src as u8)); + } + (Size::S8, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movzx Rq(dst as u8), BYTE [Rq(src as u8) + disp]); + } + (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movzx Rq(dst as u8), WORD [Rq(src as u8) + disp]); + } + _ => panic!( + "singlepass can't emit MOVZX {:?} {:?} {:?} {:?}", + sz_src, src, sz_dst, dst + ), + } + } + fn emit_movsx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location) { + match (sz_src, src, sz_dst, dst) { + (Size::S8, Location::GPR(src), Size::S32, Location::GPR(dst)) => { + dynasm!(self ; movsx Rd(dst as u8), Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Size::S32, Location::GPR(dst)) => { + dynasm!(self ; movsx Rd(dst as u8), Rw(src as u8)); + } + (Size::S8, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => { + dynasm!(self ; movsx Rd(dst as u8), BYTE [Rq(src as u8) + disp]); + } + (Size::S16, Location::Memory(src, disp), Size::S32, Location::GPR(dst)) => { + dynasm!(self ; movsx Rd(dst as u8), WORD [Rq(src as u8) + disp]); + } + (Size::S8, Location::GPR(src), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movsx Rq(dst as u8), Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movsx Rq(dst as u8), Rw(src as u8)); + } + (Size::S32, Location::GPR(src), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movsx Rq(dst as u8), Rd(src as u8)); + } + (Size::S8, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movsx Rq(dst as u8), BYTE [Rq(src as u8) + disp]); + } + (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movsx Rq(dst as u8), WORD [Rq(src as u8) + disp]); + } + (Size::S32, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { + dynasm!(self ; movsx Rq(dst as u8), DWORD [Rq(src as u8) + disp]); + } + _ => panic!( + "singlepass can't emit MOVSX {:?} {:?} {:?} {:?}", + sz_src, src, sz_dst, dst + ), + } + } + + fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location) { + match (sz, src, dst) { + (Size::S8, Location::GPR(src), Location::GPR(dst)) => { + dynasm!(self ; xchg Rb(dst as u8), Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Location::GPR(dst)) => { + dynasm!(self ; xchg Rw(dst as u8), Rw(src as u8)); + } + (Size::S32, Location::GPR(src), Location::GPR(dst)) => { + dynasm!(self ; xchg Rd(dst as u8), Rd(src as u8)); + } + (Size::S64, Location::GPR(src), Location::GPR(dst)) => { + dynasm!(self ; xchg Rq(dst as u8), Rq(src as u8)); + } + (Size::S8, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; xchg Rb(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; xchg [Rq(dst as u8) + disp], Rb(src as u8)); + } + (Size::S16, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; xchg Rw(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; xchg [Rq(dst as u8) + disp], Rw(src as u8)); + } + (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; xchg Rd(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; xchg [Rq(dst as u8) + disp], Rd(src as u8)); + } + (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; xchg Rq(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; xchg [Rq(dst as u8) + disp], Rq(src as u8)); + } + _ => panic!("singlepass can't emit XCHG {:?} {:?} {:?}", sz, src, dst), + } + } + + fn emit_lock_xadd(&mut self, sz: Size, src: Location, dst: Location) { + match (sz, src, dst) { + (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rw(src as u8)); + } + (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rd(src as u8)); + } + (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rq(src as u8)); + } + _ => panic!( + "singlepass can't emit LOCK XADD {:?} {:?} {:?}", + sz, src, dst + ), + } + } + + fn emit_lock_cmpxchg(&mut self, sz: Size, src: Location, dst: Location) { + match (sz, src, dst) { + (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rw(src as u8)); + } + (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rd(src as u8)); + } + (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rq(src as u8)); + } + _ => panic!( + "singlepass can't emit LOCK CMPXCHG {:?} {:?} {:?}", + sz, src, dst + ), + } + } + + fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR) { + dynasm!(self ; btc Rd(dst as u8), BYTE src as i8); + } + + fn emit_btc_gpr_imm8_64(&mut self, src: u8, dst: GPR) { + dynasm!(self ; btc Rq(dst as u8), BYTE src as i8); + } + + fn emit_cmovae_gpr_32(&mut self, src: GPR, dst: GPR) { + dynasm!(self ; cmovae Rd(dst as u8), Rd(src as u8)); + } + + fn emit_cmovae_gpr_64(&mut self, src: GPR, dst: GPR) { + dynasm!(self ; cmovae Rq(dst as u8), Rq(src as u8)); + } + + fn emit_vmovaps(&mut self, src: XMMOrMemory, dst: XMMOrMemory) { + match (src, dst) { + (XMMOrMemory::XMM(src), XMMOrMemory::XMM(dst)) => { + dynasm!(self ; movaps Rx(dst as u8), Rx(src as u8)) + } + (XMMOrMemory::Memory(base, disp), XMMOrMemory::XMM(dst)) => { + dynasm!(self ; movaps Rx(dst as u8), [Rq(base as u8) + disp]) + } + (XMMOrMemory::XMM(src), XMMOrMemory::Memory(base, disp)) => { + dynasm!(self ; movaps [Rq(base as u8) + disp], Rx(src as u8)) + } + _ => panic!("singlepass can't emit VMOVAPS {:?} {:?}", src, dst), + }; + } + + fn emit_vmovapd(&mut self, src: XMMOrMemory, dst: XMMOrMemory) { + match (src, dst) { + (XMMOrMemory::XMM(src), XMMOrMemory::XMM(dst)) => { + dynasm!(self ; movapd Rx(dst as u8), Rx(src as u8)) + } + (XMMOrMemory::Memory(base, disp), XMMOrMemory::XMM(dst)) => { + dynasm!(self ; movapd Rx(dst as u8), [Rq(base as u8) + disp]) + } + (XMMOrMemory::XMM(src), XMMOrMemory::Memory(base, disp)) => { + dynasm!(self ; movapd [Rq(base as u8) + disp], Rx(src as u8)) + } + _ => panic!("singlepass can't emit VMOVAPD {:?} {:?}", src, dst), + }; + } + + avx_fn!(vxorps, emit_vxorps); + avx_fn!(vxorpd, emit_vxorpd); + + avx_fn!(vaddss, emit_vaddss); + avx_fn!(vaddsd, emit_vaddsd); + + avx_fn!(vsubss, emit_vsubss); + avx_fn!(vsubsd, emit_vsubsd); + + avx_fn!(vmulss, emit_vmulss); + avx_fn!(vmulsd, emit_vmulsd); + + avx_fn!(vdivss, emit_vdivss); + avx_fn!(vdivsd, emit_vdivsd); + + avx_fn!(vmaxss, emit_vmaxss); + avx_fn!(vmaxsd, emit_vmaxsd); + + avx_fn!(vminss, emit_vminss); + avx_fn!(vminsd, emit_vminsd); + + avx_fn!(vcmpeqss, emit_vcmpeqss); + avx_fn!(vcmpeqsd, emit_vcmpeqsd); + + avx_fn!(vcmpneqss, emit_vcmpneqss); + avx_fn!(vcmpneqsd, emit_vcmpneqsd); + + avx_fn!(vcmpltss, emit_vcmpltss); + avx_fn!(vcmpltsd, emit_vcmpltsd); + + avx_fn!(vcmpless, emit_vcmpless); + avx_fn!(vcmplesd, emit_vcmplesd); + + avx_fn!(vcmpgtss, emit_vcmpgtss); + avx_fn!(vcmpgtsd, emit_vcmpgtsd); + + avx_fn!(vcmpgess, emit_vcmpgess); + avx_fn!(vcmpgesd, emit_vcmpgesd); + + avx_fn!(vcmpunordss, emit_vcmpunordss); + avx_fn!(vcmpunordsd, emit_vcmpunordsd); + + avx_fn!(vcmpordss, emit_vcmpordss); + avx_fn!(vcmpordsd, emit_vcmpordsd); + + avx_fn!(vsqrtss, emit_vsqrtss); + avx_fn!(vsqrtsd, emit_vsqrtsd); + + avx_fn!(vcvtss2sd, emit_vcvtss2sd); + avx_fn!(vcvtsd2ss, emit_vcvtsd2ss); + + avx_round_fn!(vroundss, emit_vroundss_nearest, 0); + avx_round_fn!(vroundss, emit_vroundss_floor, 1); + avx_round_fn!(vroundss, emit_vroundss_ceil, 2); + avx_round_fn!(vroundss, emit_vroundss_trunc, 3); + avx_round_fn!(vroundsd, emit_vroundsd_nearest, 0); + avx_round_fn!(vroundsd, emit_vroundsd_floor, 1); + avx_round_fn!(vroundsd, emit_vroundsd_ceil, 2); + avx_round_fn!(vroundsd, emit_vroundsd_trunc, 3); + + avx_i2f_32_fn!(vcvtsi2ss, emit_vcvtsi2ss_32); + avx_i2f_32_fn!(vcvtsi2sd, emit_vcvtsi2sd_32); + avx_i2f_64_fn!(vcvtsi2ss, emit_vcvtsi2ss_64); + avx_i2f_64_fn!(vcvtsi2sd, emit_vcvtsi2sd_64); + + fn emit_vblendvps(&mut self, src1: XMM, src2: XMMOrMemory, mask: XMM, dst: XMM) { + match src2 { + XMMOrMemory::XMM(src2) => { + dynasm!(self ; vblendvps Rx(dst as u8), Rx(mask as u8), Rx(src2 as u8), Rx(src1 as u8)) + } + XMMOrMemory::Memory(base, disp) => { + dynasm!(self ; vblendvps Rx(dst as u8), Rx(mask as u8), [Rq(base as u8) + disp], Rx(src1 as u8)) + } + } + } + + fn emit_vblendvpd(&mut self, src1: XMM, src2: XMMOrMemory, mask: XMM, dst: XMM) { + match src2 { + XMMOrMemory::XMM(src2) => { + dynasm!(self ; vblendvpd Rx(dst as u8), Rx(mask as u8), Rx(src2 as u8), Rx(src1 as u8)) + } + XMMOrMemory::Memory(base, disp) => { + dynasm!(self ; vblendvpd Rx(dst as u8), Rx(mask as u8), [Rq(base as u8) + disp], Rx(src1 as u8)) + } + } + } + + fn emit_ucomiss(&mut self, src: XMMOrMemory, dst: XMM) { + match src { + XMMOrMemory::XMM(x) => dynasm!(self ; ucomiss Rx(dst as u8), Rx(x as u8)), + XMMOrMemory::Memory(base, disp) => { + dynasm!(self ; ucomiss Rx(dst as u8), [Rq(base as u8) + disp]) + } + } + } + + fn emit_ucomisd(&mut self, src: XMMOrMemory, dst: XMM) { + match src { + XMMOrMemory::XMM(x) => dynasm!(self ; ucomisd Rx(dst as u8), Rx(x as u8)), + XMMOrMemory::Memory(base, disp) => { + dynasm!(self ; ucomisd Rx(dst as u8), [Rq(base as u8) + disp]) + } + } + } + + fn emit_cvttss2si_32(&mut self, src: XMMOrMemory, dst: GPR) { + match src { + XMMOrMemory::XMM(x) => dynasm!(self ; cvttss2si Rd(dst as u8), Rx(x as u8)), + XMMOrMemory::Memory(base, disp) => { + dynasm!(self ; cvttss2si Rd(dst as u8), [Rq(base as u8) + disp]) + } + } + } + + fn emit_cvttss2si_64(&mut self, src: XMMOrMemory, dst: GPR) { + match src { + XMMOrMemory::XMM(x) => dynasm!(self ; cvttss2si Rq(dst as u8), Rx(x as u8)), + XMMOrMemory::Memory(base, disp) => { + dynasm!(self ; cvttss2si Rq(dst as u8), [Rq(base as u8) + disp]) + } + } + } + + fn emit_cvttsd2si_32(&mut self, src: XMMOrMemory, dst: GPR) { + match src { + XMMOrMemory::XMM(x) => dynasm!(self ; cvttsd2si Rd(dst as u8), Rx(x as u8)), + XMMOrMemory::Memory(base, disp) => { + dynasm!(self ; cvttsd2si Rd(dst as u8), [Rq(base as u8) + disp]) + } + } + } + + fn emit_cvttsd2si_64(&mut self, src: XMMOrMemory, dst: GPR) { + match src { + XMMOrMemory::XMM(x) => dynasm!(self ; cvttsd2si Rq(dst as u8), Rx(x as u8)), + XMMOrMemory::Memory(base, disp) => { + dynasm!(self ; cvttsd2si Rq(dst as u8), [Rq(base as u8) + disp]) + } + } + } + + fn emit_test_gpr_64(&mut self, reg: GPR) { + dynasm!(self ; test Rq(reg as u8), Rq(reg as u8)); + } + + fn emit_ud2(&mut self) { + dynasm!(self ; ud2); + } + fn emit_ret(&mut self) { + dynasm!(self ; ret); + } + + fn emit_call_label(&mut self, label: Self::Label) { + dynasm!(self ; call =>label); + } + fn emit_call_location(&mut self, loc: Location) { + match loc { + Location::GPR(x) => dynasm!(self ; call Rq(x as u8)), + Location::Memory(base, disp) => dynasm!(self ; call QWORD [Rq(base as u8) + disp]), + _ => panic!("singlepass can't emit CALL {:?}", loc), + } + } + + fn emit_bkpt(&mut self) { + dynasm!(self ; int 0x3); + } + + fn emit_host_redirection(&mut self, target: GPR) { + self.emit_jmp_location(Location::GPR(target)); + } + + fn emit_inline_breakpoint(&mut self, ty: InlineBreakpointType) { + dynasm!(self + ; ud2 + ; .byte 0x0f ; .byte 0xb9u8 as i8 // ud + ; int -1 + ; .byte ty as u8 as i8 + ); + } +} diff --git a/lib/compiler-singlepass/src/lib.rs b/lib/compiler-singlepass/src/lib.rs index 19da0d0f8..b4f4e3087 100644 --- a/lib/compiler-singlepass/src/lib.rs +++ b/lib/compiler-singlepass/src/lib.rs @@ -5,11 +5,20 @@ //! including Blockchains and Edge computing where quick compilation //! times are a must, and JIT bombs should never happen. //! -//! Compared to Cranelift and LLVM, Singlepass is much faster to compile. +//! Compared to Cranelift and LLVM, Singlepass compiles much faster but has worse +//! runtime performance. +//! //! > Note: Singlepass currently depends on Rust nightly features. +#![feature(proc_macro_hygiene)] + mod compiler; mod config; +//mod codegen_x64; +mod common_decl; +mod emitter_x64; +mod machine; +mod x64_decl; pub use crate::compiler::SinglepassCompiler; pub use crate::config::SinglepassConfig; diff --git a/lib/compiler-singlepass/src/machine.rs b/lib/compiler-singlepass/src/machine.rs new file mode 100644 index 000000000..cd2ff719a --- /dev/null +++ b/lib/compiler-singlepass/src/machine.rs @@ -0,0 +1,513 @@ +use crate::common_decl::*; +use crate::emitter_x64::*; +use crate::x64_decl::{new_machine_state, X64Register}; +use smallvec::smallvec; +use smallvec::SmallVec; +use std::collections::HashSet; +use wasmparser::Type as WpType; + +struct MachineStackOffset(usize); + +pub struct Machine { + used_gprs: HashSet, + used_xmms: HashSet, + stack_offset: MachineStackOffset, + save_area_offset: Option, + pub state: MachineState, + pub(crate) track_state: bool, +} + +impl Machine { + pub fn new() -> Self { + Machine { + used_gprs: HashSet::new(), + used_xmms: HashSet::new(), + stack_offset: MachineStackOffset(0), + save_area_offset: None, + state: new_machine_state(), + track_state: true, + } + } + + pub fn get_stack_offset(&self) -> usize { + self.stack_offset.0 + } + + pub fn get_used_gprs(&self) -> Vec { + self.used_gprs.iter().cloned().collect() + } + + pub fn get_used_xmms(&self) -> Vec { + self.used_xmms.iter().cloned().collect() + } + + pub fn get_vmctx_reg() -> GPR { + GPR::R15 + } + + /// Picks an unused general purpose register for local/stack/argument use. + /// + /// This method does not mark the register as used. + pub fn pick_gpr(&self) -> Option { + use GPR::*; + static REGS: &'static [GPR] = &[RSI, RDI, R8, R9, R10, R11]; + for r in REGS { + if !self.used_gprs.contains(r) { + return Some(*r); + } + } + None + } + + /// Picks an unused general purpose register for internal temporary use. + /// + /// This method does not mark the register as used. + pub fn pick_temp_gpr(&self) -> Option { + use GPR::*; + static REGS: &'static [GPR] = &[RAX, RCX, RDX]; + for r in REGS { + if !self.used_gprs.contains(r) { + return Some(*r); + } + } + None + } + + /// Acquires a temporary GPR. + pub fn acquire_temp_gpr(&mut self) -> Option { + let gpr = self.pick_temp_gpr(); + if let Some(x) = gpr { + self.used_gprs.insert(x); + } + gpr + } + + /// Releases a temporary GPR. + pub fn release_temp_gpr(&mut self, gpr: GPR) { + assert!(self.used_gprs.remove(&gpr)); + } + + /// Specify that a given register is in use. + pub fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR { + assert!(!self.used_gprs.contains(&gpr)); + self.used_gprs.insert(gpr); + gpr + } + + /// Picks an unused XMM register. + /// + /// This method does not mark the register as used. + pub fn pick_xmm(&self) -> Option { + use XMM::*; + static REGS: &'static [XMM] = &[XMM3, XMM4, XMM5, XMM6, XMM7]; + for r in REGS { + if !self.used_xmms.contains(r) { + return Some(*r); + } + } + None + } + + /// Picks an unused XMM register for internal temporary use. + /// + /// This method does not mark the register as used. + pub fn pick_temp_xmm(&self) -> Option { + use XMM::*; + static REGS: &'static [XMM] = &[XMM0, XMM1, XMM2]; + for r in REGS { + if !self.used_xmms.contains(r) { + return Some(*r); + } + } + None + } + + /// Acquires a temporary XMM register. + pub fn acquire_temp_xmm(&mut self) -> Option { + let xmm = self.pick_temp_xmm(); + if let Some(x) = xmm { + self.used_xmms.insert(x); + } + xmm + } + + /// Releases a temporary XMM register. + pub fn release_temp_xmm(&mut self, xmm: XMM) { + assert_eq!(self.used_xmms.remove(&xmm), true); + } + + /// Acquires locations from the machine state. + /// + /// If the returned locations are used for stack value, `release_location` needs to be called on them; + /// Otherwise, if the returned locations are used for locals, `release_location` does not need to be called on them. + pub fn acquire_locations( + &mut self, + assembler: &mut E, + tys: &[(WpType, MachineValue)], + zeroed: bool, + ) -> SmallVec<[Location; 1]> { + let mut ret = smallvec![]; + let mut delta_stack_offset: usize = 0; + + for (ty, mv) in tys { + let loc = match *ty { + WpType::F32 | WpType::F64 => self.pick_xmm().map(Location::XMM), + WpType::I32 | WpType::I64 => self.pick_gpr().map(Location::GPR), + _ => unreachable!(), + }; + + let loc = if let Some(x) = loc { + x + } else { + self.stack_offset.0 += 8; + delta_stack_offset += 8; + Location::Memory(GPR::RBP, -(self.stack_offset.0 as i32)) + }; + if let Location::GPR(x) = loc { + self.used_gprs.insert(x); + self.state.register_values[X64Register::GPR(x).to_index().0] = mv.clone(); + } else if let Location::XMM(x) = loc { + self.used_xmms.insert(x); + self.state.register_values[X64Register::XMM(x).to_index().0] = mv.clone(); + } else { + self.state.stack_values.push(mv.clone()); + } + self.state.wasm_stack.push(WasmAbstractValue::Runtime); + ret.push(loc); + } + + if delta_stack_offset != 0 { + assembler.emit_sub( + Size::S64, + Location::Imm32(delta_stack_offset as u32), + Location::GPR(GPR::RSP), + ); + } + if zeroed { + for i in 0..tys.len() { + assembler.emit_mov(Size::S64, Location::Imm32(0), ret[i]); + } + } + ret + } + + /// Releases locations used for stack value. + pub fn release_locations(&mut self, assembler: &mut E, locs: &[Location]) { + let mut delta_stack_offset: usize = 0; + + for loc in locs.iter().rev() { + match *loc { + Location::GPR(ref x) => { + assert_eq!(self.used_gprs.remove(x), true); + self.state.register_values[X64Register::GPR(*x).to_index().0] = + MachineValue::Undefined; + } + Location::XMM(ref x) => { + assert_eq!(self.used_xmms.remove(x), true); + self.state.register_values[X64Register::XMM(*x).to_index().0] = + MachineValue::Undefined; + } + Location::Memory(GPR::RBP, x) => { + if x >= 0 { + unreachable!(); + } + let offset = (-x) as usize; + if offset != self.stack_offset.0 { + unreachable!(); + } + self.stack_offset.0 -= 8; + delta_stack_offset += 8; + self.state.stack_values.pop().unwrap(); + } + _ => {} + } + self.state.wasm_stack.pop().unwrap(); + } + + if delta_stack_offset != 0 { + assembler.emit_add( + Size::S64, + Location::Imm32(delta_stack_offset as u32), + Location::GPR(GPR::RSP), + ); + } + } + + pub fn release_locations_only_regs(&mut self, locs: &[Location]) { + for loc in locs.iter().rev() { + match *loc { + Location::GPR(ref x) => { + assert_eq!(self.used_gprs.remove(x), true); + self.state.register_values[X64Register::GPR(*x).to_index().0] = + MachineValue::Undefined; + } + Location::XMM(ref x) => { + assert_eq!(self.used_xmms.remove(x), true); + self.state.register_values[X64Register::XMM(*x).to_index().0] = + MachineValue::Undefined; + } + _ => {} + } + // Wasm state popping is deferred to `release_locations_only_osr_state`. + } + } + + pub fn release_locations_only_stack( + &mut self, + assembler: &mut E, + locs: &[Location], + ) { + let mut delta_stack_offset: usize = 0; + + for loc in locs.iter().rev() { + match *loc { + Location::Memory(GPR::RBP, x) => { + if x >= 0 { + unreachable!(); + } + let offset = (-x) as usize; + if offset != self.stack_offset.0 { + unreachable!(); + } + self.stack_offset.0 -= 8; + delta_stack_offset += 8; + self.state.stack_values.pop().unwrap(); + } + _ => {} + } + // Wasm state popping is deferred to `release_locations_only_osr_state`. + } + + if delta_stack_offset != 0 { + assembler.emit_add( + Size::S64, + Location::Imm32(delta_stack_offset as u32), + Location::GPR(GPR::RSP), + ); + } + } + + pub fn release_locations_only_osr_state(&mut self, n: usize) { + for _ in 0..n { + self.state.wasm_stack.pop().unwrap(); + } + } + + pub fn release_locations_keep_state(&self, assembler: &mut E, locs: &[Location]) { + let mut delta_stack_offset: usize = 0; + let mut stack_offset = self.stack_offset.0; + + for loc in locs.iter().rev() { + match *loc { + Location::Memory(GPR::RBP, x) => { + if x >= 0 { + unreachable!(); + } + let offset = (-x) as usize; + if offset != stack_offset { + unreachable!(); + } + stack_offset -= 8; + delta_stack_offset += 8; + } + _ => {} + } + } + + if delta_stack_offset != 0 { + assembler.emit_add( + Size::S64, + Location::Imm32(delta_stack_offset as u32), + Location::GPR(GPR::RSP), + ); + } + } + + pub fn init_locals( + &mut self, + a: &mut E, + n: usize, + n_params: usize, + ) -> Vec { + // Use callee-saved registers for locals. + fn get_local_location(idx: usize) -> Location { + match idx { + 0 => Location::GPR(GPR::R12), + 1 => Location::GPR(GPR::R13), + 2 => Location::GPR(GPR::R14), + 3 => Location::GPR(GPR::RBX), + _ => Location::Memory(GPR::RBP, -(((idx - 3) * 8) as i32)), + } + } + + let mut locations: Vec = vec![]; + let mut allocated: usize = 0; + + // Determine locations for parameters. + for i in 0..n_params { + let loc = Self::get_param_location(i + 1); + locations.push(match loc { + Location::GPR(_) => { + let old_idx = allocated; + allocated += 1; + get_local_location(old_idx) + } + Location::Memory(_, _) => { + let old_idx = allocated; + allocated += 1; + get_local_location(old_idx) + } + _ => unreachable!(), + }); + } + + // Determine locations for normal locals. + for _ in n_params..n { + locations.push(get_local_location(allocated)); + allocated += 1; + } + + for (i, loc) in locations.iter().enumerate() { + match *loc { + Location::GPR(x) => { + self.state.register_values[X64Register::GPR(x).to_index().0] = + MachineValue::WasmLocal(i); + } + Location::Memory(_, _) => { + self.state.stack_values.push(MachineValue::WasmLocal(i)); + } + _ => unreachable!(), + } + } + + // How many machine stack slots did all the locals use? + let num_mem_slots = locations + .iter() + .filter(|&&loc| match loc { + Location::Memory(_, _) => true, + _ => false, + }) + .count(); + + // Move RSP down to reserve space for machine stack slots. + if num_mem_slots > 0 { + a.emit_sub( + Size::S64, + Location::Imm32((num_mem_slots * 8) as u32), + Location::GPR(GPR::RSP), + ); + self.stack_offset.0 += num_mem_slots * 8; + } + + // Save callee-saved registers. + for loc in locations.iter() { + if let Location::GPR(x) = *loc { + a.emit_push(Size::S64, *loc); + self.stack_offset.0 += 8; + self.state.stack_values.push(MachineValue::PreserveRegister( + X64Register::GPR(x).to_index(), + )); + } + } + + // Save R15 for vmctx use. + a.emit_push(Size::S64, Location::GPR(GPR::R15)); + self.stack_offset.0 += 8; + self.state.stack_values.push(MachineValue::PreserveRegister( + X64Register::GPR(GPR::R15).to_index(), + )); + + // Save the offset of static area. + self.save_area_offset = Some(MachineStackOffset(self.stack_offset.0)); + + // Load in-register parameters into the allocated locations. + for i in 0..n_params { + let loc = Self::get_param_location(i + 1); + match loc { + Location::GPR(_) => { + a.emit_mov(Size::S64, loc, locations[i]); + } + Location::Memory(_, _) => match locations[i] { + Location::GPR(_) => { + a.emit_mov(Size::S64, loc, locations[i]); + } + Location::Memory(_, _) => { + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), locations[i]); + } + _ => unreachable!(), + }, + _ => unreachable!(), + } + } + + // Load vmctx. + a.emit_mov( + Size::S64, + Self::get_param_location(0), + Location::GPR(GPR::R15), + ); + + // Initialize all normal locals to zero. + for i in n_params..n { + a.emit_mov(Size::S64, Location::Imm32(0), locations[i]); + } + + locations + } + + pub fn finalize_locals(&mut self, a: &mut E, locations: &[Location]) { + // Unwind stack to the "save area". + a.emit_lea( + Size::S64, + Location::Memory( + GPR::RBP, + -(self.save_area_offset.as_ref().unwrap().0 as i32), + ), + Location::GPR(GPR::RSP), + ); + + // Restore R15 used by vmctx. + a.emit_pop(Size::S64, Location::GPR(GPR::R15)); + + // Restore callee-saved registers. + for loc in locations.iter().rev() { + if let Location::GPR(_) = *loc { + a.emit_pop(Size::S64, *loc); + } + } + } + + pub fn get_param_location(idx: usize) -> Location { + match idx { + 0 => Location::GPR(GPR::RDI), + 1 => Location::GPR(GPR::RSI), + 2 => Location::GPR(GPR::RDX), + 3 => Location::GPR(GPR::RCX), + 4 => Location::GPR(GPR::R8), + 5 => Location::GPR(GPR::R9), + _ => Location::Memory(GPR::RBP, (16 + (idx - 6) * 8) as i32), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use dynasmrt::x64::Assembler; + + #[test] + fn test_release_locations_keep_state_nopanic() { + let mut machine = Machine::new(); + let mut assembler = Assembler::new().unwrap(); + let locs = machine.acquire_locations( + &mut assembler, + &(0..10) + .map(|_| (WpType::I32, MachineValue::Undefined)) + .collect::>(), + false, + ); + + machine.release_locations_keep_state(&mut assembler, &locs); + } +} diff --git a/lib/compiler-singlepass/src/x64_decl.rs b/lib/compiler-singlepass/src/x64_decl.rs new file mode 100644 index 000000000..1181500e0 --- /dev/null +++ b/lib/compiler-singlepass/src/x64_decl.rs @@ -0,0 +1,222 @@ +//! X64 structures. + +use crate::common_decl::{MachineState, MachineValue, RegisterIndex}; +use std::collections::BTreeMap; +use wasm_common::Type; + +/// General-purpose registers. +#[repr(u8)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum GPR { + /// RAX register + RAX, + /// RCX register + RCX, + /// RDX register + RDX, + /// RBX register + RBX, + /// RSP register + RSP, + /// RBP register + RBP, + /// RSI register + RSI, + /// RDI register + RDI, + /// R8 register + R8, + /// R9 register + R9, + /// R10 register + R10, + /// R11 register + R11, + /// R12 register + R12, + /// R13 register + R13, + /// R14 register + R14, + /// R15 register + R15, +} + +/// XMM registers. +#[repr(u8)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum XMM { + /// XMM register 0 + XMM0, + /// XMM register 1 + XMM1, + /// XMM register 2 + XMM2, + /// XMM register 3 + XMM3, + /// XMM register 4 + XMM4, + /// XMM register 5 + XMM5, + /// XMM register 6 + XMM6, + /// XMM register 7 + XMM7, + /// XMM register 8 + XMM8, + /// XMM register 9 + XMM9, + /// XMM register 10 + XMM10, + /// XMM register 11 + XMM11, + /// XMM register 12 + XMM12, + /// XMM register 13 + XMM13, + /// XMM register 14 + XMM14, + /// XMM register 15 + XMM15, +} + +/// A machine register under the x86-64 architecture. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum X64Register { + /// General-purpose registers. + GPR(GPR), + /// XMM (floating point/SIMD) registers. + XMM(XMM), +} + +impl X64Register { + /// Returns the index of the register. + pub fn to_index(&self) -> RegisterIndex { + match *self { + X64Register::GPR(x) => RegisterIndex(x as usize), + X64Register::XMM(x) => RegisterIndex(x as usize + 16), + } + } + + /// Converts a DWARD regnum to X64Register. + pub fn from_dwarf_regnum(x: u16) -> Option { + Some(match x { + 0 => X64Register::GPR(GPR::RAX), + 1 => X64Register::GPR(GPR::RDX), + 2 => X64Register::GPR(GPR::RCX), + 3 => X64Register::GPR(GPR::RBX), + 4 => X64Register::GPR(GPR::RSI), + 5 => X64Register::GPR(GPR::RDI), + 6 => X64Register::GPR(GPR::RBP), + 7 => X64Register::GPR(GPR::RSP), + 8 => X64Register::GPR(GPR::R8), + 9 => X64Register::GPR(GPR::R9), + 10 => X64Register::GPR(GPR::R10), + 11 => X64Register::GPR(GPR::R11), + 12 => X64Register::GPR(GPR::R12), + 13 => X64Register::GPR(GPR::R13), + 14 => X64Register::GPR(GPR::R14), + 15 => X64Register::GPR(GPR::R15), + + 17 => X64Register::XMM(XMM::XMM0), + 18 => X64Register::XMM(XMM::XMM1), + 19 => X64Register::XMM(XMM::XMM2), + 20 => X64Register::XMM(XMM::XMM3), + 21 => X64Register::XMM(XMM::XMM4), + 22 => X64Register::XMM(XMM::XMM5), + 23 => X64Register::XMM(XMM::XMM6), + 24 => X64Register::XMM(XMM::XMM7), + _ => return None, + }) + } + + /// Returns the instruction prefix for `movq %this_reg, ?(%rsp)`. + /// + /// To build an instruction, append the memory location as a 32-bit + /// offset to the stack pointer to this prefix. + pub fn prefix_mov_to_stack(&self) -> Option<&'static [u8]> { + Some(match *self { + X64Register::GPR(gpr) => match gpr { + GPR::RDI => &[0x48, 0x89, 0xbc, 0x24], + GPR::RSI => &[0x48, 0x89, 0xb4, 0x24], + GPR::RDX => &[0x48, 0x89, 0x94, 0x24], + GPR::RCX => &[0x48, 0x89, 0x8c, 0x24], + GPR::R8 => &[0x4c, 0x89, 0x84, 0x24], + GPR::R9 => &[0x4c, 0x89, 0x8c, 0x24], + _ => return None, + }, + X64Register::XMM(xmm) => match xmm { + XMM::XMM0 => &[0x66, 0x0f, 0xd6, 0x84, 0x24], + XMM::XMM1 => &[0x66, 0x0f, 0xd6, 0x8c, 0x24], + XMM::XMM2 => &[0x66, 0x0f, 0xd6, 0x94, 0x24], + XMM::XMM3 => &[0x66, 0x0f, 0xd6, 0x9c, 0x24], + XMM::XMM4 => &[0x66, 0x0f, 0xd6, 0xa4, 0x24], + XMM::XMM5 => &[0x66, 0x0f, 0xd6, 0xac, 0x24], + XMM::XMM6 => &[0x66, 0x0f, 0xd6, 0xb4, 0x24], + XMM::XMM7 => &[0x66, 0x0f, 0xd6, 0xbc, 0x24], + _ => return None, + }, + }) + } +} + +/// An allocator that allocates registers for function arguments according to the System V ABI. +#[derive(Default)] +pub struct ArgumentRegisterAllocator { + n_gprs: usize, + n_xmms: usize, +} + +impl ArgumentRegisterAllocator { + /// Allocates a register for argument type `ty`. Returns `None` if no register is available for this type. + pub fn next(&mut self, ty: Type) -> Option { + static GPR_SEQ: &'static [GPR] = + &[GPR::RDI, GPR::RSI, GPR::RDX, GPR::RCX, GPR::R8, GPR::R9]; + static XMM_SEQ: &'static [XMM] = &[ + XMM::XMM0, + XMM::XMM1, + XMM::XMM2, + XMM::XMM3, + XMM::XMM4, + XMM::XMM5, + XMM::XMM6, + XMM::XMM7, + ]; + match ty { + Type::I32 | Type::I64 => { + if self.n_gprs < GPR_SEQ.len() { + let gpr = GPR_SEQ[self.n_gprs]; + self.n_gprs += 1; + Some(X64Register::GPR(gpr)) + } else { + None + } + } + Type::F32 | Type::F64 => { + if self.n_xmms < XMM_SEQ.len() { + let xmm = XMM_SEQ[self.n_xmms]; + self.n_xmms += 1; + Some(X64Register::XMM(xmm)) + } else { + None + } + } + _ => todo!( + "ArgumentRegisterAllocator::next: Unsupported type: {:?}", + ty + ), + } + } +} + +/// Create a new `MachineState` with default values. +pub fn new_machine_state() -> MachineState { + MachineState { + stack_values: vec![], + register_values: vec![MachineValue::Undefined; 16 + 8], + prev_frame: BTreeMap::new(), + wasm_stack: vec![], + wasm_stack_private_depth: 0, + wasm_inst_offset: std::usize::MAX, + } +} diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index cf3774241..f99422635 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -9,7 +9,9 @@ use crate::target::Target; use crate::FunctionBodyData; use crate::ModuleTranslationState; use wasm_common::entity::PrimaryMap; -use wasm_common::{Features, FunctionType, LocalFunctionIndex, MemoryIndex, TableIndex}; +use wasm_common::{ + Features, FunctionIndex, FunctionType, LocalFunctionIndex, MemoryIndex, TableIndex, +}; use wasmer_runtime::Module; use wasmer_runtime::{MemoryPlan, TablePlan}; use wasmparser::{validate, OperatorValidatorConfig, ValidatingParserConfig}; @@ -80,8 +82,30 @@ pub trait Compiler { /// let func = instance.exports.func("my_func"); /// func.call(&[Value::I32(1)]); /// ``` - fn compile_wasm_trampolines( + fn compile_function_call_trampolines( &self, signatures: &[FunctionType], ) -> Result, CompileError>; + + /// Compile the trampolines to call a dynamic function defined in + /// a host, from a Wasm module. + /// + /// This allows us to create dynamic Wasm functions, such as: + /// + /// ```ignore + /// fn my_func(values: Vec) -> Vec { + /// // do something + /// } + /// + /// let my_func_type = FuncType::new(vec![Type::I32], vec![Type::I32]); + /// let imports = imports!{ + /// "namespace" => { + /// "my_func" => Func::new_dynamic(my_func_type, my_func),s + /// } + /// } + /// ``` + fn compile_dynamic_function_trampolines( + &self, + module: &Module, + ) -> Result, CompileError>; } diff --git a/lib/compiler/src/function.rs b/lib/compiler/src/function.rs index c1de96f5d..6f25205e0 100644 --- a/lib/compiler/src/function.rs +++ b/lib/compiler/src/function.rs @@ -99,7 +99,7 @@ impl Compilation { self.functions.is_empty() } - /// Gets functions jump table offsets. + /// Gets functions relocations. pub fn get_relocations(&self) -> PrimaryMap> { self.functions .iter() @@ -107,7 +107,7 @@ impl Compilation { .collect::>() } - /// Gets functions jump table offsets. + /// Gets functions bodies. pub fn get_function_bodies(&self) -> PrimaryMap { self.functions .iter() @@ -123,7 +123,7 @@ impl Compilation { .collect::>() } - /// Gets functions jump table offsets. + /// Gets functions frame info. pub fn get_frame_info(&self) -> PrimaryMap { self.functions .iter() @@ -138,6 +138,14 @@ impl Compilation { .map(|(_, section)| section.bytes.clone()) .collect::>() } + + /// Gets relocations that apply to custom sections. + pub fn get_custom_section_relocations(&self) -> PrimaryMap> { + self.custom_sections + .iter() + .map(|(_, section)| section.relocations.clone()) + .collect::>() + } } impl<'a> IntoIterator for &'a Compilation { diff --git a/lib/compiler/src/lib.rs b/lib/compiler/src/lib.rs index 19edf11b0..0cbd228ef 100644 --- a/lib/compiler/src/lib.rs +++ b/lib/compiler/src/lib.rs @@ -78,6 +78,7 @@ pub use crate::unwind::{CompiledFunctionUnwindInfo, FDERelocEntry, FunctionTable pub use wasm_common::Features; +#[cfg(feature = "translator")] /// wasmparser is exported as a module to slim compiler dependencies pub mod wasmparser { pub use wasmparser::*; diff --git a/lib/compiler/src/relocation.rs b/lib/compiler/src/relocation.rs index 39b889bcb..62975174c 100644 --- a/lib/compiler/src/relocation.rs +++ b/lib/compiler/src/relocation.rs @@ -15,7 +15,7 @@ use crate::{Addend, CodeOffset, JumpTable}; use serde::{Deserialize, Serialize}; use std::fmt; use wasm_common::entity::PrimaryMap; -use wasm_common::{FunctionIndex, LocalFunctionIndex}; +use wasm_common::LocalFunctionIndex; use wasmer_runtime::libcalls::LibCall; /// Relocation kinds for every ISA. @@ -105,15 +105,15 @@ impl Relocation { /// The function returns the relocation address and the delta. pub fn for_address(&self, start: usize, target_func_address: u64) -> (usize, u64) { match self.kind { - RelocationKind::Abs8 => unsafe { + RelocationKind::Abs8 => { let reloc_address = start + self.offset as usize; let reloc_addend = self.addend as isize; let reloc_abs = (target_func_address) .checked_add(reloc_addend as u64) .unwrap(); (reloc_address, reloc_abs) - }, - RelocationKind::X86PCRel4 => unsafe { + } + RelocationKind::X86PCRel4 => { let reloc_address = start + self.offset as usize; let reloc_addend = self.addend as isize; let reloc_delta_u32 = (target_func_address as u32) diff --git a/lib/compiler/src/section.rs b/lib/compiler/src/section.rs index 36a0e4405..16cf45a76 100644 --- a/lib/compiler/src/section.rs +++ b/lib/compiler/src/section.rs @@ -6,6 +6,7 @@ //! it can be patched later by the engine (native or JIT). use crate::std::vec::Vec; +use crate::Relocation; use serde::{Deserialize, Serialize}; use wasm_common::entity::entity_impl; @@ -23,8 +24,7 @@ pub enum CustomSectionProtection { Read, // We don't include `ReadWrite` here because it would complicate freeze // and resumption of executing Modules. - // We also currently don't include `ReadExecute` as we don't have a way - // to represent relocations for this kind of section. + // TODO: add `ReadExecute`. } /// A Section for a `Compilation`. @@ -33,8 +33,9 @@ pub enum CustomSectionProtection { /// in the emitted module. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct CustomSection { - /// The protection + /// Memory protection that applies to this section. pub protection: CustomSectionProtection, + /// The bytes corresponding to this section. /// /// > Note: These bytes have to be at-least 8-byte aligned @@ -42,6 +43,9 @@ pub struct CustomSection { /// > We might need to create another field for alignment in case it's /// > needed in the future. pub bytes: SectionBody, + + /// Relocations that apply to this custom section. + pub relocations: Vec, } /// The bytes in the section. @@ -49,14 +53,9 @@ pub struct CustomSection { pub struct SectionBody(#[serde(with = "serde_bytes")] Vec); impl SectionBody { - /// Extend the section with the bytes given. - pub fn extend(&mut self, contents: &[u8]) { - self.0.extend(contents); - } - - /// Extends the section by appending bytes from another section. - pub fn append(&mut self, body: &SectionBody) { - self.0.extend(&body.0); + /// Create a new section body with the given contents. + pub fn new_with_vec(contents: Vec) -> Self { + Self(contents) } /// Returns a raw pointer to the section's buffer. @@ -68,4 +67,9 @@ impl SectionBody { pub fn len(&self) -> usize { self.0.len() } + + /// Returns whether or not the section body is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } } diff --git a/lib/compiler/src/target.rs b/lib/compiler/src/target.rs index 19c046b73..ed230edf1 100644 --- a/lib/compiler/src/target.rs +++ b/lib/compiler/src/target.rs @@ -2,23 +2,22 @@ use enumset::{EnumSet, EnumSetType}; pub use target_lexicon::{Architecture, CallingConvention, OperatingSystem, Triple}; -use crate::std::boxed::Box; #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] use raw_cpuid::CpuId; -/// The nomenclature is inspired by the [raw-cpuid crate]. +/// The nomenclature is inspired by the [`cpuid` crate]. /// The list of supported features was initially retrieved from -/// [cranelift-native]. +/// [`cranelift-native`]. /// -/// The `CpuFeature` enum vaues are likely to grow closer to the -/// original cpuid. However, we prefer to start small and grow from there. +/// The `CpuFeature` enum values are likely to grow closer to the +/// original `cpuid`. However, we prefer to start small and grow from there. /// /// If you would like to use a flag that doesn't exist yet here, please /// open a PR. /// -/// [cpuid crate]: https://docs.rs/cpuid/0.1.1/cpuid/enum.CpuFeature.html -/// [cranelift-native]: https://github.com/bytecodealliance/cranelift/blob/6988545fd20249b084c53f4761b8c861266f5d31/cranelift-native/src/lib.rs#L51-L92 -#[allow(missing_docs)] +/// [`cpuid` crate]: https://docs.rs/cpuid/0.1.1/cpuid/enum.CpuFeature.html +/// [`cranelift-native`]: https://github.com/bytecodealliance/cranelift/blob/6988545fd20249b084c53f4761b8c861266f5d31/cranelift-native/src/lib.rs#L51-L92 +#[allow(missing_docs, clippy::derive_hash_xor_eq)] #[derive(EnumSetType, Debug, Hash)] pub enum CpuFeature { // X86 features @@ -48,47 +47,47 @@ impl CpuFeature { if let Some(info) = cpuid.get_feature_info() { if info.has_sse2() { - features.insert(CpuFeature::SSE2); + features.insert(Self::SSE2); } if info.has_sse3() { - features.insert(CpuFeature::SSE3); + features.insert(Self::SSE3); } if info.has_ssse3() { - features.insert(CpuFeature::SSSE3); + features.insert(Self::SSSE3); } if info.has_sse41() { - features.insert(CpuFeature::SSE41); + features.insert(Self::SSE41); } if info.has_sse42() { - features.insert(CpuFeature::SSE42); + features.insert(Self::SSE42); } if info.has_popcnt() { - features.insert(CpuFeature::POPCNT); + features.insert(Self::POPCNT); } if info.has_avx() { - features.insert(CpuFeature::AVX); + features.insert(Self::AVX); } } if let Some(info) = cpuid.get_extended_feature_info() { if info.has_bmi1() { - features.insert(CpuFeature::BMI1); + features.insert(Self::BMI1); } if info.has_bmi2() { - features.insert(CpuFeature::BMI2); + features.insert(Self::BMI2); } if info.has_avx2() { - features.insert(CpuFeature::AVX2); + features.insert(Self::AVX2); } if info.has_avx512dq() { - features.insert(CpuFeature::AVX512DQ); + features.insert(Self::AVX512DQ); } if info.has_avx512vl() { - features.insert(CpuFeature::AVX512VL); + features.insert(Self::AVX512VL); } } if let Some(info) = cpuid.get_extended_function_info() { if info.has_lzcnt() { - features.insert(CpuFeature::LZCNT); + features.insert(Self::LZCNT); } } features @@ -111,8 +110,8 @@ pub struct Target { impl Target { /// Creates a new target given a triple - pub fn new(triple: Triple, cpu_features: EnumSet) -> Target { - Target { + pub fn new(triple: Triple, cpu_features: EnumSet) -> Self { + Self { triple, cpu_features, } @@ -131,8 +130,8 @@ impl Target { /// The default for the Target will use the HOST as the triple impl Default for Target { - fn default() -> Target { - Target { + fn default() -> Self { + Self { triple: Triple::host(), cpu_features: CpuFeature::for_host(), } diff --git a/lib/compiler/src/translator/error.rs b/lib/compiler/src/translator/error.rs index d114adea7..c2191d0a8 100644 --- a/lib/compiler/src/translator/error.rs +++ b/lib/compiler/src/translator/error.rs @@ -1,5 +1,4 @@ use crate::WasmError; -use thiserror::Error; use wasmparser::BinaryReaderError; /// Return an `Err(WasmError::Unsupported(msg))` where `msg` the string built by calling `format!` diff --git a/lib/compiler/src/unwind.rs b/lib/compiler/src/unwind.rs index 205235d14..f91ff0f87 100644 --- a/lib/compiler/src/unwind.rs +++ b/lib/compiler/src/unwind.rs @@ -42,26 +42,26 @@ impl CompiledFunctionUnwindInfo { /// Retuns true is no unwind info data. pub fn is_empty(&self) -> bool { match self { - CompiledFunctionUnwindInfo::Windows(d) => d.is_empty(), - CompiledFunctionUnwindInfo::FrameLayout(c, _, _) => c.is_empty(), + Self::Windows(d) => d.is_empty(), + Self::FrameLayout(c, _, _) => c.is_empty(), } } /// Returns size of serilized unwind info. pub fn len(&self) -> usize { match self { - CompiledFunctionUnwindInfo::Windows(d) => d.len(), - CompiledFunctionUnwindInfo::FrameLayout(c, _, _) => c.len(), + Self::Windows(d) => d.len(), + Self::FrameLayout(c, _, _) => c.len(), } } /// Serializes data into byte array. pub fn serialize(&self, dest: &mut [u8], relocs: &mut Vec) { match self { - CompiledFunctionUnwindInfo::Windows(d) => { + Self::Windows(d) => { dest.copy_from_slice(d); } - CompiledFunctionUnwindInfo::FrameLayout(code, _fde_offset, r) => { + Self::FrameLayout(code, _fde_offset, r) => { dest.copy_from_slice(code); r.iter().for_each(move |r| { assert_eq!(r.2, 8); diff --git a/lib/engine-jit/src/engine.rs b/lib/engine-jit/src/engine.rs index 10d55b846..651719ef0 100644 --- a/lib/engine-jit/src/engine.rs +++ b/lib/engine-jit/src/engine.rs @@ -4,8 +4,8 @@ use crate::{CodeMemory, CompiledModule}; use std::collections::HashMap; use std::sync::{Arc, Mutex}; use wasm_common::entity::PrimaryMap; -use wasm_common::{FunctionType, LocalFunctionIndex, MemoryIndex, SignatureIndex, TableIndex}; -use wasmer_compiler::{Compilation, CompileError, FunctionBody, Target}; +use wasm_common::{FunctionIndex, FunctionType, LocalFunctionIndex, SignatureIndex}; +use wasmer_compiler::{CompileError, FunctionBody}; #[cfg(feature = "compiler")] use wasmer_compiler::{Compiler, CompilerConfig}; use wasmer_engine::{ @@ -13,8 +13,7 @@ use wasmer_engine::{ SerializeError, Tunables, }; use wasmer_runtime::{ - InstanceHandle, MemoryPlan, Module, SignatureRegistry, TablePlan, VMFunctionBody, - VMSharedSignatureIndex, VMTrampoline, + InstanceHandle, Module, SignatureRegistry, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, }; /// A WebAssembly `JIT` Engine. @@ -40,7 +39,7 @@ impl JITEngine { Self { inner: Arc::new(Mutex::new(JITEngineInner { compiler: Some(compiler), - trampolines: HashMap::new(), + function_call_trampolines: HashMap::new(), code_memory: CodeMemory::new(), signatures: SignatureRegistry::new(), })), @@ -66,7 +65,7 @@ impl JITEngine { inner: Arc::new(Mutex::new(JITEngineInner { #[cfg(feature = "compiler")] compiler: None, - trampolines: HashMap::new(), + function_call_trampolines: HashMap::new(), code_memory: CodeMemory::new(), signatures: SignatureRegistry::new(), })), @@ -108,8 +107,8 @@ impl Engine for JITEngine { } /// Retrieves a trampoline given a signature - fn trampoline(&self, sig: VMSharedSignatureIndex) -> Option { - self.compiler().trampoline(sig) + fn function_call_trampoline(&self, sig: VMSharedSignatureIndex) -> Option { + self.compiler().function_call_trampoline(sig) } /// Validates a WebAssembly module @@ -129,7 +128,7 @@ impl Engine for JITEngine { resolver: &dyn Resolver, ) -> Result { let compiled_module = compiled_module.downcast_ref::().unwrap(); - unsafe { compiled_module.instantiate(&self, resolver, Box::new(())) } + compiled_module.instantiate(&self, resolver, Box::new(())) } /// Finish the instantiation of a WebAssembly module @@ -139,7 +138,7 @@ impl Engine for JITEngine { handle: &InstanceHandle, ) -> Result<(), InstantiationError> { let compiled_module = compiled_module.downcast_ref::().unwrap(); - unsafe { compiled_module.finish_instantiation(&handle) } + compiled_module.finish_instantiation(&handle) } /// Serializes a WebAssembly module @@ -174,7 +173,7 @@ pub struct JITEngineInner { #[cfg(feature = "compiler")] compiler: Option>, /// Pointers to trampoline functions used to enter particular signatures - trampolines: HashMap, + function_call_trampolines: HashMap, /// The code memory is responsible of publishing the compiled /// functions to memory. code_memory: CodeMemory, @@ -208,12 +207,19 @@ impl JITEngineInner { } /// Compile the given function bodies. - pub(crate) fn allocate<'data>( + pub(crate) fn allocate( &mut self, module: &Module, functions: &PrimaryMap, - trampolines: &PrimaryMap, - ) -> Result, CompileError> { + function_call_trampolines: &PrimaryMap, + dynamic_function_trampolines: &PrimaryMap, + ) -> Result< + ( + PrimaryMap, + PrimaryMap, + ), + CompileError, + > { // Allocate all of the compiled functions into executable memory, // copying over their contents. let allocated_functions = @@ -226,10 +232,10 @@ impl JITEngineInner { )) })?; - for (sig_index, compiled_function) in trampolines.iter() { + for (sig_index, compiled_function) in function_call_trampolines.iter() { let func_type = module.signatures.get(sig_index).unwrap(); let index = self.signatures.register(&func_type); - if self.trampolines.contains_key(&index) { + if self.function_call_trampolines.contains_key(&index) { // We don't need to allocate the trampoline in case // it's signature is already allocated. continue; @@ -239,16 +245,34 @@ impl JITEngineInner { .allocate_for_function(&compiled_function) .map_err(|message| { CompileError::Resource(format!( - "failed to allocate memory for trampolines: {}", + "failed to allocate memory for function call trampolines: {}", message )) })? .as_ptr(); let trampoline = unsafe { std::mem::transmute::<*const VMFunctionBody, VMTrampoline>(ptr) }; - self.trampolines.insert(index, trampoline); + self.function_call_trampolines.insert(index, trampoline); } - Ok(allocated_functions) + + let allocated_dynamic_function_trampolines = dynamic_function_trampolines + .values() + .map(|compiled_function| { + let ptr = self + .code_memory + .allocate_for_function(&compiled_function) + .map_err(|message| { + CompileError::Resource(format!( + "failed to allocate memory for dynamic function trampolines: {}", + message + )) + })? + .as_ptr(); + Ok(ptr) + }) + .collect::, CompileError>>()?; + + Ok((allocated_functions, allocated_dynamic_function_trampolines)) } /// Make memory containing compiled code executable. @@ -262,7 +286,7 @@ impl JITEngineInner { } /// Gets the trampoline pre-registered for a particular signature - pub fn trampoline(&self, sig: VMSharedSignatureIndex) -> Option { - self.trampolines.get(&sig).cloned() + pub fn function_call_trampoline(&self, sig: VMSharedSignatureIndex) -> Option { + self.function_call_trampolines.get(&sig).cloned() } } diff --git a/lib/engine-jit/src/function_table.rs b/lib/engine-jit/src/function_table.rs index f4fd92c44..9da8a811c 100644 --- a/lib/engine-jit/src/function_table.rs +++ b/lib/engine-jit/src/function_table.rs @@ -27,6 +27,11 @@ impl FunctionTable { self.functions.len() } + /// Returns whether or not the function table is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Adds a function to the table based off of the start offset, end offset, and unwind offset. /// /// The offsets are from the "module base", which is provided when the table is published. @@ -131,6 +136,11 @@ impl FunctionTable { self.functions.len() } + /// Returns whether or not the function table is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Adds a function to the table based off of the start offset, end offset, and unwind offset. /// /// The offsets are from the "module base", which is provided when the table is published. diff --git a/lib/engine-jit/src/link.rs b/lib/engine-jit/src/link.rs index 27cbe4404..2f65dfebc 100644 --- a/lib/engine-jit/src/link.rs +++ b/lib/engine-jit/src/link.rs @@ -4,71 +4,80 @@ use std::ptr::write_unaligned; use wasm_common::entity::{EntityRef, PrimaryMap}; use wasm_common::LocalFunctionIndex; use wasmer_compiler::{ - JumpTable, JumpTableOffsets, RelocationKind, RelocationTarget, Relocations, SectionBody, - SectionIndex, + JumpTable, JumpTableOffsets, Relocation, RelocationKind, RelocationTarget, Relocations, + SectionBody, SectionIndex, }; use wasmer_runtime::Module; use wasmer_runtime::VMFunctionBody; +fn apply_relocation( + body: usize, + r: &Relocation, + allocated_functions: &PrimaryMap, + jt_offsets: &PrimaryMap, + allocated_sections: &PrimaryMap, +) { + let target_func_address: usize = match r.reloc_target { + RelocationTarget::LocalFunc(index) => { + let fatptr: *const [VMFunctionBody] = allocated_functions[index]; + fatptr as *const VMFunctionBody as usize + } + RelocationTarget::LibCall(libcall) => libcall.function_pointer(), + RelocationTarget::CustomSection(custom_section) => { + allocated_sections[custom_section].as_ptr() as usize + } + RelocationTarget::JumpTable(func_index, jt) => { + let offset = *jt_offsets + .get(func_index) + .and_then(|ofs| ofs.get(JumpTable::new(jt.index()))) + .expect("func jump table"); + let fatptr: *const [VMFunctionBody] = allocated_functions[func_index]; + fatptr as *const VMFunctionBody as usize + offset as usize + } + }; + + match r.kind { + #[cfg(target_pointer_width = "64")] + RelocationKind::Abs8 => unsafe { + let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64); + write_unaligned(reloc_address as *mut u64, reloc_delta); + }, + #[cfg(target_pointer_width = "32")] + RelocationKind::X86PCRel4 => unsafe { + let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64); + write_unaligned(reloc_address as *mut u32, reloc_delta); + }, + #[cfg(target_pointer_width = "32")] + RelocationKind::X86CallPCRel4 => { + let (reloc_address, reloc_delta) = r.for_address(body, target_func_address as u64); + write_unaligned(reloc_address as *mut u32, reloc_delta); + } + RelocationKind::X86PCRelRodata4 => {} + _ => panic!("Relocation kind unsupported in the current architecture"), + } +} + /// Links a module, patching the allocated functions with the /// required relocations and jump tables. pub fn link_module( - module: &Module, + _module: &Module, allocated_functions: &PrimaryMap, jt_offsets: &PrimaryMap, - relocations: Relocations, + function_relocations: Relocations, allocated_sections: &PrimaryMap, + section_relocations: &PrimaryMap>, ) { - for (i, function_relocs) in relocations.into_iter() { + for (i, section_relocs) in section_relocations.iter() { + let body = allocated_sections[i].as_ptr() as usize; + for r in section_relocs { + apply_relocation(body, r, allocated_functions, jt_offsets, allocated_sections); + } + } + for (i, function_relocs) in function_relocations.into_iter() { + let fatptr: *const [VMFunctionBody] = allocated_functions[i]; + let body = fatptr as *const VMFunctionBody as usize; for r in function_relocs { - let target_func_address: usize = match r.reloc_target { - RelocationTarget::LocalFunc(index) => { - let fatptr: *const [VMFunctionBody] = allocated_functions[index]; - fatptr as *const VMFunctionBody as usize - } - RelocationTarget::LibCall(libcall) => libcall.function_pointer(), - RelocationTarget::CustomSection(custom_section) => { - allocated_sections[custom_section].as_ptr() as usize - } - RelocationTarget::JumpTable(func_index, jt) => { - let offset = *jt_offsets - .get(func_index) - .and_then(|ofs| ofs.get(JumpTable::new(jt.index()))) - .expect("func jump table"); - let fatptr: *const [VMFunctionBody] = allocated_functions[func_index]; - fatptr as *const VMFunctionBody as usize + offset as usize - } - }; - - let fatptr: *const [VMFunctionBody] = allocated_functions[i]; - let body = fatptr as *const VMFunctionBody; - match r.kind { - #[cfg(target_pointer_width = "64")] - RelocationKind::Abs8 => unsafe { - let (reloc_address, reloc_delta) = - r.for_address(body as usize, target_func_address as u64); - write_unaligned(reloc_address as *mut u64, reloc_delta); - }, - #[cfg(target_pointer_width = "32")] - RelocationKind::X86PCRel4 => unsafe { - let (reloc_address, reloc_delta) = - r.for_address(body as usize, target_func_address as u64); - write_unaligned(reloc_address as *mut u32, reloc_delta); - }, - #[cfg(target_pointer_width = "32")] - RelocationKind::X86CallPCRel4 => { - let (reloc_address, reloc_delta) = - r.for_address(body as usize, target_func_address as u64); - write_unaligned(reloc_address as *mut u32, reloc_delta); - } - RelocationKind::X86CallPLTRel4 => unsafe { - let (reloc_address, reloc_delta) = - r.for_address(body as usize, target_func_address as u64); - write_unaligned(reloc_address as *mut u32, reloc_delta as u32); - }, - RelocationKind::X86PCRelRodata4 => {} - _ => panic!("Relocation kind unsupported in the current architecture"), - } + apply_relocation(body, r, allocated_functions, jt_offsets, allocated_sections); } } } diff --git a/lib/engine-jit/src/module.rs b/lib/engine-jit/src/module.rs index fe0fc24b7..4718f701e 100644 --- a/lib/engine-jit/src/module.rs +++ b/lib/engine-jit/src/module.rs @@ -4,25 +4,23 @@ use crate::engine::{JITEngine, JITEngineInner}; use crate::link::link_module; use crate::serialize::{SerializableCompilation, SerializableModule}; -use serde::{Deserialize, Serialize}; use std::any::Any; use std::sync::{Arc, Mutex}; -use wasm_common::entity::{BoxedSlice, EntityRef, PrimaryMap}; +use wasm_common::entity::{BoxedSlice, PrimaryMap}; use wasm_common::{ - DataInitializer, LocalFunctionIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, - MemoryIndex, OwnedDataInitializer, SignatureIndex, TableIndex, + DataInitializer, FunctionIndex, LocalFunctionIndex, MemoryIndex, OwnedDataInitializer, + SignatureIndex, TableIndex, }; use wasmer_compiler::CompileError; #[cfg(feature = "compiler")] use wasmer_compiler::ModuleEnvironment; use wasmer_engine::{ register_frame_info, resolve_imports, CompiledModule as BaseCompiledModule, DeserializeError, - Engine, GlobalFrameInfoRegistration, InstantiationError, LinkError, Resolver, RuntimeError, - SerializableFunctionFrameInfo, SerializeError, Tunables, + Engine, GlobalFrameInfoRegistration, InstantiationError, Resolver, RuntimeError, + SerializableFunctionFrameInfo, SerializeError, }; use wasmer_runtime::{ - InstanceHandle, LinearMemory, Module, SignatureRegistry, Table, VMFunctionBody, - VMGlobalDefinition, VMSharedSignatureIndex, + InstanceHandle, Module, SignatureRegistry, VMFunctionBody, VMSharedSignatureIndex, }; use wasmer_runtime::{MemoryPlan, TablePlan}; @@ -32,6 +30,7 @@ pub struct CompiledModule { serializable: SerializableModule, finished_functions: BoxedSlice, + finished_dynamic_function_trampolines: BoxedSlice, signatures: BoxedSlice, frame_info_registration: Mutex>>, } @@ -44,9 +43,7 @@ impl CompiledModule { let mut jit_compiler = jit.compiler_mut(); let tunables = jit.tunables(); - let translation = environ - .translate(data) - .map_err(|error| CompileError::Wasm(error))?; + let translation = environ.translate(data).map_err(CompileError::Wasm)?; let memory_plans: PrimaryMap = translation .module @@ -79,11 +76,14 @@ impl CompiledModule { .values() .cloned() .collect::>(); - let trampolines = compiler - .compile_wasm_trampolines(&func_types)? + let function_call_trampolines = compiler + .compile_function_call_trampolines(&func_types)? .into_iter() .collect::>(); + let dynamic_function_trampolines = + compiler.compile_dynamic_function_trampolines(&translation.module)?; + let data_initializers = translation .data_initializers .iter() @@ -102,8 +102,10 @@ impl CompiledModule { function_relocations: compilation.get_relocations(), function_jt_offsets: compilation.get_jt_offsets(), function_frame_info: frame_infos, - trampolines, + function_call_trampolines, + dynamic_function_trampolines, custom_sections: compilation.get_custom_sections(), + custom_section_relocations: compilation.get_custom_section_relocations(), }; let serializable = SerializableModule { compilation: serializable_compilation, @@ -134,15 +136,14 @@ impl CompiledModule { } /// Deserialize a CompiledModule - pub fn deserialize(jit: &JITEngine, bytes: &[u8]) -> Result { + pub fn deserialize(jit: &JITEngine, bytes: &[u8]) -> Result { // let r = flexbuffers::Reader::get_root(bytes).map_err(|e| DeserializeError::CorruptedBinary(format!("{:?}", e)))?; // let serializable = SerializableModule::deserialize(r).map_err(|e| DeserializeError::CorruptedBinary(format!("{:?}", e)))?; let serializable: SerializableModule = bincode::deserialize(bytes) .map_err(|e| DeserializeError::CorruptedBinary(format!("{:?}", e)))?; - Self::from_parts(&mut jit.compiler_mut(), serializable) - .map_err(|e| DeserializeError::Compiler(e)) + Self::from_parts(&mut jit.compiler_mut(), serializable).map_err(DeserializeError::Compiler) } /// Construct a `CompiledModule` from component parts. @@ -150,10 +151,11 @@ impl CompiledModule { jit_compiler: &mut JITEngineInner, serializable: SerializableModule, ) -> Result { - let finished_functions = jit_compiler.allocate( + let (finished_functions, finished_dynamic_function_trampolines) = jit_compiler.allocate( &serializable.module, &serializable.compilation.function_bodies, - &serializable.compilation.trampolines, + &serializable.compilation.function_call_trampolines, + &serializable.compilation.dynamic_function_trampolines, )?; link_module( @@ -162,6 +164,7 @@ impl CompiledModule { &serializable.compilation.function_jt_offsets, serializable.compilation.function_relocations.clone(), &serializable.compilation.custom_sections, + &serializable.compilation.custom_section_relocations, ); // Compute indices into the shared signature table. @@ -181,6 +184,8 @@ impl CompiledModule { Ok(Self { serializable, finished_functions: finished_functions.into_boxed_slice(), + finished_dynamic_function_trampolines: finished_dynamic_function_trampolines + .into_boxed_slice(), signatures: signatures.into_boxed_slice(), frame_info_registration: Mutex::new(None), }) @@ -212,6 +217,7 @@ impl CompiledModule { &self.module(), &sig_registry, resolver, + &self.finished_dynamic_function_trampolines, self.memory_plans(), self.table_plans(), ) diff --git a/lib/engine-jit/src/serialize.rs b/lib/engine-jit/src/serialize.rs index 12bb9b8bc..789ce6dc8 100644 --- a/lib/engine-jit/src/serialize.rs +++ b/lib/engine-jit/src/serialize.rs @@ -2,7 +2,8 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use wasm_common::entity::PrimaryMap; use wasm_common::{ - Features, LocalFunctionIndex, MemoryIndex, OwnedDataInitializer, SignatureIndex, TableIndex, + Features, FunctionIndex, LocalFunctionIndex, MemoryIndex, OwnedDataInitializer, SignatureIndex, + TableIndex, }; use wasmer_compiler::{FunctionBody, JumpTableOffsets, Relocation, SectionBody, SectionIndex}; use wasmer_engine::SerializableFunctionFrameInfo; @@ -19,8 +20,10 @@ pub struct SerializableCompilation { // to allow lazy frame_info deserialization, we convert it to it's lazy binary // format upon serialization. pub function_frame_info: PrimaryMap, - pub trampolines: PrimaryMap, + pub function_call_trampolines: PrimaryMap, + pub dynamic_function_trampolines: PrimaryMap, pub custom_sections: PrimaryMap, + pub custom_section_relocations: PrimaryMap>, } /// Serializable struct that is able to serialize from and to diff --git a/lib/engine-native/src/engine.rs b/lib/engine-native/src/engine.rs index a18d2da47..12b92cd38 100644 --- a/lib/engine-native/src/engine.rs +++ b/lib/engine-native/src/engine.rs @@ -151,7 +151,7 @@ impl Engine for NativeEngine { } /// Retrieves a trampoline given a signature - fn trampoline(&self, sig: VMSharedSignatureIndex) -> Option { + fn function_call_trampoline(&self, sig: VMSharedSignatureIndex) -> Option { self.inner().trampoline(sig) } diff --git a/lib/engine-native/src/module.rs b/lib/engine-native/src/module.rs index 92aff177f..d05953412 100644 --- a/lib/engine-native/src/module.rs +++ b/lib/engine-native/src/module.rs @@ -15,7 +15,7 @@ use tempfile::NamedTempFile; use wasm_common::entity::{BoxedSlice, EntityRef, PrimaryMap}; use wasm_common::{ DataInitializer, LocalFunctionIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, - MemoryIndex, OwnedDataInitializer, SignatureIndex, TableIndex, + MemoryIndex, OwnedDataInitializer, SignatureIndex, TableIndex, FunctionIndex, }; use wasmer_compiler::CompileError; #[cfg(feature = "compiler")] @@ -38,6 +38,7 @@ pub struct NativeModule { metadata: ModuleMetadata, library: Library, finished_functions: BoxedSlice, + finished_dynamic_function_trampolines: BoxedSlice, signatures: BoxedSlice, } @@ -83,18 +84,22 @@ impl NativeModule { table_plans.clone(), )?; - // Compile the trampolines + // Compile the function call trampolines let func_types = translation .module .signatures .values() .cloned() .collect::>(); - let trampolines = compiler - .compile_wasm_trampolines(&func_types)? + let function_call_trampolines = compiler + .compile_function_call_trampolines(&func_types)? .into_iter() .collect::>(); + // Compile the dynamic function trampolines + let dynamic_function_trampolines = compiler + .compile_dynamic_function_trampolines(&translation.module)?; + let data_initializers = translation .data_initializers .iter() @@ -168,7 +173,7 @@ impl NativeModule { // } // Add functions - for (function_local_index, function) in function_bodies.iter() { + for (function_local_index, function) in function_bodies.into_iter() { let function_name = Self::get_function_name(&metadata, function_local_index); obj.declare(&function_name, Decl::function().global()) .map_err(to_compile_error)?; @@ -176,9 +181,18 @@ impl NativeModule { .map_err(to_compile_error)?; } - // Add trampolines - for (signature_index, function) in trampolines.iter() { - let function_name = Self::get_trampoline_name(&metadata, signature_index); + // Add function call trampolines + for (signature_index, function) in function_call_trampolines.into_iter() { + let function_name = Self::get_function_call_trampoline_name(&metadata, signature_index); + obj.declare(&function_name, Decl::function().global()) + .map_err(to_compile_error)?; + obj.define(&function_name, function.body.clone()) + .map_err(to_compile_error)?; + } + + // Add dynamic function trampolines + for (func_index, function) in dynamic_function_trampolines.into_iter() { + let function_name = Self::get_dynamic_function_trampoline_name(&metadata, func_index); obj.declare(&function_name, Decl::function().global()) .map_err(to_compile_error)?; obj.define(&function_name, function.body.clone()) @@ -267,8 +281,12 @@ impl NativeModule { format!("wasmer_function_{}_{}", metadata.prefix, index.index()) } - fn get_trampoline_name(metadata: &ModuleMetadata, index: SignatureIndex) -> String { - format!("wasmer_trampoline_{}_{}", metadata.prefix, index.index()) + fn get_function_call_trampoline_name(metadata: &ModuleMetadata, index: SignatureIndex) -> String { + format!("wasmer_trampoline_function_call_{}_{}", metadata.prefix, index.index()) + } + + fn get_dynamic_function_trampoline_name(metadata: &ModuleMetadata, index: FunctionIndex) -> String { + format!("wasmer_trampoline_dynamic_function_{}_{}", metadata.prefix, index.index()) } /// Construct a `NativeModule` from component parts. @@ -299,6 +317,39 @@ impl NativeModule { } } + // Retrieve function call trampolines (for all signatures in the module) + for (sig_index, func_type) in metadata.module.signatures.iter() { + let function_name = Self::get_function_call_trampoline_name(&metadata, sig_index); + unsafe { + let trampoline: Symbol = lib + .get(function_name.as_bytes()) + .map_err(to_compile_error)?; + engine_inner.add_trampoline(&func_type, *trampoline); + } + } + + // Retrieve dynamic function trampolines (only for imported functions) + let mut finished_dynamic_function_trampolines: PrimaryMap = + PrimaryMap::with_capacity(metadata.module.num_imported_funcs); + for func_index in metadata.module.functions.keys().take(metadata.module.num_imported_funcs) { + let function_name = Self::get_dynamic_function_trampoline_name(&metadata, func_index); + unsafe { + let trampoline: Symbol<*const VMFunctionBody> = lib + .get(function_name.as_bytes()) + .map_err(to_compile_error)?; + finished_dynamic_function_trampolines.push(*trampoline); + } + } + + // Leaving frame infos from now, as they are not yet used + // however they might be useful for the future. + // let frame_infos = compilation + // .get_frame_info() + // .values() + // .map(|frame_info| SerializableFunctionFrameInfo::Processed(frame_info.clone())) + // .collect::>(); + // Self::from_parts(&mut engine_inner, lib, metadata, ) + // Compute indices into the shared signature table. let signatures = { let signature_registry = engine_inner.signatures(); @@ -310,29 +361,13 @@ impl NativeModule { .collect::>() }; - for (sig_index, func_type) in metadata.module.signatures.iter() { - let function_name = Self::get_trampoline_name(&metadata, sig_index); - unsafe { - let trampoline: Symbol = lib - .get(function_name.as_bytes()) - .map_err(to_compile_error)?; - engine_inner.add_trampoline(&func_type, *trampoline); - } - } - // Leaving frame infos from now, as they are not yet used - // however they might be useful for the future. - // let frame_infos = compilation - // .get_frame_info() - // .values() - // .map(|frame_info| SerializableFunctionFrameInfo::Processed(frame_info.clone())) - // .collect::>(); - // Self::from_parts(&mut engine_inner, lib, metadata, ) - Ok(Self { sharedobject_path, metadata, library: lib, finished_functions: finished_functions.into_boxed_slice(), + finished_dynamic_function_trampolines: finished_dynamic_function_trampolines + .into_boxed_slice(), signatures: signatures.into_boxed_slice(), }) } @@ -410,6 +445,7 @@ impl NativeModule { &self.module(), &sig_registry, resolver, + &self.finished_dynamic_function_trampolines, self.memory_plans(), self.table_plans(), ) diff --git a/lib/engine/src/engine.rs b/lib/engine/src/engine.rs index d3ddf965d..8610ef1af 100644 --- a/lib/engine/src/engine.rs +++ b/lib/engine/src/engine.rs @@ -15,7 +15,7 @@ use wasmer_runtime::{InstanceHandle, VMSharedSignatureIndex, VMTrampoline}; /// such as: JIT or Native. pub trait Engine { /// Get the tunables - fn tunables(&self) -> &Tunables; + fn tunables(&self) -> &dyn Tunables; /// Register a signature fn register_signature(&self, func_type: &FunctionType) -> VMSharedSignatureIndex; @@ -24,13 +24,13 @@ pub trait Engine { fn lookup_signature(&self, sig: VMSharedSignatureIndex) -> Option; /// Retrieves a trampoline given a signature - fn trampoline(&self, sig: VMSharedSignatureIndex) -> Option; + fn function_call_trampoline(&self, sig: VMSharedSignatureIndex) -> Option; /// Validates a WebAssembly module fn validate(&self, binary: &[u8]) -> Result<(), CompileError>; /// Compile a WebAssembly binary - fn compile(&self, binary: &[u8]) -> Result, CompileError>; + fn compile(&self, binary: &[u8]) -> Result, CompileError>; /// Instantiates a WebAssembly module unsafe fn instantiate( @@ -50,13 +50,13 @@ pub trait Engine { fn serialize(&self, compiled_module: &dyn CompiledModule) -> Result, SerializeError>; /// Deserializes a WebAssembly module - fn deserialize(&self, bytes: &[u8]) -> Result, DeserializeError>; + fn deserialize(&self, bytes: &[u8]) -> Result, DeserializeError>; /// Deserializes a WebAssembly module from a path fn deserialize_from_file( &self, file_ref: &Path, - ) -> Result, DeserializeError> { + ) -> Result, DeserializeError> { let bytes = std::fs::read(file_ref)?; self.deserialize(&bytes) } diff --git a/lib/engine/src/module.rs b/lib/engine/src/module.rs index a15a69536..29ee02041 100644 --- a/lib/engine/src/module.rs +++ b/lib/engine/src/module.rs @@ -1,6 +1,3 @@ -use crate::error::InstantiationError; -use std::sync::Arc; -use wasmer_runtime::InstanceHandle; use wasmer_runtime::Module; use downcast_rs::{impl_downcast, Downcast}; diff --git a/lib/engine/src/resolver.rs b/lib/engine/src/resolver.rs index 5ae87b749..c3fa83592 100644 --- a/lib/engine/src/resolver.rs +++ b/lib/engine/src/resolver.rs @@ -3,11 +3,11 @@ use crate::error::{ImportError, LinkError}; use more_asserts::assert_ge; -use wasm_common::entity::PrimaryMap; -use wasm_common::{ExternType, ImportIndex, MemoryIndex, TableIndex}; +use wasm_common::entity::{BoxedSlice, EntityRef, PrimaryMap}; +use wasm_common::{ExternType, FunctionIndex, ImportIndex, MemoryIndex, TableIndex}; use wasmer_runtime::{ - Export, Imports, SignatureRegistry, VMFunctionImport, VMGlobalImport, VMMemoryImport, - VMTableImport, + Export, Imports, SignatureRegistry, VMFunctionBody, VMFunctionImport, VMFunctionKind, + VMGlobalImport, VMMemoryImport, VMTableImport, }; use wasmer_runtime::{MemoryPlan, TablePlan}; @@ -43,15 +43,15 @@ fn get_extern_from_import(module: &Module, import_index: &ImportIndex) -> Extern ExternType::Function(func) } ImportIndex::Table(index) => { - let table = module.tables[*index].clone(); + let table = module.tables[*index]; ExternType::Table(table) } ImportIndex::Memory(index) => { - let memory = module.memories[*index].clone(); + let memory = module.memories[*index]; ExternType::Memory(memory) } ImportIndex::Global(index) => { - let global = module.globals[*index].clone(); + let global = module.globals[*index]; ExternType::Global(global) } } @@ -65,19 +65,19 @@ fn get_extern_from_export( ) -> ExternType { match export { Export::Function(ref f) => { - let func = signatures.lookup(f.signature).unwrap().clone(); + let func = signatures.lookup(f.signature).unwrap(); ExternType::Function(func) } Export::Table(ref t) => { - let table = t.plan().table.clone(); + let table = t.plan().table; ExternType::Table(table) } Export::Memory(ref m) => { - let memory = m.plan().memory.clone(); + let memory = m.plan().memory; ExternType::Memory(memory) } Export::Global(ref g) => { - let global = g.global.clone(); + let global = g.global; ExternType::Global(global) } } @@ -91,6 +91,7 @@ pub fn resolve_imports( module: &Module, signatures: &SignatureRegistry, resolver: &dyn Resolver, + finished_dynamic_function_trampolines: &BoxedSlice, memory_plans: &PrimaryMap, _table_plans: &PrimaryMap, ) -> Result { @@ -122,8 +123,21 @@ pub fn resolve_imports( } match resolved { Export::Function(ref f) => { + let address = match f.kind { + VMFunctionKind::Dynamic => { + // If this is a dynamic imported function, + // the address of the funciton is the address of the + // reverse trampoline. + let index = FunctionIndex::new(function_imports.len()); + finished_dynamic_function_trampolines[index] + + // TODO: We should check that the f.vmctx actually matches + // the shape of `VMDynamicFunctionImportContext` + } + VMFunctionKind::Static => f.address, + }; function_imports.push(VMFunctionImport { - body: f.address, + body: address, vmctx: f.vmctx, }); } diff --git a/lib/engine/src/serialize.rs b/lib/engine/src/serialize.rs index 42964a24e..c5f1749a8 100644 --- a/lib/engine/src/serialize.rs +++ b/lib/engine/src/serialize.rs @@ -100,7 +100,7 @@ impl<'de> Deserialize<'de> for SerializableFunctionFrameInfo { where D: Deserializer<'de>, { - Ok(SerializableFunctionFrameInfo::Unprocessed( + Ok(Self::Unprocessed( deserializer.deserialize_byte_buf(FunctionFrameInfoVisitor)?, )) } diff --git a/lib/engine/src/trap/error.rs b/lib/engine/src/trap/error.rs index 7f066b633..3d7f99ab7 100644 --- a/lib/engine/src/trap/error.rs +++ b/lib/engine/src/trap/error.rs @@ -124,12 +124,16 @@ impl RuntimeError { .any(|pc| info.should_process_frame(*pc).unwrap_or(false)) { // We drop the read lock, to get a write one. + // Note: this is not guaranteed because it's a RwLock: + // the following code may cause deadlocks. + // TODO: clean up this code drop(info); - let mut info = FRAME_INFO.write().unwrap(); - for pc in frames.iter() { - drop(info.maybe_process_frame(*pc)); + { + let mut info = FRAME_INFO.write().unwrap(); + for pc in frames.iter() { + info.maybe_process_frame(*pc); + } } - drop(info); FRAME_INFO.read().unwrap() } else { info @@ -182,7 +186,7 @@ impl fmt::Display for RuntimeError { for frame in self.trace().iter() { let name = frame.module_name(); let func_index = frame.func_index(); - writeln!(f, "")?; + writeln!(f)?; write!(f, " at ")?; match frame.func_name() { Some(name) => match rustc_demangle::try_demangle(name) { diff --git a/lib/engine/src/trap/frame_info.rs b/lib/engine/src/trap/frame_info.rs index 5678b9c32..98a7cb52a 100644 --- a/lib/engine/src/trap/frame_info.rs +++ b/lib/engine/src/trap/frame_info.rs @@ -66,7 +66,7 @@ impl ModuleFrameInfo { } fn process_function_debug_info(&mut self, local_index: LocalFunctionIndex) { - let mut func = self.frame_infos.get_mut(local_index).unwrap(); + let func = self.frame_infos.get_mut(local_index).unwrap(); let processed: CompiledFunctionFrameInfo = match func { SerializableFunctionFrameInfo::Processed(_) => { // This should be a no-op on processed info @@ -187,7 +187,8 @@ impl GlobalFrameInfo { pub fn maybe_process_frame(&mut self, pc: usize) -> Option<()> { let module = self.module_info_mut(pc)?; let func = module.function_info(pc)?; - module.process_function_debug_info(func.local_index); + let func_local_index = func.local_index; + module.process_function_debug_info(func_local_index); Some(()) } @@ -246,7 +247,7 @@ pub fn register( }; assert!(functions.insert(end, func).is_none()); } - if functions.len() == 0 { + if functions.is_empty() { return None; } @@ -266,7 +267,7 @@ pub fn register( ModuleFrameInfo { start: min, functions, - module: module.clone(), + module, frame_infos, }, ); diff --git a/lib/engine/src/tunables.rs b/lib/engine/src/tunables.rs index 080f44977..78ef182e0 100644 --- a/lib/engine/src/tunables.rs +++ b/lib/engine/src/tunables.rs @@ -1,9 +1,10 @@ use crate::error::LinkError; use wasm_common::entity::{EntityRef, PrimaryMap}; use wasm_common::{ - GlobalIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryIndex, MemoryType, - TableIndex, TableType, + LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryIndex, MemoryType, TableIndex, + TableType, }; +use wasmer_runtime::MemoryError; use wasmer_runtime::{LinearMemory, Module, Table, VMGlobalDefinition}; use wasmer_runtime::{MemoryPlan, TablePlan}; @@ -16,7 +17,7 @@ pub trait Tunables { fn table_plan(&self, table: TableType) -> TablePlan; /// Create a memory given a memory type - fn create_memory(&self, memory_type: MemoryPlan) -> Result; + fn create_memory(&self, memory_type: MemoryPlan) -> Result; /// Create a memory given a memory type fn create_table(&self, table_type: TablePlan) -> Result; @@ -32,7 +33,10 @@ pub trait Tunables { PrimaryMap::with_capacity(module.memories.len() - num_imports); for index in num_imports..module.memories.len() { let plan = memory_plans[MemoryIndex::new(index)].clone(); - memories.push(self.create_memory(plan).map_err(LinkError::Resource)?); + memories.push( + self.create_memory(plan) + .map_err(|e| LinkError::Resource(format!("Failed to create memory: {}", e)))?, + ); } Ok(memories) } diff --git a/lib/runtime/Cargo.toml b/lib/runtime/Cargo.toml index 7a8fce1af..5022f05ae 100644 --- a/lib/runtime/Cargo.toml +++ b/lib/runtime/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] wasm-common = { path = "../wasm-common", version = "0.16.2", features = ["enable-serde"] } region = "2.1.2" -libc = { version = "0.2", default-features = false } +libc = { version = "0.2.70", default-features = false } memoffset = "0.5.4" indexmap = { version = "1.3.2", features = ["serde-1"] } thiserror = "1.0.16" diff --git a/lib/runtime/build.rs b/lib/runtime/build.rs index bd8ebe10f..206d99276 100644 --- a/lib/runtime/build.rs +++ b/lib/runtime/build.rs @@ -1,3 +1,5 @@ +//! Runtime build script compiles C code using setjmp for trap handling. + fn main() { println!("cargo:rerun-if-changed=src/trap/helpers.c"); cc::Build::new() diff --git a/lib/runtime/src/export.rs b/lib/runtime/src/export.rs index 36675ff5e..52ed8df59 100644 --- a/lib/runtime/src/export.rs +++ b/lib/runtime/src/export.rs @@ -2,8 +2,8 @@ use crate::memory::LinearMemory; use crate::module::{MemoryPlan, TablePlan}; use crate::table::Table; use crate::vmcontext::{ - VMContext, VMFunctionBody, VMGlobalDefinition, VMMemoryDefinition, VMSharedSignatureIndex, - VMTableDefinition, + VMContext, VMFunctionBody, VMFunctionKind, VMGlobalDefinition, VMMemoryDefinition, + VMSharedSignatureIndex, VMTableDefinition, }; use wasm_common::GlobalType; @@ -34,11 +34,13 @@ pub struct ExportFunction { /// /// Note that this indexes within the module associated with `vmctx`. pub signature: VMSharedSignatureIndex, + /// The function kind (it defines how it's the signature that provided `address` have) + pub kind: VMFunctionKind, } impl From for Export { - fn from(func: ExportFunction) -> Export { - Export::Function(func) + fn from(func: ExportFunction) -> Self { + Self::Function(func) } } @@ -59,8 +61,8 @@ impl ExportTable { } impl From for Export { - fn from(table: ExportTable) -> Export { - Export::Table(table) + fn from(table: ExportTable) -> Self { + Self::Table(table) } } @@ -81,8 +83,8 @@ impl ExportMemory { } impl From for Export { - fn from(memory: ExportMemory) -> Export { - Export::Memory(memory) + fn from(memory: ExportMemory) -> Self { + Self::Memory(memory) } } @@ -96,7 +98,7 @@ pub struct ExportGlobal { } impl From for Export { - fn from(global: ExportGlobal) -> Export { - Export::Global(global) + fn from(global: ExportGlobal) -> Self { + Self::Global(global) } } diff --git a/lib/runtime/src/imports.rs b/lib/runtime/src/imports.rs index 7fb510346..d311d863b 100644 --- a/lib/runtime/src/imports.rs +++ b/lib/runtime/src/imports.rs @@ -1,5 +1,4 @@ use crate::vmcontext::{VMFunctionImport, VMGlobalImport, VMMemoryImport, VMTableImport}; -use std::collections::HashSet; use wasm_common::entity::{BoxedSlice, PrimaryMap}; use wasm_common::{FunctionIndex, GlobalIndex, MemoryIndex, TableIndex}; diff --git a/lib/runtime/src/instance.rs b/lib/runtime/src/instance.rs index 4aa847d7d..a7444256c 100644 --- a/lib/runtime/src/instance.rs +++ b/lib/runtime/src/instance.rs @@ -3,13 +3,13 @@ //! `InstanceHandle` is a reference-counting handle for an `Instance`. use crate::export::Export; use crate::imports::Imports; -use crate::memory::LinearMemory; +use crate::memory::{LinearMemory, MemoryError}; use crate::table::Table; use crate::trap::{catch_traps, init_traps, Trap, TrapCode}; use crate::vmcontext::{ VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, - VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, - VMTableDefinition, VMTableImport, + VMFunctionKind, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, + VMSharedSignatureIndex, VMTableDefinition, VMTableImport, }; use crate::{ExportFunction, ExportGlobal, ExportMemory, ExportTable}; use crate::{Module, TableElements, VMOffsets}; @@ -18,7 +18,7 @@ use more_asserts::assert_lt; use std::alloc::{self, Layout}; use std::any::Any; use std::cell::{Cell, RefCell}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::convert::TryFrom; use std::sync::Arc; use std::{mem, ptr, slice}; @@ -294,6 +294,11 @@ impl Instance { }; ExportFunction { address, + // Any function received is already static at this point as: + // 1. All locally defined functions in the Wasm have a static signature. + // 2. All the imported functions are already static (because + // they point to the trampolines rather than the dynamic addresses). + kind: VMFunctionKind::Static, signature, vmctx, } @@ -431,7 +436,7 @@ impl Instance { &self, memory_index: LocalMemoryIndex, delta: IntoPages, - ) -> Option + ) -> Result where IntoPages: Into, { @@ -459,7 +464,7 @@ impl Instance { &self, memory_index: MemoryIndex, delta: IntoPages, - ) -> Option + ) -> Result where IntoPages: Into, { @@ -815,6 +820,7 @@ impl InstanceHandle { vmctx: VMContext {}, }; let layout = instance.alloc_layout(); + #[allow(clippy::cast_ptr_alignment)] let instance_ptr = alloc::alloc(layout) as *mut Instance; if instance_ptr.is_null() { alloc::handle_alloc_error(layout); @@ -979,7 +985,7 @@ impl InstanceHandle { &self, memory_index: LocalMemoryIndex, delta: IntoPages, - ) -> Option + ) -> Result where IntoPages: Into, { @@ -1070,7 +1076,7 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), Trap> { let size = usize::try_from(table.size()).unwrap(); if size < start + init.elements.len() { - return Err(Trap::wasm(TrapCode::TableSetterOutOfBounds).into()); + return Err(Trap::wasm(TrapCode::TableSetterOutOfBounds)); } } @@ -1095,6 +1101,7 @@ fn get_memory_init_start(init: &DataInitializer<'_>, instance: &Instance) -> usi start } +#[allow(clippy::mut_from_ref)] /// Return a byte-slice view of a memory's data. unsafe fn get_memory_slice<'instance>( init: &DataInitializer<'_>, @@ -1121,7 +1128,7 @@ fn check_memory_init_bounds( unsafe { let mem_slice = get_memory_slice(init, instance); if mem_slice.get_mut(start..start + init.data.len()).is_none() { - return Err(Trap::wasm(TrapCode::HeapSetterOutOfBounds).into()); + return Err(Trap::wasm(TrapCode::HeapSetterOutOfBounds)); } } } @@ -1158,7 +1165,7 @@ fn initialize_tables(instance: &Instance) -> Result<(), Trap> { .checked_add(init.elements.len()) .map_or(true, |end| end > table.size() as usize) { - return Err(Trap::wasm(TrapCode::TableAccessOutOfBounds).into()); + return Err(Trap::wasm(TrapCode::TableAccessOutOfBounds)); } for (i, func_idx) in init.elements.iter().enumerate() { @@ -1213,7 +1220,7 @@ fn initialize_memories( .checked_add(init.data.len()) .map_or(true, |end| end > memory.current_length) { - return Err(Trap::wasm(TrapCode::HeapAccessOutOfBounds).into()); + return Err(Trap::wasm(TrapCode::HeapAccessOutOfBounds)); } unsafe { diff --git a/lib/runtime/src/lib.rs b/lib/runtime/src/lib.rs index bad8455d5..1ff5d9468 100644 --- a/lib/runtime/src/lib.rs +++ b/lib/runtime/src/lib.rs @@ -39,7 +39,7 @@ pub mod libcalls; pub use crate::export::*; pub use crate::imports::Imports; pub use crate::instance::InstanceHandle; -pub use crate::memory::LinearMemory; +pub use crate::memory::{LinearMemory, MemoryError}; pub use crate::mmap::Mmap; pub use crate::module::{ ExportsIterator, ImportsIterator, MemoryPlan, MemoryStyle, Module, TableElements, TablePlan, @@ -50,9 +50,10 @@ pub use crate::sig_registry::SignatureRegistry; pub use crate::table::Table; pub use crate::trap::*; pub use crate::vmcontext::{ - VMBuiltinFunctionIndex, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, - VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, - VMTableDefinition, VMTableImport, VMTrampoline, + VMBuiltinFunctionIndex, VMCallerCheckedAnyfunc, VMContext, VMDynamicFunctionImportContext, + VMFunctionBody, VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, + VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, + VMTrampoline, }; pub use crate::vmoffsets::{TargetSharedSignatureIndex, VMOffsets}; diff --git a/lib/runtime/src/memory.rs b/lib/runtime/src/memory.rs index 4a96256ca..513de069b 100644 --- a/lib/runtime/src/memory.rs +++ b/lib/runtime/src/memory.rs @@ -7,7 +7,34 @@ use crate::module::{MemoryPlan, MemoryStyle}; use crate::vmcontext::VMMemoryDefinition; use more_asserts::{assert_ge, assert_le}; use std::cell::RefCell; -use wasm_common::Pages; +use thiserror::Error; +use wasm_common::{Bytes, Pages}; + +/// Error type describing things that can go wrong when operating on Wasm Memories. +#[derive(Error, Debug, Clone, PartialEq, Hash)] +pub enum MemoryError { + /// Low level error with mmap. + #[error("Error when allocating memory: {0}")] + Region(String), + /// The operation would cause the size of the memory to exceed the maximum or would cause + /// an overflow leading to unindexable memory. + #[error("The memory could not grow: current size {} pages, requested increase: {} pages", current.0, attempted_delta.0)] + CouldNotGrow { + /// The current size in pages. + current: Pages, + /// The attempted amount to grow by in pages. + attempted_delta: Pages, + }, + /// The operation would cause the size of the memory size exceed the maximum. + #[error("The memory plan is invalid because {}", reason)] + InvalidMemoryPlan { + /// The reason why the memory plan is invalid. + reason: String, + }, + /// A user defined error value, used for error cases not listed above. + #[error("A user-defined error occurred: {0}")] + Generic(String), +} /// A linear memory instance. #[derive(Debug)] @@ -40,13 +67,23 @@ struct WasmMmap { impl LinearMemory { /// Create a new linear memory instance with specified minimum and maximum number of wasm pages. - pub fn new(plan: &MemoryPlan) -> Result { + pub fn new(plan: &MemoryPlan) -> Result { // `maximum` cannot be set to more than `65536` pages. assert_le!(plan.memory.minimum, Pages::max_value()); assert!( plan.memory.maximum.is_none() || plan.memory.maximum.unwrap() <= Pages::max_value() ); + if plan.memory.maximum.is_some() && plan.memory.maximum.unwrap() < plan.memory.minimum { + return Err(MemoryError::InvalidMemoryPlan { + reason: format!( + "the maximum ({} pages) is less than the minimum ({} pages)", + plan.memory.maximum.unwrap().0, + plan.memory.minimum.0 + ), + }); + } + let offset_guard_bytes = plan.offset_guard_size as usize; // If we have an offset guard, or if we're doing the static memory @@ -71,7 +108,8 @@ impl LinearMemory { let mapped_bytes = mapped_pages.bytes(); let mmap = WasmMmap { - alloc: Mmap::accessible_reserved(mapped_bytes.0, request_bytes)?, + alloc: Mmap::accessible_reserved(mapped_bytes.0, request_bytes) + .map_err(MemoryError::Region)?, size: plan.memory.minimum, }; @@ -98,7 +136,7 @@ impl LinearMemory { /// /// Returns `None` if memory can't be grown by the specified amount /// of wasm pages. - pub fn grow(&self, delta: IntoPages) -> Option + pub fn grow(&self, delta: IntoPages) -> Result where IntoPages: Into, { @@ -106,20 +144,24 @@ impl LinearMemory { let delta: Pages = delta.into(); let mut mmap = self.mmap.borrow_mut(); if delta.0 == 0 { - return Some(mmap.size); + return Ok(mmap.size); } - let new_pages = match mmap.size.checked_add(delta) { - Some(new_pages) => new_pages, - // Linear memory size overflow. - None => return None, - }; + let new_pages = mmap + .size + .checked_add(delta) + .ok_or_else(|| MemoryError::CouldNotGrow { + current: mmap.size, + attempted_delta: delta, + })?; let prev_pages = mmap.size; if let Some(maximum) = self.maximum { if new_pages > maximum { - // Linear memory size would exceed the declared maximum. - return None; + return Err(MemoryError::CouldNotGrow { + current: mmap.size, + attempted_delta: delta, + }); } } @@ -128,7 +170,10 @@ impl LinearMemory { // limit here. if new_pages >= Pages::max_value() { // Linear memory size would exceed the index range. - return None; + return Err(MemoryError::CouldNotGrow { + current: mmap.size, + attempted_delta: delta, + }); } let delta_bytes = delta.bytes().0; @@ -139,9 +184,16 @@ impl LinearMemory { // If the new size is within the declared maximum, but needs more memory than we // have on hand, it's a dynamic heap and it can move. let guard_bytes = self.offset_guard_size; - let request_bytes = new_bytes.checked_add(guard_bytes)?; + let request_bytes = + new_bytes + .checked_add(guard_bytes) + .ok_or_else(|| MemoryError::CouldNotGrow { + current: new_pages, + attempted_delta: Bytes(guard_bytes).into(), + })?; - let mut new_mmap = Mmap::accessible_reserved(new_bytes, request_bytes).ok()?; + let mut new_mmap = + Mmap::accessible_reserved(new_bytes, request_bytes).map_err(MemoryError::Region)?; let copy_len = mmap.alloc.len() - self.offset_guard_size; new_mmap.as_mut_slice()[..copy_len].copy_from_slice(&mmap.alloc.as_slice()[..copy_len]); @@ -149,12 +201,14 @@ impl LinearMemory { mmap.alloc = new_mmap; } else if delta_bytes > 0 { // Make the newly allocated pages accessible. - mmap.alloc.make_accessible(prev_bytes, delta_bytes).ok()?; + mmap.alloc + .make_accessible(prev_bytes, delta_bytes) + .map_err(MemoryError::Region)?; } mmap.size = new_pages; - Some(prev_pages) + Ok(prev_pages) } /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. @@ -170,7 +224,7 @@ impl LinearMemory { /// /// This function is used in the `wasmer_runtime::Instance` to retrieve /// the host memory pointer and interact with the host memory directly. - pub fn as_mut_ptr(&self) -> *mut LinearMemory { - self as *const LinearMemory as *mut LinearMemory + pub fn as_mut_ptr(&self) -> *mut Self { + self as *const Self as *mut Self } } diff --git a/lib/runtime/src/module.rs b/lib/runtime/src/module.rs index c5311835c..3472fb6f1 100644 --- a/lib/runtime/src/module.rs +++ b/lib/runtime/src/module.rs @@ -258,7 +258,7 @@ impl Module { } ImportIndex::Global(i) => { let global_type = self.globals.get(i.clone()).unwrap(); - ExternType::Global(global_type.clone()) + ExternType::Global(*global_type) } }; ImportType::new(module, field, extern_type) @@ -347,7 +347,7 @@ impl Module { /// Get the Module name pub fn name(&self) -> String { match self.name { - Some(ref name) => format!("{}", name), + Some(ref name) => name.to_string(), None => "".to_string(), } } @@ -387,21 +387,21 @@ impl + Sized> ExportsIterator { /// Get only the memories pub fn memories(self) -> impl Iterator> + Sized { self.iter.filter_map(|extern_| match extern_.ty() { - ExternType::Memory(ty) => Some(ExportType::new(extern_.name(), ty.clone())), + ExternType::Memory(ty) => Some(ExportType::new(extern_.name(), *ty)), _ => None, }) } /// Get only the tables pub fn tables(self) -> impl Iterator> + Sized { self.iter.filter_map(|extern_| match extern_.ty() { - ExternType::Table(ty) => Some(ExportType::new(extern_.name(), ty.clone())), + ExternType::Table(ty) => Some(ExportType::new(extern_.name(), *ty)), _ => None, }) } /// Get only the globals pub fn globals(self) -> impl Iterator> + Sized { self.iter.filter_map(|extern_| match extern_.ty() { - ExternType::Global(ty) => Some(ExportType::new(extern_.name(), ty.clone())), + ExternType::Global(ty) => Some(ExportType::new(extern_.name(), *ty)), _ => None, }) } @@ -443,33 +443,21 @@ impl + Sized> ImportsIterator { /// Get only the memories pub fn memories(self) -> impl Iterator> + Sized { self.iter.filter_map(|extern_| match extern_.ty() { - ExternType::Memory(ty) => Some(ImportType::new( - extern_.module(), - extern_.name(), - ty.clone(), - )), + ExternType::Memory(ty) => Some(ImportType::new(extern_.module(), extern_.name(), *ty)), _ => None, }) } /// Get only the tables pub fn tables(self) -> impl Iterator> + Sized { self.iter.filter_map(|extern_| match extern_.ty() { - ExternType::Table(ty) => Some(ImportType::new( - extern_.module(), - extern_.name(), - ty.clone(), - )), + ExternType::Table(ty) => Some(ImportType::new(extern_.module(), extern_.name(), *ty)), _ => None, }) } /// Get only the globals pub fn globals(self) -> impl Iterator> + Sized { self.iter.filter_map(|extern_| match extern_.ty() { - ExternType::Global(ty) => Some(ImportType::new( - extern_.module(), - extern_.name(), - ty.clone(), - )), + ExternType::Global(ty) => Some(ImportType::new(extern_.module(), extern_.name(), *ty)), _ => None, }) } diff --git a/lib/runtime/src/table.rs b/lib/runtime/src/table.rs index 0ad3952c9..c640eaf4f 100644 --- a/lib/runtime/src/table.rs +++ b/lib/runtime/src/table.rs @@ -156,7 +156,7 @@ impl Table { /// /// This function is used in the `wasmer_runtime::Instance` to retrieve /// the host table pointer and interact with the host table directly. - pub fn as_mut_ptr(&self) -> *mut Table { - self as *const Table as *mut Table + pub fn as_mut_ptr(&self) -> *mut Self { + self as *const Self as *mut Self } } diff --git a/lib/runtime/src/trap/helpers.c b/lib/runtime/src/trap/helpers.c index 643692224..1a7f4946c 100644 --- a/lib/runtime/src/trap/helpers.c +++ b/lib/runtime/src/trap/helpers.c @@ -17,21 +17,3 @@ void Unwind(void *JmpBuf) { jmp_buf *buf = (jmp_buf*) JmpBuf; longjmp(*buf, 1); } - - -#ifdef __APPLE__ -#include - -void* GetPcFromUContext(ucontext_t *cx) { - return (void*) cx->uc_mcontext->__ss.__rip; -} -#endif - -#if defined(__linux__) && defined(__aarch64__) -#include - -void* GetPcFromUContext(ucontext_t *cx) { - return (void*) cx->uc_mcontext.pc; -} - -#endif // __linux__ && __aarch64__ diff --git a/lib/runtime/src/trap/traphandlers.rs b/lib/runtime/src/trap/traphandlers.rs index 24bb8113c..ff2660863 100644 --- a/lib/runtime/src/trap/traphandlers.rs +++ b/lib/runtime/src/trap/traphandlers.rs @@ -114,9 +114,9 @@ cfg_if::cfg_if! { // exception was handled by a custom exception handler, so we // keep executing. if jmp_buf.is_null() { - return false; + false } else if jmp_buf as usize == 1 { - return true; + true } else { Unwind(jmp_buf) } @@ -161,18 +161,11 @@ cfg_if::cfg_if! { let cx = &*(cx as *const libc::ucontext_t); cx.uc_mcontext.gregs[libc::REG_EIP as usize] as *const u8 } else if #[cfg(all(target_os = "linux", target_arch = "aarch64"))] { - // libc doesn't seem to support Linux/aarch64 at the moment? - extern "C" { - fn GetPcFromUContext(cx: *mut libc::c_void) -> *const u8; - } - GetPcFromUContext(cx) + let cx = &*(cx as *const libc::ucontext_t); + cx.uc_mcontext.pc as *const u8 } else if #[cfg(target_os = "macos")] { - // FIXME(rust-lang/libc#1702) - once that lands and is - // released we should inline the definition here - extern "C" { - fn GetPcFromUContext(cx: *mut libc::c_void) -> *const u8; - } - GetPcFromUContext(cx) + let cx = &*(cx as *const libc::ucontext_t); + (*cx.uc_mcontext).__ss.__rip as *const u8 } else { compile_error!("unsupported platform"); } @@ -356,7 +349,7 @@ impl Trap { /// Internally saves a backtrace when constructed. pub fn wasm(trap_code: TrapCode) -> Self { let backtrace = Backtrace::new_unresolved(); - Trap::Wasm { + Self::Wasm { trap_code, backtrace, } @@ -367,7 +360,7 @@ impl Trap { /// Internally saves a backtrace when constructed. pub fn oom() -> Self { let backtrace = Backtrace::new_unresolved(); - Trap::OOM { backtrace } + Self::OOM { backtrace } } } @@ -447,8 +440,8 @@ enum UnwindReason { } impl CallThreadState { - fn new(vmctx: *mut VMContext) -> CallThreadState { - CallThreadState { + fn new(vmctx: *mut VMContext) -> Self { + Self { unwind: Cell::new(UnwindReason::None), vmctx, jmp_buf: Cell::new(ptr::null()), @@ -458,7 +451,7 @@ impl CallThreadState { } } - fn with(mut self, closure: impl FnOnce(&CallThreadState) -> i32) -> Result<(), Trap> { + fn with(mut self, closure: impl FnOnce(&Self) -> i32) -> Result<(), Trap> { tls::with(|prev| { self.prev = prev.map(|p| p as *const _); let ret = tls::set(&self, || closure(&self)); @@ -545,7 +538,7 @@ impl CallThreadState { }; let result = call_handler(&handler); i.instance().signal_handler.set(Some(handler)); - return result; + result }) { self.handling_trap.set(false); return 1 as *const _; @@ -715,7 +708,7 @@ fn setup_unix_signalstack() -> Result<(), Trap> { impl Drop for Tls { fn drop(&mut self) { let (ptr, size) = match self { - Tls::Allocated { + Self::Allocated { mmap_ptr, mmap_size, } => (*mmap_ptr, *mmap_size), diff --git a/lib/runtime/src/vmcontext.rs b/lib/runtime/src/vmcontext.rs index 61697a0d9..abdc8c8cc 100644 --- a/lib/runtime/src/vmcontext.rs +++ b/lib/runtime/src/vmcontext.rs @@ -46,6 +46,52 @@ mod test_vmfunction_import { } } +/// The `VMDynamicFunctionImportContext` is the context that dynamic +/// functions will receive when called (rather than `vmctx`). +/// A dynamic function is a function for which we don't know the signature +/// until runtime. +/// +/// As such, we need to expose the dynamic function `context` +/// containing the relevant context for running the function indicated +/// in `address`. +#[repr(C)] +pub struct VMDynamicFunctionImportContext { + /// The address of the inner dynamic function. + /// + /// Note: The function must be on the form of + /// `(*mut T, *mut VMContext, SignatureIndex, *mut i128)`. + pub address: *const VMFunctionBody, + + /// The context that the inner dynamic function will receive. + pub ctx: T, +} + +#[cfg(test)] +mod test_vmdynamicfunction_import_context { + use super::VMDynamicFunctionImportContext; + use crate::{Module, VMOffsets}; + use memoffset::offset_of; + use std::mem::size_of; + + #[test] + fn check_vmdynamicfunction_import_context_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); + assert_eq!( + size_of::>(), + usize::from(offsets.size_of_vmdynamicfunction_import_context()) + ); + assert_eq!( + offset_of!(VMDynamicFunctionImportContext, address), + usize::from(offsets.vmdynamicfunction_import_context_address()) + ); + assert_eq!( + offset_of!(VMDynamicFunctionImportContext, ctx), + usize::from(offsets.vmdynamicfunction_import_context_ctx()) + ); + } +} + /// A placeholder byte-sized type which is just used to provide some amount of type /// safety when dealing with pointers to JIT-compiled function bodies. Note that it's /// deliberately not Copy, as we shouldn't be carelessly copying function body bytes @@ -64,6 +110,26 @@ mod test_vmfunction_body { } } +/// A function kind. +#[derive(Debug, Copy, Clone, PartialEq)] +#[repr(C)] +pub enum VMFunctionKind { + /// A function is static when it's address matches the signature: + /// (vmctx, vmctx, arg1, arg2...) -> (result1, result2, ...) + /// + /// This is the default for functions that are defined: + /// 1. In the Host, natively + /// 2. In the WebAssembly file + Static, + + /// A function is dynamic when it's address matches the signature: + /// (ctx, &[Type]) -> Vec + /// + /// This is the default for functions that are defined: + /// 1. In the Host, dynamically + Dynamic, +} + /// The fields compiled code needs to access to utilize a WebAssembly table /// imported from another instance. #[derive(Debug, Copy, Clone)] @@ -321,6 +387,7 @@ mod test_vmtable_definition { #[derive(Debug, Copy, Clone)] #[repr(C, align(16))] pub struct VMGlobalDefinition { + // TODO: use `UnsafeCell` here, make this not Copy; there's probably a ton of UB in this code right now storage: [u8; 16], // If more elements are added here, remember to add offset_of tests below! } diff --git a/lib/runtime/src/vmoffsets.rs b/lib/runtime/src/vmoffsets.rs index 0bf8dfd00..f2b9f73ba 100644 --- a/lib/runtime/src/vmoffsets.rs +++ b/lib/runtime/src/vmoffsets.rs @@ -92,6 +92,30 @@ impl VMOffsets { } } +/// Offsets for [`VMDynamicFunctionImportContext`]. +/// +/// [`VMDynamicFunctionImportContext`]: crate::vmcontext::VMDynamicFunctionImportContext +impl VMOffsets { + /// The offset of the `address` field. + #[allow(clippy::erasing_op)] + pub fn vmdynamicfunction_import_context_address(&self) -> u8 { + 0 * self.pointer_size + } + + /// The offset of the `ctx` field. + #[allow(clippy::identity_op)] + pub fn vmdynamicfunction_import_context_ctx(&self) -> u8 { + 1 * self.pointer_size + } + + /// Return the size of [`VMDynamicFunctionImportContext`]. + /// + /// [`VMDynamicFunctionImportContext`]: crate::vmcontext::VMDynamicFunctionImportContext + pub fn size_of_vmdynamicfunction_import_context(&self) -> u8 { + 2 * self.pointer_size + } +} + /// Offsets for `*const VMFunctionBody`. impl VMOffsets { /// The size of the `current_elements` field. diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index 84fa37b6d..833eb8c25 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -14,7 +14,7 @@ bincode = "1" byteorder = "1.3" thiserror = "1" generational-arena = { version = "0.2", features = ["serde"] } -libc = "0.2.60" +libc = { version = "0.2.70", default-features = false } tracing = "0.1" getrandom = "0.1" time = "0.1" diff --git a/lib/wasi/src/state/mod.rs b/lib/wasi/src/state/mod.rs index 3ebbb3b7d..f0a3e6871 100644 --- a/lib/wasi/src/state/mod.rs +++ b/lib/wasi/src/state/mod.rs @@ -19,7 +19,6 @@ mod types; pub use self::builder::*; pub use self::types::*; use crate::syscalls::types::*; -use crate::WasiEnv; use generational_arena::Arena; pub use generational_arena::Index as Inode; use serde::{Deserialize, Serialize}; @@ -52,15 +51,6 @@ const STDOUT_DEFAULT_RIGHTS: __wasi_rights_t = __WASI_RIGHT_FD_DATASYNC | __WASI_RIGHT_POLL_FD_READWRITE; const STDERR_DEFAULT_RIGHTS: __wasi_rights_t = STDOUT_DEFAULT_RIGHTS; -/// Get WasiState from a Ctx -/// -/// # Safety -/// - This function must be called on a `WasiEnv` that was created with `WasiState` -/// in the data field -pub unsafe fn get_wasi_state<'a>(env: &'a mut WasiEnv) -> &'a mut WasiState { - env.state_mut() -} - /// A completely aribtrary "big enough" number used as the upper limit for /// the number of symlinks that can be traversed when resolving a path pub const MAX_SYMLINKS: u32 = 128; diff --git a/lib/wasi/src/syscalls/mod.rs b/lib/wasi/src/syscalls/mod.rs index eff295d25..58085566b 100644 --- a/lib/wasi/src/syscalls/mod.rs +++ b/lib/wasi/src/syscalls/mod.rs @@ -1001,7 +1001,7 @@ pub fn fd_readdir( .map(|(name, inode)| { let entry = &state.fs.inodes[*inode]; ( - format!("{}", entry.name), + entry.name.to_string(), entry.stat.st_filetype, entry.stat.st_ino, ) @@ -1456,7 +1456,7 @@ pub fn path_filestat_get( flags & __WASI_LOOKUP_SYMLINK_FOLLOW != 0, )); let stat = if state.fs.inodes[file_inode].is_preopened { - state.fs.inodes[file_inode].stat.clone() + state.fs.inodes[file_inode].stat } else { wasi_try!(state .fs @@ -1998,7 +1998,7 @@ pub fn path_remove_directory( ), } - if let Err(_) = std::fs::remove_dir(path_str) { + if std::fs::remove_dir(path_str).is_err() { // reinsert to prevent FS from being in bad state if let Kind::Dir { ref mut entries, .. diff --git a/lib/wasi/src/utils.rs b/lib/wasi/src/utils.rs index e9a4e4ad8..bba0a3a27 100644 --- a/lib/wasi/src/utils.rs +++ b/lib/wasi/src/utils.rs @@ -1,4 +1,4 @@ -use wasmer::{ExternType, ImportType, Module}; +use wasmer::{ExternType, Module}; #[allow(dead_code)] /// Check if a provided module is compiled for some version of WASI. diff --git a/lib/wasm-common/src/indexes.rs b/lib/wasm-common/src/indexes.rs index 69a15e831..e761f9d72 100644 --- a/lib/wasm-common/src/indexes.rs +++ b/lib/wasm-common/src/indexes.rs @@ -55,6 +55,7 @@ entity_impl!(MemoryIndex); /// Index type of a signature (imported or local) inside the WebAssembly module. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +#[repr(transparent)] pub struct SignatureIndex(u32); entity_impl!(SignatureIndex); diff --git a/lib/wasm-common/src/native.rs b/lib/wasm-common/src/native.rs index d718a8394..d45739f3d 100644 --- a/lib/wasm-common/src/native.rs +++ b/lib/wasm-common/src/native.rs @@ -261,39 +261,26 @@ pub struct FunctionBody(*mut u8); /// Represents a function that can be used by WebAssembly. #[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub struct Func { +pub struct Func { address: *const FunctionBody, - env: Option<*mut Env>, _phantom: PhantomData<(Args, Rets)>, } unsafe impl Send for Func {} -impl Func +impl Func where Args: WasmTypeList, Rets: WasmTypeList, - Env: Sized, { /// Creates a new `Func`. - pub fn new(func: F) -> Self + pub fn new(func: F) -> Self where - F: HostFunction, + F: HostFunction, + T: HostFunctionKind, + E: Sized, { Self { - env: None, - address: func.to_raw(), - _phantom: PhantomData, - } - } - - /// Creates a new `Func` with a given `env`. - pub fn new_env(env: &mut Env, func: F) -> Self - where - F: HostFunction, - { - Self { - env: Some(env), address: func.to_raw(), _phantom: PhantomData, } @@ -304,11 +291,6 @@ where FunctionType::new(Args::wasm_types(), Rets::wasm_types()) } - /// Get the type of the Func - pub fn env(&self) -> Option<*mut Env> { - self.env - } - /// Get the address of the Func pub fn address(&self) -> *const FunctionBody { self.address diff --git a/lib/wasm-common/src/ref.rs b/lib/wasm-common/src/ref.rs index 20bc8f7f1..0727707ce 100644 --- a/lib/wasm-common/src/ref.rs +++ b/lib/wasm-common/src/ref.rs @@ -78,12 +78,12 @@ impl AnyRef { any: data, host_info: None, }; - AnyRef::Other(OtherRef(Rc::new(RefCell::new(info)))) + Self::Other(OtherRef(Rc::new(RefCell::new(info)))) } /// Creates a `Null` reference. pub fn null() -> Self { - AnyRef::Null + Self::Null } /// Returns the data stored in the reference if available. @@ -93,7 +93,7 @@ impl AnyRef { /// Panics if the variant isn't `AnyRef::Other`. pub fn data(&self) -> cell::Ref> { match self { - AnyRef::Other(OtherRef(r)) => cell::Ref::map(r.borrow(), |r| &r.any), + Self::Other(OtherRef(r)) => cell::Ref::map(r.borrow(), |r| &r.any), _ => panic!("expected AnyRef::Other"), } } @@ -102,11 +102,9 @@ impl AnyRef { /// values that compare as equal). pub fn ptr_eq(&self, other: &AnyRef) -> bool { match (self, other) { - (AnyRef::Null, AnyRef::Null) => true, - (AnyRef::Ref(InternalRef(ref a)), AnyRef::Ref(InternalRef(ref b))) => { - a.ptr_eq(b.as_ref()) - } - (AnyRef::Other(OtherRef(ref a)), AnyRef::Other(OtherRef(ref b))) => Rc::ptr_eq(a, b), + (Self::Null, AnyRef::Null) => true, + (Self::Ref(InternalRef(ref a)), Self::Ref(InternalRef(ref b))) => a.ptr_eq(b.as_ref()), + (Self::Other(OtherRef(ref a)), Self::Other(OtherRef(ref b))) => Rc::ptr_eq(a, b), _ => false, } } @@ -118,9 +116,9 @@ impl AnyRef { /// Panics if `AnyRef` is already borrowed or `AnyRef` is `Null`. pub fn host_info(&self) -> Option>> { match self { - AnyRef::Null => panic!("null"), - AnyRef::Ref(r) => r.0.host_info(), - AnyRef::Other(r) => { + Self::Null => panic!("null"), + Self::Ref(r) => r.0.host_info(), + Self::Other(r) => { let info = cell::RefMut::map(r.0.borrow_mut(), |b| &mut b.host_info); if info.is_none() { return None; @@ -137,9 +135,9 @@ impl AnyRef { /// Panics if `AnyRef` is already borrowed or `AnyRef` is `Null`. pub fn set_host_info(&self, info: Option>) { match self { - AnyRef::Null => panic!("null"), - AnyRef::Ref(r) => r.0.set_host_info(info), - AnyRef::Other(r) => { + Self::Null => panic!("null"), + Self::Ref(r) => r.0.set_host_info(info), + Self::Other(r) => { r.0.borrow_mut().host_info = info; } } @@ -149,9 +147,9 @@ impl AnyRef { impl fmt::Debug for AnyRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - AnyRef::Null => write!(f, "null"), - AnyRef::Ref(_) => write!(f, "anyref"), - AnyRef::Other(_) => write!(f, "other ref"), + Self::Null => write!(f, "null"), + Self::Ref(_) => write!(f, "anyref"), + Self::Other(_) => write!(f, "other ref"), } } } @@ -175,14 +173,14 @@ pub struct HostRef(Rc>>); impl HostRef { /// Creates a new `HostRef` from `T`. - pub fn new(item: T) -> HostRef { - let anyref_data: Weak> = Weak::new(); + pub fn new(item: T) -> Self { + let anyref_data: Weak = Weak::new(); let content = ContentBox { content: item, host_info: None, anyref_data, }; - HostRef(Rc::new(RefCell::new(content))) + Self(Rc::new(RefCell::new(content))) } /// Immutably borrows the wrapped data. @@ -205,7 +203,7 @@ impl HostRef { /// Returns true if the two `HostRef`'s point to the same value (not just /// values that compare as equal). - pub fn ptr_eq(&self, other: &HostRef) -> bool { + pub fn ptr_eq(&self, other: &Self) -> bool { Rc::ptr_eq(&self.0, &other.0) } @@ -253,8 +251,8 @@ impl InternalRefBase for HostRef { } impl Clone for HostRef { - fn clone(&self) -> HostRef { - HostRef(self.0.clone()) + fn clone(&self) -> Self { + Self(self.0.clone()) } } diff --git a/lib/wasm-common/src/types.rs b/lib/wasm-common/src/types.rs index 178cd63c3..e17017ea2 100644 --- a/lib/wasm-common/src/types.rs +++ b/lib/wasm-common/src/types.rs @@ -34,7 +34,7 @@ impl Type { /// `I64`, `F32`, `F64`, `V128`). pub fn is_num(&self) -> bool { match self { - Type::I32 | Type::I64 | Type::F32 | Type::F64 | Type::V128 => true, + Self::I32 | Self::I64 | Self::F32 | Self::F64 | Self::V128 => true, _ => false, } } @@ -42,7 +42,7 @@ impl Type { /// Returns true if `Type` matches either of the reference types. pub fn is_ref(&self) -> bool { match self { - Type::AnyRef | Type::FuncRef => true, + Self::AnyRef | Self::FuncRef => true, _ => false, } } @@ -171,7 +171,7 @@ macro_rules! accessors { /// Attempt to return the underlying type of this external type, /// returning `None` if it is a different type. pub fn $get(&self) -> Option<&$ty> { - if let ExternType::$variant(e) = self { + if let Self::$variant(e) = self { Some(e) } else { None @@ -200,10 +200,10 @@ impl ExternType { /// Check if two externs are compatible pub fn is_compatible_with(&self, other: &Self) -> bool { match (self, other) { - (ExternType::Function(a), ExternType::Function(b)) => a == b, - (ExternType::Global(a), ExternType::Global(b)) => is_global_compatible(a, b), - (ExternType::Table(a), ExternType::Table(b)) => is_table_compatible(a, b), - (ExternType::Memory(a), ExternType::Memory(b)) => is_memory_compatible(a, b), + (Self::Function(a), Self::Function(b)) => a == b, + (Self::Global(a), Self::Global(b)) => is_global_compatible(a, b), + (Self::Table(a), Self::Table(b)) => is_table_compatible(a, b), + (Self::Memory(a), Self::Memory(b)) => is_memory_compatible(a, b), // The rest of possibilities, are not compatible _ => false, } @@ -288,16 +288,16 @@ pub enum Mutability { } impl From for Mutability { - fn from(val: bool) -> Mutability { + fn from(val: bool) -> Self { match val { - false => Mutability::Const, - true => Mutability::Var, + false => Self::Const, + true => Self::Var, } } } impl From for bool { - fn from(val: Mutability) -> bool { + fn from(val: Mutability) -> Self { match val { Mutability::Const => false, Mutability::Var => true, @@ -374,20 +374,20 @@ impl GlobalInit { /// Get the `GlobalInit` from a given `Value` pub fn from_value(value: Value) -> Self { match value { - Value::I32(i) => GlobalInit::I32Const(i), - Value::I64(i) => GlobalInit::I64Const(i), - Value::F32(f) => GlobalInit::F32Const(f), - Value::F64(f) => GlobalInit::F64Const(f), + Value::I32(i) => Self::I32Const(i), + Value::I64(i) => Self::I64Const(i), + Value::F32(f) => Self::F32Const(f), + Value::F64(f) => Self::F64Const(f), _ => unimplemented!("GlobalInit from_value for {:?}", value), } } /// Get the `Value` from the Global init value pub fn to_value(&self) -> Value { match self { - GlobalInit::I32Const(i) => Value::I32(*i), - GlobalInit::I64Const(i) => Value::I64(*i), - GlobalInit::F32Const(f) => Value::F32(*f), - GlobalInit::F64Const(f) => Value::F64(*f), + Self::I32Const(i) => Value::I32(*i), + Self::I64Const(i) => Value::I64(*i), + Self::F32Const(f) => Value::F32(*f), + Self::F64Const(f) => Value::F64(*f), _ => unimplemented!("GlobalInit to_value for {:?}", self), } } @@ -414,8 +414,8 @@ pub struct TableType { impl TableType { /// Creates a new table descriptor which will contain the specified /// `element` and have the `limits` applied to its length. - pub fn new(ty: Type, minimum: u32, maximum: Option) -> TableType { - TableType { + pub fn new(ty: Type, minimum: u32, maximum: Option) -> Self { + Self { ty, minimum, maximum, @@ -453,15 +453,11 @@ pub struct MemoryType { impl MemoryType { /// Creates a new descriptor for a WebAssembly memory given the specified /// limits of the memory. - pub fn new( - minimum: IntoPages, - maximum: Option, - shared: bool, - ) -> MemoryType + pub fn new(minimum: IntoPages, maximum: Option, shared: bool) -> Self where IntoPages: Into, { - MemoryType { + Self { minimum: minimum.into(), maximum: maximum.map(|m| m.into()), shared, diff --git a/lib/wasm-common/src/units.rs b/lib/wasm-common/src/units.rs index df9cf3343..79a9a6eef 100644 --- a/lib/wasm-common/src/units.rs +++ b/lib/wasm-common/src/units.rs @@ -69,59 +69,59 @@ impl fmt::Debug for Bytes { } impl From for Bytes { - fn from(pages: Pages) -> Bytes { - Bytes((pages.0 as usize) * WASM_PAGE_SIZE) + fn from(pages: Pages) -> Self { + Self((pages.0 as usize) * WASM_PAGE_SIZE) } } impl From for Bytes { fn from(other: usize) -> Self { - Bytes(other) + Self(other) } } impl Sub for Pages where - T: Into, + T: Into, { - type Output = Pages; - fn sub(self, rhs: T) -> Pages { - Pages(self.0 - rhs.into().0) + type Output = Self; + fn sub(self, rhs: T) -> Self { + Self(self.0 - rhs.into().0) } } impl Add for Pages where - T: Into, + T: Into, { - type Output = Pages; - fn add(self, rhs: T) -> Pages { - Pages(self.0 + rhs.into().0) + type Output = Self; + fn add(self, rhs: T) -> Self { + Self(self.0 + rhs.into().0) } } impl From for Pages { - fn from(bytes: Bytes) -> Pages { - Pages((bytes.0 / WASM_PAGE_SIZE) as u32) + fn from(bytes: Bytes) -> Self { + Self((bytes.0 / WASM_PAGE_SIZE) as u32) } } impl Sub for Bytes where - T: Into, + T: Into, { - type Output = Bytes; - fn sub(self, rhs: T) -> Bytes { - Bytes(self.0 - rhs.into().0) + type Output = Self; + fn sub(self, rhs: T) -> Self { + Self(self.0 - rhs.into().0) } } impl Add for Bytes where - T: Into, + T: Into, { - type Output = Bytes; - fn add(self, rhs: T) -> Bytes { - Bytes(self.0 + rhs.into().0) + type Output = Self; + fn add(self, rhs: T) -> Self { + Self(self.0 + rhs.into().0) } } diff --git a/lib/wasm-common/src/values.rs b/lib/wasm-common/src/values.rs index 20c8d63f0..49d3900ca 100644 --- a/lib/wasm-common/src/values.rs +++ b/lib/wasm-common/src/values.rs @@ -37,7 +37,7 @@ macro_rules! accessors { /// Attempt to access the underlying value of this `Value`, returning /// `None` if it is not the correct type. pub fn $get(&self) -> Option<$ty> { - if let Value::$variant($bind) = self { + if let Self::$variant($bind) = self { Some($cvt) } else { None @@ -58,43 +58,53 @@ macro_rules! accessors { impl Value { /// Returns a null `anyref` value. - pub fn null() -> Value { - Value::AnyRef(AnyRef::null()) + pub fn null() -> Self { + Self::AnyRef(AnyRef::null()) } /// Returns the corresponding [`Type`] for this `Value`. pub fn ty(&self) -> Type { match self { - Value::I32(_) => Type::I32, - Value::I64(_) => Type::I64, - Value::F32(_) => Type::F32, - Value::F64(_) => Type::F64, - Value::AnyRef(_) => Type::AnyRef, - Value::FuncRef(_) => Type::FuncRef, - Value::V128(_) => Type::V128, + Self::I32(_) => Type::I32, + Self::I64(_) => Type::I64, + Self::F32(_) => Type::F32, + Self::F64(_) => Type::F64, + Self::AnyRef(_) => Type::AnyRef, + Self::FuncRef(_) => Type::FuncRef, + Self::V128(_) => Type::V128, } } /// Writes it's value to a given pointer + /// + /// # Safety + /// `p` must be: + /// - Sufficiently aligned for the Rust equivalent of the type in `self` + /// - Non-null and pointing to valid, mutable memory pub unsafe fn write_value_to(&self, p: *mut i128) { match self { - Value::I32(i) => ptr::write(p as *mut i32, *i), - Value::I64(i) => ptr::write(p as *mut i64, *i), - Value::F32(u) => ptr::write(p as *mut f32, *u), - Value::F64(u) => ptr::write(p as *mut f64, *u), - Value::V128(b) => ptr::write(p as *mut u128, *b), + Self::I32(i) => ptr::write(p as *mut i32, *i), + Self::I64(i) => ptr::write(p as *mut i64, *i), + Self::F32(u) => ptr::write(p as *mut f32, *u), + Self::F64(u) => ptr::write(p as *mut f64, *u), + Self::V128(b) => ptr::write(p as *mut u128, *b), _ => unimplemented!("Value::write_value_to"), } } /// Gets a `Value` given a pointer and a `Type` + /// + /// # Safety + /// `p` must be: + /// - Properly aligned to the specified `ty`'s Rust equivalent + /// - Non-null and pointing to valid memory pub unsafe fn read_value_from(p: *const i128, ty: Type) -> Value { match ty { - Type::I32 => Value::I32(ptr::read(p as *const i32)), - Type::I64 => Value::I64(ptr::read(p as *const i64)), - Type::F32 => Value::F32(ptr::read(p as *const f32)), - Type::F64 => Value::F64(ptr::read(p as *const f64)), - Type::V128 => Value::V128(ptr::read(p as *const u128)), + Type::I32 => Self::I32(ptr::read(p as *const i32)), + Type::I64 => Self::I64(ptr::read(p as *const i64)), + Type::F32 => Self::F32(ptr::read(p as *const f32)), + Type::F64 => Self::F64(ptr::read(p as *const f64)), + Type::V128 => Self::V128(ptr::read(p as *const u128)), _ => unimplemented!("Value::read_value_from"), } } @@ -115,7 +125,7 @@ impl Value { /// This will return `Some` for both the `AnyRef` and `FuncRef` types. pub fn anyref(&self) -> Option { match self { - Value::AnyRef(e) => Some(e.clone()), + Self::AnyRef(e) => Some(e.clone()), _ => None, } } @@ -134,13 +144,13 @@ impl Value { impl fmt::Debug for Value { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Value::I32(v) => write!(f, "I32({:?})", v), - Value::I64(v) => write!(f, "I64({:?})", v), - Value::F32(v) => write!(f, "F32({:?})", v), - Value::F64(v) => write!(f, "F64({:?})", v), - Value::AnyRef(v) => write!(f, "AnyRef({:?})", v), - Value::FuncRef(_) => write!(f, "FuncRef"), - Value::V128(v) => write!(f, "V128({:?})", v), + Self::I32(v) => write!(f, "I32({:?})", v), + Self::I64(v) => write!(f, "I64({:?})", v), + Self::F32(v) => write!(f, "F32({:?})", v), + Self::F64(v) => write!(f, "F64({:?})", v), + Self::AnyRef(v) => write!(f, "AnyRef({:?})", v), + Self::FuncRef(_) => write!(f, "FuncRef"), + Self::V128(v) => write!(f, "V128({:?})", v), } } } @@ -148,49 +158,49 @@ impl fmt::Debug for Value { impl ToString for Value { fn to_string(&self) -> String { match self { - Value::I32(v) => format!("{}", v), - Value::I64(v) => format!("{}", v), - Value::F32(v) => format!("{}", v), - Value::F64(v) => format!("{}", v), - Value::AnyRef(_) => format!("anyref"), - Value::FuncRef(_) => format!("funcref"), - Value::V128(v) => format!("{}", v), + Self::I32(v) => v.to_string(), + Self::I64(v) => v.to_string(), + Self::F32(v) => v.to_string(), + Self::F64(v) => v.to_string(), + Self::AnyRef(_) => "anyref".to_string(), + Self::FuncRef(_) => "funcref".to_string(), + Self::V128(v) => v.to_string(), } } } impl From for Value { - fn from(val: i32) -> Value { - Value::I32(val) + fn from(val: i32) -> Self { + Self::I32(val) } } impl From for Value { - fn from(val: i64) -> Value { - Value::I64(val) + fn from(val: i64) -> Self { + Self::I64(val) } } impl From for Value { - fn from(val: f32) -> Value { - Value::F32(val) + fn from(val: f32) -> Self { + Self::F32(val) } } impl From for Value { - fn from(val: f64) -> Value { - Value::F64(val) + fn from(val: f64) -> Self { + Self::F64(val) } } impl From for Value { - fn from(val: AnyRef) -> Value { - Value::AnyRef(val) + fn from(val: AnyRef) -> Self { + Self::AnyRef(val) } } // impl From for Value { -// fn from(val: T) -> Value { -// Value::FuncRef(val) +// fn from(val: T) -> Self { +// Self::FuncRef(val) // } // } diff --git a/src/commands/cache.rs b/src/commands/cache.rs index e07b17e64..f5aa21be8 100644 --- a/src/commands/cache.rs +++ b/src/commands/cache.rs @@ -33,7 +33,7 @@ impl Cache { if cache_dir.exists() { fs::remove_dir_all(cache_dir.clone())?; } - fs::create_dir_all(cache_dir.clone())?; + fs::create_dir_all(cache_dir)?; eprintln!("Wasmer cache cleaned successfully."); Ok(()) } diff --git a/src/commands/run.rs b/src/commands/run.rs index 13aacc150..f9725b736 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -2,7 +2,7 @@ use crate::common::get_cache_dir; use crate::store::StoreOptions; use crate::suggestions::suggest_function_exports; use crate::warning; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{anyhow, Context, Result}; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; @@ -84,7 +84,7 @@ impl Run { fn inner_execute(&self) -> Result<()> { let module = self .get_module() - .with_context(|| format!("module instantiation failed"))?; + .with_context(|| "module instantiation failed")?; // Do we want to invoke a function? if let Some(ref invoke) = self.invoke { let imports = imports! {}; @@ -118,14 +118,14 @@ impl Run { return self .wasi .execute(module, version, program_name, self.args.clone()) - .with_context(|| format!("WASI execution failed")); + .with_context(|| "WASI execution failed"); } } // Try to instantiate the wasm file, with no provided imports let imports = imports! {}; let instance = Instance::new(&module, &imports)?; - let start: Function = self.try_find_function(&instance, "_start", &vec![])?; + let start: Function = self.try_find_function(&instance, "_start", &[])?; start.call(&[])?; Ok(()) @@ -169,8 +169,8 @@ impl Run { .cache_key .as_ref() .and_then(|key| WasmHash::from_str(&key).ok()) - .unwrap_or(WasmHash::generate(&contents)); - let module = match unsafe { cache.load(&store, hash) } { + .unwrap_or_else(|| WasmHash::generate(&contents)); + match unsafe { cache.load(&store, hash) } { Ok(module) => module, Err(e) => { match e { @@ -186,8 +186,7 @@ impl Run { cache.store(hash, module.clone())?; module } - }; - module + } } else { Module::new(&store, &contents)? }; @@ -201,13 +200,13 @@ impl Run { &self, instance: &Instance, name: &str, - args: &Vec, + args: &[String], ) -> Result { Ok(instance .exports .get_function(&name) .map_err(|e| { - if instance.module().info().functions.len() == 0 { + if instance.module().info().functions.is_empty() { anyhow!("The module has no exported functions to call.") } else { let suggested_functions = suggest_function_exports(instance.module(), ""); @@ -246,7 +245,7 @@ impl Run { &self, instance: &Instance, invoke: &str, - args: &Vec, + args: &[String], ) -> Result> { let func: Function = self.try_find_function(&instance, invoke, args)?; let func_ty = func.ty(); diff --git a/src/commands/run/wasi.rs b/src/commands/run/wasi.rs index c5e1280e6..cea6ddb64 100644 --- a/src/commands/run/wasi.rs +++ b/src/commands/run/wasi.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use std::path::PathBuf; use wasmer::{Function, Instance, Memory, Module}; @@ -118,7 +118,7 @@ impl Wasi { start .call(&[]) - .with_context(|| format!("failed to run WASI `_start` function"))?; + .with_context(|| "failed to run WASI `_start` function")?; Ok(()) } diff --git a/src/commands/self_update.rs b/src/commands/self_update.rs index a77d4a3d3..2218107c3 100644 --- a/src/commands/self_update.rs +++ b/src/commands/self_update.rs @@ -1,5 +1,5 @@ //! When wasmer self-update is executed, this is what gets executed -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; #[cfg(not(target_os = "windows"))] use std::process::{Command, Stdio}; use structopt::StructOpt; @@ -11,8 +11,7 @@ pub struct SelfUpdate {} impl SelfUpdate { /// Runs logic for the `self-update` subcommand pub fn execute(&self) -> Result<()> { - self.inner_execute() - .context(format!("failed to self-update wasmer")) + self.inner_execute().context("failed to self-update wasmer") } #[cfg(not(target_os = "windows"))] diff --git a/src/commands/wast.rs b/src/commands/wast.rs index be1d9e3df..081384944 100644 --- a/src/commands/wast.rs +++ b/src/commands/wast.rs @@ -30,8 +30,7 @@ impl Wast { let (store, _engine_name, _compiler_name) = self.compiler.get_store()?; let mut wast = WastSpectest::new_with_spectest(store); wast.fail_fast = self.fail_fast; - wast.run_file(&self.path) - .with_context(|| format!("tests failed"))?; + wast.run_file(&self.path).with_context(|| "tests failed")?; eprintln!("Wast tests succeeded for `{}`.", self.path.display()); Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index c402fda5e..ce0d42754 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,9 @@ #![doc(html_favicon_url = "https://wasmer.io/static/icons/favicon.ico")] #![doc(html_logo_url = "https://avatars3.githubusercontent.com/u/44205449?s=200&v=4")] +#[macro_use] +extern crate anyhow; + pub mod commands; pub mod common; #[macro_use] diff --git a/src/store.rs b/src/store.rs index d260b5be9..1d2fad6b0 100644 --- a/src/store.rs +++ b/src/store.rs @@ -2,7 +2,7 @@ //! commands. use crate::common::WasmFeatures; -use anyhow::{bail, Error, Result}; +use anyhow::{Error, Result}; use std::str::FromStr; use std::string::ToString; use std::sync::Arc; @@ -76,25 +76,25 @@ impl FromStr for Compiler { impl StoreOptions { fn get_compiler(&self) -> Result { if self.cranelift { - return Ok(Compiler::Cranelift); + Ok(Compiler::Cranelift) } else if self.llvm { - return Ok(Compiler::LLVM); + Ok(Compiler::LLVM) } else if self.singlepass { - return Ok(Compiler::Singlepass); + Ok(Compiler::Singlepass) } else if let Some(backend) = self.backend.clone() { warning!( "the `--backend={0}` flag is deprecated, please use `--{0}` instead", backend ); - return Compiler::from_str(&backend); + Compiler::from_str(&backend) } else { // Auto mode, we choose the best compiler for that platform if cfg!(feature = "cranelift") && cfg!(target_arch = "x86_64") { - return Ok(Compiler::Cranelift); + Ok(Compiler::Cranelift) } else if cfg!(feature = "singlepass") && cfg!(target_arch = "x86_64") { - return Ok(Compiler::Singlepass); + Ok(Compiler::Singlepass) } else if cfg!(feature = "llvm") { - return Ok(Compiler::LLVM); + Ok(Compiler::LLVM) } else { bail!("There are no available compilers for your architecture") } @@ -138,7 +138,7 @@ impl StoreOptions { compiler.to_string() ), }; - return Ok(config); + Ok(config) } /// Gets the compiler config diff --git a/src/utils.rs b/src/utils.rs index aa80e68f6..6a0fd056e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -6,5 +6,5 @@ pub fn wasmer_should_print_color() -> bool { env::var("WASMER_COLOR") .ok() .and_then(|inner| inner.parse::().ok()) - .unwrap_or(atty::is(atty::Stream::Stdout)) + .unwrap_or_else(|| atty::is(atty::Stream::Stdout)) } diff --git a/tests/lib/test-generator/src/lib.rs b/tests/lib/test-generator/src/lib.rs index 3da80eedf..629174f5b 100644 --- a/tests/lib/test-generator/src/lib.rs +++ b/tests/lib/test-generator/src/lib.rs @@ -70,20 +70,16 @@ pub fn build_ignores_from_textfile(path: PathBuf) -> anyhow::Result { } else { (line, None) }; - if line.len() == 0 { + if line.is_empty() { continue; } - match target { - Some(t) => { - // We skip the ignore if doesn't apply to the current - // host target - if !host.contains(&t) { - continue; - } - } - None => {} + // We skip the ignore if doesn't apply to the current + // host target + if target.map(|t| !host.contains(&t)).unwrap_or(false) { + continue; } + ignores.insert(line); } Ok(ignores) diff --git a/tests/lib/test-utils/Cargo.toml b/tests/lib/test-utils/Cargo.toml index 3a9457695..3e9f62d78 100644 --- a/tests/lib/test-utils/Cargo.toml +++ b/tests/lib/test-utils/Cargo.toml @@ -12,6 +12,8 @@ wasmer-compiler = { path = "../../../lib/compiler", version = "0.16.2" } wasmer-compiler-singlepass = { path = "../../../lib/compiler-singlepass", version = "0.16.2", optional = true } wasmer-compiler-cranelift = { path = "../../../lib/compiler-cranelift", version = "0.16.2", optional = true } wasmer-compiler-llvm = { path = "../../../lib/compiler-llvm", version = "0.16.2", optional = true } +wasmer-engine-jit = { path = "../../../lib/engine-jit", version = "0.16.2" } +wasmer = { path = "../../../lib/api", version = "0.16.2", deafult-features = false } [features] compiler = [] diff --git a/tests/lib/test-utils/src/lib.rs b/tests/lib/test-utils/src/lib.rs index 4a9dd834c..bc737e636 100644 --- a/tests/lib/test-utils/src/lib.rs +++ b/tests/lib/test-utils/src/lib.rs @@ -1,13 +1,16 @@ #![cfg(feature = "compiler")] +use std::sync::Arc; +use wasmer::{Store, Tunables}; use wasmer_compiler::{CompilerConfig, Features, Target}; +use wasmer_engine_jit::JITEngine; pub fn get_compiler_config_from_str( compiler_name: &str, try_nan_canonicalization: bool, features: Features, ) -> Box { - /// We use the current host target for testing locally + // We use the current host target for testing locally let target = Target::default(); match compiler_name { @@ -34,3 +37,10 @@ pub fn get_compiler_config_from_str( _ => panic!("Compiler {} not supported", compiler_name), } } + +/// for when you need a store but you don't care about the details +pub fn get_default_store() -> Store { + let compiler_config = get_compiler_config_from_str("cranelift", false, Features::default()); + let tunables = Tunables::for_target(compiler_config.target().triple()); + Store::new(Arc::new(JITEngine::new(&*compiler_config, tunables))) +} diff --git a/tests/lib/wast/src/lib.rs b/tests/lib/wast/src/lib.rs index be1e65030..9742817a8 100644 --- a/tests/lib/wast/src/lib.rs +++ b/tests/lib/wast/src/lib.rs @@ -4,10 +4,7 @@ #![warn(unused_import_braces)] #![deny(unstable_features)] #![cfg_attr(feature = "clippy", plugin(clippy(conf_file = "../../clippy.toml")))] -#![cfg_attr( - feature = "cargo-clippy", - allow(clippy::new_without_default, clippy::new_without_default_derive) -)] +#![cfg_attr(feature = "cargo-clippy", allow(clippy::new_without_default))] #![cfg_attr( feature = "cargo-clippy", warn( diff --git a/tests/lib/wast/src/spectest.rs b/tests/lib/wast/src/spectest.rs index 4eadfc492..be90476d6 100644 --- a/tests/lib/wast/src/spectest.rs +++ b/tests/lib/wast/src/spectest.rs @@ -26,7 +26,7 @@ pub fn spectest_importobject(store: &Store) -> ImportObject { let table = Table::new(store, ty, Val::AnyRef(AnyRef::Null)).unwrap(); let ty = MemoryType::new(1, Some(2), false); - let memory = Memory::new(store, ty); + let memory = Memory::new(store, ty).unwrap(); imports! { "spectest" => { diff --git a/tests/lib/wast/src/wast.rs b/tests/lib/wast/src/wast.rs index 104063cf4..91de2fe14 100644 --- a/tests/lib/wast/src/wast.rs +++ b/tests/lib/wast/src/wast.rs @@ -42,7 +42,7 @@ impl Wast { /// A list of instantiation failures to allow pub fn allow_instantiation_failures(&mut self, failures: &[&str]) { - for failure_str in failures.iter() { + for &failure_str in failures.iter() { self.allowed_instantiation_failures .insert(failure_str.to_string()); } @@ -195,7 +195,7 @@ impl Wast { wast::QuoteModule::Quote(_) => return Ok(()), }; let bytes = module.encode()?; - if let Ok(_) = self.module(None, &bytes) { + if self.module(None, &bytes).is_ok() { bail!("expected malformed module to fail to instantiate"); } } @@ -238,29 +238,26 @@ impl Wast { let mut errors = Vec::with_capacity(ast.directives.len()); for directive in ast.directives { let sp = directive.span(); - match self.run_directive(directive) { - Err(e) => { - let message = format!("{}", e); - // If depends on an instance that doesn't exist - if message.contains("no previous instance found") { - continue; - } - // We don't compute it, comes from instantiating an instance - // that we expected to fail. - if self.current.is_none() && self.current_is_allowed_failure { - continue; - } - let (line, col) = sp.linecol_in(wast); - errors.push(DirectiveError { - line: line + 1, - col, - message: message, - }); - if self.fail_fast { - break; - } + if let Err(e) = self.run_directive(directive) { + let message = format!("{}", e); + // If depends on an instance that doesn't exist + if message.contains("no previous instance found") { + continue; + } + // We don't compute it, comes from instantiating an instance + // that we expected to fail. + if self.current.is_none() && self.current_is_allowed_failure { + continue; + } + let (line, col) = sp.linecol_in(wast); + errors.push(DirectiveError { + line: line + 1, + col, + message, + }); + if self.fail_fast { + break; } - Ok(_) => {} } } if !errors.is_empty() { @@ -491,7 +488,7 @@ impl NaNCheck for f32 { } fn is_canonical_nan(&self) -> bool { - return (self.to_bits() & 0x7fff_ffff) == 0x7fc0_0000; + (self.to_bits() & 0x7fff_ffff) == 0x7fc0_0000 } } diff --git a/tests/misc.rs b/tests/misc.rs new file mode 100644 index 000000000..fbf1b724b --- /dev/null +++ b/tests/misc.rs @@ -0,0 +1,41 @@ +use test_utils::get_default_store; +use wasmer::{Memory, MemoryError, MemoryType, Pages}; + +#[test] +fn growing_memory_with_api() { + let desc = MemoryType::new(Pages(10), Some(Pages(16)), false); + let store = get_default_store(); + + let memory = Memory::new(&store, desc).unwrap(); + + assert_eq!(memory.size(), Pages(10)); + let result = memory.grow(Pages(2)).unwrap(); + assert_eq!(result, Pages(10)); + assert_eq!(memory.size(), Pages(12)); + + let result = memory.grow(Pages(10)); + assert_eq!( + result, + Err(MemoryError::CouldNotGrow { + current: 12.into(), + attempted_delta: 10.into(), + }) + ); + + let bad_desc = MemoryType::new(Pages(15), Some(Pages(10)), false); + let bad_result = Memory::new(&store, bad_desc); + + // due to stack overflow with a modern nightly, we can't update CI to use a version of nightly which will make this work + /*assert!(matches!( + bad_result, + Err(MemoryError::InvalidMemoryPlan { .. }) + ));*/ + + assert!( + if let Err(MemoryError::InvalidMemoryPlan { .. }) = bad_result { + true + } else { + false + } + ); +} diff --git a/tests/wast.rs b/tests/wast.rs index b691d97e4..10a733837 100644 --- a/tests/wast.rs +++ b/tests/wast.rs @@ -47,8 +47,8 @@ fn run_wast(wast_path: &str, compiler: &str) -> anyhow::Result<()> { native.set_deterministic_prefixer(native_prefixer); let store = Store::new(Arc::new(native)); let mut wast = Wast::new_with_spectest(store); - if compiler == "singlepass" { - // We don't support multivalue yet in singlepass + if compiler == "singlepass" || compiler == "llvm" { + // We don't support multivalue yet in singlepass or llvm wast.allow_instantiation_failures(&[ "Validation error: invalid result arity: func type returns multiple values", "Validation error: blocks, loops, and ifs accept no parameters when multi-value is not enabled"