From c7c1507b8b20c841252043995062f83ef39482df Mon Sep 17 00:00:00 2001 From: John Sharratt's Shared Account Date: Wed, 17 Aug 2022 17:45:48 +1000 Subject: [PATCH 01/40] Implemented LinearMemory trait --- Cargo.lock | 1 + lib/api/src/js/export.rs | 6 + lib/api/src/js/externals/memory.rs | 35 +- lib/api/src/js/function_env.rs | 7 +- lib/api/src/js/imports.rs | 30 ++ lib/api/src/js/mod.rs | 6 + lib/api/src/js/native.rs | 1 - lib/api/src/js/store.rs | 5 + lib/api/src/sys/externals/memory.rs | 16 +- lib/api/src/sys/externals/memory_view.rs | 2 +- lib/api/src/sys/mod.rs | 2 +- .../src/translator/code_translator.rs | 7 +- lib/compiler/src/engine/resolver.rs | 2 +- lib/compiler/src/translator/environ.rs | 2 +- lib/types/Cargo.toml | 3 + lib/types/src/error.rs | 48 +- lib/types/src/lib.rs | 6 +- lib/types/src/memory.rs | 88 +++- lib/vm/src/instance/allocator.rs | 4 +- lib/vm/src/instance/mod.rs | 16 +- lib/vm/src/lib.rs | 7 +- lib/vm/src/memory.rs | 461 +++++++++++------- lib/vm/src/store.rs | 20 + lib/vm/src/vmcontext.rs | 161 ++---- 24 files changed, 600 insertions(+), 336 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9328944bd..04df81af3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3194,6 +3194,7 @@ dependencies = [ "enum-iterator", "enumset", "indexmap", + "memoffset", "more-asserts", "rkyv", "serde", diff --git a/lib/api/src/js/export.rs b/lib/api/src/js/export.rs index f396e617c..024189901 100644 --- a/lib/api/src/js/export.rs +++ b/lib/api/src/js/export.rs @@ -7,6 +7,7 @@ use std::fmt; use wasm_bindgen::{JsCast, JsValue}; use wasmer_types::{ExternType, FunctionType, GlobalType, MemoryType, TableType}; +/// Represents linear memory that is managed by the javascript runtime #[derive(Clone, Debug, PartialEq)] pub struct VMMemory { pub(crate) memory: Memory, @@ -20,6 +21,11 @@ impl VMMemory { pub(crate) fn new(memory: Memory, ty: MemoryType) -> Self { Self { memory, ty } } + + /// Attempts to clone this memory (if its clonable) + pub(crate) fn try_clone(&self) -> Option { + Some(self.clone()) + } } #[derive(Clone, Debug, PartialEq)] diff --git a/lib/api/src/js/externals/memory.rs b/lib/api/src/js/externals/memory.rs index 6c5229ce9..2b66d8ed7 100644 --- a/lib/api/src/js/externals/memory.rs +++ b/lib/api/src/js/externals/memory.rs @@ -6,7 +6,6 @@ use crate::js::{MemoryAccessError, MemoryType}; use std::marker::PhantomData; use std::mem::MaybeUninit; use std::slice; -use thiserror::Error; #[cfg(feature = "tracing")] use tracing::warn; @@ -16,22 +15,7 @@ use wasmer_types::Pages; use super::MemoryView; -/// Error type describing things that can go wrong when operating on Wasm Memories. -#[derive(Error, Debug, Clone, PartialEq, Hash)] -pub enum MemoryError { - /// The operation would cause the size of the memory to exceed the maximum or would cause - /// an overflow leading to unindexable memory. - #[error("The memory could not grow: current size {} pages, requested increase: {} pages", current.0, attempted_delta.0)] - CouldNotGrow { - /// The current size in pages. - current: Pages, - /// The attempted amount to grow by in pages. - attempted_delta: Pages, - }, - /// A user defined error value, used for error cases not listed above. - #[error("A user-defined error occurred: {0}")] - Generic(String), -} +pub use wasmer_types::MemoryError; #[wasm_bindgen] extern "C" { @@ -116,6 +100,17 @@ impl Memory { Ok(Self::from_vm_export(store, vm_memory)) } + /// Creates a new host `Memory` from provided JavaScript memory. + pub fn new_raw(store: &mut impl AsStoreMut, js_memory: js_sys::WebAssembly::Memory, ty: MemoryType) -> Result { + let vm_memory = VMMemory::new(js_memory, ty); + Ok(Self::from_vm_export(store, vm_memory)) + } + + /// Create a memory object from an existing memory and attaches it to the store + pub fn new_from_existing(new_store: &mut impl AsStoreMut, memory: VMMemory) -> Self { + Self::from_vm_export(new_store, memory) + } + /// Returns the [`MemoryType`] of the `Memory`. /// /// # Example @@ -210,6 +205,12 @@ impl Memory { } } + /// Attempts to clone this memory (if its clonable) + pub fn try_clone(&self, store: &impl AsStoreRef) -> Option { + let mem = self.handle.get(store.as_store_ref().objects()); + mem.try_clone() + } + /// Checks whether this `Global` can be used with the given context. pub fn is_from_store(&self, store: &impl AsStoreRef) -> bool { self.handle.store_id() == store.as_store_ref().objects().id() diff --git a/lib/api/src/js/function_env.rs b/lib/api/src/js/function_env.rs index 1705f1a47..524fc7a2d 100644 --- a/lib/api/src/js/function_env.rs +++ b/lib/api/src/js/function_env.rs @@ -36,7 +36,7 @@ impl FunctionEnv { } /// Get the data as reference - pub fn as_ref<'a>(&self, store: &'a impl AsStoreMut) -> &'a T + pub fn as_ref<'a>(&self, store: &'a impl AsStoreRef) -> &'a T where T: Any + Send + 'static + Sized, { @@ -112,6 +112,11 @@ impl FunctionEnvMut<'_, T> { self.func_env.as_mut(&mut self.store_mut) } + /// Borrows a new immmutable reference + pub fn as_ref(&self) -> FunctionEnv { + self.func_env.clone() + } + /// Borrows a new mutable reference pub fn as_mut<'a>(&'a mut self) -> FunctionEnvMut<'a, T> { FunctionEnvMut { diff --git a/lib/api/src/js/imports.rs b/lib/api/src/js/imports.rs index 95a993099..d3b2c42ef 100644 --- a/lib/api/src/js/imports.rs +++ b/lib/api/src/js/imports.rs @@ -174,6 +174,36 @@ impl Imports { } imports } + + /// Iterates through all the imports in this structure + pub fn iter<'a>(&'a self) -> ImportsIterator<'a> { + ImportsIterator::new(self) + } +} + +pub struct ImportsIterator<'a> { + iter: std::collections::hash_map::Iter<'a, (String, String), Extern> +} + +impl<'a> ImportsIterator<'a> +{ + fn new(imports: &'a Imports) -> Self { + let iter = imports.map.iter(); + Self { iter } + } +} + +impl<'a> Iterator +for ImportsIterator<'a> { + type Item = (&'a str, &'a str, &'a Extern); + + fn next(&mut self) -> Option { + self.iter + .next() + .map(|(k, v)| { + (k.0.as_str(), k.1.as_str(), v) + }) + } } impl IntoIterator for &Imports { diff --git a/lib/api/src/js/mod.rs b/lib/api/src/js/mod.rs index 14594839a..a88509086 100644 --- a/lib/api/src/js/mod.rs +++ b/lib/api/src/js/mod.rs @@ -73,6 +73,12 @@ pub use crate::js::types::{ pub use crate::js::value::Value; pub use crate::js::value::Value as Val; +pub mod vm { + //! The `vm` module re-exports wasmer-vm types. + + pub use crate::js::export::VMMemory; +} + pub use wasmer_types::is_wasm; pub use wasmer_types::{ Bytes, ExportIndex, GlobalInit, LocalFunctionIndex, Pages, ValueType, WASM_MAX_PAGES, diff --git a/lib/api/src/js/native.rs b/lib/api/src/js/native.rs index 0e9066eda..69b350652 100644 --- a/lib/api/src/js/native.rs +++ b/lib/api/src/js/native.rs @@ -11,7 +11,6 @@ use std::marker::PhantomData; use crate::js::externals::Function; use crate::js::store::{AsStoreMut, AsStoreRef, StoreHandle}; -use crate::js::FunctionEnv; use crate::js::{FromToNativeWasmType, RuntimeError, WasmTypeList}; // use std::panic::{catch_unwind, AssertUnwindSafe}; use crate::js::export::VMFunction; diff --git a/lib/api/src/js/store.rs b/lib/api/src/js/store.rs index b6c10e846..66d6d58ec 100644 --- a/lib/api/src/js/store.rs +++ b/lib/api/src/js/store.rs @@ -263,6 +263,11 @@ mod objects { self.id } + /// Sets the ID of this store + pub fn set_id(&mut self, id: StoreId) { + self.id = id; + } + /// Returns a pair of mutable references from two handles. /// /// Panics if both handles point to the same object. diff --git a/lib/api/src/sys/externals/memory.rs b/lib/api/src/sys/externals/memory.rs index c0db55fa8..64ab9a136 100644 --- a/lib/api/src/sys/externals/memory.rs +++ b/lib/api/src/sys/externals/memory.rs @@ -10,7 +10,7 @@ use std::mem::MaybeUninit; use std::slice; #[cfg(feature = "tracing")] use tracing::warn; -use wasmer_types::Pages; +use wasmer_types::{Pages, LinearMemory}; use wasmer_vm::{InternalStoreHandle, MemoryError, StoreHandle, VMExtern, VMMemory}; use super::MemoryView; @@ -60,6 +60,13 @@ impl Memory { }) } + /// Create a memory object from an existing memory and attaches it to the store + pub fn new_from_existing(new_store: &mut impl AsStoreMut, memory: VMMemory) -> Self { + Self { + handle: StoreHandle::new(new_store.objects_mut(), memory) + } + } + /// Returns the [`MemoryType`] of the `Memory`. /// /// # Example @@ -142,6 +149,13 @@ impl Memory { self.handle.store_id() == store.as_store_ref().objects().id() } + /// Attempts to clone this memory (if its clonable) + pub fn try_clone(&self, store: &impl AsStoreRef) -> Option { + let mem = self.handle.get(store.as_store_ref().objects()); + mem.try_clone() + .map(|mem| mem.into()) + } + pub(crate) fn to_vm_extern(&self) -> VMExtern { VMExtern::Memory(self.handle.internal_handle()) } diff --git a/lib/api/src/sys/externals/memory_view.rs b/lib/api/src/sys/externals/memory_view.rs index 891700c08..a638acb6b 100644 --- a/lib/api/src/sys/externals/memory_view.rs +++ b/lib/api/src/sys/externals/memory_view.rs @@ -4,7 +4,7 @@ use std::convert::TryInto; use std::marker::PhantomData; use std::mem::MaybeUninit; use std::slice; -use wasmer_types::Pages; +use wasmer_types::{Pages, LinearMemory}; use super::memory::MemoryBuffer; use super::Memory; diff --git a/lib/api/src/sys/mod.rs b/lib/api/src/sys/mod.rs index 1c272a2a6..b6f9088c3 100644 --- a/lib/api/src/sys/mod.rs +++ b/lib/api/src/sys/mod.rs @@ -58,7 +58,7 @@ pub mod vm { pub use wasmer_vm::{ MemoryError, MemoryStyle, TableStyle, VMExtern, VMMemory, VMMemoryDefinition, VMTable, - VMTableDefinition, + VMTableDefinition, }; } diff --git a/lib/compiler-cranelift/src/translator/code_translator.rs b/lib/compiler-cranelift/src/translator/code_translator.rs index c750d6c32..60da48347 100644 --- a/lib/compiler-cranelift/src/translator/code_translator.rs +++ b/lib/compiler-cranelift/src/translator/code_translator.rs @@ -1071,8 +1071,7 @@ pub fn translate_operator( expected, timeout, )?; - state.push1(res); - } + state.push1(res); } Operator::MemoryAtomicNotify { memarg } => { let heap_index = MemoryIndex::from_u32(memarg.memory); let heap = state.get_heap(builder.func, memarg.memory, environ)?; @@ -1080,8 +1079,8 @@ pub fn translate_operator( let addr = state.pop1(); // 32 (fixed) let addr = fold_atomic_mem_addr(addr, memarg, I32, builder); let res = - environ.translate_atomic_notify(builder.cursor(), heap_index, heap, addr, count)?; - state.push1(res); + environ.translate_atomic_notify(builder.cursor(), heap_index, heap, addr, count)?; + state.push1(res); } Operator::I32AtomicLoad { memarg } => { translate_atomic_load(I32, I32, memarg, builder, state, environ)? diff --git a/lib/compiler/src/engine/resolver.rs b/lib/compiler/src/engine/resolver.rs index 207380bb8..94d40a502 100644 --- a/lib/compiler/src/engine/resolver.rs +++ b/lib/compiler/src/engine/resolver.rs @@ -4,7 +4,7 @@ use crate::LinkError; use more_asserts::assert_ge; use wasmer_types::entity::{BoxedSlice, EntityRef, PrimaryMap}; use wasmer_types::{ - ExternType, FunctionIndex, ImportError, ImportIndex, MemoryIndex, ModuleInfo, TableIndex, + ExternType, FunctionIndex, ImportError, ImportIndex, MemoryIndex, ModuleInfo, TableIndex, LinearMemory, }; use wasmer_vm::{ diff --git a/lib/compiler/src/translator/environ.rs b/lib/compiler/src/translator/environ.rs index e172d92b0..cd00a7bc0 100644 --- a/lib/compiler/src/translator/environ.rs +++ b/lib/compiler/src/translator/environ.rs @@ -1,8 +1,8 @@ // This file contains code from external sources. // Attributions: https://github.com/wasmerio/wasmer/blob/master/ATTRIBUTIONS.md use super::state::ModuleTranslationState; -use crate::lib::std::borrow::ToOwned; use crate::lib::std::string::ToString; +use crate::lib::std::borrow::ToOwned; use crate::lib::std::{boxed::Box, string::String, vec::Vec}; use crate::translate_module; use crate::wasmparser::{Operator, Range, Type}; diff --git a/lib/types/Cargo.toml b/lib/types/Cargo.toml index e10aebf87..3ee22d504 100644 --- a/lib/types/Cargo.toml +++ b/lib/types/Cargo.toml @@ -21,6 +21,9 @@ enum-iterator = "0.7.0" target-lexicon = { version = "0.12.2", default-features = false } enumset = "1.0" +[dev-dependencies] +memoffset = "0.6" + [features] default = ["std"] std = [] diff --git a/lib/types/src/error.rs b/lib/types/src/error.rs index 53555d623..0e0a1eb12 100644 --- a/lib/types/src/error.rs +++ b/lib/types/src/error.rs @@ -1,5 +1,5 @@ //! The WebAssembly possible errors -use crate::ExternType; +use crate::{ExternType, Pages}; use std::io; use thiserror::Error; @@ -37,6 +37,48 @@ pub enum DeserializeError { Compiler(#[from] CompileError), } +/// Error type describing things that can go wrong when operating on Wasm Memories. +#[derive(Error, Debug, Clone, PartialEq, Hash)] +pub enum MemoryError { + /// Low level error with mmap. + #[error("Error when allocating memory: {0}")] + Region(String), + /// The operation would cause the size of the memory to exceed the maximum or would cause + /// an overflow leading to unindexable memory. + #[error("The memory could not grow: current size {} pages, requested increase: {} pages", current.0, attempted_delta.0)] + CouldNotGrow { + /// The current size in pages. + current: Pages, + /// The attempted amount to grow by in pages. + attempted_delta: Pages, + }, + /// The operation would cause the size of the memory size exceed the maximum. + #[error("The memory is invalid because {}", reason)] + InvalidMemory { + /// The reason why the provided memory is invalid. + reason: String, + }, + /// Caller asked for more minimum memory than we can give them. + #[error("The minimum requested ({} pages) memory is greater than the maximum allowed memory ({} pages)", min_requested.0, max_allowed.0)] + MinimumMemoryTooLarge { + /// The number of pages requested as the minimum amount of memory. + min_requested: Pages, + /// The maximum amount of memory we can allocate. + max_allowed: Pages, + }, + /// Caller asked for a maximum memory greater than we can give them. + #[error("The maximum requested memory ({} pages) is greater than the maximum allowed memory ({} pages)", max_requested.0, max_allowed.0)] + MaximumMemoryTooLarge { + /// The number of pages requested as the maximum amount of memory. + max_requested: Pages, + /// The number of pages requested as the maximum amount of memory. + max_allowed: Pages, + }, + /// A user defined error value, used for error cases not listed above. + #[error("A user-defined error occurred: {0}")] + Generic(String), +} + /// An ImportError. /// /// Note: this error is not standard to WebAssembly, but it's @@ -52,6 +94,10 @@ pub enum ImportError { /// This error occurs when an import was expected but not provided. #[error("unknown import. Expected {0:?}")] UnknownImport(ExternType), + + /// Memory Error + #[error("memory error. {0}")] + MemoryError(String), } /// An error while preinstantiating a module. diff --git a/lib/types/src/lib.rs b/lib/types/src/lib.rs index 3fcfbe950..3f71079c9 100644 --- a/lib/types/src/lib.rs +++ b/lib/types/src/lib.rs @@ -77,7 +77,7 @@ pub use crate::compilation::target::{ pub use crate::serialize::{MetadataHeader, SerializableCompilation, SerializableModule}; pub use error::{ CompileError, DeserializeError, ImportError, MiddlewareError, ParseCpuFeatureError, - PreInstantiationError, SerializeError, WasmError, WasmResult, + PreInstantiationError, SerializeError, WasmError, WasmResult, MemoryError, }; /// The entity module, with common helpers for Rust structures @@ -103,7 +103,9 @@ pub use types::{ pub use value::{RawValue, ValueType}; pub use crate::libcalls::LibCall; -pub use crate::memory::MemoryStyle; +pub use crate::memory::{ + MemoryStyle, LinearMemory, VMMemoryDefinition +}; pub use crate::table::TableStyle; pub use crate::trapcode::TrapCode; pub use crate::vmoffsets::{TargetSharedSignatureIndex, VMBuiltinFunctionIndex, VMOffsets}; diff --git a/lib/types/src/memory.rs b/lib/types/src/memory.rs index fafcb955a..6ebac15e7 100644 --- a/lib/types/src/memory.rs +++ b/lib/types/src/memory.rs @@ -2,12 +2,16 @@ use crate::{Pages, ValueType}; use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; +use core::ptr::NonNull; use std::convert::{TryFrom, TryInto}; use std::iter::Sum; use std::ops::{Add, AddAssign}; +use super::MemoryType; +use super::MemoryError; + /// Implementation styles for WebAssembly linear memory. -#[derive(Debug, Clone, PartialEq, Eq, Hash, RkyvSerialize, RkyvDeserialize, Archive)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, RkyvSerialize, RkyvDeserialize, Archive)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] #[archive(as = "Self")] pub enum MemoryStyle { @@ -84,6 +88,9 @@ pub unsafe trait MemorySize: Copy { /// Zero value used for `WasmPtr::is_null`. const ZERO: Self::Offset; + /// One value used for counting. + const ONE: Self::Offset; + /// Convert an `Offset` to a `Native`. fn offset_to_native(offset: Self::Offset) -> Self::Native; @@ -98,6 +105,7 @@ unsafe impl MemorySize for Memory32 { type Offset = u32; type Native = i32; const ZERO: Self::Offset = 0; + const ONE: Self::Offset = 1; fn offset_to_native(offset: Self::Offset) -> Self::Native { offset as Self::Native } @@ -113,6 +121,7 @@ unsafe impl MemorySize for Memory64 { type Offset = u64; type Native = i64; const ZERO: Self::Offset = 0; + const ONE: Self::Offset = 1; fn offset_to_native(offset: Self::Offset) -> Self::Native { offset as Self::Native } @@ -120,3 +129,80 @@ unsafe impl MemorySize for Memory64 { native as Self::Offset } } + +/// Represents memory that is used by the WebAsssembly module +pub trait LinearMemory +where Self: std::fmt::Debug + Send +{ + /// Returns the type for this memory. + fn ty(&self) -> MemoryType; + + /// Returns the size of hte memory in pages + fn size(&self) -> Pages; + + /// Returns the memory style for this memory. + fn style(&self) -> MemoryStyle; + + /// Grow memory by the specified amount of wasm pages. + /// + /// Returns `None` if memory can't be grown by the specified amount + /// of wasm pages. + fn grow(&mut self, delta: Pages) -> Result; + + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. + fn vmmemory(&self) -> NonNull; + + /// Attempts to clone this memory (if its clonable) + fn try_clone(&self) -> Option>; +} + +/// The fields compiled code needs to access to utilize a WebAssembly linear +/// memory defined within the instance, namely the start address and the +/// size in bytes. +#[derive(Debug, Copy, Clone)] +#[repr(C)] +pub struct VMMemoryDefinition { + /// The start address which is always valid, even if the memory grows. + pub base: *mut u8, + + /// The current logical size of this linear memory in bytes. + pub current_length: usize, +} + +/// # Safety +/// This data is safe to share between threads because it's plain data that +/// is the user's responsibility to synchronize. +unsafe impl Send for VMMemoryDefinition {} +/// # Safety +/// This data is safe to share between threads because it's plain data that +/// is the user's responsibility to synchronize. And it's `Copy` so there's +/// really no difference between passing it by reference or by value as far as +/// correctness in a multi-threaded context is concerned. +unsafe impl Sync for VMMemoryDefinition {} + +#[cfg(test)] +mod test_vmmemory_definition { + use super::VMMemoryDefinition; + use crate::VMOffsets; + use memoffset::offset_of; + use std::mem::size_of; + use crate::ModuleInfo; + + #[test] + fn check_vmmemory_definition_offsets() { + let module = ModuleInfo::new(); + let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); + assert_eq!( + size_of::(), + usize::from(offsets.size_of_vmmemory_definition()) + ); + assert_eq!( + offset_of!(VMMemoryDefinition, base), + usize::from(offsets.vmmemory_definition_base()) + ); + assert_eq!( + offset_of!(VMMemoryDefinition, current_length), + usize::from(offsets.vmmemory_definition_current_length()) + ); + } +} diff --git a/lib/vm/src/instance/allocator.rs b/lib/vm/src/instance/allocator.rs index 29804c746..e00625f5e 100644 --- a/lib/vm/src/instance/allocator.rs +++ b/lib/vm/src/instance/allocator.rs @@ -1,11 +1,11 @@ use super::{Instance, InstanceHandle}; -use crate::vmcontext::{VMMemoryDefinition, VMTableDefinition}; +use crate::vmcontext::VMTableDefinition; use std::alloc::{self, Layout}; use std::convert::TryFrom; use std::mem; use std::ptr::{self, NonNull}; use wasmer_types::entity::EntityRef; -use wasmer_types::VMOffsets; +use wasmer_types::{VMOffsets, VMMemoryDefinition}; use wasmer_types::{LocalMemoryIndex, LocalTableIndex, ModuleInfo}; /// This is an intermediate type that manages the raw allocation and diff --git a/lib/vm/src/instance/mod.rs b/lib/vm/src/instance/mod.rs index bf0be8a29..ab11b2b55 100644 --- a/lib/vm/src/instance/mod.rs +++ b/lib/vm/src/instance/mod.rs @@ -10,14 +10,13 @@ mod allocator; use crate::export::VMExtern; use crate::imports::Imports; -use crate::memory::MemoryError; use crate::store::{InternalStoreHandle, StoreObjects}; use crate::table::TableElement; use crate::trap::{catch_traps, Trap, TrapCode}; use crate::vmcontext::{ VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionContext, - VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, - VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, + VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, + VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, memory_copy, memory_fill, }; use crate::{FunctionBodyPtr, MaybeInstanceOwned, TrapHandlerFn, VMFunctionBody}; use crate::{VMFuncRef, VMFunction, VMGlobal, VMMemory, VMTable}; @@ -37,7 +36,8 @@ use wasmer_types::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, use wasmer_types::{ DataIndex, DataInitializer, ElemIndex, ExportIndex, FunctionIndex, GlobalIndex, GlobalInit, LocalFunctionIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryIndex, - ModuleInfo, Pages, SignatureIndex, TableIndex, TableInitializer, VMOffsets, + ModuleInfo, Pages, SignatureIndex, TableIndex, TableInitializer, VMOffsets, LinearMemory, + MemoryError, VMMemoryDefinition }; /// A WebAssembly instance. @@ -632,7 +632,7 @@ impl Instance { let memory = self.memory(memory_index); // The following memory copy is not synchronized and is not atomic: - unsafe { memory.memory_copy(dst, src, len) } + unsafe { memory_copy(&memory, dst, src, len) } } /// Perform a `memory.copy` on an imported memory. @@ -646,7 +646,7 @@ impl Instance { let import = self.imported_memory(memory_index); let memory = unsafe { import.definition.as_ref() }; // The following memory copy is not synchronized and is not atomic: - unsafe { memory.memory_copy(dst, src, len) } + unsafe { memory_copy(memory, dst, src, len) } } /// Perform the `memory.fill` operation on a locally defined memory. @@ -663,7 +663,7 @@ impl Instance { ) -> Result<(), Trap> { let memory = self.memory(memory_index); // The following memory fill is not synchronized and is not atomic: - unsafe { memory.memory_fill(dst, val, len) } + unsafe { memory_fill(&memory, dst, val, len) } } /// Perform the `memory.fill` operation on an imported memory. @@ -681,7 +681,7 @@ impl Instance { let import = self.imported_memory(memory_index); let memory = unsafe { import.definition.as_ref() }; // The following memory fill is not synchronized and is not atomic: - unsafe { memory.memory_fill(dst, val, len) } + unsafe { memory_fill(memory, dst, val, len) } } /// Performs the `memory.init` operation. diff --git a/lib/vm/src/lib.rs b/lib/vm/src/lib.rs index 1b30f1e61..88b209a24 100644 --- a/lib/vm/src/lib.rs +++ b/lib/vm/src/lib.rs @@ -45,7 +45,8 @@ pub use crate::function_env::VMFunctionEnvironment; pub use crate::global::*; pub use crate::imports::Imports; pub use crate::instance::{InstanceAllocator, InstanceHandle}; -pub use crate::memory::{MemoryError, VMMemory}; +pub use crate::memory::VMMemory; +pub use wasmer_types::MemoryError; pub use crate::mmap::Mmap; pub use crate::probestack::PROBESTACK; pub use crate::sig_registry::SignatureRegistry; @@ -56,11 +57,11 @@ pub use crate::table::{TableElement, VMTable}; pub use crate::trap::*; pub use crate::vmcontext::{ VMCallerCheckedAnyfunc, VMContext, VMDynamicFunctionContext, VMFunctionContext, - VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, + VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, }; pub use wasmer_types::LibCall; -pub use wasmer_types::MemoryStyle; +pub use wasmer_types::{MemoryStyle, VMMemoryDefinition}; use wasmer_types::RawValue; pub use wasmer_types::TableStyle; pub use wasmer_types::{TargetSharedSignatureIndex, VMBuiltinFunctionIndex, VMOffsets}; diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index 787d5f390..265c5c259 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -5,88 +5,160 @@ //! //! `Memory` is to WebAssembly linear memories what `Table` is to WebAssembly tables. -use crate::vmcontext::VMMemoryDefinition; use crate::{mmap::Mmap, store::MaybeInstanceOwned}; use more_asserts::assert_ge; use std::cell::UnsafeCell; use std::convert::TryInto; use std::ptr::NonNull; -use thiserror::Error; -use wasmer_types::{Bytes, MemoryStyle, MemoryType, Pages}; - -/// Error type describing things that can go wrong when operating on Wasm Memories. -#[derive(Error, Debug, Clone, Eq, PartialEq, Hash)] -pub enum MemoryError { - /// Low level error with mmap. - #[error("Error when allocating memory: {0}")] - Region(String), - /// The operation would cause the size of the memory to exceed the maximum or would cause - /// an overflow leading to unindexable memory. - #[error("The memory could not grow: current size {} pages, requested increase: {} pages", current.0, attempted_delta.0)] - CouldNotGrow { - /// The current size in pages. - current: Pages, - /// The attempted amount to grow by in pages. - attempted_delta: Pages, - }, - /// The operation would cause the size of the memory size exceed the maximum. - #[error("The memory is invalid because {}", reason)] - InvalidMemory { - /// The reason why the provided memory is invalid. - reason: String, - }, - /// Caller asked for more minimum memory than we can give them. - #[error("The minimum requested ({} pages) memory is greater than the maximum allowed memory ({} pages)", min_requested.0, max_allowed.0)] - MinimumMemoryTooLarge { - /// The number of pages requested as the minimum amount of memory. - min_requested: Pages, - /// The maximum amount of memory we can allocate. - max_allowed: Pages, - }, - /// Caller asked for a maximum memory greater than we can give them. - #[error("The maximum requested memory ({} pages) is greater than the maximum allowed memory ({} pages)", max_requested.0, max_allowed.0)] - MaximumMemoryTooLarge { - /// The number of pages requested as the maximum amount of memory. - max_requested: Pages, - /// The number of pages requested as the maximum amount of memory. - max_allowed: Pages, - }, - /// A user defined error value, used for error cases not listed above. - #[error("A user-defined error occurred: {0}")] - Generic(String), -} - -/// A linear memory instance. -pub struct VMMemory { - // The underlying allocation. - mmap: WasmMmap, - - // The optional maximum size in wasm pages of this linear memory. - maximum: Option, - - /// The WebAssembly linear memory description. - memory: MemoryType, - - /// Our chosen implementation style. - style: MemoryStyle, - - // Size in bytes of extra guard pages after the end to optimize loads and stores with - // constant offsets. - offset_guard_size: usize, - - /// The owned memory definition used by the generated code - vm_memory_definition: MaybeInstanceOwned, -} +use wasmer_types::{Bytes, MemoryStyle, MemoryType, Pages, MemoryError, LinearMemory, VMMemoryDefinition}; +// The memory mapped area #[derive(Debug)] struct WasmMmap { // Our OS allocation of mmap'd memory. alloc: Mmap, // The current logical size in wasm pages of this linear memory. size: Pages, + /// The owned memory definition used by the generated code + vm_memory_definition: MaybeInstanceOwned, } -impl VMMemory { +impl WasmMmap +{ + fn get_vm_memory_definition(&self) -> NonNull { + self.vm_memory_definition.as_ptr() + } + + fn size(&self) -> Pages { + unsafe { + let md_ptr = self.get_vm_memory_definition(); + let md = md_ptr.as_ref(); + Bytes::from(md.current_length).try_into().unwrap() + } + } + + fn grow(&mut self, delta: Pages, conf: VMMemoryConfig) -> Result { + // Optimization of memory.grow 0 calls. + if delta.0 == 0 { + return Ok(self.size); + } + + let new_pages = self + .size + .checked_add(delta) + .ok_or(MemoryError::CouldNotGrow { + current: self.size, + attempted_delta: delta, + })?; + let prev_pages = self.size; + + if let Some(maximum) = conf.maximum { + if new_pages > maximum { + return Err(MemoryError::CouldNotGrow { + current: self.size, + attempted_delta: delta, + }); + } + } + + // Wasm linear memories are never allowed to grow beyond what is + // indexable. If the memory has no maximum, enforce the greatest + // limit here. + if new_pages >= Pages::max_value() { + // Linear memory size would exceed the index range. + return Err(MemoryError::CouldNotGrow { + current: self.size, + attempted_delta: delta, + }); + } + + let delta_bytes = delta.bytes().0; + let prev_bytes = prev_pages.bytes().0; + let new_bytes = new_pages.bytes().0; + + if new_bytes > self.alloc.len() - conf.offset_guard_size { + // If the new size is within the declared maximum, but needs more memory than we + // have on hand, it's a dynamic heap and it can move. + let guard_bytes = conf.offset_guard_size; + let request_bytes = + new_bytes + .checked_add(guard_bytes) + .ok_or_else(|| MemoryError::CouldNotGrow { + current: new_pages, + attempted_delta: Bytes(guard_bytes).try_into().unwrap(), + })?; + + let mut new_mmap = + Mmap::accessible_reserved(new_bytes, request_bytes).map_err(MemoryError::Region)?; + + let copy_len = self.alloc.len() - conf.offset_guard_size; + new_mmap.as_mut_slice()[..copy_len] + .copy_from_slice(&self.alloc.as_slice()[..copy_len]); + + self.alloc = new_mmap; + } else if delta_bytes > 0 { + // Make the newly allocated pages accessible. + self + .alloc + .make_accessible(prev_bytes, delta_bytes) + .map_err(MemoryError::Region)?; + } + + self.size = new_pages; + + // update memory definition + unsafe { + let mut md_ptr = self.vm_memory_definition.as_ptr(); + let md = md_ptr.as_mut(); + md.current_length = new_pages.bytes().0; + md.base = self.alloc.as_mut_ptr() as _; + } + + Ok(prev_pages) + } +} + +/// A linear memory instance. +#[derive(Debug, Clone)] +struct VMMemoryConfig { + // The optional maximum size in wasm pages of this linear memory. + maximum: Option, + /// The WebAssembly linear memory description. + memory: MemoryType, + /// Our chosen implementation style. + style: MemoryStyle, + // Size in bytes of extra guard pages after the end to optimize loads and stores with + // constant offsets. + offset_guard_size: usize, +} + +impl VMMemoryConfig +{ + fn ty(&self, minimum: Pages) -> MemoryType { + let mut out = self.memory; + out.minimum = minimum; + + out + } + + fn style(&self) -> MemoryStyle { + self.style + } +} + +/// A linear memory instance. +#[derive(Debug)] +pub struct VMOwnedMemory { + // The underlying allocation. + mmap: WasmMmap, + // Configuration of this memory + config: VMMemoryConfig, +} + +unsafe impl Send for VMOwnedMemory { } +unsafe impl Sync for VMOwnedMemory { } + +impl VMOwnedMemory { /// Create a new linear memory instance with specified minimum and maximum number of wasm pages. /// /// This creates a `Memory` with owned metadata: this can be used to create a memory @@ -154,18 +226,11 @@ impl VMMemory { let mapped_pages = memory.minimum; let mapped_bytes = mapped_pages.bytes(); - let mut mmap = WasmMmap { - alloc: Mmap::accessible_reserved(mapped_bytes.0, request_bytes) - .map_err(MemoryError::Region)?, - size: memory.minimum, - }; - - let base_ptr = mmap.alloc.as_mut_ptr(); + let mut alloc = Mmap::accessible_reserved(mapped_bytes.0, request_bytes) + .map_err(MemoryError::Region)?; + let base_ptr = alloc.as_mut_ptr(); let mem_length = memory.minimum.bytes().0; - Ok(Self { - mmap, - maximum: memory.maximum, - offset_guard_size: offset_guard_bytes, + let mmap = WasmMmap { vm_memory_definition: if let Some(mem_loc) = vm_memory_location { { let mut ptr = mem_loc; @@ -180,127 +245,155 @@ impl VMMemory { current_length: mem_length, }))) }, - memory: *memory, - style: style.clone(), + alloc, + size: memory.minimum, + }; + + Ok(Self { + mmap: mmap, + config: VMMemoryConfig { + maximum: memory.maximum, + offset_guard_size: offset_guard_bytes, + memory: *memory, + style: style.clone(), + } }) } +} - /// Get the `VMMemoryDefinition`. - fn get_vm_memory_definition(&self) -> NonNull { - self.vm_memory_definition.as_ptr() +impl LinearMemory +for VMOwnedMemory +{ + /// Returns the type for this memory. + fn ty(&self) -> MemoryType { + let minimum = self.mmap.size(); + self.config.ty(minimum) } - /// Returns the type for this memory. - pub fn ty(&self) -> MemoryType { - let minimum = self.size(); - let mut out = self.memory; - out.minimum = minimum; - - out + /// Returns the size of hte memory in pages + fn size(&self) -> Pages { + self.mmap.size() } /// Returns the memory style for this memory. - pub fn style(&self) -> &MemoryStyle { - &self.style - } - - /// Returns the number of allocated wasm pages. - pub fn size(&self) -> Pages { - // TODO: investigate this function for race conditions - unsafe { - let md_ptr = self.get_vm_memory_definition(); - let md = md_ptr.as_ref(); - Bytes::from(md.current_length).try_into().unwrap() - } + fn style(&self) -> MemoryStyle { + self.config.style() } /// Grow memory by the specified amount of wasm pages. /// /// Returns `None` if memory can't be grown by the specified amount /// of wasm pages. - pub fn grow(&mut self, delta: Pages) -> Result { - // Optimization of memory.grow 0 calls. - if delta.0 == 0 { - return Ok(self.mmap.size); - } - - let new_pages = self - .mmap - .size - .checked_add(delta) - .ok_or(MemoryError::CouldNotGrow { - current: self.mmap.size, - attempted_delta: delta, - })?; - let prev_pages = self.mmap.size; - - if let Some(maximum) = self.maximum { - if new_pages > maximum { - return Err(MemoryError::CouldNotGrow { - current: self.mmap.size, - attempted_delta: delta, - }); - } - } - - // Wasm linear memories are never allowed to grow beyond what is - // indexable. If the memory has no maximum, enforce the greatest - // limit here. - if new_pages >= Pages::max_value() { - // Linear memory size would exceed the index range. - return Err(MemoryError::CouldNotGrow { - current: self.mmap.size, - attempted_delta: delta, - }); - } - - let delta_bytes = delta.bytes().0; - let prev_bytes = prev_pages.bytes().0; - let new_bytes = new_pages.bytes().0; - - if new_bytes > self.mmap.alloc.len() - self.offset_guard_size { - // If the new size is within the declared maximum, but needs more memory than we - // have on hand, it's a dynamic heap and it can move. - let guard_bytes = self.offset_guard_size; - let request_bytes = - new_bytes - .checked_add(guard_bytes) - .ok_or_else(|| MemoryError::CouldNotGrow { - current: new_pages, - attempted_delta: Bytes(guard_bytes).try_into().unwrap(), - })?; - - let mut new_mmap = - Mmap::accessible_reserved(new_bytes, request_bytes).map_err(MemoryError::Region)?; - - let copy_len = self.mmap.alloc.len() - self.offset_guard_size; - new_mmap.as_mut_slice()[..copy_len] - .copy_from_slice(&self.mmap.alloc.as_slice()[..copy_len]); - - self.mmap.alloc = new_mmap; - } else if delta_bytes > 0 { - // Make the newly allocated pages accessible. - self.mmap - .alloc - .make_accessible(prev_bytes, delta_bytes) - .map_err(MemoryError::Region)?; - } - - self.mmap.size = new_pages; - - // update memory definition - unsafe { - let mut md_ptr = self.get_vm_memory_definition(); - let md = md_ptr.as_mut(); - md.current_length = new_pages.bytes().0; - md.base = self.mmap.alloc.as_mut_ptr() as _; - } - - Ok(prev_pages) + fn grow(&mut self, delta: Pages) -> Result { + self.mmap.grow(delta, self.config.clone()) } /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. - pub fn vmmemory(&self) -> NonNull { - self.get_vm_memory_definition() + fn vmmemory(&self) -> NonNull { + self.mmap.vm_memory_definition.as_ptr() + } + + /// Owned memory can not be cloned (this will always return None) + fn try_clone(&self) -> Option> { + None + } +} + +impl Into +for VMOwnedMemory +{ + fn into(self) -> VMMemory { + VMMemory(Box::new(self)) + } +} + +/// Represents linear memory that can be either owned or shared +#[derive(Debug)] +pub struct VMMemory(Box); + +impl Into +for Box +{ + fn into(self) -> VMMemory { + VMMemory(self) + } +} + +impl LinearMemory +for VMMemory +{ + /// Returns the type for this memory. + fn ty(&self) -> MemoryType { + self.0.ty() + } + + /// Returns the size of hte memory in pages + fn size(&self) -> Pages { + self.0.size() + } + + /// Grow memory by the specified amount of wasm pages. + /// + /// Returns `None` if memory can't be grown by the specified amount + /// of wasm pages. + fn grow(&mut self, delta: Pages) -> Result { + self.0.grow(delta) + } + + /// Returns the memory style for this memory. + fn style(&self) -> MemoryStyle { + self.0.style() + } + + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. + fn vmmemory(&self) -> NonNull { + self.0.vmmemory() + } + + /// Attempts to clone this memory (if its clonable) + fn try_clone(&self) -> Option> { + self.0.try_clone() + } +} + +impl VMMemory +{ + /// Creates a new linear memory instance of the correct type with specified + /// minimum and maximum number of wasm pages. + /// + /// This creates a `Memory` with owned metadata: this can be used to create a memory + /// that will be imported into Wasm modules. + pub fn new(memory: &MemoryType, style: &MemoryStyle) -> Result { + Ok( + Self(Box::new(VMOwnedMemory::new(memory, style)?)) + ) + } + + /// Create a new linear memory instance with specified minimum and maximum number of wasm pages. + /// + /// This creates a `Memory` with metadata owned by a VM, pointed to by + /// `vm_memory_location`: this can be used to create a local memory. + /// + /// # Safety + /// - `vm_memory_location` must point to a valid location in VM memory. + pub unsafe fn from_definition( + memory: &MemoryType, + style: &MemoryStyle, + vm_memory_location: NonNull, + ) -> Result { + Ok( + Self(Box::new(VMOwnedMemory::from_definition(memory, style, vm_memory_location)?)) + ) + } + + /// Creates VMMemory from a custom implementation - the following into implementations + /// are natively supported + /// - VMOwnedMemory -> VMMemory + /// - VMSharedMemory -> VMMemory + /// - Box -> VMMemory + pub fn from_custom(memory: IntoVMMemory) -> VMMemory + where IntoVMMemory: Into + { + memory.into() } } diff --git a/lib/vm/src/store.rs b/lib/vm/src/store.rs index 4faee579a..25afdb7c1 100644 --- a/lib/vm/src/store.rs +++ b/lib/vm/src/store.rs @@ -266,3 +266,23 @@ impl MaybeInstanceOwned { } } } + +impl std::fmt::Debug +for MaybeInstanceOwned +where T: std::fmt::Debug +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MaybeInstanceOwned::Host(p) => { + write!(f, "host(")?; + p.as_ref().fmt(f)?; + write!(f, ")") + }, + MaybeInstanceOwned::Instance(p) => { + write!(f, "instance(")?; + unsafe { p.as_ref().fmt(f)? }; + write!(f, ")") + } + } + } +} \ No newline at end of file diff --git a/lib/vm/src/vmcontext.rs b/lib/vm/src/vmcontext.rs index bb1cb6cc9..6940f43a2 100644 --- a/lib/vm/src/vmcontext.rs +++ b/lib/vm/src/vmcontext.rs @@ -15,7 +15,7 @@ use crate::{VMBuiltinFunctionIndex, VMFunction}; use std::convert::TryFrom; use std::ptr::{self, NonNull}; use std::u32; -use wasmer_types::RawValue; +use wasmer_types::{RawValue, VMMemoryDefinition}; /// Union representing the first parameter passed when calling a function. /// @@ -303,120 +303,67 @@ mod test_vmglobal_import { } } -/// The fields compiled code needs to access to utilize a WebAssembly linear -/// memory defined within the instance, namely the start address and the -/// size in bytes. -#[derive(Debug, Copy, Clone)] -#[repr(C)] -pub struct VMMemoryDefinition { - /// The start address which is always valid, even if the memory grows. - pub base: *mut u8, +/// Do an unsynchronized, non-atomic `memory.copy` for the memory. +/// +/// # Errors +/// +/// Returns a `Trap` error when the source or destination ranges are out of +/// bounds. +/// +/// # Safety +/// The memory is not copied atomically and is not synchronized: it's the +/// caller's responsibility to synchronize. +pub(crate) unsafe fn memory_copy(mem: &VMMemoryDefinition, dst: u32, src: u32, len: u32) -> Result<(), Trap> { + // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy + if src + .checked_add(len) + .map_or(true, |n| usize::try_from(n).unwrap() > mem.current_length) + || dst + .checked_add(len) + .map_or(true, |m| usize::try_from(m).unwrap() > mem.current_length) + { + return Err(Trap::lib(TrapCode::HeapAccessOutOfBounds)); + } - /// The current logical size of this linear memory in bytes. - pub current_length: usize, + let dst = usize::try_from(dst).unwrap(); + let src = usize::try_from(src).unwrap(); + + // Bounds and casts are checked above, by this point we know that + // everything is safe. + let dst = mem.base.add(dst); + let src = mem.base.add(src); + ptr::copy(src, dst, len as usize); + + Ok(()) } +/// Perform the `memory.fill` operation for the memory in an unsynchronized, +/// non-atomic way. +/// +/// # Errors +/// +/// Returns a `Trap` error if the memory range is out of bounds. +/// /// # Safety -/// This data is safe to share between threads because it's plain data that -/// is the user's responsibility to synchronize. -unsafe impl Send for VMMemoryDefinition {} -/// # Safety -/// This data is safe to share between threads because it's plain data that -/// is the user's responsibility to synchronize. And it's `Copy` so there's -/// really no difference between passing it by reference or by value as far as -/// correctness in a multi-threaded context is concerned. -unsafe impl Sync for VMMemoryDefinition {} - -impl VMMemoryDefinition { - /// Do an unsynchronized, non-atomic `memory.copy` for the memory. - /// - /// # Errors - /// - /// Returns a `Trap` error when the source or destination ranges are out of - /// bounds. - /// - /// # Safety - /// The memory is not copied atomically and is not synchronized: it's the - /// caller's responsibility to synchronize. - pub(crate) unsafe fn memory_copy(&self, dst: u32, src: u32, len: u32) -> Result<(), Trap> { - // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy - if src - .checked_add(len) - .map_or(true, |n| usize::try_from(n).unwrap() > self.current_length) - || dst - .checked_add(len) - .map_or(true, |m| usize::try_from(m).unwrap() > self.current_length) - { - return Err(Trap::lib(TrapCode::HeapAccessOutOfBounds)); - } - - let dst = usize::try_from(dst).unwrap(); - let src = usize::try_from(src).unwrap(); - - // Bounds and casts are checked above, by this point we know that - // everything is safe. - let dst = self.base.add(dst); - let src = self.base.add(src); - ptr::copy(src, dst, len as usize); - - Ok(()) +/// The memory is not filled atomically and is not synchronized: it's the +/// caller's responsibility to synchronize. +pub(crate) unsafe fn memory_fill(mem: &VMMemoryDefinition, dst: u32, val: u32, len: u32) -> Result<(), Trap> { + if dst + .checked_add(len) + .map_or(true, |m| usize::try_from(m).unwrap() > mem.current_length) + { + return Err(Trap::lib(TrapCode::HeapAccessOutOfBounds)); } - /// Perform the `memory.fill` operation for the memory in an unsynchronized, - /// non-atomic way. - /// - /// # Errors - /// - /// Returns a `Trap` error if the memory range is out of bounds. - /// - /// # Safety - /// The memory is not filled atomically and is not synchronized: it's the - /// caller's responsibility to synchronize. - pub(crate) unsafe fn memory_fill(&self, dst: u32, val: u32, len: u32) -> Result<(), Trap> { - if dst - .checked_add(len) - .map_or(true, |m| usize::try_from(m).unwrap() > self.current_length) - { - return Err(Trap::lib(TrapCode::HeapAccessOutOfBounds)); - } + let dst = isize::try_from(dst).unwrap(); + let val = val as u8; - let dst = isize::try_from(dst).unwrap(); - let val = val as u8; + // Bounds and casts are checked above, by this point we know that + // everything is safe. + let dst = mem.base.offset(dst); + ptr::write_bytes(dst, val, len as usize); - // Bounds and casts are checked above, by this point we know that - // everything is safe. - let dst = self.base.offset(dst); - ptr::write_bytes(dst, val, len as usize); - - Ok(()) - } -} - -#[cfg(test)] -mod test_vmmemory_definition { - use super::VMMemoryDefinition; - use crate::VMOffsets; - use memoffset::offset_of; - use std::mem::size_of; - use wasmer_types::ModuleInfo; - - #[test] - fn check_vmmemory_definition_offsets() { - let module = ModuleInfo::new(); - let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); - assert_eq!( - size_of::(), - usize::from(offsets.size_of_vmmemory_definition()) - ); - assert_eq!( - offset_of!(VMMemoryDefinition, base), - usize::from(offsets.vmmemory_definition_base()) - ); - assert_eq!( - offset_of!(VMMemoryDefinition, current_length), - usize::from(offsets.vmmemory_definition_current_length()) - ); - } + Ok(()) } /// The fields compiled code needs to access to utilize a WebAssembly table From f599df0240ae60cd3045fbdf594f1af80c928308 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Thu, 18 Aug 2022 16:58:04 +0200 Subject: [PATCH 02/40] Fix linter --- lib/api/src/js/externals/memory.rs | 6 +- lib/api/src/js/imports.rs | 12 ++-- lib/api/src/js/mod.rs | 2 +- lib/api/src/sys/externals/memory.rs | 7 +-- lib/api/src/sys/externals/memory_view.rs | 2 +- lib/api/src/sys/mod.rs | 2 +- .../src/translator/code_translator.rs | 7 ++- lib/compiler/src/engine/resolver.rs | 3 +- lib/compiler/src/translator/environ.rs | 2 +- lib/types/src/lib.rs | 8 +-- lib/types/src/memory.rs | 9 +-- lib/vm/src/instance/allocator.rs | 2 +- lib/vm/src/instance/mod.rs | 12 ++-- lib/vm/src/lib.rs | 8 +-- lib/vm/src/memory.rs | 58 ++++++++----------- lib/vm/src/store.rs | 10 ++-- lib/vm/src/vmcontext.rs | 14 ++++- 17 files changed, 82 insertions(+), 82 deletions(-) diff --git a/lib/api/src/js/externals/memory.rs b/lib/api/src/js/externals/memory.rs index 2b66d8ed7..599b261e2 100644 --- a/lib/api/src/js/externals/memory.rs +++ b/lib/api/src/js/externals/memory.rs @@ -101,7 +101,11 @@ impl Memory { } /// Creates a new host `Memory` from provided JavaScript memory. - pub fn new_raw(store: &mut impl AsStoreMut, js_memory: js_sys::WebAssembly::Memory, ty: MemoryType) -> Result { + pub fn new_raw( + store: &mut impl AsStoreMut, + js_memory: js_sys::WebAssembly::Memory, + ty: MemoryType, + ) -> Result { let vm_memory = VMMemory::new(js_memory, ty); Ok(Self::from_vm_export(store, vm_memory)) } diff --git a/lib/api/src/js/imports.rs b/lib/api/src/js/imports.rs index d3b2c42ef..63a199abb 100644 --- a/lib/api/src/js/imports.rs +++ b/lib/api/src/js/imports.rs @@ -182,27 +182,23 @@ impl Imports { } pub struct ImportsIterator<'a> { - iter: std::collections::hash_map::Iter<'a, (String, String), Extern> + iter: std::collections::hash_map::Iter<'a, (String, String), Extern>, } -impl<'a> ImportsIterator<'a> -{ +impl<'a> ImportsIterator<'a> { fn new(imports: &'a Imports) -> Self { let iter = imports.map.iter(); Self { iter } } } -impl<'a> Iterator -for ImportsIterator<'a> { +impl<'a> Iterator for ImportsIterator<'a> { type Item = (&'a str, &'a str, &'a Extern); fn next(&mut self) -> Option { self.iter .next() - .map(|(k, v)| { - (k.0.as_str(), k.1.as_str(), v) - }) + .map(|(k, v)| (k.0.as_str(), k.1.as_str(), v)) } } diff --git a/lib/api/src/js/mod.rs b/lib/api/src/js/mod.rs index a88509086..1e660a530 100644 --- a/lib/api/src/js/mod.rs +++ b/lib/api/src/js/mod.rs @@ -75,7 +75,7 @@ pub use crate::js::value::Value as Val; pub mod vm { //! The `vm` module re-exports wasmer-vm types. - + pub use crate::js::export::VMMemory; } diff --git a/lib/api/src/sys/externals/memory.rs b/lib/api/src/sys/externals/memory.rs index 64ab9a136..459863155 100644 --- a/lib/api/src/sys/externals/memory.rs +++ b/lib/api/src/sys/externals/memory.rs @@ -10,7 +10,7 @@ use std::mem::MaybeUninit; use std::slice; #[cfg(feature = "tracing")] use tracing::warn; -use wasmer_types::{Pages, LinearMemory}; +use wasmer_types::{LinearMemory, Pages}; use wasmer_vm::{InternalStoreHandle, MemoryError, StoreHandle, VMExtern, VMMemory}; use super::MemoryView; @@ -63,7 +63,7 @@ impl Memory { /// Create a memory object from an existing memory and attaches it to the store pub fn new_from_existing(new_store: &mut impl AsStoreMut, memory: VMMemory) -> Self { Self { - handle: StoreHandle::new(new_store.objects_mut(), memory) + handle: StoreHandle::new(new_store.objects_mut(), memory), } } @@ -152,8 +152,7 @@ impl Memory { /// Attempts to clone this memory (if its clonable) pub fn try_clone(&self, store: &impl AsStoreRef) -> Option { let mem = self.handle.get(store.as_store_ref().objects()); - mem.try_clone() - .map(|mem| mem.into()) + mem.try_clone().map(|mem| mem.into()) } pub(crate) fn to_vm_extern(&self) -> VMExtern { diff --git a/lib/api/src/sys/externals/memory_view.rs b/lib/api/src/sys/externals/memory_view.rs index a638acb6b..5822c4113 100644 --- a/lib/api/src/sys/externals/memory_view.rs +++ b/lib/api/src/sys/externals/memory_view.rs @@ -4,7 +4,7 @@ use std::convert::TryInto; use std::marker::PhantomData; use std::mem::MaybeUninit; use std::slice; -use wasmer_types::{Pages, LinearMemory}; +use wasmer_types::{LinearMemory, Pages}; use super::memory::MemoryBuffer; use super::Memory; diff --git a/lib/api/src/sys/mod.rs b/lib/api/src/sys/mod.rs index b6f9088c3..1c272a2a6 100644 --- a/lib/api/src/sys/mod.rs +++ b/lib/api/src/sys/mod.rs @@ -58,7 +58,7 @@ pub mod vm { pub use wasmer_vm::{ MemoryError, MemoryStyle, TableStyle, VMExtern, VMMemory, VMMemoryDefinition, VMTable, - VMTableDefinition, + VMTableDefinition, }; } diff --git a/lib/compiler-cranelift/src/translator/code_translator.rs b/lib/compiler-cranelift/src/translator/code_translator.rs index 60da48347..c750d6c32 100644 --- a/lib/compiler-cranelift/src/translator/code_translator.rs +++ b/lib/compiler-cranelift/src/translator/code_translator.rs @@ -1071,7 +1071,8 @@ pub fn translate_operator( expected, timeout, )?; - state.push1(res); } + state.push1(res); + } Operator::MemoryAtomicNotify { memarg } => { let heap_index = MemoryIndex::from_u32(memarg.memory); let heap = state.get_heap(builder.func, memarg.memory, environ)?; @@ -1079,8 +1080,8 @@ pub fn translate_operator( let addr = state.pop1(); // 32 (fixed) let addr = fold_atomic_mem_addr(addr, memarg, I32, builder); let res = - environ.translate_atomic_notify(builder.cursor(), heap_index, heap, addr, count)?; - state.push1(res); + environ.translate_atomic_notify(builder.cursor(), heap_index, heap, addr, count)?; + state.push1(res); } Operator::I32AtomicLoad { memarg } => { translate_atomic_load(I32, I32, memarg, builder, state, environ)? diff --git a/lib/compiler/src/engine/resolver.rs b/lib/compiler/src/engine/resolver.rs index 94d40a502..2b218bde3 100644 --- a/lib/compiler/src/engine/resolver.rs +++ b/lib/compiler/src/engine/resolver.rs @@ -4,7 +4,8 @@ use crate::LinkError; use more_asserts::assert_ge; use wasmer_types::entity::{BoxedSlice, EntityRef, PrimaryMap}; use wasmer_types::{ - ExternType, FunctionIndex, ImportError, ImportIndex, MemoryIndex, ModuleInfo, TableIndex, LinearMemory, + ExternType, FunctionIndex, ImportError, ImportIndex, LinearMemory, MemoryIndex, ModuleInfo, + TableIndex, }; use wasmer_vm::{ diff --git a/lib/compiler/src/translator/environ.rs b/lib/compiler/src/translator/environ.rs index cd00a7bc0..e172d92b0 100644 --- a/lib/compiler/src/translator/environ.rs +++ b/lib/compiler/src/translator/environ.rs @@ -1,8 +1,8 @@ // This file contains code from external sources. // Attributions: https://github.com/wasmerio/wasmer/blob/master/ATTRIBUTIONS.md use super::state::ModuleTranslationState; -use crate::lib::std::string::ToString; use crate::lib::std::borrow::ToOwned; +use crate::lib::std::string::ToString; use crate::lib::std::{boxed::Box, string::String, vec::Vec}; use crate::translate_module; use crate::wasmparser::{Operator, Range, Type}; diff --git a/lib/types/src/lib.rs b/lib/types/src/lib.rs index 3f71079c9..5a2e57ef0 100644 --- a/lib/types/src/lib.rs +++ b/lib/types/src/lib.rs @@ -76,8 +76,8 @@ pub use crate::compilation::target::{ }; pub use crate::serialize::{MetadataHeader, SerializableCompilation, SerializableModule}; pub use error::{ - CompileError, DeserializeError, ImportError, MiddlewareError, ParseCpuFeatureError, - PreInstantiationError, SerializeError, WasmError, WasmResult, MemoryError, + CompileError, DeserializeError, ImportError, MemoryError, MiddlewareError, + ParseCpuFeatureError, PreInstantiationError, SerializeError, WasmError, WasmResult, }; /// The entity module, with common helpers for Rust structures @@ -103,9 +103,7 @@ pub use types::{ pub use value::{RawValue, ValueType}; pub use crate::libcalls::LibCall; -pub use crate::memory::{ - MemoryStyle, LinearMemory, VMMemoryDefinition -}; +pub use crate::memory::{LinearMemory, MemoryStyle, VMMemoryDefinition}; pub use crate::table::TableStyle; pub use crate::trapcode::TrapCode; pub use crate::vmoffsets::{TargetSharedSignatureIndex, VMBuiltinFunctionIndex, VMOffsets}; diff --git a/lib/types/src/memory.rs b/lib/types/src/memory.rs index 6ebac15e7..a06b3379c 100644 --- a/lib/types/src/memory.rs +++ b/lib/types/src/memory.rs @@ -1,14 +1,14 @@ use crate::{Pages, ValueType}; +use core::ptr::NonNull; use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; -use core::ptr::NonNull; use std::convert::{TryFrom, TryInto}; use std::iter::Sum; use std::ops::{Add, AddAssign}; -use super::MemoryType; use super::MemoryError; +use super::MemoryType; /// Implementation styles for WebAssembly linear memory. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, RkyvSerialize, RkyvDeserialize, Archive)] @@ -132,7 +132,8 @@ unsafe impl MemorySize for Memory64 { /// Represents memory that is used by the WebAsssembly module pub trait LinearMemory -where Self: std::fmt::Debug + Send +where + Self: std::fmt::Debug + Send, { /// Returns the type for this memory. fn ty(&self) -> MemoryType; @@ -183,10 +184,10 @@ unsafe impl Sync for VMMemoryDefinition {} #[cfg(test)] mod test_vmmemory_definition { use super::VMMemoryDefinition; + use crate::ModuleInfo; use crate::VMOffsets; use memoffset::offset_of; use std::mem::size_of; - use crate::ModuleInfo; #[test] fn check_vmmemory_definition_offsets() { diff --git a/lib/vm/src/instance/allocator.rs b/lib/vm/src/instance/allocator.rs index e00625f5e..c6ca74193 100644 --- a/lib/vm/src/instance/allocator.rs +++ b/lib/vm/src/instance/allocator.rs @@ -5,8 +5,8 @@ use std::convert::TryFrom; use std::mem; use std::ptr::{self, NonNull}; use wasmer_types::entity::EntityRef; -use wasmer_types::{VMOffsets, VMMemoryDefinition}; use wasmer_types::{LocalMemoryIndex, LocalTableIndex, ModuleInfo}; +use wasmer_types::{VMMemoryDefinition, VMOffsets}; /// This is an intermediate type that manages the raw allocation and /// metadata when creating an [`Instance`]. diff --git a/lib/vm/src/instance/mod.rs b/lib/vm/src/instance/mod.rs index ab11b2b55..98e2423d1 100644 --- a/lib/vm/src/instance/mod.rs +++ b/lib/vm/src/instance/mod.rs @@ -14,9 +14,9 @@ use crate::store::{InternalStoreHandle, StoreObjects}; use crate::table::TableElement; use crate::trap::{catch_traps, Trap, TrapCode}; use crate::vmcontext::{ - VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionContext, - VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, - VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, memory_copy, memory_fill, + memory_copy, memory_fill, VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, + VMFunctionContext, VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, + VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, }; use crate::{FunctionBodyPtr, MaybeInstanceOwned, TrapHandlerFn, VMFunctionBody}; use crate::{VMFuncRef, VMFunction, VMGlobal, VMMemory, VMTable}; @@ -35,9 +35,9 @@ use std::sync::Arc; use wasmer_types::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap}; use wasmer_types::{ DataIndex, DataInitializer, ElemIndex, ExportIndex, FunctionIndex, GlobalIndex, GlobalInit, - LocalFunctionIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryIndex, - ModuleInfo, Pages, SignatureIndex, TableIndex, TableInitializer, VMOffsets, LinearMemory, - MemoryError, VMMemoryDefinition + LinearMemory, LocalFunctionIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, + MemoryError, MemoryIndex, ModuleInfo, Pages, SignatureIndex, TableIndex, TableInitializer, + VMMemoryDefinition, VMOffsets, }; /// A WebAssembly instance. diff --git a/lib/vm/src/lib.rs b/lib/vm/src/lib.rs index 88b209a24..05ae87d4a 100644 --- a/lib/vm/src/lib.rs +++ b/lib/vm/src/lib.rs @@ -46,7 +46,6 @@ pub use crate::global::*; pub use crate::imports::Imports; pub use crate::instance::{InstanceAllocator, InstanceHandle}; pub use crate::memory::VMMemory; -pub use wasmer_types::MemoryError; pub use crate::mmap::Mmap; pub use crate::probestack::PROBESTACK; pub use crate::sig_registry::SignatureRegistry; @@ -57,13 +56,14 @@ pub use crate::table::{TableElement, VMTable}; pub use crate::trap::*; pub use crate::vmcontext::{ VMCallerCheckedAnyfunc, VMContext, VMDynamicFunctionContext, VMFunctionContext, - VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, - VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, + VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, VMMemoryImport, + VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, }; pub use wasmer_types::LibCall; -pub use wasmer_types::{MemoryStyle, VMMemoryDefinition}; +pub use wasmer_types::MemoryError; use wasmer_types::RawValue; pub use wasmer_types::TableStyle; +pub use wasmer_types::{MemoryStyle, VMMemoryDefinition}; pub use wasmer_types::{TargetSharedSignatureIndex, VMBuiltinFunctionIndex, VMOffsets}; #[deprecated( diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index 265c5c259..5ed4c9cf7 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -10,7 +10,9 @@ use more_asserts::assert_ge; use std::cell::UnsafeCell; use std::convert::TryInto; use std::ptr::NonNull; -use wasmer_types::{Bytes, MemoryStyle, MemoryType, Pages, MemoryError, LinearMemory, VMMemoryDefinition}; +use wasmer_types::{ + Bytes, LinearMemory, MemoryError, MemoryStyle, MemoryType, Pages, VMMemoryDefinition, +}; // The memory mapped area #[derive(Debug)] @@ -23,8 +25,7 @@ struct WasmMmap { vm_memory_definition: MaybeInstanceOwned, } -impl WasmMmap -{ +impl WasmMmap { fn get_vm_memory_definition(&self) -> NonNull { self.vm_memory_definition.as_ptr() } @@ -92,14 +93,12 @@ impl WasmMmap Mmap::accessible_reserved(new_bytes, request_bytes).map_err(MemoryError::Region)?; let copy_len = self.alloc.len() - conf.offset_guard_size; - new_mmap.as_mut_slice()[..copy_len] - .copy_from_slice(&self.alloc.as_slice()[..copy_len]); + new_mmap.as_mut_slice()[..copy_len].copy_from_slice(&self.alloc.as_slice()[..copy_len]); self.alloc = new_mmap; } else if delta_bytes > 0 { // Make the newly allocated pages accessible. - self - .alloc + self.alloc .make_accessible(prev_bytes, delta_bytes) .map_err(MemoryError::Region)?; } @@ -132,8 +131,7 @@ struct VMMemoryConfig { offset_guard_size: usize, } -impl VMMemoryConfig -{ +impl VMMemoryConfig { fn ty(&self, minimum: Pages) -> MemoryType { let mut out = self.memory; out.minimum = minimum; @@ -155,8 +153,8 @@ pub struct VMOwnedMemory { config: VMMemoryConfig, } -unsafe impl Send for VMOwnedMemory { } -unsafe impl Sync for VMOwnedMemory { } +unsafe impl Send for VMOwnedMemory {} +unsafe impl Sync for VMOwnedMemory {} impl VMOwnedMemory { /// Create a new linear memory instance with specified minimum and maximum number of wasm pages. @@ -248,7 +246,7 @@ impl VMOwnedMemory { alloc, size: memory.minimum, }; - + Ok(Self { mmap: mmap, config: VMMemoryConfig { @@ -256,14 +254,12 @@ impl VMOwnedMemory { offset_guard_size: offset_guard_bytes, memory: *memory, style: style.clone(), - } + }, }) } } -impl LinearMemory -for VMOwnedMemory -{ +impl LinearMemory for VMOwnedMemory { /// Returns the type for this memory. fn ty(&self) -> MemoryType { let minimum = self.mmap.size(); @@ -299,9 +295,7 @@ for VMOwnedMemory } } -impl Into -for VMOwnedMemory -{ +impl Into for VMOwnedMemory { fn into(self) -> VMMemory { VMMemory(Box::new(self)) } @@ -311,17 +305,13 @@ for VMOwnedMemory #[derive(Debug)] pub struct VMMemory(Box); -impl Into -for Box -{ +impl Into for Box { fn into(self) -> VMMemory { VMMemory(self) } } -impl LinearMemory -for VMMemory -{ +impl LinearMemory for VMMemory { /// Returns the type for this memory. fn ty(&self) -> MemoryType { self.0.ty() @@ -356,17 +346,14 @@ for VMMemory } } -impl VMMemory -{ +impl VMMemory { /// Creates a new linear memory instance of the correct type with specified /// minimum and maximum number of wasm pages. /// /// This creates a `Memory` with owned metadata: this can be used to create a memory /// that will be imported into Wasm modules. pub fn new(memory: &MemoryType, style: &MemoryStyle) -> Result { - Ok( - Self(Box::new(VMOwnedMemory::new(memory, style)?)) - ) + Ok(Self(Box::new(VMOwnedMemory::new(memory, style)?))) } /// Create a new linear memory instance with specified minimum and maximum number of wasm pages. @@ -381,9 +368,11 @@ impl VMMemory style: &MemoryStyle, vm_memory_location: NonNull, ) -> Result { - Ok( - Self(Box::new(VMOwnedMemory::from_definition(memory, style, vm_memory_location)?)) - ) + Ok(Self(Box::new(VMOwnedMemory::from_definition( + memory, + style, + vm_memory_location, + )?))) } /// Creates VMMemory from a custom implementation - the following into implementations @@ -392,7 +381,8 @@ impl VMMemory /// - VMSharedMemory -> VMMemory /// - Box -> VMMemory pub fn from_custom(memory: IntoVMMemory) -> VMMemory - where IntoVMMemory: Into + where + IntoVMMemory: Into, { memory.into() } diff --git a/lib/vm/src/store.rs b/lib/vm/src/store.rs index 25afdb7c1..57630f15c 100644 --- a/lib/vm/src/store.rs +++ b/lib/vm/src/store.rs @@ -267,9 +267,9 @@ impl MaybeInstanceOwned { } } -impl std::fmt::Debug -for MaybeInstanceOwned -where T: std::fmt::Debug +impl std::fmt::Debug for MaybeInstanceOwned +where + T: std::fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -277,7 +277,7 @@ where T: std::fmt::Debug write!(f, "host(")?; p.as_ref().fmt(f)?; write!(f, ")") - }, + } MaybeInstanceOwned::Instance(p) => { write!(f, "instance(")?; unsafe { p.as_ref().fmt(f)? }; @@ -285,4 +285,4 @@ where T: std::fmt::Debug } } } -} \ No newline at end of file +} diff --git a/lib/vm/src/vmcontext.rs b/lib/vm/src/vmcontext.rs index 6940f43a2..9d28adae3 100644 --- a/lib/vm/src/vmcontext.rs +++ b/lib/vm/src/vmcontext.rs @@ -313,7 +313,12 @@ mod test_vmglobal_import { /// # Safety /// The memory is not copied atomically and is not synchronized: it's the /// caller's responsibility to synchronize. -pub(crate) unsafe fn memory_copy(mem: &VMMemoryDefinition, dst: u32, src: u32, len: u32) -> Result<(), Trap> { +pub(crate) unsafe fn memory_copy( + mem: &VMMemoryDefinition, + dst: u32, + src: u32, + len: u32, +) -> Result<(), Trap> { // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy if src .checked_add(len) @@ -347,7 +352,12 @@ pub(crate) unsafe fn memory_copy(mem: &VMMemoryDefinition, dst: u32, src: u32, l /// # Safety /// The memory is not filled atomically and is not synchronized: it's the /// caller's responsibility to synchronize. -pub(crate) unsafe fn memory_fill(mem: &VMMemoryDefinition, dst: u32, val: u32, len: u32) -> Result<(), Trap> { +pub(crate) unsafe fn memory_fill( + mem: &VMMemoryDefinition, + dst: u32, + val: u32, + len: u32, +) -> Result<(), Trap> { if dst .checked_add(len) .map_or(true, |m| usize::try_from(m).unwrap() > mem.current_length) From 7db6d3de78a2d7d66b8e6a83709ad5eea7d0ac17 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Thu, 18 Aug 2022 17:40:32 +0200 Subject: [PATCH 03/40] Made clippy happy --- lib/compiler/src/engine/resolver.rs | 2 +- lib/vm/src/memory.rs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/compiler/src/engine/resolver.rs b/lib/compiler/src/engine/resolver.rs index 2b218bde3..561ab998b 100644 --- a/lib/compiler/src/engine/resolver.rs +++ b/lib/compiler/src/engine/resolver.rs @@ -150,7 +150,7 @@ pub fn resolve_imports( bound: import_bound, .. }, - ) = (export_memory_style.clone(), &import_memory_style) + ) = (export_memory_style, &import_memory_style) { assert_ge!(bound, *import_bound); } diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index 5ed4c9cf7..7a8205579 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -248,12 +248,12 @@ impl VMOwnedMemory { }; Ok(Self { - mmap: mmap, + mmap, config: VMMemoryConfig { maximum: memory.maximum, offset_guard_size: offset_guard_bytes, memory: *memory, - style: style.clone(), + style: *style, }, }) } @@ -295,9 +295,9 @@ impl LinearMemory for VMOwnedMemory { } } -impl Into for VMOwnedMemory { - fn into(self) -> VMMemory { - VMMemory(Box::new(self)) +impl From for VMMemory { + fn from(mem: VMOwnedMemory) -> Self { + Self(Box::new(mem)) } } @@ -305,9 +305,9 @@ impl Into for VMOwnedMemory { #[derive(Debug)] pub struct VMMemory(Box); -impl Into for Box { - fn into(self) -> VMMemory { - VMMemory(self) +impl From> for VMMemory { + fn from(mem: Box) -> Self { + Self(mem) } } From 6cf99a63f6426469f8bc94f6f811ee0036992abc Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Fri, 19 Aug 2022 12:15:55 +0200 Subject: [PATCH 04/40] Removed from_vm_export in favor of from_vm_extern --- lib/api/src/js/externals/memory.rs | 15 ++++++--------- lib/api/src/js/externals/mod.rs | 2 +- lib/api/src/js/instance.rs | 2 +- lib/api/src/sys/externals/memory.rs | 5 ++--- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/api/src/js/externals/memory.rs b/lib/api/src/js/externals/memory.rs index 599b261e2..015769b89 100644 --- a/lib/api/src/js/externals/memory.rs +++ b/lib/api/src/js/externals/memory.rs @@ -97,7 +97,8 @@ impl Memory { .map_err(|_e| MemoryError::Generic("Error while creating the memory".to_owned()))?; let vm_memory = VMMemory::new(js_memory, ty); - Ok(Self::from_vm_export(store, vm_memory)) + let handle = StoreHandle::new(store.objects_mut(), vm_memory); + Ok(Self::from_vm_extern(store, handle.internal_handle())) } /// Creates a new host `Memory` from provided JavaScript memory. @@ -107,12 +108,14 @@ impl Memory { ty: MemoryType, ) -> Result { let vm_memory = VMMemory::new(js_memory, ty); - Ok(Self::from_vm_export(store, vm_memory)) + let handle = StoreHandle::new(store.objects_mut(), vm_memory); + Ok(Self::from_vm_extern(store, handle.internal_handle())) } /// Create a memory object from an existing memory and attaches it to the store pub fn new_from_existing(new_store: &mut impl AsStoreMut, memory: VMMemory) -> Self { - Self::from_vm_export(new_store, memory) + let handle = StoreHandle::new(new_store.objects_mut(), memory); + Self::from_vm_extern(new_store, handle.internal_handle()) } /// Returns the [`MemoryType`] of the `Memory`. @@ -192,12 +195,6 @@ impl Memory { Ok(Pages(new_pages)) } - pub(crate) fn from_vm_export(store: &mut impl AsStoreMut, vm_memory: VMMemory) -> Self { - Self { - handle: StoreHandle::new(store.objects_mut(), vm_memory), - } - } - pub(crate) fn from_vm_extern( store: &mut impl AsStoreMut, internal: InternalStoreHandle, diff --git a/lib/api/src/js/externals/mod.rs b/lib/api/src/js/externals/mod.rs index 4a3c80315..2748e08b1 100644 --- a/lib/api/src/js/externals/mod.rs +++ b/lib/api/src/js/externals/mod.rs @@ -46,7 +46,7 @@ impl Extern { } /// Create an `Extern` from an `wasmer_compiler::Export`. - pub fn from_vm_export(store: &mut impl AsStoreMut, export: Export) -> Self { + pub fn from_vm_extern(store: &mut impl AsStoreMut, export: Export) -> Self { match export { Export::Function(f) => Self::Function(Function::from_vm_extern(store, f)), Export::Memory(m) => Self::Memory(Memory::from_vm_extern(store, m)), diff --git a/lib/api/src/js/instance.rs b/lib/api/src/js/instance.rs index 6d2762adf..1b1eb0898 100644 --- a/lib/api/src/js/instance.rs +++ b/lib/api/src/js/instance.rs @@ -105,7 +105,7 @@ impl Instance { })?; let export: Export = Export::from_js_value(js_export, &mut store, extern_type)?.into(); - let extern_ = Extern::from_vm_export(&mut store, export); + let extern_ = Extern::from_vm_extern(&mut store, export); Ok((name.to_string(), extern_)) }) .collect::>()?; diff --git a/lib/api/src/sys/externals/memory.rs b/lib/api/src/sys/externals/memory.rs index 459863155..688d86c46 100644 --- a/lib/api/src/sys/externals/memory.rs +++ b/lib/api/src/sys/externals/memory.rs @@ -62,9 +62,8 @@ impl Memory { /// Create a memory object from an existing memory and attaches it to the store pub fn new_from_existing(new_store: &mut impl AsStoreMut, memory: VMMemory) -> Self { - Self { - handle: StoreHandle::new(new_store.objects_mut(), memory), - } + let handle = StoreHandle::new(new_store.objects_mut(), memory); + Self::from_vm_extern(new_store, handle.internal_handle()) } /// Returns the [`MemoryType`] of the `Memory`. From 553745cd3dd06d43694985628cf0e88706d273e5 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Fri, 19 Aug 2022 16:00:38 +0200 Subject: [PATCH 05/40] Move LinearMemory trait and VMMemoryDefinition to wasmer_vm (from wasmer_types) --- lib/api/src/sys/externals/memory.rs | 4 +- lib/api/src/sys/externals/memory_view.rs | 3 +- lib/compiler/src/engine/resolver.rs | 8 +-- lib/types/src/lib.rs | 2 +- lib/types/src/memory.rs | 82 ------------------------ lib/vm/src/instance/allocator.rs | 3 +- lib/vm/src/instance/mod.rs | 6 +- lib/vm/src/lib.rs | 4 +- lib/vm/src/memory.rs | 82 +++++++++++++++++++++++- lib/vm/src/vmcontext.rs | 4 +- 10 files changed, 97 insertions(+), 101 deletions(-) diff --git a/lib/api/src/sys/externals/memory.rs b/lib/api/src/sys/externals/memory.rs index 688d86c46..285acb231 100644 --- a/lib/api/src/sys/externals/memory.rs +++ b/lib/api/src/sys/externals/memory.rs @@ -10,8 +10,8 @@ use std::mem::MaybeUninit; use std::slice; #[cfg(feature = "tracing")] use tracing::warn; -use wasmer_types::{LinearMemory, Pages}; -use wasmer_vm::{InternalStoreHandle, MemoryError, StoreHandle, VMExtern, VMMemory}; +use wasmer_types::Pages; +use wasmer_vm::{InternalStoreHandle, LinearMemory, MemoryError, StoreHandle, VMExtern, VMMemory}; use super::MemoryView; diff --git a/lib/api/src/sys/externals/memory_view.rs b/lib/api/src/sys/externals/memory_view.rs index 5822c4113..a872ea7c2 100644 --- a/lib/api/src/sys/externals/memory_view.rs +++ b/lib/api/src/sys/externals/memory_view.rs @@ -4,7 +4,8 @@ use std::convert::TryInto; use std::marker::PhantomData; use std::mem::MaybeUninit; use std::slice; -use wasmer_types::{LinearMemory, Pages}; +use wasmer_types::Pages; +use wasmer_vm::LinearMemory; use super::memory::MemoryBuffer; use super::Memory; diff --git a/lib/compiler/src/engine/resolver.rs b/lib/compiler/src/engine/resolver.rs index 561ab998b..87fa45f30 100644 --- a/lib/compiler/src/engine/resolver.rs +++ b/lib/compiler/src/engine/resolver.rs @@ -4,13 +4,13 @@ use crate::LinkError; use more_asserts::assert_ge; use wasmer_types::entity::{BoxedSlice, EntityRef, PrimaryMap}; use wasmer_types::{ - ExternType, FunctionIndex, ImportError, ImportIndex, LinearMemory, MemoryIndex, ModuleInfo, - TableIndex, + ExternType, FunctionIndex, ImportError, ImportIndex, MemoryIndex, ModuleInfo, TableIndex, }; use wasmer_vm::{ - FunctionBodyPtr, Imports, MemoryStyle, StoreObjects, TableStyle, VMExtern, VMFunctionBody, - VMFunctionImport, VMFunctionKind, VMGlobalImport, VMMemoryImport, VMTableImport, + FunctionBodyPtr, Imports, LinearMemory, MemoryStyle, StoreObjects, TableStyle, VMExtern, + VMFunctionBody, VMFunctionImport, VMFunctionKind, VMGlobalImport, VMMemoryImport, + VMTableImport, }; /// Get an `ExternType` given a import index. diff --git a/lib/types/src/lib.rs b/lib/types/src/lib.rs index 5a2e57ef0..4eebc53e2 100644 --- a/lib/types/src/lib.rs +++ b/lib/types/src/lib.rs @@ -103,7 +103,7 @@ pub use types::{ pub use value::{RawValue, ValueType}; pub use crate::libcalls::LibCall; -pub use crate::memory::{LinearMemory, MemoryStyle, VMMemoryDefinition}; +pub use crate::memory::MemoryStyle; pub use crate::table::TableStyle; pub use crate::trapcode::TrapCode; pub use crate::vmoffsets::{TargetSharedSignatureIndex, VMBuiltinFunctionIndex, VMOffsets}; diff --git a/lib/types/src/memory.rs b/lib/types/src/memory.rs index a06b3379c..2c986d908 100644 --- a/lib/types/src/memory.rs +++ b/lib/types/src/memory.rs @@ -1,5 +1,4 @@ use crate::{Pages, ValueType}; -use core::ptr::NonNull; use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -7,9 +6,6 @@ use std::convert::{TryFrom, TryInto}; use std::iter::Sum; use std::ops::{Add, AddAssign}; -use super::MemoryError; -use super::MemoryType; - /// Implementation styles for WebAssembly linear memory. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, RkyvSerialize, RkyvDeserialize, Archive)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] @@ -129,81 +125,3 @@ unsafe impl MemorySize for Memory64 { native as Self::Offset } } - -/// Represents memory that is used by the WebAsssembly module -pub trait LinearMemory -where - Self: std::fmt::Debug + Send, -{ - /// Returns the type for this memory. - fn ty(&self) -> MemoryType; - - /// Returns the size of hte memory in pages - fn size(&self) -> Pages; - - /// Returns the memory style for this memory. - fn style(&self) -> MemoryStyle; - - /// Grow memory by the specified amount of wasm pages. - /// - /// Returns `None` if memory can't be grown by the specified amount - /// of wasm pages. - fn grow(&mut self, delta: Pages) -> Result; - - /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. - fn vmmemory(&self) -> NonNull; - - /// Attempts to clone this memory (if its clonable) - fn try_clone(&self) -> Option>; -} - -/// The fields compiled code needs to access to utilize a WebAssembly linear -/// memory defined within the instance, namely the start address and the -/// size in bytes. -#[derive(Debug, Copy, Clone)] -#[repr(C)] -pub struct VMMemoryDefinition { - /// The start address which is always valid, even if the memory grows. - pub base: *mut u8, - - /// The current logical size of this linear memory in bytes. - pub current_length: usize, -} - -/// # Safety -/// This data is safe to share between threads because it's plain data that -/// is the user's responsibility to synchronize. -unsafe impl Send for VMMemoryDefinition {} -/// # Safety -/// This data is safe to share between threads because it's plain data that -/// is the user's responsibility to synchronize. And it's `Copy` so there's -/// really no difference between passing it by reference or by value as far as -/// correctness in a multi-threaded context is concerned. -unsafe impl Sync for VMMemoryDefinition {} - -#[cfg(test)] -mod test_vmmemory_definition { - use super::VMMemoryDefinition; - use crate::ModuleInfo; - use crate::VMOffsets; - use memoffset::offset_of; - use std::mem::size_of; - - #[test] - fn check_vmmemory_definition_offsets() { - let module = ModuleInfo::new(); - let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); - assert_eq!( - size_of::(), - usize::from(offsets.size_of_vmmemory_definition()) - ); - assert_eq!( - offset_of!(VMMemoryDefinition, base), - usize::from(offsets.vmmemory_definition_base()) - ); - assert_eq!( - offset_of!(VMMemoryDefinition, current_length), - usize::from(offsets.vmmemory_definition_current_length()) - ); - } -} diff --git a/lib/vm/src/instance/allocator.rs b/lib/vm/src/instance/allocator.rs index c6ca74193..e81af2550 100644 --- a/lib/vm/src/instance/allocator.rs +++ b/lib/vm/src/instance/allocator.rs @@ -1,12 +1,13 @@ use super::{Instance, InstanceHandle}; +use crate::memory::VMMemoryDefinition; use crate::vmcontext::VMTableDefinition; use std::alloc::{self, Layout}; use std::convert::TryFrom; use std::mem; use std::ptr::{self, NonNull}; use wasmer_types::entity::EntityRef; +use wasmer_types::VMOffsets; use wasmer_types::{LocalMemoryIndex, LocalTableIndex, ModuleInfo}; -use wasmer_types::{VMMemoryDefinition, VMOffsets}; /// This is an intermediate type that manages the raw allocation and /// metadata when creating an [`Instance`]. diff --git a/lib/vm/src/instance/mod.rs b/lib/vm/src/instance/mod.rs index 98e2423d1..c91a8f63f 100644 --- a/lib/vm/src/instance/mod.rs +++ b/lib/vm/src/instance/mod.rs @@ -19,6 +19,7 @@ use crate::vmcontext::{ VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, }; use crate::{FunctionBodyPtr, MaybeInstanceOwned, TrapHandlerFn, VMFunctionBody}; +use crate::{LinearMemory, VMMemoryDefinition}; use crate::{VMFuncRef, VMFunction, VMGlobal, VMMemory, VMTable}; pub use allocator::InstanceAllocator; use memoffset::offset_of; @@ -35,9 +36,8 @@ use std::sync::Arc; use wasmer_types::entity::{packed_option::ReservedValue, BoxedSlice, EntityRef, PrimaryMap}; use wasmer_types::{ DataIndex, DataInitializer, ElemIndex, ExportIndex, FunctionIndex, GlobalIndex, GlobalInit, - LinearMemory, LocalFunctionIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, - MemoryError, MemoryIndex, ModuleInfo, Pages, SignatureIndex, TableIndex, TableInitializer, - VMMemoryDefinition, VMOffsets, + LocalFunctionIndex, LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryError, + MemoryIndex, ModuleInfo, Pages, SignatureIndex, TableIndex, TableInitializer, VMOffsets, }; /// A WebAssembly instance. diff --git a/lib/vm/src/lib.rs b/lib/vm/src/lib.rs index 05ae87d4a..16af4df4b 100644 --- a/lib/vm/src/lib.rs +++ b/lib/vm/src/lib.rs @@ -45,7 +45,7 @@ pub use crate::function_env::VMFunctionEnvironment; pub use crate::global::*; pub use crate::imports::Imports; pub use crate::instance::{InstanceAllocator, InstanceHandle}; -pub use crate::memory::VMMemory; +pub use crate::memory::{LinearMemory, VMMemory, VMMemoryDefinition}; pub use crate::mmap::Mmap; pub use crate::probestack::PROBESTACK; pub use crate::sig_registry::SignatureRegistry; @@ -61,9 +61,9 @@ pub use crate::vmcontext::{ }; pub use wasmer_types::LibCall; pub use wasmer_types::MemoryError; +pub use wasmer_types::MemoryStyle; use wasmer_types::RawValue; pub use wasmer_types::TableStyle; -pub use wasmer_types::{MemoryStyle, VMMemoryDefinition}; pub use wasmer_types::{TargetSharedSignatureIndex, VMBuiltinFunctionIndex, VMOffsets}; #[deprecated( diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index 7a8205579..a82c48295 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -10,9 +10,7 @@ use more_asserts::assert_ge; use std::cell::UnsafeCell; use std::convert::TryInto; use std::ptr::NonNull; -use wasmer_types::{ - Bytes, LinearMemory, MemoryError, MemoryStyle, MemoryType, Pages, VMMemoryDefinition, -}; +use wasmer_types::{Bytes, MemoryError, MemoryStyle, MemoryType, Pages}; // The memory mapped area #[derive(Debug)] @@ -387,3 +385,81 @@ impl VMMemory { memory.into() } } + +/// Represents memory that is used by the WebAsssembly module +pub trait LinearMemory +where + Self: std::fmt::Debug + Send, +{ + /// Returns the type for this memory. + fn ty(&self) -> MemoryType; + + /// Returns the size of hte memory in pages + fn size(&self) -> Pages; + + /// Returns the memory style for this memory. + fn style(&self) -> MemoryStyle; + + /// Grow memory by the specified amount of wasm pages. + /// + /// Returns `None` if memory can't be grown by the specified amount + /// of wasm pages. + fn grow(&mut self, delta: Pages) -> Result; + + /// Return a `VMMemoryDefinition` for exposing the memory to compiled wasm code. + fn vmmemory(&self) -> NonNull; + + /// Attempts to clone this memory (if its clonable) + fn try_clone(&self) -> Option>; +} + +/// The fields compiled code needs to access to utilize a WebAssembly linear +/// memory defined within the instance, namely the start address and the +/// size in bytes. +#[derive(Debug, Copy, Clone)] +#[repr(C)] +pub struct VMMemoryDefinition { + /// The start address which is always valid, even if the memory grows. + pub base: *mut u8, + + /// The current logical size of this linear memory in bytes. + pub current_length: usize, +} + +/// # Safety +/// This data is safe to share between threads because it's plain data that +/// is the user's responsibility to synchronize. +unsafe impl Send for VMMemoryDefinition {} +/// # Safety +/// This data is safe to share between threads because it's plain data that +/// is the user's responsibility to synchronize. And it's `Copy` so there's +/// really no difference between passing it by reference or by value as far as +/// correctness in a multi-threaded context is concerned. +unsafe impl Sync for VMMemoryDefinition {} + +#[cfg(test)] +mod test_vmmemory_definition { + use super::VMMemoryDefinition; + use crate::ModuleInfo; + use crate::VMOffsets; + use memoffset::offset_of; + use std::mem::size_of; + + #[test] + fn check_vmmemory_definition_offsets() { + let module = ModuleInfo::new(); + let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); + assert_eq!( + size_of::(), + usize::from(offsets.size_of_vmmemory_definition()) + ); + assert_eq!( + offset_of!(VMMemoryDefinition, base), + usize::from(offsets.vmmemory_definition_base()) + ); + assert_eq!( + offset_of!(VMMemoryDefinition, current_length), + usize::from(offsets.vmmemory_definition_current_length()) + ); + } +} diff --git a/lib/vm/src/vmcontext.rs b/lib/vm/src/vmcontext.rs index 9d28adae3..cea7b116e 100644 --- a/lib/vm/src/vmcontext.rs +++ b/lib/vm/src/vmcontext.rs @@ -6,7 +6,7 @@ use crate::global::VMGlobal; use crate::instance::Instance; -use crate::memory::VMMemory; +use crate::memory::{VMMemory, VMMemoryDefinition}; use crate::store::InternalStoreHandle; use crate::trap::{Trap, TrapCode}; use crate::VMFunctionBody; @@ -15,7 +15,7 @@ use crate::{VMBuiltinFunctionIndex, VMFunction}; use std::convert::TryFrom; use std::ptr::{self, NonNull}; use std::u32; -use wasmer_types::{RawValue, VMMemoryDefinition}; +use wasmer_types::RawValue; /// Union representing the first parameter passed when calling a function. /// From 57632ab0ffbe896b771c146e5782ed3ae25164c7 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Sat, 20 Aug 2022 22:17:21 +0300 Subject: [PATCH 06/40] Update Cargo.lock dependencies plotters 0.3.2 and pest 2.2.1 were yanked, issued `cargo update` to resolve. --- Cargo.lock | 159 +++++++++++++++++++++++++++++------------------------ 1 file changed, 86 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9328944bd..597c3303b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ed72e1635e121ca3e79420540282af22da58be50de153d36f81ddc6b83aa9e" +dependencies = [ + "libc", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -48,9 +57,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.60" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c794e162a5eff65c72ef524dfe393eb923c354e350bb78b9c7383df13f3bc142" +checksum = "1485d4d2cc45e7b201ee3767015c96faa5904387c9d87c6efdd0fb511f12d305" [[package]] name = "arbitrary" @@ -81,7 +90,7 @@ checksum = "c98233c6673d8601ab23e77eb38f999c51100d46c5703b17288c57fddf3a1ffe" dependencies = [ "bstr", "doc-comment", - "predicates 2.1.1", + "predicates", "predicates-core", "predicates-tree", "wait-timeout", @@ -215,9 +224,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" [[package]] name = "bytecheck" @@ -270,7 +279,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6358dedf60f4d9b8db43ad187391afe959746101346fe51bb978126bec61dfb" dependencies = [ - "clap 3.2.16", + "clap 3.2.17", "heck", "indexmap", "log", @@ -312,10 +321,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.20" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6127248204b9aba09a362f6c930ef6a78f2c1b2215f8a7b398c06e1083f17af0" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ + "iana-time-zone", "js-sys", "num-integer", "num-traits", @@ -351,9 +361,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.16" +version = "3.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3dbbb6653e7c55cc8595ad3e1f7be8f32aba4eb7ff7f0fd1163d4f3d137c0a9" +checksum = "29e724a68d9319343bb3328c9cc2dfde263f4b3142ee1059a9980580171c954b" dependencies = [ "atty", "bitflags", @@ -368,9 +378,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.2.15" +version = "3.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" +checksum = "13547f7012c01ab4a0e8f8967730ada8f9fdf419e8b6c792788f39cf4e46eefa" dependencies = [ "heck", "proc-macro-error", @@ -474,6 +484,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "corosensei" version = "0.1.3" @@ -755,12 +771,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "difference" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" - [[package]] name = "difflib" version = "0.4.0" @@ -871,9 +881,9 @@ dependencies = [ [[package]] name = "either" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "enum-iterator" @@ -987,9 +997,9 @@ dependencies = [ [[package]] name = "float-cmp" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" dependencies = [ "num-traits", ] @@ -1162,6 +1172,19 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iana-time-zone" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "js-sys", + "wasm-bindgen", + "winapi", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1207,15 +1230,15 @@ dependencies = [ [[package]] name = "inline-c" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b370741c5913d9b3765d280493b9b97658818410fd64cbf6e8d333b42a1049e" +checksum = "340dd3d6102fa919bd20987024a6d84954c36ec691ac1efea37742ee983c8dd5" dependencies = [ "assert_cmd", "cc", "inline-c-macro", "lazy_static", - "predicates 1.0.8", + "predicates", "regex", "rustc_version 0.3.3", "target-lexicon 0.11.2", @@ -1302,9 +1325,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.127" +version = "0.2.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b" +checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" [[package]] name = "libfuzzer-sys" @@ -1385,9 +1408,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a79b39c93a7a5a27eeaf9a23b5ff43f1b9e0ad6b1cdd441140ae53c35613fc7" +checksum = "95af15f345b17af2efc8ead6080fb8bc376f8cec1b35277b935637595fe77498" dependencies = [ "libc", ] @@ -1541,9 +1564,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e" [[package]] name = "oorandom" @@ -1568,9 +1591,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.2.0" +version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4" +checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" [[package]] name = "output_vt100" @@ -1620,9 +1643,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pest" -version = "2.2.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69486e2b8c2d2aeb9762db7b4e00b0331156393555cff467f4163ff06821eef8" +checksum = "4b0560d531d1febc25a3c9398a62a71256c0178f2e3443baedd9ad4bb8c9deb4" dependencies = [ "thiserror", "ucd-trie", @@ -1642,9 +1665,9 @@ checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "plotters" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9428003b84df1496fb9d6eeee9c5f8145cb41ca375eb0dad204328888832811f" +checksum = "716b4eeb6c4a1d3ecc956f75b43ec2e8e8ba80026413e70a3f41fd3313d3492b" dependencies = [ "num-traits", "plotters-backend", @@ -1661,9 +1684,9 @@ checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" [[package]] name = "plotters-svg" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0918736323d1baff32ee0eade54984f6f201ad7e97d5cfb5d6ab4a358529615" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" dependencies = [ "plotters-backend", ] @@ -1674,19 +1697,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" -[[package]] -name = "predicates" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df" -dependencies = [ - "difference", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", -] - [[package]] name = "predicates" version = "2.1.1" @@ -1694,8 +1704,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5aab5be6e4732b473071984b3164dbbfb7a3674d30ea5ff44410b6bcd960c3c" dependencies = [ "difflib", + "float-cmp", "itertools", + "normalize-line-endings", "predicates-core", + "regex", ] [[package]] @@ -2181,9 +2194,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.142" +version = "1.0.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590c437916fb6b221e1d00df6e3294f3fccd70ca7e92541c475d6ed6ef5fee2" +checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860" dependencies = [ "serde_derive", ] @@ -2209,9 +2222,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.142" +version = "1.0.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b5b8d809babe02f538c2cfec6f2c1ed10804c0e5a6a041a049a4f5588ccc2e" +checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00" dependencies = [ "proc-macro2", "quote", @@ -2886,9 +2899,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8905fd25fdadeb0e7e8bf43a9f46f9f972d6291ad0c7a32573b88dd13a6cfa6b" +checksum = "d443c5a7daae71697d97ec12ad70b4fe8766d3a0f4db16158ac8b781365892f7" dependencies = [ "leb128", ] @@ -2996,7 +3009,7 @@ dependencies = [ "atty", "bytesize", "cfg-if 1.0.0", - "clap 3.2.16", + "clap 3.2.17", "colored 2.0.0", "dirs", "distance", @@ -3057,7 +3070,7 @@ dependencies = [ "atty", "bytesize", "cfg-if 1.0.0", - "clap 3.2.16", + "clap 3.2.17", "colored 2.0.0", "distance", "fern", @@ -3378,21 +3391,21 @@ checksum = "718ed7c55c2add6548cca3ddd6383d738cd73b892df400e96b9aa876f0141d7a" [[package]] name = "wasmparser" -version = "0.88.0" +version = "0.89.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb8cf7dd82407fe68161bedcd57fde15596f32ebf6e9b3bdbf3ae1da20e38e5e" +checksum = "ab5d3e08b13876f96dd55608d03cd4883a0545884932d5adf11925876c96daef" dependencies = [ "indexmap", ] [[package]] name = "wasmprinter" -version = "0.2.38" +version = "0.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f2786f19a25211ddfa331e28b7579a6d6880f5f4b18d21253cd90274aa4c21" +checksum = "aa9e5ee2f56cc8a5da489558114e8c118e5a8416d96aefe63dcf1b5b05b858c6" dependencies = [ "anyhow", - "wasmparser 0.88.0", + "wasmparser 0.89.1", ] [[package]] @@ -3415,23 +3428,23 @@ dependencies = [ [[package]] name = "wast" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186c474c4f9bb92756b566d592a16591b4526b1a4841171caa3f31d7fe330d96" +checksum = "ea0ab19660e3ea6891bba69167b9be40fad00fb1fe3dd39c5eebcee15607131b" dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder 0.15.0", + "wasm-encoder 0.16.0", ] [[package]] name = "wat" -version = "1.0.47" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d4bc4724b4f02a482c8cab053dac5ef26410f264c06ce914958f9a42813556" +checksum = "8f775282def4d5bffd94d60d6ecd57bfe6faa46171cdbf8d32bd5458842b1e3e" dependencies = [ - "wast 45.0.0", + "wast 46.0.0", ] [[package]] @@ -3661,9 +3674,9 @@ checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "x11-dl" -version = "2.19.1" +version = "2.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea26926b4ce81a6f5d9d0f3a0bc401e5a37c6ae14a1bfaa8ff6099ca80038c59" +checksum = "0c83627bc137605acc00bb399c7b908ef460b621fc37c953db2b09f88c449ea6" dependencies = [ "lazy_static", "libc", From 57d918e3f590cc9b0b804992c8e0208e3a394569 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Sun, 21 Aug 2022 16:12:52 +0300 Subject: [PATCH 07/40] Bring libwasmer-headless.a from 22MiB to 7.2MiB (on my machine) By using lto optimization flags. A qjs.wasm executable using the headless engine now weights 4.6MiB: % du -sh qjs-he*.out 20M qjs-headfull.out 4.6M qjs-headless.out --- Makefile | 4 ++-- lib/c-api/Cargo.toml | 2 +- lib/wasi/Cargo.toml | 3 --- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a6be323b4..aa38bd87c 100644 --- a/Makefile +++ b/Makefile @@ -389,7 +389,7 @@ build-wasmer-headless-minimal: RUSTFLAGS += -C panic=abort build-wasmer-headless-minimal: RUSTFLAGS="${RUSTFLAGS}" xargo build --target $(HOST_TARGET) --release --manifest-path=lib/cli/Cargo.toml --no-default-features --features headless-minimal --bin wasmer-headless ifeq ($(IS_DARWIN), 1) - strip -u target/$(HOST_TARGET)/release/wasmer-headless + strip target/$(HOST_TARGET)/release/wasmer-headless else ifeq ($(IS_WINDOWS), 1) strip --strip-unneeded target/$(HOST_TARGET)/release/wasmer-headless.exe else @@ -455,7 +455,7 @@ build-capi-llvm-universal: capi-setup # Headless (we include the minimal to be able to run) build-capi-headless: capi-setup - RUSTFLAGS="${RUSTFLAGS} -C panic=abort" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS} -C panic=abort -C link-dead-code -C lto -O -C embed-bitcode=yes" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features compiler-headless,wasi build-capi-headless-ios: capi-setup diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index d275c0fff..072da4e3d 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -17,7 +17,7 @@ edition = "2018" # a conflict with the existing `wasmer` crate, see below. name = "wasmer" # ##lib.name## # ^ DO NOT REMOVE, it's used the `Makefile`, see `build-docs-capi`. -crate-type = ["cdylib", "rlib", "staticlib"] +crate-type = ["staticlib", "cdylib"] #"cdylib", "rlib", "staticlib"] [dependencies] # We rename `wasmer` to `wasmer-api` to avoid the conflict with this diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index bfa746e29..798d13046 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -10,9 +10,6 @@ license = "MIT" readme = "README.md" edition = "2018" -[lib] -crate-type = ["cdylib", "rlib"] - [dependencies] cfg-if = "1.0" thiserror = "1" From 306347e1b90a7c6dd057a417ba5d52aa41516f04 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Thu, 18 Aug 2022 13:50:40 +0300 Subject: [PATCH 08/40] create-exe: fix serialized create-exe not working Unified generated objects interfaces: both static and serialized objects expose the same function signature to create the module: wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name); --- lib/cli/src/c_gen/staticlib_header.rs | 4 +- lib/cli/src/commands/create_exe.rs | 78 ++++--- lib/cli/src/commands/create_obj.rs | 12 +- lib/cli/src/commands/wasmer_create_exe_main.c | 14 +- ...eate_exe.h => wasmer_deserialize_module.h} | 2 +- .../commands/wasmer_static_create_exe_main.c | 192 ------------------ 6 files changed, 54 insertions(+), 248 deletions(-) rename lib/cli/src/commands/{wasmer_create_exe.h => wasmer_deserialize_module.h} (84%) delete mode 100644 lib/cli/src/commands/wasmer_static_create_exe_main.c diff --git a/lib/cli/src/c_gen/staticlib_header.rs b/lib/cli/src/c_gen/staticlib_header.rs index bd203c644..6449ef8b4 100644 --- a/lib/cli/src/c_gen/staticlib_header.rs +++ b/lib/cli/src/c_gen/staticlib_header.rs @@ -58,8 +58,8 @@ wasm_byte_vec_t generate_serialized_data() { return module_byte_vec; } -wasm_module_t* wasmer_static_module_new(wasm_store_t* store, const char* wasm_name) { - // wasm_name intentionally unused for now: will be used in the future. +wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name) { + // module_name intentionally unused for now: will be used in the future. wasm_byte_vec_t module_byte_vec = generate_serialized_data(); wasm_module_t* module = wasm_module_deserialize(store, &module_byte_vec); free(module_byte_vec.data); diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 69b9abb42..470dd96a6 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -21,8 +21,7 @@ use wasmer_object::{emit_serialized, get_object_for_target}; pub type PrefixerFn = Box String + Send>; const WASMER_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_create_exe_main.c"); -#[cfg(feature = "static-artifact-create")] -const WASMER_STATIC_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_static_create_exe_main.c"); +const WASMER_DESERIALIZE_HEADER: &str = include_str!("wasmer_deserialize_module.h"); #[derive(Debug, Clone)] struct CrossCompile { @@ -307,9 +306,39 @@ impl CreateExe { .map_err(|err| anyhow::anyhow!(err.to_string()))?; writer.flush()?; drop(writer); + // Write down header file that includes deserialize function + { + let mut writer = BufWriter::new(File::create("static_defs.h")?); + writer.write_all(WASMER_DESERIALIZE_HEADER.as_bytes())?; + writer.flush()?; + } - let cli_given_triple = self.target_triple.clone(); - self.compile_c(wasm_object_path, cli_given_triple, output_path)?; + // write C src to disk + let c_src_path = Path::new("wasmer_main.c"); + #[cfg(not(windows))] + let c_src_obj = PathBuf::from("wasmer_main.o"); + #[cfg(windows)] + let c_src_obj = PathBuf::from("wasmer_main.obj"); + + { + let mut c_src_file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&c_src_path) + .context("Failed to open C source code file")?; + c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; + } + run_c_compile(c_src_path, &c_src_obj, self.target_triple.clone()) + .context("Failed to compile C source code")?; + LinkCode { + object_paths: vec![c_src_obj, wasm_object_path], + output_path, + additional_libraries: self.libraries.clone(), + target: self.target_triple.clone(), + ..Default::default() + } + .run() + .context("Failed to link objects together")?; } #[cfg(not(feature = "static-artifact-create"))] ObjectFormat::Symbols => { @@ -379,42 +408,6 @@ impl CreateExe { Ok(()) } - fn compile_c( - &self, - wasm_object_path: PathBuf, - target_triple: Option, - output_path: PathBuf, - ) -> anyhow::Result<()> { - // write C src to disk - let c_src_path = Path::new("wasmer_main.c"); - #[cfg(not(windows))] - let c_src_obj = PathBuf::from("wasmer_main.o"); - #[cfg(windows)] - let c_src_obj = PathBuf::from("wasmer_main.obj"); - - { - let mut c_src_file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&c_src_path) - .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; - } - run_c_compile(c_src_path, &c_src_obj, target_triple.clone()) - .context("Failed to compile C source code")?; - LinkCode { - object_paths: vec![c_src_obj, wasm_object_path], - output_path, - additional_libraries: self.libraries.clone(), - target: target_triple, - ..Default::default() - } - .run() - .context("Failed to link objects together")?; - - Ok(()) - } - fn compile_zig( &self, output_path: PathBuf, @@ -449,7 +442,7 @@ impl CreateExe { .write(true) .open(&c_src_path) .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_STATIC_MAIN_C_SOURCE)?; + c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; } if !header_code_path.is_dir() { @@ -531,7 +524,7 @@ impl CreateExe { .write(true) .open(&c_src_path) .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_STATIC_MAIN_C_SOURCE)?; + c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; } if !header_code_path.is_dir() { @@ -666,6 +659,7 @@ fn run_c_compile( .arg("-O2") .arg("-c") .arg(path_to_c_src) + .arg("-I.") .arg("-I") .arg(get_wasmer_include_directory()?); diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs index f9113e45f..c8e3534ba 100644 --- a/lib/cli/src/commands/create_obj.rs +++ b/lib/cli/src/commands/create_obj.rs @@ -15,7 +15,7 @@ use std::process::Command; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; -const WASMER_SERIALIZED_HEADER: &[u8] = include_bytes!("wasmer_create_exe.h"); +const WASMER_SERIALIZED_HEADER: &[u8] = include_bytes!("wasmer_deserialize_module.h"); #[derive(Debug, Parser)] /// The options for the `wasmer create-exe` subcommand @@ -152,6 +152,16 @@ impl CreateObj { self.output.display(), header_output.display(), ); + eprintln!("\n---\n"); + eprintln!( + r#"To use, link the object file to your executable and call the `wasmer_object_module_new` function defined in the header file. For example, in the C language: + + #include "{}" + + wasm_module_t *module = wasmer_object_module_new(store, "my_module_name"); + "#, + header_output.display(), + ); Ok(()) } diff --git a/lib/cli/src/commands/wasmer_create_exe_main.c b/lib/cli/src/commands/wasmer_create_exe_main.c index a58fb5c73..3fe98f62c 100644 --- a/lib/cli/src/commands/wasmer_create_exe_main.c +++ b/lib/cli/src/commands/wasmer_create_exe_main.c @@ -1,7 +1,5 @@ - #include "wasmer.h" -//#include "my_wasm.h" - +#include "static_defs.h" #include #include #include @@ -11,8 +9,8 @@ // TODO: make this define templated so that the Rust code can toggle it on/off #define WASI -extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); -extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); +extern wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name) asm("wasmer_object_module_new"); + static void print_wasmer_error() { int error_len = wasmer_last_error_length(); @@ -95,11 +93,7 @@ int main(int argc, char *argv[]) { wasm_engine_t *engine = wasm_engine_new_with_config(config); wasm_store_t *store = wasm_store_new(engine); - wasm_byte_vec_t module_byte_vec = { - .size = WASMER_MODULE_LENGTH, - .data = (const char*)&WASMER_MODULE_DATA, - }; - wasm_module_t *module = wasm_module_deserialize(store, &module_byte_vec); + wasm_module_t *module = wasmer_object_module_new(store, "module"); if (!module) { fprintf(stderr, "Failed to create module\n"); diff --git a/lib/cli/src/commands/wasmer_create_exe.h b/lib/cli/src/commands/wasmer_deserialize_module.h similarity index 84% rename from lib/cli/src/commands/wasmer_create_exe.h rename to lib/cli/src/commands/wasmer_deserialize_module.h index 98705d4fc..f0ef229aa 100644 --- a/lib/cli/src/commands/wasmer_create_exe.h +++ b/lib/cli/src/commands/wasmer_deserialize_module.h @@ -10,7 +10,7 @@ extern "C" { extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); -wasm_module_t* wasmer_module_new(wasm_store_t* store, const char* wasm_name) { +wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name) { wasm_byte_vec_t module_byte_vec = { .size = WASMER_MODULE_LENGTH, .data = (const char*)&WASMER_MODULE_DATA, diff --git a/lib/cli/src/commands/wasmer_static_create_exe_main.c b/lib/cli/src/commands/wasmer_static_create_exe_main.c deleted file mode 100644 index 2276ed3bc..000000000 --- a/lib/cli/src/commands/wasmer_static_create_exe_main.c +++ /dev/null @@ -1,192 +0,0 @@ -#include "wasmer.h" -#include "static_defs.h" -#include -#include -#include - -#define own - -// TODO: make this define templated so that the Rust code can toggle it on/off -#define WASI - -extern wasm_module_t* wasmer_module_new(wasm_store_t* store) asm("wasmer_module_new"); -extern wasm_module_t* wasmer_static_module_new(wasm_store_t* store,const char* wasm_name) asm("wasmer_static_module_new"); - - -static void print_wasmer_error() { - int error_len = wasmer_last_error_length(); - printf("Error len: `%d`\n", error_len); - char *error_str = (char *)malloc(error_len); - wasmer_last_error_message(error_str, error_len); - printf("%s\n", error_str); - free(error_str); -} - -#ifdef WASI -static void pass_mapdir_arg(wasi_config_t *wasi_config, char *mapdir) { - int colon_location = strchr(mapdir, ':') - mapdir; - if (colon_location == 0) { - // error malformed argument - fprintf(stderr, "Expected mapdir argument of the form alias:directory\n"); - exit(-1); - } - - char *alias = (char *)malloc(colon_location + 1); - memcpy(alias, mapdir, colon_location); - alias[colon_location] = '\0'; - - int dir_len = strlen(mapdir) - colon_location; - char *dir = (char *)malloc(dir_len + 1); - memcpy(dir, &mapdir[colon_location + 1], dir_len); - dir[dir_len] = '\0'; - - wasi_config_mapdir(wasi_config, alias, dir); - free(alias); - free(dir); -} - -// We try to parse out `--dir` and `--mapdir` ahead of time and process those -// specially. All other arguments are passed to the guest program. -static void handle_arguments(wasi_config_t *wasi_config, int argc, - char *argv[]) { - for (int i = 1; i < argc; ++i) { - // We probably want special args like `--dir` and `--mapdir` to not be - // passed directly - if (strcmp(argv[i], "--dir") == 0) { - // next arg is a preopen directory - if ((i + 1) < argc) { - i++; - wasi_config_preopen_dir(wasi_config, argv[i]); - } else { - fprintf(stderr, "--dir expects a following argument specifying which " - "directory to preopen\n"); - exit(-1); - } - } else if (strcmp(argv[i], "--mapdir") == 0) { - // next arg is a mapdir - if ((i + 1) < argc) { - i++; - pass_mapdir_arg(wasi_config, argv[i]); - } else { - fprintf(stderr, - "--mapdir expects a following argument specifying which " - "directory to preopen in the form alias:directory\n"); - exit(-1); - } - } else if (strncmp(argv[i], "--dir=", strlen("--dir=")) == 0) { - // this arg is a preopen dir - char *dir = argv[i] + strlen("--dir="); - wasi_config_preopen_dir(wasi_config, dir); - } else if (strncmp(argv[i], "--mapdir=", strlen("--mapdir=")) == 0) { - // this arg is a mapdir - char *mapdir = argv[i] + strlen("--mapdir="); - pass_mapdir_arg(wasi_config, mapdir); - } else { - // guest argument - wasi_config_arg(wasi_config, argv[i]); - } - } -} -#endif - -int main(int argc, char *argv[]) { - wasm_config_t *config = wasm_config_new(); - wasm_engine_t *engine = wasm_engine_new_with_config(config); - wasm_store_t *store = wasm_store_new(engine); - - wasm_module_t *module = wasmer_static_module_new(store, "module"); - - if (!module) { - fprintf(stderr, "Failed to create module\n"); - print_wasmer_error(); - return -1; - } - - // We have now finished the memory buffer book keeping and we have a valid - // Module. - -#ifdef WASI - wasi_config_t *wasi_config = wasi_config_new(argv[0]); - handle_arguments(wasi_config, argc, argv); - - wasi_env_t *wasi_env = wasi_env_new(store, wasi_config); - if (!wasi_env) { - fprintf(stderr, "Error building WASI env!\n"); - print_wasmer_error(); - return 1; - } -#endif - - wasm_importtype_vec_t import_types; - wasm_module_imports(module, &import_types); - - wasm_extern_vec_t imports; - wasm_extern_vec_new_uninitialized(&imports, import_types.size); - wasm_importtype_vec_delete(&import_types); - -#ifdef WASI - bool get_imports_result = wasi_get_imports(store, wasi_env, module, &imports); - - if (!get_imports_result) { - fprintf(stderr, "Error getting WASI imports!\n"); - print_wasmer_error(); - - return 1; - } -#endif - - wasm_instance_t *instance = wasm_instance_new(store, module, &imports, NULL); - - if (!instance) { - fprintf(stderr, "Failed to create instance\n"); - print_wasmer_error(); - return -1; - } - -#ifdef WASI - // Read the exports. - wasm_extern_vec_t exports; - wasm_instance_exports(instance, &exports); - wasm_memory_t* mem = NULL; - for (size_t i = 0; i < exports.size; i++) { - mem = wasm_extern_as_memory(exports.data[i]); - if (mem) { - break; - } - } - - if (!mem) { - fprintf(stderr, "Failed to create instance: Could not find memory in exports\n"); - print_wasmer_error(); - return -1; - } - wasi_env_set_memory(wasi_env, mem); - - own wasm_func_t *start_function = wasi_get_start_function(instance); - if (!start_function) { - fprintf(stderr, "`_start` function not found\n"); - print_wasmer_error(); - return -1; - } - - wasm_val_vec_t args = WASM_EMPTY_VEC; - wasm_val_vec_t results = WASM_EMPTY_VEC; - own wasm_trap_t *trap = wasm_func_call(start_function, &args, &results); - if (trap) { - fprintf(stderr, "Trap is not NULL: TODO:\n"); - return -1; - } -#endif - - // TODO: handle non-WASI start (maybe with invoke?) - -#ifdef WASI - wasi_env_delete(wasi_env); - wasm_extern_vec_delete(&exports); -#endif - wasm_instance_delete(instance); - wasm_module_delete(module); - wasm_store_delete(store); - wasm_engine_delete(engine); - return 0; -} From 8844994d3a08c0becce72377600cf19342fc2a80 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Thu, 18 Aug 2022 14:28:27 +0300 Subject: [PATCH 09/40] tests/integration/cli: fix tests failing --- Makefile | 2 +- tests/integration/cli/build.rs | 2 +- tests/integration/cli/src/assets.rs | 68 +++++++++----------------- tests/integration/cli/tests/run.rs | 12 ++--- tests/integration/cli/tests/version.rs | 12 +++-- 5 files changed, 37 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index a6be323b4..67bff7ec6 100644 --- a/Makefile +++ b/Makefile @@ -535,7 +535,7 @@ test-examples: $(CARGO_BINARY) test $(CARGO_TARGET) --release $(compiler_features) --features wasi --examples test-integration: - $(CARGO_BINARY) test $(CARGO_TARGET) --no-fail-fast -p wasmer-integration-tests-cli + $(CARGO_BINARY) test $(CARGO_TARGET) --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture test-integration-ios: $(CARGO_BINARY) test $(CARGO_TARGET) -p wasmer-integration-tests-ios diff --git a/tests/integration/cli/build.rs b/tests/integration/cli/build.rs index 81caa36d6..9057ee24f 100644 --- a/tests/integration/cli/build.rs +++ b/tests/integration/cli/build.rs @@ -1,6 +1,6 @@ fn main() { println!( - "cargo:rustc-env=TARGET={}", + "cargo:rustc-env=CARGO_BUILD_TARGET={}", std::env::var("TARGET").unwrap() ); } diff --git a/tests/integration/cli/src/assets.rs b/tests/integration/cli/src/assets.rs index c3700707d..6f7b9a24b 100644 --- a/tests/integration/cli/src/assets.rs +++ b/tests/integration/cli/src/assets.rs @@ -3,74 +3,49 @@ use std::path::PathBuf; pub const C_ASSET_PATH: &str = concat!( env!("CARGO_MANIFEST_DIR"), - "/../../../lib/c-api/examples/assets" + "/../../../lib/c-api/examples/assets/" ); -pub const ASSET_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../tests/examples"); +pub const ASSET_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../tests/examples/"); -pub const WASMER_INCLUDE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../lib/c-api"); +pub const WASMER_INCLUDE_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../lib/c-api/"); #[cfg(feature = "debug")] -pub const WASMER_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../target/debug/wasmer"); - -#[cfg(not(feature = "debug"))] -pub const WASMER_PATH: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../target/release/wasmer" -); - +pub const WASMER_TARGET_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../target/debug/"); #[cfg(feature = "debug")] -pub const WASMER_TARGET_PATH: &str = concat!( +pub const WASMER_TARGET_PATH_2: &str = concat!( env!("CARGO_MANIFEST_DIR"), - "/../../../", - env!("CARGO_CFG_TARGET"), - "/debug/wasmer" + "/../../../target/", + env!("CARGO_BUILD_TARGET"), + "/debug/" ); /* env var TARGET is set by tests/integration/cli/build.rs on compile-time */ #[cfg(not(feature = "debug"))] -pub const WASMER_TARGET_PATH: &str = concat!( +pub const WASMER_TARGET_PATH: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/../../../target/release/"); +#[cfg(not(feature = "debug"))] +pub const WASMER_TARGET_PATH2: &str = concat!( env!("CARGO_MANIFEST_DIR"), "/../../../target/", - env!("TARGET"), - "/release/wasmer" + env!("CARGO_BUILD_TARGET"), + "/release/" ); #[cfg(not(windows))] -pub const LIBWASMER_PATH: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../target/release/libwasmer.a" -); +pub const LIBWASMER_FILENAME: &str = "libwasmer.a"; #[cfg(windows)] -pub const LIBWASMER_PATH: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../target/release/wasmer.lib" -); - -#[cfg(not(windows))] -pub const LIBWASMER_TARGET_PATH: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../target", - env!("TARGET"), - "/release/libwasmer.a" -); - -#[cfg(windows)] -pub const LIBWASMER_TARGET_PATH: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../", - env!("TARGET"), - "/release/wasmer.lib" -); +pub const LIBWASMER_PATH: &str = "wasmer.lib"; /// Get the path to the `libwasmer.a` static library. pub fn get_libwasmer_path() -> PathBuf { let mut ret = PathBuf::from( - env::var("WASMER_TEST_LIBWASMER_PATH").unwrap_or_else(|_| LIBWASMER_PATH.to_string()), + env::var("WASMER_TEST_LIBWASMER_PATH") + .unwrap_or_else(|_| format!("{}{}", WASMER_TARGET_PATH, LIBWASMER_FILENAME)), ); if !ret.exists() { - ret = PathBuf::from(LIBWASMER_TARGET_PATH.to_string()); + ret = PathBuf::from(format!("{}{}", WASMER_TARGET_PATH2, LIBWASMER_FILENAME)); } if !ret.exists() { panic!("Could not find libwasmer path! {:?}", ret); @@ -81,10 +56,11 @@ pub fn get_libwasmer_path() -> PathBuf { /// Get the path to the `wasmer` executable to be used in this test. pub fn get_wasmer_path() -> PathBuf { let mut ret = PathBuf::from( - env::var("WASMER_TEST_WASMER_PATH").unwrap_or_else(|_| WASMER_PATH.to_string()), + env::var("WASMER_TEST_WASMER_PATH") + .unwrap_or_else(|_| format!("{}wasmer", WASMER_TARGET_PATH)), ); if !ret.exists() { - ret = PathBuf::from(WASMER_TARGET_PATH.to_string()); + ret = PathBuf::from(format!("{}wasmer", WASMER_TARGET_PATH2)); } if !ret.exists() { panic!("Could not find wasmer executable path! {:?}", ret); diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 45e1901f0..bb97b67ca 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -2,7 +2,7 @@ use anyhow::bail; use std::process::Command; -use wasmer_integration_tests_cli::{ASSET_PATH, C_ASSET_PATH, WASMER_PATH}; +use wasmer_integration_tests_cli::{get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; fn wasi_test_wasm_path() -> String { format!("{}/{}", C_ASSET_PATH, "qjs.wasm") @@ -18,7 +18,7 @@ fn test_no_start_wat_path() -> String { #[test] fn run_wasi_works() -> anyhow::Result<()> { - let output = Command::new(WASMER_PATH) + let output = Command::new(get_wasmer_path()) .arg("run") .arg(wasi_test_wasm_path()) .arg("--") @@ -44,7 +44,7 @@ fn run_wasi_works() -> anyhow::Result<()> { #[test] fn run_no_imports_wasm_works() -> anyhow::Result<()> { - let output = Command::new(WASMER_PATH) + let output = Command::new(get_wasmer_path()) .arg("run") .arg(test_no_imports_wat_path()) .output()?; @@ -82,7 +82,7 @@ fn run_invoke_works_with_nomain_wasi() -> anyhow::Result<()> { let random = rand::random::(); let module_file = std::env::temp_dir().join(&format!("{random}.wat")); std::fs::write(&module_file, wasi_wat.as_bytes()).unwrap(); - let output = Command::new(WASMER_PATH) + let output = Command::new(get_wasmer_path()) .arg("run") .arg(&module_file) .output()?; @@ -94,7 +94,7 @@ fn run_invoke_works_with_nomain_wasi() -> anyhow::Result<()> { panic!(); } - let output = Command::new(WASMER_PATH) + let output = Command::new(get_wasmer_path()) .arg("run") .arg("--invoke") .arg("_start") @@ -114,7 +114,7 @@ fn run_invoke_works_with_nomain_wasi() -> anyhow::Result<()> { #[test] fn run_no_start_wasm_report_error() -> anyhow::Result<()> { - let output = Command::new(WASMER_PATH) + let output = Command::new(get_wasmer_path()) .arg("run") .arg(test_no_start_wat_path()) .output()?; diff --git a/tests/integration/cli/tests/version.rs b/tests/integration/cli/tests/version.rs index 543237c23..f304749d7 100644 --- a/tests/integration/cli/tests/version.rs +++ b/tests/integration/cli/tests/version.rs @@ -1,16 +1,17 @@ use anyhow::bail; use std::process::Command; -use wasmer_integration_tests_cli::WASMER_PATH; +use wasmer_integration_tests_cli::get_wasmer_path; const WASMER_VERSION: &str = env!("CARGO_PKG_VERSION"); #[test] fn version_string_is_correct() -> anyhow::Result<()> { let expected_version_output = format!("wasmer {}\n", WASMER_VERSION); + let wasmer_path = get_wasmer_path(); let outputs = [ - Command::new(WASMER_PATH).arg("--version").output()?, - Command::new(WASMER_PATH).arg("-V").output()?, + Command::new(&wasmer_path).arg("--version").output()?, + Command::new(&wasmer_path).arg("-V").output()?, ]; for output in &outputs { @@ -34,10 +35,11 @@ fn version_string_is_correct() -> anyhow::Result<()> { #[test] fn help_text_contains_version() -> anyhow::Result<()> { let expected_version_output = format!("wasmer {}", WASMER_VERSION); + let wasmer_path = get_wasmer_path(); let outputs = [ - Command::new(WASMER_PATH).arg("--help").output()?, - Command::new(WASMER_PATH).arg("-h").output()?, + Command::new(&wasmer_path).arg("--help").output()?, + Command::new(&wasmer_path).arg("-h").output()?, ]; for output in &outputs { From 0352403c932d8c5147d2536296276974a4eb5a9e Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Thu, 18 Aug 2022 14:29:28 +0300 Subject: [PATCH 10/40] Add CLI integration tests in CI test workflow --- .github/workflows/test-sys.yaml | 18 ++++++++++++++++++ Makefile | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index 1a1267f4a..ae7089335 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -199,6 +199,14 @@ jobs: '${{ runner.tool_cache }}/cargo-sccache/bin/sccache' -s echo 'RUSTC_WRAPPER=${{ runner.tool_cache }}/cargo-sccache/bin/sccache' >> $GITHUB_ENV shell: bash + - name: Test + if: matrix.run_test && matrix.os != 'windows-2019' + run: | + make + env: + TARGET: ${{ matrix.target }} + TARGET_DIR: target/${{ matrix.target }}/release + CARGO_TARGET: --target ${{ matrix.target }} - name: Test if: matrix.run_test && matrix.os != 'windows-2019' run: | @@ -215,6 +223,16 @@ jobs: TARGET: ${{ matrix.target }} TARGET_DIR: target/${{ matrix.target }}/release CARGO_TARGET: --target ${{ matrix.target }} + - name: Test integration CLI + if: matrix.run_test && matrix.os != 'windows-2019' + run: | + make && make build-capi && make package-capi && make package + export WASMER_DIR=`pwd`/package + make test-integration-cli + env: + TARGET: ${{ matrix.target }} + TARGET_DIR: target/${{ matrix.target }}/release + CARGO_TARGET: --target ${{ matrix.target }} - name: Test if: matrix.run_test && matrix.os == 'windows-2019' run: | diff --git a/Makefile b/Makefile index 67bff7ec6..810677016 100644 --- a/Makefile +++ b/Makefile @@ -534,7 +534,7 @@ test-examples: $(CARGO_BINARY) test $(CARGO_TARGET) $(compiler_features) --features wasi --examples $(CARGO_BINARY) test $(CARGO_TARGET) --release $(compiler_features) --features wasi --examples -test-integration: +test-integration-cli: $(CARGO_BINARY) test $(CARGO_TARGET) --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture test-integration-ios: From e6e06a5b086ed5ccb458f205252fc80be185e284 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Thu, 18 Aug 2022 14:54:42 +0300 Subject: [PATCH 11/40] tests/integration/cli: add tests for create-{exe,obj} & --object-format flag Test that serialized and symbol formats work both in create-exe and create-obj. --- tests/integration/cli/tests/create_exe.rs | 173 +++++++++++++++++++++- 1 file changed, 171 insertions(+), 2 deletions(-) diff --git a/tests/integration/cli/tests/create_exe.rs b/tests/integration/cli/tests/create_exe.rs index abb213c58..a550e7294 100644 --- a/tests/integration/cli/tests/create_exe.rs +++ b/tests/integration/cli/tests/create_exe.rs @@ -26,6 +26,8 @@ struct WasmerCreateExe { native_executable_path: PathBuf, /// Compiler with which to compile the Wasm. compiler: Compiler, + /// Extra CLI flags + extra_cli_flags: Vec<&'static str>, } impl Default for WasmerCreateExe { @@ -40,17 +42,19 @@ impl Default for WasmerCreateExe { wasm_path: PathBuf::from(create_exe_test_wasm_path()), native_executable_path, compiler: Compiler::Cranelift, + extra_cli_flags: vec![], } } } impl WasmerCreateExe { - fn run(&self) -> anyhow::Result<()> { + fn run(&self) -> anyhow::Result> { let output = Command::new(&self.wasmer_path) .current_dir(&self.current_dir) .arg("create-exe") .arg(&self.wasm_path.canonicalize()?) .arg(&self.compiler.to_flag()) + .args(self.extra_cli_flags.iter()) .arg("-o") .arg(&self.native_executable_path) .output()?; @@ -64,7 +68,66 @@ impl WasmerCreateExe { .expect("stderr is not utf8! need to handle arbitrary bytes") ); } - Ok(()) + Ok(output.stdout) + } +} + +/// Data used to run the `wasmer compile` command. +#[derive(Debug)] +struct WasmerCreateObj { + /// The directory to operate in. + current_dir: PathBuf, + /// Path to wasmer executable used to run the command. + wasmer_path: PathBuf, + /// Path to the Wasm file to compile. + wasm_path: PathBuf, + /// Path to the object file produced by compiling the Wasm. + output_object_path: PathBuf, + /// Compiler with which to compile the Wasm. + compiler: Compiler, + /// Extra CLI flags + extra_cli_flags: Vec<&'static str>, +} + +impl Default for WasmerCreateObj { + fn default() -> Self { + #[cfg(not(windows))] + let output_object_path = PathBuf::from("wasm.o"); + #[cfg(windows)] + let output_object_path = PathBuf::from("wasm.obj"); + Self { + current_dir: std::env::current_dir().unwrap(), + wasmer_path: get_wasmer_path(), + wasm_path: PathBuf::from(create_exe_test_wasm_path()), + output_object_path, + compiler: Compiler::Cranelift, + extra_cli_flags: vec![], + } + } +} + +impl WasmerCreateObj { + fn run(&self) -> anyhow::Result> { + let output = Command::new(&self.wasmer_path) + .current_dir(&self.current_dir) + .arg("create-obj") + .arg(&self.wasm_path.canonicalize()?) + .arg(&self.compiler.to_flag()) + .args(self.extra_cli_flags.iter()) + .arg("-o") + .arg(&self.output_object_path) + .output()?; + + if !output.status.success() { + bail!( + "wasmer create-obj failed with: stdout: {}\n\nstderr: {}", + std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"), + std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes") + ); + } + Ok(output.stdout) } } @@ -160,3 +223,109 @@ fn create_exe_works_with_file() -> anyhow::Result<()> { Ok(()) } + +#[test] +fn create_exe_serialized_works() -> anyhow::Result<()> { + let temp_dir = tempfile::tempdir()?; + let operating_dir: PathBuf = temp_dir.path().to_owned(); + + let wasm_path = operating_dir.join(create_exe_test_wasm_path()); + #[cfg(not(windows))] + let executable_path = operating_dir.join("wasm.out"); + #[cfg(windows)] + let executable_path = operating_dir.join("wasm.exe"); + + let output: Vec = WasmerCreateExe { + current_dir: operating_dir.clone(), + wasm_path, + native_executable_path: executable_path.clone(), + compiler: Compiler::Cranelift, + extra_cli_flags: vec!["--object-format", "serialized"], + ..Default::default() + } + .run() + .context("Failed to create-exe wasm with Wasmer")?; + + let result = run_code( + &operating_dir, + &executable_path, + &["--eval".to_string(), "function greet(name) { return JSON.stringify('Hello, ' + name); }; print(greet('World'));".to_string()], + ) + .context("Failed to run generated executable")?; + let result_lines = result.lines().collect::>(); + assert_eq!(result_lines, vec!["\"Hello, World\""],); + + let output_str = String::from_utf8_lossy(&output); + assert!( + output_str.contains("Serialized"), + "create-exe output doesn't mention `serialized` format keyword:\n{}", + output_str + ); + + Ok(()) +} + +fn create_obj(args: Vec<&'static str>, keyword_needle: &str, keyword: &str) -> anyhow::Result<()> { + let temp_dir = tempfile::tempdir()?; + let operating_dir: PathBuf = temp_dir.path().to_owned(); + + let wasm_path = operating_dir.join(create_exe_test_wasm_path()); + + #[cfg(not(windows))] + let object_path = operating_dir.join("wasm.o"); + #[cfg(windows)] + let object_path = operating_dir.join("wasm.obj"); + + let output: Vec = WasmerCreateObj { + current_dir: operating_dir.clone(), + wasm_path, + output_object_path: object_path.clone(), + compiler: Compiler::Cranelift, + extra_cli_flags: args, + ..Default::default() + } + .run() + .context("Failed to create-obj wasm with Wasmer")?; + + assert!( + object_path.exists(), + "create-obj successfully completed but object output file `{}` missing", + object_path.display() + ); + let mut object_header_path = object_path.clone(); + object_header_path.set_extension("h"); + assert!( + object_header_path.exists(), + "create-obj successfully completed but object output header file `{}` missing", + object_header_path.display() + ); + + let output_str = String::from_utf8_lossy(&output); + assert!( + output_str.contains(keyword_needle), + "create-obj output doesn't mention `{}` format keyword:\n{}", + keyword, + output_str + ); + + Ok(()) +} + +#[test] +fn create_obj_default() -> anyhow::Result<()> { + create_obj(vec![], "Symbols", "symbols") +} + +#[test] +fn create_obj_symbols() -> anyhow::Result<()> { + create_obj(vec!["--object-format", "symbols"], "Symbols", "symbols") +} + +#[test] +fn create_obj_serialized() -> anyhow::Result<()> { + create_obj( + vec!["--object-format", "serialized"], + "Serialized", + "serialized", + ) +} From 9da3d515c460341d97afd21c49e6e81caa63a82d Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Thu, 18 Aug 2022 14:57:51 +0300 Subject: [PATCH 12/40] tests/integration/cli: test that giving object as input to create-exe works --- tests/integration/cli/tests/create_exe.rs | 78 +++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/integration/cli/tests/create_exe.rs b/tests/integration/cli/tests/create_exe.rs index a550e7294..45969f7c6 100644 --- a/tests/integration/cli/tests/create_exe.rs +++ b/tests/integration/cli/tests/create_exe.rs @@ -329,3 +329,81 @@ fn create_obj_serialized() -> anyhow::Result<()> { "serialized", ) } + +fn create_exe_with_object_input(args: Vec<&'static str>) -> anyhow::Result<()> { + let temp_dir = tempfile::tempdir()?; + let operating_dir: PathBuf = temp_dir.path().to_owned(); + + let wasm_path = operating_dir.join(create_exe_test_wasm_path()); + + #[cfg(not(windows))] + let object_path = operating_dir.join("wasm.o"); + #[cfg(windows)] + let object_path = operating_dir.join("wasm.obj"); + + WasmerCreateObj { + current_dir: operating_dir.clone(), + wasm_path, + output_object_path: object_path.clone(), + compiler: Compiler::Cranelift, + extra_cli_flags: args, + ..Default::default() + } + .run() + .context("Failed to create-obj wasm with Wasmer")?; + + assert!( + object_path.exists(), + "create-obj successfully completed but object output file `{}` missing", + object_path.display() + ); + let mut object_header_path = object_path.clone(); + object_header_path.set_extension("h"); + assert!( + object_header_path.exists(), + "create-obj successfully completed but object output header file `{}` missing", + object_header_path.display() + ); + + #[cfg(not(windows))] + let executable_path = operating_dir.join("wasm.out"); + #[cfg(windows)] + let executable_path = operating_dir.join("wasm.exe"); + + WasmerCreateExe { + current_dir: operating_dir.clone(), + wasm_path: object_path, + native_executable_path: executable_path.clone(), + compiler: Compiler::Cranelift, + extra_cli_flags: vec!["--header", "wasm.h"], + ..Default::default() + } + .run() + .context("Failed to create-exe wasm with Wasmer")?; + + let result = run_code( + &operating_dir, + &executable_path, + &["--eval".to_string(), "function greet(name) { return JSON.stringify('Hello, ' + name); }; print(greet('World'));".to_string()], + ) + .context("Failed to run generated executable")?; + let result_lines = result.lines().collect::>(); + assert_eq!(result_lines, vec!["\"Hello, World\""],); + + Ok(()) +} + +#[test] +fn create_exe_with_object_input_default() -> anyhow::Result<()> { + create_exe_with_object_input(vec![]) +} + +#[test] +fn create_exe_with_object_input_symbols() -> anyhow::Result<()> { + create_exe_with_object_input(vec!["--object-format", "symbols"]) +} + +#[test] +fn create_exe_with_object_input_serialized() -> anyhow::Result<()> { + create_exe_with_object_input(vec!["--object-format", "serialized"]) +} From bc9ed1a5c1d24adb01fe80bbfabf5aea82c31689 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Sat, 20 Aug 2022 21:55:28 +0300 Subject: [PATCH 13/40] create-exe: use absolute paths everywhere --- lib/cli/src/commands/create_exe.rs | 135 +++++++++++++++++------------ 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 470dd96a6..ff0d20f90 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -39,6 +39,7 @@ struct CrossCompileSetup { target: Triple, zig_binary_path: PathBuf, library: PathBuf, + working_dir: PathBuf, } #[derive(Debug, Parser)] @@ -110,6 +111,11 @@ pub struct CreateExe { impl CreateExe { /// Runs logic for the `compile` subcommand pub fn execute(&self) -> Result<()> { + let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); + let working_dir = tempfile::tempdir()?; + let starting_cd = env::current_dir()?; + let output_path = starting_cd.join(&self.output); + /* Making library_path, tarball zig_binary_path flags require that target_triple flag * is set cannot be encoded with structopt, so we have to perform cli flag validation * manually here */ @@ -149,10 +155,6 @@ impl CreateExe { }) .unwrap_or_default(); - let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); - let working_dir = tempfile::tempdir()?; - let starting_cd = env::current_dir()?; - let output_path = starting_cd.join(&self.output); env::set_current_dir(&working_dir)?; let cross_compilation: Option = if let Some(mut cross_subc) = @@ -255,6 +257,7 @@ impl CreateExe { target, zig_binary_path, library, + working_dir: working_dir.path().to_path_buf(), }) } else { None @@ -267,30 +270,37 @@ impl CreateExe { println!("Format: {:?}", object_format); #[cfg(not(windows))] - let wasm_object_path = PathBuf::from("wasm.o"); + let wasm_object_path = working_dir.path().join("wasm.o"); #[cfg(windows)] - let wasm_object_path = PathBuf::from("wasm.obj"); + let wasm_object_path = working_dir.path().join("wasm.obj"); let wasm_module_path = starting_cd.join(&self.path); + let static_defs_header_path: PathBuf = working_dir.path().join("static_defs.h"); + if let Some(header_path) = self.header.as_ref() { /* In this case, since a header file is given, the input file is expected to be an * object created with `create-obj` subcommand */ let header_path = starting_cd.join(&header_path); - std::fs::copy(&header_path, Path::new("static_defs.h")) + std::fs::copy(&header_path, &static_defs_header_path) .context("Could not access given header file")?; + let object_file_path = wasm_module_path; if let Some(setup) = cross_compilation.as_ref() { self.compile_zig( output_path, - wasm_module_path, - std::path::Path::new("static_defs.h").into(), + object_file_path, + static_defs_header_path, setup, )?; } else { self.link( - output_path, - wasm_module_path, - std::path::Path::new("static_defs.h").into(), + static_defs_header_path, + LinkCode { + object_paths: vec![object_file_path, "main_obj.obj".into()], + output_path, + working_dir: working_dir.path().to_path_buf(), + ..Default::default() + }, )?; } } else { @@ -308,17 +318,17 @@ impl CreateExe { drop(writer); // Write down header file that includes deserialize function { - let mut writer = BufWriter::new(File::create("static_defs.h")?); + let mut writer = BufWriter::new(File::create(&static_defs_header_path)?); writer.write_all(WASMER_DESERIALIZE_HEADER.as_bytes())?; writer.flush()?; } // write C src to disk - let c_src_path = Path::new("wasmer_main.c"); + let c_src_path: PathBuf = working_dir.path().join("wasmer_main.c"); #[cfg(not(windows))] - let c_src_obj = PathBuf::from("wasmer_main.o"); + let c_src_obj: PathBuf = working_dir.path().join("wasmer_main.o"); #[cfg(windows)] - let c_src_obj = PathBuf::from("wasmer_main.obj"); + let c_src_obj: PathBuf = working_dir.path().join("wasmer_main.obj"); { let mut c_src_file = fs::OpenOptions::new() @@ -328,8 +338,13 @@ impl CreateExe { .context("Failed to open C source code file")?; c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; } - run_c_compile(c_src_path, &c_src_obj, self.target_triple.clone()) - .context("Failed to compile C source code")?; + run_c_compile( + &c_src_path, + &c_src_obj, + static_defs_header_path, + self.target_triple.clone(), + ) + .context("Failed to compile C source code")?; LinkCode { object_paths: vec![c_src_obj, wasm_object_path], output_path, @@ -365,27 +380,31 @@ impl CreateExe { ); // Write object file with functions let object_file_path: std::path::PathBuf = - std::path::Path::new("functions.o").into(); + working_dir.path().join("functions.o"); let mut writer = BufWriter::new(File::create(&object_file_path)?); obj.write_stream(&mut writer) .map_err(|err| anyhow::anyhow!(err.to_string()))?; writer.flush()?; // Write down header file that includes pointer arrays and the deserialize function - let mut writer = BufWriter::new(File::create("static_defs.h")?); + let mut writer = BufWriter::new(File::create(&static_defs_header_path)?); writer.write_all(header_file_src.as_bytes())?; writer.flush()?; if let Some(setup) = cross_compilation.as_ref() { self.compile_zig( output_path, object_file_path, - std::path::Path::new("static_defs.h").into(), + static_defs_header_path, setup, )?; } else { self.link( - output_path, - object_file_path, - std::path::Path::new("static_defs.h").into(), + static_defs_header_path, + LinkCode { + object_paths: vec![object_file_path, "main_obj.obj".into()], + output_path, + working_dir: working_dir.path().to_path_buf(), + ..Default::default() + }, )?; } } @@ -412,15 +431,21 @@ impl CreateExe { &self, output_path: PathBuf, object_path: PathBuf, - mut header_code_path: PathBuf, + mut header_path: PathBuf, setup: &CrossCompileSetup, ) -> anyhow::Result<()> { - let c_src_path = Path::new("wasmer_main.c"); + debug_assert!( + header_path.is_absolute(), + "compile_zig() called with relative header file path {}", + header_path.display() + ); let CrossCompileSetup { ref target, ref zig_binary_path, ref library, + ref working_dir, } = setup; + let c_src_path = working_dir.join("wasmer_main.c"); let mut libwasmer_path = library.to_path_buf(); println!("Library Path: {}", libwasmer_path.display()); @@ -445,12 +470,8 @@ impl CreateExe { c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; } - if !header_code_path.is_dir() { - header_code_path.pop(); - } - - if header_code_path.display().to_string().is_empty() { - header_code_path = std::env::current_dir()?; + if !header_path.is_dir() { + header_path.pop(); } /* Compile main function */ @@ -471,7 +492,7 @@ impl CreateExe { .arg(&format!("-L{}", libwasmer_path.display())) .arg(&format!("-l:{}", lib_filename)) .arg(&format!("-I{}", include_dir.display())) - .arg(&format!("-I{}", header_code_path.display())); + .arg(&format!("-I{}", header_path.display())); if !zig_triple.contains("windows") { cmd_mut = cmd_mut.arg("-lunwind"); } @@ -493,18 +514,13 @@ impl CreateExe { } #[cfg(feature = "static-artifact-create")] - fn link( - &self, - output_path: PathBuf, - object_path: PathBuf, - mut header_code_path: PathBuf, - ) -> anyhow::Result<()> { - let linkcode = LinkCode { - object_paths: vec![object_path, "main_obj.obj".into()], - output_path, - ..Default::default() - }; - let c_src_path = Path::new("wasmer_main.c"); + fn link(&self, mut header_path: PathBuf, linkcode: LinkCode) -> anyhow::Result<()> { + debug_assert!( + header_path.is_absolute(), + "link() called with relative header file path {}", + header_path.display() + ); + let c_src_path: PathBuf = linkcode.working_dir.join("wasmer_main.c"); let mut libwasmer_path = get_libwasmer_path()? .canonicalize() .context("Failed to find libwasmer")?; @@ -527,12 +543,8 @@ impl CreateExe { c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; } - if !header_code_path.is_dir() { - header_code_path.pop(); - } - - if header_code_path.display().to_string().is_empty() { - header_code_path = std::env::current_dir()?; + if !header_path.is_dir() { + header_path.pop(); } /* Compile main function */ @@ -561,7 +573,7 @@ impl CreateExe { .arg("-ldl") .arg("-lm") .arg("-pthread") - .arg(&format!("-I{}", header_code_path.display())) + .arg(&format!("-I{}", header_path.display())) .arg("-v") .arg("-o") .arg("main_obj.obj") @@ -645,8 +657,15 @@ fn get_libwasmer_cache_path() -> anyhow::Result { fn run_c_compile( path_to_c_src: &Path, output_name: &Path, + mut header_path: PathBuf, target: Option, ) -> anyhow::Result<()> { + debug_assert!( + header_path.is_absolute(), + "run_c_compile() called with relative header file path {}", + header_path.display() + ); + #[cfg(not(windows))] let c_compiler = "cc"; // We must use a C++ compiler on Windows because wasm.h uses `static_assert` @@ -654,14 +673,17 @@ fn run_c_compile( #[cfg(windows)] let c_compiler = "clang++"; + if !header_path.is_dir() { + header_path.pop(); + } + let mut command = Command::new(c_compiler); let command = command .arg("-O2") .arg("-c") .arg(path_to_c_src) - .arg("-I.") - .arg("-I") - .arg(get_wasmer_include_directory()?); + .arg(&format!("-I{}", header_path.display())) + .arg(&format!("-I{}", get_wasmer_include_directory()?.display())); let command = if let Some(target) = target { command.arg("-target").arg(format!("{}", target)) @@ -700,6 +722,8 @@ struct LinkCode { libwasmer_path: PathBuf, /// The target to link the executable for. target: Option, + /// Working directory + working_dir: PathBuf, } impl Default for LinkCode { @@ -716,6 +740,7 @@ impl Default for LinkCode { output_path: PathBuf::from("a.out"), libwasmer_path: get_libwasmer_path().unwrap(), target: None, + working_dir: env::current_dir().expect("could not get current dir from environment"), } } } From 4fdbd344af18f625eef1009528cf7f06a5c917b0 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Sat, 20 Aug 2022 22:04:37 +0300 Subject: [PATCH 14/40] create-obj: remove dead code --- lib/cli/src/commands/create_obj.rs | 63 +----------------------------- 1 file changed, 1 insertion(+), 62 deletions(-) diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs index c8e3534ba..79995fd6f 100644 --- a/lib/cli/src/commands/create_obj.rs +++ b/lib/cli/src/commands/create_obj.rs @@ -1,5 +1,4 @@ -#![allow(dead_code)] -//! Create a standalone native executable for a given Wasm file. +//! Create a compiled standalone object file for a given Wasm file. use super::ObjectFormat; use crate::{commands::PrefixerFn, store::CompilerOptions}; @@ -11,7 +10,6 @@ use std::fs::File; use std::io::prelude::*; use std::io::BufWriter; use std::path::PathBuf; -use std::process::Command; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; @@ -166,62 +164,3 @@ impl CreateObj { Ok(()) } } -fn link( - output_path: PathBuf, - object_path: PathBuf, - header_code_path: PathBuf, -) -> anyhow::Result<()> { - let libwasmer_path = get_libwasmer_path()? - .canonicalize() - .context("Failed to find libwasmer")?; - println!( - "link output {:?}", - Command::new("cc") - .arg(&header_code_path) - .arg(&format!("-L{}", libwasmer_path.display())) - //.arg(&format!("-I{}", header_code_path.display())) - .arg("-pie") - .arg("-o") - .arg("header_obj.o") - .output()? - ); - //ld -relocatable a.o b.o -o c.o - - println!( - "link output {:?}", - Command::new("ld") - .arg("-relocatable") - .arg(&object_path) - .arg("header_obj.o") - .arg("-o") - .arg(&output_path) - .output()? - ); - - Ok(()) -} - -/// path to the static libwasmer -fn get_libwasmer_path() -> anyhow::Result { - let mut path = get_wasmer_dir()?; - path.push("lib"); - - // TODO: prefer headless Wasmer if/when it's a separate library. - #[cfg(not(windows))] - path.push("libwasmer.a"); - #[cfg(windows)] - path.push("wasmer.lib"); - - Ok(path) -} -fn get_wasmer_dir() -> anyhow::Result { - Ok(PathBuf::from( - env::var("WASMER_DIR") - .or_else(|e| { - option_env!("WASMER_INSTALL_PREFIX") - .map(str::to_string) - .ok_or(e) - }) - .context("Trying to read env var `WASMER_DIR`")?, - )) -} From 12eea421877279c5b4728bb0b03d2e3ab248fc7c Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Fri, 19 Aug 2022 18:13:39 +0200 Subject: [PATCH 15/40] Added some test for LinearMemory trait --- lib/vm/src/memory.rs | 101 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 1 deletion(-) diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index a82c48295..a127b701d 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -376,7 +376,6 @@ impl VMMemory { /// Creates VMMemory from a custom implementation - the following into implementations /// are natively supported /// - VMOwnedMemory -> VMMemory - /// - VMSharedMemory -> VMMemory /// - Box -> VMMemory pub fn from_custom(memory: IntoVMMemory) -> VMMemory where @@ -463,3 +462,103 @@ mod test_vmmemory_definition { ); } } + +#[cfg(test)] +mod test_linearmemory { + use super::LinearMemory; + use super::VMMemoryDefinition; + use crate::store::MaybeInstanceOwned; + use crate::VMMemory; + use std::cell::UnsafeCell; + use std::ptr::read_unaligned; + use std::ptr::write_unaligned; + use std::ptr::NonNull; + use wasmer_types::{MemoryError, MemoryStyle, MemoryType, Pages, WASM_PAGE_SIZE}; + + #[derive(Debug)] + struct VMTinyMemory { + mem: [u8; WASM_PAGE_SIZE], + } + + unsafe impl Send for VMTinyMemory {} + unsafe impl Sync for VMTinyMemory {} + + impl VMTinyMemory { + pub fn new() -> Result { + Ok(VMTinyMemory { + mem: [0; WASM_PAGE_SIZE], + }) + } + } + + impl LinearMemory for VMTinyMemory { + fn ty(&self) -> MemoryType { + MemoryType { + minimum: Pages::from(1u32), + maximum: Some(Pages::from(1u32)), + shared: false, + } + } + fn size(&self) -> Pages { + Pages::from(1u32) + } + fn style(&self) -> MemoryStyle { + MemoryStyle::Static { + bound: Pages::from(1u32), + offset_guard_size: 0, + } + } + fn grow(&mut self, delta: Pages) -> Result { + Err(MemoryError::CouldNotGrow { + current: Pages::from(1u32), + attempted_delta: delta, + }) + } + fn vmmemory(&self) -> NonNull { + MaybeInstanceOwned::Host(Box::new(UnsafeCell::new(VMMemoryDefinition { + base: self.mem.as_ptr() as _, + current_length: WASM_PAGE_SIZE, + }))) + .as_ptr() + } + fn try_clone(&self) -> Option> { + None + } + } + + impl From for VMMemory { + fn from(mem: VMTinyMemory) -> Self { + Self(Box::new(mem)) + } + } + + #[test] + fn check_linearmemory() { + let mut memory = VMTinyMemory::new().unwrap(); + assert!(memory.grow(Pages::from(2u32)).is_err()); + assert_eq!(memory.size(), Pages::from(1u32)); + + let vmemdef = memory.vmmemory(); + let raw_ptr: *mut u8 = unsafe { vmemdef.as_ref().base }; + unsafe { + write_unaligned(raw_ptr, 1); + assert_eq!(read_unaligned(raw_ptr), 1); + write_unaligned(raw_ptr.add(100), 200); + assert_eq!(read_unaligned(raw_ptr.add(100)), 200); + } + // re-borrow + let vmemdef = memory.vmmemory(); + let raw_ptr: *mut u8 = unsafe { vmemdef.as_ref().base }; + unsafe { + assert_eq!(read_unaligned(raw_ptr), 1); + } + // borrow as VMMemory + let vmmemory: VMMemory = VMMemory::from_custom(memory); + assert_eq!(vmmemory.size(), Pages::from(1u32)); + let raw_ptr = unsafe { vmmemory.vmmemory().as_ref().base }; + unsafe { + assert_eq!(read_unaligned(raw_ptr), 1); + assert_eq!(read_unaligned(raw_ptr.add(100)), 200); + } + } +} From 54b0e5c2ff4fe95d95b6f4da95b6b4f12637a549 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Mon, 22 Aug 2022 15:17:36 +0300 Subject: [PATCH 16/40] scripts/publish.py: replace toposort dependency with python std graphlib module Python standard library already includes a topological sorter: https://docs.python.org/3/library/graphlib.html We don't need a third party dependency in scripts/publish.py. Closes #2965 --- scripts/publish.py | 439 +++++++++++++++++++++++++++++---------------- 1 file changed, 283 insertions(+), 156 deletions(-) diff --git a/scripts/publish.py b/scripts/publish.py index 9afcafc07..3076bf413 100644 --- a/scripts/publish.py +++ b/scripts/publish.py @@ -1,199 +1,326 @@ #! /usr/bin/env python3 -# This is a script for publishing the wasmer crates to crates.io. -# It should be run in the root of wasmer like `python3 scripts/publish.py --no-dry-run`. -# By default the script executes a test run and does not publish the crates to crates.io. - -# install dependencies: -# pip3 install toposort +"""This is a script for publishing the wasmer crates to crates.io. +It should be run in the root of wasmer like `python3 scripts/publish.py --help`.""" import argparse +import itertools import os import re import subprocess import time - -from typing import Optional +import sys +import typing +from pprint import pprint try: - from toposort import toposort_flatten + # try import tomllib from standard Python library: New in version 3.11. + import tomllib +except ModuleNotFoundError: + try: + # if tomllib is missing, this is an older python3 version. Try + # importing the equivalent third-party library tomli: + import tomli as tomllib + except ModuleNotFoundError: + print("Please install tomli, `pip3 install tomli`") + sys.exit(1) + +try: + from graphlib import TopologicalSorter except ImportError: - print("Please install toposort, `pip3 install toposort`") + print("Incompatible python3 version: lacks graphlib standard library module.") + sys.exit(1) - -# TODO: find this automatically -target_version = "3.0.0-beta" - -# TODO: generate this by parsing toml files -dep_graph = { - "wasmer-types": set([]), - "wasmer-derive": set([]), - "wasmer-vm": set(["wasmer-types"]), - "wasmer-compiler": set(["wasmer-types"]), - "wasmer-compiler-singlepass": set(["wasmer-types", "wasmer-compiler"]), - "wasmer-compiler-cranelift": set(["wasmer-types", "wasmer-compiler"]), - "wasmer-compiler-cli": set( - [ - "wasmer-compiler", - "wasmer-types", - "wasmer-compiler-singlepass", - "wasmer-compiler-cranelift", - ] - ), - "wasmer-object": set(["wasmer-types", "wasmer-compiler"]), - "wasmer-compiler-llvm": set(["wasmer-compiler", "wasmer-vm", "wasmer-types"]), - "wasmer": set( - [ - "wasmer-vm", - "wasmer-compiler", - "wasmer-derive", - "wasmer-types", - "wasmer-compiler-singlepass", - "wasmer-compiler-cranelift", - "wasmer-compiler-llvm", - ] - ), - "wasmer-vfs": set([]), - "wasmer-vbus": set([]), - "wasmer-vnet": set([]), - "wasmer-wasi-local-networking": set([]), - "wasmer-cache": set(["wasmer"]), - "wasmer-wasi": set(["wasmer", "wasmer-wasi-types", "wasmer-vfs", "wasmer-vbus", "wasmer-vnet"]), - "wasmer-wasi-types": set(["wasmer-types"]), - "wasmer-wasi-experimental-io-devices": set(["wasmer-wasi"]), - "wasmer-emscripten": set(["wasmer"]), - "wasmer-c-api": set( - [ - "wasmer", - "wasmer-compiler", - "wasmer-compiler-cranelift", - "wasmer-compiler-singlepass", - "wasmer-compiler-llvm", - "wasmer-emscripten", - "wasmer-middlewares", - "wasmer-wasi", - "wasmer-types", - ] - ), - "wasmer-middlewares": set(["wasmer", "wasmer-types", "wasmer-vm"]), - "wasmer-wast": set(["wasmer", "wasmer-wasi", "wasmer-vfs"]), - "wasmer-cli": set( - [ - "wasmer", - "wasmer-compiler", - "wasmer-compiler-cranelift", - "wasmer-compiler-singlepass", - "wasmer-compiler-llvm", - "wasmer-emscripten", - "wasmer-vm", - "wasmer-wasi", - "wasmer-wasi-experimental-io-devices", - "wasmer-wast", - "wasmer-cache", - "wasmer-types", - "wasmer-vfs", - ] - ), +SETTINGS = { + # extra features to put when publishing, for example wasmer-cli needs a + # compiler by default otherwise it won't work standalone + "publish_features": { + "wasmer-cli": "default,cranelift", + }, + # workspace members we want to publish but whose path doesn't start by + # "./lib/" + "non-lib-workspace-members": set(["tests/lib/wast"]), } -# where each crate is located in the `lib` directory -# TODO: this could also be generated from the toml files -location = { - "wasmer-compiler-cli": "cli-compiler", - "wasmer-types": "types", - "wasmer-derive": "derive", - "wasmer-vm": "vm", - "wasmer-compiler": "compiler", - "wasmer-object": "object", - "wasmer-compiler-singlepass": "compiler-singlepass", - "wasmer-compiler-cranelift": "compiler-cranelift", - "wasmer-compiler-llvm": "compiler-llvm", - "wasmer-cache": "cache", - "wasmer": "api", - "wasmer-wasi": "wasi", - "wasmer-wasi-types": "wasi-types", - "wasmer-emscripten": "emscripten", - "wasmer-wasi-experimental-io-devices": "wasi-experimental-io-devices", - "wasmer-c-api": "c-api", - "wasmer-middlewares": "middlewares", - "wasmer-vfs": "vfs", - "wasmer-vbus": "vbus", - "wasmer-vnet": "vnet", - "wasmer-cli": "cli", - "wasmer-wasi-local-networking": "wasi-local-networking", - "wasmer-wast": "../tests/lib/wast", -} -no_dry_run = False - - -def get_latest_version_for_crate(crate_name: str) -> Optional[str]: - output = subprocess.run(["cargo", "search", crate_name], capture_output=True) - rexp_src = '^{} = "([^"]+)"'.format(crate_name) +def get_latest_version_for_crate(crate_name: str) -> typing.Optional[str]: + """Fetches the latest version of a given crate name in local cargo registry.""" + output = subprocess.run( + ["cargo", "search", crate_name], capture_output=True, check=True + ) + rexp_src = f'^{crate_name} = "([^"]+)"' prog = re.compile(rexp_src) haystack = output.stdout.decode("utf-8") for line in haystack.splitlines(): result = prog.match(line) if result: return result.group(1) + return None -def is_crate_already_published(crate_name: str) -> bool: - found_string = get_latest_version_for_crate(crate_name) - if found_string is None: - return False +class Crate: + """Represents a workspace crate that is to be published to crates.io.""" - return target_version == found_string + def __init__( + self, + dependencies: typing.List[str], + cargo_manifest: dict, + cargo_file_path="Cargo.toml", + ): + self.name = cargo_manifest["package"]["name"] + self.dependencies = dependencies + self.cargo_manifest = cargo_manifest + if not os.path.isabs(cargo_file_path): + cargo_file_path = os.path.join(os.getcwd(), cargo_file_path) + self.cargo_file_path = cargo_file_path + + def __str__(self): + return f"{self.name}: {self.dependencies} {self.cargo_file_path} {self.path()}" + + def path(self) -> str: + """Return the absolute filesystem path containing this crate.""" + return os.path.dirname(self.cargo_file_path) -def publish_crate(crate: str): - starting_dir = os.getcwd() - os.chdir("lib/{}".format(location[crate])) +class Publisher: + """A class responsible for figuring out dependencies, + creating a topological sorting in order to publish them + to crates.io in a valid order.""" - global no_dry_run - if no_dry_run: - output = subprocess.run(["cargo", "publish"]) - else: - print("In dry-run: not publishing crate `{}`".format(crate)) - output = subprocess.run(["cargo", "publish", "--dry-run"]) + def __init__(self, version=None, dry_run=True, verbose=True): + self.dry_run = dry_run + self.verbose = verbose - os.chdir(starting_dir) + # open workspace Cargo.toml + with open("Cargo.toml", "rb") as file: + data = tomllib.load(file) + + if version is None: + version = data["package"]["version"] + self.version = version + + if self.verbose and not self.dry_run: + print(f"Publishing version {self.version}") + elif self.verbose and self.dry_run: + print(f"Publishing version {self.version} dry run!") + + # define helper function + check_local_dep_fn = lambda t: isinstance(t[1], dict) and "path" in t[1] + members = set( + map( + lambda p: p + "/Cargo.toml", + filter( + lambda path: ( + path.startswith("lib/") and os.path.exists(path + "/Cargo.toml") + ) + or path in SETTINGS["non-lib-workspace-members"], + itertools.chain( + data["workspace"]["members"], + map( + lambda p: p[1]["path"], + filter(check_local_dep_fn, data["dependencies"].items()), + ), + ), + ), + ) + ) + crates = [] + for member in members: + with open(member, "rb") as file: + member_data = tomllib.load(file) + + def return_dependencies(toml) -> typing.List[str]: + acc = set() + stack = [toml] + while len(stack) > 0: + toml = stack.pop() + if "dependencies" in toml: + acc.update( + list( + map( + lambda dep: dep[1]["package"] + if "package" in dep[1] + else dep[0], + filter( + check_local_dep_fn, toml["dependencies"].items() + ), + ) + ) + ) + if "target" in toml: + stack.append(toml["target"]) + for key, value in toml.items(): + if key.startswith("cfg"): + stack.append(value) + return list(acc) + + dependencies = return_dependencies(member_data) + crates.append(Crate(dependencies, member_data, cargo_file_path=member)) + + self.crates = crates + self.crate_index: typing.Dict[str, Crate] = {c.name: c for c in crates} + + self.create_publish_order() + + self.starting_dir = os.getcwd() + + def create_publish_order(self): + """Creates a valid publish order by topologically + sorting crates using the dependency graph.""" + topological_sorter = TopologicalSorter() + + for crate in self.crates: + topological_sorter.add(crate.name, *crate.dependencies) + + self.publish_order: typing.List[str] = [*topological_sorter.static_order()] + + def is_crate_already_published(self, crate_name: str) -> bool: + """Checks if a given crate name is already published + with the version string we intend to publish with.""" + found_string: str = get_latest_version_for_crate(crate_name) + if found_string is None: + return False + + return self.version == found_string + + def publish_crate(self, crate_name: str): + # pylint: disable=broad-except + """Publish a given crate by name.""" + status = None + try: + crate = self.crate_index[crate_name] + os.chdir(crate.cargo_file_path) + if self.dry_run: + print(f"In dry-run: not publishing crate `{crate_name}`") + output = subprocess.run(["cargo", "publish", "--dry-run"], check=True) + if self.verbose: + print(output) + else: + output = subprocess.run(["cargo", "publish"], check=True) + if self.verbose: + print(output) + if self.verbose: + print("Success.") + except Exception as exc: + if self.verbose: + print(f"Failed to publish {crate_name}.") + print(exc) + status = exc + finally: + os.chdir(self.starting_dir) + return status + + def publish(self): + # pylint: disable=too-many-branches + """Publish all packages in workspace.""" + if self.verbose and self.dry_run: + print("(Dry run, not actually publishing anything)") + if self.verbose: + print("Publishing order:") + pprint(self.publish_order) + status = {} + failures = 0 + + for crate_name in self.publish_order: + print(f"Publishing `{crate_name}`...") + if not self.is_crate_already_published(crate_name): + status[crate_name] = self.publish_crate(crate_name) + if status[crate_name]: + failures = +1 + else: + print(f"`{crate_name}` was already published!") + continue + + # sleep for 16 seconds between crates to ensure the crates.io + # index has time to update + if not self.dry_run: + print( + "Sleeping for 16 seconds to allow the `crates.io` index to update..." + ) + time.sleep(16) + else: + print("In dry-run: not sleeping for crates.io to update.") + if failures > 0 and self.verbose: + print(f"encountered {failures} failures.") + for key, value in status.items(): + if value is None: + result = "ok" + else: + result = str(value) + print(f"{key}\t{result}") + if self.verbose: + print(f"Published {len(status) - failures} crates.") + return failures def main(): + """Main executable function.""" os.environ["WASMER_PUBLISH_SCRIPT_IS_RUNNING"] = "1" parser = argparse.ArgumentParser( description="Publish the Wasmer crates to crates.io" ) - parser.add_argument( - "--no-dry-run", + subparsers = parser.add_subparsers(dest="subcommand") + health_cmd = subparsers.add_parser( + "health-check", + help="""Check the dependency graph is a tree, meaning a non-cyclic + planar graph. Combine with verbosity to print a topological sorting + of the graph.""", + ) + health_cmd.add_argument( + "-v", + "--verbose", default=False, action="store_true", - help="Run the script without actually publishing anything to crates.io", + help="Be verbose.", + ) + health_cmd.add_argument( + "--print-dependencies", + default=False, + action="store_true", + help="For each crate, print its dependencies.", + ) + publish_cmd = subparsers.add_parser( + "publish", help="Publish Wasmer crates to crates.io." + ) + publish_cmd.add_argument( + "--version", + default=None, + type=str, + help="""Define the semver target triple (Default is automatically + read from workspace Cargo.toml.""", + ) + publish_cmd.add_argument( + "--dry-run", + default=False, + action="store_true", + help="""Run the script without actually publishing anything to + crates.io""", + ) + publish_cmd.add_argument( + "-v", + "--verbose", + default=False, + action="store_true", + help="Be verbose.", ) - args = vars(parser.parse_args()) - global no_dry_run - no_dry_run = args["no_dry_run"] + args = parser.parse_args() + if args.subcommand == "health-check": + verbose = args.verbose + publisher = Publisher(verbose=verbose) + if verbose: + print(f"Version is {publisher.version}") + pprint(publisher.publish_order) + if args.print_dependencies: + for crate in publisher.crates: + print(f"{crate.name}: {crate.dependencies}") + return 0 - # get the order to publish the crates in - order = list(toposort_flatten(dep_graph, sort=True)) - - for crate in order: - print("Publishing `{}`...".format(crate)) - if not is_crate_already_published(crate): - publish_crate(crate) - else: - print("`{}` was already published!".format(crate)) - continue - # sleep for 16 seconds between crates to ensure the crates.io index has time to update - # this can be optimized with knowledge of our dep graph via toposort; we can even publish - # crates in parallel; however this is out of scope for the first version of this script - if no_dry_run: - print("Sleeping for 16 seconds to allow the `crates.io` index to update...") - time.sleep(16) - else: - print("In dry-run: not sleeping for crates.io to update.") + if args.subcommand == "publish": + verbose = args.verbose + publisher = Publisher(dry_run=args.dry_run, verbose=verbose) + return publisher.publish() + return 0 if __name__ == "__main__": From 73844bb7d2f4761efb44a98edcdbcd6692c54780 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Mon, 22 Aug 2022 15:01:28 +0200 Subject: [PATCH 17/40] Move LinearMemory test from vm::Memory to Tunables --- lib/api/src/sys/tunables.rs | 79 ++++++++++++++++++++++++++++ lib/vm/src/memory.rs | 102 +----------------------------------- 2 files changed, 80 insertions(+), 101 deletions(-) diff --git a/lib/api/src/sys/tunables.rs b/lib/api/src/sys/tunables.rs index 00663cab7..1be12e59c 100644 --- a/lib/api/src/sys/tunables.rs +++ b/lib/api/src/sys/tunables.rs @@ -134,6 +134,8 @@ impl Tunables for BaseTunables { #[cfg(test)] mod tests { + use crate::sys::tunables; + use super::*; #[test] @@ -174,4 +176,81 @@ mod tests { s => panic!("Unexpected memory style: {:?}", s), } } + + use std::cell::UnsafeCell; + use std::ptr::read_unaligned; + use std::ptr::write_unaligned; + use std::ptr::NonNull; + use wasmer_types::{MemoryError, MemoryStyle, MemoryType, Pages, WASM_PAGE_SIZE}; + use wasmer_vm::{LinearMemory, MaybeInstanceOwned}; + + #[derive(Debug)] + struct VMTinyMemory { + mem: [u8; WASM_PAGE_SIZE], + } + + unsafe impl Send for VMTinyMemory {} + unsafe impl Sync for VMTinyMemory {} + + impl VMTinyMemory { + pub fn new() -> Result { + Ok(VMTinyMemory { + mem: [0; WASM_PAGE_SIZE], + }) + } + } + + impl LinearMemory for VMTinyMemory { + fn ty(&self) -> MemoryType { + MemoryType { + minimum: Pages::from(1u32), + maximum: Some(Pages::from(1u32)), + shared: false, + } + } + fn size(&self) -> Pages { + Pages::from(1u32) + } + fn style(&self) -> MemoryStyle { + MemoryStyle::Static { + bound: Pages::from(1u32), + offset_guard_size: 0, + } + } + fn grow(&mut self, delta: Pages) -> Result { + Err(MemoryError::CouldNotGrow { + current: Pages::from(1u32), + attempted_delta: delta, + }) + } + fn vmmemory(&self) -> NonNull { + MaybeInstanceOwned::Host(Box::new(UnsafeCell::new(VMMemoryDefinition { + base: self.mem.as_ptr() as _, + current_length: WASM_PAGE_SIZE, + }))) + .as_ptr() + } + fn try_clone(&self) -> Option> { + None + } + } + + impl From for wasmer_vm::VMMemory { + fn from(mem: VMTinyMemory) -> Self { + Self(Box::new(mem)) + } + } + + #[test] + fn check_linearmemory() { + let mut memory = VMTinyMemory::new().unwrap(); + assert!(memory.grow(Pages::from(2u32)).is_err()); + assert_eq!(memory.size(), Pages::from(1u32)); + let target = Target::default(); + let tunables = BaseTunables::for_target(&target); + let vmmemory = + unsafe { tunables.create_vm_memory(&memory.ty(), &memory.style(), memory.vmmemory()) }; + let vmmemory = vmmemory.unwrap(); + assert_eq!(vmmemory.size(), Pages::from(1u32)); + } } diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index a127b701d..516da4dee 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -301,7 +301,7 @@ impl From for VMMemory { /// Represents linear memory that can be either owned or shared #[derive(Debug)] -pub struct VMMemory(Box); +pub struct VMMemory(pub Box); impl From> for VMMemory { fn from(mem: Box) -> Self { @@ -462,103 +462,3 @@ mod test_vmmemory_definition { ); } } - -#[cfg(test)] -mod test_linearmemory { - use super::LinearMemory; - use super::VMMemoryDefinition; - use crate::store::MaybeInstanceOwned; - use crate::VMMemory; - use std::cell::UnsafeCell; - use std::ptr::read_unaligned; - use std::ptr::write_unaligned; - use std::ptr::NonNull; - use wasmer_types::{MemoryError, MemoryStyle, MemoryType, Pages, WASM_PAGE_SIZE}; - - #[derive(Debug)] - struct VMTinyMemory { - mem: [u8; WASM_PAGE_SIZE], - } - - unsafe impl Send for VMTinyMemory {} - unsafe impl Sync for VMTinyMemory {} - - impl VMTinyMemory { - pub fn new() -> Result { - Ok(VMTinyMemory { - mem: [0; WASM_PAGE_SIZE], - }) - } - } - - impl LinearMemory for VMTinyMemory { - fn ty(&self) -> MemoryType { - MemoryType { - minimum: Pages::from(1u32), - maximum: Some(Pages::from(1u32)), - shared: false, - } - } - fn size(&self) -> Pages { - Pages::from(1u32) - } - fn style(&self) -> MemoryStyle { - MemoryStyle::Static { - bound: Pages::from(1u32), - offset_guard_size: 0, - } - } - fn grow(&mut self, delta: Pages) -> Result { - Err(MemoryError::CouldNotGrow { - current: Pages::from(1u32), - attempted_delta: delta, - }) - } - fn vmmemory(&self) -> NonNull { - MaybeInstanceOwned::Host(Box::new(UnsafeCell::new(VMMemoryDefinition { - base: self.mem.as_ptr() as _, - current_length: WASM_PAGE_SIZE, - }))) - .as_ptr() - } - fn try_clone(&self) -> Option> { - None - } - } - - impl From for VMMemory { - fn from(mem: VMTinyMemory) -> Self { - Self(Box::new(mem)) - } - } - - #[test] - fn check_linearmemory() { - let mut memory = VMTinyMemory::new().unwrap(); - assert!(memory.grow(Pages::from(2u32)).is_err()); - assert_eq!(memory.size(), Pages::from(1u32)); - - let vmemdef = memory.vmmemory(); - let raw_ptr: *mut u8 = unsafe { vmemdef.as_ref().base }; - unsafe { - write_unaligned(raw_ptr, 1); - assert_eq!(read_unaligned(raw_ptr), 1); - write_unaligned(raw_ptr.add(100), 200); - assert_eq!(read_unaligned(raw_ptr.add(100)), 200); - } - // re-borrow - let vmemdef = memory.vmmemory(); - let raw_ptr: *mut u8 = unsafe { vmemdef.as_ref().base }; - unsafe { - assert_eq!(read_unaligned(raw_ptr), 1); - } - // borrow as VMMemory - let vmmemory: VMMemory = VMMemory::from_custom(memory); - assert_eq!(vmmemory.size(), Pages::from(1u32)); - let raw_ptr = unsafe { vmmemory.vmmemory().as_ref().base }; - unsafe { - assert_eq!(read_unaligned(raw_ptr), 1); - assert_eq!(read_unaligned(raw_ptr.add(100)), 200); - } - } -} From bae459138f0e9a13190af977ad6d9193fb44e5d0 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Mon, 22 Aug 2022 16:52:23 +0200 Subject: [PATCH 18/40] LinearMemory test also include custom Tunables now --- lib/api/src/sys/tunables.rs | 81 ++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/lib/api/src/sys/tunables.rs b/lib/api/src/sys/tunables.rs index 1be12e59c..d27b86266 100644 --- a/lib/api/src/sys/tunables.rs +++ b/lib/api/src/sys/tunables.rs @@ -178,8 +178,6 @@ mod tests { } use std::cell::UnsafeCell; - use std::ptr::read_unaligned; - use std::ptr::write_unaligned; use std::ptr::NonNull; use wasmer_types::{MemoryError, MemoryStyle, MemoryType, Pages, WASM_PAGE_SIZE}; use wasmer_vm::{LinearMemory, MaybeInstanceOwned}; @@ -219,7 +217,7 @@ mod tests { } fn grow(&mut self, delta: Pages) -> Result { Err(MemoryError::CouldNotGrow { - current: Pages::from(1u32), + current: Pages::from(100u32), attempted_delta: delta, }) } @@ -241,16 +239,77 @@ mod tests { } } + struct TinyTunables; + impl Tunables for TinyTunables { + fn memory_style(&self, _memory: &MemoryType) -> MemoryStyle { + MemoryStyle::Static { + bound: Pages::from(1u32), + offset_guard_size: 0, + } + } + + /// Construct a `TableStyle` for the provided `TableType` + fn table_style(&self, _table: &TableType) -> TableStyle { + TableStyle::CallerChecksSignature + } + fn create_host_memory( + &self, + _ty: &MemoryType, + _style: &MemoryStyle, + ) -> Result { + let memory = VMTinyMemory::new().unwrap(); + Ok(VMMemory::from_custom(memory)) + } + unsafe fn create_vm_memory( + &self, + _ty: &MemoryType, + _style: &MemoryStyle, + _vm_definition_location: NonNull, + ) -> Result { + let memory = VMTinyMemory::new().unwrap(); + Ok(VMMemory::from_custom(memory)) + } + + /// Create a table owned by the host given a [`TableType`] and a [`TableStyle`]. + fn create_host_table(&self, ty: &TableType, style: &TableStyle) -> Result { + VMTable::new(ty, style) + } + + /// Create a table owned by the VM given a [`TableType`] and a [`TableStyle`]. + /// + /// # Safety + /// - `vm_definition_location` must point to a valid location in VM memory. + unsafe fn create_vm_table( + &self, + ty: &TableType, + style: &TableStyle, + vm_definition_location: NonNull, + ) -> Result { + VMTable::from_definition(ty, style, vm_definition_location) + } + } + #[test] fn check_linearmemory() { - let mut memory = VMTinyMemory::new().unwrap(); - assert!(memory.grow(Pages::from(2u32)).is_err()); - assert_eq!(memory.size(), Pages::from(1u32)); - let target = Target::default(); - let tunables = BaseTunables::for_target(&target); - let vmmemory = - unsafe { tunables.create_vm_memory(&memory.ty(), &memory.style(), memory.vmmemory()) }; - let vmmemory = vmmemory.unwrap(); + let tunables = TinyTunables {}; + let vmmemory = tunables.create_host_memory( + &MemoryType::new(1u32, Some(100u32), true), + &MemoryStyle::Static { + bound: Pages::from(1u32), + offset_guard_size: 0u64, + }, + ); + let mut vmmemory = vmmemory.unwrap(); + assert!(vmmemory.grow(Pages::from(2u32)).is_err()); assert_eq!(vmmemory.size(), Pages::from(1u32)); + //unsafe { tunables.create_vm_memory(&memory.ty(), &memory.style(), memory.vmmemory()) }; + assert_eq!(vmmemory.size(), Pages::from(1u32)); + assert_eq!( + vmmemory.grow(Pages::from(0u32)).err().unwrap(), + MemoryError::CouldNotGrow { + current: Pages::from(100u32), + attempted_delta: Pages::from(0u32) + } + ); } } From 6e0d58f0d43e04ff10999d327390f0aae8681dd7 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Mon, 22 Aug 2022 17:42:07 +0200 Subject: [PATCH 19/40] Cleanup LinearMemory test --- lib/api/src/sys/tunables.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/api/src/sys/tunables.rs b/lib/api/src/sys/tunables.rs index d27b86266..60abdac53 100644 --- a/lib/api/src/sys/tunables.rs +++ b/lib/api/src/sys/tunables.rs @@ -134,8 +134,6 @@ impl Tunables for BaseTunables { #[cfg(test)] mod tests { - use crate::sys::tunables; - use super::*; #[test] @@ -302,8 +300,6 @@ mod tests { let mut vmmemory = vmmemory.unwrap(); assert!(vmmemory.grow(Pages::from(2u32)).is_err()); assert_eq!(vmmemory.size(), Pages::from(1u32)); - //unsafe { tunables.create_vm_memory(&memory.ty(), &memory.style(), memory.vmmemory()) }; - assert_eq!(vmmemory.size(), Pages::from(1u32)); assert_eq!( vmmemory.grow(Pages::from(0u32)).err().unwrap(), MemoryError::CouldNotGrow { From f6fc1886ce59facb3f434190c69b43405f4c51bb Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Tue, 23 Aug 2022 10:57:43 +0300 Subject: [PATCH 20/40] scripts/publish.py: validate crates version before publishing Also use extra feature flags if defined in SETTINGS. --- scripts/publish.py | 54 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/scripts/publish.py b/scripts/publish.py index 3076bf413..6cfc938df 100644 --- a/scripts/publish.py +++ b/scripts/publish.py @@ -1,7 +1,17 @@ #! /usr/bin/env python3 """This is a script for publishing the wasmer crates to crates.io. -It should be run in the root of wasmer like `python3 scripts/publish.py --help`.""" +It should be run in the root of wasmer like `python3 scripts/publish.py --help`. + +Please lint with pylint and format with black. + +$ pylint scripts/publish.py + +-------------------------------------------------------------------- +Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00) + +$ black scripts/publish.py +""" import argparse import itertools @@ -77,10 +87,16 @@ class Crate: def __str__(self): return f"{self.name}: {self.dependencies} {self.cargo_file_path} {self.path()}" + @property def path(self) -> str: """Return the absolute filesystem path containing this crate.""" return os.path.dirname(self.cargo_file_path) + @property + def version(self) -> str: + """Return the crate's version according to its manifest.""" + return self.cargo_manifest["package"]["version"] + class Publisher: """A class responsible for figuring out dependencies, @@ -88,8 +104,8 @@ class Publisher: to crates.io in a valid order.""" def __init__(self, version=None, dry_run=True, verbose=True): - self.dry_run = dry_run - self.verbose = verbose + self.dry_run: bool = dry_run + self.verbose: bool = verbose # open workspace Cargo.toml with open("Cargo.toml", "rb") as file: @@ -97,7 +113,7 @@ class Publisher: if version is None: version = data["package"]["version"] - self.version = version + self.version: str = version if self.verbose and not self.dry_run: print(f"Publishing version {self.version}") @@ -157,6 +173,21 @@ class Publisher: dependencies = return_dependencies(member_data) crates.append(Crate(dependencies, member_data, cargo_file_path=member)) + invalids: typing.List[str] = [] + for crate in crates: + if crate.version != self.version: + print( + f"Crate {crate.name} is version {crate.version} but" + f" we're publishing for version {self.version}" + ) + invalids.append(crate.name) + + if len(invalids) > 0: + raise Exception( + f"Some crates have a different version than the" + f" one we're publishing ({self.version}): {invalids}" + ) + self.crates = crates self.crate_index: typing.Dict[str, Crate] = {c.name: c for c in crates} @@ -189,14 +220,23 @@ class Publisher: status = None try: crate = self.crate_index[crate_name] - os.chdir(crate.cargo_file_path) + os.chdir(crate.path) + extra_args = [] + if crate_name in SETTINGS["publish_features"]: + extra_args = ["--features", SETTINGS["publish_features"][crate_name]] if self.dry_run: print(f"In dry-run: not publishing crate `{crate_name}`") - output = subprocess.run(["cargo", "publish", "--dry-run"], check=True) + command = ["cargo", "publish", "--dry-run"] + extra_args + if self.verbose: + print(*command) + output = subprocess.run(command, check=True) if self.verbose: print(output) else: - output = subprocess.run(["cargo", "publish"], check=True) + command = ["cargo", "publish"] + extra_args + if self.verbose: + print(*command) + output = subprocess.run(command, check=True) if self.verbose: print(output) if self.verbose: From 2953093fb151e177b9cd885750f06c2c050e5dce Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Tue, 23 Aug 2022 11:28:43 +0200 Subject: [PATCH 21/40] Some more cleanup --- lib/api/src/sys/tunables.rs | 2 +- lib/vm/src/instance/allocator.rs | 3 +- lib/vm/src/lib.rs | 6 ++-- lib/vm/src/memory.rs | 53 +------------------------------- lib/vm/src/vmcontext.rs | 53 +++++++++++++++++++++++++++++++- 5 files changed, 58 insertions(+), 59 deletions(-) diff --git a/lib/api/src/sys/tunables.rs b/lib/api/src/sys/tunables.rs index 60abdac53..0a1eadd76 100644 --- a/lib/api/src/sys/tunables.rs +++ b/lib/api/src/sys/tunables.rs @@ -265,7 +265,7 @@ mod tests { _vm_definition_location: NonNull, ) -> Result { let memory = VMTinyMemory::new().unwrap(); - Ok(VMMemory::from_custom(memory)) + Ok(memory.into()) } /// Create a table owned by the host given a [`TableType`] and a [`TableStyle`]. diff --git a/lib/vm/src/instance/allocator.rs b/lib/vm/src/instance/allocator.rs index e81af2550..29804c746 100644 --- a/lib/vm/src/instance/allocator.rs +++ b/lib/vm/src/instance/allocator.rs @@ -1,6 +1,5 @@ use super::{Instance, InstanceHandle}; -use crate::memory::VMMemoryDefinition; -use crate::vmcontext::VMTableDefinition; +use crate::vmcontext::{VMMemoryDefinition, VMTableDefinition}; use std::alloc::{self, Layout}; use std::convert::TryFrom; use std::mem; diff --git a/lib/vm/src/lib.rs b/lib/vm/src/lib.rs index 16af4df4b..19982e54f 100644 --- a/lib/vm/src/lib.rs +++ b/lib/vm/src/lib.rs @@ -45,7 +45,7 @@ pub use crate::function_env::VMFunctionEnvironment; pub use crate::global::*; pub use crate::imports::Imports; pub use crate::instance::{InstanceAllocator, InstanceHandle}; -pub use crate::memory::{LinearMemory, VMMemory, VMMemoryDefinition}; +pub use crate::memory::{LinearMemory, VMMemory}; pub use crate::mmap::Mmap; pub use crate::probestack::PROBESTACK; pub use crate::sig_registry::SignatureRegistry; @@ -56,8 +56,8 @@ pub use crate::table::{TableElement, VMTable}; pub use crate::trap::*; pub use crate::vmcontext::{ VMCallerCheckedAnyfunc, VMContext, VMDynamicFunctionContext, VMFunctionContext, - VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, VMMemoryImport, - VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, + VMFunctionImport, VMFunctionKind, VMGlobalDefinition, VMGlobalImport, VMMemoryDefinition, + VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTrampoline, }; pub use wasmer_types::LibCall; pub use wasmer_types::MemoryError; diff --git a/lib/vm/src/memory.rs b/lib/vm/src/memory.rs index 516da4dee..f071365d1 100644 --- a/lib/vm/src/memory.rs +++ b/lib/vm/src/memory.rs @@ -5,7 +5,7 @@ //! //! `Memory` is to WebAssembly linear memories what `Table` is to WebAssembly tables. -use crate::{mmap::Mmap, store::MaybeInstanceOwned}; +use crate::{mmap::Mmap, store::MaybeInstanceOwned, vmcontext::VMMemoryDefinition}; use more_asserts::assert_ge; use std::cell::UnsafeCell; use std::convert::TryInto; @@ -411,54 +411,3 @@ where /// Attempts to clone this memory (if its clonable) fn try_clone(&self) -> Option>; } - -/// The fields compiled code needs to access to utilize a WebAssembly linear -/// memory defined within the instance, namely the start address and the -/// size in bytes. -#[derive(Debug, Copy, Clone)] -#[repr(C)] -pub struct VMMemoryDefinition { - /// The start address which is always valid, even if the memory grows. - pub base: *mut u8, - - /// The current logical size of this linear memory in bytes. - pub current_length: usize, -} - -/// # Safety -/// This data is safe to share between threads because it's plain data that -/// is the user's responsibility to synchronize. -unsafe impl Send for VMMemoryDefinition {} -/// # Safety -/// This data is safe to share between threads because it's plain data that -/// is the user's responsibility to synchronize. And it's `Copy` so there's -/// really no difference between passing it by reference or by value as far as -/// correctness in a multi-threaded context is concerned. -unsafe impl Sync for VMMemoryDefinition {} - -#[cfg(test)] -mod test_vmmemory_definition { - use super::VMMemoryDefinition; - use crate::ModuleInfo; - use crate::VMOffsets; - use memoffset::offset_of; - use std::mem::size_of; - - #[test] - fn check_vmmemory_definition_offsets() { - let module = ModuleInfo::new(); - let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); - assert_eq!( - size_of::(), - usize::from(offsets.size_of_vmmemory_definition()) - ); - assert_eq!( - offset_of!(VMMemoryDefinition, base), - usize::from(offsets.vmmemory_definition_base()) - ); - assert_eq!( - offset_of!(VMMemoryDefinition, current_length), - usize::from(offsets.vmmemory_definition_current_length()) - ); - } -} diff --git a/lib/vm/src/vmcontext.rs b/lib/vm/src/vmcontext.rs index cea7b116e..766a8708d 100644 --- a/lib/vm/src/vmcontext.rs +++ b/lib/vm/src/vmcontext.rs @@ -6,7 +6,7 @@ use crate::global::VMGlobal; use crate::instance::Instance; -use crate::memory::{VMMemory, VMMemoryDefinition}; +use crate::memory::VMMemory; use crate::store::InternalStoreHandle; use crate::trap::{Trap, TrapCode}; use crate::VMFunctionBody; @@ -677,3 +677,54 @@ pub type VMTrampoline = unsafe extern "C" fn( *const VMFunctionBody, // function we're actually calling *mut RawValue, // space for arguments and return values ); + +/// The fields compiled code needs to access to utilize a WebAssembly linear +/// memory defined within the instance, namely the start address and the +/// size in bytes. +#[derive(Debug, Copy, Clone)] +#[repr(C)] +pub struct VMMemoryDefinition { + /// The start address which is always valid, even if the memory grows. + pub base: *mut u8, + + /// The current logical size of this linear memory in bytes. + pub current_length: usize, +} + +/// # Safety +/// This data is safe to share between threads because it's plain data that +/// is the user's responsibility to synchronize. +unsafe impl Send for VMMemoryDefinition {} +/// # Safety +/// This data is safe to share between threads because it's plain data that +/// is the user's responsibility to synchronize. And it's `Copy` so there's +/// really no difference between passing it by reference or by value as far as +/// correctness in a multi-threaded context is concerned. +unsafe impl Sync for VMMemoryDefinition {} + +#[cfg(test)] +mod test_vmmemory_definition { + use super::VMMemoryDefinition; + use crate::ModuleInfo; + use crate::VMOffsets; + use memoffset::offset_of; + use std::mem::size_of; + + #[test] + fn check_vmmemory_definition_offsets() { + let module = ModuleInfo::new(); + let offsets = VMOffsets::new(size_of::<*mut u8>() as u8, &module); + assert_eq!( + size_of::(), + usize::from(offsets.size_of_vmmemory_definition()) + ); + assert_eq!( + offset_of!(VMMemoryDefinition, base), + usize::from(offsets.vmmemory_definition_base()) + ); + assert_eq!( + offset_of!(VMMemoryDefinition, current_length), + usize::from(offsets.vmmemory_definition_current_length()) + ); + } +} From d0dd8303060ed834c944996c5562108462f5fb3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 13:38:08 +0200 Subject: [PATCH 22/40] Initial update for migration-to-3.0.0 for MemoryView --- docs/migration_to_3.0.0.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/migration_to_3.0.0.md b/docs/migration_to_3.0.0.md index 2a87147af..68e116187 100644 --- a/docs/migration_to_3.0.0.md +++ b/docs/migration_to_3.0.0.md @@ -117,6 +117,9 @@ env_mut.memory = Some(instance.exports.get_memory("memory")); env_mut.alloc_guest_memory = Some(instance.exports.get_typed_function("__alloc")); ``` +### New `MemoryView` API + +Adding the new memory view API (and show how people can update from previous usage) ### Managing imports From e2daf4f5970b3e329c3279499db1ab1fa3909d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 14:39:28 +0200 Subject: [PATCH 23/40] Test metadata slice len before accessing to prevent panic --- lib/compiler/src/engine/artifact.rs | 25 ++++++++++++++++++++++++- lib/types/src/error.rs | 8 ++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 2608723a0..f362ed4de 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -116,9 +116,32 @@ impl Artifact { "The provided bytes are not wasmer-universal".to_string(), )); } + + if bytes.len() < ArtifactBuild::MAGIC_HEADER.len() { + return Err(DeserializeError::InvalidByteLength { + expected: ArtifactBuild::MAGIC_HEADER.len(), + got: bytes.len(), + }); + } + let bytes = &bytes[ArtifactBuild::MAGIC_HEADER.len()..]; let metadata_len = MetadataHeader::parse(bytes)?; - let metadata_slice: &[u8] = &bytes[MetadataHeader::LEN..][..metadata_len]; + if bytes.len() < MetadataHeader::LEN { + return Err(DeserializeError::InvalidByteLength { + expected: MetadataHeader::LEN, + got: bytes.len(), + }); + } + + let metadata_slice: &[u8] = &bytes[MetadataHeader::LEN..]; + if metadata_slice.len() < metadata_len { + return Err(DeserializeError::InvalidByteLength { + expected: metadata_len + MetadataHeader::LEN, + got: bytes.len(), + }); + } + + let metadata_slice: &[u8] = &metadata_slice[..metadata_len]; let serializable = SerializableModule::deserialize(metadata_slice)?; let artifact = ArtifactBuild::from_serializable(serializable); let mut inner_engine = engine.inner_mut(); diff --git a/lib/types/src/error.rs b/lib/types/src/error.rs index 0e0a1eb12..8f99f9419 100644 --- a/lib/types/src/error.rs +++ b/lib/types/src/error.rs @@ -35,6 +35,14 @@ pub enum DeserializeError { /// trying to allocate the required resources. #[error(transparent)] Compiler(#[from] CompileError), + /// Input artifact bytes have an invalid length + #[error("invalid input bytes: expected {expected} bytes, got {got}")] + InvalidByteLength { + /// How many bytes were expected + expected: usize, + /// How many bytes the artifact contained + got: usize, + } } /// Error type describing things that can go wrong when operating on Wasm Memories. From 9ea5c941b4ed5ffd927ada37fd88a0713e7f142d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 14:56:07 +0200 Subject: [PATCH 24/40] Prevent more panics in Artifact::deserialize --- lib/compiler/src/engine/artifact.rs | 86 +++++++++++++++++++++++++---- lib/types/src/error.rs | 2 +- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index f362ed4de..6446ee05e 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -132,7 +132,7 @@ impl Artifact { got: bytes.len(), }); } - + let metadata_slice: &[u8] = &bytes[MetadataHeader::LEN..]; if metadata_slice.len() < metadata_len { return Err(DeserializeError::InvalidByteLength { @@ -140,7 +140,7 @@ impl Artifact { got: bytes.len(), }); } - + let metadata_slice: &[u8] = &metadata_slice[..metadata_len]; let serializable = SerializableModule::deserialize(metadata_slice)?; let artifact = ArtifactBuild::from_serializable(serializable); @@ -596,6 +596,26 @@ impl Artifact { )) } + fn get_byte_slice(input: &[u8], start: usize, end: usize) -> Option<&[u8]> { + if start == end && input.len() > start { + return Some(&input[start..end]); + } else if start < end && input.len() > start && input.len() >= end { + Some(&input[start..end]) + } else { + None + } + } + + fn get_byte_slice_mut(input: &mut [u8], start: usize, end: usize) -> Option<&mut [u8]> { + if start == end && input.len() > start { + return Some(&mut input[start..end]); + } else if start < end && input.len() > start && input.len() >= end { + Some(&mut input[start..end]) + } else { + None + } + } + /// Deserialize a ArtifactBuild from an object file /// /// # Safety @@ -606,15 +626,27 @@ impl Artifact { bytes: &[u8], ) -> Result { let metadata_len = MetadataHeader::parse(bytes)?; + let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len()).ok_or( + DeserializeError::InvalidByteLength { + expected: MetadataHeader::LEN, + got: bytes.len(), + }, + )?; - let metadata_slice = &bytes[MetadataHeader::LEN..][..metadata_len]; let metadata: ModuleMetadata = ModuleMetadata::deserialize(metadata_slice)?; const WORD_SIZE: usize = mem::size_of::(); let mut byte_buffer = [0u8; WORD_SIZE]; let mut cur_offset = MetadataHeader::LEN + metadata_len; - byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + + let byte_buffer_slice = Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE) + .ok_or(DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + })?; + + byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; let num_finished_functions = usize::from_ne_bytes(byte_buffer); @@ -626,8 +658,15 @@ impl Artifact { // read finished functions in order now... for _i in 0..num_finished_functions { - byte_buffer[0..WORD_SIZE] - .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + let byte_buffer_slice = + Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE).ok_or( + DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + }, + )?; + + byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); cur_offset += WORD_SIZE; @@ -648,12 +687,24 @@ impl Artifact { // read trampolines in order let mut finished_function_call_trampolines = PrimaryMap::new(); - byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + + let byte_buffer_slice = Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE) + .ok_or(DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + })?; + byte_buffer[0..WORD_SIZE].clone_from_slice(&byte_buffer_slice); cur_offset += WORD_SIZE; let num_function_trampolines = usize::from_ne_bytes(byte_buffer); for _ in 0..num_function_trampolines { - byte_buffer[0..WORD_SIZE] - .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + let byte_buffer_slice = + Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE).ok_or( + DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + }, + )?; + byte_buffer[0..WORD_SIZE].clone_from_slice(&byte_buffer_slice); cur_offset += WORD_SIZE; let trampoline_ptr_bytes = usize::from_ne_bytes(byte_buffer); let trampoline = mem::transmute::(trampoline_ptr_bytes); @@ -663,12 +714,23 @@ impl Artifact { // read dynamic function trampolines in order now... let mut finished_dynamic_function_trampolines = PrimaryMap::new(); - byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + let byte_buffer_slice = Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE) + .ok_or(DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + })?; + byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; let num_dynamic_trampoline_functions = usize::from_ne_bytes(byte_buffer); for _i in 0..num_dynamic_trampoline_functions { - byte_buffer[0..WORD_SIZE] - .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + let byte_buffer_slice = + Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE).ok_or( + DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + }, + )?; + byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); cur_offset += WORD_SIZE; diff --git a/lib/types/src/error.rs b/lib/types/src/error.rs index 8f99f9419..e9b8dcd9e 100644 --- a/lib/types/src/error.rs +++ b/lib/types/src/error.rs @@ -42,7 +42,7 @@ pub enum DeserializeError { expected: usize, /// How many bytes the artifact contained got: usize, - } + }, } /// Error type describing things that can go wrong when operating on Wasm Memories. From bb9f95b2f843d031d84bcc91c67a27c390856c91 Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Tue, 23 Aug 2022 15:04:55 +0200 Subject: [PATCH 25/40] Revert "Lower libwasmer headless size" --- Makefile | 4 ++-- lib/c-api/Cargo.toml | 2 +- lib/wasi/Cargo.toml | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 00d35e272..810677016 100644 --- a/Makefile +++ b/Makefile @@ -389,7 +389,7 @@ build-wasmer-headless-minimal: RUSTFLAGS += -C panic=abort build-wasmer-headless-minimal: RUSTFLAGS="${RUSTFLAGS}" xargo build --target $(HOST_TARGET) --release --manifest-path=lib/cli/Cargo.toml --no-default-features --features headless-minimal --bin wasmer-headless ifeq ($(IS_DARWIN), 1) - strip target/$(HOST_TARGET)/release/wasmer-headless + strip -u target/$(HOST_TARGET)/release/wasmer-headless else ifeq ($(IS_WINDOWS), 1) strip --strip-unneeded target/$(HOST_TARGET)/release/wasmer-headless.exe else @@ -455,7 +455,7 @@ build-capi-llvm-universal: capi-setup # Headless (we include the minimal to be able to run) build-capi-headless: capi-setup - RUSTFLAGS="${RUSTFLAGS} -C panic=abort -C link-dead-code -C lto -O -C embed-bitcode=yes" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ + RUSTFLAGS="${RUSTFLAGS} -C panic=abort" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features compiler-headless,wasi build-capi-headless-ios: capi-setup diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index 072da4e3d..d275c0fff 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -17,7 +17,7 @@ edition = "2018" # a conflict with the existing `wasmer` crate, see below. name = "wasmer" # ##lib.name## # ^ DO NOT REMOVE, it's used the `Makefile`, see `build-docs-capi`. -crate-type = ["staticlib", "cdylib"] #"cdylib", "rlib", "staticlib"] +crate-type = ["cdylib", "rlib", "staticlib"] [dependencies] # We rename `wasmer` to `wasmer-api` to avoid the conflict with this diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index 798d13046..bfa746e29 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -10,6 +10,9 @@ license = "MIT" readme = "README.md" edition = "2018" +[lib] +crate-type = ["cdylib", "rlib"] + [dependencies] cfg-if = "1.0" thiserror = "1" From 6988025cd0fc9e0d42ce5852fef2f9e5e8f200d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 15:24:13 +0200 Subject: [PATCH 26/40] Fix make lint --- lib/cli/src/c_gen/mod.rs | 4 +- .../src/artifact_builders/artifact_builder.rs | 2 +- lib/compiler/src/engine/artifact.rs | 67 +++++++------------ lib/emscripten/src/lib.rs | 2 +- lib/types/src/error.rs | 2 +- lib/types/src/serialize.rs | 2 +- lib/vm/src/instance/mod.rs | 4 +- lib/vm/src/libcalls.rs | 26 +++---- lib/wasi/src/syscalls/mod.rs | 6 +- 9 files changed, 50 insertions(+), 65 deletions(-) diff --git a/lib/cli/src/c_gen/mod.rs b/lib/cli/src/c_gen/mod.rs index addc30a8c..dbb56490a 100644 --- a/lib/cli/src/c_gen/mod.rs +++ b/lib/cli/src/c_gen/mod.rs @@ -125,7 +125,7 @@ impl CType { #[allow(clippy::borrowed_box)] let ret: CType = return_value .as_ref() - .map(|i: &Box| (&**i).clone()) + .map(|i: &Box| (**i).clone()) .unwrap_or_default(); ret.generate_c(w); w.push(' '); @@ -183,7 +183,7 @@ impl CType { #[allow(clippy::borrowed_box)] let ret: CType = return_value .as_ref() - .map(|i: &Box| (&**i).clone()) + .map(|i: &Box| (**i).clone()) .unwrap_or_default(); ret.generate_c(w); w.push(' '); diff --git a/lib/compiler/src/artifact_builders/artifact_builder.rs b/lib/compiler/src/artifact_builders/artifact_builder.rs index aef2db988..098cd0b8f 100644 --- a/lib/compiler/src/artifact_builders/artifact_builder.rs +++ b/lib/compiler/src/artifact_builders/artifact_builder.rs @@ -201,7 +201,7 @@ impl ArtifactCreate for ArtifactBuild { } fn data_initializers(&self) -> &[OwnedDataInitializer] { - &*self.serializable.data_initializers + &self.serializable.data_initializers } fn memory_styles(&self) -> &PrimaryMap { diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 6446ee05e..40bf5c371 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -366,7 +366,7 @@ impl Artifact { // Get pointers to where metadata about local tables should live in VM memory. let (allocator, memory_definition_locations, table_definition_locations) = - InstanceAllocator::new(&*module); + InstanceAllocator::new(&module); let finished_memories = tunables .create_memories( context, @@ -423,7 +423,7 @@ impl Artifact { .iter() .map(|init| DataInitializer { location: init.location.clone(), - data: &*init.data, + data: &init.data, }) .collect::>(); handle @@ -596,26 +596,17 @@ impl Artifact { )) } + #[cfg(feature = "static-artifact-load")] fn get_byte_slice(input: &[u8], start: usize, end: usize) -> Option<&[u8]> { - if start == end && input.len() > start { - return Some(&input[start..end]); - } else if start < end && input.len() > start && input.len() >= end { + if (start == end && input.len() > start) + || (start < end && input.len() > start && input.len() >= end) + { Some(&input[start..end]) } else { None } } - fn get_byte_slice_mut(input: &mut [u8], start: usize, end: usize) -> Option<&mut [u8]> { - if start == end && input.len() > start { - return Some(&mut input[start..end]); - } else if start < end && input.len() > start && input.len() >= end { - Some(&mut input[start..end]) - } else { - None - } - } - /// Deserialize a ArtifactBuild from an object file /// /// # Safety @@ -640,7 +631,7 @@ impl Artifact { let mut cur_offset = MetadataHeader::LEN + metadata_len; - let byte_buffer_slice = Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE) + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) .ok_or(DeserializeError::InvalidByteLength { expected: cur_offset + WORD_SIZE, got: bytes.len(), @@ -658,13 +649,11 @@ impl Artifact { // read finished functions in order now... for _i in 0..num_finished_functions { - let byte_buffer_slice = - Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE).ok_or( - DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - }, - )?; + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) + .ok_or(DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + })?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); @@ -688,23 +677,21 @@ impl Artifact { // read trampolines in order let mut finished_function_call_trampolines = PrimaryMap::new(); - let byte_buffer_slice = Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE) + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) .ok_or(DeserializeError::InvalidByteLength { expected: cur_offset + WORD_SIZE, got: bytes.len(), })?; - byte_buffer[0..WORD_SIZE].clone_from_slice(&byte_buffer_slice); + byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; let num_function_trampolines = usize::from_ne_bytes(byte_buffer); for _ in 0..num_function_trampolines { - let byte_buffer_slice = - Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE).ok_or( - DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - }, - )?; - byte_buffer[0..WORD_SIZE].clone_from_slice(&byte_buffer_slice); + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) + .ok_or(DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + })?; + byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; let trampoline_ptr_bytes = usize::from_ne_bytes(byte_buffer); let trampoline = mem::transmute::(trampoline_ptr_bytes); @@ -714,7 +701,7 @@ impl Artifact { // read dynamic function trampolines in order now... let mut finished_dynamic_function_trampolines = PrimaryMap::new(); - let byte_buffer_slice = Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE) + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) .ok_or(DeserializeError::InvalidByteLength { expected: cur_offset + WORD_SIZE, got: bytes.len(), @@ -723,13 +710,11 @@ impl Artifact { cur_offset += WORD_SIZE; let num_dynamic_trampoline_functions = usize::from_ne_bytes(byte_buffer); for _i in 0..num_dynamic_trampoline_functions { - let byte_buffer_slice = - Self::get_byte_slice(&bytes, cur_offset, cur_offset + WORD_SIZE).ok_or( - DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - }, - )?; + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) + .ok_or(DeserializeError::InvalidByteLength { + expected: cur_offset + WORD_SIZE, + got: bytes.len(), + })?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); cur_offset += WORD_SIZE; diff --git a/lib/emscripten/src/lib.rs b/lib/emscripten/src/lib.rs index 1e4b7af80..216171e35 100644 --- a/lib/emscripten/src/lib.rs +++ b/lib/emscripten/src/lib.rs @@ -105,7 +105,7 @@ impl EmEnv { /// Get a reference to the memory pub fn memory(&self, _mem_idx: u32) -> Memory { - (&*self.memory.read().unwrap()).as_ref().cloned().unwrap() + (*self.memory.read().unwrap()).as_ref().cloned().unwrap() } pub fn set_functions(&mut self, funcs: EmscriptenFunctions) { diff --git a/lib/types/src/error.rs b/lib/types/src/error.rs index e9b8dcd9e..66f5ce765 100644 --- a/lib/types/src/error.rs +++ b/lib/types/src/error.rs @@ -46,7 +46,7 @@ pub enum DeserializeError { } /// Error type describing things that can go wrong when operating on Wasm Memories. -#[derive(Error, Debug, Clone, PartialEq, Hash)] +#[derive(Error, Debug, Clone, PartialEq, Eq, Hash)] pub enum MemoryError { /// Low level error with mmap. #[error("Error when allocating memory: {0}")] diff --git a/lib/types/src/serialize.rs b/lib/types/src/serialize.rs index 4bf7bfac0..7df4fca9b 100644 --- a/lib/types/src/serialize.rs +++ b/lib/types/src/serialize.rs @@ -144,7 +144,7 @@ impl SerializableModule { /// Returns data initializers to pass to `InstanceHandle::initialize` pub fn data_initializers(&self) -> &[OwnedDataInitializer] { - &*self.data_initializers + &self.data_initializers } /// Returns the memory styles associated with this `Artifact`. diff --git a/lib/vm/src/instance/mod.rs b/lib/vm/src/instance/mod.rs index c91a8f63f..0f189a75e 100644 --- a/lib/vm/src/instance/mod.rs +++ b/lib/vm/src/instance/mod.rs @@ -116,7 +116,7 @@ impl Instance { } pub(crate) fn module_ref(&self) -> &ModuleInfo { - &*self.module + &self.module } fn context(&self) -> &StoreObjects { @@ -868,7 +868,7 @@ impl InstanceHandle { let instance = instance_handle.instance_mut(); let vmctx_ptr = instance.vmctx_ptr(); (instance.funcrefs, instance.imported_funcrefs) = build_funcrefs( - &*instance.module, + &instance.module, context, &imports, &instance.functions, diff --git a/lib/vm/src/libcalls.rs b/lib/vm/src/libcalls.rs index f9b1b2870..9274237f1 100644 --- a/lib/vm/src/libcalls.rs +++ b/lib/vm/src/libcalls.rs @@ -189,7 +189,7 @@ pub unsafe extern "C" fn wasmer_vm_imported_memory32_grow( /// `vmctx` must be dereferenceable. #[no_mangle] pub unsafe extern "C" fn wasmer_vm_memory32_size(vmctx: *mut VMContext, memory_index: u32) -> u32 { - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); let memory_index = LocalMemoryIndex::from_u32(memory_index); instance.memory_size(memory_index).0 @@ -205,7 +205,7 @@ pub unsafe extern "C" fn wasmer_vm_imported_memory32_size( vmctx: *mut VMContext, memory_index: u32, ) -> u32 { - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); let memory_index = MemoryIndex::from_u32(memory_index); instance.imported_memory_size(memory_index).0 @@ -303,7 +303,7 @@ pub unsafe extern "C" fn wasmer_vm_table_fill( /// `vmctx` must be dereferenceable. #[no_mangle] pub unsafe extern "C" fn wasmer_vm_table_size(vmctx: *mut VMContext, table_index: u32) -> u32 { - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); let table_index = LocalTableIndex::from_u32(table_index); instance.table_size(table_index) @@ -319,7 +319,7 @@ pub unsafe extern "C" fn wasmer_vm_imported_table_size( vmctx: *mut VMContext, table_index: u32, ) -> u32 { - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); let table_index = TableIndex::from_u32(table_index); instance.imported_table_size(table_index) @@ -336,7 +336,7 @@ pub unsafe extern "C" fn wasmer_vm_table_get( table_index: u32, elem_index: u32, ) -> RawTableElement { - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); let table_index = LocalTableIndex::from_u32(table_index); // TODO: type checking, maybe have specialized accessors @@ -495,7 +495,7 @@ pub unsafe extern "C" fn wasmer_vm_func_ref( vmctx: *mut VMContext, function_index: u32, ) -> VMFuncRef { - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); let function_index = FunctionIndex::from_u32(function_index); instance.func_ref(function_index).unwrap() @@ -510,7 +510,7 @@ pub unsafe extern "C" fn wasmer_vm_func_ref( pub unsafe extern "C" fn wasmer_vm_elem_drop(vmctx: *mut VMContext, elem_index: u32) { on_host_stack(|| { let elem_index = ElemIndex::from_u32(elem_index); - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); instance.elem_drop(elem_index); }) } @@ -530,7 +530,7 @@ pub unsafe extern "C" fn wasmer_vm_memory32_copy( ) { let result = { let memory_index = LocalMemoryIndex::from_u32(memory_index); - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); instance.local_memory_copy(memory_index, dst, src, len) }; if let Err(trap) = result { @@ -553,7 +553,7 @@ pub unsafe extern "C" fn wasmer_vm_imported_memory32_copy( ) { let result = { let memory_index = MemoryIndex::from_u32(memory_index); - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); instance.imported_memory_copy(memory_index, dst, src, len) }; if let Err(trap) = result { @@ -576,7 +576,7 @@ pub unsafe extern "C" fn wasmer_vm_memory32_fill( ) { let result = { let memory_index = LocalMemoryIndex::from_u32(memory_index); - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); instance.local_memory_fill(memory_index, dst, val, len) }; if let Err(trap) = result { @@ -599,7 +599,7 @@ pub unsafe extern "C" fn wasmer_vm_imported_memory32_fill( ) { let result = { let memory_index = MemoryIndex::from_u32(memory_index); - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); instance.imported_memory_fill(memory_index, dst, val, len) }; if let Err(trap) = result { @@ -624,7 +624,7 @@ pub unsafe extern "C" fn wasmer_vm_memory32_init( let result = { let memory_index = MemoryIndex::from_u32(memory_index); let data_index = DataIndex::from_u32(data_index); - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); instance.memory_init(memory_index, data_index, dst, src, len) }; if let Err(trap) = result { @@ -641,7 +641,7 @@ pub unsafe extern "C" fn wasmer_vm_memory32_init( pub unsafe extern "C" fn wasmer_vm_data_drop(vmctx: *mut VMContext, data_index: u32) { on_host_stack(|| { let data_index = DataIndex::from_u32(data_index); - let instance = (&*vmctx).instance(); + let instance = (*vmctx).instance(); instance.data_drop(data_index) }) } diff --git a/lib/wasi/src/syscalls/mod.rs b/lib/wasi/src/syscalls/mod.rs index 0c5266e6e..e972946c0 100644 --- a/lib/wasi/src/syscalls/mod.rs +++ b/lib/wasi/src/syscalls/mod.rs @@ -307,7 +307,7 @@ pub fn args_get( let env = ctx.data(); let (memory, mut state) = env.get_memory_and_wasi_state(&ctx, 0); - let result = write_buffer_array(&memory, &*state.args, argv, argv_buf); + let result = write_buffer_array(&memory, &state.args, argv, argv_buf); debug!( "=> args:\n{}", @@ -433,7 +433,7 @@ pub fn environ_get( let (memory, mut state) = env.get_memory_and_wasi_state(&ctx, 0); trace!(" -> State envs: {:?}", state.envs); - write_buffer_array(&memory, &*state.envs, environ, environ_buf) + write_buffer_array(&memory, &state.envs, environ, environ_buf) } /// ### `environ_sizes_get()` @@ -5555,7 +5555,7 @@ pub unsafe fn sock_send_file( sock, __WASI_RIGHT_SOCK_SEND, |socket| { - let buf = (&buf[..]).to_vec(); + let buf = (buf[..]).to_vec(); socket.send_bytes::(Bytes::from(buf)) } )); From 36f4c44c6c83110440ee40364a53e2966c8f301f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 15:32:18 +0200 Subject: [PATCH 27/40] Fix typo --- examples/exports_memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/exports_memory.rs b/examples/exports_memory.rs index 5f8b0a9d2..8cb29e2e6 100644 --- a/examples/exports_memory.rs +++ b/examples/exports_memory.rs @@ -75,7 +75,7 @@ fn main() -> Result<(), Box> { println!("String offset: {:?}", ptr.offset()); println!("String length: {:?}", length); - // We now know where to fin our string, let's read it. + // We now know where to find our string, let's read it. // // We will get bytes out of the memory so we need to // decode them into a string. From b6a6897ef52efaff2b54d458e897762f2d04b421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 15:52:50 +0200 Subject: [PATCH 28/40] Add proper description of MemoryView API --- docs/migration_to_3.0.0.md | 57 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/docs/migration_to_3.0.0.md b/docs/migration_to_3.0.0.md index 68e116187..c115578cd 100644 --- a/docs/migration_to_3.0.0.md +++ b/docs/migration_to_3.0.0.md @@ -117,9 +117,62 @@ env_mut.memory = Some(instance.exports.get_memory("memory")); env_mut.alloc_guest_memory = Some(instance.exports.get_typed_function("__alloc")); ``` -### New `MemoryView` API +### New `MemoryView` API (preparation for shared memory) -Adding the new memory view API (and show how people can update from previous usage) +Reading from memory has slightly changed compared to 2.x: + +```rust +// 2.x +let memory = instance.exports.get_memory("mem")?; +println!("Memory size (pages) {:?}", memory.size()); +println!("Memory size (bytes) {:?}", memory.data_size()); + +let load = instance + .exports + .get_native_function::<(), (WasmPtr, i32)>("load")?; + +let (ptr, length) = load.call(&mut store)?; +let str = ptr.get_utf8_string(memory, length as u32).unwrap(); +println!("Memory contents: {:?}", str); +``` + +```rust +// 3.x +let memory = instance.exports.get_memory("mem")?; +let memory_view = memory.view(&store); +println!("Memory size (pages) {:?}", memory_view.size()); +println!("Memory size (bytes) {:?}", memory_view.data_size()); + +let load: TypedFunction<(), (WasmPtr, i32)> = + instance.exports.get_typed_function(&mut store, "load")?; + +let (ptr, length) = load.call(&mut store)?; +let memory_view = memory.view(&store); +let str = ptr.read_utf8_string(&memory_view, length as u32).unwrap(); +println!("Memory contents: {:?}", str); +``` + +The reason for this change is that in the future this will enable +safely sharing memory across threads. The same thing goes for reading slices: + +```rust +// 2.x +let new_str = b"Hello, Wasmer!"; +let values = ptr.deref(memory, 0, new_str.len() as u32).unwrap(); +for i in 0..new_str.len() { + values[i].set(new_str[i]); +} +``` + +```rust +// 3.x +let memory_view = memory.view(&store); // (can be reused) +let new_str = b"Hello, Wasmer!"; +let values = ptr.slice(&memory_view, new_str.len() as u32).unwrap(); +for i in 0..new_str.len() { + values.index(i as u64).write(new_str[i]).unwrap(); +} +``` ### Managing imports From c1205e39be28582cb287d5b4c52a478cca04f3f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 15:59:11 +0200 Subject: [PATCH 29/40] Add update-migration to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7756a471e..e2822cc82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Looking for changes that affect our C API? See the [C API Changelog](lib/c-api/C - #[3096](https://github.com/wasmerio/wasmer/pull/3096) create-exe: use cached wasmer tarballs for network fetches ### Changed +- #[3131](https://github.com/wasmerio/wasmer/pull/3131) Update migration docs for MemoryView changes ### Fixed From e00be2380d0157f0d6dd89b7b7e063b520c3182c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 16:10:12 +0200 Subject: [PATCH 30/40] Remove manual checks in Artifact::deserialize, prefer function instead --- lib/compiler/src/engine/artifact.rs | 34 +++++++++++++---------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 40bf5c371..d98dbe4e2 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -117,29 +117,26 @@ impl Artifact { )); } - if bytes.len() < ArtifactBuild::MAGIC_HEADER.len() { - return Err(DeserializeError::InvalidByteLength { + let bytes = Self::get_byte_slice(bytes, 0, ArtifactBuild::MAGIC_HEADER.len()).ok_or( + DeserializeError::InvalidByteLength { expected: ArtifactBuild::MAGIC_HEADER.len(), got: bytes.len(), - }); - } + }, + )?; - let bytes = &bytes[ArtifactBuild::MAGIC_HEADER.len()..]; let metadata_len = MetadataHeader::parse(bytes)?; - if bytes.len() < MetadataHeader::LEN { - return Err(DeserializeError::InvalidByteLength { - expected: MetadataHeader::LEN, + let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len()).ok_or( + DeserializeError::InvalidByteLength { + expected: ArtifactBuild::MAGIC_HEADER.len(), got: bytes.len(), - }); - } - - let metadata_slice: &[u8] = &bytes[MetadataHeader::LEN..]; - if metadata_slice.len() < metadata_len { - return Err(DeserializeError::InvalidByteLength { - expected: metadata_len + MetadataHeader::LEN, - got: bytes.len(), - }); - } + }, + )?; + let metadata_slice = Self::get_byte_slice(metadata_slice, 0, metadata_len).ok_or( + DeserializeError::InvalidByteLength { + expected: metadata_len, + got: metadata_slice.len(), + }, + )?; let metadata_slice: &[u8] = &metadata_slice[..metadata_len]; let serializable = SerializableModule::deserialize(metadata_slice)?; @@ -596,7 +593,6 @@ impl Artifact { )) } - #[cfg(feature = "static-artifact-load")] fn get_byte_slice(input: &[u8], start: usize, end: usize) -> Option<&[u8]> { if (start == end && input.len() > start) || (start < end && input.len() > start && input.len() >= end) From 21342e540f3dfe5579dbab13b4cadd63c8baa406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 16:25:08 +0200 Subject: [PATCH 31/40] Adress review comment: Use Result<...> instead of Option for quick return --- lib/compiler/src/engine/artifact.rs | 73 +++++------------------------ 1 file changed, 13 insertions(+), 60 deletions(-) diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index d98dbe4e2..08e598b60 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -117,26 +117,11 @@ impl Artifact { )); } - let bytes = Self::get_byte_slice(bytes, 0, ArtifactBuild::MAGIC_HEADER.len()).ok_or( - DeserializeError::InvalidByteLength { - expected: ArtifactBuild::MAGIC_HEADER.len(), - got: bytes.len(), - }, - )?; + let bytes = Self::get_byte_slice(bytes, 0, ArtifactBuild::MAGIC_HEADER.len())?; let metadata_len = MetadataHeader::parse(bytes)?; - let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len()).ok_or( - DeserializeError::InvalidByteLength { - expected: ArtifactBuild::MAGIC_HEADER.len(), - got: bytes.len(), - }, - )?; - let metadata_slice = Self::get_byte_slice(metadata_slice, 0, metadata_len).ok_or( - DeserializeError::InvalidByteLength { - expected: metadata_len, - got: metadata_slice.len(), - }, - )?; + let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len())?; + let metadata_slice = Self::get_byte_slice(metadata_slice, 0, metadata_len)?; let metadata_slice: &[u8] = &metadata_slice[..metadata_len]; let serializable = SerializableModule::deserialize(metadata_slice)?; @@ -593,13 +578,13 @@ impl Artifact { )) } - fn get_byte_slice(input: &[u8], start: usize, end: usize) -> Option<&[u8]> { + fn get_byte_slice(input: &[u8], start: usize, end: usize) -> Result<&[u8], DeserializeError> { if (start == end && input.len() > start) || (start < end && input.len() > start && input.len() >= end) { - Some(&input[start..end]) + Ok(&input[start..end]) } else { - None + Err(DeserializeError::InvalidByteLength { expected: end - start, got: input.len() }) } } @@ -613,13 +598,7 @@ impl Artifact { bytes: &[u8], ) -> Result { let metadata_len = MetadataHeader::parse(bytes)?; - let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len()).ok_or( - DeserializeError::InvalidByteLength { - expected: MetadataHeader::LEN, - got: bytes.len(), - }, - )?; - + let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len())?; let metadata: ModuleMetadata = ModuleMetadata::deserialize(metadata_slice)?; const WORD_SIZE: usize = mem::size_of::(); @@ -627,12 +606,7 @@ impl Artifact { let mut cur_offset = MetadataHeader::LEN + metadata_len; - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) - .ok_or(DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - })?; - + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; @@ -645,12 +619,7 @@ impl Artifact { // read finished functions in order now... for _i in 0..num_finished_functions { - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) - .ok_or(DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - })?; - + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); cur_offset += WORD_SIZE; @@ -673,20 +642,12 @@ impl Artifact { // read trampolines in order let mut finished_function_call_trampolines = PrimaryMap::new(); - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) - .ok_or(DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - })?; + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; let num_function_trampolines = usize::from_ne_bytes(byte_buffer); for _ in 0..num_function_trampolines { - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) - .ok_or(DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - })?; + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; let trampoline_ptr_bytes = usize::from_ne_bytes(byte_buffer); @@ -697,20 +658,12 @@ impl Artifact { // read dynamic function trampolines in order now... let mut finished_dynamic_function_trampolines = PrimaryMap::new(); - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) - .ok_or(DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - })?; + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; let num_dynamic_trampoline_functions = usize::from_ne_bytes(byte_buffer); for _i in 0..num_dynamic_trampoline_functions { - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE) - .ok_or(DeserializeError::InvalidByteLength { - expected: cur_offset + WORD_SIZE, - got: bytes.len(), - })?; + let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); cur_offset += WORD_SIZE; From 0df69e4a6479a16667e5f1f3aa96edbdc8d2b70c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 16:27:54 +0200 Subject: [PATCH 32/40] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7756a471e..68ab2b5e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Looking for changes that affect our C API? See the [C API Changelog](lib/c-api/C ### Changed ### Fixed +- #[3130](https://github.com/wasmerio/wasmer/pull/3130) Remove panics from Artifact::deserialize ## 3.0.0-beta - 2022/08/08 From f659a35229bd7f26a9537e698050c8b9d428c7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 16:30:50 +0200 Subject: [PATCH 33/40] Make "make lint" pass --- lib/compiler/src/engine/artifact.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 08e598b60..b721c120a 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -584,7 +584,10 @@ impl Artifact { { Ok(&input[start..end]) } else { - Err(DeserializeError::InvalidByteLength { expected: end - start, got: input.len() }) + Err(DeserializeError::InvalidByteLength { + expected: end - start, + got: input.len(), + }) } } @@ -619,7 +622,8 @@ impl Artifact { // read finished functions in order now... for _i in 0..num_finished_functions { - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; + let byte_buffer_slice = + Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); cur_offset += WORD_SIZE; @@ -647,7 +651,8 @@ impl Artifact { cur_offset += WORD_SIZE; let num_function_trampolines = usize::from_ne_bytes(byte_buffer); for _ in 0..num_function_trampolines { - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; + let byte_buffer_slice = + Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); cur_offset += WORD_SIZE; let trampoline_ptr_bytes = usize::from_ne_bytes(byte_buffer); @@ -663,7 +668,8 @@ impl Artifact { cur_offset += WORD_SIZE; let num_dynamic_trampoline_functions = usize::from_ne_bytes(byte_buffer); for _i in 0..num_dynamic_trampoline_functions { - let byte_buffer_slice = Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; + let byte_buffer_slice = + Self::get_byte_slice(bytes, cur_offset, cur_offset + WORD_SIZE)?; byte_buffer[0..WORD_SIZE].clone_from_slice(byte_buffer_slice); let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); cur_offset += WORD_SIZE; From 18abab157c6ae5cffd673db3ab1066b65c9f7d67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 16:56:16 +0200 Subject: [PATCH 34/40] Add sanity test for Artifact::deserialize --- tests/compilers/serialize.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/compilers/serialize.rs b/tests/compilers/serialize.rs index 2610f83da..4bca9000f 100644 --- a/tests/compilers/serialize.rs +++ b/tests/compilers/serialize.rs @@ -1,6 +1,13 @@ use anyhow::Result; use wasmer::*; +#[test] +fn sanity_test_artifact_deserialize() { + let engine = Engine::headless(); + let result = unsafe { Artifact::deserialize(&engine, &[]) }; + assert!(result.is_err()); +} + #[compiler_test(serialize)] fn test_serialize(config: crate::Config) -> Result<()> { let mut store = config.store(); From ff8f43b0aec4d21a37ec9e2724514ffa01e8f0be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Tue, 23 Aug 2022 18:03:13 +0200 Subject: [PATCH 35/40] Fix error in MetadataHeader::parse --- lib/compiler/src/engine/artifact.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index b721c120a..d49780127 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -117,7 +117,7 @@ impl Artifact { )); } - let bytes = Self::get_byte_slice(bytes, 0, ArtifactBuild::MAGIC_HEADER.len())?; + let bytes = Self::get_byte_slice(bytes, ArtifactBuild::MAGIC_HEADER.len(), bytes.len())?; let metadata_len = MetadataHeader::parse(bytes)?; let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len())?; From 056268cb588315c03e3f72adf71a8a5f429cf4fa Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Sun, 21 Aug 2022 16:12:52 +0300 Subject: [PATCH 36/40] Bring libwasmer-headless.a from 22MiB to 7.2MiB (on my machine) By using lto optimization flags. A qjs.wasm executable using the headless engine now weights 4.6MiB: % du -sh qjs-he*.out 20M qjs-headfull.out 4.6M qjs-headless.out --- Makefile | 9 +++++++-- lib/c-api/Cargo.toml | 2 +- lib/wasi/Cargo.toml | 3 --- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 810677016..8355918c5 100644 --- a/Makefile +++ b/Makefile @@ -389,7 +389,7 @@ build-wasmer-headless-minimal: RUSTFLAGS += -C panic=abort build-wasmer-headless-minimal: RUSTFLAGS="${RUSTFLAGS}" xargo build --target $(HOST_TARGET) --release --manifest-path=lib/cli/Cargo.toml --no-default-features --features headless-minimal --bin wasmer-headless ifeq ($(IS_DARWIN), 1) - strip -u target/$(HOST_TARGET)/release/wasmer-headless + strip target/$(HOST_TARGET)/release/wasmer-headless else ifeq ($(IS_WINDOWS), 1) strip --strip-unneeded target/$(HOST_TARGET)/release/wasmer-headless.exe else @@ -455,8 +455,13 @@ build-capi-llvm-universal: capi-setup # Headless (we include the minimal to be able to run) build-capi-headless: capi-setup - RUSTFLAGS="${RUSTFLAGS} -C panic=abort" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ +ifeq ($(CARGO_TARGET),) + RUSTFLAGS="${RUSTFLAGS} -C panic=abort -C link-dead-code -C lto -O -C embed-bitcode=yes" $(CARGO_BINARY) build --target $(HOST_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features compiler-headless,wasi +else + RUSTFLAGS="${RUSTFLAGS} -C panic=abort -C link-dead-code -C lto -O -C embed-bitcode=yes" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ + --no-default-features --features compiler-headless,wasi +endif build-capi-headless-ios: capi-setup RUSTFLAGS="${RUSTFLAGS} -C panic=abort" cargo lipo --manifest-path lib/c-api/Cargo.toml --release \ diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index d275c0fff..072da4e3d 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -17,7 +17,7 @@ edition = "2018" # a conflict with the existing `wasmer` crate, see below. name = "wasmer" # ##lib.name## # ^ DO NOT REMOVE, it's used the `Makefile`, see `build-docs-capi`. -crate-type = ["cdylib", "rlib", "staticlib"] +crate-type = ["staticlib", "cdylib"] #"cdylib", "rlib", "staticlib"] [dependencies] # We rename `wasmer` to `wasmer-api` to avoid the conflict with this diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index bfa746e29..798d13046 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -10,9 +10,6 @@ license = "MIT" readme = "README.md" edition = "2018" -[lib] -crate-type = ["cdylib", "rlib"] - [dependencies] cfg-if = "1.0" thiserror = "1" From a869daea916863cb63e6fcebe955417aebd3b5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 24 Aug 2022 18:07:35 +0200 Subject: [PATCH 37/40] Fix cache path not being present during installation of cross-tarball --- lib/cli/src/commands/create_exe.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index ff0d20f90..9756b9102 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -984,6 +984,7 @@ mod http_fetch { .expect("Could not join downloading thread"); match super::get_libwasmer_cache_path() { Ok(mut cache_path) => { + let _ = std::fs::create_dir_all(&cache_path); cache_path.push(&filename); if !cache_path.exists() { if let Err(err) = std::fs::copy(&filename, &cache_path) { From d8ed8d7c6c0c8dbe784ea1b0f547968b947bf66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 25 Aug 2022 12:39:08 +0200 Subject: [PATCH 38/40] Fix issue in Artifact::deserialize --- lib/compiler/src/engine/artifact.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index d49780127..b436c2da8 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -123,7 +123,6 @@ impl Artifact { let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len())?; let metadata_slice = Self::get_byte_slice(metadata_slice, 0, metadata_len)?; - let metadata_slice: &[u8] = &metadata_slice[..metadata_len]; let serializable = SerializableModule::deserialize(metadata_slice)?; let artifact = ArtifactBuild::from_serializable(serializable); let mut inner_engine = engine.inner_mut(); @@ -602,6 +601,7 @@ impl Artifact { ) -> Result { let metadata_len = MetadataHeader::parse(bytes)?; let metadata_slice = Self::get_byte_slice(bytes, MetadataHeader::LEN, bytes.len())?; + let metadata_slice = Self::get_byte_slice(metadata_slice, 0, metadata_len)?; let metadata: ModuleMetadata = ModuleMetadata::deserialize(metadata_slice)?; const WORD_SIZE: usize = mem::size_of::(); From e65612085313b51a40f71d1dff3ed9ad2c4a9997 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Fri, 26 Aug 2022 12:46:02 +0200 Subject: [PATCH 39/40] Bumped rust toolchain to 1.61 --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 791d8759b..4213d88dc 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.59 +1.61 From 02b36a74d9d4bebafddddaa412c68a7d4e6d07fe Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Fri, 26 Aug 2022 15:23:22 +0200 Subject: [PATCH 40/40] Also bumpped CI to 1.61 --- .github/workflows/benchmark.yaml | 2 +- .github/workflows/build.yml | 4 ++-- .github/workflows/cloudcompiler.yaml | 2 +- .github/workflows/coverage.yaml | 2 +- .github/workflows/documentation.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test-js.yaml | 2 +- .github/workflows/test-sys.yaml | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 3c27a2dd0..a95994c99 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -24,7 +24,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 - name: Configure cargo data directory # After this point, all cargo registry and crate data is stored in # $GITHUB_WORKSPACE/.cargo_home. This allows us to cache only the files diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3b793205c..8411bd32a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -100,7 +100,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 target: ${{ matrix.target }} - uses: Swatinem/rust-cache@v1 if: matrix.use_sccache != true @@ -270,7 +270,7 @@ jobs: - uses: actions/checkout@v2 - uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 target: aarch64-unknown-linux-gnu - name: Build cross image run: | diff --git a/.github/workflows/cloudcompiler.yaml b/.github/workflows/cloudcompiler.yaml index 059b87c82..a296b312f 100644 --- a/.github/workflows/cloudcompiler.yaml +++ b/.github/workflows/cloudcompiler.yaml @@ -21,7 +21,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 target: ${{ matrix.target }} - name: Install wasm32-wasi target shell: bash diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index ce0475081..545696a35 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -21,7 +21,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 - name: Install LLVM (Linux) run: | curl --proto '=https' --tlsv1.2 -sSf https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz -L -o llvm.tar.xz diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index ddcfcc98f..0096c315d 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -16,7 +16,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 - name: Install LLVM shell: bash run: | diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index ad1de3fcb..9e3a58101 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -18,7 +18,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 components: rustfmt, clippy - name: Install LLVM (Linux) run: | diff --git a/.github/workflows/test-js.yaml b/.github/workflows/test-js.yaml index a8007cb35..486eac08b 100644 --- a/.github/workflows/test-js.yaml +++ b/.github/workflows/test-js.yaml @@ -33,7 +33,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 - name: Install NodeJS uses: actions/setup-node@v2 diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index ae7089335..3afacb5b5 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -110,7 +110,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.59 + toolchain: 1.61 target: ${{ matrix.target }} - uses: Swatinem/rust-cache@v1 if: matrix.use_sccache != true