Merge branch 'master' into feat-c-api-cross-compilation-2

CHANGELOG.md

@@ -14,11 +14,13 @@
 - [#2054](https://github.com/wasmerio/wasmer/pull/2054) Add `wasm_config_delete` to the Wasm C API.
 
 ### Changed
-- [#2042](https://github.com/wasmerio/wasmer/pull/2042) Parse more exotic environment variables in `wasmer run`.
-- [#2041](https://github.com/wasmerio/wasmer/pull/2041) Documentation diagrams now have a solid white background rather than a transparent background.
 - [#2056](https://github.com/wasmerio/wasmer/pull/2056) Change back to depend on the `enumset` crate instead of `wasmer_enumset`
 
 ### Fixed
+- [#2069](https://github.com/wasmerio/wasmer/pull/2069) Use the new documentation for `include/README.md` in the Wasmer package.
+- [#2042](https://github.com/wasmerio/wasmer/pull/2042) Parse more exotic environment variables in `wasmer run`.
+- [#2041](https://github.com/wasmerio/wasmer/pull/2041) Documentation diagrams now have a solid white background rather than a transparent background.
+- [#2070](https://github.com/wasmerio/wasmer/pull/2070) Do not drain the entire captured stream at first read with `wasi_env_read_stdout` or `_stderr` in the C API.
 - [#2058](https://github.com/wasmerio/wasmer/pull/2058) Expose WASI versions to C correctly.
 - [#2044](https://github.com/wasmerio/wasmer/pull/2044) Do not build C headers on docs.rs.
 

Cargo.lock

@@ -1528,15 +1528,6 @@ dependencies = [
  "rand_core",
 ]
 
-[[package]]
-name = "raw-cpuid"
-version = "9.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c27cb5785b85bd05d4eb171556c9a1a514552e26123aeae6bb7d811353148026"
-dependencies = [
- "bitflags",
-]
-
 [[package]]
 name = "raw-window-handle"
 version = "0.3.3"
@@ -2400,7 +2391,6 @@ version = "1.0.1"
 dependencies = [
  "enumset",
  "hashbrown 0.9.1",
- "raw-cpuid",
  "serde",
  "serde_bytes",
  "smallvec",

Makefile

@@ -2,10 +2,12 @@
 ifneq ($(OS), Windows_NT)
 	ARCH := $(shell uname -m)
 	UNAME_S := $(shell uname -s)
+	LIBC ?= $(shell ldd 2>&1 | grep -o musl | head -n1)
 else
 	# We can assume, if in windows it will likely be in x86_64
 	ARCH := x86_64
 	UNAME_S :=
+	LIBC ?=
 endif
 
 # Which compilers we build. These have dependencies that may not be on the system.
@@ -38,14 +40,21 @@ ifeq ($(ARCH), x86_64)
 	test_compilers_engines += cranelift-jit
 	# LLVM could be enabled if not in Windows
 	ifneq ($(OS), Windows_NT)
-		# Native engine doesn't work on Windows yet.
+		ifneq ($(LIBC), musl)
-		test_compilers_engines += cranelift-native
+			# Native engine doesn't work on Windows and musl yet.
+			test_compilers_engines += cranelift-native
+		endif
 		# Singlepass doesn't work yet on Windows.
 		compilers += singlepass
 		# Singlepass doesn't work with the native engine.
 		test_compilers_engines += singlepass-jit
 		ifneq (, $(findstring llvm,$(compilers)))
-			test_compilers_engines += llvm-jit llvm-native
+			test_compilers_engines += llvm-jit
+
+			ifneq ($(LIBC), musl)
+				# Native engine doesn't work on musl yet.
+				test_compilers_engines += llvm-native
+			endif
 		endif
 	endif
 endif
@@ -79,9 +88,9 @@ compilers := $(filter-out ,$(compilers))
 test_compilers_engines := $(filter-out ,$(test_compilers_engines))
 
 ifneq ($(OS), Windows_NT)
-	bold := $(shell tput bold)
+	bold := $(shell tput bold 2>/dev/null || echo -n '')
-	green := $(shell tput setaf 2)
+	green := $(shell tput setaf 2 2>/dev/null || echo -n '')
-	reset := $(shell tput sgr0)
+	reset := $(shell tput sgr0 2>/dev/null || echo -n '')
 endif
 
 
@@ -90,6 +99,10 @@ compiler_features := --features "$(compiler_features_spaced)"
 
 HOST_TARGET=$(shell rustup show | grep 'Default host: ' | cut -d':' -f2 | tr -d ' ')
 
+ifneq (, $(LIBC))
+$(info C standard library: $(bold)$(green)$(LIBC)$(reset))
+endif
+
 $(info Host target: $(bold)$(green)$(HOST_TARGET)$(reset))
 $(info Available compilers: $(bold)$(green)${compilers}$(reset))
 $(info Compilers features: $(bold)$(green)${compiler_features}$(reset))
@@ -363,7 +376,7 @@ package-capi:
 	cp lib/c-api/wasmer.h* package/include
 	cp lib/c-api/wasmer_wasm.h* package/include
 	cp lib/c-api/wasm.h* package/include
-	cp lib/c-api/doc/deprecated/index.md package/include/README.md
+	cp lib/c-api/README.md package/include/README.md
 ifeq ($(OS), Windows_NT)
 	cp target/release/wasmer_c_api.dll package/lib
 	cp target/release/wasmer_c_api.lib package/lib
@@ -376,7 +389,10 @@ ifeq ($(UNAME_S), Darwin)
 	# Fix the rpath for the dylib
 	install_name_tool -id "@rpath/libwasmer.dylib" package/lib/libwasmer.dylib
 else
-	cp target/release/libwasmer_c_api.so package/lib/libwasmer.so
+	# In some cases the .so may not be available, for example when building against musl (static linking)
+	if [ -f target/release/libwasmer_c_api.so ]; then \
+		cp target/release/libwasmer_c_api.so package/lib/libwasmer.so ;\
+	fi;
 	cp target/release/libwasmer_c_api.a package/lib/libwasmer.a
 endif
 endif

examples/engine_cross_compilation.rs

@@ -102,7 +102,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 #[cfg(not(any(
     windows,
     // We don't support yet crosscompilation in macOS with Apple Silicon
-    all(target_os = "macos", target_arch = "aarch64")
+    all(target_os = "macos", target_arch = "aarch64"),
+    target_env = "musl",
 )))]
 fn test_cross_compilation() -> Result<(), Box<dyn std::error::Error>> {
     main()

examples/engine_headless.rs

@@ -147,7 +147,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 }
 
 #[test]
-#[cfg(not(any(windows, target_arch = "aarch64")))]
+#[cfg(not(any(windows, target_arch = "aarch64", target_env = "musl")))]
 fn test_engine_headless() -> Result<(), Box<dyn std::error::Error>> {
     main()
 }

examples/engine_native.rs

@@ -87,7 +87,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 }
 
 #[test]
-#[cfg(not(target_arch = "aarch64"))]
+#[cfg(not(any(target_arch = "aarch64", target_env = "musl")))]
 fn test_engine_native() -> Result<(), Box<dyn std::error::Error>> {
     main()
 }

lib/c-api/src/wasm_c_api/wasi/mod.rs

@@ -13,6 +13,7 @@ use super::{
 // required due to really weird Rust resolution rules for macros
 // https://github.com/rust-lang/rust/issues/57966
 use crate::error::{update_last_error, CApiError};
+use std::cmp::min;
 use std::convert::TryFrom;
 use std::ffi::CStr;
 use std::os::raw::c_char;
@@ -275,12 +276,16 @@ pub unsafe extern "C" fn wasi_env_read_stderr(
 
 fn read_inner(wasi_file: &mut Box<dyn WasiFile>, inner_buffer: &mut [u8]) -> isize {
     if let Some(oc) = wasi_file.downcast_mut::<capture_files::OutputCapturer>() {
-        let mut num_bytes_written = 0;
+        let total_to_read = min(inner_buffer.len(), oc.buffer.len());
-        for (address, value) in inner_buffer.iter_mut().zip(oc.buffer.drain(..)) {
+
+        for (address, value) in inner_buffer
+            .iter_mut()
+            .zip(oc.buffer.drain(..total_to_read))
+        {
             *address = value;
-            num_bytes_written += 1;
         }
-        num_bytes_written
+
+        total_to_read as isize
     } else {
         -1
     }

lib/c-api/tests/test-wasi.c

@@ -58,6 +58,7 @@ int main(int argc, const char* argv[]) {
   const char* js_string = "function greet(name) { return JSON.stringify('Hello, ' + name); }; print(greet('World'));";
   wasi_config_arg(config, "--eval");
   wasi_config_arg(config, js_string);
+  wasi_config_capture_stdout(config);
 
   wasi_env_t* wasi_env = wasi_env_new(config);
   if (!wasi_env) {
@@ -125,16 +126,35 @@ int main(int argc, const char* argv[]) {
     return 1;
   }
 
-  char buffer[BUF_SIZE] = { 0 };
+  {
-  size_t result = BUF_SIZE;
+    FILE *memory_stream;
-  for (size_t i = 0;
+    char* stdout;
-       // TODO: this code is too clever, make the control flow more obvious here
+    size_t stdout_size = 0;
-       result == BUF_SIZE &&
+
-               (result = wasi_env_read_stdout(wasi_env, buffer, BUF_SIZE));
+    memory_stream = open_memstream(&stdout, &stdout_size);
-       ++i) {
+
-     printf("%.*s", BUF_SIZE, buffer);
+    if (NULL == memory_stream) {
+      printf("> Error creating a memory stream.\n");
+      return 1;
+    }
+
+    char buffer[BUF_SIZE] = { 0 };
+    size_t data_read_size = BUF_SIZE;
+
+    do {
+      data_read_size = wasi_env_read_stdout(wasi_env, buffer, BUF_SIZE);
+
+      if (data_read_size > 0) {
+        stdout_size += data_read_size;
+        fwrite(buffer, sizeof(char), data_read_size, memory_stream);
+      }
+    } while (BUF_SIZE == data_read_size);
+
+    fclose(memory_stream);
+
+    printf("WASI Stdout: `%.*s`\n", (int) stdout_size, stdout);
   }
-  printf("\n");
+
 
   wasm_extern_vec_delete(&exports);
   wasm_extern_vec_delete(&imports);

lib/compiler/Cargo.toml

@@ -22,9 +22,6 @@ thiserror = "1.0"
 serde_bytes = { version = "0.11", optional = true }
 smallvec = "1.6" 
 
-[target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]
-raw-cpuid = "9.0"
-
 [features]
 default = ["std", "enable-serde"]
 # This feature is for compiler implementors, it enables using `Compiler` and

lib/compiler/src/target.rs

@@ -8,9 +8,6 @@ pub use target_lexicon::{
     Triple,
 };
 
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-use raw_cpuid::CpuId;
-
 /// The nomenclature is inspired by the [`cpuid` crate].
 /// The list of supported features was initially retrieved from
 /// [`cranelift-native`].
@@ -39,6 +36,7 @@ pub enum CpuFeature {
     AVX2,
     AVX512DQ,
     AVX512VL,
+    AVX512F,
     LZCNT,
     // ARM features
     // Risc-V features
@@ -49,52 +47,48 @@ impl CpuFeature {
     /// Retrieves the features for the current Host
     pub fn for_host() -> EnumSet<Self> {
         let mut features = EnumSet::new();
-        let cpuid = CpuId::new();
 
-        if let Some(info) = cpuid.get_feature_info() {
+        if std::is_x86_feature_detected!("sse2") {
-            if info.has_sse2() {
+            features.insert(Self::SSE2);
-                features.insert(Self::SSE2);
-            }
-            if info.has_sse3() {
-                features.insert(Self::SSE3);
-            }
-            if info.has_ssse3() {
-                features.insert(Self::SSSE3);
-            }
-            if info.has_sse41() {
-                features.insert(Self::SSE41);
-            }
-            if info.has_sse42() {
-                features.insert(Self::SSE42);
-            }
-            if info.has_popcnt() {
-                features.insert(Self::POPCNT);
-            }
-            if info.has_avx() {
-                features.insert(Self::AVX);
-            }
         }
-        if let Some(info) = cpuid.get_extended_feature_info() {
+        if std::is_x86_feature_detected!("sse3") {
-            if info.has_bmi1() {
+            features.insert(Self::SSE3);
-                features.insert(Self::BMI1);
-            }
-            if info.has_bmi2() {
-                features.insert(Self::BMI2);
-            }
-            if info.has_avx2() {
-                features.insert(Self::AVX2);
-            }
-            if info.has_avx512dq() {
-                features.insert(Self::AVX512DQ);
-            }
-            if info.has_avx512vl() {
-                features.insert(Self::AVX512VL);
-            }
         }
-        if let Some(info) = cpuid.get_extended_function_info() {
+        if std::is_x86_feature_detected!("ssse3") {
-            if info.has_lzcnt() {
+            features.insert(Self::SSSE3);
-                features.insert(Self::LZCNT);
+        }
-            }
+        if std::is_x86_feature_detected!("sse4.1") {
+            features.insert(Self::SSE41);
+        }
+        if std::is_x86_feature_detected!("sse4.2") {
+            features.insert(Self::SSE42);
+        }
+        if std::is_x86_feature_detected!("popcnt") {
+            features.insert(Self::POPCNT);
+        }
+        if std::is_x86_feature_detected!("avx") {
+            features.insert(Self::AVX);
+        }
+        if std::is_x86_feature_detected!("bmi1") {
+            features.insert(Self::BMI1);
+        }
+        if std::is_x86_feature_detected!("bmi2") {
+            features.insert(Self::BMI2);
+        }
+        if std::is_x86_feature_detected!("avx2") {
+            features.insert(Self::AVX2);
+        }
+        if std::is_x86_feature_detected!("avx512dq") {
+            features.insert(Self::AVX512DQ);
+        }
+        if std::is_x86_feature_detected!("avx512vl") {
+            features.insert(Self::AVX512VL);
+        }
+        if std::is_x86_feature_detected!("avx512f") {
+            features.insert(Self::AVX512F);
+        }
+        if std::is_x86_feature_detected!("lzcnt") {
+            features.insert(Self::LZCNT);
         }
         features
     }
@@ -135,6 +129,7 @@ impl FromStr for CpuFeature {
             "avx2" => Ok(Self::AVX2),
             "avx512dq" => Ok(Self::AVX512DQ),
             "avx512vl" => Ok(Self::AVX512VL),
+            "avx512f" => Ok(Self::AVX512F),
             "lzcnt" => Ok(Self::LZCNT),
             _ => Err(ParseCpuFeatureError::Missing(s.to_string())),
         }
@@ -156,6 +151,7 @@ impl ToString for CpuFeature {
             Self::AVX2 => "avx2",
             Self::AVX512DQ => "avx512dq",
             Self::AVX512VL => "avx512vl",
+            Self::AVX512F => "avx512f",
             Self::LZCNT => "lzcnt",
         }
         .to_string()

tests/compilers/traps.rs

@@ -43,6 +43,7 @@ fn test_trap_return() -> Result<()> {
         feature = "test-singlepass",
         feature = "test-native",
         target_arch = "aarch64",
+        target_env = "musl",
     ),
     ignore
 )]
@@ -130,6 +131,7 @@ fn test_trap_trace_cb() -> Result<()> {
         feature = "test-singlepass",
         feature = "test-native",
         target_arch = "aarch64",
+        target_env = "musl",
     ),
     ignore
 )]
@@ -169,6 +171,7 @@ fn test_trap_stack_overflow() -> Result<()> {
         feature = "test-llvm",
         feature = "test-native",
         target_arch = "aarch64",
+        target_env = "musl",
     ),
     ignore
 )]
@@ -210,6 +213,7 @@ RuntimeError: unreachable
         feature = "test-llvm",
         feature = "test-native",
         target_arch = "aarch64",
+        target_env = "musl",
     ),
     ignore
 )]
@@ -424,7 +428,8 @@ fn mismatched_arguments() -> Result<()> {
         feature = "test-singlepass",
         feature = "test-llvm",
         feature = "test-native",
-        all(target_os = "macos", target_arch = "aarch64")
+        all(target_os = "macos", target_arch = "aarch64"),
+        target_env = "musl",
     ),
     ignore
 )]
@@ -464,6 +469,7 @@ RuntimeError: indirect call type mismatch
         feature = "test-llvm",
         feature = "test-native",
         target_arch = "aarch64",
+        target_env = "musl",
     ),
     ignore
 )]