diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 9e7bd97f3..ec416d9c8 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -234,9 +234,9 @@ fn wasmer_main_inner() -> Result<(), anyhow::Error> { WasmerCLIOptions::Run(Run::from_binfmt_args()) } else { match command.unwrap_or(&"".to_string()).as_ref() { - "add" | "cache" | "compile" | "config" | "create-exe" | "help" | "inspect" | "init" - | "run" | "self-update" | "validate" | "wast" | "binfmt" | "list" | "login" - | "publish" => WasmerCLIOptions::parse(), + "add" | "cache" | "compile" | "config" | "create-obj" | "create-exe" | "help" + | "inspect" | "init" | "run" | "self-update" | "validate" | "wast" | "binfmt" + | "list" | "login" | "publish" => WasmerCLIOptions::parse(), _ => { WasmerCLIOptions::try_parse_from(args.iter()).unwrap_or_else(|e| { match e.kind() { diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index ed3190c7c..8220fe81c 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -112,7 +112,7 @@ pub(crate) struct CrossCompile { pub(crate) struct CrossCompileSetup { pub(crate) target: Triple, pub(crate) zig_binary_path: Option, - pub(crate) library: Option, + pub(crate) library: PathBuf, } /// Given a pirita file, determines whether the file has one @@ -167,7 +167,21 @@ impl CreateExe { if input_path.is_dir() { return Err(anyhow::anyhow!("input path cannot be a directory")); - } else if let Ok(pirita) = WebCMmap::parse(input_path.clone(), &ParseOptions::default()) { + } + + let (_, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + println!("Compiler: {}", compiler_type.to_string()); + println!("Target: {}", target.triple()); + println!("Format: {:?}", object_format); + println!( + "Using path `{}` as libwasmer path.", + cross_compilation.library.display() + ); + if !cross_compilation.library.exists() { + return Err(anyhow::anyhow!("library path does not exist")); + } + + if let Ok(pirita) = WebCMmap::parse(input_path.clone(), &ParseOptions::default()) { // pirita file let temp = tempdir::TempDir::new("pirita-compile")?; let tempdir = match self.debug_dir.as_ref() { @@ -394,7 +408,9 @@ pub(super) fn compile_pirita_into_directory( )); } - let prefix_map = PrefixMapCompilation::from_input(&atoms_from_file, prefixes, false)?; + let prefix_map = PrefixMapCompilation::from_input(&atoms_from_file, prefixes, false) + .with_context(|| anyhow::anyhow!("compile_pirita_into_directory"))?; + let module_infos = conpile_atoms( &atoms_from_file, &target_dir.join("atoms"), @@ -439,7 +455,7 @@ pub(super) fn compile_pirita_into_directory( } /// Prefix map used during compilation of object files -#[derive(Debug, Default)] +#[derive(Debug, Default, PartialEq)] struct PrefixMapCompilation { /// Sha256 hashes for the input files input_hashes: BTreeMap, @@ -450,17 +466,45 @@ struct PrefixMapCompilation { compilation_objects: BTreeMap>, } +#[test] +fn test_prefix_parsing() { + let tempdir = tempdir::TempDir::new("test-prefix-parsing").unwrap(); + let path = tempdir.path(); + std::fs::write(path.join("test.obj"), b"").unwrap(); + let str1 = format!("ATOM_NAME:{}:PREFIX", path.join("test.obj").display()); + let prefix = PrefixMapCompilation::from_input( + &[("ATOM_NAME".to_string(), b"".to_vec())], + &[str1.to_string()], + false, + ); + assert_eq!( + prefix.unwrap(), + PrefixMapCompilation { + input_hashes: BTreeMap::new(), + manual_prefixes: vec![("ATOM_NAME".to_string(), "PREFIX".to_string())] + .into_iter() + .collect(), + compilation_objects: vec![("ATOM_NAME".to_string(), b"".to_vec())] + .into_iter() + .collect(), + } + ); +} + impl PrefixMapCompilation { /// Sets up the prefix map from a collection like "sha123123" or "wasmfile:sha123123" or "wasmfile:/tmp/filepath/:sha123123" fn from_input( atoms: &[(String, Vec)], prefixes: &[String], - compilation_object_mode: bool, + only_validate_prefixes: bool, ) -> Result { + // No atoms: no prefixes necessary if atoms.is_empty() { return Ok(Self::default()); } + // default case: no prefixes have been specified: + // for all atoms, calculate the sha256 if prefixes.is_empty() { return Ok(Self { input_hashes: atoms @@ -472,6 +516,7 @@ impl PrefixMapCompilation { }); } + // if prefixes are specified, have to match the atom names exactly if prefixes.len() != atoms.len() { return Err(anyhow::anyhow!( "invalid mapping of prefix and atoms: expected prefixes for {} atoms, got {} prefixes", @@ -479,72 +524,56 @@ impl PrefixMapCompilation { )); } - // Only on single atom mapping is using a raw input allowed, all other prefix - // have to carry something like "nameofatom:sha256hash" instead of just "sha256hash" - if atoms.len() == 1 && !compilation_object_mode { - let prefix = &prefixes[0]; - let (atom_name, _atom_bytes) = &atoms[0]; - let atom_prefix = format!("{atom_name}:"); - - if prefix.contains(':') && !prefix.contains(&atom_prefix) { - return Err(anyhow::anyhow!("invalid prefix in prefix {prefix}")); - } - - let prefix_without_atom_name = prefix.replacen(&atom_prefix, "", 1); - if !prefix_without_atom_name - .chars() - .all(|c| c.is_alphanumeric() || c == '_' || c == '-') - { - return Err(anyhow::anyhow!("invalid prefix {prefix}")); - } - return Ok(Self { - input_hashes: BTreeMap::new(), - manual_prefixes: IntoIterator::into_iter([( - atom_name.clone(), - prefix_without_atom_name, - )]) - .collect(), - compilation_objects: BTreeMap::new(), - }); - } - + let available_atoms = atoms.iter().map(|(k, _)| k.clone()).collect::>(); let mut manual_prefixes = BTreeMap::new(); let mut compilation_objects = BTreeMap::new(); - for (atom_name, _atom_bytes) in atoms { - let prefix_start_str = format!("{atom_name}:"); - let prefix = match prefixes.iter().find(|p| p.contains(&prefix_start_str)) { - Some(s) => s, - None => { - return Err(anyhow::anyhow!( - "could not find prefix for atom {atom_name:?}" - )) - } - }; - let prefix_without_atom_name = prefix.replacen(&prefix_start_str, "", 1); - match prefix_without_atom_name - .split(':') - .collect::>() - .as_slice() - { - &[path, prefix] => { - let bytes = std::fs::read(path).map_err(|e| { - anyhow::anyhow!("could not read file for prefix {prefix} ({path}): {e}") - })?; - compilation_objects.insert(atom_name.clone(), bytes); - } - _ => { - if compilation_object_mode { - return Err(anyhow::anyhow!("invalid prefix format {prefix}")); + + for p in prefixes.iter() { + let prefix_split = p.split(':').collect::>(); + + match prefix_split.as_slice() { + // ATOM:PATH:PREFIX + &[atom, path, prefix] => { + if only_validate_prefixes { + // only insert the prefix in order to not error out of the fs::read(path) + manual_prefixes.insert(atom.to_string(), prefix.to_string()); + } else { + let atom_hash = atoms + .iter() + .find_map(|(name, _)| if name == atom { Some(prefix) } else { None }) + .ok_or_else(|| anyhow::anyhow!("no atom {atom:?} found, for prefix {p:?}, available atoms are {available_atoms:?}"))?; + + let bytes = std::fs::read(path).map_err(|e| { + anyhow::anyhow!("could not read file for prefix {p} ({path}): {e}") + })?; + + compilation_objects.insert(atom.to_string(), bytes); + manual_prefixes.insert(atom.to_string(), atom_hash.to_string()); } } - }; - if !prefix_without_atom_name - .chars() - .all(|c| c.is_alphanumeric() || c == '_' || c == '-') - { - return Err(anyhow::anyhow!("invalid prefix {prefix}")); + // atom + path, but default SHA256 prefix + &[atom, path] => { + let atom_hash = atoms + .iter() + .find_map(|(name, bytes)| if name == atom { Some(Self::hash_for_bytes(bytes)) } else { None }) + .ok_or_else(|| anyhow::anyhow!("no atom {atom:?} found, for prefix {p:?}, available atoms are {available_atoms:?}"))?; + manual_prefixes.insert(atom.to_string(), atom_hash.to_string()); + + if !only_validate_prefixes { + let bytes = std::fs::read(path).map_err(|e| { + anyhow::anyhow!("could not read file for prefix {p} ({path}): {e}") + })?; + compilation_objects.insert(atom.to_string(), bytes); + } + } + // only prefix if atoms.len() == 1 + &[prefix] if atoms.len() == 1 => { + manual_prefixes.insert(atoms[0].0.clone(), prefix.to_string()); + } + _ => { + return Err(anyhow::anyhow!("invalid --precompiled-atom {p:?} - correct format is ATOM:PATH:PREFIX or ATOM:PATH")); + } } - manual_prefixes.insert(atom_name.clone(), prefix_without_atom_name); } Ok(Self { @@ -764,7 +793,9 @@ pub(super) fn prepare_directory_from_single_wasm_file( target_paths.push((atom_name, atom_path)); } - let prefix_map = PrefixMapCompilation::from_input(&atoms_from_file, prefix, false)?; + let prefix_map = PrefixMapCompilation::from_input(&atoms_from_file, prefix, false) + .with_context(|| anyhow::anyhow!("prepare_directory_from_single_wasm_file"))?; + let module_infos = conpile_atoms( &atoms_from_file, &target_dir.join("atoms"), @@ -846,7 +877,8 @@ fn create_header_files_in_dir( anyhow::anyhow!("cannot create /include dir in {}: {e}", directory.display()) })?; - let prefixes = PrefixMapCompilation::from_input(atoms, prefixes, false)?; + let prefixes = PrefixMapCompilation::from_input(atoms, prefixes, false) + .with_context(|| anyhow::anyhow!("create_header_files_in_dir"))?; for atom in entrypoint.atoms.iter_mut() { let atom_name = &atom.atom; @@ -921,7 +953,9 @@ fn link_exe_from_dir( let entrypoint = get_entrypoint(directory).with_context(|| anyhow::anyhow!("link exe from dir"))?; - let prefixes = PrefixMapCompilation::from_input(atoms, prefixes, true)?; + let prefixes = PrefixMapCompilation::from_input(atoms, prefixes, true) + .with_context(|| anyhow::anyhow!("link_exe_from_dir"))?; + let wasmer_main_c = generate_wasmer_main_c(&entrypoint, &prefixes).map_err(|e| { anyhow::anyhow!( "could not generate wasmer_main.c in dir {}: {e}", @@ -936,10 +970,7 @@ fn link_exe_from_dir( ) })?; - let library_path = cross_compilation - .library - .as_ref() - .ok_or_else(|| anyhow::anyhow!("libwasmer.a / wasmer.lib not found"))?; + let library_path = &cross_compilation.library; let mut object_paths = entrypoint .atoms @@ -1132,10 +1163,6 @@ fn link_objects_system_linker( let libwasmer_path = libwasmer_path .canonicalize() .context("Failed to find libwasmer")?; - println!( - "Using path `{}` as libwasmer path.", - libwasmer_path.display() - ); let mut command = Command::new(linker_cmd); let command = command .arg("-Wall") @@ -1450,6 +1477,10 @@ pub(super) mod utils { .ok() .map(|(filename, tarball_dir)| tarball_dir.join(&filename)) }; + + let library = + library.ok_or_else(|| anyhow::anyhow!("libwasmer.a / wasmer.lib not found"))?; + let ccs = CrossCompileSetup { target: target.clone(), zig_binary_path, diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs index 988ee0c07..39aefb284 100644 --- a/lib/cli/src/commands/create_obj.rs +++ b/lib/cli/src/commands/create_obj.rs @@ -81,11 +81,22 @@ impl CreateObj { Some(s) => s.as_path(), None => temp_dir.path(), }; + std::fs::create_dir_all(&output_directory_path)?; let object_format = self.object_format.unwrap_or_default(); let prefix = match self.prefix.as_ref() { Some(s) => vec![s.clone()], None => Vec::new(), }; + + let target = crate::commands::create_exe::utils::target_triple_to_target( + &target_triple, + &self.cpu_features, + ); + let (_, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + println!("Compiler: {}", compiler_type.to_string()); + println!("Target: {}", target.triple()); + println!("Format: {:?}", object_format); + let atoms = if let Ok(pirita) = WebCMmap::parse(input_path.clone(), &ParseOptions::default()) { crate::commands::create_exe::compile_pirita_into_directory( @@ -119,7 +130,7 @@ impl CreateObj { output_directory_path.join("atoms").display() ) })? - .filter_map(|path| Some(path.ok()?.path())) + .filter_map(|path| Some(path.ok()?.path().canonicalize().ok()?)) .collect::>(); if file_paths.is_empty() { @@ -130,7 +141,20 @@ impl CreateObj { } if file_paths.len() == 1 { - std::fs::copy(&file_paths[0], &self.output)?; + if let Some(parent) = self.output.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::copy( + std::env::current_dir().unwrap().join(&file_paths[0]), + std::env::current_dir().unwrap().join(&self.output), + ) + .map_err(|e| { + anyhow::anyhow!( + "{} -> {}: {e}", + &file_paths[0].display(), + self.output.display() + ) + })?; } else { let keys = atoms .iter() @@ -150,7 +174,7 @@ impl CreateObj { eprintln!( "✔ Object compiled successfully to directory `{}`", - self.output.display() + self.output.canonicalize().unwrap().display() ); Ok(()) diff --git a/tests/integration/cli/tests/create_exe.rs b/tests/integration/cli/tests/create_exe.rs index 88393d165..831712d75 100644 --- a/tests/integration/cli/tests/create_exe.rs +++ b/tests/integration/cli/tests/create_exe.rs @@ -31,7 +31,7 @@ struct WasmerCreateExe { /// Compiler with which to compile the Wasm. compiler: Compiler, /// Extra CLI flags - extra_cli_flags: Vec<&'static str>, + extra_cli_flags: Vec, } impl Default for WasmerCreateExe { @@ -53,19 +53,22 @@ impl Default for WasmerCreateExe { impl WasmerCreateExe { fn run(&self) -> anyhow::Result> { - let output = Command::new(&self.wasmer_path) - .current_dir(&self.current_dir) - .arg("create-exe") - .arg(&self.wasm_path.canonicalize()?) - .arg(&self.compiler.to_flag()) - .args(self.extra_cli_flags.iter()) - .arg("-o") - .arg(&self.native_executable_path) - .output()?; + let mut output = Command::new(&self.wasmer_path); + output.current_dir(&self.current_dir); + output.arg("create-exe"); + output.arg(&self.wasm_path.canonicalize()?); + output.arg(&self.compiler.to_flag()); + output.args(self.extra_cli_flags.iter()); + output.arg("-o"); + output.arg(&self.native_executable_path); + + let cmd = format!("{:?}", output); + + let output = output.output()?; if !output.status.success() { bail!( - "wasmer create-exe failed with: stdout: {}\n\nstderr: {}", + "{cmd}\r\n failed with: stdout: {}\n\nstderr: {}", std::str::from_utf8(&output.stdout) .expect("stdout is not utf8! need to handle arbitrary bytes"), std::str::from_utf8(&output.stderr) @@ -90,7 +93,7 @@ struct WasmerCreateObj { /// Compiler with which to compile the Wasm. compiler: Compiler, /// Extra CLI flags - extra_cli_flags: Vec<&'static str>, + extra_cli_flags: Vec, } impl Default for WasmerCreateObj { @@ -112,19 +115,22 @@ impl Default for WasmerCreateObj { impl WasmerCreateObj { fn run(&self) -> anyhow::Result> { - let output = Command::new(&self.wasmer_path) - .current_dir(&self.current_dir) - .arg("create-obj") - .arg(&self.wasm_path.canonicalize()?) - .arg(&self.compiler.to_flag()) - .args(self.extra_cli_flags.iter()) - .arg("-o") - .arg(&self.output_object_path) - .output()?; + let mut output = Command::new(&self.wasmer_path); + output.current_dir(&self.current_dir); + output.arg("create-obj"); + output.arg(&self.wasm_path.canonicalize()?); + output.arg(&self.compiler.to_flag()); + output.args(self.extra_cli_flags.iter()); + output.arg("-o"); + output.arg(&self.output_object_path); + + let cmd = format!("{:?}", output); + + let output = output.output()?; if !output.status.success() { bail!( - "wasmer create-obj failed with: stdout: {}\n\nstderr: {}", + "{cmd}\r\n failed with: stdout: {}\n\nstderr: {}", std::str::from_utf8(&output.stdout) .expect("stdout is not utf8! need to handle arbitrary bytes"), std::str::from_utf8(&output.stderr) @@ -180,7 +186,7 @@ fn create_exe_works_multi_command() -> anyhow::Result<()> { let executable_path = operating_dir.join("multicommand.exe"); WasmerCreateExe { - current_dir: std::env::current_dir().unwrap(), // operating_dir.clone(), + current_dir: operating_dir.clone(), wasm_path, native_executable_path: executable_path.clone(), compiler: Compiler::Cranelift, @@ -192,14 +198,14 @@ fn create_exe_works_multi_command() -> anyhow::Result<()> { let _result = run_code( &operating_dir, &executable_path, - &["--command".to_string(), "wabt".to_string()], + &["--command".to_string(), "wasm2wat".to_string()], ) .context("Failed to run generated executable")?; let result = run_code( &operating_dir, &executable_path, - &["-c".to_string(), "wabt".to_string()], + &["-c".to_string(), "wasm-validate".to_string()], ) .context("Failed to run generated executable")?; @@ -281,11 +287,11 @@ fn create_exe_serialized_works() -> anyhow::Result<()> { let executable_path = operating_dir.join("wasm.exe"); let output: Vec = WasmerCreateExe { - current_dir: operating_dir.clone(), + current_dir: std::env::current_dir().unwrap(), wasm_path, native_executable_path: executable_path.clone(), compiler: Compiler::Cranelift, - extra_cli_flags: vec!["--object-format", "serialized"], + extra_cli_flags: vec!["--object-format".to_string(), "serialized".to_string()], ..Default::default() } .run() @@ -310,7 +316,7 @@ fn create_exe_serialized_works() -> anyhow::Result<()> { Ok(()) } -fn create_obj(args: Vec<&'static str>, keyword_needle: &str, keyword: &str) -> anyhow::Result<()> { +fn create_obj(args: Vec, keyword_needle: &str, keyword: &str) -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -333,16 +339,6 @@ fn create_obj(args: Vec<&'static str>, keyword_needle: &str, keyword: &str) -> a "create-obj successfully completed but object output file `{}` missing", object_path.display() ); - let object_header_path = operating_dir - .as_path() - .join("wasm") - .join("include") - .join("static_defs_main.h"); - assert!( - object_header_path.exists(), - "create-obj successfully completed but object output header file `{}` missing", - object_header_path.display() - ); let output_str = String::from_utf8_lossy(&output); assert!( @@ -362,19 +358,23 @@ fn create_obj_default() -> anyhow::Result<()> { #[test] fn create_obj_symbols() -> anyhow::Result<()> { - create_obj(vec!["--object-format", "symbols"], "Symbols", "symbols") + create_obj( + vec!["--object-format".to_string(), "symbols".to_string()], + "Symbols", + "symbols", + ) } #[test] fn create_obj_serialized() -> anyhow::Result<()> { create_obj( - vec!["--object-format", "serialized"], + vec!["--object-format".to_string(), "serialized".to_string()], "Serialized", "serialized", ) } -fn create_exe_with_object_input(args: Vec<&'static str>) -> anyhow::Result<()> { +fn create_exe_with_object_input(mut args: Vec) -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -385,9 +385,14 @@ fn create_exe_with_object_input(args: Vec<&'static str>) -> anyhow::Result<()> { #[cfg(windows)] let object_path = operating_dir.join("wasm.obj"); + args.push("--prefix".to_string()); + args.push("abc123".to_string()); + args.push("--debug-dir".to_string()); + args.push("tmp".to_string()); + WasmerCreateObj { current_dir: operating_dir.clone(), - wasm_path, + wasm_path: wasm_path.clone(), output_object_path: object_path.clone(), compiler: Compiler::Cranelift, extra_cli_flags: args, @@ -401,13 +406,6 @@ fn create_exe_with_object_input(args: Vec<&'static str>) -> anyhow::Result<()> { "create-obj successfully completed but object output file `{}` missing", object_path.display() ); - let mut object_header_path = object_path.clone(); - object_header_path.set_extension("h"); - assert!( - object_header_path.exists(), - "create-obj successfully completed but object output header file `{}` missing", - object_header_path.display() - ); #[cfg(not(windows))] let executable_path = operating_dir.join("wasm.out"); @@ -415,11 +413,14 @@ fn create_exe_with_object_input(args: Vec<&'static str>) -> anyhow::Result<()> { let executable_path = operating_dir.join("wasm.exe"); WasmerCreateExe { - current_dir: operating_dir.clone(), - wasm_path: object_path, + current_dir: std::env::current_dir().unwrap(), + wasm_path: wasm_path, native_executable_path: executable_path.clone(), compiler: Compiler::Cranelift, - extra_cli_flags: vec!["--header", "wasm.h"], + extra_cli_flags: vec![ + "--precompiled-atom".to_string(), + format!("qjs:{}:abc123", object_path.display()), + ], ..Default::default() } .run() @@ -444,10 +445,13 @@ fn create_exe_with_object_input_default() -> anyhow::Result<()> { #[test] fn create_exe_with_object_input_symbols() -> anyhow::Result<()> { - create_exe_with_object_input(vec!["--object-format", "symbols"]) + create_exe_with_object_input(vec!["--object-format".to_string(), "symbols".to_string()]) } #[test] fn create_exe_with_object_input_serialized() -> anyhow::Result<()> { - create_exe_with_object_input(vec!["--object-format", "serialized"]) + create_exe_with_object_input(vec![ + "--object-format".to_string(), + "serialized".to_string(), + ]) }