Debug why linking WASMER_METADATA doesn't work for multiple files

Cargo.lock

@@ -2333,6 +2333,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd"
 
+[[package]]
+name = "pathdiff"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
+
 [[package]]
 name = "pbkdf2"
 version = "0.11.0"
@@ -4312,6 +4318,7 @@ dependencies = [
  "log",
  "minisign",
  "nuke-dir",
+ "pathdiff",
  "prettytable-rs",
  "regex",
  "reqwest",

lib/cli/Cargo.toml

@@ -85,6 +85,7 @@ log = "0.4.17"
 minisign = "0.7.2"
 semver = "1.0.14"
 rpassword = "7.2.0"
+pathdiff = "0.2.1"
 
 [build-dependencies]
 chrono = { version = "^0.4", default-features = false, features = [ "std", "clock" ] }

lib/cli/src/commands/create_exe.rs

@@ -52,14 +52,15 @@ pub struct CreateExe {
     target_triple: Option<Triple>,
 
     /// Optional object format (serialized | symbols)
+    #[clap(long)]
     object_format: Option<ObjectFormat>,
 
-    #[clap(short = 'm')]
+    #[clap(long, short = 'm', multiple = true, number_of_values = 1)]
     cpu_features: Vec<CpuFeature>,
 
     /// Additional libraries to link against.
     /// This is useful for fixing linker errors that may occur on some systems.
-    #[clap(short = 'l')]
+    #[clap(long, short = 'l')]
     libraries: Vec<String>,
 
     #[clap(flatten)]
@@ -208,6 +209,10 @@ pub(super) fn compile_pirita_into_directory(
     triple: &Triple,
     object_format: ObjectFormat,
 ) -> anyhow::Result<()> {
+    std::fs::create_dir_all(target_dir)
+        .map_err(|e| anyhow::anyhow!("cannot create / dir in {}: {e}", target_dir.display()))?;
+
+    let target_dir = target_dir.canonicalize()?;
     let target = &utils::target_triple_to_target(&triple, cpu_features);
 
     std::fs::create_dir_all(target_dir.join("volumes")).map_err(|e| {
@@ -221,6 +226,8 @@ pub(super) fn compile_pirita_into_directory(
     let volume_name = "VOLUMES";
     let volume_path = target_dir.join("volumes").join("volume.o");
     write_volume_obj(&volume_bytes, &volume_name, &volume_path, &target)?;
+    let volume_path = volume_path.canonicalize()?;
+    let volume_path = pathdiff::diff_paths(&volume_path, &target_dir).unwrap();
 
     std::fs::create_dir_all(target_dir.join("atoms")).map_err(|e| {
         anyhow::anyhow!("cannot create /atoms dir in {}: {e}", target_dir.display())
@@ -261,15 +268,22 @@ pub(super) fn compile_pirita_into_directory(
         object_format,
     )?;
 
+    // target_dir
     let mut atoms = Vec::new();
     for (atom_name, atom_path, opt_header_path) in target_paths {
-        atoms.push(CommandEntrypoint {
+        if let Ok(atom_path) = atom_path.canonicalize() {
-            // TODO: improve, "--command pip" should be able to invoke atom "python" with args "-m pip"
+            let header_path = opt_header_path.and_then(|p| p.canonicalize().ok());
-            command: atom_name.clone(),
+            let atom_path =
-            atom: atom_name.clone(),
+                pathdiff::diff_paths(&atom_path, &target_dir).unwrap_or_else(|| atom_path.clone());
-            path: atom_path,
+            let header_path = header_path.and_then(|h| pathdiff::diff_paths(&h, &target_dir));
-            header: opt_header_path,
+            atoms.push(CommandEntrypoint {
-        });
+                // TODO: improve, "--command pip" should be able to invoke atom "python" with args "-m pip"
+                command: atom_name.clone(),
+                atom: atom_name.clone(),
+                path: atom_path,
+                header: header_path,
+            });
+        }
     }
 
     let entrypoint = Entrypoint {
@@ -307,14 +321,17 @@ fn conpile_atoms(
     use std::io::BufWriter;
     use std::io::Write;
 
-    for (atom_name, data) in atoms {
+    for (a, data) in atoms {
         let (store, _) = compiler.get_store_for_target(target.clone())?;
+        let atom_name = utils::normalize_atom_name(a);
         let output_object_path = output_dir.join(format!("{atom_name}.o"));
-
+        let module_name = format!(
+            "WASMER_MODULE_{}",
+            utils::normalize_atom_name(&atom_name).to_uppercase()
+        );
         match object_format {
             ObjectFormat::Symbols => {
                 let output_header_path = header_dir.join(format!("static_defs_{atom_name}.h"));
-
                 let engine = store.engine();
                 let engine_inner = engine.inner();
                 let compiler = engine_inner.compiler()?;
@@ -324,9 +341,16 @@ fn conpile_atoms(
                 let prefixer: Option<PrefixerFn> = Some(Box::new(move |_| {
                     utils::normalize_atom_name(&atom_name_copy)
                 }));
+                println!("generating object file {module_name}");
                 let (module_info, obj, metadata_length, symbol_registry) =
                     Artifact::generate_object(
-                        compiler, &data, prefixer, &target, tunables, features,
+                        compiler,
+                        &data,
+                        &module_name,
+                        prefixer,
+                        &target,
+                        tunables,
+                        features,
                     )?;
 
                 let header_file_src = crate::c_gen::staticlib_header::generate_header_file(
@@ -350,15 +374,7 @@ fn conpile_atoms(
                     Module::from_binary(&store, &data).context("failed to compile Wasm")?;
                 let bytes = module.serialize()?;
                 let mut obj = get_object_for_target(target.triple())?;
-                emit_serialized(
+                emit_serialized(&mut obj, &bytes, target.triple(), &module_name)?;
-                    &mut obj,
-                    &bytes,
-                    target.triple(),
-                    &format!(
-                        "WASMER_MODULE_{}",
-                        utils::normalize_atom_name(atom_name).to_uppercase()
-                    ),
-                )?;
 
                 let mut writer = BufWriter::new(File::create(&output_object_path)?);
                 obj.write_stream(&mut writer)
@@ -454,6 +470,11 @@ pub(super) fn prepare_directory_from_single_wasm_file(
     let bytes = std::fs::read(wasm_file)?;
     let target = &utils::target_triple_to_target(&triple, cpu_features);
 
+    std::fs::create_dir_all(target_dir)
+        .map_err(|e| anyhow::anyhow!("cannot create / dir in {}: {e}", target_dir.display()))?;
+
+    let target_dir = target_dir.canonicalize()?;
+
     std::fs::create_dir_all(target_dir.join("atoms")).map_err(|e| {
         anyhow::anyhow!("cannot create /atoms dir in {}: {e}", target_dir.display())
     })?;
@@ -465,8 +486,8 @@ pub(super) fn prepare_directory_from_single_wasm_file(
 
     for (atom_name, atom_bytes) in all_files.iter() {
         atoms_from_file.push((atom_name.clone(), atom_bytes.to_vec()));
-        let atom_path = target_dir.join("atoms").join(format!("{atom_name}.o"));
+        let mut atom_path = target_dir.join("atoms").join(format!("{atom_name}.o"));
-        let header_path = match object_format {
+        let mut header_path = match object_format {
             ObjectFormat::Symbols => {
                 std::fs::create_dir_all(target_dir.join("include")).map_err(|e| {
                     anyhow::anyhow!(
@@ -483,6 +504,11 @@ pub(super) fn prepare_directory_from_single_wasm_file(
             }
             ObjectFormat::Serialized => None,
         };
+        if let Ok(a) = atom_path.canonicalize() {
+            let opt_header_path = header_path.and_then(|p| p.canonicalize().ok());
+            atom_path = pathdiff::diff_paths(&a, &target_dir).unwrap_or_else(|| a.clone());
+            header_path = opt_header_path.and_then(|h| pathdiff::diff_paths(&h, &target_dir));
+        }
         target_paths.push((atom_name, atom_path, header_path));
     }
 
@@ -583,6 +609,10 @@ fn link_exe_from_dir(
             .filter_map(|v| directory.join(&v.obj_file).canonicalize().ok()),
     );
 
+    println!("linking volume objects: {:#?}", entrypoint.volumes);
+    println!("object_paths: {:#?}", object_paths);
+    println!("object_paths: {:#?}", entrypoint.atoms);
+
     let zig_triple = utils::triple_to_zig_triple(&cross_compilation.target);
     let include_dirs = entrypoint
         .atoms
@@ -642,10 +672,22 @@ fn link_exe_from_dir(
                 cmd.arg("-lc");
             }
 
+            let mut include_dirs = include_dirs
+                .iter()
+                .map(|i| format!("{}", i.display()))
+                .collect::<Vec<_>>();
+            include_dirs.sort();
+            include_dirs.dedup();
             for include_dir in include_dirs {
-                cmd.arg(format!("-I{}", include_dir.display()));
+                cmd.arg(format!("-I{include_dir}"));
             }
 
+            let mut include_path = library_path.clone();
+            include_path.pop();
+            include_path.pop();
+            include_path.push("include");
+            cmd.arg(format!("-I{}", include_path.display()));
+
             cmd.arg("-lunwind");
             cmd.arg("-OReleaseSafe");
             cmd.arg("-fno-compiler-rt");
@@ -657,7 +699,7 @@ fn link_exe_from_dir(
             cmd.arg(&format!("-femit-bin={}", out_path.display()));
             cmd.args(&object_paths);
             cmd.arg(&library_path);
-            cmd.arg(directory.join("wasmer_main.c"));
+            cmd.arg(directory.join("wasmer_main.c").canonicalize().unwrap());
 
             if zig_triple.contains("windows") {
                 let mut winsdk_path = library_path.clone();
@@ -673,6 +715,7 @@ fn link_exe_from_dir(
                 cmd.args(files_winsdk);
             }
 
+            println!("{cmd:?}");
             let compilation = cmd.output().context("Could not execute `zig`")?;
 
             if !compilation.status.success() {
@@ -737,11 +780,12 @@ fn generate_wasmer_main_c_inner(
             for a in atom_names.iter() {
                 let atom_name = utils::normalize_atom_name(&a);
                 let atom_name_uppercase = atom_name.to_uppercase();
+                let module_name = format!("WASMER_MODULE_{atom_name_uppercase}");
 
                 c_code_to_add.push_str(&format!(
                     "
-                extern size_t {atom_name_uppercase}_LENGTH asm(\"{atom_name_uppercase}_LENGTH\");
+                extern size_t {module_name}_LENGTH asm(\"{module_name}_LENGTH\");
-                extern char {atom_name_uppercase}_DATA asm(\"{atom_name_uppercase}_DATA\");
+                extern char {module_name}_DATA asm(\"{module_name}_DATA\");
                 "
                 ));
 
@@ -757,8 +801,8 @@ fn generate_wasmer_main_c_inner(
                 } else {
                     c_code_to_instantiate.push_str(&format!("
                     wasm_byte_vec_t atom_{atom_name}_byte_vec = {{
-                        .size = {atom_name_uppercase}_LENGTH,
+                        .size = {module_name}_LENGTH,
-                        .data = &{atom_name_uppercase}_DATA,
+                        .data = &{module_name}_DATA,
                     }};
                     wasm_module_t *atom_{atom_name} = wasm_module_deserialize(store, &atom_{atom_name}_byte_vec);
                     if (!atom_{atom_name}) {{
@@ -772,15 +816,38 @@ fn generate_wasmer_main_c_inner(
                 deallocate_module.push_str(&format!("wasm_module_delete(atom_{atom_name});"));
             }
 
+            println!("entrypoint volumes: {:?}", entrypoint.volumes);
+            let volumes_str = entrypoint
+                .volumes
+                .iter()
+                .map(|v| utils::normalize_atom_name(&v.name).to_uppercase())
+                .map(|uppercase| {
+                    vec![
+                        format!("extern size_t {uppercase}_LENGTH asm(\"{uppercase}_LENGTH\");"),
+                        format!("extern char {uppercase}_DATA asm(\"{uppercase}_DATA\");"),
+                    ]
+                    .join("\r\n")
+                })
+                .collect::<Vec<_>>();
+
             let base_str = if compile_static {
                 WASMER_STATIC_MAIN_C_SOURCE
             } else {
                 WASMER_MAIN_C_SOURCE
             };
 
+            let volumes_str = volumes_str.join("\r\n");
             let return_str = base_str
-                .replace("#define WASI", "#define WASI\r\n#define WASI_PIRITA")
+                .replace(
+                    "#define WASI",
+                    if !volumes_str.trim().is_empty() {
+                        "#define WASI\r\n#define WASI_PIRITA"
+                    } else {
+                        "#define WASI"
+                    },
+                )
                 .replace("// DECLARE_MODULES", &c_code_to_add)
+                .replace("// DECLARE_VOLUMES", &volumes_str)
                 .replace("wasm_module_delete(module);", &deallocate_module);
 
             if atom_names.len() == 1 {

lib/cli/src/commands/create_obj.rs

@@ -48,10 +48,10 @@ pub struct CreateObj {
     /// This flag accepts two options: `symbols` or `serialized`.
     /// - (default) `symbols` creates an object where all functions and metadata of the module are regular object symbols
     /// - `serialized` creates an object where the module is zero-copy serialized as raw data
-    #[clap(name = "OBJECT_FORMAT", long = "object-format", verbatim_doc_comment)]
+    #[clap(name = "OBJECT_FORMAT", long, verbatim_doc_comment)]
     object_format: Option<ObjectFormat>,
 
-    #[clap(short = 'm', multiple = true, number_of_values = 1)]
+    #[clap(long, short = 'm', multiple = true, number_of_values = 1)]
     cpu_features: Vec<CpuFeature>,
 
     #[clap(flatten)]

lib/cli/src/commands/wasmer_create_exe_main.c

@@ -1,24 +1,16 @@
 
 #include "wasmer.h"
-//#include "my_wasm.h"
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+// EXTRA_HEADERS
 
 #define own
 
-
 #define WASI
-
+ 
-#ifdef WASI_PIRITA
+// DECLARE_VOLUMES
-extern size_t VOLUMES_LENGTH asm("VOLUMES_LENGTH");
-extern char VOLUMES_DATA asm("VOLUMES_DATA");
 // DECLARE_MODULES
-#else
-extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH");
-extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA");
-#endif
 
 static void print_wasmer_error() {
   int error_len = wasmer_last_error_length();
@@ -137,7 +129,7 @@ int main(int argc, char *argv[]) {
       module,
       filesystem,
       &imports,
-      selected_atom,
+      selected_atom
       );
   if (!wasi_env) {
     printf("Error setting filesystem\n");

lib/cli/src/commands/wasmer_static_create_exe_main.c

@@ -1,18 +1,15 @@
 #include "wasmer.h"
-#include "static_defs.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+// EXTRA_HEADERS
 
 #define own
 
 // TODO: make this define templated so that the Rust code can toggle it on/off
 #define WASI
 
-#ifdef WASI_PIRITA
+// DECLARE_VOLUMES
-extern size_t VOLUMES_LENGTH asm("VOLUMES_LENGTH");
-extern char VOLUMES_DATA asm("VOLUMES_DATA");
-#endif
 
 extern wasm_module_t* wasmer_object_module_new(wasm_store_t* store,const char* wasm_name) asm("wasmer_object_module_new");
 
@@ -97,11 +94,15 @@ int main(int argc, char *argv[]) {
   wasm_engine_t *engine = wasm_engine_new_with_config(config);
   wasm_store_t *store = wasm_store_new(engine);
 
-  #ifdef WASI_PIRITA
+  const char* selected_atom = "main";
-    // INSTANTIATE_MODULES
+  for (int i = 1; i < argc; i++) {
-  #else
+      if (strcmp(argv[i], "--command") == 0 && i + 1 < argc) {
-    wasm_module_t *module = wasmer_object_module_new(store, "module");
+        selected_atom = argv[i + 1];
-  #endif
+        break;
+      }
+  }
+
+  // INSTANTIATE_MODULES
 
   if (!module) {
     fprintf(stderr, "Failed to create module\n");

lib/compiler/src/engine/artifact.rs

@@ -51,9 +51,6 @@ pub struct Artifact {
 #[cfg(feature = "static-artifact-create")]
 pub type PrefixerFn = Box<dyn Fn(&[u8]) -> String + Send>;
 
-#[cfg(feature = "static-artifact-create")]
-const WASMER_METADATA_SYMBOL: &[u8] = b"WASMER_METADATA";
-
 impl Artifact {
     /// Compile a data buffer into a `ArtifactBuild`, which may then be instantiated.
     #[cfg(feature = "compiler")]
@@ -472,6 +469,7 @@ impl Artifact {
     pub fn generate_object<'data>(
         compiler: &dyn Compiler,
         data: &[u8],
+        object_name: &str,
         prefixer: Option<PrefixerFn>,
         target: &'data Target,
         tunables: &dyn Tunables,
@@ -550,7 +548,7 @@ impl Artifact {
             )?;
         let mut obj = get_object_for_target(target_triple).map_err(to_compile_error)?;
 
-        emit_data(&mut obj, WASMER_METADATA_SYMBOL, &metadata_binary, 1)
+        emit_data(&mut obj, object_name.as_bytes(), &metadata_binary, 1)
             .map_err(to_compile_error)?;
 
         emit_compilation(&mut obj, compilation, &symbol_registry, target_triple)

lib/registry/src/lib.rs

@@ -91,7 +91,6 @@ impl fmt::Display for LocalPackage {
 }
 
 impl LocalPackage {
-
     pub fn get_path(&self) -> Result<PathBuf, String> {
         Ok(self.path.clone())
     }