Fix entrypoint serialization

lib/cli/Cargo.toml

@@ -36,7 +36,7 @@ wasmer-wasi = { version = "=3.1.0", path = "../wasi", optional = true }
 wasmer-wasi-experimental-io-devices = { version = "=3.1.0", path = "../wasi-experimental-io-devices", optional = true, features = ["link_external_libs"] }
 wasmer-wast = { version = "=3.1.0", path = "../../tests/lib/wast", optional = true }
 wasmer-cache = { version = "=3.1.0", path = "../cache", optional = true }
-wasmer-types = { version = "=3.1.0", path = "../types" }
+wasmer-types = { version = "=3.1.0", path = "../types", features = ["enable-serde"] }
 wasmer-registry = { version = "=4.0.0", path = "../registry" }
 wasmer-object = { version = "=3.1.0", path = "../object", optional = true }
 wasmer-vfs  = { version = "=3.1.0", path = "../vfs", default-features = false, features = ["host-fs"] }
@@ -169,7 +169,6 @@ enable-serde = [
   "wasmer/enable-serde",
   "wasmer-vm/enable-serde",
   "wasmer-compiler/enable-serde",
-  "wasmer-types/enable-serde",
   "wasmer-wasi/enable-serde",
 ]
 

lib/cli/src/commands/create_exe.rs

@@ -11,6 +11,7 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use wasmer::*;
 use wasmer_object::{emit_serialized, get_object_for_target};
+use wasmer_types::ModuleInfo;
 
 use webc::{ParseOptions, WebCMmap};
 
@@ -137,6 +138,8 @@ pub struct CommandEntrypoint {
     pub path: PathBuf,
     /// Optional path to the static_defs.h header file, relative to the entrypoint.json parent dir
     pub header: Option<PathBuf>,
+    /// Module info, set when the wasm file is compiled
+    pub module_info: Option<ModuleInfo>,
 }
 
 /// Volume object file (name + path to object file)
@@ -162,15 +165,7 @@ impl CreateExe {
             utils::get_cross_compile_setup(&mut cc, &target_triple, &starting_cd)?;
 
         if input_path.is_dir() {
-            // assumes that the input directory has been created with create-obj
+            return Err(anyhow::anyhow!("input path cannot be a directory"));
-            link_exe_from_dir(
-                &input_path,
-                output_path,
-                &cross_compilation,
-                &self.libraries,
-                true,
-                &self.precompiled_atom,
-            )?;
         } else if let Ok(pirita) = WebCMmap::parse(input_path.clone(), &ParseOptions::default()) {
             // pirita file
             let temp = tempdir::TempDir::new("pirita-compile")?;
@@ -179,7 +174,7 @@ impl CreateExe {
                 None => temp.path().to_path_buf(),
             };
             std::fs::create_dir_all(&tempdir)?;
-            compile_pirita_into_directory(
+            let atoms = compile_pirita_into_directory(
                 &pirita,
                 &tempdir,
                 &self.compiler,
@@ -188,6 +183,9 @@ impl CreateExe {
                 self.object_format.unwrap_or_default(),
                 &self.precompiled_atom,
             )?;
+            get_module_infos(&input_path, &atoms)?;
+            let entrypoint = get_entrypoint(&tempdir)?;
+            create_header_files_in_dir(&tempdir, &entrypoint, &self.precompiled_atom)?;
             link_exe_from_dir(
                 &tempdir,
                 output_path,
@@ -204,7 +202,7 @@ impl CreateExe {
                 None => temp.path().to_path_buf(),
             };
             std::fs::create_dir_all(&tempdir)?;
-            prepare_directory_from_single_wasm_file(
+            let atoms = prepare_directory_from_single_wasm_file(
                 &input_path,
                 &tempdir,
                 &self.compiler,
@@ -213,6 +211,9 @@ impl CreateExe {
                 self.object_format.unwrap_or_default(),
                 &self.precompiled_atom,
             )?;
+            get_module_infos(&input_path, &atoms)?;
+            let entrypoint = get_entrypoint(&tempdir)?;
+            create_header_files_in_dir(&tempdir, &entrypoint, &self.precompiled_atom)?;
             link_exe_from_dir(
                 &tempdir,
                 output_path,
@@ -240,6 +241,42 @@ impl CreateExe {
     }
 }
 
+fn write_entrypoint(directory: &Path, entrypoint: &Entrypoint) -> Result<(), anyhow::Error> {
+    std::fs::write(
+        directory.join("entrypoint.json"),
+        serde_json::to_string_pretty(&entrypoint).unwrap(),
+    )
+    .map_err(|e| {
+        anyhow::anyhow!(
+            "cannot create entrypoint.json dir in {}: {e}",
+            directory.display()
+        )
+    })
+}
+
+fn get_entrypoint(directory: &Path) -> Result<Entrypoint, anyhow::Error> {
+    let entrypoint_json =
+        std::fs::read_to_string(directory.join("entrypoint.json")).map_err(|e| {
+            anyhow::anyhow!(
+                "could not read entrypoint.json in {}: {e}",
+                directory.display()
+            )
+        })?;
+
+    let entrypoint: Entrypoint = serde_json::from_str(&entrypoint_json).map_err(|e| {
+        anyhow::anyhow!(
+            "could not parse entrypoint.json in {}: {e}",
+            directory.display()
+        )
+    })?;
+
+    if entrypoint.atoms.is_empty() {
+        return Err(anyhow::anyhow!("file has no atoms to compile"));
+    }
+
+    Ok(entrypoint)
+}
+
 /// Given a pirita file, compiles the .wasm files into the target directory
 pub(super) fn compile_pirita_into_directory(
     pirita: &WebCMmap,
@@ -249,7 +286,7 @@ pub(super) fn compile_pirita_into_directory(
     triple: &Triple,
     object_format: ObjectFormat,
     prefixes: &[String],
-) -> anyhow::Result<()> {
+) -> anyhow::Result<Vec<(String, Vec<u8>)>> {
     std::fs::create_dir_all(target_dir)
         .map_err(|e| anyhow::anyhow!("cannot create / dir in {}: {e}", target_dir.display()))?;
 
@@ -307,7 +344,7 @@ pub(super) fn compile_pirita_into_directory(
     }
 
     let prefix_map = PrefixMapCompilation::from_input(&atoms_from_file, prefixes, false)?;
-    conpile_atoms(
+    let module_infos = conpile_atoms(
         &atoms_from_file,
         &target_dir.join("atoms"),
         compiler,
@@ -329,9 +366,10 @@ pub(super) fn compile_pirita_into_directory(
         atoms.push(CommandEntrypoint {
             // TODO: improve, "--command pip" should be able to invoke atom "python" with args "-m pip"
             command: command_name,
-            atom: atom_name,
+            atom: atom_name.clone(),
             path: atom_path,
             header: header_path,
+            module_info: module_infos.get(&atom_name).cloned(),
         });
     }
 
@@ -344,18 +382,9 @@ pub(super) fn compile_pirita_into_directory(
         object_format,
     };
 
-    std::fs::write(
+    write_entrypoint(&target_dir, &entrypoint)?;
-        target_dir.join("entrypoint.json"),
-        serde_json::to_string_pretty(&entrypoint).unwrap_or_default(),
-    )
-    .map_err(|e| {
-        anyhow::anyhow!(
-            "cannot create entrypoint.json dir in {}: {e}",
-            target_dir.display()
-        )
-    })?;
 
-    Ok(())
+    Ok(atoms_from_file)
 }
 
 /// Prefix map used during compilation of object files
@@ -504,11 +533,12 @@ fn conpile_atoms(
     target: &Target,
     object_format: ObjectFormat,
     prefixes: &PrefixMapCompilation,
-) -> Result<()> {
+) -> Result<BTreeMap<String, ModuleInfo>, anyhow::Error> {
     use std::fs::File;
     use std::io::BufWriter;
     use std::io::Write;
 
+    let mut module_infos = BTreeMap::new();
     for (a, data) in atoms {
         let prefix = prefixes
             .get_prefix_for_atom(a)
@@ -526,7 +556,7 @@ fn conpile_atoms(
                 let tunables = store.tunables();
                 let prefix_copy = prefix.to_string();
                 let prefixer: Option<PrefixerFn> = Some(Box::new(move |_| prefix_copy.to_string()));
-                let (_, obj, _, _) = Artifact::generate_object(
+                let (module_info, obj, _, _) = Artifact::generate_object(
                     compiler,
                     data,
                     &module_name,
@@ -535,6 +565,7 @@ fn conpile_atoms(
                     tunables,
                     features,
                 )?;
+                module_infos.insert(atom_name, module_info);
                 // Write object file with functions
                 let mut writer = BufWriter::new(File::create(&output_object_path)?);
                 obj.write_stream(&mut writer)
@@ -555,7 +586,7 @@ fn conpile_atoms(
         }
     }
 
-    Ok(())
+    Ok(module_infos)
 }
 
 /// Compile the C code.
@@ -637,7 +668,7 @@ pub(super) fn prepare_directory_from_single_wasm_file(
     cpu_features: &[CpuFeature],
     object_format: ObjectFormat,
     prefix: &[String],
-) -> anyhow::Result<()> {
+) -> anyhow::Result<Vec<(String, Vec<u8>)>, anyhow::Error> {
     let bytes = std::fs::read(wasm_file)?;
     let target = &utils::target_triple_to_target(triple, cpu_features);
 
@@ -684,7 +715,7 @@ pub(super) fn prepare_directory_from_single_wasm_file(
     }
 
     let prefix_map = PrefixMapCompilation::from_input(&atoms_from_file, prefix, false)?;
-    conpile_atoms(
+    let module_infos = conpile_atoms(
         &atoms_from_file,
         &target_dir.join("atoms"),
         compiler,
@@ -701,6 +732,7 @@ pub(super) fn prepare_directory_from_single_wasm_file(
             atom: atom_name.clone(),
             path: atom_path,
             header: opt_header_path,
+            module_info: module_infos.get(atom_name).cloned(),
         });
     }
 
@@ -710,32 +742,50 @@ pub(super) fn prepare_directory_from_single_wasm_file(
         object_format,
     };
 
-    std::fs::write(
+    write_entrypoint(&target_dir, &entrypoint)?;
-        target_dir.join("entrypoint.json"),
-        serde_json::to_string_pretty(&entrypoint).unwrap_or_default(),
-    )
-    .map_err(|e| {
-        anyhow::anyhow!(
-            "cannot create entrypoint.json dir in {}: {e}",
-            target_dir.display()
-        )
-    })?;
 
-    Ok(())
+    Ok(all_files)
+}
+
+// Given the input file paths, correctly resolves the .wasm files,
+// reads the module info from the wasm module and writes the ModuleInfo for each file
+// into the entrypoint.json file
+fn get_module_infos(
+    directory: &Path,
+    atoms: &[(String, Vec<u8>)],
+) -> Result<BTreeMap<String, ModuleInfo>, anyhow::Error> {
+    let mut entrypoint = get_entrypoint(directory)?;
+
+    let mut module_infos = BTreeMap::new();
+    for (atom_name, atom_bytes) in atoms {
+        let store = Store::default();
+        let module = unsafe { Module::deserialize(&store, atom_bytes.as_slice()) }
+            .map_err(|e| anyhow::anyhow!("could not deserialize module {atom_name}: {e}"))?;
+        if let Some(s) = entrypoint
+            .atoms
+            .iter_mut()
+            .find(|a| a.atom.as_str() == atom_name.as_str())
+        {
+            s.module_info = Some(module.info().clone());
+            module_infos.insert(atom_name.clone(), module.info().clone());
+        }
+    }
+
+    write_entrypoint(directory, &entrypoint)?;
+
+    Ok(module_infos)
 }
 
 /// Create the static_defs.h header files in the /include directory
 fn create_header_files_in_dir(
     directory: &Path,
-    atoms: &[(String, Vec<u8>)],
     entrypoint: &Entrypoint,
     prefixes: &[String],
-    object_format: ObjectFormat,
 ) -> anyhow::Result<()> {
     use object::{Object, ObjectSection};
     use wasmer_types::compilation::symbols::ModuleMetadataSymbolRegistry;
 
-    if object_format == ObjectFormat::Serialized {
+    if entrypoint.object_format == ObjectFormat::Serialized {
         return Ok(());
     }
 
@@ -754,16 +804,9 @@ fn create_header_files_in_dir(
     for atom in entrypoint.atoms.iter() {
         let atom_name = &atom.atom;
         let prefix = prefixes
-            .get_prefix_for_atom(&atom_name)
+            .get_prefix_for_atom(atom_name)
             .ok_or_else(|| anyhow::anyhow!("cannot get prefix for atom {atom_name}"))?;
 
-        let atom_bytes = atoms
-            .iter()
-            .find_map(|(name, bytes)| if name == atom_name { Some(bytes) } else { None })
-            .ok_or_else(|| {
-                anyhow::anyhow!("could not find bytes for atom {atom_name} in generate-header step")
-            })?;
-
         let object_file_src = directory.join(&atom.path);
         let object_file = std::fs::read(&object_file_src)
             .map_err(|e| anyhow::anyhow!("could not read {}: {e}", object_file_src.display()))?;
@@ -777,22 +820,19 @@ fn create_header_files_in_dir(
         })?;
         let metadata_length = section.size();
 
-        let store = Store::default();
+        let module_info = atom
-        let module_info =
+            .module_info
-            unsafe { Module::deserialize(&store, atom_bytes.as_slice()) }.map_err(|e| {
+            .as_ref()
-                anyhow::anyhow!(
+            .ok_or_else(|| anyhow::anyhow!("no module info for atom {atom_name:?}"))?;
-                    "could not deserialized module {atom_name} in generate-header step: {e}"
-                )
-            })?;
 
         let header_file_path = directory
             .join("include")
             .join(format!("static_defs_{prefix}.h"));
 
         let header_file_src = crate::c_gen::staticlib_header::generate_header_file(
-            &atom_name,
+            atom_name,
             &format!("WASMER_{prefix}_METADATA"),
-            &module_info.info(),
+            module_info,
             &ModuleMetadataSymbolRegistry { prefix },
             metadata_length as usize,
         );
@@ -815,24 +855,7 @@ fn link_exe_from_dir(
     debug: bool,
     prefixes: &[String],
 ) -> anyhow::Result<()> {
-    let entrypoint_json =
+    let entrypoint = get_entrypoint(directory)?;
-        std::fs::read_to_string(directory.join("entrypoint.json")).map_err(|e| {
-            anyhow::anyhow!(
-                "could not read entrypoint.json in {}: {e}",
-                directory.display()
-            )
-        })?;
-
-    let entrypoint: Entrypoint = serde_json::from_str(&entrypoint_json).map_err(|e| {
-        anyhow::anyhow!(
-            "could not parse entrypoint.json in {}: {e}",
-            directory.display()
-        )
-    })?;
-
-    if entrypoint.atoms.is_empty() {
-        return Err(anyhow::anyhow!("file has no atoms to compile"));
-    }
 
     let prefix_map = entrypoint
         .atoms

lib/cli/src/commands/create_obj.rs

@@ -128,6 +128,11 @@ impl CreateObj {
 
         if file_paths.len() == 1 {
             std::fs::copy(&file_paths[0], &self.output)?;
+        } else {
+            std::fs::create_dir_all(&self.output.join("atoms"))?;
+            for f in file_paths {
+                std::fs::copy(&f, self.output.join(f.file_name().unwrap()))?;
+            }
         }
 
         eprintln!(

lib/types/src/module.rs

@@ -67,6 +67,34 @@ impl From<(String, String, u32)> for ImportKey {
     }
 }
 
+#[cfg(feature = "enable-serde")]
+mod serde_imports {
+
+    use crate::ImportIndex;
+    use crate::ImportKey;
+    use indexmap::IndexMap;
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+    type InitialType = IndexMap<ImportKey, ImportIndex>;
+    type SerializedType = Vec<(ImportKey, ImportIndex)>;
+    // IndexMap<ImportKey, ImportIndex>
+    // Vec<
+    pub fn serialize<S: Serializer>(s: &InitialType, serializer: S) -> Result<S::Ok, S::Error> {
+        let vec: SerializedType = s
+            .iter()
+            .map(|(a, b)| (a.clone(), b.clone()))
+            .collect::<Vec<_>>();
+        vec.serialize(serializer)
+    }
+
+    pub fn deserialize<'de, D: Deserializer<'de>>(
+        deserializer: D,
+    ) -> Result<InitialType, D::Error> {
+        let serialized = <SerializedType as Deserialize>::deserialize(deserializer)?;
+        Ok(serialized.into_iter().collect())
+    }
+}
+
 /// A translated WebAssembly module, excluding the function bodies and
 /// memory initializers.
 #[derive(Debug, Clone, Default)]
@@ -89,6 +117,7 @@ pub struct ModuleInfo {
     /// Keeping the `index_of_the_import` is important, as there can be
     /// two same references to the same import, and we don't want to confuse
     /// them.
+    #[cfg_attr(feature = "enable-serde", serde(with = "serde_imports"))]
     pub imports: IndexMap<ImportKey, ImportIndex>,
 
     /// Exported entities.