mii/wasmer
1
0
Fork 0
mirror of https://github.com/mii443/wasmer.git synced 2025-12-13 05:48:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Debug why linking WASMER_METADATA doesn't work for multiple files

Browse Source
This commit is contained in:
Felix Schütt
2022-12-22 13:58:40 +01:00
parent 3672e03f96
commit c7986832a2
8 changed files with 125 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Cargo.lock generated
View File

@@ -2333,6 +2333,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd"
[[package]]
name = "pathdiff"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
[[package]]
name = "pbkdf2"
version = "0.11.0"
@@ -4312,6 +4318,7 @@ dependencies = [
"log",
"minisign",
"nuke-dir",
"pathdiff",
"prettytable-rs",
"regex",
"reqwest",

1
lib/cli/Cargo.toml
View File

@@ -85,6 +85,7 @@ log = "0.4.17"
minisign = "0.7.2"
semver = "1.0.14"
rpassword = "7.2.0"
pathdiff = "0.2.1"
[build-dependencies]
chrono = { version = "^0.4", default-features = false, features = [ "std", "clock" ] }

117
lib/cli/src/commands/create_exe.rs
View File

@@ -52,14 +52,15 @@ pub struct CreateExe {
target_triple: Option<Triple>,
/// Optional object format (serialized | symbols)
#[clap(long)]
object_format: Option<ObjectFormat>,
#[clap(short = 'm')]
#[clap(long, short = 'm', multiple = true, number_of_values = 1)]
cpu_features: Vec<CpuFeature>,
/// Additional libraries to link against.
/// This is useful for fixing linker errors that may occur on some systems.
#[clap(short = 'l')]
#[clap(long, short = 'l')]
libraries: Vec<String>,
#[clap(flatten)]
@@ -208,6 +209,10 @@ pub(super) fn compile_pirita_into_directory(
triple: &Triple,
object_format: ObjectFormat,
) -> anyhow::Result<()> {
std::fs::create_dir_all(target_dir)
.map_err(|e| anyhow::anyhow!("cannot create / dir in {}: {e}", target_dir.display()))?;
let target_dir = target_dir.canonicalize()?;
let target = &utils::target_triple_to_target(&triple, cpu_features);
std::fs::create_dir_all(target_dir.join("volumes")).map_err(|e| {
@@ -221,6 +226,8 @@ pub(super) fn compile_pirita_into_directory(
let volume_name = "VOLUMES";
let volume_path = target_dir.join("volumes").join("volume.o");
write_volume_obj(&volume_bytes, &volume_name, &volume_path, &target)?;
let volume_path = volume_path.canonicalize()?;
let volume_path = pathdiff::diff_paths(&volume_path, &target_dir).unwrap();
std::fs::create_dir_all(target_dir.join("atoms")).map_err(|e| {
anyhow::anyhow!("cannot create /atoms dir in {}: {e}", target_dir.display())
@@ -261,16 +268,23 @@ pub(super) fn compile_pirita_into_directory(
object_format,
)?;
// target_dir
let mut atoms = Vec::new();
for (atom_name, atom_path, opt_header_path) in target_paths {
if let Ok(atom_path) = atom_path.canonicalize() {
let header_path = opt_header_path.and_then(|p| p.canonicalize().ok());
let atom_path =
pathdiff::diff_paths(&atom_path, &target_dir).unwrap_or_else(|| atom_path.clone());
let header_path = header_path.and_then(|h| pathdiff::diff_paths(&h, &target_dir));
atoms.push(CommandEntrypoint {
// TODO: improve, "--command pip" should be able to invoke atom "python" with args "-m pip"
command: atom_name.clone(),
atom: atom_name.clone(),
path: atom_path,
header: opt_header_path,
header: header_path,
});
}
}
let entrypoint = Entrypoint {
atoms,
@@ -307,14 +321,17 @@ fn conpile_atoms(
use std::io::BufWriter;
use std::io::Write;
for (atom_name, data) in atoms {
for (a, data) in atoms {
let (store, _) = compiler.get_store_for_target(target.clone())?;
let atom_name = utils::normalize_atom_name(a);
let output_object_path = output_dir.join(format!("{atom_name}.o"));
let module_name = format!(
"WASMER_MODULE_{}",
utils::normalize_atom_name(&atom_name).to_uppercase()
);
match object_format {
ObjectFormat::Symbols => {
let output_header_path = header_dir.join(format!("static_defs_{atom_name}.h"));
let engine = store.engine();
let engine_inner = engine.inner();
let compiler = engine_inner.compiler()?;
@@ -324,9 +341,16 @@ fn conpile_atoms(
let prefixer: Option<PrefixerFn> = Some(Box::new(move |_| {
utils::normalize_atom_name(&atom_name_copy)
}));
println!("generating object file {module_name}");
let (module_info, obj, metadata_length, symbol_registry) =
Artifact::generate_object(
compiler, &data, prefixer, &target, tunables, features,
compiler,
&data,
&module_name,
prefixer,
&target,
tunables,
features,
)?;
let header_file_src = crate::c_gen::staticlib_header::generate_header_file(
@@ -350,15 +374,7 @@ fn conpile_atoms(
Module::from_binary(&store, &data).context("failed to compile Wasm")?;
let bytes = module.serialize()?;
let mut obj = get_object_for_target(target.triple())?;
emit_serialized(
&mut obj,
&bytes,
target.triple(),
&format!(
"WASMER_MODULE_{}",
utils::normalize_atom_name(atom_name).to_uppercase()
),
)?;
emit_serialized(&mut obj, &bytes, target.triple(), &module_name)?;
let mut writer = BufWriter::new(File::create(&output_object_path)?);
obj.write_stream(&mut writer)
@@ -454,6 +470,11 @@ pub(super) fn prepare_directory_from_single_wasm_file(
let bytes = std::fs::read(wasm_file)?;
let target = &utils::target_triple_to_target(&triple, cpu_features);
std::fs::create_dir_all(target_dir)
.map_err(|e| anyhow::anyhow!("cannot create / dir in {}: {e}", target_dir.display()))?;
let target_dir = target_dir.canonicalize()?;
std::fs::create_dir_all(target_dir.join("atoms")).map_err(|e| {
anyhow::anyhow!("cannot create /atoms dir in {}: {e}", target_dir.display())
})?;
@@ -465,8 +486,8 @@ pub(super) fn prepare_directory_from_single_wasm_file(
for (atom_name, atom_bytes) in all_files.iter() {
atoms_from_file.push((atom_name.clone(), atom_bytes.to_vec()));
let atom_path = target_dir.join("atoms").join(format!("{atom_name}.o"));
let header_path = match object_format {
let mut atom_path = target_dir.join("atoms").join(format!("{atom_name}.o"));
let mut header_path = match object_format {
ObjectFormat::Symbols => {
std::fs::create_dir_all(target_dir.join("include")).map_err(|e| {
anyhow::anyhow!(
@@ -483,6 +504,11 @@ pub(super) fn prepare_directory_from_single_wasm_file(
}
ObjectFormat::Serialized => None,
};
if let Ok(a) = atom_path.canonicalize() {
let opt_header_path = header_path.and_then(|p| p.canonicalize().ok());
atom_path = pathdiff::diff_paths(&a, &target_dir).unwrap_or_else(|| a.clone());
header_path = opt_header_path.and_then(|h| pathdiff::diff_paths(&h, &target_dir));
}
target_paths.push((atom_name, atom_path, header_path));
}
@@ -583,6 +609,10 @@ fn link_exe_from_dir(
.filter_map(|v| directory.join(&v.obj_file).canonicalize().ok()),
);
println!("linking volume objects: {:#?}", entrypoint.volumes);
println!("object_paths: {:#?}", object_paths);
println!("object_paths: {:#?}", entrypoint.atoms);
let zig_triple = utils::triple_to_zig_triple(&cross_compilation.target);
let include_dirs = entrypoint
.atoms
@@ -642,10 +672,22 @@ fn link_exe_from_dir(
cmd.arg("-lc");
}
let mut include_dirs = include_dirs
.iter()
.map(|i| format!("{}", i.display()))
.collect::<Vec<_>>();
include_dirs.sort();
include_dirs.dedup();
for include_dir in include_dirs {
cmd.arg(format!("-I{}", include_dir.display()));
cmd.arg(format!("-I{include_dir}"));
}
let mut include_path = library_path.clone();
include_path.pop();
include_path.pop();
include_path.push("include");
cmd.arg(format!("-I{}", include_path.display()));
cmd.arg("-lunwind");
cmd.arg("-OReleaseSafe");
cmd.arg("-fno-compiler-rt");
@@ -657,7 +699,7 @@ fn link_exe_from_dir(
cmd.arg(&format!("-femit-bin={}", out_path.display()));
cmd.args(&object_paths);
cmd.arg(&library_path);
cmd.arg(directory.join("wasmer_main.c"));
cmd.arg(directory.join("wasmer_main.c").canonicalize().unwrap());
if zig_triple.contains("windows") {
let mut winsdk_path = library_path.clone();
@@ -673,6 +715,7 @@ fn link_exe_from_dir(
cmd.args(files_winsdk);
}
println!("{cmd:?}");
let compilation = cmd.output().context("Could not execute `zig`")?;
if !compilation.status.success() {
@@ -737,11 +780,12 @@ fn generate_wasmer_main_c_inner(
for a in atom_names.iter() {
let atom_name = utils::normalize_atom_name(&a);
let atom_name_uppercase = atom_name.to_uppercase();
let module_name = format!("WASMER_MODULE_{atom_name_uppercase}");
c_code_to_add.push_str(&format!(
"
extern size_t {atom_name_uppercase}_LENGTH asm(\"{atom_name_uppercase}_LENGTH\");
extern char {atom_name_uppercase}_DATA asm(\"{atom_name_uppercase}_DATA\");
extern size_t {module_name}_LENGTH asm(\"{module_name}_LENGTH\");
extern char {module_name}_DATA asm(\"{module_name}_DATA\");
"
));
@@ -757,8 +801,8 @@ fn generate_wasmer_main_c_inner(
} else {
c_code_to_instantiate.push_str(&format!("
wasm_byte_vec_t atom_{atom_name}_byte_vec = {{
.size = {atom_name_uppercase}_LENGTH,
.data = &{atom_name_uppercase}_DATA,
.size = {module_name}_LENGTH,
.data = &{module_name}_DATA,
}};
wasm_module_t *atom_{atom_name} = wasm_module_deserialize(store, &atom_{atom_name}_byte_vec);
if (!atom_{atom_name}) {{
@@ -772,15 +816,38 @@ fn generate_wasmer_main_c_inner(
deallocate_module.push_str(&format!("wasm_module_delete(atom_{atom_name});"));
}
println!("entrypoint volumes: {:?}", entrypoint.volumes);
let volumes_str = entrypoint
.volumes
.iter()
.map(|v| utils::normalize_atom_name(&v.name).to_uppercase())
.map(|uppercase| {
vec![
format!("extern size_t {uppercase}_LENGTH asm(\"{uppercase}_LENGTH\");"),
format!("extern char {uppercase}_DATA asm(\"{uppercase}_DATA\");"),
]
.join("\r\n")
})
.collect::<Vec<_>>();
let base_str = if compile_static {
WASMER_STATIC_MAIN_C_SOURCE
} else {
WASMER_MAIN_C_SOURCE
};
let volumes_str = volumes_str.join("\r\n");
let return_str = base_str
.replace("#define WASI", "#define WASI\r\n#define WASI_PIRITA")
.replace(
"#define WASI",
if !volumes_str.trim().is_empty() {
"#define WASI\r\n#define WASI_PIRITA"
} else {
"#define WASI"
},
)
.replace("// DECLARE_MODULES", &c_code_to_add)
.replace("// DECLARE_VOLUMES", &volumes_str)
.replace("wasm_module_delete(module);", &deallocate_module);
if atom_names.len() == 1 {

4
lib/cli/src/commands/create_obj.rs
View File

@@ -48,10 +48,10 @@ pub struct CreateObj {
/// This flag accepts two options: `symbols` or `serialized`.
/// - (default) `symbols` creates an object where all functions and metadata of the module are regular object symbols
/// - `serialized` creates an object where the module is zero-copy serialized as raw data
#[clap(name = "OBJECT_FORMAT", long = "object-format", verbatim_doc_comment)]
#[clap(name = "OBJECT_FORMAT", long, verbatim_doc_comment)]
object_format: Option<ObjectFormat>,
#[clap(short = 'm', multiple = true, number_of_values = 1)]
#[clap(long, short = 'm', multiple = true, number_of_values = 1)]
cpu_features: Vec<CpuFeature>,
#[clap(flatten)]

14
lib/cli/src/commands/wasmer_create_exe_main.c
View File

@@ -1,24 +1,16 @@
#include "wasmer.h"
//#include "my_wasm.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// EXTRA_HEADERS
#define own
#define WASI
#ifdef WASI_PIRITA
extern size_t VOLUMES_LENGTH asm("VOLUMES_LENGTH");
extern char VOLUMES_DATA asm("VOLUMES_DATA");
// DECLARE_VOLUMES
// DECLARE_MODULES
#else
extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH");
extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA");
#endif
static void print_wasmer_error() {
int error_len = wasmer_last_error_length();
@@ -137,7 +129,7 @@ int main(int argc, char *argv[]) {
module,
filesystem,
&imports,
selected_atom,
selected_atom
);
if (!wasi_env) {
printf("Error setting filesystem\n");

19
lib/cli/src/commands/wasmer_static_create_exe_main.c
View File

@@ -1,18 +1,15 @@
#include "wasmer.h"
#include "static_defs.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// EXTRA_HEADERS
#define own
// TODO: make this define templated so that the Rust code can toggle it on/off
#define WASI
#ifdef WASI_PIRITA
extern size_t VOLUMES_LENGTH asm("VOLUMES_LENGTH");
extern char VOLUMES_DATA asm("VOLUMES_DATA");
#endif
// DECLARE_VOLUMES
extern wasm_module_t* wasmer_object_module_new(wasm_store_t* store,const char* wasm_name) asm("wasmer_object_module_new");
@@ -97,11 +94,15 @@ int main(int argc, char *argv[]) {
wasm_engine_t *engine = wasm_engine_new_with_config(config);
wasm_store_t *store = wasm_store_new(engine);
#ifdef WASI_PIRITA
const char* selected_atom = "main";
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--command") == 0 && i + 1 < argc) {
selected_atom = argv[i + 1];
break;
}
}
// INSTANTIATE_MODULES
#else
wasm_module_t *module = wasmer_object_module_new(store, "module");
#endif
if (!module) {
fprintf(stderr, "Failed to create module\n");

6
lib/compiler/src/engine/artifact.rs
View File

@@ -51,9 +51,6 @@ pub struct Artifact {
#[cfg(feature = "static-artifact-create")]
pub type PrefixerFn = Box<dyn Fn(&[u8]) -> String + Send>;
#[cfg(feature = "static-artifact-create")]
const WASMER_METADATA_SYMBOL: &[u8] = b"WASMER_METADATA";
impl Artifact {
/// Compile a data buffer into a `ArtifactBuild`, which may then be instantiated.
#[cfg(feature = "compiler")]
@@ -472,6 +469,7 @@ impl Artifact {
pub fn generate_object<'data>(
compiler: &dyn Compiler,
data: &[u8],
object_name: &str,
prefixer: Option<PrefixerFn>,
target: &'data Target,
tunables: &dyn Tunables,
@@ -550,7 +548,7 @@ impl Artifact {
)?;
let mut obj = get_object_for_target(target_triple).map_err(to_compile_error)?;
emit_data(&mut obj, WASMER_METADATA_SYMBOL, &metadata_binary, 1)
emit_data(&mut obj, object_name.as_bytes(), &metadata_binary, 1)
.map_err(to_compile_error)?;
emit_compilation(&mut obj, compilation, &symbol_registry, target_triple)

1
lib/registry/src/lib.rs
View File

@@ -91,7 +91,6 @@ impl fmt::Display for LocalPackage {
}
impl LocalPackage {
pub fn get_path(&self) -> Result<PathBuf, String> {
Ok(self.path.clone())
}