diff --git a/Cargo.lock b/Cargo.lock index 0adde1037..fb7582c8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1317,7 +1317,7 @@ checksum = "64fba5a42bd76a17cad4bfa00de168ee1cbfa06a5e8ce992ae880218c05641a9" dependencies = [ "byteorder", "dynasm", - "memmap2", + "memmap2 0.5.10", ] [[package]] @@ -2484,6 +2484,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memmap2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -4027,6 +4036,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shared-buffer" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a4987984c21cbd48a683c00975a0a9652d1882e8e986c42223f65440f4668f" +dependencies = [ + "bytes", + "memmap2 0.6.2", +] + [[package]] name = "shell-words" version = "1.1.0" @@ -5661,7 +5680,7 @@ dependencies = [ "hashbrown 0.11.2", "lazy_static", "leb128", - "memmap2", + "memmap2 0.5.10", "more-asserts", "region", "serde", @@ -6446,9 +6465,9 @@ dependencies = [ [[package]] name = "webc" -version = "5.0.2" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42af14e63ed784e4f813bd5fb35bc84016fa8b245879809547247da6051107f0" +checksum = "2041e22d0f634bec4f43f1fb77620fd68fb1adaa3f0a721fa682dcdb2e916840" dependencies = [ "anyhow", "base64", @@ -6457,7 +6476,6 @@ dependencies = [ "indexmap", "leb128", "lexical-sort", - "memmap2", "once_cell", "path-clean", "rand", @@ -6465,6 +6483,7 @@ dependencies = [ "serde_cbor", "serde_json", "sha2", + "shared-buffer", "thiserror", "url", "walkdir", diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index d21a57d87..babd148d2 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -514,7 +514,7 @@ fn serialize_volume_to_webc_v1(volume: &WebcVolume) -> Vec { if let Some(contents) = volume.read_file(&*path) { files.insert( webc::v1::DirOrFile::File(path.to_string().into()), - contents.into(), + contents.to_vec(), ); } } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 0cd094cec..f008378ed 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -29,7 +29,7 @@ bincode = { version = "1.3" } chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } derivative = { version = "^2" } bytes = "1" -webc = { version = "5.0.2", default-features = false } +webc = { version = "5.0.3", default-features = false } serde_cbor = { version = "0.11.2", optional = true } anyhow = { version = "1.0.66" } lazy_static = "1.4" diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 104ab4162..4320e6ced 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -1,21 +1,22 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, - path::Path, + path::{Path, PathBuf}, sync::Arc, }; use anyhow::{Context, Error}; use futures::{stream::FuturesUnordered, TryStreamExt}; use once_cell::sync::OnceCell; -use virtual_fs::{FileSystem, WebcVolumeFileSystem}; -use webc::compat::Container; +use virtual_fs::{FileSystem, OverlayFileSystem, WebcVolumeFileSystem}; +use webc::compat::{Container, Volume}; use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, runtime::{ package_loader::PackageLoader, resolver::{ - DependencyGraph, ItemLocation, PackageId, PackageSummary, Resolution, ResolvedPackage, + DependencyGraph, ItemLocation, PackageId, PackageSummary, Resolution, + ResolvedFileSystemMapping, ResolvedPackage, }, }, }; @@ -231,17 +232,7 @@ async fn fetch_dependencies( Ok(packages) } -fn filesystem( - packages: &HashMap, - pkg: &ResolvedPackage, -) -> Result { - // FIXME: Take the [fs] table into account - // See for more - let root = &packages[&pkg.root_package]; - let fs = WebcVolumeFileSystem::mount_all(root); - Ok(fs) -} - +/// How many bytes worth of files does a directory contain? fn count_file_system(fs: &dyn FileSystem, path: &Path) -> u64 { let mut total = 0; @@ -263,3 +254,156 @@ fn count_file_system(fs: &dyn FileSystem, path: &Path) -> u64 { total } + +/// Given a set of [`ResolvedFileSystemMapping`]s and the [`Container`] for each +/// package in a dependency tree, construct the resulting filesystem. +/// +/// # Note to future readers +/// +/// Sooo... this code is a bit convoluted because we're constrained by the +/// filesystem implementations we've got available. +/// +/// Ideally, we would create a WebcVolumeFileSystem for each volume we're +/// using, then we'd have a single "union" filesystem which lets you mount +/// filesystem objects under various paths and can deal with conflicts. +/// +/// The OverlayFileSystem lets us make files from multiple filesystem +/// implementations available at the same time, however all of the +/// filesystems will be mounted at "/", when the user wants to mount volumes +/// at arbitrary locations. +/// +/// The TmpFileSystem *does* allow mounting at non-root paths, however it can't +/// handle nested paths (e.g. mounting to "/lib" and "/lib/python3.10" - see +/// https://github.com/wasmerio/wasmer/issues/3678 for more) and you aren't +/// allowed to mount to "/" because it's a special directory that already +/// exists. +/// +/// As a result, we'll duct-tape things together and hope for the best 🤞 +fn filesystem( + packages: &HashMap, + pkg: &ResolvedPackage, +) -> Result { + let mut filesystems = Vec::new(); + let mut volumes: HashMap<&PackageId, BTreeMap> = HashMap::new(); + + let mut mountings: Vec<_> = pkg.filesystem.iter().collect(); + mountings.sort_by_key(|m| std::cmp::Reverse(m.mount_path.as_path())); + + for ResolvedFileSystemMapping { + mount_path, + volume_name, + package, + } in &pkg.filesystem + { + // Note: We want to reuse existing Volume instances if we can. That way + // we can keep the memory usage down. A webc::compat::Volume is + // reference-counted, anyway. + let container_volumes = match volumes.entry(package) { + std::collections::hash_map::Entry::Occupied(entry) => &*entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => { + // looks like we need to insert it + let container = packages.get(package) + .with_context(|| format!( + "\"{}\" wants to use the \"{}\" package, but it isn't in the dependency tree", + pkg.root_package, + package, + ))?; + &*entry.insert(container.volumes()) + } + }; + + let volume = container_volumes.get(volume_name).with_context(|| { + format!("The \"{package}\" package doesn't have a \"{volume_name}\" volume") + })?; + + let fs = NestedFileSystem::new( + mount_path.clone(), + WebcVolumeFileSystem::new(volume.clone()), + ); + filesystems.push(fs); + } + + let fs = OverlayFileSystem::new(virtual_fs::EmptyFileSystem::default(), filesystems); + + Ok(fs) +} + +/// A [`FileSystem`] implementation that exposes some other filesystem under a +/// nested directory. +#[derive(Debug, Clone, PartialEq)] +struct NestedFileSystem { + path: PathBuf, + inner: F, +} + +impl NestedFileSystem { + fn new(path: PathBuf, inner: F) -> Self { + NestedFileSystem { path, inner } + } + + fn strip_prefix(&self, path: &Path) -> Result { + let path = path + .strip_prefix(&self.path) + .map_err(|_| virtual_fs::FsError::BaseNotDirectory)?; + + // Don't forget to make the path absolute again. + Ok(Path::new("/").join(path)) + } +} + +impl FileSystem for NestedFileSystem +where + F: FileSystem, +{ + fn read_dir(&self, path: &Path) -> virtual_fs::Result { + let path = self.strip_prefix(path)?; + self.inner.read_dir(&path) + } + + fn create_dir(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.strip_prefix(path)?; + self.inner.create_dir(&path) + } + + fn remove_dir(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.strip_prefix(path)?; + self.inner.remove_dir(&path) + } + + fn rename(&self, from: &Path, to: &Path) -> virtual_fs::Result<()> { + let from = self.strip_prefix(from)?; + let to = self.strip_prefix(to)?; + self.inner.rename(&from, &to) + } + + fn metadata(&self, path: &Path) -> virtual_fs::Result { + let path = self.strip_prefix(path)?; + self.inner.metadata(&path) + } + + fn remove_file(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.strip_prefix(path)?; + self.inner.remove_file(&path) + } + + fn new_open_options(&self) -> virtual_fs::OpenOptions { + virtual_fs::OpenOptions::new(self) + } +} + +impl virtual_fs::FileOpener for NestedFileSystem +where + F: FileSystem, +{ + fn open( + &self, + path: &Path, + conf: &virtual_fs::OpenOptionsConfig, + ) -> virtual_fs::Result> { + let path = self.strip_prefix(path)?; + self.inner + .new_open_options() + .options(conf.clone()) + .open(path) + } +} diff --git a/lib/wasix/src/runtime/resolver/inputs.rs b/lib/wasix/src/runtime/resolver/inputs.rs index 51fddcb78..c5fd83a7d 100644 --- a/lib/wasix/src/runtime/resolver/inputs.rs +++ b/lib/wasix/src/runtime/resolver/inputs.rs @@ -188,7 +188,7 @@ pub struct PackageInfo { impl PackageInfo { pub fn from_manifest(manifest: &Manifest) -> Result { let WapmAnnotations { name, version, .. } = manifest - .package_annotation("wapm")? + .wapm()? .context("Unable to find the \"wapm\" annotations")?; let dependencies = manifest @@ -210,7 +210,7 @@ impl PackageInfo { }) .collect(); - let filesystem = filesystem_mapping_from_manifest(manifest); + let filesystem = filesystem_mapping_from_manifest(manifest)?; Ok(PackageInfo { name, @@ -230,15 +230,40 @@ impl PackageInfo { } } -fn filesystem_mapping_from_manifest(_manifest: &Manifest) -> Vec { - // FIXME(Michael-F-Bryan): wapm2pirita never added filesystem mappings to the manifest - // That means we need to - // - Figure out whether filesystem mappings belong to the whole package or - // if each command has their own set of mappings - // - Update wapm-targz-to-pirita to copy the [fs] table across - // - Re-generate all packages on WAPM - // - Update this function to copy metadata into our internal datastructures - Vec::new() +fn filesystem_mapping_from_manifest( + manifest: &Manifest, +) -> Result, serde_cbor::Error> { + match manifest.filesystem()? { + Some(webc::metadata::annotations::FileSystemMappings(mappings)) => { + let mappings = mappings + .into_iter() + .map(|mapping| FileSystemMapping { + volume_name: mapping.volume_name, + mount_path: mapping.mount_path, + dependency_name: mapping.from, + }) + .collect(); + + Ok(mappings) + } + None => { + // A "fs" annotation hasn't been attached to this package. This was + // the case when *.webc files were generated by wapm2pirita version + // 1.0.29 and earlier. + // + // To maintain compatibility with those older packages, we'll say + // that the "atom" volume from the current package is mounted to "/" + // and contains all files in the package. + tracing::debug!( + "No \"fs\" package annotations found. Mounting the \"atom\" volume to \"/\" for compatibility." + ); + Ok(vec![FileSystemMapping { + volume_name: "atom".to_string(), + mount_path: "/".to_string(), + dependency_name: None, + }]) + } + } } #[derive(Debug, Clone, PartialEq, Eq)]