Implement wasmer run {url}

Cargo.lock

@@ -2600,6 +2600,7 @@ dependencies = [
  "serde_urlencoded",
  "tokio",
  "tokio-rustls",
+ "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",
@@ -4179,7 +4180,9 @@ dependencies = [
  "anyhow",
  "dirs 4.0.0",
  "flate2",
+ "futures-util",
  "graphql_client",
+ "hex",
  "lzma-rs",
  "reqwest",
  "semver 1.0.14",
@@ -4187,9 +4190,11 @@ dependencies = [
  "serde_json",
  "tar",
  "thiserror",
+ "tokio",
  "toml",
  "url",
  "wapm-toml",
+ "webc",
  "whoami",
 ]
 

lib/cli/src/commands/run.rs

@@ -11,6 +11,8 @@ use std::collections::HashMap;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::str::FromStr;
+use std::sync::Arc;
+use url::Url;
 use wasmer::FunctionEnv;
 use wasmer::*;
 #[cfg(feature = "cache")]
@@ -827,6 +829,10 @@ pub(crate) fn try_run_package_or_file(
 ) -> Result<(), anyhow::Error> {
     let debug_msgs_allowed = isatty::stdout_isatty();
 
+    if let Ok(url) = url::Url::parse(&format!("{}", r.path.display())) {
+        return try_run_url(&url, args, r, debug);
+    }
+
     // Check "r.path" is a file or a package / command name
     if r.path.exists() {
         if r.path.is_dir() && r.path.join("wapm.toml").exists() {
@@ -908,3 +914,30 @@ pub(crate) fn try_run_package_or_file(
     // else: local package not found - try to download and install package
     try_autoinstall_package(args, &sv, package_download_info, r.force_install)
 }
+
+fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(), anyhow::Error> {
+    let checksum = wasmer_registry::get_remote_webc_checksum(&url)
+        .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
+
+    if !wasmer_registry::get_all_installed_webc_packages()
+        .iter()
+        .any(|p| p.checksum == checksum)
+    {
+        let sp = start_spinner(format!("Installing {}", url));
+
+        wasmer_registry::install_webc_package(&url, &checksum)
+            .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
+
+        if let Some(sp) = sp {
+            sp.close();
+        }
+    }
+
+    let webc_install_path = wasmer_registry::get_webc_dir()
+        .ok_or_else(|| anyhow::anyhow!("Error installing package: webc download failed"))?
+        .join(checksum);
+
+    let mut r = r.clone();
+    r.path = webc_install_path;
+    r.execute()
+}

lib/registry/Cargo.toml

@@ -10,7 +10,8 @@ dirs = "4.0.0"
 graphql_client = "0.11.0"
 serde = { version = "1.0.145", features = ["derive"] }
 anyhow = "1.0.65" 
-reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json"] }
+reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json", "stream"] }
+futures-util = "0.3.25"
 whoami = "1.2.3" 
 serde_json = "1.0.85"
 url = "2.3.1"
@@ -21,3 +22,6 @@ tar = "0.4.38"
 flate2 = "1.0.24"
 semver = "1.0.14"
 lzma-rs = "0.2.0"
+webc = { version ="3.0.1", features = ["mmap"] }
+hex = "0.4.3"
+tokio = "1.21.2"

lib/registry/src/lib.rs

@@ -1,11 +1,17 @@
 use std::collections::BTreeMap;
 use std::env;
 use std::fmt;
+use std::io::Write;
 use std::path::{Path, PathBuf};
+use std::ptr::read;
 use std::time::Duration;
 
+use reqwest::header::ACCEPT;
+use reqwest::header::RANGE;
 use serde::Deserialize;
 use serde::Serialize;
+use std::ops::Range;
+use url::Url;
 
 pub mod graphql {
 
@@ -20,7 +26,7 @@ pub mod graphql {
     #[cfg(target_os = "wasi")]
     use {wasm_bus_reqwest::prelude::header::*, wasm_bus_reqwest::prelude::*};
 
-    mod proxy {
+    pub mod proxy {
         //! Code for dealing with setting things up to proxy network requests
         use thiserror::Error;
 
@@ -940,6 +946,10 @@ pub fn get_checkouts_dir() -> Option<PathBuf> {
     Some(get_wasmer_root_dir()?.join("checkouts"))
 }
 
+pub fn get_webc_dir() -> Option<PathBuf> {
+    Some(get_wasmer_root_dir()?.join("webc"))
+}
+
 /// Returs the path to the directory where all packages on this computer are being stored
 pub fn get_global_install_dir(registry_host: &str) -> Option<PathBuf> {
     Some(get_checkouts_dir()?.join(registry_host))
@@ -1180,6 +1190,178 @@ pub fn get_all_available_registries() -> Result<Vec<String>, String> {
     Ok(registries)
 }
 
+#[derive(Debug, PartialEq, Clone)]
+pub struct RemoteWebcInfo {
+    pub checksum: String,
+    pub manifest: webc::Manifest,
+}
+
+pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), String> {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(async { install_webc_package_inner(url, checksum).await })
+}
+
+async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), String> {
+    use futures_util::StreamExt;
+
+    let path = get_webc_dir().ok_or_else(|| format!("no webc dir"))?;
+
+    let _ = std::fs::create_dir_all(&path);
+
+    let webc_path = path.join(checksum);
+
+    let mut file =
+        std::fs::File::create(&webc_path).map_err(|e| format!("{}: {e}", webc_path.display()))?;
+
+    let client = {
+        let builder = reqwest::Client::builder();
+
+        #[cfg(not(target_os = "wasi"))]
+        let builder = if let Some(proxy) =
+            crate::graphql::proxy::maybe_set_up_proxy().map_err(|e| format!("{e}"))?
+        {
+            builder.proxy(proxy)
+        } else {
+            builder
+        };
+
+        builder.build().map_err(|e| format!("{e}"))?
+    };
+
+    let res = client
+        .get(url.clone())
+        .header(ACCEPT, "application/webc")
+        .send()
+        .await
+        .map_err(|e| format!("{e}"))?;
+
+    let mut stream = res.bytes_stream();
+
+    while let Some(item) = stream.next().await {
+        let item = item.map_err(|e| format!("{e}"))?;
+        file.write_all(&item);
+    }
+
+    Ok(())
+}
+
+/// Returns a list of all installed webc packages
+pub fn get_all_installed_webc_packages() -> Vec<RemoteWebcInfo> {
+    let dir = match get_webc_dir() {
+        Some(s) => s,
+        None => return Vec::new(),
+    };
+
+    let read_dir = match std::fs::read_dir(dir) {
+        Ok(s) => s,
+        Err(_) => return Vec::new(),
+    };
+
+    read_dir
+        .filter_map(|r| Some(r.ok()?.path()))
+        .filter_map(|path| {
+            webc::WebCMmap::parse(
+                path,
+                &webc::ParseOptions {
+                    parse_atoms: false,
+                    parse_volumes: false,
+                    parse_manifest: true,
+                    ..Default::default()
+                },
+            )
+            .ok()
+        })
+        .filter_map(|webc| {
+            let mut checksum = webc.checksum.as_ref().map(|s| &s.data)?.to_vec();
+            while checksum.last().copied() == Some(0) {
+                checksum.pop();
+            }
+            let hex_string = hex::encode(&checksum);
+            Some(RemoteWebcInfo {
+                checksum: hex_string,
+                manifest: webc.manifest.clone(),
+            })
+        })
+        .collect()
+}
+
+/// Returns the checksum of the .webc file, so that we can check whether the
+/// file is already installed before downloading it
+pub fn get_remote_webc_checksum(url: &Url) -> Result<String, String> {
+    let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
+    let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
+    let mut checksum = webc::WebC::get_checksum_bytes(&data)
+        .map_err(|e| format!("{e}"))?
+        .to_vec();
+    while checksum.last().copied() == Some(0) {
+        checksum.pop();
+    }
+    let hex_string = hex::encode(&checksum);
+    Ok(hex_string)
+}
+
+/// Before fetching the entire file from a remote URL, just fetch the manifest
+/// so we can see if the package has already been installed
+pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, String> {
+    // Request up unti manifest size / manifest len
+    let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
+    let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
+    let mut checksum = webc::WebC::get_checksum_bytes(&data)
+        .map_err(|e| format!("{e}"))?
+        .to_vec();
+    while checksum.last().copied() == Some(0) {
+        checksum.pop();
+    }
+    let hex_string = hex::encode(&checksum);
+
+    let (manifest_start, manifest_len) =
+        webc::WebC::get_manifest_offset_size(&data).map_err(|e| format!("{e}"))?;
+    let data_with_manifest =
+        get_webc_bytes(url, Some(0..manifest_start + manifest_len)).map_err(|e| format!("{e}"))?;
+    let manifest = webc::WebC::get_manifest(&data_with_manifest).map_err(|e| format!("{e}"))?;
+    Ok(RemoteWebcInfo {
+        checksum: hex_string,
+        manifest,
+    })
+}
+
+fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, String> {
+    let client = {
+        let builder = reqwest::blocking::Client::builder();
+
+        #[cfg(not(target_os = "wasi"))]
+        let builder = if let Some(proxy) =
+            crate::graphql::proxy::maybe_set_up_proxy().map_err(|e| format!("{e}"))?
+        {
+            builder.proxy(proxy)
+        } else {
+            builder
+        };
+
+        builder.build().map_err(|e| format!("{e}"))?
+    };
+
+    Ok(client.get(url.clone()).header(ACCEPT, "application/webc"))
+}
+
+fn get_webc_bytes(url: &Url, range: Option<Range<usize>>) -> Result<Vec<u8>, String> {
+    // curl -r 0-500 -L https://wapm.dev/syrusakbary/python -H "Accept: application/webc" --output python.webc
+
+    let mut res = setup_webc_client(url)?;
+
+    if let Some(range) = range.as_ref() {
+        res = res.header(RANGE, format!("bytes={}-{}", range.start, range.end));
+    }
+
+    let res = res.send().map_err(|e| format!("{e}"))?;
+    let bytes = res.bytes().map_err(|e| format!("{e}"))?;
+
+    Ok(bytes.to_vec())
+}
+
 // TODO: this test is segfaulting only on linux-musl, no other OS
 // See https://github.com/wasmerio/wasmer/pull/3215
 #[cfg(not(target_env = "musl"))]