Add downloading .tar.gz URLs and new caching in checkouts dir

Cargo.lock

@@ -1086,9 +1086,9 @@ dependencies = [
 
 [[package]]
 name = "filetime"
-version = "0.2.18"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b9663d381d07ae25dc88dbdf27df458faa83a9b25336bcac83d5e452b5fc9d3"
+checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
@@ -1130,6 +1130,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fs_extra"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
+
 [[package]]
 name = "fuchsia-cprng"
 version = "0.1.1"
@@ -4059,6 +4065,7 @@ dependencies = [
  "dirs",
  "distance",
  "fern",
+ "hex",
  "http_req",
  "isatty",
  "libc",
@@ -4299,13 +4306,16 @@ version = "3.0.2"
 dependencies = [
  "anyhow",
  "dirs",
+ "filetime",
  "flate2",
+ "fs_extra",
  "futures-util",
  "graphql_client",
  "hex",
  "log",
  "lzma-rs",
  "rand 0.8.5",
+ "regex",
  "reqwest",
  "semver 1.0.14",
  "serde",
@@ -4313,6 +4323,7 @@ dependencies = [
  "tar",
  "tempdir",
  "thiserror",
+ "tldextract",
  "tokio",
  "toml",
  "url",

lib/cli/Cargo.toml

@@ -73,6 +73,7 @@ webc = { version = "3.0.1", optional = true }
 isatty = "0.1.9"
 dialoguer = "0.10.2"
 tldextract = "0.6.0"
+hex = "0.4.3"
 
 [build-dependencies]
 chrono = { version = "^0.4", default-features = false, features = [ "std", "clock" ] }

lib/cli/src/cli.rs

@@ -15,7 +15,6 @@ use crate::commands::{
 };
 use crate::error::PrettyError;
 use clap::{CommandFactory, ErrorKind, Parser};
-use std::{fmt, str::FromStr};
 
 #[derive(Parser, Debug)]
 #[cfg_attr(
@@ -243,218 +242,9 @@ fn wasmer_main_inner() -> Result<(), anyhow::Error> {
         }
     };
 
-    // Check if the file is a package name
-    if let WasmerCLIOptions::Run(r) = &options {
-        #[cfg(not(feature = "debug"))]
-        let debug = false;
-        #[cfg(feature = "debug")]
-        let debug = r.options.debug;
-        return crate::commands::try_run_package_or_file(&args, r, debug);
-    }
-
     options.execute()
 }
 
-#[derive(Debug, Clone, PartialEq, Default)]
-pub(crate) struct SplitVersion {
-    pub(crate) original: String,
-    pub(crate) registry: Option<String>,
-    pub(crate) package: String,
-    pub(crate) version: Option<String>,
-    pub(crate) command: Option<String>,
-}
-
-impl fmt::Display for SplitVersion {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let version = self.version.as_deref().unwrap_or("latest");
-        let command = self
-            .command
-            .as_ref()
-            .map(|s| format!(":{s}"))
-            .unwrap_or_default();
-        write!(f, "{}@{version}{command}", self.package)
-    }
-}
-
-#[test]
-fn test_split_version() {
-    assert_eq!(
-        SplitVersion::parse("registry.wapm.io/graphql/python/python").unwrap(),
-        SplitVersion {
-            original: "registry.wapm.io/graphql/python/python".to_string(),
-            registry: Some("https://registry.wapm.io/graphql".to_string()),
-            package: "python/python".to_string(),
-            version: None,
-            command: None,
-        }
-    );
-    assert_eq!(
-        SplitVersion::parse("registry.wapm.io/python/python").unwrap(),
-        SplitVersion {
-            original: "registry.wapm.io/python/python".to_string(),
-            registry: Some("https://registry.wapm.io/graphql".to_string()),
-            package: "python/python".to_string(),
-            version: None,
-            command: None,
-        }
-    );
-    assert_eq!(
-        SplitVersion::parse("namespace/name@version:command").unwrap(),
-        SplitVersion {
-            original: "namespace/name@version:command".to_string(),
-            registry: None,
-            package: "namespace/name".to_string(),
-            version: Some("version".to_string()),
-            command: Some("command".to_string()),
-        }
-    );
-    assert_eq!(
-        SplitVersion::parse("namespace/name@version").unwrap(),
-        SplitVersion {
-            original: "namespace/name@version".to_string(),
-            registry: None,
-            package: "namespace/name".to_string(),
-            version: Some("version".to_string()),
-            command: None,
-        }
-    );
-    assert_eq!(
-        SplitVersion::parse("namespace/name").unwrap(),
-        SplitVersion {
-            original: "namespace/name".to_string(),
-            registry: None,
-            package: "namespace/name".to_string(),
-            version: None,
-            command: None,
-        }
-    );
-    assert_eq!(
-        SplitVersion::parse("registry.wapm.io/namespace/name").unwrap(),
-        SplitVersion {
-            original: "registry.wapm.io/namespace/name".to_string(),
-            registry: Some("https://registry.wapm.io/graphql".to_string()),
-            package: "namespace/name".to_string(),
-            version: None,
-            command: None,
-        }
-    );
-    assert_eq!(
-        format!("{}", SplitVersion::parse("namespace").unwrap_err()),
-        "Invalid package version: \"namespace\"".to_string(),
-    );
-}
-
-impl SplitVersion {
-    pub fn parse(s: &str) -> Result<SplitVersion, anyhow::Error> {
-        s.parse()
-    }
-}
-
-impl FromStr for SplitVersion {
-    type Err = anyhow::Error;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let command = WasmerCLIOptions::command();
-        let mut prohibited_package_names = command.get_subcommands().map(|s| s.get_name());
-
-        let re1 = regex::Regex::new(r#"(.*)/(.*)@(.*):(.*)"#).unwrap();
-        let re2 = regex::Regex::new(r#"(.*)/(.*)@(.*)"#).unwrap();
-        let re3 = regex::Regex::new(r#"(.*)/(.*)"#).unwrap();
-        let re4 = regex::Regex::new(r#"(.*)/(.*):(.*)"#).unwrap();
-
-        let mut no_version = false;
-
-        let captures = if re1.is_match(s) {
-            re1.captures(s)
-                .map(|c| {
-                    c.iter()
-                        .flatten()
-                        .map(|m| m.as_str().to_owned())
-                        .collect::<Vec<_>>()
-                })
-                .unwrap_or_default()
-        } else if re2.is_match(s) {
-            re2.captures(s)
-                .map(|c| {
-                    c.iter()
-                        .flatten()
-                        .map(|m| m.as_str().to_owned())
-                        .collect::<Vec<_>>()
-                })
-                .unwrap_or_default()
-        } else if re4.is_match(s) {
-            no_version = true;
-            re4.captures(s)
-                .map(|c| {
-                    c.iter()
-                        .flatten()
-                        .map(|m| m.as_str().to_owned())
-                        .collect::<Vec<_>>()
-                })
-                .unwrap_or_default()
-        } else if re3.is_match(s) {
-            re3.captures(s)
-                .map(|c| {
-                    c.iter()
-                        .flatten()
-                        .map(|m| m.as_str().to_owned())
-                        .collect::<Vec<_>>()
-                })
-                .unwrap_or_default()
-        } else {
-            return Err(anyhow::anyhow!("Invalid package version: {s:?}"));
-        };
-
-        let mut namespace = match captures.get(1).cloned() {
-            Some(s) => s,
-            None => {
-                return Err(anyhow::anyhow!(
-                    "Invalid package version: {s:?}: no namespace"
-                ))
-            }
-        };
-
-        let name = match captures.get(2).cloned() {
-            Some(s) => s,
-            None => return Err(anyhow::anyhow!("Invalid package version: {s:?}: no name")),
-        };
-
-        let mut registry = None;
-        if namespace.contains('/') {
-            let (r, n) = namespace.rsplit_once('/').unwrap();
-            let mut real_registry = r.to_string();
-            if !real_registry.ends_with("graphql") {
-                real_registry = format!("{real_registry}/graphql");
-            }
-            if !real_registry.contains("://") {
-                real_registry = format!("https://{real_registry}");
-            }
-            registry = Some(real_registry);
-            namespace = n.to_string();
-        }
-
-        let sv = SplitVersion {
-            original: s.to_string(),
-            registry,
-            package: format!("{namespace}/{name}"),
-            version: if no_version {
-                None
-            } else {
-                captures.get(3).cloned()
-            },
-            command: captures.get(if no_version { 3 } else { 4 }).cloned(),
-        };
-
-        let svp = sv.package.clone();
-        anyhow::ensure!(
-            !prohibited_package_names.any(|s| s == sv.package.trim()),
-            "Invalid package name {svp:?}"
-        );
-
-        Ok(sv)
-    }
-}
-
 fn print_help(verbose: bool) -> Result<(), anyhow::Error> {
     let mut cmd = WasmerCLIOptions::command();
     if verbose {

lib/cli/src/commands/add.rs

@@ -4,8 +4,6 @@ use anyhow::{Context, Error};
 use clap::Parser;
 use wasmer_registry::{Bindings, PartialWapmConfig, ProgrammingLanguage};
 
-use crate::cli::SplitVersion;
-
 /// Add a WAPM package's bindings to your application.
 #[derive(Debug, Parser)]
 pub struct Add {
@@ -26,7 +24,7 @@ pub struct Add {
     pip: bool,
     /// The packages to add (e.g. "wasmer/wasmer-pack@0.5.0" or "python/python")
     #[clap(parse(try_from_str))]
-    packages: Vec<SplitVersion>,
+    packages: Vec<wasmer_registry::Package>,
 }
 
 impl Add {
@@ -103,11 +101,11 @@ impl Add {
 
 fn lookup_bindings_for_package(
     registry: &str,
-    pkg: &SplitVersion,
+    pkg: &wasmer_registry::Package,
     language: &ProgrammingLanguage,
 ) -> Result<Bindings, Error> {
     let all_bindings =
-        wasmer_registry::list_bindings(registry, &pkg.package, pkg.version.as_deref())?;
+        wasmer_registry::list_bindings(registry, &pkg.package(), pkg.version.as_deref())?;
 
     match all_bindings.iter().find(|b| b.language == *language) {
         Some(b) => {

lib/cli/src/commands/run.rs

@@ -1,7 +1,7 @@
-use crate::cli::SplitVersion;
 use crate::common::get_cache_dir;
 #[cfg(feature = "debug")]
 use crate::logging;
+use crate::package_source::PackageSource;
 use crate::store::{CompilerType, StoreOptions};
 use crate::suggestions::suggest_function_exports;
 use crate::warning;
@@ -11,12 +11,10 @@ use std::collections::HashMap;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::str::FromStr;
-use url::Url;
 use wasmer::FunctionEnv;
 use wasmer::*;
 #[cfg(feature = "cache")]
 use wasmer_cache::{Cache, FileSystemCache, Hash};
-use wasmer_registry::PackageDownloadInfo;
 use wasmer_types::Type as ValueType;
 #[cfg(feature = "webc_runner")]
 use wasmer_wasi::runners::{Runner, WapmContainer};
@@ -27,6 +25,17 @@ mod wasi;
 #[cfg(feature = "wasi")]
 use wasi::Wasi;
 
+/// The options for the `wasmer run` subcommand, runs either a package, URL or a file
+#[derive(Debug, Parser, Clone, Default)]
+pub struct Run {
+    /// File to run
+    #[clap(name = "SOURCE", parse(try_from_str))]
+    pub(crate) path: PackageSource,
+    /// Options to run the file / package / URL with
+    #[clap(flatten)]
+    pub(crate) options: RunWithoutFile,
+}
+
 /// Same as `wasmer run`, but without the required `path` argument (injected previously)
 #[derive(Debug, Parser, Clone, Default)]
 pub struct RunWithoutFile {
@@ -83,103 +92,65 @@ pub struct RunWithoutFile {
     pub(crate) args: Vec<String>,
 }
 
-#[allow(dead_code)]
+/// Same as `Run`, but uses a resolved local file path.
-fn is_dir(e: &walkdir::DirEntry) -> bool {
+#[derive(Debug, Clone, Default)]
-    let meta = match e.metadata() {
+pub struct RunWithPathBuf {
-        Ok(o) => o,
-        Err(_) => return false,
-    };
-    meta.is_dir()
-}
-
-impl RunWithoutFile {
-    /// Given a local path, returns the `Run` command (overriding the `--path` argument).
-    pub fn into_run_args(
-        mut self,
-        package_root_dir: PathBuf, // <- package dir
-        command: Option<&str>,
-        _debug_output_allowed: bool,
-    ) -> Result<Run, anyhow::Error> {
-        let (manifest, pathbuf) =
-            wasmer_registry::get_executable_file_from_path(&package_root_dir, command)?;
-
-        #[cfg(feature = "wasi")]
-        {
-            let default = HashMap::default();
-            let fs = manifest.fs.as_ref().unwrap_or(&default);
-            for (alias, real_dir) in fs.iter() {
-                let real_dir = package_root_dir.join(&real_dir);
-                if !real_dir.exists() {
-                    if _debug_output_allowed {
-                        println!(
-                            "warning: cannot map {alias:?} to {}: directory does not exist",
-                            real_dir.display()
-                        );
-                    }
-                    continue;
-                }
-
-                self.wasi.map_dir(alias, real_dir.clone());
-            }
-        }
-
-        Ok(Run {
-            path: pathbuf,
-            options: RunWithoutFile {
-                force_install: self.force_install,
-                #[cfg(feature = "cache")]
-                disable_cache: self.disable_cache,
-                invoke: self.invoke,
-                // If the RunWithoutFile was constructed via a package name,
-                // the correct syntax is "package:command-name" (--command-name would be
-                // interpreted as a CLI argument for the .wasm file)
-                command_name: None,
-                #[cfg(feature = "cache")]
-                cache_key: self.cache_key,
-                store: self.store,
-                #[cfg(feature = "wasi")]
-                wasi: self.wasi,
-                #[cfg(feature = "io-devices")]
-                enable_experimental_io_devices: self.enable_experimental_io_devices,
-                #[cfg(feature = "debug")]
-                debug: self.debug,
-                #[cfg(feature = "debug")]
-                verbose: self.verbose,
-                args: self.args,
-            },
-        })
-    }
-}
-
-#[derive(Debug, Parser, Clone, Default)]
-/// The options for the `wasmer run` subcommand
-pub struct Run {
     /// File to run
-    #[clap(name = "FILE", parse(from_os_str))]
     pub(crate) path: PathBuf,
-
+    /// Options for running the file
-    #[clap(flatten)]
     pub(crate) options: RunWithoutFile,
 }
 
-impl Deref for Run {
+impl Deref for RunWithPathBuf {
     type Target = RunWithoutFile;
     fn deref(&self) -> &Self::Target {
         &self.options
     }
 }
 
-impl Run {
+impl RunWithPathBuf {
     /// Execute the run command
     pub fn execute(&self) -> Result<()> {
+        let mut self_clone = self.clone();
+
+        if self_clone.path.is_dir() {
+            let (manifest, pathbuf) = wasmer_registry::get_executable_file_from_path(
+                &self_clone.path,
+                self_clone.command_name.as_deref(),
+            )?;
+
+            #[cfg(feature = "wasi")]
+            {
+                let default = HashMap::default();
+                let fs = manifest.fs.as_ref().unwrap_or(&default);
+                for (alias, real_dir) in fs.iter() {
+                    let real_dir = self_clone.path.join(&real_dir);
+                    if !real_dir.exists() {
+                        #[cfg(feature = "debug")]
+                        if self_clone.debug {
+                            println!(
+                                "warning: cannot map {alias:?} to {}: directory does not exist",
+                                real_dir.display()
+                            );
+                        }
+                        continue;
+                    }
+
+                    self_clone.options.wasi.map_dir(alias, real_dir.clone());
+                }
+            }
+
+            self_clone.path = pathbuf;
+        }
+
         #[cfg(feature = "debug")]
         if self.debug {
-            logging::set_up_logging(self.verbose.unwrap_or(0)).unwrap();
+            logging::set_up_logging(self_clone.verbose.unwrap_or(0)).unwrap();
         }
-        self.inner_execute().with_context(|| {
+        self_clone.inner_execute().with_context(|| {
             format!(
                 "failed to run `{}`{}",
-                self.path.display(),
+                self_clone.path.display(),
                 if CompilerType::enabled().is_empty() {
                     " (no compilers enabled)"
                 } else {
@@ -586,6 +557,19 @@ impl Run {
             .collect::<Result<Vec<_>>>()?;
         Ok(func.call(ctx, &invoke_args)?)
     }
+}
+
+impl Run {
+    /// Executes the `wasmer run` command
+    pub fn execute(&self) -> Result<(), anyhow::Error> {
+        // downloads and installs the package if necessary
+        let path_to_run = self.path.download_and_get_filepath()?;
+        RunWithPathBuf {
+            path: path_to_run,
+            options: self.options.clone(),
+        }
+        .execute()
+    }
 
     /// Create Run instance for arguments/env,
     /// assuming we're being run from a CFP binfmt interpreter.
@@ -599,37 +583,17 @@ impl Run {
 
     #[cfg(target_os = "linux")]
     fn from_binfmt_args_fallible() -> Result<Run> {
-        let argv = std::env::args_os().collect::<Vec<_>>();
+        let argv = std::env::args().collect::<Vec<_>>();
         let (_interpreter, executable, original_executable, args) = match &argv[..] {
             [a, b, c, d @ ..] => (a, b, c, d),
             _ => {
                 bail!("Wasmer binfmt interpreter needs at least three arguments (including $0) - must be registered as binfmt interpreter with the CFP flags. (Got arguments: {:?})", argv);
             }
         };
-        // TODO: Optimally, args and env would be passed as an UTF-8 Vec.
-        // (Can be pulled out of std::os::unix::ffi::OsStrExt)
-        // But I don't want to duplicate or rewrite run.rs today.
-        let args = args
-            .iter()
-            .enumerate()
-            .map(|(i, s)| {
-                s.clone().into_string().map_err(|s| {
-                    anyhow!(
-                        "Cannot convert argument {} ({:?}) to UTF-8 string",
-                        i + 1,
-                        s
-                    )
-                })
-            })
-            .collect::<Result<Vec<_>>>()?;
-        let original_executable = original_executable
-            .clone()
-            .into_string()
-            .map_err(|s| anyhow!("Cannot convert executable name {:?} to UTF-8 string", s))?;
         let store = StoreOptions::default();
         // TODO: store.compiler.features.all = true; ?
         Ok(Self {
-            path: executable.into(),
+            path: PackageSource::parse(executable),
             options: RunWithoutFile {
                 args,
                 command_name: Some(original_executable),
@@ -639,324 +603,9 @@ impl Run {
             },
         })
     }
+
     #[cfg(not(target_os = "linux"))]
     fn from_binfmt_args_fallible() -> Result<Run> {
         bail!("binfmt_misc is only available on linux.")
     }
 }
-
-fn start_spinner(msg: String) -> Option<spinoff::Spinner> {
-    if !isatty::stdout_isatty() {
-        return None;
-    }
-    #[cfg(target_os = "windows")]
-    {
-        use colored::control;
-        let _ = control::set_virtual_terminal(true);
-    }
-    Some(spinoff::Spinner::new(
-        spinoff::Spinners::Dots,
-        msg,
-        spinoff::Color::White,
-    ))
-}
-
-/// Before looking up a command from the registry, try to see if we have
-/// the command already installed
-fn try_run_local_command(
-    args: &[String],
-    sv: &SplitVersion,
-    debug_msgs_allowed: bool,
-) -> Result<(), ExecuteLocalPackageError> {
-    let result = wasmer_registry::try_finding_local_command(&sv.original).ok_or_else(|| {
-        ExecuteLocalPackageError::BeforeExec(anyhow::anyhow!(
-            "could not find command {} locally",
-            sv.original
-        ))
-    })?;
-    let package_dir = result
-        .get_path()
-        .map_err(|e| ExecuteLocalPackageError::BeforeExec(anyhow::anyhow!("{e}")))?;
-
-    // Try auto-installing the remote package
-    let args_without_package = fixup_args(args, &sv.original);
-    let mut run_args = RunWithoutFile::try_parse_from(args_without_package.iter())
-        .map_err(|e| ExecuteLocalPackageError::DuringExec(e.into()))?;
-    run_args.command_name = sv.command.clone();
-
-    run_args
-        .into_run_args(package_dir, sv.command.as_deref(), debug_msgs_allowed)
-        .map_err(ExecuteLocalPackageError::DuringExec)?
-        .execute()
-        .map_err(ExecuteLocalPackageError::DuringExec)
-}
-
-pub(crate) fn try_autoinstall_package(
-    args: &[String],
-    sv: &SplitVersion,
-    package: Option<PackageDownloadInfo>,
-    force_install: bool,
-) -> Result<(), anyhow::Error> {
-    use std::io::Write;
-    let mut sp = start_spinner(format!("Installing package {} ...", sv.package));
-    let debug_msgs_allowed = sp.is_some();
-    let v = sv.version.as_deref();
-    let result = wasmer_registry::install_package(
-        sv.registry.as_deref(),
-        &sv.package,
-        v,
-        package,
-        force_install,
-    );
-    if let Some(sp) = sp.take() {
-        sp.clear();
-    }
-    let _ = std::io::stdout().flush();
-    let (_, package_dir) = match result {
-        Ok(o) => o,
-        Err(e) => {
-            return Err(anyhow::anyhow!("{e}"));
-        }
-    };
-
-    // Try auto-installing the remote package
-    let args_without_package = fixup_args(args, &sv.original);
-    let mut run_args = RunWithoutFile::try_parse_from(args_without_package.iter())?;
-    run_args.command_name = sv.command.clone();
-
-    run_args
-        .into_run_args(package_dir, sv.command.as_deref(), debug_msgs_allowed)?
-        .execute()
-}
-
-// We need to distinguish between errors that happen
-// before vs. during execution
-enum ExecuteLocalPackageError {
-    BeforeExec(anyhow::Error),
-    DuringExec(anyhow::Error),
-}
-
-fn try_execute_local_package(
-    args: &[String],
-    sv: &SplitVersion,
-    debug_msgs_allowed: bool,
-) -> Result<(), ExecuteLocalPackageError> {
-    let package = wasmer_registry::get_local_package(None, &sv.package, sv.version.as_deref())
-        .ok_or_else(|| {
-            ExecuteLocalPackageError::BeforeExec(anyhow::anyhow!("no local package {sv:?} found"))
-        })?;
-
-    let package_dir = package
-        .get_path()
-        .map_err(|e| ExecuteLocalPackageError::BeforeExec(anyhow::anyhow!("{e}")))?;
-
-    // Try finding the local package
-    let args_without_package = fixup_args(args, &sv.original);
-
-    RunWithoutFile::try_parse_from(args_without_package.iter())
-        .map_err(|e| ExecuteLocalPackageError::DuringExec(e.into()))?
-        .into_run_args(package_dir, sv.command.as_deref(), debug_msgs_allowed)
-        .map_err(ExecuteLocalPackageError::DuringExec)?
-        .execute()
-        .map_err(|e| ExecuteLocalPackageError::DuringExec(e.context(anyhow::anyhow!("{}", sv))))
-}
-
-fn try_lookup_command(sv: &mut SplitVersion) -> Result<PackageDownloadInfo, anyhow::Error> {
-    use std::io::Write;
-    let mut sp = start_spinner(format!("Looking up command {} ...", sv.package));
-
-    for registry in wasmer_registry::get_all_available_registries().unwrap_or_default() {
-        let result = wasmer_registry::query_command_from_registry(&registry, &sv.package);
-        if let Some(s) = sp.take() {
-            s.clear();
-        }
-        let _ = std::io::stdout().flush();
-        let command = sv.package.clone();
-        if let Ok(o) = result {
-            sv.package = o.package.clone();
-            sv.version = Some(o.version.clone());
-            sv.command = Some(command);
-            return Ok(o);
-        }
-    }
-
-    if let Some(sp) = sp.take() {
-        sp.clear();
-    }
-    let _ = std::io::stdout().flush();
-    Err(anyhow::anyhow!("command {sv} not found"))
-}
-
-/// Removes the difference between "wasmer run {file} arg1 arg2" and "wasmer {file} arg1 arg2"
-fn fixup_args(args: &[String], command: &str) -> Vec<String> {
-    let mut args_without_package = args.to_vec();
-    if args_without_package.get(1).map(|s| s.as_str()) == Some(command) {
-        let _ = args_without_package.remove(1);
-    } else if args_without_package.get(2).map(|s| s.as_str()) == Some(command) {
-        let _ = args_without_package.remove(1);
-        let _ = args_without_package.remove(1);
-    }
-    args_without_package
-}
-
-#[test]
-fn test_fixup_args() {
-    let first_args = vec![
-        format!("wasmer"),
-        format!("run"),
-        format!("python/python"),
-        format!("--arg1"),
-        format!("--arg2"),
-    ];
-
-    let second_args = vec![
-        format!("wasmer"), // no "run"
-        format!("python/python"),
-        format!("--arg1"),
-        format!("--arg2"),
-    ];
-
-    let arg1_transformed = fixup_args(&first_args, "python/python");
-    let arg2_transformed = fixup_args(&second_args, "python/python");
-
-    assert_eq!(arg1_transformed, arg2_transformed);
-}
-
-pub(crate) fn try_run_package_or_file(
-    args: &[String],
-    r: &Run,
-    debug: bool,
-) -> Result<(), anyhow::Error> {
-    let debug_msgs_allowed = isatty::stdout_isatty();
-
-    // Check "r.path" is a file or a package / command name
-    if r.path.exists() {
-        if r.path.is_dir() && r.path.join("wapm.toml").exists() {
-            let args_without_package = fixup_args(args, &format!("{}", r.path.display()));
-            return RunWithoutFile::try_parse_from(args_without_package.iter())?
-                .into_run_args(
-                    r.path.clone(),
-                    r.command_name.as_deref(),
-                    debug_msgs_allowed,
-                )?
-                .execute();
-        }
-        return r.execute();
-    }
-
-    // c:// might be parsed as a URL on Windows
-    let url_string = format!("{}", r.path.display());
-    if let Ok(url) = url::Url::parse(&url_string) {
-        if url.scheme() == "http" || url.scheme() == "https" {
-            match try_run_url(&url, args, r, debug) {
-                Err(ExecuteLocalPackageError::BeforeExec(_)) => {}
-                Err(ExecuteLocalPackageError::DuringExec(e)) => return Err(e),
-                Ok(o) => return Ok(o),
-            }
-        }
-    }
-
-    let package = format!("{}", r.path.display());
-
-    let mut is_fake_sv = false;
-    let mut sv = match SplitVersion::parse(&package) {
-        Ok(o) => o,
-        Err(_) => {
-            let mut fake_sv = SplitVersion {
-                original: package.to_string(),
-                registry: None,
-                package: package.to_string(),
-                version: None,
-                command: None,
-            };
-            is_fake_sv = true;
-            match try_run_local_command(args, &fake_sv, debug) {
-                Ok(()) => return Ok(()),
-                Err(ExecuteLocalPackageError::DuringExec(e)) => return Err(e),
-                _ => {}
-            }
-            match try_lookup_command(&mut fake_sv) {
-                Ok(o) => SplitVersion {
-                    original: package.to_string(),
-                    registry: None,
-                    package: o.package,
-                    version: Some(o.version),
-                    command: r.command_name.clone(),
-                },
-                Err(e) => {
-                    return Err(
-                        anyhow::anyhow!("No package for command {package:?} found, file {package:?} not found either")
-                        .context(e)
-                        .context(anyhow::anyhow!("{}", r.path.display()))
-                    );
-                }
-            }
-        }
-    };
-
-    if sv.command.is_none() {
-        sv.command = r.command_name.clone();
-    }
-
-    if sv.command.is_none() && is_fake_sv {
-        sv.command = Some(package);
-    }
-
-    let mut package_download_info = None;
-    if !sv.package.contains('/') {
-        if let Ok(o) = try_lookup_command(&mut sv) {
-            package_download_info = Some(o);
-        }
-    }
-
-    match try_execute_local_package(args, &sv, debug_msgs_allowed) {
-        Ok(o) => return Ok(o),
-        Err(ExecuteLocalPackageError::DuringExec(e)) => return Err(e),
-        _ => {}
-    }
-
-    if debug && isatty::stdout_isatty() {
-        eprintln!("finding local package {} failed", sv);
-    }
-
-    // else: local package not found - try to download and install package
-    try_autoinstall_package(args, &sv, package_download_info, r.force_install)
-}
-
-fn try_run_url(
-    url: &Url,
-    _args: &[String],
-    r: &Run,
-    _debug: bool,
-) -> Result<(), ExecuteLocalPackageError> {
-    let checksum = wasmer_registry::get_remote_webc_checksum(url).map_err(|e| {
-        ExecuteLocalPackageError::BeforeExec(anyhow::anyhow!("error fetching {url}: {e}"))
-    })?;
-
-    let packages = wasmer_registry::get_all_installed_webc_packages();
-
-    if !packages.iter().any(|p| p.checksum == checksum) {
-        let sp = start_spinner(format!("Installing {}", url));
-
-        let result = wasmer_registry::install_webc_package(url, &checksum);
-
-        result.map_err(|e| {
-            ExecuteLocalPackageError::BeforeExec(anyhow::anyhow!("error fetching {url}: {e}"))
-        })?;
-
-        if let Some(sp) = sp {
-            sp.clear();
-        }
-    }
-
-    let webc_dir = wasmer_registry::get_webc_dir();
-
-    let webc_install_path = webc_dir
-        .context("Error installing package: no webc dir")
-        .map_err(ExecuteLocalPackageError::BeforeExec)?
-        .join(checksum);
-
-    let mut r = r.clone();
-    r.path = webc_install_path;
-    r.execute().map_err(ExecuteLocalPackageError::DuringExec)
-}

lib/cli/src/lib.rs

@@ -24,6 +24,7 @@ pub mod c_gen;
 pub mod cli;
 #[cfg(feature = "debug")]
 pub mod logging;
+pub mod package_source;
 pub mod store;
 pub mod suggestions;
 pub mod utils;

lib/cli/src/package_source.rs

@@ -0,0 +1,114 @@
+//! Module for parsing and installing packages
+
+use anyhow::Context;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+use url::Url;
+
+/// Source of a package
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum PackageSource {
+    /// Download from a URL
+    Url(Url),
+    /// Run a local file
+    File(String),
+    /// Download from a package
+    Package(wasmer_registry::Package),
+}
+
+impl Default for PackageSource {
+    fn default() -> Self {
+        PackageSource::File(String::new())
+    }
+}
+
+impl FromStr for PackageSource {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::parse(s)
+    }
+}
+
+impl PackageSource {
+    /// Parses a package source and transforms it to a URL or a File
+    pub fn parse(s: &str) -> Result<Self, String> {
+        // If the file is a http:// URL, run the URL
+        if let Ok(url) = url::Url::parse(s) {
+            if url.scheme() == "http" || url.scheme() == "https" {
+                return Ok(Self::Url(url));
+            }
+        }
+
+        Ok(match wasmer_registry::Package::from_str(s) {
+            Ok(o) => Self::Package(o),
+            Err(_) => Self::File(s.to_string()),
+        })
+    }
+
+    /// Downloads the package (if any) to the installation directory, returns the path
+    /// of the package directory (containing the wapm.toml)
+    pub fn download_and_get_filepath(&self) -> Result<PathBuf, anyhow::Error> {
+        let url = match self {
+            Self::File(f) => {
+                let path = Path::new(&f).to_path_buf();
+                return if path.exists() {
+                    Ok(path)
+                } else {
+                    Err(anyhow::anyhow!(
+                        "invalid package name, could not find file {f}"
+                    ))
+                };
+            }
+            Self::Url(u) => {
+                if let Some(path) = wasmer_registry::Package::is_url_already_installed(u) {
+                    return Ok(path);
+                } else {
+                    u.clone()
+                }
+            }
+            Self::Package(p) => {
+                if let Some(path) = p.already_installed() {
+                    return Ok(path);
+                } else {
+                    p.url()?
+                }
+            }
+        };
+
+        let extra = if let Self::Package(p) = self {
+            format!(", file {} does not exist either", p.file())
+        } else {
+            String::new()
+        };
+
+        let mut sp = start_spinner(format!("Installing package {url} ..."));
+        let opt_path = wasmer_registry::install_package(&url);
+        if let Some(sp) = sp.take() {
+            use std::io::Write;
+            sp.clear();
+            let _ = std::io::stdout().flush();
+        }
+
+        let (_, path) = opt_path
+            .with_context(|| anyhow::anyhow!("could not install package from URL {url}{extra}"))?;
+
+        Ok(path)
+    }
+}
+
+fn start_spinner(msg: String) -> Option<spinoff::Spinner> {
+    if !isatty::stdout_isatty() {
+        return None;
+    }
+    #[cfg(target_os = "windows")]
+    {
+        use colored::control;
+        let _ = control::set_virtual_terminal(true);
+    }
+    Some(spinoff::Spinner::new(
+        spinoff::Spinners::Dots,
+        msg,
+        spinoff::Color::White,
+    ))
+}

lib/registry/Cargo.toml

@@ -30,3 +30,7 @@ hex = "0.4.3"
 tokio = "1.21.2"
 tempdir = "0.3.7"
 log = "0.4.17"
+regex = "1.7.0"
+fs_extra = "1.2.0"
+filetime = "0.2.19"
+tldextract = "0.6.0"

lib/registry/src/lib.rs

@@ -16,20 +16,18 @@ use std::fmt;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
-use std::{
-    collections::BTreeMap,
-    fmt::{Display, Formatter},
-};
 use url::Url;
 
 pub mod config;
 pub mod graphql;
 pub mod login;
+pub mod package;
 pub mod queries;
 pub mod utils;
 
 pub use crate::{
     config::{format_graphql, PartialWapmConfig},
+    package::Package,
     queries::get_bindings_query::ProgrammingLanguage,
 };
 
@@ -157,9 +155,9 @@ pub fn get_executable_file_from_path(
             } else if commands.len() == 1 {
                 Ok(&commands[0])
             } else {
-                Err(anyhow::anyhow!("  -> wasmer run {name}@{version} --command-name={0} OR wasmer run {name}@{version}:{0}", commands.first().map(|f| f.get_name()).unwrap()))
+                Err(anyhow::anyhow!("  -> wasmer run {name}@{version} --command-name={0}", commands.first().map(|f| f.get_name()).unwrap()))
                 .context(anyhow::anyhow!("{}", commands.iter().map(|c| format!("`{}`", c.get_name())).collect::<Vec<_>>().join(", ")))
-                .context(anyhow::anyhow!("You can run any of those by using the --command-name=COMMAND flag or : postfix"))
+                .context(anyhow::anyhow!("You can run any of those by using the --command-name=COMMAND flag"))
                 .context(anyhow::anyhow!("The `{name}@{version}` package doesn't have a default entrypoint, but has multiple available commands:"))
             }?
         }
@@ -320,7 +318,7 @@ pub fn query_command_from_registry(
     });
 
     let response: get_package_by_command_query::ResponseData = execute_query(registry_url, "", &q)
-        .map_err(|e| format!("Error sending GetPackageByCommandQuery:  {e}"))?;
+        .map_err(|e| format!("Error sending GetPackageByCommandQuery:  {e}"))?;
 
     let command = response
         .get_command
@@ -353,7 +351,7 @@ pub enum QueryPackageError {
 }
 
 impl fmt::Display for QueryPackageError {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             QueryPackageError::ErrorSendingQuery(q) => write!(f, "error sending query: {q}"),
             QueryPackageError::NoPackageFound { name, version } => {
@@ -450,6 +448,7 @@ fn test_get_if_package_has_new_version() {
 pub fn get_if_package_has_new_version(
     #[cfg(test)] test_name: &str,
     registry_url: &str,
+    namespace: &str,
     name: &str,
     version: Option<String>,
     max_timeout: Duration,
@@ -462,10 +461,6 @@ pub fn get_if_package_has_new_version(
         Err(_) => return Err(format!("invalid host: {registry_url}")),
     };
 
-    let (namespace, name) = name
-        .split_once('/')
-        .ok_or_else(|| format!("missing namespace / name for {name:?}"))?;
-
     #[cfg(not(test))]
     let global_install_dir = get_global_install_dir(&host);
     #[cfg(test)]
@@ -524,7 +519,13 @@ pub fn get_if_package_has_new_version(
         .filter_map(|entry| {
             let entry = entry.ok()?;
             let version = semver::Version::parse(entry.file_name().to_str()?).ok()?;
-            let modified = entry.metadata().ok()?.modified().ok()?;
+            let modified = entry
+                .path()
+                .join("wapm.toml")
+                .metadata()
+                .ok()?
+                .modified()
+                .ok()?;
             let older_than_timeout = modified.elapsed().ok()? > max_timeout;
             Some((version, older_than_timeout))
         })
@@ -603,9 +604,7 @@ pub fn query_package_from_registry(
         })?;
 
     let v = response.package_version.as_ref().ok_or_else(|| {
-        QueryPackageError::ErrorSendingQuery(format!(
+        QueryPackageError::ErrorSendingQuery(format!("no package version for {name:?}"))
-            "Invalid response for crate {name:?}: no package version: {response:#?}"
-        ))
     })?;
 
     let manifest = toml::from_str::<wapm_toml::Manifest>(&v.manifest).map_err(|e| {
@@ -777,139 +776,80 @@ where
     Ok(())
 }
 
-/// Given a triple of [registry, name, version], downloads and installs the
+/// Installs the .tar.gz if it doesn't yet exist, returns the
-/// .tar.gz if it doesn't yet exist, returns the (package dir, entrypoint .wasm file path)
+/// (package dir, entrypoint .wasm file path)
 pub fn install_package(
     #[cfg(test)] test_name: &str,
-    registry: Option<&str>,
+    url: &Url,
-    name: &str,
+) -> Result<(LocalPackage, PathBuf), anyhow::Error> {
-    version: Option<&str>,
+    use fs_extra::dir::copy;
-    package_download_info: Option<PackageDownloadInfo>,
-    force_install: bool,
-) -> Result<(LocalPackage, PathBuf), String> {
-    let package_info = match package_download_info {
-        Some(s) => s,
-        None => {
-            let registries = match registry {
-                Some(s) => vec![s.to_string()],
-                None => {
-                    #[cfg(test)]
-                    {
-                        get_all_available_registries(test_name)?
-                    }
-                    #[cfg(not(test))]
-                    {
-                        get_all_available_registries()?
-                    }
-                }
-            };
-            let mut url_of_package = None;
 
-            let version_str = match version {
+    let host = url
-                None => name.to_string(),
-                Some(v) => format!("{name}@{v}"),
-            };
-
-            let registries_searched = registries
-                .iter()
-                .filter_map(|s| url::Url::parse(s).ok())
-                .filter_map(|s| Some(s.host_str()?.to_string()))
-                .collect::<Vec<_>>();
-
-            let mut errors = BTreeMap::new();
-
-            for r in registries.iter() {
-                if !force_install {
-                    #[cfg(not(test))]
-                    let package_has_new_version = get_if_package_has_new_version(
-                        r,
-                        name,
-                        version.map(|s| s.to_string()),
-                        Duration::from_secs(60 * 5),
-                    )?;
-                    #[cfg(test)]
-                    let package_has_new_version = get_if_package_has_new_version(
-                        test_name,
-                        r,
-                        name,
-                        version.map(|s| s.to_string()),
-                        Duration::from_secs(60 * 5),
-                    )?;
-                    if let GetIfPackageHasNewVersionResult::UseLocalAlreadyInstalled {
-                        registry_host,
-                        namespace,
-                        name,
-                        version,
-                        path,
-                    } = package_has_new_version
-                    {
-                        return Ok((
-                            LocalPackage {
-                                registry: registry_host,
-                                name: format!("{namespace}/{name}"),
-                                version,
-                            },
-                            path,
-                        ));
-                    }
-                }
-
-                match query_package_from_registry(r, name, version) {
-                    Ok(o) => {
-                        url_of_package = Some((r, o));
-                        break;
-                    }
-                    Err(e) => {
-                        errors.insert(r.clone(), e);
-                    }
-                }
-            }
-
-            let errors = errors
-                .into_iter()
-                .map(|(registry, e)| format!("  {registry}: {e}"))
-                .collect::<Vec<_>>()
-                .join("\r\n");
-
-            let (_, package_info) = url_of_package.ok_or_else(|| {
-                format!("Package {version_str} not found in registries {registries_searched:?}.\r\n\r\nErrors:\r\n\r\n{errors}")
-            })?;
-
-            package_info
-        }
-    };
-
-    let host = url::Url::parse(&package_info.registry)
-        .map_err(|e| format!("invalid url: {}: {e}", package_info.registry))?
         .host_str()
-        .ok_or_else(|| format!("invalid url: {}", package_info.registry))?
+        .ok_or_else(|| anyhow::anyhow!("invalid url: {}", url))?
         .to_string();
 
-    #[cfg(test)]
+    let tempdir = tempdir::TempDir::new(&format!("download-{host}"))
-    let dir = get_package_local_dir(
+        .map_err(|e| anyhow::anyhow!("could not create download temp dir: {e}"))?;
-        test_name,
-        &host,
-        &package_info.package,
-        &package_info.version,
-    )?;
-    #[cfg(not(test))]
-    let dir = get_package_local_dir(&host, &package_info.package, &package_info.version)?;
 
-    let version = package_info.version;
+    let target_targz_path = tempdir.path().join("package.tar.gz");
-    let name = package_info.package;
+    let unpacked_targz_path = tempdir.path().join("package");
+    std::fs::create_dir_all(&unpacked_targz_path).map_err(|e| {
+        anyhow::anyhow!(
+            "could not create dir {}: {e}",
+            unpacked_targz_path.display()
+        )
+    })?;
 
-    if !dir.join("wapm.toml").exists() || force_install {
+    get_targz_bytes(url, None, Some(target_targz_path.clone()))
-        download_and_unpack_targz(&package_info.url, &dir, false).map_err(|e| format!("{e}"))?;
+        .map_err(|e| anyhow::anyhow!("failed to download {url}: {e}"))?;
-    }
 
-    Ok((
+    try_unpack_targz(
-        LocalPackage {
+        target_targz_path.as_path(),
-            registry: package_info.registry,
+        unpacked_targz_path.as_path(),
-            name,
+        false,
-            version,
+    )
-        },
+    .with_context(|| anyhow::anyhow!("Could not unpack file downloaded from {url}"))?;
-        dir,
+
-    ))
+    // read {unpacked}/wapm.toml to get the name + version number
+    let toml_path = unpacked_targz_path.join("wapm.toml");
+    let toml = std::fs::read_to_string(&toml_path)
+        .map_err(|e| anyhow::anyhow!("error reading {}: {e}", toml_path.display()))?;
+    let toml_parsed = toml::from_str::<wapm_toml::Manifest>(&toml)
+        .map_err(|e| anyhow::anyhow!("error parsing {}: {e}", toml_path.display()))?;
+
+    let package = LocalPackage {
+        registry: host,
+        name: toml_parsed.package.name,
+        version: toml_parsed.package.version.to_string(),
+    };
+
+    let installation_path = package.get_path().map_err(|e| {
+        anyhow::anyhow!(
+            "could not determine installation path for {}: {e}",
+            package.name
+        )
+    })?;
+
+    std::fs::create_dir_all(&installation_path).map_err(|e| {
+        anyhow::anyhow!(
+            "could not create installation path for {}: {e}",
+            package.name
+        )
+    })?;
+
+    let mut options = fs_extra::dir::CopyOptions::new();
+    options.content_only = true;
+    options.overwrite = true;
+    copy(&unpacked_targz_path, &installation_path, &options)?;
+
+    #[cfg(not(target_os = "wasi"))]
+    let _ = filetime::set_file_mtime(
+        installation_path.join("wapm.toml"),
+        filetime::FileTime::now(),
+    );
+
+    Ok((package, installation_path))
 }
 
 pub fn whoami(
@@ -1043,6 +983,7 @@ async fn install_webc_package_inner(
         let builder = reqwest::Client::builder();
         let builder = crate::graphql::proxy::maybe_set_up_proxy(builder)?;
         builder
+            .redirect(reqwest::redirect::Policy::limited(10))
             .build()
             .map_err(|e| anyhow::anyhow!("{e}"))
             .context("install_webc_package: failed to build reqwest Client")?
@@ -1140,18 +1081,18 @@ pub fn get_checksum_hash(bytes: &[u8]) -> String {
     while checksum.last().copied() == Some(0) {
         checksum.pop();
     }
-    hex::encode(&checksum)
+    hex::encode(&checksum).chars().take(64).collect()
 }
 
 /// Returns the checksum of the .webc file, so that we can check whether the
 /// file is already installed before downloading it
 pub fn get_remote_webc_checksum(url: &Url) -> Result<String, anyhow::Error> {
     let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
-    let data = get_webc_bytes(url, Some(0..request_max_bytes))
+    let data = get_webc_bytes(url, Some(0..request_max_bytes), None)
-        .with_context(|| format!("get_webc_bytes failed on {url}"))?;
+        .with_context(|| anyhow::anyhow!("note: use --registry to change the registry URL"))?
+        .unwrap();
     let checksum = webc::WebC::get_checksum_bytes(&data)
-        .map_err(|e| anyhow::anyhow!("{e}"))
+        .map_err(|e| anyhow::anyhow!("{e}"))?
-        .context("get_checksum_bytes failed")?
         .to_vec();
     Ok(get_checksum_hash(&checksum))
 }
@@ -1161,7 +1102,7 @@ pub fn get_remote_webc_checksum(url: &Url) -> Result<String, anyhow::Error> {
 pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Error> {
     // Request up unti manifest size / manifest len
     let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
-    let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
+    let data = get_webc_bytes(url, Some(0..request_max_bytes), None)?.unwrap();
     let checksum = webc::WebC::get_checksum_bytes(&data)
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("WebC::get_checksum_bytes failed")?
@@ -1171,7 +1112,8 @@ pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Err
     let (manifest_start, manifest_len) = webc::WebC::get_manifest_offset_size(&data)
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("WebC::get_manifest_offset_size failed")?;
-    let data_with_manifest = get_webc_bytes(url, Some(0..manifest_start + manifest_len))?;
+    let data_with_manifest =
+        get_webc_bytes(url, Some(0..manifest_start + manifest_len), None)?.unwrap();
     let manifest = webc::WebC::get_manifest(&data_with_manifest)
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("WebC::get_manifest failed")?;
@@ -1181,39 +1123,99 @@ pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Err
     })
 }
 
-fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, anyhow::Error> {
+fn setup_client(
+    url: &Url,
+    application_type: &'static str,
+) -> Result<reqwest::blocking::RequestBuilder, anyhow::Error> {
     let client = {
         let builder = reqwest::blocking::Client::builder();
         let builder = crate::graphql::proxy::maybe_set_up_proxy_blocking(builder)
             .context("setup_webc_client")?;
         builder
+            .redirect(reqwest::redirect::Policy::limited(10))
             .build()
             .map_err(|e| anyhow::anyhow!("{e}"))
             .context("setup_webc_client: builder.build() failed")?
     };
 
-    Ok(client.get(url.clone()).header(ACCEPT, "application/webc"))
+    Ok(client.get(url.clone()).header(ACCEPT, application_type))
 }
 
-fn get_webc_bytes(url: &Url, range: Option<Range<usize>>) -> Result<Vec<u8>, anyhow::Error> {
+fn get_webc_bytes(
+    url: &Url,
+    range: Option<Range<usize>>,
+    stream_response_into: Option<PathBuf>,
+) -> Result<Option<Vec<u8>>, anyhow::Error> {
+    get_bytes(url, range, "application/webc", stream_response_into)
+}
+
+fn get_targz_bytes(
+    url: &Url,
+    range: Option<Range<usize>>,
+    stream_response_into: Option<PathBuf>,
+) -> Result<Option<Vec<u8>>, anyhow::Error> {
+    get_bytes(url, range, "application/tar+gzip", stream_response_into)
+}
+
+fn get_bytes(
+    url: &Url,
+    range: Option<Range<usize>>,
+    application_type: &'static str,
+    stream_response_into: Option<PathBuf>,
+) -> Result<Option<Vec<u8>>, anyhow::Error> {
     // curl -r 0-500 -L https://wapm.dev/syrusakbary/python -H "Accept: application/webc" --output python.webc
 
-    let mut res = setup_webc_client(url)?;
+    let mut res = setup_client(url, application_type)?;
 
     if let Some(range) = range.as_ref() {
         res = res.header(RANGE, format!("bytes={}-{}", range.start, range.end));
     }
 
-    let res = res
+    let mut res = res
         .send()
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("send() failed")?;
-    let bytes = res
-        .bytes()
-        .map_err(|e| anyhow::anyhow!("{e}"))
-        .context("bytes() failed")?;
 
-    Ok(bytes.to_vec())
+    if res.status().is_redirection() {
+        return Err(anyhow::anyhow!("redirect: {:?}", res.status()));
+    }
+
+    if res.status().is_server_error() {
+        return Err(anyhow::anyhow!("server error: {:?}", res.status()));
+    }
+
+    if res.status().is_client_error() {
+        return Err(anyhow::anyhow!("client error: {:?}", res.status()));
+    }
+
+    if let Some(path) = stream_response_into.as_ref() {
+        let mut file = std::fs::File::create(&path).map_err(|e| {
+            anyhow::anyhow!("failed to download {url} into {}: {e}", path.display())
+        })?;
+
+        res.copy_to(&mut file)
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .map_err(|e| {
+                anyhow::anyhow!("failed to download {url} into {}: {e}", path.display())
+            })?;
+
+        Ok(None)
+    } else {
+        let bytes = res
+            .bytes()
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .context("bytes() failed")?;
+
+        if (range.is_none() || range.unwrap().start == 0)
+            && bytes[0..webc::MAGIC.len()] != webc::MAGIC[..]
+        {
+            let bytes = bytes.iter().copied().take(100).collect::<Vec<_>>();
+            let first_50_bytes = String::from_utf8_lossy(&bytes);
+            return Err(anyhow::anyhow!("invalid webc bytes: {first_50_bytes:?}"));
+        }
+
+        Ok(Some(bytes.to_vec()))
+    }
 }
 
 // TODO: this test is segfaulting only on linux-musl, no other OS
@@ -1269,14 +1271,10 @@ fn test_install_package() {
 
     let all_installed_packages = get_all_local_packages(TEST_NAME, Some(registry));
 
-    println!("all_installed_packages: {all_installed_packages:#?}");
-
     let is_installed = all_installed_packages
         .iter()
         .any(|p| p.name == "wasmer/wabt" && p.version == "1.0.29");
 
-    println!("is_installed: {is_installed:#?}");
-
     if !is_installed {
         let panic_str = get_all_local_packages(TEST_NAME, Some(registry))
             .iter()
@@ -1316,8 +1314,8 @@ pub struct BindingsGenerator {
     pub command: String,
 }
 
-impl Display for BindingsGenerator {
+impl fmt::Display for BindingsGenerator {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let BindingsGenerator {
             package_name,
             version,

lib/registry/src/package.rs

@@ -0,0 +1,151 @@
+use crate::PartialWapmConfig;
+use std::path::PathBuf;
+use std::{fmt, str::FromStr};
+use url::Url;
+
+#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Package {
+    pub namespace: String,
+    pub name: String,
+    pub version: Option<String>,
+}
+
+impl fmt::Display for Package {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.file())
+    }
+}
+
+impl Package {
+    /// Checks whether the package is already installed, if yes, returns the path to the root dir
+    pub fn already_installed(&self) -> Option<PathBuf> {
+        let checkouts_dir = crate::get_checkouts_dir()?;
+        let hash = self.get_hash();
+        let found = std::fs::read_dir(&checkouts_dir)
+            .ok()?
+            .filter_map(|e| Some(e.ok()?.file_name().to_str()?.to_string()))
+            .find(|s| match self.version.as_ref() {
+                None => s.contains(&hash),
+                Some(v) => s.contains(&hash) && s.ends_with(v),
+            })?;
+        Some(checkouts_dir.join(found))
+    }
+
+    pub fn is_url_already_installed(_url: &Url) -> Option<PathBuf> {
+        None // TODO
+    }
+
+    pub fn get_hash(&self) -> String {
+        hex::encode(&format!(
+            "{}",
+            self.get_url_without_version().unwrap_or_default()
+        ))
+        .chars()
+        .take(64)
+        .collect()
+    }
+
+    fn get_url_without_version(&self) -> Result<String, anyhow::Error> {
+        Ok(format!(
+            "{}/{}/{}",
+            self.url()?.origin().ascii_serialization(),
+            self.namespace,
+            self.name
+        ))
+    }
+
+    /// Returns the filename for this package
+    pub fn file(&self) -> String {
+        let version = self
+            .version
+            .as_ref()
+            .map(|f| format!("@{f}"))
+            .unwrap_or_default();
+        format!("{}/{}{version}", self.namespace, self.name)
+    }
+
+    /// Returns the {namespace}/{name} package name
+    pub fn package(&self) -> String {
+        format!("{}/{}", self.namespace, self.name)
+    }
+
+    pub fn url(&self) -> Result<Url, anyhow::Error> {
+        let config = PartialWapmConfig::from_file()
+            .map_err(|e| anyhow::anyhow!("could not read wapm config: {e}"))?;
+        let registry = config.registry.get_current_registry();
+        let registry_tld = tldextract::TldExtractor::new(tldextract::TldOption::default())
+            .extract(&registry)
+            .map_err(|e| anyhow::anyhow!("Invalid registry: {}: {e}", registry))?;
+
+        let registry_tld = format!(
+            "{}.{}",
+            registry_tld.domain.as_deref().unwrap_or(""),
+            registry_tld.suffix.as_deref().unwrap_or(""),
+        );
+
+        let version = self
+            .version
+            .as_ref()
+            .map(|f| format!("@{f}"))
+            .unwrap_or_default();
+        let url = format!(
+            "https://{registry_tld}/{}/{}{version}",
+            self.namespace, self.name
+        );
+        url::Url::parse(&url).map_err(|e| anyhow::anyhow!("error parsing {url}: {e}"))
+    }
+
+    /// Returns the path to the installation directory.
+    /// Does not check whether the installation directory already exists.
+    pub fn get_path(&self) -> Result<PathBuf, anyhow::Error> {
+        let checkouts_dir =
+            crate::get_checkouts_dir().ok_or_else(|| anyhow::anyhow!("no checkouts dir"))?;
+        match self.version.as_ref() {
+            Some(v) => Ok(checkouts_dir.join(format!("{}@{}", self.get_hash(), v))),
+            None => Ok(checkouts_dir.join(&self.get_hash())),
+        }
+    }
+}
+
+impl FromStr for Package {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let regex = regex::Regex::new(r#"^([a-zA-Z0-9\-_]+)/([a-zA-Z0-9\-_]+)(@(.*))?$"#).unwrap();
+
+        let captures = regex
+            .captures(s.trim())
+            .map(|c| {
+                c.iter()
+                    .flatten()
+                    .map(|m| m.as_str().to_owned())
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default();
+
+        match captures.len() {
+            // namespace/package
+            3 => {
+                let namespace = captures[1].to_string();
+                let name = captures[2].to_string();
+                Ok(Package {
+                    namespace,
+                    name,
+                    version: None,
+                })
+            }
+            // namespace/package@version
+            5 => {
+                let namespace = captures[1].to_string();
+                let name = captures[2].to_string();
+                let version = captures[4].to_string();
+                Ok(Package {
+                    namespace,
+                    name,
+                    version: Some(version),
+                })
+            }
+            other => Err(anyhow::anyhow!("invalid package {other}")),
+        }
+    }
+}