From e8c422d5207d6191dd7c537f2d7cc6b3d743ee1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Thu, 10 Nov 2022 14:23:52 +0100
Subject: [PATCH 01/16] Implement wasmer run {url}

---
 Cargo.lock                  |   5 +
 lib/cli/src/commands/run.rs |  33 +++++++
 lib/registry/Cargo.toml     |   8 +-
 lib/registry/src/lib.rs     | 184 +++++++++++++++++++++++++++++++++++-
 4 files changed, 227 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7f7d22b5e6e..d3fb5166582 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2600,6 +2600,7 @@ dependencies = [
  "serde_urlencoded",
  "tokio",
  "tokio-rustls",
+ "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",
@@ -4179,7 +4180,9 @@ dependencies = [
  "anyhow",
  "dirs 4.0.0",
  "flate2",
+ "futures-util",
  "graphql_client",
+ "hex",
  "lzma-rs",
  "reqwest",
  "semver 1.0.14",
@@ -4187,9 +4190,11 @@ dependencies = [
  "serde_json",
  "tar",
  "thiserror",
+ "tokio",
  "toml",
  "url",
  "wapm-toml",
+ "webc",
  "whoami",
 ]
 
diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs
index 2b5dad57bb9..96dd2f5efe9 100644
--- a/lib/cli/src/commands/run.rs
+++ b/lib/cli/src/commands/run.rs
@@ -11,6 +11,8 @@ use std::collections::HashMap;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::str::FromStr;
+use std::sync::Arc;
+use url::Url;
 use wasmer::FunctionEnv;
 use wasmer::*;
 #[cfg(feature = "cache")]
@@ -827,6 +829,10 @@ pub(crate) fn try_run_package_or_file(
 ) -> Result<(), anyhow::Error> {
     let debug_msgs_allowed = isatty::stdout_isatty();
 
+    if let Ok(url) = url::Url::parse(&format!("{}", r.path.display())) {
+        return try_run_url(&url, args, r, debug);
+    }
+
     // Check "r.path" is a file or a package / command name
     if r.path.exists() {
         if r.path.is_dir() && r.path.join("wapm.toml").exists() {
@@ -908,3 +914,30 @@ pub(crate) fn try_run_package_or_file(
     // else: local package not found - try to download and install package
     try_autoinstall_package(args, &sv, package_download_info, r.force_install)
 }
+
+fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(), anyhow::Error> {
+    let checksum = wasmer_registry::get_remote_webc_checksum(&url)
+        .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
+
+    if !wasmer_registry::get_all_installed_webc_packages()
+        .iter()
+        .any(|p| p.checksum == checksum)
+    {
+        let sp = start_spinner(format!("Installing {}", url));
+
+        wasmer_registry::install_webc_package(&url, &checksum)
+            .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
+
+        if let Some(sp) = sp {
+            sp.close();
+        }
+    }
+
+    let webc_install_path = wasmer_registry::get_webc_dir()
+        .ok_or_else(|| anyhow::anyhow!("Error installing package: webc download failed"))?
+        .join(checksum);
+
+    let mut r = r.clone();
+    r.path = webc_install_path;
+    r.execute()
+}
diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml
index 03a3a97ea9d..40213575b19 100644
--- a/lib/registry/Cargo.toml
+++ b/lib/registry/Cargo.toml
@@ -10,7 +10,8 @@ dirs = "4.0.0"
 graphql_client = "0.11.0"
 serde = { version = "1.0.145", features = ["derive"] }
 anyhow = "1.0.65" 
-reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json"] }
+reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json", "stream"] }
+futures-util = "0.3.25"
 whoami = "1.2.3" 
 serde_json = "1.0.85"
 url = "2.3.1"
@@ -20,4 +21,7 @@ wapm-toml = "0.2.0"
 tar = "0.4.38"
 flate2 = "1.0.24"
 semver = "1.0.14"
-lzma-rs = "0.2.0"
\ No newline at end of file
+lzma-rs = "0.2.0"
+webc = { version ="3.0.1", features = ["mmap"] }
+hex = "0.4.3"
+tokio = "1.21.2"
\ No newline at end of file
diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index 4d034d5815d..20bd75f97dc 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -1,11 +1,17 @@
 use std::collections::BTreeMap;
 use std::env;
 use std::fmt;
+use std::io::Write;
 use std::path::{Path, PathBuf};
+use std::ptr::read;
 use std::time::Duration;
 
+use reqwest::header::ACCEPT;
+use reqwest::header::RANGE;
 use serde::Deserialize;
 use serde::Serialize;
+use std::ops::Range;
+use url::Url;
 
 pub mod graphql {
 
@@ -20,7 +26,7 @@ pub mod graphql {
     #[cfg(target_os = "wasi")]
     use {wasm_bus_reqwest::prelude::header::*, wasm_bus_reqwest::prelude::*};
 
-    mod proxy {
+    pub mod proxy {
         //! Code for dealing with setting things up to proxy network requests
         use thiserror::Error;
 
@@ -940,6 +946,10 @@ pub fn get_checkouts_dir() -> Option<PathBuf> {
     Some(get_wasmer_root_dir()?.join("checkouts"))
 }
 
+pub fn get_webc_dir() -> Option<PathBuf> {
+    Some(get_wasmer_root_dir()?.join("webc"))
+}
+
 /// Returs the path to the directory where all packages on this computer are being stored
 pub fn get_global_install_dir(registry_host: &str) -> Option<PathBuf> {
     Some(get_checkouts_dir()?.join(registry_host))
@@ -1180,6 +1190,178 @@ pub fn get_all_available_registries() -> Result<Vec<String>, String> {
     Ok(registries)
 }
 
+#[derive(Debug, PartialEq, Clone)]
+pub struct RemoteWebcInfo {
+    pub checksum: String,
+    pub manifest: webc::Manifest,
+}
+
+pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), String> {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(async { install_webc_package_inner(url, checksum).await })
+}
+
+async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), String> {
+    use futures_util::StreamExt;
+
+    let path = get_webc_dir().ok_or_else(|| format!("no webc dir"))?;
+
+    let _ = std::fs::create_dir_all(&path);
+
+    let webc_path = path.join(checksum);
+
+    let mut file =
+        std::fs::File::create(&webc_path).map_err(|e| format!("{}: {e}", webc_path.display()))?;
+
+    let client = {
+        let builder = reqwest::Client::builder();
+
+        #[cfg(not(target_os = "wasi"))]
+        let builder = if let Some(proxy) =
+            crate::graphql::proxy::maybe_set_up_proxy().map_err(|e| format!("{e}"))?
+        {
+            builder.proxy(proxy)
+        } else {
+            builder
+        };
+
+        builder.build().map_err(|e| format!("{e}"))?
+    };
+
+    let res = client
+        .get(url.clone())
+        .header(ACCEPT, "application/webc")
+        .send()
+        .await
+        .map_err(|e| format!("{e}"))?;
+
+    let mut stream = res.bytes_stream();
+
+    while let Some(item) = stream.next().await {
+        let item = item.map_err(|e| format!("{e}"))?;
+        file.write_all(&item);
+    }
+
+    Ok(())
+}
+
+/// Returns a list of all installed webc packages
+pub fn get_all_installed_webc_packages() -> Vec<RemoteWebcInfo> {
+    let dir = match get_webc_dir() {
+        Some(s) => s,
+        None => return Vec::new(),
+    };
+
+    let read_dir = match std::fs::read_dir(dir) {
+        Ok(s) => s,
+        Err(_) => return Vec::new(),
+    };
+
+    read_dir
+        .filter_map(|r| Some(r.ok()?.path()))
+        .filter_map(|path| {
+            webc::WebCMmap::parse(
+                path,
+                &webc::ParseOptions {
+                    parse_atoms: false,
+                    parse_volumes: false,
+                    parse_manifest: true,
+                    ..Default::default()
+                },
+            )
+            .ok()
+        })
+        .filter_map(|webc| {
+            let mut checksum = webc.checksum.as_ref().map(|s| &s.data)?.to_vec();
+            while checksum.last().copied() == Some(0) {
+                checksum.pop();
+            }
+            let hex_string = hex::encode(&checksum);
+            Some(RemoteWebcInfo {
+                checksum: hex_string,
+                manifest: webc.manifest.clone(),
+            })
+        })
+        .collect()
+}
+
+/// Returns the checksum of the .webc file, so that we can check whether the
+/// file is already installed before downloading it
+pub fn get_remote_webc_checksum(url: &Url) -> Result<String, String> {
+    let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
+    let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
+    let mut checksum = webc::WebC::get_checksum_bytes(&data)
+        .map_err(|e| format!("{e}"))?
+        .to_vec();
+    while checksum.last().copied() == Some(0) {
+        checksum.pop();
+    }
+    let hex_string = hex::encode(&checksum);
+    Ok(hex_string)
+}
+
+/// Before fetching the entire file from a remote URL, just fetch the manifest
+/// so we can see if the package has already been installed
+pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, String> {
+    // Request up unti manifest size / manifest len
+    let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
+    let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
+    let mut checksum = webc::WebC::get_checksum_bytes(&data)
+        .map_err(|e| format!("{e}"))?
+        .to_vec();
+    while checksum.last().copied() == Some(0) {
+        checksum.pop();
+    }
+    let hex_string = hex::encode(&checksum);
+
+    let (manifest_start, manifest_len) =
+        webc::WebC::get_manifest_offset_size(&data).map_err(|e| format!("{e}"))?;
+    let data_with_manifest =
+        get_webc_bytes(url, Some(0..manifest_start + manifest_len)).map_err(|e| format!("{e}"))?;
+    let manifest = webc::WebC::get_manifest(&data_with_manifest).map_err(|e| format!("{e}"))?;
+    Ok(RemoteWebcInfo {
+        checksum: hex_string,
+        manifest,
+    })
+}
+
+fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, String> {
+    let client = {
+        let builder = reqwest::blocking::Client::builder();
+
+        #[cfg(not(target_os = "wasi"))]
+        let builder = if let Some(proxy) =
+            crate::graphql::proxy::maybe_set_up_proxy().map_err(|e| format!("{e}"))?
+        {
+            builder.proxy(proxy)
+        } else {
+            builder
+        };
+
+        builder.build().map_err(|e| format!("{e}"))?
+    };
+
+    Ok(client.get(url.clone()).header(ACCEPT, "application/webc"))
+}
+
+fn get_webc_bytes(url: &Url, range: Option<Range<usize>>) -> Result<Vec<u8>, String> {
+    // curl -r 0-500 -L https://wapm.dev/syrusakbary/python -H "Accept: application/webc" --output python.webc
+
+    let mut res = setup_webc_client(url)?;
+
+    if let Some(range) = range.as_ref() {
+        res = res.header(RANGE, format!("bytes={}-{}", range.start, range.end));
+    }
+
+    let res = res.send().map_err(|e| format!("{e}"))?;
+    let bytes = res.bytes().map_err(|e| format!("{e}"))?;
+
+    Ok(bytes.to_vec())
+}
+
 // TODO: this test is segfaulting only on linux-musl, no other OS
 // See https://github.com/wasmerio/wasmer/pull/3215
 #[cfg(not(target_env = "musl"))]

From 75bedbaa7157da534d4598fe939cf92bbd551aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Thu, 10 Nov 2022 14:38:44 +0100
Subject: [PATCH 02/16] Add integration test, fix make lint

---
 lib/cli/Cargo.toml                 |  2 +-
 lib/cli/src/commands/run.rs        |  5 ++---
 lib/registry/src/lib.rs            | 11 ++++-------
 tests/integration/cli/tests/run.rs | 26 ++++++++++++++++++++++++++
 4 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml
index 4f18cf35864..3d878069ba3 100644
--- a/lib/cli/Cargo.toml
+++ b/lib/cli/Cargo.toml
@@ -56,7 +56,7 @@ log = { version = "0.4", optional = true }
 tempfile = "3"
 tempdir = "0.3.7"
 http_req  = { version="^0.8", default-features = false, features = ["rust-tls"], optional = true }
-reqwest = { version = "^0.11", default-features = false, feature = ["rustls-tls", "json"], optional = true }
+reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json"], optional = true }
 serde = { version = "1.0.147", features = ["derive"], optional = true }
 dirs = { version = "4.0", optional = true }
 serde_json = { version = "1.0", optional = true }
diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs
index 96dd2f5efe9..f074a5cb5c0 100644
--- a/lib/cli/src/commands/run.rs
+++ b/lib/cli/src/commands/run.rs
@@ -11,7 +11,6 @@ use std::collections::HashMap;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::str::FromStr;
-use std::sync::Arc;
 use url::Url;
 use wasmer::FunctionEnv;
 use wasmer::*;
@@ -916,7 +915,7 @@ pub(crate) fn try_run_package_or_file(
 }
 
 fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(), anyhow::Error> {
-    let checksum = wasmer_registry::get_remote_webc_checksum(&url)
+    let checksum = wasmer_registry::get_remote_webc_checksum(url)
         .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
 
     if !wasmer_registry::get_all_installed_webc_packages()
@@ -925,7 +924,7 @@ fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(),
     {
         let sp = start_spinner(format!("Installing {}", url));
 
-        wasmer_registry::install_webc_package(&url, &checksum)
+        wasmer_registry::install_webc_package(url, &checksum)
             .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
 
         if let Some(sp) = sp {
diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index 20bd75f97dc..d9dff102e59 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -3,7 +3,6 @@ use std::env;
 use std::fmt;
 use std::io::Write;
 use std::path::{Path, PathBuf};
-use std::ptr::read;
 use std::time::Duration;
 
 use reqwest::header::ACCEPT;
@@ -1207,7 +1206,7 @@ pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), String> {
 async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), String> {
     use futures_util::StreamExt;
 
-    let path = get_webc_dir().ok_or_else(|| format!("no webc dir"))?;
+    let path = get_webc_dir().ok_or_else(|| "no webc dir".to_string())?;
 
     let _ = std::fs::create_dir_all(&path);
 
@@ -1242,7 +1241,7 @@ async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), Str
 
     while let Some(item) = stream.next().await {
         let item = item.map_err(|e| format!("{e}"))?;
-        file.write_all(&item);
+        file.write_all(&item).map_err(|e| format!("{e}"))?;
     }
 
     Ok(())
@@ -1268,7 +1267,6 @@ pub fn get_all_installed_webc_packages() -> Vec<RemoteWebcInfo> {
                 &webc::ParseOptions {
                     parse_atoms: false,
                     parse_volumes: false,
-                    parse_manifest: true,
                     ..Default::default()
                 },
             )
@@ -1319,9 +1317,8 @@ pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, String> {
 
     let (manifest_start, manifest_len) =
         webc::WebC::get_manifest_offset_size(&data).map_err(|e| format!("{e}"))?;
-    let data_with_manifest =
-        get_webc_bytes(url, Some(0..manifest_start + manifest_len)).map_err(|e| format!("{e}"))?;
-    let manifest = webc::WebC::get_manifest(&data_with_manifest).map_err(|e| format!("{e}"))?;
+    let data_with_manifest = get_webc_bytes(url, Some(0..manifest_start + manifest_len))?;
+    let manifest = webc::WebC::get_manifest(&data_with_manifest).map_err(|e| e.to_string())?;
     Ok(RemoteWebcInfo {
         checksum: hex_string,
         manifest,
diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs
index dafb0d3f960..c5105e62476 100644
--- a/tests/integration/cli/tests/run.rs
+++ b/tests/integration/cli/tests/run.rs
@@ -232,6 +232,32 @@ fn test_wasmer_run_pirita_works() -> anyhow::Result<()> {
     Ok(())
 }
 
+#[cfg(feature = "webc_runner")]
+#[test]
+fn test_wasmer_run_pirita_url_works() -> anyhow::Result<()> {
+    let output = Command::new(get_wasmer_path())
+        .arg("run")
+        .arg("https://wapm.dev/syrusakbary/python")
+        .arg("--")
+        .arg("-c")
+        .arg("print(\"hello\")")
+        .output()?;
+
+    let stdout = std::str::from_utf8(&output.stdout)
+        .expect("stdout is not utf8! need to handle arbitrary bytes");
+
+    if stdout != "hello\n" {
+        bail!(
+            "1 running python.wasmer failed with: stdout: {}\n\nstderr: {}",
+            stdout,
+            std::str::from_utf8(&output.stderr)
+                .expect("stderr is not utf8! need to handle arbitrary bytes")
+        );
+    }
+
+    Ok(())
+}
+
 #[test]
 fn test_wasmer_run_works_with_dir() -> anyhow::Result<()> {
     let temp_dir = tempfile::TempDir::new()?;

From bcf259550a00a138b62858f01f14b0e3a308bad5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 11 Nov 2022 12:51:59 +0100
Subject: [PATCH 03/16] Convert from String -> anyhow::Error

---
 lib/cli/src/commands/run.rs |  2 +-
 lib/registry/src/lib.rs     | 36 +++++++++++++++++++++++++-----------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs
index f074a5cb5c0..6c901a3b43a 100644
--- a/lib/cli/src/commands/run.rs
+++ b/lib/cli/src/commands/run.rs
@@ -933,7 +933,7 @@ fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(),
     }
 
     let webc_install_path = wasmer_registry::get_webc_dir()
-        .ok_or_else(|| anyhow::anyhow!("Error installing package: webc download failed"))?
+        .context("Error installing package: no webc dir")?
         .join(checksum);
 
     let mut r = r.clone();
diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index d9dff102e59..a191d72e15e 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -5,6 +5,7 @@ use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
+use anyhow::Context;
 use reqwest::header::ACCEPT;
 use reqwest::header::RANGE;
 use serde::Deserialize;
@@ -1195,7 +1196,7 @@ pub struct RemoteWebcInfo {
     pub manifest: webc::Manifest,
 }
 
-pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), String> {
+pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), anyhow::Error> {
     tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .build()
@@ -1203,31 +1204,36 @@ pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), String> {
         .block_on(async { install_webc_package_inner(url, checksum).await })
 }
 
-async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), String> {
+async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), anyhow::Error> {
     use futures_util::StreamExt;
 
-    let path = get_webc_dir().ok_or_else(|| "no webc dir".to_string())?;
+    let path = get_webc_dir().ok_or_else(|| anyhow::anyhow!("no webc dir"))?;
 
     let _ = std::fs::create_dir_all(&path);
 
     let webc_path = path.join(checksum);
 
-    let mut file =
-        std::fs::File::create(&webc_path).map_err(|e| format!("{}: {e}", webc_path.display()))?;
+    let mut file = std::fs::File::create(&webc_path)
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context(anyhow::anyhow!("{}", webc_path.display()))?;
 
     let client = {
         let builder = reqwest::Client::builder();
 
         #[cfg(not(target_os = "wasi"))]
-        let builder = if let Some(proxy) =
-            crate::graphql::proxy::maybe_set_up_proxy().map_err(|e| format!("{e}"))?
+        let builder = if let Some(proxy) = crate::graphql::proxy::maybe_set_up_proxy()
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .context("install_webc_package: failed to setup proxy for reqwest Client")?
         {
             builder.proxy(proxy)
         } else {
             builder
         };
 
-        builder.build().map_err(|e| format!("{e}"))?
+        builder
+            .build()
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .context("install_webc_package: failed to build reqwest Client")?
     };
 
     let res = client
@@ -1235,13 +1241,21 @@ async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), Str
         .header(ACCEPT, "application/webc")
         .send()
         .await
-        .map_err(|e| format!("{e}"))?;
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context(anyhow::anyhow!("install_webc_package: failed to GET {url}"))?;
 
     let mut stream = res.bytes_stream();
 
     while let Some(item) = stream.next().await {
-        let item = item.map_err(|e| format!("{e}"))?;
-        file.write_all(&item).map_err(|e| format!("{e}"))?;
+        let item = item
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .context(anyhow::anyhow!("install_webc_package: failed to GET {url}"))?;
+        file.write_all(&item)
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .context(anyhow::anyhow!(
+                "install_webc_package: failed to write chunk to {}",
+                webc_path.display()
+            ))?;
     }
 
     Ok(())

From e9f4b6c0fa14dbf8faaceead8aa9225026d05e7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 11 Nov 2022 13:52:05 +0100
Subject: [PATCH 04/16] Refactor builder / proxy setup into separate function

---
 lib/registry/src/lib.rs | 113 +++++++++++++++++++++++-----------------
 1 file changed, 64 insertions(+), 49 deletions(-)

diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index a191d72e15e..05d8db43e9d 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -42,6 +42,37 @@ pub mod graphql {
             ConnectionError(String),
         }
 
+        pub fn maybe_set_up_proxy_blocking(
+            builder: reqwest::blocking::ClientBuilder,
+        ) -> anyhow::Result<reqwest::blocking::ClientBuilder> {
+            #[cfg(not(target_os = "wasi"))]
+            use anyhow::Context;
+            #[cfg(not(target_os = "wasi"))]
+            if let Some(proxy) = maybe_set_up_proxy_inner()
+                .map_err(|e| anyhow::anyhow!("{e}"))
+                .context("install_webc_package: failed to setup proxy for reqwest Client")?
+            {
+                return Ok(builder.proxy(proxy));
+            }
+            Ok(builder)
+        }
+
+        #[must_use]
+        pub fn maybe_set_up_proxy(
+            builder: reqwest::ClientBuilder,
+        ) -> anyhow::Result<reqwest::ClientBuilder> {
+            #[cfg(not(target_os = "wasi"))]
+            use anyhow::Context;
+            #[cfg(not(target_os = "wasi"))]
+            if let Some(proxy) = maybe_set_up_proxy_inner()
+                .map_err(|e| anyhow::anyhow!("{e}"))
+                .context("install_webc_package: failed to setup proxy for reqwest Client")?
+            {
+                return Ok(builder.proxy(proxy));
+            }
+            Ok(builder)
+        }
+
         /// Tries to set up a proxy
         ///
         /// This function reads from wapm config's `proxy.url` first, then checks
@@ -54,7 +85,7 @@ pub mod graphql {
         /// A return value of `Ok(None)` means that there was no attempt to set up a proxy,
         /// `Ok(Some(proxy))` means that the proxy was set up successfully, and `Err(e)` that
         /// there was a failure while attempting to set up the proxy.
-        pub fn maybe_set_up_proxy() -> anyhow::Result<Option<reqwest::Proxy>> {
+        fn maybe_set_up_proxy_inner() -> anyhow::Result<Option<reqwest::Proxy>> {
             use std::env;
             let proxy = if let Ok(proxy_url) =
                 env::var("ALL_PROXY").or_else(|_| env::var("all_proxy"))
@@ -142,13 +173,7 @@ pub mod graphql {
     {
         let client = {
             let builder = Client::builder();
-
-            #[cfg(not(target_os = "wasi"))]
-            let builder = if let Some(proxy) = proxy::maybe_set_up_proxy()? {
-                builder.proxy(proxy)
-            } else {
-                builder
-            };
+            let builder = proxy::maybe_set_up_proxy_blocking(builder)?;
             builder.build()?
         };
 
@@ -201,13 +226,7 @@ pub mod graphql {
     {
         let client = {
             let builder = Client::builder();
-
-            #[cfg(not(target_os = "wasi"))]
-            let builder = if let Some(proxy) = proxy::maybe_set_up_proxy()? {
-                builder.proxy(proxy)
-            } else {
-                builder
-            };
+            let builder = proxy::maybe_set_up_proxy_blocking(builder)?;
             builder.build()?
         };
 
@@ -1219,17 +1238,7 @@ async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), any
 
     let client = {
         let builder = reqwest::Client::builder();
-
-        #[cfg(not(target_os = "wasi"))]
-        let builder = if let Some(proxy) = crate::graphql::proxy::maybe_set_up_proxy()
-            .map_err(|e| anyhow::anyhow!("{e}"))
-            .context("install_webc_package: failed to setup proxy for reqwest Client")?
-        {
-            builder.proxy(proxy)
-        } else {
-            builder
-        };
-
+        let builder = crate::graphql::proxy::maybe_set_up_proxy(builder)?;
         builder
             .build()
             .map_err(|e| anyhow::anyhow!("{e}"))
@@ -1302,11 +1311,12 @@ pub fn get_all_installed_webc_packages() -> Vec<RemoteWebcInfo> {
 
 /// Returns the checksum of the .webc file, so that we can check whether the
 /// file is already installed before downloading it
-pub fn get_remote_webc_checksum(url: &Url) -> Result<String, String> {
+pub fn get_remote_webc_checksum(url: &Url) -> Result<String, anyhow::Error> {
     let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
-    let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
+    let data = get_webc_bytes(url, Some(0..request_max_bytes)).context("get_webc_bytes failed")?;
     let mut checksum = webc::WebC::get_checksum_bytes(&data)
-        .map_err(|e| format!("{e}"))?
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context("get_checksum_bytes failed")?
         .to_vec();
     while checksum.last().copied() == Some(0) {
         checksum.pop();
@@ -1317,48 +1327,47 @@ pub fn get_remote_webc_checksum(url: &Url) -> Result<String, String> {
 
 /// Before fetching the entire file from a remote URL, just fetch the manifest
 /// so we can see if the package has already been installed
-pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, String> {
+pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Error> {
     // Request up unti manifest size / manifest len
     let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
     let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
     let mut checksum = webc::WebC::get_checksum_bytes(&data)
-        .map_err(|e| format!("{e}"))?
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context("WebC::get_checksum_bytes failed")?
         .to_vec();
     while checksum.last().copied() == Some(0) {
         checksum.pop();
     }
     let hex_string = hex::encode(&checksum);
 
-    let (manifest_start, manifest_len) =
-        webc::WebC::get_manifest_offset_size(&data).map_err(|e| format!("{e}"))?;
+    let (manifest_start, manifest_len) = webc::WebC::get_manifest_offset_size(&data)
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context("WebC::get_manifest_offset_size failed")?;
     let data_with_manifest = get_webc_bytes(url, Some(0..manifest_start + manifest_len))?;
-    let manifest = webc::WebC::get_manifest(&data_with_manifest).map_err(|e| e.to_string())?;
+    let manifest = webc::WebC::get_manifest(&data_with_manifest)
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context("WebC::get_manifest failed")?;
     Ok(RemoteWebcInfo {
         checksum: hex_string,
         manifest,
     })
 }
 
-fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, String> {
+fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, anyhow::Error> {
     let client = {
         let builder = reqwest::blocking::Client::builder();
-
-        #[cfg(not(target_os = "wasi"))]
-        let builder = if let Some(proxy) =
-            crate::graphql::proxy::maybe_set_up_proxy().map_err(|e| format!("{e}"))?
-        {
-            builder.proxy(proxy)
-        } else {
-            builder
-        };
-
-        builder.build().map_err(|e| format!("{e}"))?
+        let builder = crate::graphql::proxy::maybe_set_up_proxy_blocking(builder)
+            .context("setup_webc_client")?;
+        builder
+            .build()
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .context("setup_webc_client: builder.build() failed")?
     };
 
     Ok(client.get(url.clone()).header(ACCEPT, "application/webc"))
 }
 
-fn get_webc_bytes(url: &Url, range: Option<Range<usize>>) -> Result<Vec<u8>, String> {
+fn get_webc_bytes(url: &Url, range: Option<Range<usize>>) -> Result<Vec<u8>, anyhow::Error> {
     // curl -r 0-500 -L https://wapm.dev/syrusakbary/python -H "Accept: application/webc" --output python.webc
 
     let mut res = setup_webc_client(url)?;
@@ -1367,8 +1376,14 @@ fn get_webc_bytes(url: &Url, range: Option<Range<usize>>) -> Result<Vec<u8>, Str
         res = res.header(RANGE, format!("bytes={}-{}", range.start, range.end));
     }
 
-    let res = res.send().map_err(|e| format!("{e}"))?;
-    let bytes = res.bytes().map_err(|e| format!("{e}"))?;
+    let res = res
+        .send()
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context("send() failed")?;
+    let bytes = res
+        .bytes()
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context("bytes() failed")?;
 
     Ok(bytes.to_vec())
 }

From 43c7c4e91b199a6a03bd4623838bb3914de17d83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 11 Nov 2022 14:02:38 +0100
Subject: [PATCH 05/16] Return error on 404

---
 lib/registry/src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index 05d8db43e9d..2ad05259850 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -1250,6 +1250,7 @@ async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), any
         .header(ACCEPT, "application/webc")
         .send()
         .await
+        .and_then(|response| response.error_for_status())
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context(anyhow::anyhow!("install_webc_package: failed to GET {url}"))?;
 

From b705db438c4963ff5527f03cba8eaa4a270d9b3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 11 Nov 2022 14:09:02 +0100
Subject: [PATCH 06/16] Fix "make lint"

---
 lib/registry/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index 2ad05259850..09199005a0b 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -57,7 +57,6 @@ pub mod graphql {
             Ok(builder)
         }
 
-        #[must_use]
         pub fn maybe_set_up_proxy(
             builder: reqwest::ClientBuilder,
         ) -> anyhow::Result<reqwest::ClientBuilder> {

From 108a5598ec5fc0c7c0eacfbaa8bee180d9f10afa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 11 Nov 2022 15:46:27 +0100
Subject: [PATCH 07/16] Remove duplicated code + add comments for checksum
 function

---
 lib/registry/src/lib.rs | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index 09199005a0b..e82355673ec 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -1296,11 +1296,8 @@ pub fn get_all_installed_webc_packages() -> Vec<RemoteWebcInfo> {
             .ok()
         })
         .filter_map(|webc| {
-            let mut checksum = webc.checksum.as_ref().map(|s| &s.data)?.to_vec();
-            while checksum.last().copied() == Some(0) {
-                checksum.pop();
-            }
-            let hex_string = hex::encode(&checksum);
+            let checksum = webc.checksum.as_ref().map(|s| &s.data)?.to_vec();
+            let hex_string = get_checksum_hash(&checksum);
             Some(RemoteWebcInfo {
                 checksum: hex_string,
                 manifest: webc.manifest.clone(),
@@ -1309,6 +1306,22 @@ pub fn get_all_installed_webc_packages() -> Vec<RemoteWebcInfo> {
         .collect()
 }
 
+/// The checksum of the webc file has a bunch of zeros at the end
+/// (it's currently encoded that way in the webc format). This function
+/// strips the zeros because otherwise the filename would become too long.
+///
+/// So:
+///
+/// `3ea47cb0000000000000` -> `3ea47cb`
+///
+pub fn get_checksum_hash(bytes: &[u8]) -> String {
+    let mut checksum = bytes.to_vec();
+    while checksum.last().copied() == Some(0) {
+        checksum.pop();
+    }
+    hex::encode(&checksum)
+}
+
 /// Returns the checksum of the .webc file, so that we can check whether the
 /// file is already installed before downloading it
 pub fn get_remote_webc_checksum(url: &Url) -> Result<String, anyhow::Error> {
@@ -1318,11 +1331,7 @@ pub fn get_remote_webc_checksum(url: &Url) -> Result<String, anyhow::Error> {
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("get_checksum_bytes failed")?
         .to_vec();
-    while checksum.last().copied() == Some(0) {
-        checksum.pop();
-    }
-    let hex_string = hex::encode(&checksum);
-    Ok(hex_string)
+    Ok(get_checksum_hash(&checksum))
 }
 
 /// Before fetching the entire file from a remote URL, just fetch the manifest
@@ -1335,10 +1344,7 @@ pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Err
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("WebC::get_checksum_bytes failed")?
         .to_vec();
-    while checksum.last().copied() == Some(0) {
-        checksum.pop();
-    }
-    let hex_string = hex::encode(&checksum);
+    let hex_string = get_checksum_hash(&checksum);
 
     let (manifest_start, manifest_len) = webc::WebC::get_manifest_offset_size(&data)
         .map_err(|e| anyhow::anyhow!("{e}"))

From 7d89d6de214f61143902f637b59f9921b242d71f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 11 Nov 2022 15:50:25 +0100
Subject: [PATCH 08/16] Fix "make lint"

---
 lib/registry/src/lib.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index e82355673ec..3846392bd0f 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -1327,7 +1327,7 @@ pub fn get_checksum_hash(bytes: &[u8]) -> String {
 pub fn get_remote_webc_checksum(url: &Url) -> Result<String, anyhow::Error> {
     let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
     let data = get_webc_bytes(url, Some(0..request_max_bytes)).context("get_webc_bytes failed")?;
-    let mut checksum = webc::WebC::get_checksum_bytes(&data)
+    let checksum = webc::WebC::get_checksum_bytes(&data)
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("get_checksum_bytes failed")?
         .to_vec();
@@ -1340,7 +1340,7 @@ pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Err
     // Request up unti manifest size / manifest len
     let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
     let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
-    let mut checksum = webc::WebC::get_checksum_bytes(&data)
+    let checksum = webc::WebC::get_checksum_bytes(&data)
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("WebC::get_checksum_bytes failed")?
         .to_vec();

From 7a59e66060f3bde259b18aa49ac7662633ffaebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 18 Nov 2022 10:13:52 +0100
Subject: [PATCH 09/16] Fix merge issues

---
 lib/registry/src/graphql.rs | 2 +-
 lib/registry/src/lib.rs     | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/registry/src/graphql.rs b/lib/registry/src/graphql.rs
index 26cf05c5313..6f71281cd77 100644
--- a/lib/registry/src/graphql.rs
+++ b/lib/registry/src/graphql.rs
@@ -9,7 +9,7 @@ use std::time::Duration;
 #[cfg(target_os = "wasi")]
 use {wasm_bus_reqwest::prelude::header::*, wasm_bus_reqwest::prelude::*};
 
-mod proxy {
+pub(crate) mod proxy {
     //! Code for dealing with setting things up to proxy network requests
     use thiserror::Error;
 
diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index ace1508a93b..07534996882 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -1,5 +1,9 @@
 use std::collections::BTreeMap;
 use std::fmt;
+use anyhow::Context;
+use url::Url;
+use reqwest::header::{RANGE, ACCEPT};
+use core::ops::Range;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
@@ -1048,7 +1052,7 @@ pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Err
     })
 }
 
-fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, anyhow::Error> {
+fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, anyhow::Error> {    
     let client = {
         let builder = reqwest::blocking::Client::builder();
         let builder = crate::graphql::proxy::maybe_set_up_proxy_blocking(builder)

From 2fa8bd75fec576df1ce260cfa347373902139eef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 18 Nov 2022 12:04:27 +0100
Subject: [PATCH 10/16] Fix merge issues for proxy_blocking

---
 lib/registry/src/graphql.rs | 41 +++++++++++++++++++++++++++++--------
 lib/registry/src/lib.rs     | 12 +++++------
 2 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/lib/registry/src/graphql.rs b/lib/registry/src/graphql.rs
index eabd23c855c..6a14eecc1de 100644
--- a/lib/registry/src/graphql.rs
+++ b/lib/registry/src/graphql.rs
@@ -25,6 +25,37 @@ pub(crate) mod proxy {
         ConnectionError(String),
     }
 
+    pub fn maybe_set_up_proxy_blocking(
+        builder: reqwest::blocking::ClientBuilder,
+    ) -> anyhow::Result<reqwest::blocking::ClientBuilder> {
+        #[cfg(not(target_os = "wasi"))]
+        use anyhow::Context;
+        #[cfg(not(target_os = "wasi"))]
+        if let Some(proxy) = maybe_set_up_proxy_inner()
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .context("install_webc_package: failed to setup proxy for reqwest Client")?
+        {
+            return Ok(builder.proxy(proxy));
+        }
+        Ok(builder)
+    }
+
+    #[must_use]
+    pub fn maybe_set_up_proxy(
+        builder: reqwest::ClientBuilder,
+    ) -> anyhow::Result<reqwest::ClientBuilder> {
+        #[cfg(not(target_os = "wasi"))]
+        use anyhow::Context;
+        #[cfg(not(target_os = "wasi"))]
+        if let Some(proxy) = maybe_set_up_proxy_inner()
+            .map_err(|e| anyhow::anyhow!("{e}"))
+            .context("install_webc_package: failed to setup proxy for reqwest Client")?
+        {
+            return Ok(builder.proxy(proxy));
+        }
+        Ok(builder)
+    }
+
     /// Tries to set up a proxy
     ///
     /// This function reads from wapm config's `proxy.url` first, then checks
@@ -37,7 +68,7 @@ pub(crate) mod proxy {
     /// A return value of `Ok(None)` means that there was no attempt to set up a proxy,
     /// `Ok(Some(proxy))` means that the proxy was set up successfully, and `Err(e)` that
     /// there was a failure while attempting to set up the proxy.
-    pub fn maybe_set_up_proxy() -> anyhow::Result<Option<reqwest::Proxy>> {
+    fn maybe_set_up_proxy_inner() -> anyhow::Result<Option<reqwest::Proxy>> {
         use std::env;
         let proxy = if let Ok(proxy_url) = env::var("ALL_PROXY").or_else(|_| env::var("all_proxy"))
         {
@@ -120,13 +151,7 @@ pub fn whoami_distro() -> String {
 
 fn setup_client() -> Result<Client, anyhow::Error> {
     let builder = Client::builder();
-
-    let builder = if let Some(proxy) = proxy::maybe_set_up_proxy()? {
-        builder.proxy(proxy)
-    } else {
-        builder
-    };
-
+    let builder = proxy::maybe_set_up_proxy_blocking(builder)?;
     builder.build().map_err(|e| e.into())
 }
 
diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index 7b68ba68445..905a0d2c1c0 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -8,11 +8,11 @@
 //! curl -sSfL https://registry.wapm.io/graphql/schema.graphql > lib/registry/graphql/schema.graphql
 //! ```
 
-use std::fmt;
+use crate::config::Registries;
 use anyhow::Context;
-use url::Url;
-use reqwest::header::{RANGE, ACCEPT};
 use core::ops::Range;
+use reqwest::header::{ACCEPT, RANGE};
+use std::fmt;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
@@ -20,6 +20,7 @@ use std::{
     collections::BTreeMap,
     fmt::{Display, Formatter},
 };
+use url::Url;
 
 pub mod config;
 pub mod graphql;
@@ -31,9 +32,6 @@ pub use crate::{
     graphql::get_bindings_query::ProgrammingLanguage,
 };
 
-use crate::config::Registries;
-use anyhow::Context;
-
 pub static GLOBAL_CONFIG_FILE_NAME: &str = if cfg!(target_os = "wasi") {
     "/.private/wapm.toml"
 } else {
@@ -1069,7 +1067,7 @@ pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Err
     })
 }
 
-fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, anyhow::Error> {    
+fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, anyhow::Error> {
     let client = {
         let builder = reqwest::blocking::Client::builder();
         let builder = crate::graphql::proxy::maybe_set_up_proxy_blocking(builder)

From 1c4a039ff27a2f8dc59f57cc9f0404455220cf06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 18 Nov 2022 12:20:09 +0100
Subject: [PATCH 11/16] Fix make lint

---
 lib/registry/src/graphql.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/registry/src/graphql.rs b/lib/registry/src/graphql.rs
index 6a14eecc1de..c473f22ddd8 100644
--- a/lib/registry/src/graphql.rs
+++ b/lib/registry/src/graphql.rs
@@ -40,7 +40,6 @@ pub(crate) mod proxy {
         Ok(builder)
     }
 
-    #[must_use]
     pub fn maybe_set_up_proxy(
         builder: reqwest::ClientBuilder,
     ) -> anyhow::Result<reqwest::ClientBuilder> {

From 57b3263adac4859271f44000fc0d9a09525affdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 18 Nov 2022 14:43:46 +0100
Subject: [PATCH 12/16] Try fixing unit tests

---
 lib/cli/src/cli.rs          |  6 +++++-
 lib/cli/src/commands/run.rs | 32 +++++++++++++++++++++++++-------
 lib/registry/src/lib.rs     | 34 +++++++++++++++++++++++++++++++---
 3 files changed, 61 insertions(+), 11 deletions(-)

diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs
index 34c2c1835d2..586fc630edd 100644
--- a/lib/cli/src/cli.rs
+++ b/lib/cli/src/cli.rs
@@ -252,7 +252,11 @@ fn wasmer_main_inner() -> Result<(), anyhow::Error> {
         let debug = false;
         #[cfg(feature = "debug")]
         let debug = r.options.debug;
-        return crate::commands::try_run_package_or_file(&args, r, debug);
+        #[cfg(test)]
+        let result = crate::commands::try_run_package_or_file("wasmer_main_inner", &args, r, debug);
+        #[cfg(not(test))]
+        let result = crate::commands::try_run_package_or_file(&args, r, debug);
+        return result;
     }
 
     options.execute()
diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs
index 44b2b6451df..ec1c67346e4 100644
--- a/lib/cli/src/commands/run.rs
+++ b/lib/cli/src/commands/run.rs
@@ -822,6 +822,7 @@ fn test_fixup_args() {
 }
 
 pub(crate) fn try_run_package_or_file(
+    #[cfg(test)] test_name: &str,
     args: &[String],
     r: &Run,
     debug: bool,
@@ -829,7 +830,11 @@ pub(crate) fn try_run_package_or_file(
     let debug_msgs_allowed = isatty::stdout_isatty();
 
     if let Ok(url) = url::Url::parse(&format!("{}", r.path.display())) {
-        return try_run_url(&url, args, r, debug);
+        #[cfg(test)]
+        let result = try_run_url(test_name, &url, args, r, debug);
+        #[cfg(not(test))]
+        let result = try_run_url(&url, args, r, debug);
+        return result;
     }
 
     // Check "r.path" is a file or a package / command name
@@ -914,14 +919,22 @@ pub(crate) fn try_run_package_or_file(
     try_autoinstall_package(args, &sv, package_download_info, r.force_install)
 }
 
-fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(), anyhow::Error> {
+fn try_run_url(
+    #[cfg(test)] test_name: &str,
+    url: &Url,
+    _args: &[String],
+    r: &Run,
+    _debug: bool,
+) -> Result<(), anyhow::Error> {
     let checksum = wasmer_registry::get_remote_webc_checksum(url)
         .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
 
-    if !wasmer_registry::get_all_installed_webc_packages()
-        .iter()
-        .any(|p| p.checksum == checksum)
-    {
+    #[cfg(test)]
+    let packages = wasmer_registry::get_all_installed_webc_packages(test_name);
+    #[cfg(not(test))]
+    let packages = wasmer_registry::get_all_installed_webc_packages();
+
+    if !packages.iter().any(|p| p.checksum == checksum) {
         let sp = start_spinner(format!("Installing {}", url));
 
         wasmer_registry::install_webc_package(url, &checksum)
@@ -932,7 +945,12 @@ fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(),
         }
     }
 
-    let webc_install_path = wasmer_registry::get_webc_dir()
+    #[cfg(not(test))]
+    let webc_dir = wasmer_registry::get_webc_dir();
+    #[cfg(test)]
+    let webc_dir = wasmer_registry::get_webc_dir(test_name);
+
+    let webc_install_path = webc_dir
         .context("Error installing package: no webc dir")?
         .join(checksum);
 
diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index 905a0d2c1c0..3ce896067ef 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -634,8 +634,12 @@ pub fn get_checkouts_dir(#[cfg(test)] test_name: &str) -> Option<PathBuf> {
     Some(root_dir.join("checkouts"))
 }
 
-pub fn get_webc_dir() -> Option<PathBuf> {
-    Some(get_wasmer_root_dir()?.join("webc"))
+pub fn get_webc_dir(#[cfg(test)] test_name: &str) -> Option<PathBuf> {
+    #[cfg(test)]
+    let root_dir = get_wasmer_root_dir(test_name)?;
+    #[cfg(not(test))]
+    let root_dir = get_wasmer_root_dir()?;
+    Some(root_dir.join("webc"))
 }
 
 /// Returs the path to the directory where all packages on this computer are being stored
@@ -930,9 +934,16 @@ pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), anyhow::Err
         .block_on(async { install_webc_package_inner(url, checksum).await })
 }
 
-async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), anyhow::Error> {
+async fn install_webc_package_inner(
+    #[cfg(test)] test_name: &str,
+    url: &Url,
+    checksum: &str,
+) -> Result<(), anyhow::Error> {
     use futures_util::StreamExt;
 
+    #[cfg(test)]
+    let path = get_webc_dir(test_name).ok_or_else(|| anyhow::anyhow!("no webc dir"))?;
+    #[cfg(not(test))]
     let path = get_webc_dir().ok_or_else(|| anyhow::anyhow!("no webc dir"))?;
 
     let _ = std::fs::create_dir_all(&path);
@@ -979,7 +990,24 @@ async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), any
 }
 
 /// Returns a list of all installed webc packages
+#[cfg(test)]
+pub fn get_all_installed_webc_packages(test_name: &str) -> Vec<RemoteWebcInfo> {
+    get_all_installed_webc_packages_inner(test_name)
+}
+
+#[cfg(not(test))]
 pub fn get_all_installed_webc_packages() -> Vec<RemoteWebcInfo> {
+    get_all_installed_webc_packages_inner("")
+}
+
+fn get_all_installed_webc_packages_inner(_test_name: &str) -> Vec<RemoteWebcInfo> {
+    #[cfg(test)]
+    let dir = match get_webc_dir(_test_name) {
+        Some(s) => s,
+        None => return Vec::new(),
+    };
+
+    #[cfg(not(test))]
     let dir = match get_webc_dir() {
         Some(s) => s,
         None => return Vec::new(),

From 45674a620e933410c5882f0b65501873691b91c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 18 Nov 2022 16:40:29 +0100
Subject: [PATCH 13/16] Fix install_webc_package compile error

---
 lib/cli/src/commands/run.rs |  8 ++++++--
 lib/registry/src/lib.rs     | 19 +++++++++++++++++--
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs
index ec1c67346e4..4f094e35796 100644
--- a/lib/cli/src/commands/run.rs
+++ b/lib/cli/src/commands/run.rs
@@ -937,8 +937,12 @@ fn try_run_url(
     if !packages.iter().any(|p| p.checksum == checksum) {
         let sp = start_spinner(format!("Installing {}", url));
 
-        wasmer_registry::install_webc_package(url, &checksum)
-            .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
+        #[cfg(test)]
+        let result = wasmer_registry::install_webc_package(test_name, url, &checksum);
+        #[cfg(not(test))]
+        let result = wasmer_registry::install_webc_package(url, &checksum);
+
+        let _ = result.map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
 
         if let Some(sp) = sp {
             sp.close();
diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs
index 3ce896067ef..4fefdc8db58 100644
--- a/lib/registry/src/lib.rs
+++ b/lib/registry/src/lib.rs
@@ -926,12 +926,27 @@ pub struct RemoteWebcInfo {
     pub manifest: webc::Manifest,
 }
 
-pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), anyhow::Error> {
+pub fn install_webc_package(
+    #[cfg(test)] test_name: &str,
+    url: &Url,
+    checksum: &str,
+) -> Result<(), anyhow::Error> {
     tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .build()
         .unwrap()
-        .block_on(async { install_webc_package_inner(url, checksum).await })
+        .block_on(async {
+            {
+                #[cfg(test)]
+                {
+                    install_webc_package_inner(test_name, url, checksum).await
+                }
+                #[cfg(not(test))]
+                {
+                    install_webc_package_inner(url, checksum).await
+                }
+            }
+        })
 }
 
 async fn install_webc_package_inner(

From 8883b428addf0558fb486c147f6a6eca1594c7c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Fri, 18 Nov 2022 17:05:24 +0100
Subject: [PATCH 14/16] Fix make lint

---
 lib/cli/src/commands/run.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs
index 4f094e35796..41b2823bb96 100644
--- a/lib/cli/src/commands/run.rs
+++ b/lib/cli/src/commands/run.rs
@@ -942,7 +942,7 @@ fn try_run_url(
         #[cfg(not(test))]
         let result = wasmer_registry::install_webc_package(url, &checksum);
 
-        let _ = result.map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
+        result.map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
 
         if let Some(sp) = sp {
             sp.close();

From 0d7f10ffd1771d4d79f0ec274fd8e4df2d64bda3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Sun, 20 Nov 2022 11:17:14 +0100
Subject: [PATCH 15/16] Fix #[cfg(test]] code

---
 lib/cli/src/commands/run.rs | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs
index 41b2823bb96..4947f009362 100644
--- a/lib/cli/src/commands/run.rs
+++ b/lib/cli/src/commands/run.rs
@@ -929,17 +929,11 @@ fn try_run_url(
     let checksum = wasmer_registry::get_remote_webc_checksum(url)
         .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
 
-    #[cfg(test)]
-    let packages = wasmer_registry::get_all_installed_webc_packages(test_name);
-    #[cfg(not(test))]
     let packages = wasmer_registry::get_all_installed_webc_packages();
 
     if !packages.iter().any(|p| p.checksum == checksum) {
         let sp = start_spinner(format!("Installing {}", url));
 
-        #[cfg(test)]
-        let result = wasmer_registry::install_webc_package(test_name, url, &checksum);
-        #[cfg(not(test))]
         let result = wasmer_registry::install_webc_package(url, &checksum);
 
         result.map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
@@ -949,10 +943,7 @@ fn try_run_url(
         }
     }
 
-    #[cfg(not(test))]
     let webc_dir = wasmer_registry::get_webc_dir();
-    #[cfg(test)]
-    let webc_dir = wasmer_registry::get_webc_dir(test_name);
 
     let webc_install_path = webc_dir
         .context("Error installing package: no webc dir")?

From 2f48a3c35ee38ecba0e883f37e1ab3bac7ddefb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <felix@wasmer.io>
Date: Sun, 20 Nov 2022 12:51:08 +0100
Subject: [PATCH 16/16] Fix unit tests

---
 lib/cli/src/cli.rs            |  6 +-----
 lib/cli/src/commands/login.rs |  1 +
 lib/cli/src/commands/run.rs   | 12 +-----------
 3 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs
index 586fc630edd..34c2c1835d2 100644
--- a/lib/cli/src/cli.rs
+++ b/lib/cli/src/cli.rs
@@ -252,11 +252,7 @@ fn wasmer_main_inner() -> Result<(), anyhow::Error> {
         let debug = false;
         #[cfg(feature = "debug")]
         let debug = r.options.debug;
-        #[cfg(test)]
-        let result = crate::commands::try_run_package_or_file("wasmer_main_inner", &args, r, debug);
-        #[cfg(not(test))]
-        let result = crate::commands::try_run_package_or_file(&args, r, debug);
-        return result;
+        return crate::commands::try_run_package_or_file(&args, r, debug);
     }
 
     options.execute()
diff --git a/lib/cli/src/commands/login.rs b/lib/cli/src/commands/login.rs
index 8e70b40ebfa..539f132b147 100644
--- a/lib/cli/src/commands/login.rs
+++ b/lib/cli/src/commands/login.rs
@@ -1,4 +1,5 @@
 use clap::Parser;
+#[cfg(not(test))]
 use dialoguer::Input;
 
 /// Subcommand for listing packages
diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs
index 4947f009362..52c597287ea 100644
--- a/lib/cli/src/commands/run.rs
+++ b/lib/cli/src/commands/run.rs
@@ -822,7 +822,6 @@ fn test_fixup_args() {
 }
 
 pub(crate) fn try_run_package_or_file(
-    #[cfg(test)] test_name: &str,
     args: &[String],
     r: &Run,
     debug: bool,
@@ -830,9 +829,6 @@ pub(crate) fn try_run_package_or_file(
     let debug_msgs_allowed = isatty::stdout_isatty();
 
     if let Ok(url) = url::Url::parse(&format!("{}", r.path.display())) {
-        #[cfg(test)]
-        let result = try_run_url(test_name, &url, args, r, debug);
-        #[cfg(not(test))]
         let result = try_run_url(&url, args, r, debug);
         return result;
     }
@@ -919,13 +915,7 @@ pub(crate) fn try_run_package_or_file(
     try_autoinstall_package(args, &sv, package_download_info, r.force_install)
 }
 
-fn try_run_url(
-    #[cfg(test)] test_name: &str,
-    url: &Url,
-    _args: &[String],
-    r: &Run,
-    _debug: bool,
-) -> Result<(), anyhow::Error> {
+fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(), anyhow::Error> {
     let checksum = wasmer_registry::get_remote_webc_checksum(url)
         .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;