Implement wasmer run {url}

lib/cli/Cargo.toml

@@ -56,7 +56,7 @@ log = { version = "0.4", optional = true }
 tempfile = "3"
 tempdir = "0.3.7"
 http_req  = { version="^0.8", default-features = false, features = ["rust-tls"], optional = true }
-reqwest = { version = "^0.11", default-features = false, feature = ["rustls-tls", "json"], optional = true }
+reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json"], optional = true }
 serde = { version = "1.0.147", features = ["derive"], optional = true }
 dirs = { version = "4.0", optional = true }
 serde_json = { version = "1.0", optional = true }

lib/cli/src/commands/run.rs

@@ -11,6 +11,7 @@ use std::collections::HashMap;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::str::FromStr;
+use url::Url;
 use wasmer::FunctionEnv;
 use wasmer::*;
 #[cfg(feature = "cache")]
@@ -827,6 +828,10 @@ pub(crate) fn try_run_package_or_file(
 ) -> Result<(), anyhow::Error> {
     let debug_msgs_allowed = isatty::stdout_isatty();
 
+    if let Ok(url) = url::Url::parse(&format!("{}", r.path.display())) {
+        return try_run_url(&url, args, r, debug);
+    }
+
     // Check "r.path" is a file or a package / command name
     if r.path.exists() {
         if r.path.is_dir() && r.path.join("wapm.toml").exists() {
@@ -908,3 +913,30 @@ pub(crate) fn try_run_package_or_file(
     // else: local package not found - try to download and install package
     try_autoinstall_package(args, &sv, package_download_info, r.force_install)
 }
+
+fn try_run_url(url: &Url, _args: &[String], r: &Run, _debug: bool) -> Result<(), anyhow::Error> {
+    let checksum = wasmer_registry::get_remote_webc_checksum(url)
+        .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
+
+    if !wasmer_registry::get_all_installed_webc_packages()
+        .iter()
+        .any(|p| p.checksum == checksum)
+    {
+        let sp = start_spinner(format!("Installing {}", url));
+
+        wasmer_registry::install_webc_package(url, &checksum)
+            .map_err(|e| anyhow::anyhow!("error fetching {url}: {e}"))?;
+
+        if let Some(sp) = sp {
+            sp.close();
+        }
+    }
+
+    let webc_install_path = wasmer_registry::get_webc_dir()
+        .ok_or_else(|| anyhow::anyhow!("Error installing package: webc download failed"))?
+        .join(checksum);
+
+    let mut r = r.clone();
+    r.path = webc_install_path;
+    r.execute()
+}

lib/registry/Cargo.toml

@@ -10,7 +10,8 @@ dirs = "4.0.0"
 graphql_client = "0.11.0"
 serde = { version = "1.0.145", features = ["derive"] }
 anyhow = "1.0.65" 
-reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json"] }
+reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json", "stream"] }
+futures-util = "0.3.25"
 whoami = "1.2.3" 
 serde_json = "1.0.85"
 url = "2.3.1"
@@ -20,4 +21,7 @@ wapm-toml = "0.2.0"
 tar = "0.4.38"
 flate2 = "1.0.24"
 semver = "1.0.14"
-lzma-rs = "0.2.0"
+lzma-rs = "0.2.0"
+webc = { version ="3.0.1", features = ["mmap"] }
+hex = "0.4.3"
+tokio = "1.21.2"

lib/registry/src/lib.rs

@@ -1,11 +1,16 @@
 use std::collections::BTreeMap;
 use std::env;
 use std::fmt;
+use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
+use reqwest::header::ACCEPT;
+use reqwest::header::RANGE;
 use serde::Deserialize;
 use serde::Serialize;
+use std::ops::Range;
+use url::Url;
 
 pub mod graphql {
 
@@ -20,7 +25,7 @@ pub mod graphql {
     #[cfg(target_os = "wasi")]
     use {wasm_bus_reqwest::prelude::header::*, wasm_bus_reqwest::prelude::*};
 
-    mod proxy {
+    pub mod proxy {
         //! Code for dealing with setting things up to proxy network requests
         use thiserror::Error;
 
@@ -940,6 +945,10 @@ pub fn get_checkouts_dir() -> Option<PathBuf> {
     Some(get_wasmer_root_dir()?.join("checkouts"))
 }
 
+pub fn get_webc_dir() -> Option<PathBuf> {
+    Some(get_wasmer_root_dir()?.join("webc"))
+}
+
 /// Returs the path to the directory where all packages on this computer are being stored
 pub fn get_global_install_dir(registry_host: &str) -> Option<PathBuf> {
     Some(get_checkouts_dir()?.join(registry_host))
@@ -1180,6 +1189,176 @@ pub fn get_all_available_registries() -> Result<Vec<String>, String> {
     Ok(registries)
 }
 
+#[derive(Debug, PartialEq, Clone)]
+pub struct RemoteWebcInfo {
+    pub checksum: String,
+    pub manifest: webc::Manifest,
+}
+
+pub fn install_webc_package(url: &Url, checksum: &str) -> Result<(), String> {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+        .block_on(async { install_webc_package_inner(url, checksum).await })
+}
+
+async fn install_webc_package_inner(url: &Url, checksum: &str) -> Result<(), String> {
+    use futures_util::StreamExt;
+
+    let path = get_webc_dir().ok_or_else(|| "no webc dir".to_string())?;
+
+    let _ = std::fs::create_dir_all(&path);
+
+    let webc_path = path.join(checksum);
+
+    let mut file =
+        std::fs::File::create(&webc_path).map_err(|e| format!("{}: {e}", webc_path.display()))?;
+
+    let client = {
+        let builder = reqwest::Client::builder();
+
+        #[cfg(not(target_os = "wasi"))]
+        let builder = if let Some(proxy) =
+            crate::graphql::proxy::maybe_set_up_proxy().map_err(|e| format!("{e}"))?
+        {
+            builder.proxy(proxy)
+        } else {
+            builder
+        };
+
+        builder.build().map_err(|e| format!("{e}"))?
+    };
+
+    let res = client
+        .get(url.clone())
+        .header(ACCEPT, "application/webc")
+        .send()
+        .await
+        .map_err(|e| format!("{e}"))?;
+
+    let mut stream = res.bytes_stream();
+
+    while let Some(item) = stream.next().await {
+        let item = item.map_err(|e| format!("{e}"))?;
+        file.write_all(&item).map_err(|e| format!("{e}"))?;
+    }
+
+    Ok(())
+}
+
+/// Returns a list of all installed webc packages
+pub fn get_all_installed_webc_packages() -> Vec<RemoteWebcInfo> {
+    let dir = match get_webc_dir() {
+        Some(s) => s,
+        None => return Vec::new(),
+    };
+
+    let read_dir = match std::fs::read_dir(dir) {
+        Ok(s) => s,
+        Err(_) => return Vec::new(),
+    };
+
+    read_dir
+        .filter_map(|r| Some(r.ok()?.path()))
+        .filter_map(|path| {
+            webc::WebCMmap::parse(
+                path,
+                &webc::ParseOptions {
+                    parse_atoms: false,
+                    parse_volumes: false,
+                    ..Default::default()
+                },
+            )
+            .ok()
+        })
+        .filter_map(|webc| {
+            let mut checksum = webc.checksum.as_ref().map(|s| &s.data)?.to_vec();
+            while checksum.last().copied() == Some(0) {
+                checksum.pop();
+            }
+            let hex_string = hex::encode(&checksum);
+            Some(RemoteWebcInfo {
+                checksum: hex_string,
+                manifest: webc.manifest.clone(),
+            })
+        })
+        .collect()
+}
+
+/// Returns the checksum of the .webc file, so that we can check whether the
+/// file is already installed before downloading it
+pub fn get_remote_webc_checksum(url: &Url) -> Result<String, String> {
+    let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
+    let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
+    let mut checksum = webc::WebC::get_checksum_bytes(&data)
+        .map_err(|e| format!("{e}"))?
+        .to_vec();
+    while checksum.last().copied() == Some(0) {
+        checksum.pop();
+    }
+    let hex_string = hex::encode(&checksum);
+    Ok(hex_string)
+}
+
+/// Before fetching the entire file from a remote URL, just fetch the manifest
+/// so we can see if the package has already been installed
+pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, String> {
+    // Request up unti manifest size / manifest len
+    let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
+    let data = get_webc_bytes(url, Some(0..request_max_bytes))?;
+    let mut checksum = webc::WebC::get_checksum_bytes(&data)
+        .map_err(|e| format!("{e}"))?
+        .to_vec();
+    while checksum.last().copied() == Some(0) {
+        checksum.pop();
+    }
+    let hex_string = hex::encode(&checksum);
+
+    let (manifest_start, manifest_len) =
+        webc::WebC::get_manifest_offset_size(&data).map_err(|e| format!("{e}"))?;
+    let data_with_manifest = get_webc_bytes(url, Some(0..manifest_start + manifest_len))?;
+    let manifest = webc::WebC::get_manifest(&data_with_manifest).map_err(|e| e.to_string())?;
+    Ok(RemoteWebcInfo {
+        checksum: hex_string,
+        manifest,
+    })
+}
+
+fn setup_webc_client(url: &Url) -> Result<reqwest::blocking::RequestBuilder, String> {
+    let client = {
+        let builder = reqwest::blocking::Client::builder();
+
+        #[cfg(not(target_os = "wasi"))]
+        let builder = if let Some(proxy) =
+            crate::graphql::proxy::maybe_set_up_proxy().map_err(|e| format!("{e}"))?
+        {
+            builder.proxy(proxy)
+        } else {
+            builder
+        };
+
+        builder.build().map_err(|e| format!("{e}"))?
+    };
+
+    Ok(client.get(url.clone()).header(ACCEPT, "application/webc"))
+}
+
+fn get_webc_bytes(url: &Url, range: Option<Range<usize>>) -> Result<Vec<u8>, String> {
+    // curl -r 0-500 -L https://wapm.dev/syrusakbary/python -H "Accept: application/webc" --output python.webc
+
+    let mut res = setup_webc_client(url)?;
+
+    if let Some(range) = range.as_ref() {
+        res = res.header(RANGE, format!("bytes={}-{}", range.start, range.end));
+    }
+
+    let res = res.send().map_err(|e| format!("{e}"))?;
+    let bytes = res.bytes().map_err(|e| format!("{e}"))?;
+
+    Ok(bytes.to_vec())
+}
+
 // TODO: this test is segfaulting only on linux-musl, no other OS
 // See https://github.com/wasmerio/wasmer/pull/3215
 #[cfg(not(target_env = "musl"))]

tests/integration/cli/tests/run.rs

@@ -232,6 +232,32 @@ fn test_wasmer_run_pirita_works() -> anyhow::Result<()> {
     Ok(())
 }
 
+#[cfg(feature = "webc_runner")]
+#[test]
+fn test_wasmer_run_pirita_url_works() -> anyhow::Result<()> {
+    let output = Command::new(get_wasmer_path())
+        .arg("run")
+        .arg("https://wapm.dev/syrusakbary/python")
+        .arg("--")
+        .arg("-c")
+        .arg("print(\"hello\")")
+        .output()?;
+
+    let stdout = std::str::from_utf8(&output.stdout)
+        .expect("stdout is not utf8! need to handle arbitrary bytes");
+
+    if stdout != "hello\n" {
+        bail!(
+            "1 running python.wasmer failed with: stdout: {}\n\nstderr: {}",
+            stdout,
+            std::str::from_utf8(&output.stderr)
+                .expect("stderr is not utf8! need to handle arbitrary bytes")
+        );
+    }
+
+    Ok(())
+}
+
 #[test]
 fn test_wasmer_run_works_with_dir() -> anyhow::Result<()> {
     let temp_dir = tempfile::TempDir::new()?;