diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index 9808452ce91..73f8df84cea 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -97,6 +97,9 @@ jobs: SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.SCCACHE_AZURE_CONNECTION_STRING }} steps: - uses: actions/checkout@v3 + - uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.10.0 - name: Set up libstdc++ on Linux if: matrix.build == 'linux-x64' run: | @@ -211,6 +214,7 @@ jobs: TARGET_DIR: target/${{ matrix.target }}/release CARGO_TARGET: --target ${{ matrix.target }} WAPM_DEV_TOKEN: ${{ secrets.WAPM_DEV_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} #- name: Test integration CLI # if: matrix.run_test && matrix.os == 'windows-2019' diff --git a/.gitignore b/.gitignore index 065f814be49..24affcea4df 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ api-docs-repo/ # Generated by tests on Android /avd /core +out.txt diff --git a/Cargo.lock b/Cargo.lock index 3f7b818fc9b..9ff2c79520d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3993,6 +3993,7 @@ dependencies = [ "serde", "serde_json", "spinner", + "tar", "target-lexicon 0.12.4", "tempdir", "tempfile", @@ -4230,6 +4231,7 @@ dependencies = [ "serde", "serde_json", "tar", + "tempdir", "thiserror", "toml", "url", diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index a89d14d37f2..b9d43b3aaeb 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -71,6 +71,7 @@ libc = { version = "^0.2", default-features = false } nuke-dir = { version = "0.1.0", optional = true } webc = { version = "3.0.1", optional = true } isatty = "0.1.9" +tar = "0.4" dialoguer = "0.10.2" [build-dependencies] diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 49a8e652b7c..bc69b5c0be2 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -4,10 +4,6 @@ use super::ObjectFormat; use crate::store::CompilerOptions; use anyhow::{Context, Result}; use clap::Parser; -#[cfg(feature = "http")] -use serde::{Deserialize, Serialize}; -#[cfg(feature = "http")] -use std::collections::BTreeMap; use std::env; use std::fmt::Write as _; use std::fs; @@ -372,7 +368,7 @@ impl CreateExe { )); } - let target = if let Some(target_triple) = target_triple.clone() { + let target = if let Some(target_triple) = target_triple { target_triple } else { return Err(anyhow!( @@ -407,13 +403,7 @@ impl CreateExe { v.canonicalize().unwrap_or(v) } else { { - let libwasmer_path = if target_triple.unwrap_or(Triple::host()).operating_system - == wasmer_types::OperatingSystem::Windows - { - "lib/wasmer.lib" - } else { - "lib/libwasmer.a" - }; + let libwasmer_path = "lib/libwasmer.a"; let tarball_dir; let filename = if let Some(local_tarball) = cross_subc.tarball.as_ref() { let target_file_path = local_tarball @@ -495,6 +485,7 @@ impl CreateExe { run_c_compile(&c_src_path, &c_src_obj, target_triple.clone()) .context("Failed to compile C source code")?; + LinkCode { object_paths: vec![c_src_obj, wasm_object_path], output_path, @@ -557,6 +548,8 @@ impl CreateExe { header_code_path = std::env::current_dir()?; } + println!("Output result to: {}", output_path.display()); + /* Compile main function */ let compilation = { let mut include_dir = libwasmer_path.clone(); @@ -564,28 +557,42 @@ impl CreateExe { include_dir.push("include"); let mut cmd = Command::new(zig_binary_path); - let mut cmd_mut: &mut Command = cmd - .arg("cc") - .arg("-target") - .arg(&zig_triple) - .arg(&format!("-L{}", libwasmer_path.display())) - .arg(&format!("-l:{}", lib_filename)) - .arg(&format!("-I{}", include_dir.display())) - .arg(&format!("-I{}", header_code_path.display())); - if !zig_triple.contains("windows") { - cmd_mut = cmd_mut.arg("-lunwind"); + cmd.arg("build-exe"); + cmd.arg("-target"); + cmd.arg(&zig_triple); + cmd.arg(&format!("-I{}/", include_dir.display())); + cmd.arg(&format!("-I{}/", header_code_path.display())); + if zig_triple.contains("windows") { + cmd.arg("-lc++"); + } else { + cmd.arg("-lc"); + } + cmd.arg("-lunwind"); + cmd.arg("-OReleaseSafe"); + cmd.arg("-fstrip"); + cmd.arg("-dead_strip"); + cmd.arg("-dead_strip_dylibs"); + cmd.arg("-fno-compiler-rt"); + cmd.arg(&format!("-femit-bin={}", output_path.display())); + + cmd.arg(&object_path); + cmd.arg(&c_src_path); + cmd.arg(libwasmer_path.join(&lib_filename)); + if zig_triple.contains("windows") { + let mut libwasmer_parent = libwasmer_path.clone(); + libwasmer_parent.pop(); + cmd.arg(libwasmer_parent.join("winsdk/ADVAPI32.lib")); + cmd.arg(libwasmer_parent.join("winsdk/BCRYPT.lib")); + cmd.arg(libwasmer_parent.join("winsdk/KERNEL32.lib")); + cmd.arg(libwasmer_parent.join("winsdk/USERENV.lib")); + cmd.arg(libwasmer_parent.join("winsdk/WS2_32.lib")); } - cmd_mut = cmd_mut.arg(&object_path).arg(&c_src_path); - if let Some(volume_obj) = pirita_volume_path.as_ref() { - cmd_mut = cmd_mut.arg(volume_obj.clone()); + cmd.arg(volume_obj.clone()); } - cmd_mut - .arg("-o") - .arg(&output_path) - .output() - .context("Could not execute `zig`")? + println!("{:?}", cmd); + cmd.output().context("Could not execute `zig`")? }; if !compilation.status.success() { return Err(anyhow::anyhow!(String::from_utf8_lossy( @@ -813,22 +820,22 @@ impl CreateExe { let mut include_dir = libwasmer_path.clone(); include_dir.pop(); include_dir.push("include"); - println!("include dir: {}", include_dir.display()); let mut cmd = Command::new(zig_binary_path); let mut cmd_mut: &mut Command = cmd .arg("cc") .arg("--verbose") .arg("-target") .arg(&zig_triple) - .arg(&format!("-L{}", libwasmer_path.display())) - .arg(&format!("-l:{}", lib_filename)) - .arg(&format!("-I{}", include_dir.display())); + .arg(&format!("-I{}", include_dir.display())) + .arg("-lc"); if !zig_triple.contains("windows") { cmd_mut = cmd_mut.arg("-lunwind"); } + cmd_mut .args(link_objects.into_iter()) .arg(&c_src_path) + .arg(libwasmer_path.join(lib_filename)) .arg("-o") .arg(&output_path) .output() @@ -1043,8 +1050,6 @@ impl CreateExe { .canonicalize() .context("Failed to find libwasmer")?; - println!("Using libwasmer file: {}", libwasmer_path.display()); - let lib_filename = libwasmer_path .file_name() .unwrap() @@ -1357,19 +1362,24 @@ impl LinkCode { #[cfg(feature = "http")] mod http_fetch { use anyhow::{anyhow, Context, Result}; - use http_req::{ - request::Request, - response::{Response, StatusCode}, - uri::Uri, - }; + use http_req::{request::Request, response::StatusCode, uri::Uri}; use std::convert::TryFrom; + use target_lexicon::OperatingSystem; pub fn get_latest_release() -> Result { let mut writer = Vec::new(); let uri = Uri::try_from("https://api.github.com/repos/wasmerio/wasmer/releases").unwrap(); - let response = Request::new(&uri) - .header("User-Agent", "wasmer") + // Increases rate-limiting in GitHub CI + let auth = std::env::var("GITHUB_TOKEN"); + let mut response = Request::new(&uri); + + if let Ok(token) = auth { + response.header("Authorization", &format!("Bearer {token}")); + } + + let response = response + .header("User-Agent", "wasmerio") .header("Accept", "application/vnd.github.v3+json") .timeout(Some(std::time::Duration::new(30, 0))) .send(&mut writer) @@ -1377,10 +1387,11 @@ mod http_fetch { .context("Could not lookup wasmer repository on Github.")?; if response.status_code() != StatusCode::new(200) { - return Err(anyhow!( - "Github API replied with non-200 status code: {}", - response.status_code() - )); + eprintln!( + "Warning: Github API replied with non-200 status code: {}. Response: {}", + response.status_code(), + String::from_utf8_lossy(&writer), + ); } let v: std::result::Result = serde_json::from_reader(&*writer); @@ -1443,31 +1454,64 @@ mod http_fetch { } }; + // Test if file has been already downloaded if let Ok(mut cache_path) = super::get_libwasmer_cache_path() { - match std::fs::read_dir(&cache_path).and_then(|r| { + let paths = std::fs::read_dir(&cache_path).and_then(|r| { r.map(|res| res.map(|e| e.path())) .collect::, std::io::Error>>() - }) { - Ok(mut entries) => { + }); + + if let Ok(mut entries) = paths { + entries.retain(|p| p.to_str().map(|p| p.ends_with(".tar.gz")).unwrap_or(false)); + + // create-exe on Windows is special: we use the windows-gnu64.tar.gz (GNU ABI) + // to link, not the windows-amd64.tar.gz (MSVC ABI) + if target_triple.operating_system == OperatingSystem::Windows { + entries.retain(|p| { + p.to_str() + .map(|p| p.contains("windows") && p.contains("gnu64")) + .unwrap_or(false) + }); + } else { entries.retain(|p| p.to_str().map(|p| check_arch(p)).unwrap_or(true)); entries.retain(|p| p.to_str().map(|p| check_vendor(p)).unwrap_or(true)); entries.retain(|p| p.to_str().map(|p| check_os(p)).unwrap_or(true)); entries.retain(|p| p.to_str().map(|p| check_env(p)).unwrap_or(true)); - if entries.len() == 1 { - cache_path.push(&entries[0]); - if cache_path.exists() { - eprintln!( - "Using cached tarball to cache path `{}`.", - cache_path.display() - ); - return Ok(cache_path); - } + } + + if !entries.is_empty() { + cache_path.push(&entries[0]); + if cache_path.exists() { + eprintln!( + "Using cached tarball to cache path `{}`.", + cache_path.display() + ); + return Ok(cache_path); } } - Err(_ioerr) => {} } } - if let Some(assets) = release["assets"].as_array_mut() { + + let assets = match release["assets"].as_array_mut() { + Some(s) => s, + None => { + return Err(anyhow!( + "GitHub API: no [assets] array in JSON response for latest releases" + )); + } + }; + + // create-exe on Windows is special: we use the windows-gnu64.tar.gz (GNU ABI) + // to link, not the windows-amd64.tar.gz (MSVC ABI) + if target_triple.operating_system == OperatingSystem::Windows { + assets.retain(|a| { + if let Some(name) = a["name"].as_str() { + name.contains("windows") && name.contains("gnu64") + } else { + false + } + }); + } else { assets.retain(|a| { if let Some(name) = a["name"].as_str() { check_arch(name) @@ -1482,6 +1526,7 @@ mod http_fetch { false } }); + assets.retain(|a| { if let Some(name) = a["name"].as_str() { check_os(name) @@ -1489,6 +1534,7 @@ mod http_fetch { false } }); + assets.retain(|a| { if let Some(name) = a["name"].as_str() { check_env(name) @@ -1496,128 +1542,107 @@ mod http_fetch { false } }); + } - if assets.len() == 1 { - let browser_download_url = - if let Some(url) = assets[0]["browser_download_url"].as_str() { - url.to_string() - } else { - return Err(anyhow!( - "Could not get download url from Github API response." - )); - }; - let filename = browser_download_url - .split('/') - .last() - .unwrap_or("output") - .to_string(); - let mut file = std::fs::File::create(&filename)?; - println!("Downloading {} to {}", browser_download_url, &filename); - let download_thread: std::thread::JoinHandle> = - std::thread::spawn(move || { - let uri = Uri::try_from(browser_download_url.as_str())?; - let mut response = Request::new(&uri) - .header("User-Agent", "wasmer") - .send(&mut file) - .map_err(anyhow::Error::new) - .context("Could not lookup wasmer artifact on Github.")?; - if response.status_code() == StatusCode::new(302) { - let redirect_uri = - Uri::try_from(response.headers().get("Location").unwrap().as_str()) - .unwrap(); - response = Request::new(&redirect_uri) - .header("User-Agent", "wasmer") - .send(&mut file) - .map_err(anyhow::Error::new) - .context("Could not lookup wasmer artifact on Github.")?; - } - Ok(response) - }); - match super::get_libwasmer_cache_path() { - Ok(mut cache_path) => { - cache_path.push(&filename); - if !cache_path.exists() { - if let Err(err) = std::fs::copy(&filename, &cache_path) { - eprintln!( - "Could not store tarball to cache path `{}`: {}", - cache_path.display(), - err - ); - } else { - eprintln!( - "Cached tarball to cache path `{}`.", - cache_path.display() - ); - } - } - } - Err(err) => { - eprintln!( - "Could not determine cache path for downloaded binaries.: {}", - err - ); - } - } - let _response = download_thread - .join() - .expect("Could not join downloading thread"); - match super::get_libwasmer_cache_path() { - Ok(mut cache_path) => { - cache_path.push(&filename); - if !cache_path.exists() { - if let Err(err) = std::fs::copy(&filename, &cache_path) { - eprintln!( - "Could not store tarball to cache path `{}`: {}", - cache_path.display(), - err - ); - } else { - eprintln!( - "Cached tarball to cache path `{}`.", - cache_path.display() - ); - } - } - } - Err(err) => { - eprintln!( - "Could not determine cache path for downloaded binaries.: {}", - err - ); - } + if assets.len() != 1 { + return Err(anyhow!( + "GitHub API: more that one release selected for target {target_triple}: {assets:?}" + )); + } + + let browser_download_url = if let Some(url) = assets[0]["browser_download_url"].as_str() { + url.to_string() + } else { + return Err(anyhow!( + "Could not get download url from Github API response." + )); + }; + + let filename = browser_download_url + .split('/') + .last() + .unwrap_or("output") + .to_string(); + + let download_tempdir = tempdir::TempDir::new("wasmer-download")?; + let download_path = download_tempdir.path().to_path_buf().join(&filename); + + let mut file = std::fs::File::create(&download_path)?; + eprintln!( + "Downloading {} to {}", + browser_download_url, + download_path.display() + ); + + let uri = Uri::try_from(browser_download_url.as_str())?; + let mut response = Request::new(&uri) + .header("User-Agent", "wasmer") + .send(&mut file) + .map_err(anyhow::Error::new) + .context("Could not lookup wasmer artifact on Github.")?; + if response.status_code() == StatusCode::new(302) { + let redirect_uri = + Uri::try_from(response.headers().get("Location").unwrap().as_str()).unwrap(); + response = Request::new(&redirect_uri) + .header("User-Agent", "wasmer") + .send(&mut file) + .map_err(anyhow::Error::new) + .context("Could not lookup wasmer artifact on Github.")?; + } + + let _ = response; + + match super::get_libwasmer_cache_path() { + Ok(mut cache_path) => { + cache_path.push(&filename); + if let Err(err) = std::fs::copy(&download_path, &cache_path) { + eprintln!( + "Could not store tarball to cache path `{}`: {}", + cache_path.display(), + err + ); + Err(anyhow!( + "Could not copy from {} to {}", + download_path.display(), + cache_path.display() + )) + } else { + eprintln!("Cached tarball to cache path `{}`.", cache_path.display()); + Ok(cache_path) } - return Ok(filename.into()); + } + Err(err) => { + eprintln!( + "Could not determine cache path for downloaded binaries.: {}", + err + ); + Err(anyhow!("Could not determine libwasmer cache path")) } } - Err(anyhow!("Could not get release artifact.")) } } fn untar(tarball: std::path::PathBuf, target: std::path::PathBuf) -> Result> { - let files = std::process::Command::new("tar") - .arg("-tf") - .arg(&tarball) - .output() - .expect("failed to execute process") - .stdout; + use walkdir::WalkDir; - let files_s = String::from_utf8(files)?; + wasmer_registry::try_unpack_targz(&tarball, &target, false)?; - let files = files_s - .lines() - .filter(|p| !p.ends_with('/')) - .map(|s| s.to_string()) - .collect::>(); + Ok(WalkDir::new(&target) + .into_iter() + .filter_map(|e| e.ok()) + .map(|entry| format!("{}", entry.path().display())) + .collect()) +} - let _ = std::fs::create_dir_all(&target); - let _output = std::process::Command::new("tar") - .arg("-xf") - .arg(&tarball) - .arg("-C") - .arg(&target) - .output() - .expect("failed to execute process"); - Ok(files) +fn get_zig_exe_str() -> &'static str { + #[cfg(target_os = "windows")] + { + "zig.exe" + } + #[cfg(not(target_os = "windows"))] + { + "zig" + } } fn find_zig_binary(path: Option) -> Result { @@ -1650,24 +1675,13 @@ fn find_zig_binary(path: Option) -> Result { OsStr::new("") }, )) { - p.push("zig"); + p.push(get_zig_exe_str()); if p.exists() { retval = Some(p); break; } } - retval - .or_else(|| { - #[cfg(feature = "http")] - { - try_autoinstall_zig().map(|p| p.join("zig")) - } - #[cfg(not(feature = "http"))] - { - None - } - }) - .ok_or_else(|| anyhow!("Could not find `zig` binary in PATH."))? + retval.ok_or_else(|| anyhow!("Could not find `zig` binary in PATH."))? }; let version = std::process::Command::new(&retval) @@ -1695,126 +1709,3 @@ fn find_zig_binary(path: Option) -> Result { Ok(retval) } } - -/// Tries to auto-install zig into ~/.wasmer/utils/zig/{version} -#[cfg(feature = "http")] -fn try_autoinstall_zig() -> Option { - let zig_dir = wasmer_registry::get_wasmer_root_dir()? - .join("utils") - .join("zig"); - let mut existing_version = None; - - if !zig_dir.exists() { - return install_zig(&zig_dir); - } - - if let Ok(mut rd) = std::fs::read_dir(&zig_dir) { - existing_version = rd.next().and_then(|entry| { - let string = entry.ok()?.file_name().to_str()?.to_string(); - if zig_dir.join(&string).join("zig").exists() { - Some(string) - } else { - None - } - }) - } - - if let Some(exist) = existing_version { - return Some(zig_dir.join(exist)); - } - - install_zig(&zig_dir) -} - -#[cfg(feature = "http")] -fn install_zig(target_targz_path: &Path) -> Option { - let resp = reqwest::blocking::get("https://ziglang.org/download/index.json"); - let resp = resp.ok()?; - let resp = resp.json::(); - let resp = resp.ok()?; - - let default_key = "master".to_string(); - let (latest_version, latest_version_json) = resp - .versions - .get(&default_key) - .map(|v| (&default_key, v)) - .or_else(|| resp.versions.iter().next())?; - - let latest_version = match latest_version_json.version.as_ref() { - Some(s) => s, - None => latest_version, - }; - - let install_dir = target_targz_path.join(latest_version); - if install_dir.join("zig").exists() { - return Some(install_dir); - } - - let native_host_url = latest_version_json.get_native_host_url()?; - let _ = std::fs::create_dir_all(&install_dir); - wasmer_registry::download_and_unpack_targz(&native_host_url, &install_dir, true).ok() -} - -#[cfg(feature = "http")] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -struct ZiglangOrgJson { - #[serde(flatten)] - versions: BTreeMap, -} - -#[cfg(feature = "http")] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -struct ZiglangOrgJsonTarget { - version: Option, - date: String, - src: ZiglangOrgJsonBuildTarget, - #[serde(rename = "x86_64-freebsd")] - x86_64_freebsd: Option, - #[serde(rename = "x86_64-macos")] - x86_64_macos: Option, - #[serde(rename = "aarch64-macos")] - aarch64_macos: Option, - #[serde(rename = "x86_64-windows")] - x86_64_windows: Option, - #[serde(rename = "x86_64-linux")] - x86_64_linux: Option, - #[serde(rename = "aarch64-linux")] - aarch64_linux: Option, -} - -impl ZiglangOrgJsonTarget { - pub fn get_native_host_url(&self) -> Option { - let native_host = format!("{}", target_lexicon::HOST); - if native_host.starts_with("x86_64") { - if native_host.contains("freebsd") { - Some(self.x86_64_freebsd.as_ref()?.tarball.clone()) - } else if native_host.contains("darwin") || native_host.contains("macos") { - Some(self.x86_64_macos.as_ref()?.tarball.clone()) - } else if native_host.contains("windows") { - Some(self.x86_64_windows.as_ref()?.tarball.clone()) - } else if native_host.contains("linux") { - Some(self.x86_64_linux.as_ref()?.tarball.clone()) - } else { - None - } - } else if native_host.starts_with("aarch64") { - if native_host.contains("darwin") || native_host.contains("macos") { - Some(self.aarch64_macos.as_ref()?.tarball.clone()) - } else if native_host.contains("linux") { - Some(self.aarch64_linux.as_ref()?.tarball.clone()) - } else { - None - } - } else { - None - } - } -} - -#[cfg(feature = "http")] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -struct ZiglangOrgJsonBuildTarget { - tarball: String, - shasum: String, - size: String, -} diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index 359c8723813..97dc0e702eb 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -24,4 +24,5 @@ tar = "0.4.38" flate2 = "1.0.24" semver = "1.0.14" lzma-rs = "0.2.0" +tempdir = "0.3.7" log = "0.4.17" diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index baec1ba1c19..3e8349e3fb8 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -8,6 +8,8 @@ //! curl -sSfL https://registry.wapm.io/graphql/schema.graphql > lib/registry/graphql/schema.graphql //! ``` +use crate::config::Registries; +use anyhow::Context; use std::fmt; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -26,9 +28,6 @@ pub use crate::{ graphql::get_bindings_query::ProgrammingLanguage, }; -use crate::config::Registries; -use anyhow::Context; - pub static GLOBAL_CONFIG_FILE_NAME: &str = if cfg!(target_os = "wasi") { "/.private/wapm.toml" } else { @@ -643,41 +642,18 @@ pub fn get_global_install_dir( Some(root_dir.join(registry_host)) } -/// Whether the top-level directory should be stripped -pub fn download_and_unpack_targz( - url: &str, - target_path: &Path, +/// Convenience function that will unpack .tar.gz files and .tar.bz +/// files to a target directory (does NOT remove the original .tar.gz) +pub fn try_unpack_targz>( + target_targz_path: P, + target_path: P, strip_toplevel: bool, -) -> Result { - let target_targz_path = target_path.to_path_buf().join("package.tar.gz"); - - let mut resp = - reqwest::blocking::get(url).map_err(|e| format!("failed to download {url}: {e}"))?; - - if !target_targz_path.exists() { - // create all the parent paths, only remove the created directory, not the parent dirs - let _ = std::fs::create_dir_all(&target_targz_path); - let _ = std::fs::remove_dir(&target_targz_path); - } - - { - let mut file = std::fs::File::create(&target_targz_path).map_err(|e| { - format!( - "failed to download {url} into {}: {e}", - target_targz_path.display() - ) - })?; - - resp.copy_to(&mut file).map_err(|e| format!("{e}"))?; - } - +) -> Result { + let target_targz_path = target_targz_path.as_ref(); + let target_path = target_path.as_ref(); let open_file = || { - std::fs::File::open(&target_targz_path).map_err(|e| { - format!( - "failed to download {url} into {}: {e}", - target_targz_path.display() - ) - }) + std::fs::File::open(&target_targz_path) + .map_err(|e| anyhow::anyhow!("failed to open {}: {e}", target_targz_path.display())) }; let try_decode_gz = || { @@ -685,11 +661,13 @@ pub fn download_and_unpack_targz( let gz_decoded = flate2::read::GzDecoder::new(&file); let mut ar = tar::Archive::new(gz_decoded); if strip_toplevel { - unpack_sans_parent(ar, target_path) - .map_err(|e| format!("failed to unpack {}: {e}", target_targz_path.display())) + unpack_sans_parent(ar, target_path).map_err(|e| { + anyhow::anyhow!("failed to unpack {}: {e}", target_targz_path.display()) + }) } else { - ar.unpack(target_path) - .map_err(|e| format!("failed to unpack {}: {e}", target_targz_path.display())) + ar.unpack(target_path).map_err(|e| { + anyhow::anyhow!("failed to unpack {}: {e}", target_targz_path.display()) + }) } }; @@ -697,23 +675,61 @@ pub fn download_and_unpack_targz( let file = open_file()?; let mut decomp: Vec = Vec::new(); let mut bufread = std::io::BufReader::new(&file); - lzma_rs::xz_decompress(&mut bufread, &mut decomp) - .map_err(|e| format!("failed to unpack {}: {e}", target_targz_path.display()))?; + lzma_rs::xz_decompress(&mut bufread, &mut decomp).map_err(|e| { + anyhow::anyhow!("failed to unpack {}: {e}", target_targz_path.display()) + })?; let cursor = std::io::Cursor::new(decomp); let mut ar = tar::Archive::new(cursor); if strip_toplevel { - unpack_sans_parent(ar, target_path) - .map_err(|e| format!("failed to unpack {}: {e}", target_targz_path.display())) + unpack_sans_parent(ar, target_path).map_err(|e| { + anyhow::anyhow!("failed to unpack {}: {e}", target_targz_path.display()) + }) } else { - ar.unpack(target_path) - .map_err(|e| format!("failed to unpack {}: {e}", target_targz_path.display())) + ar.unpack(target_path).map_err(|e| { + anyhow::anyhow!("failed to unpack {}: {e}", target_targz_path.display()) + }) } }; try_decode_gz().or_else(|_| try_decode_xz())?; - let _ = std::fs::remove_file(target_targz_path); + Ok(target_targz_path.to_path_buf()) +} + +/// Whether the top-level directory should be stripped +pub fn download_and_unpack_targz( + url: &str, + target_path: &Path, + strip_toplevel: bool, +) -> Result { + let tempdir = tempdir::TempDir::new("wasmer-download-targz")?; + + let target_targz_path = tempdir.path().join("package.tar.gz"); + + let mut resp = reqwest::blocking::get(url) + .map_err(|e| anyhow::anyhow!("failed to download {url}: {e}"))?; + + if !target_targz_path.exists() { + // create all the parent paths, only remove the created directory, not the parent dirs + let _ = std::fs::create_dir_all(&target_targz_path); + let _ = std::fs::remove_dir(&target_targz_path); + } + + { + let mut file = std::fs::File::create(&target_targz_path).map_err(|e| { + anyhow::anyhow!( + "failed to download {url} into {}: {e}", + target_targz_path.display() + ) + })?; + + resp.copy_to(&mut file) + .map_err(|e| anyhow::anyhow!("{e}"))?; + } + + try_unpack_targz(target_targz_path.as_path(), target_path, strip_toplevel) + .context(anyhow::anyhow!("Could not download {url}"))?; Ok(target_path.to_path_buf()) } @@ -859,7 +875,7 @@ pub fn install_package( let name = package_info.package; if !dir.join("wapm.toml").exists() || force_install { - download_and_unpack_targz(&package_info.url, &dir, false)?; + download_and_unpack_targz(&package_info.url, &dir, false).map_err(|e| format!("{e}"))?; } Ok(( diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index dafb0d3f960..c43fe52b396 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -21,6 +21,66 @@ fn test_no_start_wat_path() -> String { format!("{}/{}", ASSET_PATH, "no_start.wat") } +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[test] +fn test_cross_compile_python_windows() -> anyhow::Result<()> { + let temp_dir = tempfile::TempDir::new()?; + + let targets = &[ + "aarch64-darwin", + "x86_64-darwin", + "x86_64-linux-gnu", + "aarch64-linux-gnu", + // TODO: this test depends on the latest release -gnu64.tar.gz + // to be present, but we can't release the next release before + // the integration tests are passing, so this test depends on itself + // + // We need to first release a version without -windows being tested + // then do a second PR to test that Windows works. + // "x86_64-windows-gnu", + ]; + + for t in targets { + let python_wasmer_path = temp_dir.path().join(format!("{t}-python")); + + let mut output = Command::new(get_wasmer_path()); + + output.arg("create-exe"); + output.arg(wasi_test_python_path()); + output.arg("--target"); + output.arg(t); + output.arg("-o"); + output.arg(python_wasmer_path.clone()); + let output = output.output()?; + + let stdout = std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"); + + let stderr = std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes"); + + if !output.status.success() { + bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}"); + } + + println!("stdout: {stdout}"); + println!("stderr: {stderr}"); + + if !python_wasmer_path.exists() { + let p = std::fs::read_dir(temp_dir.path()) + .unwrap() + .filter_map(|e| Some(e.ok()?.path())) + .collect::>(); + panic!( + "target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}", + p + ); + } + } + + Ok(()) +} + #[test] fn run_wasi_works() -> anyhow::Result<()> { let output = Command::new(get_wasmer_path())