From 71631d390ab4593c30cba1c563c6d976edcd444d Mon Sep 17 00:00:00 2001 From: Taylor Thomas Date: Fri, 6 Sep 2024 11:29:26 -0600 Subject: [PATCH 1/3] feat(wkg-core): Adds new `wkg-core` crate with lockfile support This includes support for an optional config and tests for locking Signed-off-by: Taylor Thomas --- .github/workflows/ci.yml | 9 +- Cargo.lock | 19 + Cargo.toml | 11 + ci/get-oras.sh | 15 - crates/wasm-pkg-client/Cargo.toml | 20 +- crates/wasm-pkg-client/src/lib.rs | 10 +- crates/wasm-pkg-common/Cargo.toml | 20 +- crates/wkg-core/Cargo.toml | 38 ++ crates/wkg-core/src/config.rs | 137 ++++++ crates/wkg-core/src/lib.rs | 6 + crates/wkg-core/src/lock.rs | 775 ++++++++++++++++++++++++++++++ crates/wkg/Cargo.toml | 8 +- 12 files changed, 1018 insertions(+), 50 deletions(-) delete mode 100755 ci/get-oras.sh create mode 100644 crates/wkg-core/Cargo.toml create mode 100644 crates/wkg-core/src/config.rs create mode 100644 crates/wkg-core/src/lib.rs create mode 100644 crates/wkg-core/src/lock.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3b2d3a..aa1d388 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,14 +5,13 @@ name: CI jobs: run-ci: name: Run CI - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - name: Install dependencies - run: | - ci/get-oras.sh - echo "/tmp/ci-bin" >> $GITHUB_PATH - name: Run cargo test run: cargo test --all --workspace - name: Run cargo clippy diff --git a/Cargo.lock b/Cargo.lock index feed924..bbfadf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4462,6 +4462,25 @@ dependencies = [ "wit-component 0.216.0", ] +[[package]] +name = "wkg-core" +version = "0.5.0" +dependencies = [ + "anyhow", + "futures-util", + "libc", + "semver", + "serde 1.0.209", + "sha2", + "tempfile", + "tokio", + "toml 0.8.19", + "tracing", + "wasm-pkg-client", + "wasm-pkg-common", + "windows-sys 0.52.0", +] + [[package]] name = "xdg-home" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 6412767..0512733 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,14 +10,25 @@ license = "Apache-2.0 WITH LLVM-exception" [workspace.dependencies] anyhow = "1" +base64 = "0.22.0" +bytes = "1.6.0" +dirs = "5.0.1" docker_credential = "1.2.1" +futures-util = "0.3.30" oci-client = { version = "0.12", default-features = false, features = [ "rustls-tls", ] } oci-wasm = { version = "0.0.5", default-features = false, features = [ "rustls-tls", ] } +semver = "1.0.23" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tempfile = "3.10.1" +thiserror = "1.0" tokio = "1.35.1" +toml = "0.8.13" tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", default-features = false, features = [ "fmt", diff --git a/ci/get-oras.sh b/ci/get-oras.sh deleted file mode 100755 index 0eaa5ab..0000000 --- a/ci/get-oras.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -URL="https://github.com/oras-project/oras/releases/download/v1.1.0/oras_1.1.0_linux_amd64.tar.gz" -SHA256="e09e85323b24ccc8209a1506f142e3d481e6e809018537c6b3db979c891e6ad7" - -mkdir -p /tmp/ci-bin -cd /tmp/ci-bin -curl -L https://github.com/oras-project/oras/releases/download/v1.1.0/oras_1.1.0_linux_amd64.tar.gz -o oras.tgz -echo "$SHA256 oras.tgz" > oras.sum -sha256sum --check oras.sum -tar xf oras.tgz - -echo "Got /tmp/ci-bin/oras" \ No newline at end of file diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index ac60a7b..95e70d7 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -11,22 +11,22 @@ readme = "../../README.md" [dependencies] anyhow = { workspace = true } async-trait = "0.1.77" -base64 = "0.22.0" -bytes = "1.5.0" -dirs = "5.0.1" +base64 = { workspace = true } +bytes = { workspace = true } +dirs = { workspace = true } docker_credential = { workspace = true } -futures-util = { version = "0.3.29", features = ["io"] } +futures-util = { workspace = true, features = ["io"] } oci-client = { workspace = true } oci-wasm = { workspace = true } secrecy = { version = "0.8.0", features = ["serde"] } -serde = { version = "1.0.194", features = ["derive"] } -serde_json = "1.0.110" -sha2 = "0.10.8" -thiserror = "1.0.51" +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "macros"] } tokio-util = { version = "0.7.10", features = ["io", "io-util", "codec"] } -toml = "0.8.8" -tracing = "0.1.40" +toml = { workspace = true } +tracing = { workspace = true } tracing-subscriber = { workspace = true } url = "2.5.0" warg-client = "0.9.0" diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index d1d7096..3ab4a28 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -45,19 +45,17 @@ use publisher::PackagePublisher; use tokio::io::AsyncSeekExt; use tokio::sync::RwLock; use tokio_util::io::SyncIoBridge; - -use wasm_pkg_common::metadata::RegistryMetadata; -use wit_component::DecodedWasm; - -use crate::{loader::PackageLoader, local::LocalBackend, oci::OciBackend, warg::WargBackend}; - pub use wasm_pkg_common::{ config::Config, digest::ContentDigest, + metadata::RegistryMetadata, package::{PackageRef, Version}, registry::Registry, Error, }; +use wit_component::DecodedWasm; + +use crate::{loader::PackageLoader, local::LocalBackend, oci::OciBackend, warg::WargBackend}; pub use release::{Release, VersionInfo}; diff --git a/crates/wasm-pkg-common/Cargo.toml b/crates/wasm-pkg-common/Cargo.toml index f53635d..3134111 100644 --- a/crates/wasm-pkg-common/Cargo.toml +++ b/crates/wasm-pkg-common/Cargo.toml @@ -14,9 +14,9 @@ tokio = ["dep:tokio"] [dependencies] anyhow = { workspace = true } -bytes = "1.6.0" -dirs = "5.0.1" -futures-util = "0.3.30" +bytes = { workspace = true } +dirs = { workspace = true } +futures-util = { workspace = true } http = "1.1.0" reqwest = { version = "0.12.0", default-features = false, features = [ "rustls-tls", @@ -25,14 +25,14 @@ reqwest = { version = "0.12.0", default-features = false, features = [ "macos-system-configuration", "json", ], optional = true } -semver = "1.0.23" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -sha2 = "0.10.8" +semver = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } tokio = { workspace = true, features = ["io-util", "fs"], optional = true } -toml = "0.8.13" -thiserror = "1.0" -tracing = "0.1" +toml = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/crates/wkg-core/Cargo.toml b/crates/wkg-core/Cargo.toml new file mode 100644 index 0000000..619f075 --- /dev/null +++ b/crates/wkg-core/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "wkg-core" +description = "Wasm Package Tools core libraries for wkg" +repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wkg-core" +edition.workspace = true +version.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +anyhow = { workspace = true } +futures-util = { workspace = true } +semver = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt"] } +toml = { workspace = true } +tracing = { workspace = true } +wasm-pkg-common = { workspace = true } +wasm-pkg-client = { workspace = true } + +[target.'cfg(unix)'.dependencies.libc] +version = "0.2.153" + +[target.'cfg(windows)'.dependencies.windows-sys] +version = "0.52" +features = [ + "Win32_Foundation", + "Win32_Storage", + "Win32_Storage_FileSystem", + "Win32_System", + "Win32_System_IO", + "Win32_Security", + "Win32_System_Console", +] + +[dev-dependencies] +tempfile = { workspace = true } +sha2 = { workspace = true } diff --git a/crates/wkg-core/src/config.rs b/crates/wkg-core/src/config.rs new file mode 100644 index 0000000..8dfb91f --- /dev/null +++ b/crates/wkg-core/src/config.rs @@ -0,0 +1,137 @@ +//! Type definitions and functions for working with `wkg.toml` files. + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result}; +use semver::VersionReq; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWriteExt; + +/// The default name of the configuration file. +pub const CONFIG_FILE_NAME: &str = "wkg.toml"; + +/// The structure for a wkg.toml configuration file. This file is entirely optional and is used for +/// overriding and annotating wasm packages. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct Config { + /// Overrides for various packages + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overrides: Option>, + /// Additional metadata about the package. This will override any metadata already set by other + /// tools. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata: Option, +} + +impl Config { + /// Loads a configuration file from the given path. + pub async fn load_from_path(path: impl AsRef) -> Result { + let contents = tokio::fs::read_to_string(path) + .await + .context("unable to load config from file")?; + let config: Config = toml::from_str(&contents).context("unable to parse config file")?; + Ok(config) + } + + /// Attempts to load the configuration from the current directory. Most of the time, users of this + /// crate should use this function. Right now it just checks for a `wkg.toml` file in the current + /// directory, but we could add more resolution logic in the future. If the file is not found, a + /// default empty config is returned. + pub async fn load() -> Result { + let config_path = PathBuf::from(CONFIG_FILE_NAME); + if !tokio::fs::try_exists(&config_path).await? { + return Ok(Config::default()); + } + Self::load_from_path(config_path).await + } + + /// Serializes and writes the configuration to the given path. + pub async fn write(&self, path: impl AsRef) -> Result<()> { + let contents = toml::to_string_pretty(self)?; + let mut file = tokio::fs::File::create(path).await?; + file.write_all(contents.as_bytes()) + .await + .context("unable to write config to path") + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct Override { + /// A path to the package on disk. If this is set, the package will be loaded from the given + /// path. If this is not set, the package will be loaded from the registry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub path: Option, + /// Overrides the version of a package specified in a world file. This is for advanced use only + /// and may break things. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct Metadata { + /// The authors of the package. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub authors: Vec, + /// The categories of the package. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub categories: Vec, + /// The package description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// The package license. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub license: Option, + /// The package documentation URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub documentation: Option, + /// The package homepage URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub homepage: Option, + /// The package repository URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repository: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_roundtrip() { + let tempdir = tempfile::tempdir().unwrap(); + let config_path = tempdir.path().join(CONFIG_FILE_NAME); + let config = Config { + overrides: Some(HashMap::from([( + "foo:bar".to_string(), + Override { + path: Some(PathBuf::from("bar")), + version: Some(VersionReq::parse("1.0.0").unwrap()), + }, + )])), + metadata: Some(Metadata { + authors: vec!["foo".to_string(), "bar".to_string()], + categories: vec!["foo".to_string(), "bar".to_string()], + description: Some("foo".to_string()), + license: Some("foo".to_string()), + documentation: Some("foo".to_string()), + homepage: Some("foo".to_string()), + repository: Some("foo".to_string()), + }), + }; + + config + .write(&config_path) + .await + .expect("unable to write config"); + let loaded_config = Config::load_from_path(config_path) + .await + .expect("unable to load config"); + assert_eq!( + config, loaded_config, + "config loaded from file does not match original config" + ); + } +} diff --git a/crates/wkg-core/src/lib.rs b/crates/wkg-core/src/lib.rs new file mode 100644 index 0000000..0c364fd --- /dev/null +++ b/crates/wkg-core/src/lib.rs @@ -0,0 +1,6 @@ +//! A library with reusable helpers and types for the `wkg` CLI. This is intended to be used by +//! other downstream CLI tools that may need to leverage some of the same functionality provided by +//! `wkg`. + +pub mod config; +pub mod lock; diff --git a/crates/wkg-core/src/lock.rs b/crates/wkg-core/src/lock.rs new file mode 100644 index 0000000..3ad9ae2 --- /dev/null +++ b/crates/wkg-core/src/lock.rs @@ -0,0 +1,775 @@ +//! Type definitions and functions for working with `wkg.lock` files. + +use std::{ + ops::{Deref, DerefMut}, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result}; +use semver::{Version, VersionReq}; +use serde::{ser::SerializeStruct, Deserialize, Serialize}; +use tokio::{ + fs::{File, OpenOptions}, + io::AsyncWriteExt, +}; +use wasm_pkg_client::{ContentDigest, PackageRef}; + +/// The default name of the lock file. +pub const LOCK_FILE_NAME: &str = "wkg.lock"; +/// The version of the lock file for v1 +pub const LOCK_FILE_V1: u64 = 1; + +/// Represents a resolved dependency lock file. +/// +/// This is a TOML file that contains the resolved dependency information from +/// a previous build. +#[derive(Debug)] +pub struct LockFile { + /// The version of the lock file. + /// + /// Currently this is always `1`. + pub version: u64, + + /// The locked dependencies in the lock file. + /// + /// This list is sorted by the name of the locked package. + pub packages: Vec, + + locker: Locker, +} + +impl PartialEq for LockFile { + fn eq(&self, other: &Self) -> bool { + self.packages == other.packages && self.version == other.version + } +} + +impl Eq for LockFile {} + +impl Serialize for LockFile { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut state = serializer.serialize_struct("LockFile", 2)?; + state.serialize_field("version", &self.version)?; + state.serialize_field("packages", &self.packages)?; + state.end() + } +} + +impl LockFile { + /// Creates a new lock file from the given packages at the given path. This will create an empty + /// file and get an exclusive lock on the file, but will not write the data to the file unless + /// [`write`](Self::write) is called. + pub async fn new_with_path( + mut packages: Vec, + path: impl AsRef, + ) -> Result { + sort_packages(&mut packages); + let locker = Locker::open_rw(path.as_ref()).await?; + Ok(Self { + version: LOCK_FILE_V1, + packages, + locker, + }) + } + + /// Loads a lock file from the given path. If readonly is set to false, then an exclusive lock + /// will be acquired on the file. This function will block until the lock is acquired. + pub async fn load_from_path(path: impl AsRef, readonly: bool) -> Result { + let locker = if readonly { + Locker::open_ro(path.as_ref()).await + } else { + Locker::open_rw(path.as_ref()).await + }?; + let contents = tokio::fs::read_to_string(path) + .await + .context("unable to load lock file from path")?; + let mut lock_file: LockFileIntermediate = + toml::from_str(&contents).context("unable to parse lock file from path")?; + // Ensure version is correct and error if it isn't + if lock_file.version != LOCK_FILE_V1 { + return Err(anyhow::anyhow!( + "unsupported lock file version: {}", + lock_file.version + )); + } + // Ensure packages are sorted by name + sort_packages(&mut lock_file.packages); + Ok(lock_file.into_lock_file(locker)) + } + + /// Attempts to load the lock file from the current directory. Most of the time, users of this + /// crate should use this function. Right now it just checks for a `wkg.lock` file in the + /// current directory, but we could add more resolution logic in the future. If the file is not + /// found, a new file is created and a default empty lockfile is returned. This function will + /// block until the lock is acquired. + pub async fn load(readonly: bool) -> Result { + let lock_path = PathBuf::from(LOCK_FILE_NAME); + if !tokio::fs::try_exists(&lock_path).await? { + // Create a new lock file if it doesn't exist so we can then open it readonly if that is set + tokio::fs::write(&lock_path, "") + .await + .context("Unable to create lock file")?; + } + Self::load_from_path(lock_path, readonly).await + } + + /// Serializes and writes the lock file + /// + /// This function requires mutability because it needs to sort the packages before serializing. + pub async fn write(&mut self) -> Result<()> { + sort_packages(&mut self.packages); + let contents = toml::to_string_pretty(self)?; + // Truncate the file before writing to it + self.locker.file.set_len(0).await.with_context(|| { + format!( + "unable to truncate lock file at path {}", + self.locker.path.display() + ) + })?; + self.locker.file.write_all( + b"# This file is automatically generated.\n# It is not intended for manual editing.\n", + ) + .await.with_context(|| format!("unable to write lock file to path {}", self.locker.path.display()))?; + self.locker + .file + .write_all(contents.as_bytes()) + .await + .with_context(|| { + format!( + "unable to write lock file to path {}", + self.locker.path.display() + ) + })?; + // Make sure to flush and sync just to be sure the file doesn't drop and the lock is + // released too early + self.locker.file.sync_all().await.with_context(|| { + format!( + "unable to write lock file to path {}", + self.locker.path.display() + ) + }) + } +} + +/// Represents a locked package in a lock file. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct LockedPackage { + /// The name of the locked package. + pub name: PackageRef, + + /// The registry the package was resolved from. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub registry: Option, + + /// The locked version of a package. + /// + /// A package may have multiple locked versions if more than one + /// version requirement was specified for the package in `wit.toml`. + #[serde(alias = "version", default, skip_serializing_if = "Vec::is_empty")] + pub versions: Vec, +} + +/// Represents version information for a locked package. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct LockedPackageVersion { + /// The version requirement used to resolve this version (if used). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requirement: Option, + /// The version the package is locked to. + pub version: Version, + /// The digest of the package contents. + pub digest: ContentDigest, +} + +fn sort_packages(packages: &mut [LockedPackage]) { + packages.sort_unstable_by(|a, b| a.name.cmp(&b.name)); +} + +#[derive(Debug, Serialize, Deserialize)] +struct LockFileIntermediate { + version: u64, + + #[serde(alias = "package", default, skip_serializing_if = "Vec::is_empty")] + packages: Vec, +} + +impl LockFileIntermediate { + fn into_lock_file(self, locker: Locker) -> LockFile { + LockFile { + version: self.version, + packages: self.packages, + locker, + } + } +} + +/// Used to indicate the access mode of a lock file. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +enum Access { + Shared, + Exclusive, +} + +/// A wrapper around a lockable file +#[derive(Debug)] +struct Locker { + file: File, + path: PathBuf, +} + +impl Drop for Locker { + fn drop(&mut self) { + let _ = sys::unlock(&self.file); + } +} + +impl Deref for Locker { + type Target = File; + + fn deref(&self) -> &Self::Target { + &self.file + } +} + +impl DerefMut for Locker { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.file + } +} + +impl AsRef for Locker { + fn as_ref(&self) -> &File { + &self.file + } +} + +// NOTE(thomastaylor312): These lock file primitives from here on down are mostly copyed wholesale +// from the lock file implementation of cargo-component with some minor modifications to make them +// work with tokio + +impl Locker { + // NOTE(thomastaylor312): I am keeping around these try methods for possible later use. Right + // now we're ignoring the dead code + #[allow(dead_code)] + /// Attempts to acquire exclusive access to a file, returning the locked + /// version of a file. + /// + /// This function will create a file at `path` if it doesn't already exist + /// (including intermediate directories), and then it will try to acquire an + /// exclusive lock on `path`. + /// + /// If the lock cannot be immediately acquired, `Ok(None)` is returned. + /// + /// The returned file can be accessed to look at the path and also has + /// read/write access to the underlying file. + pub async fn try_open_rw(path: impl Into) -> Result> { + Self::open( + path.into(), + OpenOptions::new().read(true).write(true).create(true), + Access::Exclusive, + true, + ) + .await + } + + /// Opens exclusive access to a file, returning the locked version of a + /// file. + /// + /// This function will create a file at `path` if it doesn't already exist + /// (including intermediate directories), and then it will acquire an + /// exclusive lock on `path`. + /// + /// If the lock cannot be acquired, this function will block until it is + /// acquired. + /// + /// The returned file can be accessed to look at the path and also has + /// read/write access to the underlying file. + pub async fn open_rw(path: impl Into) -> Result { + Ok(Self::open( + path.into(), + OpenOptions::new().read(true).write(true).create(true), + Access::Exclusive, + false, + ) + .await? + .unwrap()) + } + + #[allow(dead_code)] + /// Attempts to acquire shared access to a file, returning the locked version + /// of a file. + /// + /// This function will fail if `path` doesn't already exist, but if it does + /// then it will acquire a shared lock on `path`. + /// + /// If the lock cannot be immediately acquired, `Ok(None)` is returned. + /// + /// The returned file can be accessed to look at the path and also has read + /// access to the underlying file. Any writes to the file will return an + /// error. + pub async fn try_open_ro(path: impl Into) -> Result> { + Self::open( + path.into(), + OpenOptions::new().read(true), + Access::Shared, + true, + ) + .await + } + + /// Opens shared access to a file, returning the locked version of a file. + /// + /// This function will fail if `path` doesn't already exist, but if it does + /// then it will acquire a shared lock on `path`. + /// + /// If the lock cannot be acquired, this function will block until it is + /// acquired. + /// + /// The returned file can be accessed to look at the path and also has read + /// access to the underlying file. Any writes to the file will return an + /// error. + pub async fn open_ro(path: impl Into) -> Result { + Ok(Self::open( + path.into(), + OpenOptions::new().read(true), + Access::Shared, + false, + ) + .await? + .unwrap()) + } + + async fn open( + path: PathBuf, + opts: &OpenOptions, + access: Access, + try_lock: bool, + ) -> Result> { + // If we want an exclusive lock then if we fail because of NotFound it's + // likely because an intermediate directory didn't exist, so try to + // create the directory and then continue. + let file = match opts.open(&path).await { + Ok(file) => Ok(file), + Err(e) if e.kind() == std::io::ErrorKind::NotFound && access == Access::Exclusive => { + tokio::fs::create_dir_all(path.parent().unwrap()) + .await + .with_context(|| { + format!( + "failed to create parent directories for `{path}`", + path = path.display() + ) + })?; + opts.open(&path).await + } + Err(e) => Err(e), + } + .with_context(|| format!("failed to open `{path}`", path = path.display()))?; + + // Now that the file exists, canonicalize the path for better debuggability. + let path = tokio::fs::canonicalize(path) + .await + .context("failed to canonicalize path")?; + let mut lock = Self { file, path }; + + // File locking on Unix is currently implemented via `flock`, which is known + // to be broken on NFS. We could in theory just ignore errors that happen on + // NFS, but apparently the failure mode [1] for `flock` on NFS is **blocking + // forever**, even if the "non-blocking" flag is passed! + // + // As a result, we just skip all file locks entirely on NFS mounts. That + // should avoid calling any `flock` functions at all, and it wouldn't work + // there anyway. + // + // [1]: https://github.com/rust-lang/cargo/issues/2615 + if is_on_nfs_mount(&lock.path) { + return Ok(Some(lock)); + } + + let res = match (access, try_lock) { + (Access::Shared, true) => sys::try_lock_shared(&lock.file), + (Access::Exclusive, true) => sys::try_lock_exclusive(&lock.file), + (Access::Shared, false) => { + // We have to move the lock into the thread because it requires exclusive ownership + // for dropping. We return it back out after the blocking IO. + let (l, res) = tokio::task::spawn_blocking(move || { + let res = sys::lock_shared(&lock.file); + (lock, res) + }) + .await + .context("error waiting for blocking IO")?; + lock = l; + res + } + (Access::Exclusive, false) => { + // We have to move the lock into the thread because it requires exclusive ownership + // for dropping. We return it back out after the blocking IO. + let (l, res) = tokio::task::spawn_blocking(move || { + let res = sys::lock_exclusive(&lock.file); + (lock, res) + }) + .await + .context("error waiting for blocking IO")?; + lock = l; + res + } + }; + + return match res { + Ok(_) => Ok(Some(lock)), + + // In addition to ignoring NFS which is commonly not working we also + // just ignore locking on file systems that look like they don't + // implement file locking. + Err(e) if sys::error_unsupported(&e) => Ok(Some(lock)), + + // Check to see if it was a contention error + Err(e) if try_lock && sys::error_contended(&e) => Ok(None), + + Err(e) => Err(anyhow::anyhow!(e).context(format!( + "failed to lock file `{path}`", + path = lock.path.display() + ))), + }; + + #[cfg(all(target_os = "linux", not(target_env = "musl")))] + fn is_on_nfs_mount(path: &Path) -> bool { + use std::ffi::CString; + use std::mem; + use std::os::unix::prelude::*; + + let path = match CString::new(path.as_os_str().as_bytes()) { + Ok(path) => path, + Err(_) => return false, + }; + + unsafe { + let mut buf: libc::statfs = mem::zeroed(); + let r = libc::statfs(path.as_ptr(), &mut buf); + + r == 0 && buf.f_type as u32 == libc::NFS_SUPER_MAGIC as u32 + } + } + + #[cfg(any(not(target_os = "linux"), target_env = "musl"))] + fn is_on_nfs_mount(_path: &Path) -> bool { + false + } + } +} + +#[cfg(unix)] +mod sys { + use std::io::{Error, Result}; + use std::os::unix::io::AsRawFd; + + use tokio::fs::File; + + pub(super) fn lock_shared(file: &File) -> Result<()> { + flock(file, libc::LOCK_SH) + } + + pub(super) fn lock_exclusive(file: &File) -> Result<()> { + flock(file, libc::LOCK_EX) + } + + pub(super) fn try_lock_shared(file: &File) -> Result<()> { + flock(file, libc::LOCK_SH | libc::LOCK_NB) + } + + pub(super) fn try_lock_exclusive(file: &File) -> Result<()> { + flock(file, libc::LOCK_EX | libc::LOCK_NB) + } + + pub(super) fn unlock(file: &File) -> Result<()> { + flock(file, libc::LOCK_UN) + } + + pub(super) fn error_contended(err: &Error) -> bool { + err.raw_os_error().map_or(false, |x| x == libc::EWOULDBLOCK) + } + + pub(super) fn error_unsupported(err: &Error) -> bool { + match err.raw_os_error() { + // Unfortunately, depending on the target, these may or may not be the same. + // For targets in which they are the same, the duplicate pattern causes a warning. + #[allow(unreachable_patterns)] + Some(libc::ENOTSUP | libc::EOPNOTSUPP) => true, + Some(libc::ENOSYS) => true, + _ => false, + } + } + + #[cfg(not(target_os = "solaris"))] + fn flock(file: &File, flag: libc::c_int) -> Result<()> { + let ret = unsafe { libc::flock(file.as_raw_fd(), flag) }; + if ret < 0 { + Err(Error::last_os_error()) + } else { + Ok(()) + } + } + + #[cfg(target_os = "solaris")] + fn flock(file: &File, flag: libc::c_int) -> Result<()> { + // Solaris lacks flock(), so try to emulate using fcntl() + let mut flock = libc::flock { + l_type: 0, + l_whence: 0, + l_start: 0, + l_len: 0, + l_sysid: 0, + l_pid: 0, + l_pad: [0, 0, 0, 0], + }; + flock.l_type = if flag & libc::LOCK_UN != 0 { + libc::F_UNLCK + } else if flag & libc::LOCK_EX != 0 { + libc::F_WRLCK + } else if flag & libc::LOCK_SH != 0 { + libc::F_RDLCK + } else { + panic!("unexpected flock() operation") + }; + + let mut cmd = libc::F_SETLKW; + if (flag & libc::LOCK_NB) != 0 { + cmd = libc::F_SETLK; + } + + let ret = unsafe { libc::fcntl(file.as_raw_fd(), cmd, &flock) }; + + if ret < 0 { + Err(Error::last_os_error()) + } else { + Ok(()) + } + } +} + +#[cfg(windows)] +mod sys { + use std::io::{Error, Result}; + use std::mem; + use std::os::windows::io::AsRawHandle; + + use tokio::fs::File; + use windows_sys::Win32::Foundation::HANDLE; + use windows_sys::Win32::Foundation::{ERROR_INVALID_FUNCTION, ERROR_LOCK_VIOLATION}; + use windows_sys::Win32::Storage::FileSystem::{ + LockFileEx, UnlockFile, LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY, + }; + + pub(super) fn lock_shared(file: &File) -> Result<()> { + lock_file(file, 0) + } + + pub(super) fn lock_exclusive(file: &File) -> Result<()> { + lock_file(file, LOCKFILE_EXCLUSIVE_LOCK) + } + + pub(super) fn try_lock_shared(file: &File) -> Result<()> { + lock_file(file, LOCKFILE_FAIL_IMMEDIATELY) + } + + pub(super) fn try_lock_exclusive(file: &File) -> Result<()> { + lock_file(file, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY) + } + + pub(super) fn error_contended(err: &Error) -> bool { + err.raw_os_error() + .map_or(false, |x| x == ERROR_LOCK_VIOLATION as i32) + } + + pub(super) fn error_unsupported(err: &Error) -> bool { + err.raw_os_error() + .map_or(false, |x| x == ERROR_INVALID_FUNCTION as i32) + } + + pub(super) fn unlock(file: &File) -> Result<()> { + unsafe { + let ret = UnlockFile(file.as_raw_handle() as HANDLE, 0, 0, !0, !0); + if ret == 0 { + Err(Error::last_os_error()) + } else { + Ok(()) + } + } + } + + fn lock_file(file: &File, flags: u32) -> Result<()> { + unsafe { + let mut overlapped = mem::zeroed(); + let ret = LockFileEx( + file.as_raw_handle() as HANDLE, + flags, + 0, + !0, + !0, + &mut overlapped, + ); + if ret == 0 { + Err(Error::last_os_error()) + } else { + Ok(()) + } + } + } +} + +#[cfg(test)] +mod tests { + use sha2::Digest; + + use super::*; + + #[tokio::test] + async fn test_shared_locking() { + let tempdir = tempfile::tempdir().expect("failed to create tempdir"); + let path = tempdir.path().join("test"); + + tokio::fs::write(&path, "") + .await + .expect("failed to write empty file"); + + let _locker1 = Locker::open_ro(path.clone()) + .await + .expect("failed to open reader locker"); + let _locker2 = Locker::open_ro(path.clone()) + .await + .expect("should be able to open a second reader"); + } + + #[tokio::test] + async fn test_exclusive_locking() { + let tempdir = tempfile::tempdir().expect("failed to create tempdir"); + let path = tempdir.path().join("test"); + + tokio::fs::write(&path, "") + .await + .expect("failed to write empty file"); + + let locker1 = Locker::open_rw(path.clone()) + .await + .expect("failed to open writer locker"); + let maybe_locker = Locker::try_open_rw(path.clone()) + .await + .expect("shouldn't fail with a try open"); + assert!( + maybe_locker.is_none(), + "Shouldn't be able to open a second writer" + ); + + let maybe_locker = Locker::try_open_ro(path.clone()) + .await + .expect("shouldn't fail with a try open"); + assert!(maybe_locker.is_none(), "Shouldn't be able to open a reader"); + + // A call to open_rw should block until the first locker is dropped + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let res = Locker::open_rw(path.clone()).await; + tx.send(res).expect("failed to send signal"); + }); + + // Sleep here to simulate another process finishing a write + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + drop(locker1); + + tokio::select! { + res = rx => { + assert!(res.is_ok(), "failed to open second write locker"); + } + _ = tokio::time::sleep(tokio::time::Duration::from_millis(1000)) => { + panic!("timed out waiting for second locker"); + } + } + } + + #[tokio::test] + async fn test_roundtrip() { + let tempdir = tempfile::tempdir().expect("failed to create tempdir"); + let path = tempdir.path().join(LOCK_FILE_NAME); + + let mut fakehasher = sha2::Sha256::new(); + fakehasher.update(b"fake"); + + let mut expected_deps = vec![ + LockedPackage { + name: "enterprise:holodeck".parse().unwrap(), + versions: vec![LockedPackageVersion { + version: "0.1.0".parse().unwrap(), + digest: fakehasher.clone().into(), + requirement: None, + }], + registry: None, + }, + LockedPackage { + name: "ds9:holosuite".parse().unwrap(), + versions: vec![LockedPackageVersion { + version: "0.1.0".parse().unwrap(), + digest: fakehasher.clone().into(), + requirement: None, + }], + registry: None, + }, + ]; + + let mut lock = LockFile::new_with_path(expected_deps.clone(), &path) + .await + .expect("Shouldn't fail when creating a new lock file"); + + // Now sort the expected deps and make sure the lock file deps also got sorted + sort_packages(&mut expected_deps); + assert_eq!( + lock.packages, expected_deps, + "Lock file deps should match expected deps" + ); + + // Push one more package onto the lock file before writing it + let new_package = LockedPackage { + name: "defiant:armor".parse().unwrap(), + versions: vec![LockedPackageVersion { + version: "0.1.0".parse().unwrap(), + digest: fakehasher.into(), + requirement: None, + }], + registry: None, + }; + + lock.packages.push(new_package.clone()); + expected_deps.push(new_package); + sort_packages(&mut expected_deps); + + lock.write() + .await + .expect("Shouldn't fail when writing lock file"); + + // Drop the lock file + drop(lock); + + // Parse out the file into the intermediate format and make sure the vec is in the same + // order as the expected deps + let lock_file = + std::fs::read_to_string(&path).expect("Shouldn't fail when reading lock file"); + let lock_file: LockFileIntermediate = + toml::from_str(&lock_file).expect("Shouldn't fail when parsing lock file"); + assert_eq!( + lock_file.packages, expected_deps, + "Lock file deps should match expected deps" + ); + + // Now read the lock file again and make sure everything is correct (and we can lock it + // properly) + let lock = LockFile::load_from_path(&path, true) + .await + .expect("Shouldn't fail when loading lock file"); + assert_eq!( + lock.packages, expected_deps, + "Lock file deps should match expected deps" + ); + assert_eq!(lock.version, LOCK_FILE_V1, "Lock file version should be 1"); + } +} diff --git a/crates/wkg/Cargo.toml b/crates/wkg/Cargo.toml index 9b703f4..18587cb 100644 --- a/crates/wkg/Cargo.toml +++ b/crates/wkg/Cargo.toml @@ -12,10 +12,10 @@ readme = "../../README.md" anyhow = { workspace = true } clap = { version = "4.5", features = ["derive", "wrap_help", "env"] } docker_credential = { workspace = true } -futures-util = { version = "0.3.29", features = ["io"] } +futures-util = { workspace = true, features = ["io"] } oci-client = { workspace = true } oci-wasm = { workspace = true } -tempfile = "3.10.1" +tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } @@ -24,5 +24,5 @@ wasm-pkg-client = { workspace = true } wit-component = { workspace = true } [dev-dependencies] -base64 = "0.22.1" -serde_json = "1.0.118" +base64 = { workspace = true } +serde_json = { workspace = true } From f6d712c91f08f19b90022ba94b0283a2f3a39a35 Mon Sep 17 00:00:00 2001 From: Taylor Thomas Date: Wed, 25 Sep 2024 09:27:53 -0600 Subject: [PATCH 2/3] feat(*): Adds support for building and fetching wit packages This adds a new `wit` subcommand to `wkg` that supports building wit packages and fetching/populating a deps directory. I'm sure there is much more we can do here and some obtuse edge cases that aren't supported, but I did test fetching dependencies for various worlds that used wasi:http and wasi:cli. In a follow up PR, I'll add some more integration tests Signed-off-by: Taylor Thomas --- .github/workflows/ci.yml | 16 +- Cargo.lock | 53 +- Cargo.toml | 3 + crates/wasm-pkg-client/Cargo.toml | 7 +- crates/wasm-pkg-client/src/caching/mod.rs | 17 +- crates/wasm-pkg-client/src/lib.rs | 18 +- crates/wasm-pkg-client/src/oci/mod.rs | 1 + crates/wasm-pkg-client/tests/e2e.rs | 4 + crates/wasm-pkg-common/src/config.rs | 20 +- crates/wkg-core/Cargo.toml | 5 + crates/wkg-core/src/config.rs | 44 +- crates/wkg-core/src/lib.rs | 2 + crates/wkg-core/src/lock.rs | 210 ++++-- crates/wkg-core/src/resolver.rs | 755 ++++++++++++++++++++++ crates/wkg-core/src/wit.rs | 402 ++++++++++++ crates/wkg/Cargo.toml | 2 + crates/wkg/src/main.rs | 81 ++- crates/wkg/src/wit.rs | 150 +++++ 18 files changed, 1658 insertions(+), 132 deletions(-) create mode 100644 crates/wkg-core/src/resolver.rs create mode 100644 crates/wkg-core/src/wit.rs create mode 100644 crates/wkg/src/wit.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa1d388..f47e581 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,11 +8,21 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + include: + - os: ubuntu-latest + additional_test_flags: "" + - os: windows-latest + additional_test_flags: "--no-default-features" + - os: macos-latest + additional_test_flags: "--no-default-features" steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - name: Run cargo test - run: cargo test --all --workspace + # We have to run these separately so we can deactivate a feature for one of the tests + - name: Run client tests + working-directory: ./crates/wasm-pkg-client + run: cargo test ${{ matrix.additional_test_flags }} + - name: Run other tests + run: cargo test --workspace --exclude wasm-pkg-client - name: Run cargo clippy run: cargo clippy --all --workspace \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index bbfadf4..24f2cec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1178,7 +1178,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.4.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -1482,9 +1482,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2158,7 +2158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.4.0", + "indexmap 2.5.0", ] [[package]] @@ -2946,7 +2946,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.4.0", + "indexmap 2.5.0", "serde 1.0.209", "serde_derive", "serde_json", @@ -2972,7 +2972,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "itoa", "ryu", "serde 1.0.209", @@ -3465,7 +3465,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "toml_datetime", "winnow 0.5.40", ] @@ -3476,7 +3476,7 @@ version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "serde 1.0.209", "serde_spanned", "toml_datetime", @@ -3689,7 +3689,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44b422328c3a86be288f569694aa97df958ade0cd9514ed00bc562952c6778e" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.12.1", "serde 1.0.209", "serde_with", @@ -3712,7 +3712,7 @@ dependencies = [ "dialoguer", "dirs", "futures-util", - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.12.1", "keyring", "libc", @@ -3793,7 +3793,7 @@ dependencies = [ "anyhow", "base64 0.21.7", "hex", - "indexmap 2.4.0", + "indexmap 2.5.0", "pbjson-types", "prost", "prost-types", @@ -3814,7 +3814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b950a71a544b7ac8f5a5e95f43886ac97c3fe5c7080b955b1b534037596d7be" dependencies = [ "anyhow", - "indexmap 2.4.0", + "indexmap 2.5.0", "prost", "thiserror", "warg-crypto", @@ -3903,7 +3903,7 @@ dependencies = [ "anyhow", "heck 0.4.1", "im-rc", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "petgraph", "serde 1.0.209", @@ -3952,7 +3952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6bb07c5576b608f7a2a9baa2294c1a3584a249965d695a9814a496cb6d232f" dependencies = [ "anyhow", - "indexmap 2.4.0", + "indexmap 2.5.0", "serde 1.0.209", "serde_derive", "serde_json", @@ -3968,7 +3968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47c8154d703a6b0e45acf6bd172fa002fc3c7058a9f7615e517220aeca27c638" dependencies = [ "anyhow", - "indexmap 2.4.0", + "indexmap 2.5.0", "serde 1.0.209", "serde_derive", "serde_json", @@ -4050,7 +4050,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbe55c8f9d0dbd25d9447a5a889ff90c0cc3feaa7395310d3d826b2c703eaab" dependencies = [ "bitflags 2.6.0", - "indexmap 2.4.0", + "indexmap 2.5.0", "semver", ] @@ -4063,7 +4063,7 @@ dependencies = [ "ahash", "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap 2.4.0", + "indexmap 2.5.0", "semver", ] @@ -4076,7 +4076,7 @@ dependencies = [ "ahash", "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap 2.4.0", + "indexmap 2.5.0", "semver", ] @@ -4375,7 +4375,7 @@ checksum = "f725e3885fc5890648be5c5cbc1353b755dc932aa5f1aa7de968b912a3280743" dependencies = [ "anyhow", "bitflags 2.6.0", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "serde 1.0.209", "serde_derive", @@ -4394,7 +4394,7 @@ checksum = "7e2ca3ece38ea2447a9069b43074ba73d96dde1944cba276c54e41371745f9dc" dependencies = [ "anyhow", "bitflags 2.6.0", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "serde 1.0.209", "serde_derive", @@ -4413,7 +4413,7 @@ checksum = "935a97eaffd57c3b413aa510f8f0b550a4a9fe7d59e79cd8b89a83dcb860321f" dependencies = [ "anyhow", "id-arena", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "semver", "serde 1.0.209", @@ -4431,7 +4431,7 @@ checksum = "a4d108165c1167a4ccc8a803dcf5c28e0a51d6739fd228cc7adce768632c764c" dependencies = [ "anyhow", "id-arena", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "semver", "serde 1.0.209", @@ -4448,6 +4448,7 @@ dependencies = [ "anyhow", "base64 0.22.1", "clap", + "dirs", "docker_credential", "futures-util", "oci-client", @@ -4460,25 +4461,31 @@ dependencies = [ "wasm-pkg-client", "wasm-pkg-common", "wit-component 0.216.0", + "wkg-core", ] [[package]] name = "wkg-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "futures-util", + "indexmap 2.5.0", "libc", "semver", "serde 1.0.209", "sha2", "tempfile", "tokio", + "tokio-util", "toml 0.8.19", "tracing", + "wasm-metadata 0.216.0", "wasm-pkg-client", "wasm-pkg-common", "windows-sys 0.52.0", + "wit-component 0.216.0", + "wit-parser 0.216.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0512733..93948e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ sha2 = "0.10" tempfile = "3.10.1" thiserror = "1.0" tokio = "1.35.1" +tokio-util = "0.7.10" toml = "0.8.13" tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", default-features = false, features = [ @@ -36,5 +37,7 @@ tracing-subscriber = { version = "0.3.18", default-features = false, features = ] } wasm-pkg-common = { version = "0.5.1", path = "crates/wasm-pkg-common" } wasm-pkg-client = { version = "0.5.1", path = "crates/wasm-pkg-client" } +wasm-metadata = "0.216" wit-component = "0.216" wit-parser = "0.216" +wkg-core = { version = "0.5.0", path = "crates/wkg-core" } diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index 95e70d7..025c792 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -8,6 +8,11 @@ authors.workspace = true license.workspace = true readme = "../../README.md" +[features] +default = ["_local"] +# An internal feature for making sure e2e tests can run locally but not in CI for Mac or Windows +_local = [] + [dependencies] anyhow = { workspace = true } async-trait = "0.1.77" @@ -24,7 +29,7 @@ serde_json = { workspace = true } sha2 = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "macros"] } -tokio-util = { version = "0.7.10", features = ["io", "io-util", "codec"] } +tokio-util = { workspace = true, features = ["io", "io-util", "codec"] } toml = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/wasm-pkg-client/src/caching/mod.rs b/crates/wasm-pkg-client/src/caching/mod.rs index e142548..59c0d93 100644 --- a/crates/wasm-pkg-client/src/caching/mod.rs +++ b/crates/wasm-pkg-client/src/caching/mod.rs @@ -1,4 +1,5 @@ use std::future::Future; +use std::sync::Arc; use wasm_pkg_common::{ digest::ContentDigest, @@ -46,7 +47,16 @@ pub trait Cache { /// underlying client to be used as a read-only cache. pub struct CachingClient { client: Option, - cache: T, + cache: Arc, +} + +impl Clone for CachingClient { + fn clone(&self) -> Self { + Self { + client: self.client.clone(), + cache: self.cache.clone(), + } + } } impl CachingClient { @@ -54,7 +64,10 @@ impl CachingClient { /// given, the client will be in offline or read-only mode, meaning it will only be able to return /// things that are already in the cache. pub fn new(client: Option, cache: T) -> Self { - Self { client, cache } + Self { + client, + cache: Arc::new(cache), + } } /// Returns whether or not the client is in read-only mode. diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 3ab4a28..26a50de 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -8,7 +8,7 @@ //! ```no_run //! # async fn example() -> anyhow::Result<()> { //! // Initialize client from global configuration. -//! let mut client = wasm_pkg_client::Client::with_global_defaults()?; +//! let mut client = wasm_pkg_client::Client::with_global_defaults().await?; //! //! // Get a specific package release version. //! let pkg = "example:pkg".parse()?; @@ -86,23 +86,29 @@ pub struct PublishOpts { } /// A read-only registry client. +#[derive(Clone)] pub struct Client { - config: Config, - sources: RwLock, + config: Arc, + sources: Arc>, } impl Client { /// Returns a new client with the given [`Config`]. pub fn new(config: Config) -> Self { Self { - config, + config: Arc::new(config), sources: Default::default(), } } + /// Returns a reference to the configuration this client was initialized with. + pub fn config(&self) -> &Config { + &self.config + } + /// Returns a new client configured from default global config. - pub fn with_global_defaults() -> Result { - let config = Config::global_defaults()?; + pub async fn with_global_defaults() -> Result { + let config = Config::global_defaults().await?; Ok(Self::new(config)) } diff --git a/crates/wasm-pkg-client/src/oci/mod.rs b/crates/wasm-pkg-client/src/oci/mod.rs index 5c065b1..3ab4258 100644 --- a/crates/wasm-pkg-client/src/oci/mod.rs +++ b/crates/wasm-pkg-client/src/oci/mod.rs @@ -128,6 +128,7 @@ impl OciBackend { CredentialRetrievalError::ConfigNotFound | CredentialRetrievalError::ConfigReadError | CredentialRetrievalError::NoCredentialConfigured + | CredentialRetrievalError::HelperFailure { .. } ) { tracing::debug!("Failed to look up OCI credentials: {err}"); } else { diff --git a/crates/wasm-pkg-client/tests/e2e.rs b/crates/wasm-pkg-client/tests/e2e.rs index c6954fd..01c389f 100644 --- a/crates/wasm-pkg-client/tests/e2e.rs +++ b/crates/wasm-pkg-client/tests/e2e.rs @@ -8,6 +8,10 @@ use wasm_pkg_client::{Client, Config}; const FIXTURE_WASM: &str = "./tests/testdata/binary_wit.wasm"; +#[cfg(any(target_os = "linux", feature = "_local"))] +// NOTE: These are only run on linux for CI purposes, because they rely on the docker client being +// available, and for various reasons this has proven to be problematic on both the Windows and +// MacOS runners due to it not being installed (yay licensing). #[tokio::test] async fn publish_and_fetch_smoke_test() { let _container = GenericImage::new("registry", "2") diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index b12abc6..c828177 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -73,21 +73,21 @@ impl Config { /// /// Note: This list is expected to expand in the future to include /// "workspace" config files like `./.wasm-pkg/config.toml`. - pub fn global_defaults() -> Result { + pub async fn global_defaults() -> Result { let mut config = Self::default(); - if let Some(global_config) = Self::read_global_config()? { + if let Some(global_config) = Self::read_global_config().await? { config.merge(global_config); } Ok(config) } /// Reads config from the default global config file location - pub fn read_global_config() -> Result, Error> { + pub async fn read_global_config() -> Result, Error> { let Some(config_dir) = dirs::config_dir() else { return Ok(None); }; let path = config_dir.join("wasm-pkg").join("config.toml"); - let contents = match std::fs::read_to_string(path) { + let contents = match tokio::fs::read_to_string(path).await { Ok(contents) => contents, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), Err(err) => return Err(Error::ConfigFileIoError(err)), @@ -96,8 +96,10 @@ impl Config { } /// Reads config from a TOML file at the given path. - pub fn from_file(path: impl AsRef) -> Result { - let contents = std::fs::read_to_string(path).map_err(Error::ConfigFileIoError)?; + pub async fn from_file(path: impl AsRef) -> Result { + let contents = tokio::fs::read_to_string(path) + .await + .map_err(Error::ConfigFileIoError)?; Self::from_toml(&contents) } @@ -109,9 +111,11 @@ impl Config { } /// Writes the config to a TOML file at the given path. - pub fn to_file(&self, path: impl AsRef) -> Result<(), Error> { + pub async fn to_file(&self, path: impl AsRef) -> Result<(), Error> { let toml_str = ::toml::to_string(&self).map_err(Error::invalid_config)?; - std::fs::write(path, toml_str).map_err(Error::ConfigFileIoError) + tokio::fs::write(path, toml_str) + .await + .map_err(Error::ConfigFileIoError) } /// Merges the given other config into this one. diff --git a/crates/wkg-core/Cargo.toml b/crates/wkg-core/Cargo.toml index 619f075..2455950 100644 --- a/crates/wkg-core/Cargo.toml +++ b/crates/wkg-core/Cargo.toml @@ -10,13 +10,18 @@ license.workspace = true [dependencies] anyhow = { workspace = true } futures-util = { workspace = true } +indexmap = "2.5" semver = { workspace = true } serde = { workspace = true } tokio = { workspace = true, features = ["macros", "rt"] } +tokio-util = { workspace = true, features = ["io", "io-util", "codec"] } toml = { workspace = true } tracing = { workspace = true } +wasm-metadata = { workspace = true } wasm-pkg-common = { workspace = true } wasm-pkg-client = { workspace = true } +wit-component = { workspace = true } +wit-parser = { workspace = true } [target.'cfg(unix)'.dependencies.libc] version = "0.2.153" diff --git a/crates/wkg-core/src/config.rs b/crates/wkg-core/src/config.rs index 8dfb91f..fec5753 100644 --- a/crates/wkg-core/src/config.rs +++ b/crates/wkg-core/src/config.rs @@ -9,6 +9,7 @@ use anyhow::{Context, Result}; use semver::VersionReq; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; +use wasm_metadata::{Link, LinkType, RegistryMetadata}; /// The default name of the configuration file. pub const CONFIG_FILE_NAME: &str = "wkg.toml"; @@ -73,11 +74,11 @@ pub struct Override { #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] pub struct Metadata { /// The authors of the package. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub authors: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authors: Option>, /// The categories of the package. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub categories: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub categories: Option>, /// The package description. #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, @@ -95,6 +96,37 @@ pub struct Metadata { pub repository: Option, } +impl From for wasm_metadata::RegistryMetadata { + fn from(value: Metadata) -> Self { + let mut meta = RegistryMetadata::default(); + meta.set_authors(value.authors); + meta.set_categories(value.categories); + meta.set_description(value.description); + meta.set_license(value.license); + let mut links = Vec::new(); + if let Some(documentation) = value.documentation { + links.push(Link { + ty: LinkType::Documentation, + value: documentation, + }); + } + if let Some(homepage) = value.homepage { + links.push(Link { + ty: LinkType::Homepage, + value: homepage, + }); + } + if let Some(repository) = value.repository { + links.push(Link { + ty: LinkType::Repository, + value: repository, + }); + } + meta.set_links((!links.is_empty()).then_some(links)); + meta + } +} + #[cfg(test)] mod tests { use super::*; @@ -112,8 +144,8 @@ mod tests { }, )])), metadata: Some(Metadata { - authors: vec!["foo".to_string(), "bar".to_string()], - categories: vec!["foo".to_string(), "bar".to_string()], + authors: Some(vec!["foo".to_string(), "bar".to_string()]), + categories: Some(vec!["foo".to_string(), "bar".to_string()]), description: Some("foo".to_string()), license: Some("foo".to_string()), documentation: Some("foo".to_string()), diff --git a/crates/wkg-core/src/lib.rs b/crates/wkg-core/src/lib.rs index 0c364fd..98bbf90 100644 --- a/crates/wkg-core/src/lib.rs +++ b/crates/wkg-core/src/lib.rs @@ -4,3 +4,5 @@ pub mod config; pub mod lock; +pub mod resolver; +pub mod wit; diff --git a/crates/wkg-core/src/lock.rs b/crates/wkg-core/src/lock.rs index 3ad9ae2..3695f9e 100644 --- a/crates/wkg-core/src/lock.rs +++ b/crates/wkg-core/src/lock.rs @@ -1,19 +1,23 @@ //! Type definitions and functions for working with `wkg.lock` files. use std::{ + cmp::Ordering, + collections::{BTreeSet, HashMap}, ops::{Deref, DerefMut}, path::{Path, PathBuf}, }; use anyhow::{Context, Result}; use semver::{Version, VersionReq}; -use serde::{ser::SerializeStruct, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use tokio::{ fs::{File, OpenOptions}, io::AsyncWriteExt, }; use wasm_pkg_client::{ContentDigest, PackageRef}; +use crate::resolver::{DependencyResolution, DependencyResolutionMap}; + /// The default name of the lock file. pub const LOCK_FILE_NAME: &str = "wkg.lock"; /// The version of the lock file for v1 @@ -23,7 +27,7 @@ pub const LOCK_FILE_V1: u64 = 1; /// /// This is a TOML file that contains the resolved dependency information from /// a previous build. -#[derive(Debug)] +#[derive(Debug, serde::Serialize)] pub struct LockFile { /// The version of the lock file. /// @@ -33,8 +37,9 @@ pub struct LockFile { /// The locked dependencies in the lock file. /// /// This list is sorted by the name of the locked package. - pub packages: Vec, + pub packages: BTreeSet, + #[serde(skip)] locker: Locker, } @@ -46,31 +51,18 @@ impl PartialEq for LockFile { impl Eq for LockFile {} -impl Serialize for LockFile { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let mut state = serializer.serialize_struct("LockFile", 2)?; - state.serialize_field("version", &self.version)?; - state.serialize_field("packages", &self.packages)?; - state.end() - } -} - impl LockFile { /// Creates a new lock file from the given packages at the given path. This will create an empty /// file and get an exclusive lock on the file, but will not write the data to the file unless /// [`write`](Self::write) is called. pub async fn new_with_path( - mut packages: Vec, + packages: impl IntoIterator, path: impl AsRef, ) -> Result { - sort_packages(&mut packages); let locker = Locker::open_rw(path.as_ref()).await?; Ok(Self { version: LOCK_FILE_V1, - packages, + packages: packages.into_iter().collect(), locker, }) } @@ -86,7 +78,7 @@ impl LockFile { let contents = tokio::fs::read_to_string(path) .await .context("unable to load lock file from path")?; - let mut lock_file: LockFileIntermediate = + let lock_file: LockFileIntermediate = toml::from_str(&contents).context("unable to parse lock file from path")?; // Ensure version is correct and error if it isn't if lock_file.version != LOCK_FILE_V1 { @@ -95,11 +87,31 @@ impl LockFile { lock_file.version )); } - // Ensure packages are sorted by name - sort_packages(&mut lock_file.packages); Ok(lock_file.into_lock_file(locker)) } + /// Creates a lock file from the dependency map. This will create an empty file (if it doesn't + /// exist) and get an exclusive lock on the file, but will not write the data to the file unless + /// [`write`](Self::write) is called. + pub async fn from_dependencies( + map: &DependencyResolutionMap, + path: impl AsRef, + ) -> Result { + let packages = generate_locked_packages(map); + + LockFile::new_with_path(packages, path).await + } + + /// A helper for updating the current lock file with the given dependency map. This will clear current + /// packages that are not in the dependency map and add new packages that are in the dependency + /// map. + /// + /// This function will not write the data to the file unless [`write`](Self::write) is called. + pub fn update_dependencies(&mut self, map: &DependencyResolutionMap) { + self.packages.clear(); + self.packages.extend(generate_locked_packages(map)); + } + /// Attempts to load the lock file from the current directory. Most of the time, users of this /// crate should use this function. Right now it just checks for a `wkg.lock` file in the /// current directory, but we could add more resolution logic in the future. If the file is not @@ -109,18 +121,14 @@ impl LockFile { let lock_path = PathBuf::from(LOCK_FILE_NAME); if !tokio::fs::try_exists(&lock_path).await? { // Create a new lock file if it doesn't exist so we can then open it readonly if that is set - tokio::fs::write(&lock_path, "") - .await - .context("Unable to create lock file")?; + let mut temp_lock = Self::new_with_path([], &lock_path).await?; + temp_lock.write().await?; } Self::load_from_path(lock_path, readonly).await } /// Serializes and writes the lock file - /// - /// This function requires mutability because it needs to sort the packages before serializing. pub async fn write(&mut self) -> Result<()> { - sort_packages(&mut self.packages); let contents = toml::to_string_pretty(self)?; // Truncate the file before writing to it self.locker.file.set_len(0).await.with_context(|| { @@ -152,6 +160,89 @@ impl LockFile { ) }) } + + /// Resolves a package from the lock file. + /// + /// Returns `Ok(None)` if the package cannot be resolved. + /// + /// Fails if the package cannot be resolved and the lock file is not allowed to be updated. + pub fn resolve( + &self, + registry: Option<&str>, + package_ref: &PackageRef, + requirement: &VersionReq, + ) -> Result> { + // NOTE(thomastaylor312): Using a btree map so we don't have to keep sorting the vec. The + // tradeoff is we have to clone two things here to do the fetch. That tradeoff seems fine to + // me, especially because this is used in CLI commands. + if let Some(pkg) = self.packages.get(&LockedPackage { + name: package_ref.clone(), + registry: registry.map(ToString::to_string), + versions: vec![], + }) { + if let Some(locked) = pkg + .versions + .iter() + .find(|locked| &locked.requirement == requirement) + { + tracing::info!(%package_ref, ?registry, %requirement, resolved_version = %locked.version, "dependency package was resolved by the lock file"); + return Ok(Some(locked)); + } + } + + tracing::info!(%package_ref, ?registry, %requirement, "dependency package was not in the lock file"); + Ok(None) + } +} + +fn generate_locked_packages(map: &DependencyResolutionMap) -> impl Iterator { + type PackageKey = (PackageRef, Option); + type VersionsMap = HashMap; + let mut packages: HashMap = HashMap::new(); + + for resolution in map.values() { + match resolution.key() { + Some((id, registry)) => { + let pkg = match resolution { + DependencyResolution::Registry(pkg) => pkg, + DependencyResolution::Local(_) => unreachable!(), + }; + + let prev = packages + .entry((id.clone(), registry.map(str::to_string))) + .or_default() + .insert( + pkg.requirement.to_string(), + (pkg.version.clone(), pkg.digest.clone()), + ); + + if let Some((prev, _)) = prev { + // The same requirements should resolve to the same version + assert!(prev == pkg.version) + } + } + None => continue, + } + } + + packages.into_iter().map(|((name, registry), versions)| { + let versions: Vec = versions + .into_iter() + .map(|(requirement, (version, digest))| LockedPackageVersion { + requirement: requirement + .parse() + .expect("Version requirement should have been valid. This is programmer error"), + version, + digest, + }) + .collect(); + + LockedPackage { + name, + registry, + versions, + } + }) } /// Represents a locked package in a lock file. @@ -161,7 +252,8 @@ pub struct LockedPackage { pub name: PackageRef, /// The registry the package was resolved from. - #[serde(default, skip_serializing_if = "Option::is_none")] + // NOTE(thomastaylor312): This is a string instead of using the `Registry` type because clippy + // is complaining about it being an interior mutable key type for the btreeset pub registry: Option, /// The locked version of a package. @@ -172,28 +264,39 @@ pub struct LockedPackage { pub versions: Vec, } +impl Ord for LockedPackage { + fn cmp(&self, other: &Self) -> Ordering { + if self.name == other.name { + self.registry.cmp(&other.registry) + } else { + self.name.cmp(&other.name) + } + } +} + +impl PartialOrd for LockedPackage { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// Represents version information for a locked package. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct LockedPackageVersion { - /// The version requirement used to resolve this version (if used). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub requirement: Option, + /// The version requirement used to resolve this version + pub requirement: VersionReq, /// The version the package is locked to. pub version: Version, /// The digest of the package contents. pub digest: ContentDigest, } -fn sort_packages(packages: &mut [LockedPackage]) { - packages.sort_unstable_by(|a, b| a.name.cmp(&b.name)); -} - -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] struct LockFileIntermediate { version: u64, #[serde(alias = "package", default, skip_serializing_if = "Vec::is_empty")] - packages: Vec, + packages: BTreeSet, } impl LockFileIntermediate { @@ -696,13 +799,13 @@ mod tests { let mut fakehasher = sha2::Sha256::new(); fakehasher.update(b"fake"); - let mut expected_deps = vec![ + let mut expected_deps = BTreeSet::from([ LockedPackage { name: "enterprise:holodeck".parse().unwrap(), versions: vec![LockedPackageVersion { version: "0.1.0".parse().unwrap(), digest: fakehasher.clone().into(), - requirement: None, + requirement: VersionReq::parse("=0.1.0").unwrap(), }], registry: None, }, @@ -711,37 +814,29 @@ mod tests { versions: vec![LockedPackageVersion { version: "0.1.0".parse().unwrap(), digest: fakehasher.clone().into(), - requirement: None, + requirement: VersionReq::parse("=0.1.0").unwrap(), }], registry: None, }, - ]; + ]); let mut lock = LockFile::new_with_path(expected_deps.clone(), &path) .await .expect("Shouldn't fail when creating a new lock file"); - // Now sort the expected deps and make sure the lock file deps also got sorted - sort_packages(&mut expected_deps); - assert_eq!( - lock.packages, expected_deps, - "Lock file deps should match expected deps" - ); - // Push one more package onto the lock file before writing it let new_package = LockedPackage { name: "defiant:armor".parse().unwrap(), versions: vec![LockedPackageVersion { version: "0.1.0".parse().unwrap(), digest: fakehasher.into(), - requirement: None, + requirement: VersionReq::parse("=0.1.0").unwrap(), }], registry: None, }; - lock.packages.push(new_package.clone()); - expected_deps.push(new_package); - sort_packages(&mut expected_deps); + lock.packages.insert(new_package.clone()); + expected_deps.insert(new_package); lock.write() .await @@ -750,17 +845,6 @@ mod tests { // Drop the lock file drop(lock); - // Parse out the file into the intermediate format and make sure the vec is in the same - // order as the expected deps - let lock_file = - std::fs::read_to_string(&path).expect("Shouldn't fail when reading lock file"); - let lock_file: LockFileIntermediate = - toml::from_str(&lock_file).expect("Shouldn't fail when parsing lock file"); - assert_eq!( - lock_file.packages, expected_deps, - "Lock file deps should match expected deps" - ); - // Now read the lock file again and make sure everything is correct (and we can lock it // properly) let lock = LockFile::load_from_path(&path, true) diff --git a/crates/wkg-core/src/resolver.rs b/crates/wkg-core/src/resolver.rs new file mode 100644 index 0000000..f027986 --- /dev/null +++ b/crates/wkg-core/src/resolver.rs @@ -0,0 +1,755 @@ +//! A resolver for resolving dependencies from a component registry. +// NOTE(thomastaylor312): This is copied and adapted from the `cargo-component` crate: https://github.com/bytecodealliance/cargo-component/blob/f0be1c7d9917aa97e9102e69e3b838dae38d624b/crates/core/src/registry.rs + +use std::{ + collections::{hash_map, HashMap, HashSet}, + fmt::Debug, + ops::{Deref, DerefMut}, + path::{Path, PathBuf}, + str::FromStr, +}; + +use anyhow::{bail, Context, Result}; +use futures_util::TryStreamExt; +use indexmap::{IndexMap, IndexSet}; +use semver::{Comparator, Op, Version, VersionReq}; +use tokio::io::{AsyncRead, AsyncReadExt}; +use wasm_pkg_client::{ + caching::{CachingClient, FileCache}, + Client, Config, ContentDigest, Error as WasmPkgError, PackageRef, Release, VersionInfo, +}; +use wit_component::DecodedWasm; +use wit_parser::{PackageId, PackageName, Resolve, UnresolvedPackageGroup, WorldId}; + +use crate::lock::LockFile; + +/// The name of the default registry. +pub const DEFAULT_REGISTRY_NAME: &str = "default"; + +// TODO: functions for resolving dependencies from a lock file + +/// Represents a WIT package dependency. +#[derive(Debug, Clone)] +pub enum Dependency { + /// The dependency is a registry package. + Package(RegistryPackage), + + /// The dependency is a path to a local directory or file. + Local(PathBuf), +} + +impl FromStr for Dependency { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(Self::Package(s.parse()?)) + } +} + +/// Represents a reference to a registry package. +#[derive(Debug, Clone)] +pub struct RegistryPackage { + /// The name of the package. + /// + /// If not specified, the name from the mapping will be used. + pub name: Option, + + /// The version requirement of the package. + pub version: VersionReq, + + /// The name of the component registry containing the package. + /// + /// If not specified, the default registry is used. + pub registry: Option, +} + +impl FromStr for RegistryPackage { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(Self { + name: None, + version: s + .parse() + .with_context(|| format!("'{s}' is an invalid registry package version"))?, + registry: None, + }) + } +} + +/// Represents information about a resolution of a registry package. +#[derive(Clone)] +pub struct RegistryResolution { + /// The name of the dependency that was resolved. + /// + /// This may differ from `package` if the dependency was renamed. + pub name: PackageRef, + /// The name of the package from the registry that was resolved. + pub package: PackageRef, + /// The name of the registry used to resolve the package if one was provided + pub registry: Option, + /// The version requirement that was used to resolve the package. + pub requirement: VersionReq, + /// The package version that was resolved. + pub version: Version, + /// The digest of the package contents. + pub digest: ContentDigest, + /// The client to use for fetching the package contents. + client: CachingClient, +} + +impl Debug for RegistryResolution { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("RegistryResolution") + .field("name", &self.name) + .field("package", &self.package) + .field("registry", &self.registry) + .field("requirement", &self.requirement) + .field("version", &self.version) + .field("digest", &self.digest) + .finish() + } +} + +impl RegistryResolution { + /// Fetches the raw package bytes from the registry. Returns an AsyncRead that will stream the + /// package contents + pub async fn fetch(&self) -> Result { + let stream = self + .client + .get_content( + &self.package, + &Release { + version: self.version.clone(), + content_digest: self.digest.clone(), + }, + ) + .await?; + + Ok(tokio_util::io::StreamReader::new(stream.map_err(|e| { + std::io::Error::new(std::io::ErrorKind::Other, e) + }))) + } +} + +/// Represents information about a resolution of a local file. +#[derive(Clone, Debug)] +pub struct LocalResolution { + /// The name of the dependency that was resolved. + pub name: PackageRef, + /// The path to the resolved dependency. + pub path: PathBuf, +} + +/// Represents a resolution of a dependency. +#[derive(Debug, Clone)] +#[allow(clippy::large_enum_variant)] +pub enum DependencyResolution { + /// The dependency is resolved from a registry package. + Registry(RegistryResolution), + /// The dependency is resolved from a local path. + Local(LocalResolution), +} + +impl DependencyResolution { + /// Gets the name of the dependency that was resolved. + pub fn name(&self) -> &PackageRef { + match self { + Self::Registry(res) => &res.name, + Self::Local(res) => &res.name, + } + } + + /// Gets the resolved version. + /// + /// Returns `None` if the dependency is not resolved from a registry package. + pub fn version(&self) -> Option<&Version> { + match self { + Self::Registry(res) => Some(&res.version), + Self::Local(_) => None, + } + } + + /// The key used in sorting and searching the lock file package list. + /// + /// Returns `None` if the dependency is not resolved from a registry package. + pub fn key(&self) -> Option<(&PackageRef, Option<&str>)> { + match self { + DependencyResolution::Registry(pkg) => Some((&pkg.package, pkg.registry.as_deref())), + DependencyResolution::Local(_) => None, + } + } + + /// Decodes the resolved dependency. + pub async fn decode(&self) -> Result { + // If the dependency path is a directory, assume it contains wit to parse as a package. + let bytes = match self { + DependencyResolution::Local(LocalResolution { path, .. }) + if tokio::fs::metadata(path).await?.is_dir() => + { + return Ok(DecodedDependency::Wit { + resolution: self, + package: UnresolvedPackageGroup::parse_dir(path).with_context(|| { + format!("failed to parse dependency `{path}`", path = path.display()) + })?, + }); + } + DependencyResolution::Local(LocalResolution { path, .. }) => { + tokio::fs::read(path).await.with_context(|| { + format!( + "failed to read content of dependency `{name}` at path `{path}`", + name = self.name(), + path = path.display() + ) + })? + } + DependencyResolution::Registry(res) => { + let mut reader = res.fetch().await?; + + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await?; + buf + } + }; + + if &bytes[0..4] != b"\0asm" { + return Ok(DecodedDependency::Wit { + resolution: self, + package: UnresolvedPackageGroup::parse( + // This is fake, but it's needed for the parser to work. + self.name().to_string(), + std::str::from_utf8(&bytes).with_context(|| { + format!( + "dependency `{name}` is not UTF-8 encoded", + name = self.name() + ) + })?, + )?, + }); + } + + Ok(DecodedDependency::Wasm { + resolution: self, + decoded: wit_component::decode(&bytes).with_context(|| { + format!( + "failed to decode content of dependency `{name}`", + name = self.name(), + ) + })?, + }) + } +} + +/// Represents a decoded dependency. +pub enum DecodedDependency<'a> { + /// The dependency decoded from an unresolved WIT package. + Wit { + /// The resolution related to the decoded dependency. + resolution: &'a DependencyResolution, + /// The unresolved WIT package. + package: UnresolvedPackageGroup, + }, + /// The dependency decoded from a Wasm file. + Wasm { + /// The resolution related to the decoded dependency. + resolution: &'a DependencyResolution, + /// The decoded Wasm file. + decoded: DecodedWasm, + }, +} + +impl<'a> DecodedDependency<'a> { + /// Fully resolves the dependency. + /// + /// If the dependency is an unresolved WIT package, it will assume that the + /// package has no foreign dependencies. + pub fn resolve(self) -> Result<(Resolve, PackageId, Vec)> { + match self { + Self::Wit { package, .. } => { + let mut resolve = Resolve::new(); + let source_files = package + .source_map + .source_files() + .map(Path::to_path_buf) + .collect(); + let pkg = resolve.push_group(package)?; + Ok((resolve, pkg, source_files)) + } + Self::Wasm { decoded, .. } => match decoded { + DecodedWasm::WitPackage(resolve, pkg) => Ok((resolve, pkg, Vec::new())), + DecodedWasm::Component(resolve, world) => { + let pkg = resolve.worlds[world].package.unwrap(); + Ok((resolve, pkg, Vec::new())) + } + }, + } + } + + /// Gets the package name of the decoded dependency. + pub fn package_name(&self) -> &PackageName { + match self { + Self::Wit { package, .. } => &package.main.name, + Self::Wasm { decoded, .. } => &decoded.resolve().packages[decoded.package()].name, + } + } + + /// Converts the decoded dependency into a component world. + /// + /// Returns an error if the dependency is not a decoded component. + pub fn into_component_world(self) -> Result<(Resolve, WorldId)> { + match self { + Self::Wasm { + decoded: DecodedWasm::Component(resolve, world), + .. + } => Ok((resolve, world)), + _ => bail!("dependency is not a WebAssembly component"), + } + } +} + +/// Used to resolve dependencies for a WIT package. +pub struct DependencyResolver<'a> { + client: CachingClient, + lock_file: Option<&'a LockFile>, + packages: HashMap>, + dependencies: HashMap, + resolutions: DependencyResolutionMap, +} + +impl<'a> DependencyResolver<'a> { + /// Creates a new dependency resolver. If `config` is `None`, then the resolver will be set to + /// offline mode and a lock file must be given as well. Anything that will require network + /// access will fail in offline mode. + pub fn new( + config: Option, + lock_file: Option<&'a LockFile>, + cache: FileCache, + ) -> anyhow::Result { + if config.is_none() && lock_file.is_none() { + anyhow::bail!("lock file must be provided when offline mode is enabled"); + } + let client = CachingClient::new(config.map(Client::new), cache); + Ok(DependencyResolver { + client, + lock_file, + resolutions: Default::default(), + packages: Default::default(), + dependencies: Default::default(), + }) + } + + /// Creates a new dependency resolver with the given client. This is useful when you already + /// have a client available. If the client is set to offline mode, then a lock file must be + /// given or this will error + pub fn new_with_client( + client: CachingClient, + lock_file: Option<&'a LockFile>, + ) -> anyhow::Result { + if client.is_readonly() && lock_file.is_none() { + anyhow::bail!("lock file must be provided when offline mode is enabled"); + } + Ok(DependencyResolver { + client, + lock_file, + resolutions: Default::default(), + packages: Default::default(), + dependencies: Default::default(), + }) + } + + /// Add a dependency to the resolver. If the dependency already exists, then it will be ignored. + /// To override an existing dependency, use [`override_dependency`](Self::override_dependency). + pub async fn add_dependency( + &mut self, + name: &PackageRef, + dependency: &Dependency, + ) -> Result<()> { + self.add_dependency_internal(name, dependency, false).await + } + + /// Add a dependency to the resolver. If the dependency already exists, then it will be + /// overridden. + pub async fn override_dependency( + &mut self, + name: &PackageRef, + dependency: &Dependency, + ) -> Result<()> { + self.add_dependency_internal(name, dependency, true).await + } + + async fn add_dependency_internal( + &mut self, + name: &PackageRef, + dependency: &Dependency, + force_override: bool, + ) -> Result<()> { + match dependency { + Dependency::Package(package) => { + // Dependency comes from a registry, add a dependency to the resolver + let registry_name = package.registry.as_deref().or_else(|| { + self.client.client().ok().and_then(|client| { + client + .config() + .resolve_registry(name) + .map(|reg| reg.as_ref()) + }) + }); + let package_name = package.name.clone().unwrap_or_else(|| name.clone()); + + // Resolve the version from the lock file if there is one + let locked = match self.lock_file.as_ref().and_then(|resolver| { + resolver + .resolve(registry_name, &package_name, &package.version) + .transpose() + }) { + Some(Ok(locked)) => Some(locked), + Some(Err(e)) => return Err(e), + _ => None, + }; + + if !force_override && self.resolutions.contains_key(name) { + tracing::debug!(%name, "dependency already exists and override is not set, ignoring"); + return Ok(()); + } + self.dependencies.insert( + name.to_owned(), + RegistryDependency { + package: package_name, + version: package.version.clone(), + locked: locked.map(|l| (l.version.clone(), l.digest.clone())), + }, + ); + } + Dependency::Local(p) => { + // A local path dependency, insert a resolution immediately + let res = DependencyResolution::Local(LocalResolution { + name: name.clone(), + path: p.clone(), + }); + + if !force_override && self.resolutions.contains_key(name) { + tracing::debug!(%name, "dependency already exists and override is not set, ignoring"); + return Ok(()); + } + + let prev = self.resolutions.insert(name.clone(), res); + assert!(prev.is_none()); + } + } + + Ok(()) + } + + /// Resolve all dependencies. + /// + /// This will download all dependencies that are not already present in client storage. + /// + /// Returns the dependency resolution map. + pub async fn resolve(mut self) -> Result { + let mut resolutions = self.resolutions; + for (name, dependency) in self.dependencies.into_iter() { + // We need to clone a handle to the client because we mutably borrow self below. Might + // be worth replacing the mutable borrow with a RwLock down the line. + let client = self.client.clone(); + + let (selected_version, digest) = if client.is_readonly() { + dependency + .locked + .as_ref() + .map(|(ver, digest)| (ver, Some(digest))) + .ok_or_else(|| { + anyhow::anyhow!("Couldn't find locked dependency while in offline mode") + })? + } else { + let versions = + load_package(&mut self.packages, &self.client, dependency.package.clone()) + .await? + .with_context(|| { + format!( + "package `{name}` was not found in component registry", + name = dependency.package + ) + })?; + + match &dependency.locked { + Some((version, digest)) => { + // The dependency had a lock file entry, so attempt to do an exact match first + let exact_req = VersionReq { + comparators: vec![Comparator { + op: Op::Exact, + major: version.major, + minor: Some(version.minor), + patch: Some(version.patch), + pre: version.pre.clone(), + }], + }; + + // If an exact match can't be found, fallback to the latest release to satisfy + // the version requirement; this can happen when packages are yanked. If we did + // find an exact match, return the digest for comparison after fetching the + // release + find_latest_release(versions, &exact_req).map(|v| (&v.version, Some(digest))).or_else(|| find_latest_release(versions, &dependency.version).map(|v| (&v.version, None))) + } + None => find_latest_release(versions, &dependency.version).map(|v| (&v.version, None)), + }.with_context(|| format!("component registry package `{name}` has no release matching version requirement `{version}`", name = dependency.package, version = dependency.version))? + }; + + // We need to clone a handle to the client because we mutably borrow self above. Might + // be worth replacing the mutable borrow with a RwLock down the line. + let release = client + .get_release(&dependency.package, selected_version) + .await?; + if let Some(digest) = digest { + if &release.content_digest != digest { + bail!( + "component registry package `{name}` (v`{version}`) has digest `{content}` but the lock file specifies digest `{digest}`", + name = dependency.package, + version = release.version, + content = release.content_digest, + ); + } + } + let resolution = RegistryResolution { + name: name.clone(), + package: dependency.package.clone(), + registry: self.client.client().ok().and_then(|client| { + client + .config() + .resolve_registry(&name) + .map(ToString::to_string) + }), + requirement: dependency.version.clone(), + version: release.version.clone(), + digest: release.content_digest.clone(), + client: self.client.clone(), + }; + resolutions.insert(name, DependencyResolution::Registry(resolution)); + } + + Ok(resolutions) + } +} + +async fn load_package<'b>( + packages: &'b mut HashMap>, + client: &CachingClient, + package: PackageRef, +) -> Result>> { + match packages.entry(package) { + hash_map::Entry::Occupied(e) => Ok(Some(e.into_mut())), + hash_map::Entry::Vacant(e) => match client.list_all_versions(e.key()).await { + Ok(p) => Ok(Some(e.insert(p))), + Err(WasmPkgError::PackageNotFound) => Ok(None), + Err(err) => Err(err.into()), + }, + } +} + +struct RegistryDependency { + /// The canonical package name of the registry package. In most cases, this is the same as the + /// name but could be different if the given package name has been remapped + package: PackageRef, + version: VersionReq, + locked: Option<(Version, ContentDigest)>, +} + +fn find_latest_release<'a>( + versions: &'a [VersionInfo], + req: &VersionReq, +) -> Option<&'a VersionInfo> { + versions + .iter() + .filter(|info| !info.yanked && req.matches(&info.version)) + .max_by(|a, b| a.version.cmp(&b.version)) +} + +// NOTE(thomastaylor312): This is copied from the old wit package in the cargo-component and broken +// out for some reuse. I don't know enough about resolvers to know if there is an easier way to +// write this, so any future people seeing this should feel free to refactor it if they know a +// better way to do it. + +/// Represents a map of dependency resolutions. +/// +/// The key to the map is the package name of the dependency. +#[derive(Debug, Clone, Default)] +pub struct DependencyResolutionMap(HashMap); + +impl AsRef> for DependencyResolutionMap { + fn as_ref(&self) -> &HashMap { + &self.0 + } +} + +impl Deref for DependencyResolutionMap { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for DependencyResolutionMap { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl DependencyResolutionMap { + /// Fetch all dependencies and ensure there are no circular dependencies. Returns the decoded + /// dependencies (sorted topologically), ready to use for output or adding to a [`Resolve`]. + pub async fn decode_dependencies( + &self, + ) -> Result>> { + // Start by decoding all of the dependencies + let mut deps = IndexMap::new(); + for (name, resolution) in self.0.iter() { + let decoded = resolution.decode().await?; + if let Some(prev) = deps.insert(decoded.package_name().clone(), decoded) { + anyhow::bail!( + "duplicate definitions of package `{prev}` found while decoding dependency `{name}`", + prev = prev.package_name() + ); + } + } + + // Do a topological sort of the dependencies + let mut order = IndexSet::new(); + let mut visiting = HashSet::new(); + for dep in deps.values() { + visit(dep, &deps, &mut order, &mut visiting)?; + } + + assert!(visiting.is_empty()); + + // Now that we have the right order, re-order the dependencies to match + deps.sort_by(|name_a, _, name_b, _| { + order.get_index_of(name_a).cmp(&order.get_index_of(name_b)) + }); + + Ok(deps) + } + + /// Given a path to a component or a directory containing wit, use the given dependencies to + /// generate a [`Resolve`] for the root package. + pub async fn generate_resolve(&self, dir: impl AsRef) -> Result<(Resolve, PackageId)> { + let mut merged = Resolve::default(); + + let deps = self.decode_dependencies().await?; + + // Parse the root package itself + let root = UnresolvedPackageGroup::parse_dir(&dir).with_context(|| { + format!( + "failed to parse package from directory `{dir}`", + dir = dir.as_ref().display() + ) + })?; + + let mut source_files: Vec<_> = root + .source_map + .source_files() + .map(Path::to_path_buf) + .collect(); + + // Merge all of the dependencies first + for decoded in deps.into_values() { + match decoded { + DecodedDependency::Wit { + resolution, + package, + } => { + source_files.extend(package.source_map.source_files().map(Path::to_path_buf)); + merged.push_group(package).with_context(|| { + format!( + "failed to merge dependency `{name}`", + name = resolution.name() + ) + })?; + } + DecodedDependency::Wasm { + resolution, + decoded, + } => { + let resolve = match decoded { + DecodedWasm::WitPackage(resolve, _) => resolve, + DecodedWasm::Component(resolve, _) => resolve, + }; + + merged.merge(resolve).with_context(|| { + format!( + "failed to merge world of dependency `{name}`", + name = resolution.name() + ) + })?; + } + }; + } + + let package = merged.push_group(root).with_context(|| { + format!( + "failed to merge package from directory `{dir}`", + dir = dir.as_ref().display() + ) + })?; + + Ok((merged, package)) + } +} + +fn visit<'a>( + dep: &'a DecodedDependency<'a>, + deps: &'a IndexMap, + order: &mut IndexSet, + visiting: &mut HashSet<&'a PackageName>, +) -> Result<()> { + if order.contains(dep.package_name()) { + return Ok(()); + } + + // Visit any unresolved foreign dependencies + match dep { + DecodedDependency::Wit { + package, + resolution, + } => { + for name in package.main.foreign_deps.keys() { + // Only visit known dependencies + // wit-parser will error on unknown foreign dependencies when + // the package is resolved + if let Some(dep) = deps.get(name) { + if !visiting.insert(name) { + anyhow::bail!("foreign dependency `{name}` forms a dependency cycle while parsing dependency `{other}`", other = resolution.name()); + } + + visit(dep, deps, order, visiting)?; + assert!(visiting.remove(name)); + } + } + } + DecodedDependency::Wasm { + decoded, + resolution, + } => { + // Look for foreign packages in the decoded dependency + for (_, package) in &decoded.resolve().packages { + if package.name.namespace == dep.package_name().namespace + && package.name.name == dep.package_name().name + { + continue; + } + + if let Some(dep) = deps.get(&package.name) { + if !visiting.insert(&package.name) { + anyhow::bail!("foreign dependency `{name}` forms a dependency cycle while parsing dependency `{other}`", name = package.name, other = resolution.name()); + } + + visit(dep, deps, order, visiting)?; + assert!(visiting.remove(&package.name)); + } + } + } + } + + assert!(order.insert(dep.package_name().clone())); + + Ok(()) +} diff --git a/crates/wkg-core/src/wit.rs b/crates/wkg-core/src/wit.rs new file mode 100644 index 0000000..0238826 --- /dev/null +++ b/crates/wkg-core/src/wit.rs @@ -0,0 +1,402 @@ +//! Functions for building WIT packages and fetching their dependencies. + +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + str::FromStr, +}; + +use anyhow::{Context, Result}; +use semver::{Version, VersionReq}; +use wasm_pkg_client::{ + caching::{CachingClient, FileCache}, + PackageRef, +}; +use wit_component::{DecodedWasm, WitPrinter}; +use wit_parser::{PackageId, PackageName, Resolve}; + +use crate::{ + config::Config, + lock::LockFile, + resolver::{ + DecodedDependency, Dependency, DependencyResolution, DependencyResolutionMap, + DependencyResolver, LocalResolution, RegistryPackage, + }, +}; + +/// The supported output types for WIT deps +#[derive(Debug, Clone, Copy, Default)] +pub enum OutputType { + /// Output each dependency as a WIT file in the deps directory. + Wit, + /// Output each dependency as a wasm binary file in the deps directory. + #[default] + Wasm, +} + +impl FromStr for OutputType { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result { + let lower_trim = s.trim().to_lowercase(); + match lower_trim.as_str() { + "wit" => Ok(Self::Wit), + "wasm" => Ok(Self::Wasm), + _ => Err(anyhow::anyhow!("Invalid output type: {}", s)), + } + } +} + +/// Builds a WIT package given the configuration and directory to parse. Will update the given lock +/// file with the resolved dependencies but will not write it to disk. +pub async fn build_package( + config: &Config, + wit_dir: impl AsRef, + lock_file: &mut LockFile, + client: CachingClient, +) -> Result<(PackageRef, Option, Vec)> { + let dependencies = resolve_dependencies(config, &wit_dir, Some(lock_file), client) + .await + .context("Unable to resolve dependencies")?; + + lock_file.update_dependencies(&dependencies); + + let (resolve, pkg_id) = dependencies.generate_resolve(wit_dir).await?; + + let pkg = &resolve.packages[pkg_id]; + let name = PackageRef::new( + pkg.name + .namespace + .parse() + .context("Invalid namespace found in package")?, + pkg.name + .name + .parse() + .context("Invalid name found for package")?, + ); + + let bytes = wit_component::encode(Some(true), &resolve, pkg_id)?; + + let mut producers = wasm_metadata::Producers::empty(); + producers.add( + "processed-by", + env!("CARGO_PKG_NAME"), + option_env!("WIT_VERSION_INFO").unwrap_or(env!("CARGO_PKG_VERSION")), + ); + + let mut bytes = producers + .add_to_wasm(&bytes) + .context("failed to add producers metadata to output WIT package")?; + + if let Some(meta) = config.metadata.clone() { + let meta = wasm_metadata::RegistryMetadata::from(meta); + bytes = meta.add_to_wasm(&bytes)?; + } + + Ok((name, pkg.name.version.clone(), bytes)) +} + +/// Fetches and optionally updates all dependencies for the given path and writes them in the +/// specified format. The lock file will be updated with the resolved dependencies but will not be +/// written to disk. +/// +/// This is mostly a convenience wrapper around [`resolve_dependencies`] and [`populate_dependencies`]. +pub async fn fetch_dependencies( + config: &Config, + wit_dir: impl AsRef, + lock_file: &mut LockFile, + client: CachingClient, + output: OutputType, +) -> Result<()> { + // Don't pass lock file if update is true + let dependencies = resolve_dependencies(config, &wit_dir, Some(lock_file), client).await?; + lock_file.update_dependencies(&dependencies); + populate_dependencies(wit_dir, &dependencies, output).await +} + +/// Generate the list of all packages and their version requirement from the given path (a directory +/// or file). +/// +/// This is a lower level function exposed for convenience that is used by higher level functions +/// for resolving dependencies. +pub fn get_packages( + path: impl AsRef, +) -> Result<(PackageRef, HashSet<(PackageRef, VersionReq)>)> { + let group = wit_parser::UnresolvedPackageGroup::parse_path(path)?; + + let name = PackageRef::new( + group + .main + .name + .namespace + .parse() + .context("Invalid namespace found in package")?, + group + .main + .name + .name + .parse() + .context("Invalid name found in package")?, + ); + + // Get all package refs from the main package and then from any nested packages + let packages: HashSet<(PackageRef, VersionReq)> = + packages_from_foreign_deps(group.main.foreign_deps.into_keys()) + .chain( + group + .nested + .into_iter() + .flat_map(|pkg| packages_from_foreign_deps(pkg.foreign_deps.into_keys())), + ) + .collect(); + + Ok((name, packages)) +} + +/// Builds a list of resolved dependencies loaded from the component or path containing the WIT. +/// This will configure the resolver, override any dependencies from configuration and resolve the +/// dependency map. This map can then be used in various other functions for fetching the +/// dependencies and/or building a final resolved package. +pub async fn resolve_dependencies( + config: &Config, + path: impl AsRef, + lock_file: Option<&LockFile>, + client: CachingClient, +) -> Result { + let mut resolver = DependencyResolver::new_with_client(client, lock_file)?; + // add deps from config first in case they're local deps and then add deps from the directory + if let Some(overrides) = config.overrides.as_ref() { + for (pkg, ovride) in overrides.iter() { + let pkg: PackageRef = pkg.parse().context("Unable to parse as a package ref")?; + let dep = match (ovride.path.as_ref(), ovride.version.as_ref()) { + (Some(path), None) => { + let path = tokio::fs::canonicalize(path).await?; + Dependency::Local(path) + } + (Some(path), Some(_)) => { + tracing::warn!("Ignoring version override for local package"); + let path = tokio::fs::canonicalize(path).await?; + Dependency::Local(path) + } + (None, Some(version)) => Dependency::Package(RegistryPackage { + name: Some(pkg.clone()), + version: version.to_owned(), + registry: None, + }), + (None, None) => { + tracing::warn!("Found override without version or path, ignoring"); + continue; + } + }; + resolver + .add_dependency(&pkg, &dep) + .await + .context("Unable to add dependency")?; + } + } + let (_name, packages) = get_packages(path)?; + add_packages_to_resolver(&mut resolver, packages).await?; + resolver.resolve().await +} + +/// Populate a list of dependencies into the given directory. If the directory does not exist it +/// will be created. Any existing files in the directory will be deleted. The dependencies will be +/// put into the `deps` subdirectory within the directory in the format specified by the output +/// type. Please note that if a local dep is encountered when using [`OutputType::Wasm`] and it +/// isn't a wasm binary, it will be copied directly to the directory and not packaged into a wit +/// package first +pub async fn populate_dependencies( + path: impl AsRef, + deps: &DependencyResolutionMap, + output: OutputType, +) -> Result<()> { + // Canonicalizing will error if the path doesn't exist, so we don't need to check for that + let path = tokio::fs::canonicalize(path).await?; + let metadata = tokio::fs::metadata(&path).await?; + if !metadata.is_dir() { + anyhow::bail!("Path is not a directory"); + } + let deps_path = path.join("deps"); + // Remove the whole directory if it already exists and then recreate + if let Err(e) = tokio::fs::remove_dir_all(&deps_path).await { + // If the directory doesn't exist, ignore the error + if e.kind() != std::io::ErrorKind::NotFound { + return Err(anyhow::anyhow!("Unable to remove deps directory: {e}")); + } + } + tokio::fs::create_dir_all(&deps_path).await?; + + let decoded_deps = deps.decode_dependencies().await?; + + for (name, dep) in decoded_deps.iter() { + let mut output_path = deps_path.join(name.to_string()); + + match (dep, output) { + ( + DecodedDependency::Wit { + resolution: DependencyResolution::Local(local), + .. + }, + _, + ) => { + // Local deps always need to be written to a subdirectory of deps so create that here + tokio::fs::create_dir_all(&output_path).await?; + write_local_dep(local, output_path).await?; + } + // This case shouldn't happen because registries only support wit packages. We can't get + // a resolve from the unresolved group, so error out here. Ideally we could print the + // unresolved group, but WitPrinter doesn't support that yet + ( + DecodedDependency::Wit { + resolution: DependencyResolution::Registry(_), + .. + }, + _, + ) => { + anyhow::bail!("Unable to resolve dependency, this is a programmer error"); + } + (DecodedDependency::Wasm { decoded, .. }, OutputType::Wit) => { + tokio::fs::create_dir_all(&output_path).await?; + let (resolve, pkg) = match decoded { + DecodedWasm::WitPackage(r, p) => (r, *p), + DecodedWasm::Component(r, world_id) => { + let pkg_id = r + .worlds + .iter() + .find_map(|(id, w)| { + (id == *world_id).then_some(w).and_then(|w| w.package) + }) + .ok_or_else(|| { + anyhow::anyhow!("Unable to find package for the component's world") + })?; + (r, pkg_id) + } + }; + // Print all the deps + print_wit_deps(resolve, pkg, name, &deps_path).await?; + } + // Right now WIT packages include all of their dependencies, so we don't need to fetch + // those too. In the future, we'll need to look for unsatisfied dependencies and fetch + // them + (DecodedDependency::Wasm { resolution, .. }, OutputType::Wasm) => { + // This is going to be written to a single file, so we don't create a directory here + // NOTE(thomastaylor312): This janky looking thing is to avoid chopping off the + // patch number from the release. Once `add_extension` is stabilized, we can use + // that instead + let mut file_name = output_path.file_name().unwrap().to_owned(); + file_name.push(".wasm"); + output_path.set_file_name(file_name); + match resolution { + DependencyResolution::Local(local) => { + let meta = tokio::fs::metadata(&local.path).await?; + if !meta.is_file() { + anyhow::bail!("Local dependency is not single wit package file"); + } + tokio::fs::copy(&local.path, output_path) + .await + .context("Unable to copy local dependency")?; + } + DependencyResolution::Registry(registry) => { + let mut reader = registry.fetch().await?; + let mut output_file = tokio::fs::File::create(output_path).await?; + tokio::io::copy(&mut reader, &mut output_file).await?; + output_file.sync_all().await?; + } + } + } + } + } + Ok(()) +} + +fn packages_from_foreign_deps( + deps: impl IntoIterator, +) -> impl Iterator { + deps.into_iter().filter_map(|dep| { + let name = PackageRef::new(dep.namespace.parse().ok()?, dep.name.parse().ok()?); + let version = match dep.version { + Some(v) => format!("={v}"), + None => "*".to_string(), + }; + Some(( + name, + version + .parse() + .expect("Unable to parse into version request, this is programmer error"), + )) + }) +} + +async fn add_packages_to_resolver( + resolver: &mut DependencyResolver<'_>, + packages: impl IntoIterator, +) -> Result<()> { + for (package, req) in packages { + resolver + .add_dependency( + &package, + &Dependency::Package(RegistryPackage { + name: Some(package.clone()), + version: req, + registry: None, + }), + ) + .await?; + } + Ok(()) +} + +async fn write_local_dep(local: &LocalResolution, output_path: impl AsRef) -> Result<()> { + let meta = tokio::fs::metadata(&local.path).await?; + if meta.is_file() { + tokio::fs::copy( + &local.path, + output_path.as_ref().join(local.path.file_name().unwrap()), + ) + .await?; + } else { + // For now, don't try to recurse, since most of the tools don't recurse unless + // there is a deps folder anyway, which we don't care about here + let mut dir = tokio::fs::read_dir(&local.path).await?; + while let Some(entry) = dir.next_entry().await? { + if !entry.metadata().await?.is_file() { + continue; + } + let entry_path = entry.path(); + tokio::fs::copy( + &entry_path, + output_path.as_ref().join(entry_path.file_name().unwrap()), + ) + .await?; + } + } + Ok(()) +} + +/// Recursive function that prints the top level dep given and then iterates over each of its dependencies +async fn print_wit_deps( + resolve: &Resolve, + pkg_id: PackageId, + top_level_name: &PackageName, + root_deps_dir: &PathBuf, +) -> Result<()> { + // Print the top level package first, then iterate over all dependencies and print them + let dep_path = root_deps_dir.join(top_level_name.to_string()); + tokio::fs::create_dir_all(&dep_path).await?; + let mut printer = WitPrinter::default(); + let wit = printer + .print(resolve, pkg_id, &[]) + .context("Unable to print wit")?; + tokio::fs::write(dep_path.join("package.wit"), wit).await?; + for pkg in resolve.package_direct_deps(pkg_id) { + // Get the package name + let package_name = resolve + .package_names + .iter() + .find_map(|(name, id)| (*id == pkg).then_some(name)) + .ok_or_else(|| anyhow::anyhow!("Got package ID from world that didn't exist"))?; + // Recurse into this package + Box::pin(print_wit_deps(resolve, pkg, package_name, root_deps_dir)).await?; + } + Ok(()) +} diff --git a/crates/wkg/Cargo.toml b/crates/wkg/Cargo.toml index 18587cb..8da7fe2 100644 --- a/crates/wkg/Cargo.toml +++ b/crates/wkg/Cargo.toml @@ -11,6 +11,7 @@ readme = "../../README.md" [dependencies] anyhow = { workspace = true } clap = { version = "4.5", features = ["derive", "wrap_help", "env"] } +dirs = { workspace = true } docker_credential = { workspace = true } futures-util = { workspace = true, features = ["io"] } oci-client = { workspace = true } @@ -22,6 +23,7 @@ tracing-subscriber = { workspace = true } wasm-pkg-common = { workspace = true } wasm-pkg-client = { workspace = true } wit-component = { workspace = true } +wkg-core = { workspace = true } [dev-dependencies] base64 = { workspace = true } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 101ec30..c0fa071 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -5,13 +5,18 @@ use clap::{Args, Parser, Subcommand, ValueEnum}; use futures_util::TryStreamExt; use tokio::io::AsyncWriteExt; use tracing::level_filters::LevelFilter; -use wasm_pkg_client::{Client, PublishOpts}; +use wasm_pkg_client::{ + caching::{CachingClient, FileCache}, + Client, PublishOpts, +}; use wasm_pkg_common::{config::Config, package::PackageSpec, registry::Registry}; use wit_component::DecodedWasm; mod oci; +mod wit; use oci::OciCommands; +use wit::WitCommands; #[derive(Parser, Debug)] #[command(version)] @@ -32,6 +37,43 @@ struct Common { /// The path to the configuration file. #[arg(long = "config", value_name = "CONFIG", env = "WKG_CONFIG_FILE")] config: Option, + /// The path to the cache directory. Defaults to the system cache directory. + #[arg(long = "cache", value_name = "CACHE", env = "WKG_CACHE_DIR")] + cache: Option, +} + +impl Common { + /// Helper to load the config from the given path + pub async fn load_config(&self) -> anyhow::Result { + if let Some(config_file) = self.config.as_ref() { + Config::from_file(config_file) + .await + .context(format!("error loading config file {config_file:?}")) + } else { + Config::global_defaults().await.map_err(anyhow::Error::from) + } + } + + /// Helper for loading the [`FileCache`] + pub async fn load_cache(&self) -> anyhow::Result { + let dir = if let Some(dir) = self.cache.as_ref() { + dir.clone() + } else { + dirs::cache_dir().context("unable to find cache directory")? + }; + let dir = dir.join("wkg"); + FileCache::new(dir).await + } + + /// Helper for loading a caching client. This should be the most commonly used method for + /// loading a client, but if you need to modify the config or use your own cache, you can use + /// the [`Common::load_config`] and [`Common::load_cache`] methods. + pub async fn get_client(&self) -> anyhow::Result> { + let config = self.load_config().await?; + let cache = self.load_cache().await?; + let client = Client::new(config); + Ok(CachingClient::new(Some(client), cache)) + } } #[derive(Subcommand, Debug)] @@ -44,6 +86,9 @@ enum Commands { /// Commands for interacting with OCI registries #[clap(subcommand)] Oci(OciCommands), + /// Commands for interacting with WIT files and dependencies + #[clap(subcommand)] + Wit(WitCommands), } #[derive(Args, Debug)] @@ -86,14 +131,14 @@ struct PublishArgs { /// Expected format: `:@` #[arg(long, env = "WKG_PACKAGE")] package: Option, + + #[command(flatten)] + common: Common, } impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { - let client = { - let config = Config::global_defaults()?; - Client::new(config) - }; + let client = self.common.get_client().await?; let package = if let Some(package) = self.package { Some(( @@ -106,6 +151,7 @@ impl PublishArgs { None }; let (package, version) = client + .client()? .publish_release_file( &self.file, PublishOpts { @@ -129,20 +175,14 @@ enum Format { impl GetArgs { pub async fn run(self) -> anyhow::Result<()> { let PackageSpec { package, version } = self.package_spec; - - let client = { - let mut config = if let Some(config_file) = self.common.config { - Config::from_file(&config_file) - .context(format!("error loading config file {config_file:?}"))? - } else { - Config::global_defaults()? - }; - if let Some(registry) = self.registry_args.registry.clone() { - tracing::debug!(%package, %registry, "overriding package registry"); - config.set_package_registry_override(package.clone(), registry); - } - Client::new(config) - }; + let mut config = self.common.load_config().await?; + if let Some(registry) = self.registry_args.registry.clone() { + tracing::debug!(%package, %registry, "overriding package registry"); + config.set_package_registry_override(package.clone(), registry); + } + let client = Client::new(config); + let cache = self.common.load_cache().await?; + let client = CachingClient::new(Some(client), cache); let version = match version { Some(ver) => ver, @@ -178,7 +218,7 @@ impl GetArgs { tempfile::NamedTempFile::with_prefix_in(".wkg-get", parent_dir)?.into_parts(); tracing::debug!(?tmp_path, "Created temporary file"); - let mut content_stream = client.stream_content(&package, &release).await?; + let mut content_stream = client.get_content(&package, &release).await?; let mut file = tokio::fs::File::from_std(tmp_file); while let Some(chunk) = content_stream.try_next().await? { @@ -268,5 +308,6 @@ async fn main() -> anyhow::Result<()> { Commands::Get(args) => args.run().await, Commands::Publish(args) => args.run().await, Commands::Oci(args) => args.run().await, + Commands::Wit(args) => args.run().await, } } diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs new file mode 100644 index 0000000..31689a6 --- /dev/null +++ b/crates/wkg/src/wit.rs @@ -0,0 +1,150 @@ +//! Args and commands for interacting with WIT files and dependencies +use std::path::PathBuf; + +use clap::{Args, Subcommand}; +use wkg_core::{ + lock::LockFile, + wit::{self, OutputType}, +}; + +use crate::Common; + +/// Commands for interacting with wit +#[derive(Debug, Subcommand)] +pub enum WitCommands { + /// Build a WIT package from a directory. By default, this will fetch all dependencies needed + /// and encode them in the WIT package. This will generate a lock file that can be used to fetch + /// the dependencies in the future. + Build(BuildArgs), + /// Fetch dependencies for a component. This will read the package containing the world(s) you + /// have defined in the given wit directory (`wit` by default). It will then fetch the + /// dependencies and write them to the `deps` directory along with a lock file. If no lock file + /// exists, it will fetch all dependencies. If a lock file exists, it will fetch any + /// dependencies that are not in the lock file and update the lock file. To update the lock + /// file, use the `update` command. + Fetch(FetchArgs), + /// Update the lock file with the latest dependencies. This will update all dependencies and + /// generate a new lock file. + Update(UpdateArgs), +} + +impl WitCommands { + pub async fn run(self) -> anyhow::Result<()> { + match self { + WitCommands::Build(args) => args.run().await, + WitCommands::Fetch(args) => args.run().await, + WitCommands::Update(args) => args.run().await, + } + } +} + +#[derive(Debug, Args)] +pub struct BuildArgs { + /// The directory containing the WIT files to build. + #[clap(short = 'd', long = "wit-dir", default_value = "wit")] + pub dir: PathBuf, + + /// The name of the file that should be written. This can also be a full path. Defaults to the + /// current directory with the name of the package + #[clap(short = 'o', long = "output")] + pub output: Option, + + #[clap(flatten)] + pub common: Common, +} + +#[derive(Debug, Args)] +pub struct FetchArgs { + /// The directory containing the WIT files to fetch dependencies for. + #[clap(short = 'd', long = "wit-dir", default_value = "wit")] + pub dir: PathBuf, + + /// The desired output type of the dependencies. Valid options are "wit" or "wasm" (wasm is the + /// WIT package binary format). + #[clap(short = 't', long = "type")] + pub output_type: Option, + + #[clap(flatten)] + pub common: Common, +} + +#[derive(Debug, Args)] +pub struct UpdateArgs { + /// The directory containing the WIT files to update dependencies for. + #[clap(short = 'd', long = "wit-dir", default_value = "wit")] + pub dir: PathBuf, + + /// The desired output type of the dependencies. Valid options are "wit" or "wasm" (wasm is the + /// WIT package binary format). + #[clap(short = 't', long = "type")] + pub output_type: Option, + + #[clap(flatten)] + pub common: Common, +} + +impl BuildArgs { + pub async fn run(self) -> anyhow::Result<()> { + let client = self.common.get_client().await?; + let wkg_config = wkg_core::config::Config::load().await?; + let mut lock_file = LockFile::load(false).await?; + let (pkg_ref, version, bytes) = + wit::build_package(&wkg_config, self.dir, &mut lock_file, client).await?; + let output_path = if let Some(path) = self.output { + path + } else { + let mut file_name = pkg_ref.to_string(); + if let Some(version) = version { + file_name.push_str(&format!("@{version}")); + } + file_name.push_str(".wasm"); + PathBuf::from(file_name) + }; + + tokio::fs::write(&output_path, bytes).await?; + // Now write out the lock file since everything else succeeded + lock_file.write().await?; + println!("WIT package written to {}", output_path.display()); + Ok(()) + } +} + +impl FetchArgs { + pub async fn run(self) -> anyhow::Result<()> { + let client = self.common.get_client().await?; + let wkg_config = wkg_core::config::Config::load().await?; + let mut lock_file = LockFile::load(false).await?; + wit::fetch_dependencies( + &wkg_config, + self.dir, + &mut lock_file, + client, + self.output_type.unwrap_or_default(), + ) + .await?; + // Now write out the lock file since everything else succeeded + lock_file.write().await?; + Ok(()) + } +} + +impl UpdateArgs { + pub async fn run(self) -> anyhow::Result<()> { + let client = self.common.get_client().await?; + let wkg_config = wkg_core::config::Config::load().await?; + let mut lock_file = LockFile::load(false).await?; + // Clear the lock file since we're updating it + lock_file.packages.clear(); + wit::fetch_dependencies( + &wkg_config, + self.dir, + &mut lock_file, + client, + self.output_type.unwrap_or_default(), + ) + .await?; + // Now write out the lock file since everything else succeeded + lock_file.write().await?; + todo!() + } +} From 9db8f086bf032e75e87bea637836a61e518cf2c6 Mon Sep 17 00:00:00 2001 From: Taylor Thomas Date: Wed, 25 Sep 2024 14:33:44 -0700 Subject: [PATCH 3/3] feat(wkg-core): Adds integration tests for building and fetching Signed-off-by: Taylor Thomas --- .github/workflows/ci.yml | 2 + Cargo.lock | 71 ++- crates/wkg-core/Cargo.toml | 1 + crates/wkg-core/src/wit.rs | 108 ++-- crates/wkg-core/tests/build.rs | 103 ++++ crates/wkg-core/tests/common.rs | 58 ++ crates/wkg-core/tests/fetch.rs | 57 ++ .../fixtures/cli-example/.cargo/config.toml | 2 + .../tests/fixtures/cli-example/.gitignore | 4 + .../tests/fixtures/cli-example/Cargo.toml | 15 + .../tests/fixtures/cli-example/src/lib.rs | 249 ++++++++ .../tests/fixtures/cli-example/wit/world.wit | 8 + .../fixtures/dog-fetcher/.cargo/config.toml | 2 + .../tests/fixtures/dog-fetcher/.gitignore | 3 + .../tests/fixtures/dog-fetcher/Cargo.toml | 15 + .../tests/fixtures/dog-fetcher/src/lib.rs | 88 +++ .../tests/fixtures/dog-fetcher/wit/world.wit | 7 + .../tests/fixtures/wasi-http/.gitignore | 2 + .../tests/fixtures/wasi-http/wit/handler.wit | 43 ++ .../tests/fixtures/wasi-http/wit/proxy.wit | 32 + .../tests/fixtures/wasi-http/wit/types.wit | 570 ++++++++++++++++++ crates/wkg/src/wit.rs | 10 +- 22 files changed, 1381 insertions(+), 69 deletions(-) create mode 100644 crates/wkg-core/tests/build.rs create mode 100644 crates/wkg-core/tests/common.rs create mode 100644 crates/wkg-core/tests/fetch.rs create mode 100644 crates/wkg-core/tests/fixtures/cli-example/.cargo/config.toml create mode 100644 crates/wkg-core/tests/fixtures/cli-example/.gitignore create mode 100644 crates/wkg-core/tests/fixtures/cli-example/Cargo.toml create mode 100644 crates/wkg-core/tests/fixtures/cli-example/src/lib.rs create mode 100644 crates/wkg-core/tests/fixtures/cli-example/wit/world.wit create mode 100644 crates/wkg-core/tests/fixtures/dog-fetcher/.cargo/config.toml create mode 100644 crates/wkg-core/tests/fixtures/dog-fetcher/.gitignore create mode 100644 crates/wkg-core/tests/fixtures/dog-fetcher/Cargo.toml create mode 100644 crates/wkg-core/tests/fixtures/dog-fetcher/src/lib.rs create mode 100644 crates/wkg-core/tests/fixtures/dog-fetcher/wit/world.wit create mode 100644 crates/wkg-core/tests/fixtures/wasi-http/.gitignore create mode 100644 crates/wkg-core/tests/fixtures/wasi-http/wit/handler.wit create mode 100644 crates/wkg-core/tests/fixtures/wasi-http/wit/proxy.wit create mode 100644 crates/wkg-core/tests/fixtures/wasi-http/wit/types.wit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f47e581..42e4940 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable + with: + targets: "wasm32-wasi" # We have to run these separately so we can deactivate a feature for one of the tests - name: Run client tests working-directory: ./crates/wasm-pkg-client diff --git a/Cargo.lock b/Cargo.lock index 24f2cec..25ac9ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1109,6 +1109,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.30" @@ -1155,6 +1161,12 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "group" version = "0.13.0" @@ -2289,6 +2301,15 @@ dependencies = [ "toml_edit 0.19.15", ] +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit 0.22.20", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -2567,6 +2588,12 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.7" @@ -2640,6 +2667,36 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rstest" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b423f0e62bdd61734b67cd21ff50871dfaeb9cc74f869dcd6af974fbcb19936" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e1711e7d14f74b12a58411c542185ef7fb7f2e7f8ee6e2940a883628522b42" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate 3.2.0", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.76", + "unicode-ident", +] + [[package]] name = "rust-ini" version = "0.13.0" @@ -2658,6 +2715,15 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.27" @@ -4472,6 +4538,7 @@ dependencies = [ "futures-util", "indexmap 2.5.0", "libc", + "rstest", "semver", "serde 1.0.209", "sha2", @@ -4554,7 +4621,7 @@ version = "3.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7131497b0f887e8061b430c530240063d33bf9455fa34438f388a245da69e0a5" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", "regex", @@ -4620,7 +4687,7 @@ version = "3.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37c24dc0bed72f5f90d1f8bb5b07228cbf63b3c6e9f82d82559d4bae666e7ed9" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", "syn 1.0.109", diff --git a/crates/wkg-core/Cargo.toml b/crates/wkg-core/Cargo.toml index 2455950..187531f 100644 --- a/crates/wkg-core/Cargo.toml +++ b/crates/wkg-core/Cargo.toml @@ -41,3 +41,4 @@ features = [ [dev-dependencies] tempfile = { workspace = true } sha2 = { workspace = true } +rstest = "0.22" diff --git a/crates/wkg-core/src/wit.rs b/crates/wkg-core/src/wit.rs index 0238826..d68e340 100644 --- a/crates/wkg-core/src/wit.rs +++ b/crates/wkg-core/src/wit.rs @@ -1,10 +1,6 @@ //! Functions for building WIT packages and fetching their dependencies. -use std::{ - collections::HashSet, - path::{Path, PathBuf}, - str::FromStr, -}; +use std::{collections::HashSet, path::Path, str::FromStr}; use anyhow::{Context, Result}; use semver::{Version, VersionReq}; @@ -12,7 +8,7 @@ use wasm_pkg_client::{ caching::{CachingClient, FileCache}, PackageRef, }; -use wit_component::{DecodedWasm, WitPrinter}; +use wit_component::WitPrinter; use wit_parser::{PackageId, PackageName, Resolve}; use crate::{ @@ -28,9 +24,9 @@ use crate::{ #[derive(Debug, Clone, Copy, Default)] pub enum OutputType { /// Output each dependency as a WIT file in the deps directory. + #[default] Wit, /// Output each dependency as a wasm binary file in the deps directory. - #[default] Wasm, } @@ -226,19 +222,23 @@ pub async fn populate_dependencies( } tokio::fs::create_dir_all(&deps_path).await?; + // For wit output, generate the resolve and then output each package in the resolve + if let OutputType::Wit = output { + let (resolve, pkg_id) = deps.generate_resolve(&path).await?; + return print_wit_from_resolve(&resolve, pkg_id, &deps_path).await; + } + + // If we got binary output, write them instead of the wit let decoded_deps = deps.decode_dependencies().await?; for (name, dep) in decoded_deps.iter() { - let mut output_path = deps_path.join(name.to_string()); + let mut output_path = deps_path.join(name_from_package_name(name)); - match (dep, output) { - ( - DecodedDependency::Wit { - resolution: DependencyResolution::Local(local), - .. - }, - _, - ) => { + match dep { + DecodedDependency::Wit { + resolution: DependencyResolution::Local(local), + .. + } => { // Local deps always need to be written to a subdirectory of deps so create that here tokio::fs::create_dir_all(&output_path).await?; write_local_dep(local, output_path).await?; @@ -246,39 +246,16 @@ pub async fn populate_dependencies( // This case shouldn't happen because registries only support wit packages. We can't get // a resolve from the unresolved group, so error out here. Ideally we could print the // unresolved group, but WitPrinter doesn't support that yet - ( - DecodedDependency::Wit { - resolution: DependencyResolution::Registry(_), - .. - }, - _, - ) => { + DecodedDependency::Wit { + resolution: DependencyResolution::Registry(_), + .. + } => { anyhow::bail!("Unable to resolve dependency, this is a programmer error"); } - (DecodedDependency::Wasm { decoded, .. }, OutputType::Wit) => { - tokio::fs::create_dir_all(&output_path).await?; - let (resolve, pkg) = match decoded { - DecodedWasm::WitPackage(r, p) => (r, *p), - DecodedWasm::Component(r, world_id) => { - let pkg_id = r - .worlds - .iter() - .find_map(|(id, w)| { - (id == *world_id).then_some(w).and_then(|w| w.package) - }) - .ok_or_else(|| { - anyhow::anyhow!("Unable to find package for the component's world") - })?; - (r, pkg_id) - } - }; - // Print all the deps - print_wit_deps(resolve, pkg, name, &deps_path).await?; - } // Right now WIT packages include all of their dependencies, so we don't need to fetch // those too. In the future, we'll need to look for unsatisfied dependencies and fetch // them - (DecodedDependency::Wasm { resolution, .. }, OutputType::Wasm) => { + DecodedDependency::Wasm { resolution, .. } => { // This is going to be written to a single file, so we don't create a directory here // NOTE(thomastaylor312): This janky looking thing is to avoid chopping off the // patch number from the release. Once `add_extension` is stabilized, we can use @@ -373,30 +350,29 @@ async fn write_local_dep(local: &LocalResolution, output_path: impl AsRef) Ok(()) } -/// Recursive function that prints the top level dep given and then iterates over each of its dependencies -async fn print_wit_deps( +async fn print_wit_from_resolve( resolve: &Resolve, - pkg_id: PackageId, - top_level_name: &PackageName, - root_deps_dir: &PathBuf, + top_level_id: PackageId, + root_deps_dir: &Path, ) -> Result<()> { - // Print the top level package first, then iterate over all dependencies and print them - let dep_path = root_deps_dir.join(top_level_name.to_string()); - tokio::fs::create_dir_all(&dep_path).await?; - let mut printer = WitPrinter::default(); - let wit = printer - .print(resolve, pkg_id, &[]) - .context("Unable to print wit")?; - tokio::fs::write(dep_path.join("package.wit"), wit).await?; - for pkg in resolve.package_direct_deps(pkg_id) { - // Get the package name - let package_name = resolve - .package_names - .iter() - .find_map(|(name, id)| (*id == pkg).then_some(name)) - .ok_or_else(|| anyhow::anyhow!("Got package ID from world that didn't exist"))?; - // Recurse into this package - Box::pin(print_wit_deps(resolve, pkg, package_name, root_deps_dir)).await?; + for (id, pkg) in resolve + .packages + .iter() + .filter(|(id, _)| *id != top_level_id) + { + let dep_path = root_deps_dir.join(name_from_package_name(&pkg.name)); + tokio::fs::create_dir_all(&dep_path).await?; + let mut printer = WitPrinter::default(); + let wit = printer + .print(resolve, id, &[]) + .context("Unable to print wit")?; + tokio::fs::write(dep_path.join("package.wit"), wit).await?; } Ok(()) } + +/// Given a package name, returns a valid directory/file name for it (thanks windows!) +fn name_from_package_name(package_name: &PackageName) -> String { + let package_name_str = package_name.to_string(); + package_name_str.replace([':', '@'], "-") +} diff --git a/crates/wkg-core/tests/build.rs b/crates/wkg-core/tests/build.rs new file mode 100644 index 0000000..8bc8ccd --- /dev/null +++ b/crates/wkg-core/tests/build.rs @@ -0,0 +1,103 @@ +use wit_component::DecodedWasm; +use wkg_core::{config::Config as WkgConfig, lock::LockFile}; + +mod common; + +#[tokio::test] +async fn test_build_wit() { + let (_temp, fixture_path) = common::load_fixture("wasi-http").await.unwrap(); + let lock_file = fixture_path.join("wkg.lock"); + + let mut lock = LockFile::new_with_path([], &lock_file) + .await + .expect("Should be able to create a new lock file"); + let (_temp_cache, client) = common::get_client().await.unwrap(); + let (pkg, version, bytes) = wkg_core::wit::build_package( + &WkgConfig::default(), + fixture_path.join("wit"), + &mut lock, + client, + ) + .await + .expect("Should be able to build the package"); + + assert_eq!( + pkg.to_string(), + "wasi:http", + "Should have the correct package reference" + ); + assert_eq!( + version.unwrap().to_string(), + "0.2.0", + "Should have the correct version" + ); + + // Make sure the lock file has all the correct packages in it + // NOTE: We could improve this test to check the version too + let mut names = lock + .packages + .iter() + .map(|p| p.name.to_string()) + .collect::>(); + names.sort(); + assert_eq!( + names, + vec!["wasi:cli", "wasi:clocks", "wasi:io", "wasi:random"], + "Should have the correct packages in the lock file" + ); + + // Parse the bytes and make sure it roundtrips back correctly + let parsed = wit_component::decode(&bytes).expect("Should be able to parse the bytes"); + let (resolve, pkg_id) = match parsed { + DecodedWasm::WitPackage(resolve, pkg_id) => (resolve, pkg_id), + _ => panic!("Should be a package"), + }; + + let name = resolve + .package_names + .iter() + .find_map(|(name, id)| (pkg_id == *id).then_some(name)) + .expect("Should be able to find the package name"); + + assert_eq!( + name.to_string(), + "wasi:http@0.2.0", + "Should have the correct package name" + ); + + assert!( + resolve.package_direct_deps(pkg_id).count() > 0, + "Should have direct dependencies embedded" + ); +} + +#[tokio::test] +async fn test_bad_dep_failure() { + let (_temp, fixture_path) = common::load_fixture("wasi-http").await.unwrap(); + let lock_file = fixture_path.join("wkg.lock"); + let mut lock = LockFile::new_with_path([], &lock_file) + .await + .expect("Should be able to create a new lock file"); + let (_temp_cache, client) = common::get_client().await.unwrap(); + + let world_file = fixture_path.join("wit").join("proxy.wit"); + let str_world = tokio::fs::read_to_string(&world_file) + .await + .expect("Should be able to read the world file"); + let str_world = str_world.replace( + "import wasi:cli/stdin@0.2.0;", + "import totally:not/real@0.2.0;", + ); + tokio::fs::write(world_file, str_world) + .await + .expect("Should be able to write the world file"); + + wkg_core::wit::build_package( + &WkgConfig::default(), + fixture_path.join("wit"), + &mut lock, + client, + ) + .await + .expect_err("Should error with a bad dependency"); +} diff --git a/crates/wkg-core/tests/common.rs b/crates/wkg-core/tests/common.rs new file mode 100644 index 0000000..36108e0 --- /dev/null +++ b/crates/wkg-core/tests/common.rs @@ -0,0 +1,58 @@ +use std::path::{Path, PathBuf}; + +use tempfile::TempDir; +use wasm_pkg_client::{ + caching::{CachingClient, FileCache}, + Client, +}; + +pub fn fixture_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("fixtures") +} + +pub async fn get_client() -> anyhow::Result<(TempDir, CachingClient)> { + let client = Client::with_global_defaults().await?; + let cache_temp_dir = tempfile::tempdir()?; + let cache = FileCache::new(cache_temp_dir.path()).await?; + + Ok((cache_temp_dir, CachingClient::new(Some(client), cache))) +} + +/// Loads the fixture with the given name into a temporary directory. This will copy the fixture from the tests/fixtures directory into a temporary directory and return the tempdir containing that directory (and its path) +pub async fn load_fixture(fixture: &str) -> anyhow::Result<(TempDir, PathBuf)> { + let temp_dir = tempfile::tempdir()?; + let fixture_path = fixture_dir().join(fixture); + // This will error if it doesn't exist, which is what we want + tokio::fs::metadata(&fixture_path).await?; + let copied_path = temp_dir.path().join(fixture_path.file_name().unwrap()); + copy_dir(&fixture_path, &copied_path).await?; + Ok((temp_dir, copied_path)) +} + +async fn copy_dir(source: impl AsRef, destination: impl AsRef) -> anyhow::Result<()> { + tokio::fs::create_dir_all(&destination).await?; + let mut entries = tokio::fs::read_dir(source).await?; + while let Some(entry) = entries.next_entry().await? { + let filetype = entry.file_type().await?; + if filetype.is_dir() { + // Skip the deps directory in case it is there from debugging + if entry.path().file_name().unwrap_or_default() == "deps" { + continue; + } + Box::pin(copy_dir( + entry.path(), + destination.as_ref().join(entry.file_name()), + )) + .await?; + } else { + // Skip any .lock files in the fixture + if entry.path().file_name().unwrap_or_default() == ".lock" { + continue; + } + tokio::fs::copy(entry.path(), destination.as_ref().join(entry.file_name())).await?; + } + } + Ok(()) +} diff --git a/crates/wkg-core/tests/fetch.rs b/crates/wkg-core/tests/fetch.rs new file mode 100644 index 0000000..7394dc4 --- /dev/null +++ b/crates/wkg-core/tests/fetch.rs @@ -0,0 +1,57 @@ +use std::path::Path; + +use rstest::rstest; +use tokio::process::Command; +use wkg_core::{ + config::Config, + lock::LockFile, + wit::{self, OutputType}, +}; + +mod common; + +#[rstest] +#[case("dog-fetcher", 1)] +#[case("cli-example", 2)] +#[tokio::test] +async fn test_fetch( + #[case] fixture_name: &str, + #[case] expected_deps: usize, + #[values(OutputType::Wasm, OutputType::Wit)] output: OutputType, +) { + let (_temp, fixture_path) = common::load_fixture(fixture_name).await.unwrap(); + let lock_file = fixture_path.join("wkg.lock"); + let mut lock = LockFile::new_with_path([], &lock_file) + .await + .expect("Should be able to create a new lock file"); + let (_temp_cache, client) = common::get_client().await.unwrap(); + + wit::fetch_dependencies( + &Config::default(), + fixture_path.join("wit"), + &mut lock, + client, + output, + ) + .await + .expect("Should be able to fetch the dependencies"); + + assert_eq!( + lock.packages.len(), + expected_deps, + "Should have the correct number of packages in the lock file" + ); + + // Now try to build the component to make sure the deps work + build_component(&fixture_path).await; +} + +async fn build_component(fixture_path: &Path) { + let output = Command::new(env!("CARGO")) + .current_dir(fixture_path) + .arg("build") + .output() + .await + .expect("Should be able to execute build command"); + assert!(output.status.success(), "Should be able to build the component successfully. Exited with error code: {}\nStdout:\n\n{}\n\nStderr:\n\n{}", output.status.code().unwrap_or(-1), String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); +} diff --git a/crates/wkg-core/tests/fixtures/cli-example/.cargo/config.toml b/crates/wkg-core/tests/fixtures/cli-example/.cargo/config.toml new file mode 100644 index 0000000..12dc204 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/cli-example/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = [ "wasm32-wasi" ] diff --git a/crates/wkg-core/tests/fixtures/cli-example/.gitignore b/crates/wkg-core/tests/fixtures/cli-example/.gitignore new file mode 100644 index 0000000..f7fb8b2 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/cli-example/.gitignore @@ -0,0 +1,4 @@ +wkg.lock +Cargo.lock + +deps/ diff --git a/crates/wkg-core/tests/fixtures/cli-example/Cargo.toml b/crates/wkg-core/tests/fixtures/cli-example/Cargo.toml new file mode 100644 index 0000000..a3d6273 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/cli-example/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "cli-example" +edition = "2021" +version = "0.1.0" + +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +wit-bindgen = "0.32" diff --git a/crates/wkg-core/tests/fixtures/cli-example/src/lib.rs b/crates/wkg-core/tests/fixtures/cli-example/src/lib.rs new file mode 100644 index 0000000..86b7470 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/cli-example/src/lib.rs @@ -0,0 +1,249 @@ +wit_bindgen::generate!({ generate_all }); + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use clap::{CommandFactory, FromArgMatches}; +use exports::wasi::cli::run::Guest as RunGuest; +use wasi::cli::environment; +use wasi::filesystem::preopens::get_directories; +use wasi::filesystem::types::{Descriptor, DescriptorFlags, OpenFlags, PathFlags}; +use wasi::http::types::*; + +#[derive(clap::Parser)] +#[clap(name = "hello")] +struct Hello { + /// A random string + #[clap(long = "bar")] + bar: Option, + + /// A directory to read + #[clap(long = "foo")] + foo: PathBuf, + + /// A file to read + #[clap(id = "path")] + path: PathBuf, +} + +#[derive(serde::Deserialize)] +struct DogResponse { + message: String, + status: String, +} + +struct HelloPlugin; + +// Our implementation of the wasi:cli/run interface +impl RunGuest for HelloPlugin { + fn run() -> Result<(), ()> { + // An example for reading command line arguments and environment variables + let args = environment::get_arguments(); + println!("I got some arguments: {:?}", args); + println!( + "I got some environment variables: {:?}", + environment::get_environment() + ); + + let cmd = Hello::command(); + let matches = match cmd.try_get_matches_from(args) { + Ok(matches) => matches, + Err(err) => { + eprintln!("Error parsing arguments: {}", err); + return Err(()); + } + }; + let args = match Hello::from_arg_matches(&matches) { + Ok(args) => args, + Err(err) => { + eprintln!("Error parsing arguments: {}", err); + return Err(()); + } + }; + + // An example of an outgoing HTTP request. Hopefully we'll have a helper crate to make this + // easier soon! + let req = wasi::http::outgoing_handler::OutgoingRequest::new(Fields::new()); + req.set_scheme(Some(&Scheme::Https))?; + req.set_authority(Some("dog.ceo"))?; + req.set_path_with_query(Some("/api/breeds/image/random"))?; + match wasi::http::outgoing_handler::handle(req, None) { + Ok(resp) => { + resp.subscribe().block(); + let response = resp + .get() + .expect("HTTP request response missing") + .expect("HTTP request response requested more than once") + .expect("HTTP request failed"); + if response.status() == 200 { + let response_body = response + .consume() + .expect("failed to get incoming request body"); + let body = { + let mut buf = vec![]; + let mut stream = response_body + .stream() + .expect("failed to get HTTP request response stream"); + InputStreamReader::from(&mut stream) + .read_to_end(&mut buf) + .expect("failed to read value from HTTP request response stream"); + buf + }; + let _trailers = wasi::http::types::IncomingBody::finish(response_body); + let dog_response: DogResponse = match serde_json::from_slice(&body) { + Ok(d) => d, + Err(e) => { + println!("Failed to deserialize dog response: {}", e); + DogResponse { + message: "Failed to deserialize dog response".to_string(), + status: "failure".to_string(), + } + } + }; + println!( + "{}! Here have a dog picture: {}", + dog_response.status, dog_response.message + ); + } else { + eprintln!("HTTP request failed with status code {}", response.status()); + } + } + Err(e) => { + eprintln!("Got error when trying to fetch dog: {}", e); + } + } + + // An example of writing to a file. It will be very similar to read from a file as well if + // you want to load a config file. To open a file, you have to get the directory that was + // given to the plugin. + if let Ok(dir) = get_dir("/") { + let file = dir + .open_at( + PathFlags::empty(), + "hello.txt", + OpenFlags::CREATE, + DescriptorFlags::READ | DescriptorFlags::WRITE, + ) + .expect("Should be able to access file"); + file.write(b"Hello from the plugin", 0) + .expect("Should be able to write to file"); + } + + if let Ok(dir) = get_dir(&args.foo) { + let entries = dir.read_directory().map_err(|e| { + eprintln!("Failed to read directory: {}", e); + })?; + println!("Directory entries for {}:", args.foo.display()); + while let Some(res) = entries.read_directory_entry().transpose() { + let entry = res.map_err(|e| { + eprintln!("Failed to read directory entry: {}", e); + })?; + println!("{}", entry.name); + } + } + + let file = + open_file(&args.path, OpenFlags::empty(), DescriptorFlags::READ).map_err(|e| { + eprintln!("Failed to open file: {}", e); + })?; + + let mut body = file.read_via_stream(0).map_err(|e| { + eprintln!("Failed to read file: {}", e); + })?; + let mut buf = vec![]; + InputStreamReader::from(&mut body) + .read_to_end(&mut buf) + .map_err(|e| { + eprintln!("Failed to read file: {}", e); + })?; + println!( + "The file {} has the contents {}", + args.path.display(), + String::from_utf8_lossy(&buf) + ); + + println!("Hello from the plugin"); + Ok(()) + } +} + +fn get_dir(path: impl AsRef) -> Result { + get_directories() + .into_iter() + .find_map(|(dir, dir_path)| { + (>::as_ref(&dir_path) == path.as_ref()) + .then_some(dir) + }) + .ok_or_else(|| format!("Could not find directory {}", path.as_ref().display())) +} + +/// Opens the given file. This should be the canonicalized path to the file. +fn open_file( + path: impl AsRef, + open_flags: OpenFlags, + descriptor_flags: DescriptorFlags, +) -> Result { + let dir = path + .as_ref() + .parent() + // I mean, if someone passed a path that is at the root, that probably wasn't a good idea + .ok_or_else(|| { + format!( + "Could not find parent directory of {}", + path.as_ref().display() + ) + })?; + let dir = get_dir(dir)?; + dir.open_at( + PathFlags::empty(), + path.as_ref() + .file_name() + .ok_or_else(|| format!("Path did not have a file name: {}", path.as_ref().display()))? + .to_str() + .ok_or_else(|| "Path is not a valid string".to_string())?, + open_flags, + descriptor_flags, + ) + .map_err(|e| format!("Failed to open file {}: {}", path.as_ref().display(), e)) +} + +pub struct InputStreamReader<'a> { + stream: &'a mut crate::wasi::io::streams::InputStream, +} + +impl<'a> From<&'a mut crate::wasi::io::streams::InputStream> for InputStreamReader<'a> { + fn from(stream: &'a mut crate::wasi::io::streams::InputStream) -> Self { + Self { stream } + } +} + +impl std::io::Read for InputStreamReader<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + use crate::wasi::io::streams::StreamError; + use std::io; + + let n = buf + .len() + .try_into() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + match self.stream.blocking_read(n) { + Ok(chunk) => { + let n = chunk.len(); + if n > buf.len() { + return Err(io::Error::new( + io::ErrorKind::Other, + "more bytes read than requested", + )); + } + buf[..n].copy_from_slice(&chunk); + Ok(n) + } + Err(StreamError::Closed) => Ok(0), + Err(StreamError::LastOperationFailed(e)) => { + Err(io::Error::new(io::ErrorKind::Other, e.to_debug_string())) + } + } + } +} + +export!(HelloPlugin); diff --git a/crates/wkg-core/tests/fixtures/cli-example/wit/world.wit b/crates/wkg-core/tests/fixtures/cli-example/wit/world.wit new file mode 100644 index 0000000..ee56180 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/cli-example/wit/world.wit @@ -0,0 +1,8 @@ +package test:plugin; + +world plugin { + include wasi:cli/imports@0.2.0; + import wasi:http/outgoing-handler@0.2.0; + + export wasi:cli/run@0.2.0; +} diff --git a/crates/wkg-core/tests/fixtures/dog-fetcher/.cargo/config.toml b/crates/wkg-core/tests/fixtures/dog-fetcher/.cargo/config.toml new file mode 100644 index 0000000..12dc204 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/dog-fetcher/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = [ "wasm32-wasi" ] diff --git a/crates/wkg-core/tests/fixtures/dog-fetcher/.gitignore b/crates/wkg-core/tests/fixtures/dog-fetcher/.gitignore new file mode 100644 index 0000000..745467a --- /dev/null +++ b/crates/wkg-core/tests/fixtures/dog-fetcher/.gitignore @@ -0,0 +1,3 @@ +Cargo.lock +wkg.lock +deps/ diff --git a/crates/wkg-core/tests/fixtures/dog-fetcher/Cargo.toml b/crates/wkg-core/tests/fixtures/dog-fetcher/Cargo.toml new file mode 100644 index 0000000..99ed62f --- /dev/null +++ b/crates/wkg-core/tests/fixtures/dog-fetcher/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "dog-fetcher" +edition = "2021" +version = "0.1.1" + +[workspace] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +wasi = "=0.13.1" # WASI 0.2.1 is not in the BCA repo yet +wit-bindgen = "0.32" diff --git a/crates/wkg-core/tests/fixtures/dog-fetcher/src/lib.rs b/crates/wkg-core/tests/fixtures/dog-fetcher/src/lib.rs new file mode 100644 index 0000000..63ebd74 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/dog-fetcher/src/lib.rs @@ -0,0 +1,88 @@ +mod bindings { + use crate::DogFetcher; + + wit_bindgen::generate!({ + with: { + "wasi:clocks/monotonic-clock@0.2.0": ::wasi::clocks::monotonic_clock, + "wasi:http/incoming-handler@0.2.0": generate, + "wasi:http/outgoing-handler@0.2.0": ::wasi::http::outgoing_handler, + "wasi:http/types@0.2.0": ::wasi::http::types, + "wasi:io/error@0.2.0": ::wasi::io::error, + "wasi:io/poll@0.2.0": ::wasi::io::poll, + "wasi:io/streams@0.2.0": ::wasi::io::streams, + } + }); + + export!(DogFetcher); +} + +use std::io::{Read as _, Write as _}; + +use bindings::exports::wasi::http::incoming_handler::Guest; +use wasi::http::types::*; + +#[derive(serde::Deserialize)] +struct DogResponse { + message: String, +} + +struct DogFetcher; + +impl Guest for DogFetcher { + fn handle(_request: IncomingRequest, response_out: ResponseOutparam) { + // Build a request to dog.ceo which returns a URL at which we can find a doggo + let req = wasi::http::outgoing_handler::OutgoingRequest::new(Fields::new()); + req.set_scheme(Some(&Scheme::Https)).unwrap(); + req.set_authority(Some("dog.ceo")).unwrap(); + req.set_path_with_query(Some("/api/breeds/image/random")) + .unwrap(); + + // Perform the API call to dog.ceo, expecting a URL to come back as the response body + let dog_picture_url = match wasi::http::outgoing_handler::handle(req, None) { + Ok(resp) => { + resp.subscribe().block(); + let response = resp + .get() + .expect("HTTP request response missing") + .expect("HTTP request response requested more than once") + .expect("HTTP request failed"); + if response.status() == 200 { + let response_body = response + .consume() + .expect("failed to get incoming request body"); + let body = { + let mut buf = vec![]; + let mut stream = response_body + .stream() + .expect("failed to get HTTP request response stream"); + stream + .read_to_end(&mut buf) + .expect("failed to read value from HTTP request response stream"); + buf + }; + let _trailers = wasi::http::types::IncomingBody::finish(response_body); + let dog_response: DogResponse = serde_json::from_slice(&body).unwrap(); + dog_response.message + } else { + format!("HTTP request failed with status code {}", response.status()) + } + } + Err(e) => { + format!("Got error when trying to fetch dog: {}", e) + } + }; + + // Build the HTTP response we'll send back to the user + let response = OutgoingResponse::new(Fields::new()); + response.set_status_code(200).unwrap(); + let response_body = response.body().unwrap(); + let mut write_stream = response_body.write().unwrap(); + + ResponseOutparam::set(response_out, Ok(response)); + + write_stream.write_all(dog_picture_url.as_bytes()).unwrap(); + drop(write_stream); + + OutgoingBody::finish(response_body, None).expect("failed to finish response body"); + } +} diff --git a/crates/wkg-core/tests/fixtures/dog-fetcher/wit/world.wit b/crates/wkg-core/tests/fixtures/dog-fetcher/wit/world.wit new file mode 100644 index 0000000..6d3c0f8 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/dog-fetcher/wit/world.wit @@ -0,0 +1,7 @@ +package test:hello; + +world hello { + import wasi:http/outgoing-handler@0.2.0; + + export wasi:http/incoming-handler@0.2.0; +} diff --git a/crates/wkg-core/tests/fixtures/wasi-http/.gitignore b/crates/wkg-core/tests/fixtures/wasi-http/.gitignore new file mode 100644 index 0000000..750c1a4 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/wasi-http/.gitignore @@ -0,0 +1,2 @@ +wkg.lock +deps/ diff --git a/crates/wkg-core/tests/fixtures/wasi-http/wit/handler.wit b/crates/wkg-core/tests/fixtures/wasi-http/wit/handler.wit new file mode 100644 index 0000000..a34a064 --- /dev/null +++ b/crates/wkg-core/tests/fixtures/wasi-http/wit/handler.wit @@ -0,0 +1,43 @@ +/// This interface defines a handler of incoming HTTP Requests. It should +/// be exported by components which can respond to HTTP Requests. +interface incoming-handler { + use types.{incoming-request, response-outparam}; + + /// This function is invoked with an incoming HTTP Request, and a resource + /// `response-outparam` which provides the capability to reply with an HTTP + /// Response. The response is sent by calling the `response-outparam.set` + /// method, which allows execution to continue after the response has been + /// sent. This enables both streaming to the response body, and performing other + /// work. + /// + /// The implementor of this function must write a response to the + /// `response-outparam` before returning, or else the caller will respond + /// with an error on its behalf. + handle: func( + request: incoming-request, + response-out: response-outparam + ); +} + +/// This interface defines a handler of outgoing HTTP Requests. It should be +/// imported by components which wish to make HTTP Requests. +interface outgoing-handler { + use types.{ + outgoing-request, request-options, future-incoming-response, error-code + }; + + /// This function is invoked with an outgoing HTTP Request, and it returns + /// a resource `future-incoming-response` which represents an HTTP Response + /// which may arrive in the future. + /// + /// The `options` argument accepts optional parameters for the HTTP + /// protocol's transport layer. + /// + /// This function may return an error if the `outgoing-request` is invalid + /// or not allowed to be made. Otherwise, protocol errors are reported + /// through the `future-incoming-response`. + handle: func( + request: outgoing-request, + options: option + ) -> result; +} diff --git a/crates/wkg-core/tests/fixtures/wasi-http/wit/proxy.wit b/crates/wkg-core/tests/fixtures/wasi-http/wit/proxy.wit new file mode 100644 index 0000000..687c24d --- /dev/null +++ b/crates/wkg-core/tests/fixtures/wasi-http/wit/proxy.wit @@ -0,0 +1,32 @@ +package wasi:http@0.2.0; + +/// The `wasi:http/proxy` world captures a widely-implementable intersection of +/// hosts that includes HTTP forward and reverse proxies. Components targeting +/// this world may concurrently stream in and out any number of incoming and +/// outgoing HTTP requests. +world proxy { + /// HTTP proxies have access to time and randomness. + include wasi:clocks/imports@0.2.0; + import wasi:random/random@0.2.0; + + /// Proxies have standard output and error streams which are expected to + /// terminate in a developer-facing console provided by the host. + import wasi:cli/stdout@0.2.0; + import wasi:cli/stderr@0.2.0; + + /// TODO: this is a temporary workaround until component tooling is able to + /// gracefully handle the absence of stdin. Hosts must return an eof stream + /// for this import, which is what wasi-libc + tooling will do automatically + /// when this import is properly removed. + import wasi:cli/stdin@0.2.0; + + /// This is the default handler to use when user code simply wants to make an + /// HTTP request (e.g., via `fetch()`). + import outgoing-handler; + + /// The host delivers incoming HTTP requests to a component by calling the + /// `handle` function of this exported interface. A host may arbitrarily reuse + /// or not reuse component instance when delivering incoming HTTP requests and + /// thus a component must be able to handle 0..N calls to `handle`. + export incoming-handler; +} diff --git a/crates/wkg-core/tests/fixtures/wasi-http/wit/types.wit b/crates/wkg-core/tests/fixtures/wasi-http/wit/types.wit new file mode 100644 index 0000000..755ac6a --- /dev/null +++ b/crates/wkg-core/tests/fixtures/wasi-http/wit/types.wit @@ -0,0 +1,570 @@ +/// This interface defines all of the types and methods for implementing +/// HTTP Requests and Responses, both incoming and outgoing, as well as +/// their headers, trailers, and bodies. +interface types { + use wasi:clocks/monotonic-clock@0.2.0.{duration}; + use wasi:io/streams@0.2.0.{input-stream, output-stream}; + use wasi:io/error@0.2.0.{error as io-error}; + use wasi:io/poll@0.2.0.{pollable}; + + /// This type corresponds to HTTP standard Methods. + variant method { + get, + head, + post, + put, + delete, + connect, + options, + trace, + patch, + other(string) + } + + /// This type corresponds to HTTP standard Related Schemes. + variant scheme { + HTTP, + HTTPS, + other(string) + } + + /// These cases are inspired by the IANA HTTP Proxy Error Types: + /// https://www.iana.org/assignments/http-proxy-status/http-proxy-status.xhtml#table-http-proxy-error-types + variant error-code { + DNS-timeout, + DNS-error(DNS-error-payload), + destination-not-found, + destination-unavailable, + destination-IP-prohibited, + destination-IP-unroutable, + connection-refused, + connection-terminated, + connection-timeout, + connection-read-timeout, + connection-write-timeout, + connection-limit-reached, + TLS-protocol-error, + TLS-certificate-error, + TLS-alert-received(TLS-alert-received-payload), + HTTP-request-denied, + HTTP-request-length-required, + HTTP-request-body-size(option), + HTTP-request-method-invalid, + HTTP-request-URI-invalid, + HTTP-request-URI-too-long, + HTTP-request-header-section-size(option), + HTTP-request-header-size(option), + HTTP-request-trailer-section-size(option), + HTTP-request-trailer-size(field-size-payload), + HTTP-response-incomplete, + HTTP-response-header-section-size(option), + HTTP-response-header-size(field-size-payload), + HTTP-response-body-size(option), + HTTP-response-trailer-section-size(option), + HTTP-response-trailer-size(field-size-payload), + HTTP-response-transfer-coding(option), + HTTP-response-content-coding(option), + HTTP-response-timeout, + HTTP-upgrade-failed, + HTTP-protocol-error, + loop-detected, + configuration-error, + /// This is a catch-all error for anything that doesn't fit cleanly into a + /// more specific case. It also includes an optional string for an + /// unstructured description of the error. Users should not depend on the + /// string for diagnosing errors, as it's not required to be consistent + /// between implementations. + internal-error(option) + } + + /// Defines the case payload type for `DNS-error` above: + record DNS-error-payload { + rcode: option, + info-code: option + } + + /// Defines the case payload type for `TLS-alert-received` above: + record TLS-alert-received-payload { + alert-id: option, + alert-message: option + } + + /// Defines the case payload type for `HTTP-response-{header,trailer}-size` above: + record field-size-payload { + field-name: option, + field-size: option + } + + /// Attempts to extract a http-related `error` from the wasi:io `error` + /// provided. + /// + /// Stream operations which return + /// `wasi:io/stream/stream-error::last-operation-failed` have a payload of + /// type `wasi:io/error/error` with more information about the operation + /// that failed. This payload can be passed through to this function to see + /// if there's http-related information about the error to return. + /// + /// Note that this function is fallible because not all io-errors are + /// http-related errors. + http-error-code: func(err: borrow) -> option; + + /// This type enumerates the different kinds of errors that may occur when + /// setting or appending to a `fields` resource. + variant header-error { + /// This error indicates that a `field-key` or `field-value` was + /// syntactically invalid when used with an operation that sets headers in a + /// `fields`. + invalid-syntax, + + /// This error indicates that a forbidden `field-key` was used when trying + /// to set a header in a `fields`. + forbidden, + + /// This error indicates that the operation on the `fields` was not + /// permitted because the fields are immutable. + immutable, + } + + /// Field keys are always strings. + type field-key = string; + + /// Field values should always be ASCII strings. However, in + /// reality, HTTP implementations often have to interpret malformed values, + /// so they are provided as a list of bytes. + type field-value = list; + + /// This following block defines the `fields` resource which corresponds to + /// HTTP standard Fields. Fields are a common representation used for both + /// Headers and Trailers. + /// + /// A `fields` may be mutable or immutable. A `fields` created using the + /// constructor, `from-list`, or `clone` will be mutable, but a `fields` + /// resource given by other means (including, but not limited to, + /// `incoming-request.headers`, `outgoing-request.headers`) might be be + /// immutable. In an immutable fields, the `set`, `append`, and `delete` + /// operations will fail with `header-error.immutable`. + resource fields { + + /// Construct an empty HTTP Fields. + /// + /// The resulting `fields` is mutable. + constructor(); + + /// Construct an HTTP Fields. + /// + /// The resulting `fields` is mutable. + /// + /// The list represents each key-value pair in the Fields. Keys + /// which have multiple values are represented by multiple entries in this + /// list with the same key. + /// + /// The tuple is a pair of the field key, represented as a string, and + /// Value, represented as a list of bytes. In a valid Fields, all keys + /// and values are valid UTF-8 strings. However, values are not always + /// well-formed, so they are represented as a raw list of bytes. + /// + /// An error result will be returned if any header or value was + /// syntactically invalid, or if a header was forbidden. + from-list: static func( + entries: list> + ) -> result; + + /// Get all of the values corresponding to a key. If the key is not present + /// in this `fields`, an empty list is returned. However, if the key is + /// present but empty, this is represented by a list with one or more + /// empty field-values present. + get: func(name: field-key) -> list; + + /// Returns `true` when the key is present in this `fields`. If the key is + /// syntactically invalid, `false` is returned. + has: func(name: field-key) -> bool; + + /// Set all of the values for a key. Clears any existing values for that + /// key, if they have been set. + /// + /// Fails with `header-error.immutable` if the `fields` are immutable. + set: func(name: field-key, value: list) -> result<_, header-error>; + + /// Delete all values for a key. Does nothing if no values for the key + /// exist. + /// + /// Fails with `header-error.immutable` if the `fields` are immutable. + delete: func(name: field-key) -> result<_, header-error>; + + /// Append a value for a key. Does not change or delete any existing + /// values for that key. + /// + /// Fails with `header-error.immutable` if the `fields` are immutable. + append: func(name: field-key, value: field-value) -> result<_, header-error>; + + /// Retrieve the full set of keys and values in the Fields. Like the + /// constructor, the list represents each key-value pair. + /// + /// The outer list represents each key-value pair in the Fields. Keys + /// which have multiple values are represented by multiple entries in this + /// list with the same key. + entries: func() -> list>; + + /// Make a deep copy of the Fields. Equivelant in behavior to calling the + /// `fields` constructor on the return value of `entries`. The resulting + /// `fields` is mutable. + clone: func() -> fields; + } + + /// Headers is an alias for Fields. + type headers = fields; + + /// Trailers is an alias for Fields. + type trailers = fields; + + /// Represents an incoming HTTP Request. + resource incoming-request { + + /// Returns the method of the incoming request. + method: func() -> method; + + /// Returns the path with query parameters from the request, as a string. + path-with-query: func() -> option; + + /// Returns the protocol scheme from the request. + scheme: func() -> option; + + /// Returns the authority from the request, if it was present. + authority: func() -> option; + + /// Get the `headers` associated with the request. + /// + /// The returned `headers` resource is immutable: `set`, `append`, and + /// `delete` operations will fail with `header-error.immutable`. + /// + /// The `headers` returned are a child resource: it must be dropped before + /// the parent `incoming-request` is dropped. Dropping this + /// `incoming-request` before all children are dropped will trap. + headers: func() -> headers; + + /// Gives the `incoming-body` associated with this request. Will only + /// return success at most once, and subsequent calls will return error. + consume: func() -> result; + } + + /// Represents an outgoing HTTP Request. + resource outgoing-request { + + /// Construct a new `outgoing-request` with a default `method` of `GET`, and + /// `none` values for `path-with-query`, `scheme`, and `authority`. + /// + /// * `headers` is the HTTP Headers for the Request. + /// + /// It is possible to construct, or manipulate with the accessor functions + /// below, an `outgoing-request` with an invalid combination of `scheme` + /// and `authority`, or `headers` which are not permitted to be sent. + /// It is the obligation of the `outgoing-handler.handle` implementation + /// to reject invalid constructions of `outgoing-request`. + constructor( + headers: headers + ); + + /// Returns the resource corresponding to the outgoing Body for this + /// Request. + /// + /// Returns success on the first call: the `outgoing-body` resource for + /// this `outgoing-request` can be retrieved at most once. Subsequent + /// calls will return error. + body: func() -> result; + + /// Get the Method for the Request. + method: func() -> method; + /// Set the Method for the Request. Fails if the string present in a + /// `method.other` argument is not a syntactically valid method. + set-method: func(method: method) -> result; + + /// Get the combination of the HTTP Path and Query for the Request. + /// When `none`, this represents an empty Path and empty Query. + path-with-query: func() -> option; + /// Set the combination of the HTTP Path and Query for the Request. + /// When `none`, this represents an empty Path and empty Query. Fails is the + /// string given is not a syntactically valid path and query uri component. + set-path-with-query: func(path-with-query: option) -> result; + + /// Get the HTTP Related Scheme for the Request. When `none`, the + /// implementation may choose an appropriate default scheme. + scheme: func() -> option; + /// Set the HTTP Related Scheme for the Request. When `none`, the + /// implementation may choose an appropriate default scheme. Fails if the + /// string given is not a syntactically valid uri scheme. + set-scheme: func(scheme: option) -> result; + + /// Get the HTTP Authority for the Request. A value of `none` may be used + /// with Related Schemes which do not require an Authority. The HTTP and + /// HTTPS schemes always require an authority. + authority: func() -> option; + /// Set the HTTP Authority for the Request. A value of `none` may be used + /// with Related Schemes which do not require an Authority. The HTTP and + /// HTTPS schemes always require an authority. Fails if the string given is + /// not a syntactically valid uri authority. + set-authority: func(authority: option) -> result; + + /// Get the headers associated with the Request. + /// + /// The returned `headers` resource is immutable: `set`, `append`, and + /// `delete` operations will fail with `header-error.immutable`. + /// + /// This headers resource is a child: it must be dropped before the parent + /// `outgoing-request` is dropped, or its ownership is transfered to + /// another component by e.g. `outgoing-handler.handle`. + headers: func() -> headers; + } + + /// Parameters for making an HTTP Request. Each of these parameters is + /// currently an optional timeout applicable to the transport layer of the + /// HTTP protocol. + /// + /// These timeouts are separate from any the user may use to bound a + /// blocking call to `wasi:io/poll.poll`. + resource request-options { + /// Construct a default `request-options` value. + constructor(); + + /// The timeout for the initial connect to the HTTP Server. + connect-timeout: func() -> option; + + /// Set the timeout for the initial connect to the HTTP Server. An error + /// return value indicates that this timeout is not supported. + set-connect-timeout: func(duration: option) -> result; + + /// The timeout for receiving the first byte of the Response body. + first-byte-timeout: func() -> option; + + /// Set the timeout for receiving the first byte of the Response body. An + /// error return value indicates that this timeout is not supported. + set-first-byte-timeout: func(duration: option) -> result; + + /// The timeout for receiving subsequent chunks of bytes in the Response + /// body stream. + between-bytes-timeout: func() -> option; + + /// Set the timeout for receiving subsequent chunks of bytes in the Response + /// body stream. An error return value indicates that this timeout is not + /// supported. + set-between-bytes-timeout: func(duration: option) -> result; + } + + /// Represents the ability to send an HTTP Response. + /// + /// This resource is used by the `wasi:http/incoming-handler` interface to + /// allow a Response to be sent corresponding to the Request provided as the + /// other argument to `incoming-handler.handle`. + resource response-outparam { + + /// Set the value of the `response-outparam` to either send a response, + /// or indicate an error. + /// + /// This method consumes the `response-outparam` to ensure that it is + /// called at most once. If it is never called, the implementation + /// will respond with an error. + /// + /// The user may provide an `error` to `response` to allow the + /// implementation determine how to respond with an HTTP error response. + set: static func( + param: response-outparam, + response: result, + ); + } + + /// This type corresponds to the HTTP standard Status Code. + type status-code = u16; + + /// Represents an incoming HTTP Response. + resource incoming-response { + + /// Returns the status code from the incoming response. + status: func() -> status-code; + + /// Returns the headers from the incoming response. + /// + /// The returned `headers` resource is immutable: `set`, `append`, and + /// `delete` operations will fail with `header-error.immutable`. + /// + /// This headers resource is a child: it must be dropped before the parent + /// `incoming-response` is dropped. + headers: func() -> headers; + + /// Returns the incoming body. May be called at most once. Returns error + /// if called additional times. + consume: func() -> result; + } + + /// Represents an incoming HTTP Request or Response's Body. + /// + /// A body has both its contents - a stream of bytes - and a (possibly + /// empty) set of trailers, indicating that the full contents of the + /// body have been received. This resource represents the contents as + /// an `input-stream` and the delivery of trailers as a `future-trailers`, + /// and ensures that the user of this interface may only be consuming either + /// the body contents or waiting on trailers at any given time. + resource incoming-body { + + /// Returns the contents of the body, as a stream of bytes. + /// + /// Returns success on first call: the stream representing the contents + /// can be retrieved at most once. Subsequent calls will return error. + /// + /// The returned `input-stream` resource is a child: it must be dropped + /// before the parent `incoming-body` is dropped, or consumed by + /// `incoming-body.finish`. + /// + /// This invariant ensures that the implementation can determine whether + /// the user is consuming the contents of the body, waiting on the + /// `future-trailers` to be ready, or neither. This allows for network + /// backpressure is to be applied when the user is consuming the body, + /// and for that backpressure to not inhibit delivery of the trailers if + /// the user does not read the entire body. + %stream: func() -> result; + + /// Takes ownership of `incoming-body`, and returns a `future-trailers`. + /// This function will trap if the `input-stream` child is still alive. + finish: static func(this: incoming-body) -> future-trailers; + } + + /// Represents a future which may eventaully return trailers, or an error. + /// + /// In the case that the incoming HTTP Request or Response did not have any + /// trailers, this future will resolve to the empty set of trailers once the + /// complete Request or Response body has been received. + resource future-trailers { + + /// Returns a pollable which becomes ready when either the trailers have + /// been received, or an error has occured. When this pollable is ready, + /// the `get` method will return `some`. + subscribe: func() -> pollable; + + /// Returns the contents of the trailers, or an error which occured, + /// once the future is ready. + /// + /// The outer `option` represents future readiness. Users can wait on this + /// `option` to become `some` using the `subscribe` method. + /// + /// The outer `result` is used to retrieve the trailers or error at most + /// once. It will be success on the first call in which the outer option + /// is `some`, and error on subsequent calls. + /// + /// The inner `result` represents that either the HTTP Request or Response + /// body, as well as any trailers, were received successfully, or that an + /// error occured receiving them. The optional `trailers` indicates whether + /// or not trailers were present in the body. + /// + /// When some `trailers` are returned by this method, the `trailers` + /// resource is immutable, and a child. Use of the `set`, `append`, or + /// `delete` methods will return an error, and the resource must be + /// dropped before the parent `future-trailers` is dropped. + get: func() -> option, error-code>>>; + } + + /// Represents an outgoing HTTP Response. + resource outgoing-response { + + /// Construct an `outgoing-response`, with a default `status-code` of `200`. + /// If a different `status-code` is needed, it must be set via the + /// `set-status-code` method. + /// + /// * `headers` is the HTTP Headers for the Response. + constructor(headers: headers); + + /// Get the HTTP Status Code for the Response. + status-code: func() -> status-code; + + /// Set the HTTP Status Code for the Response. Fails if the status-code + /// given is not a valid http status code. + set-status-code: func(status-code: status-code) -> result; + + /// Get the headers associated with the Request. + /// + /// The returned `headers` resource is immutable: `set`, `append`, and + /// `delete` operations will fail with `header-error.immutable`. + /// + /// This headers resource is a child: it must be dropped before the parent + /// `outgoing-request` is dropped, or its ownership is transfered to + /// another component by e.g. `outgoing-handler.handle`. + headers: func() -> headers; + + /// Returns the resource corresponding to the outgoing Body for this Response. + /// + /// Returns success on the first call: the `outgoing-body` resource for + /// this `outgoing-response` can be retrieved at most once. Subsequent + /// calls will return error. + body: func() -> result; + } + + /// Represents an outgoing HTTP Request or Response's Body. + /// + /// A body has both its contents - a stream of bytes - and a (possibly + /// empty) set of trailers, inducating the full contents of the body + /// have been sent. This resource represents the contents as an + /// `output-stream` child resource, and the completion of the body (with + /// optional trailers) with a static function that consumes the + /// `outgoing-body` resource, and ensures that the user of this interface + /// may not write to the body contents after the body has been finished. + /// + /// If the user code drops this resource, as opposed to calling the static + /// method `finish`, the implementation should treat the body as incomplete, + /// and that an error has occured. The implementation should propogate this + /// error to the HTTP protocol by whatever means it has available, + /// including: corrupting the body on the wire, aborting the associated + /// Request, or sending a late status code for the Response. + resource outgoing-body { + + /// Returns a stream for writing the body contents. + /// + /// The returned `output-stream` is a child resource: it must be dropped + /// before the parent `outgoing-body` resource is dropped (or finished), + /// otherwise the `outgoing-body` drop or `finish` will trap. + /// + /// Returns success on the first call: the `output-stream` resource for + /// this `outgoing-body` may be retrieved at most once. Subsequent calls + /// will return error. + write: func() -> result; + + /// Finalize an outgoing body, optionally providing trailers. This must be + /// called to signal that the response is complete. If the `outgoing-body` + /// is dropped without calling `outgoing-body.finalize`, the implementation + /// should treat the body as corrupted. + /// + /// Fails if the body's `outgoing-request` or `outgoing-response` was + /// constructed with a Content-Length header, and the contents written + /// to the body (via `write`) does not match the value given in the + /// Content-Length. + finish: static func( + this: outgoing-body, + trailers: option + ) -> result<_, error-code>; + } + + /// Represents a future which may eventaully return an incoming HTTP + /// Response, or an error. + /// + /// This resource is returned by the `wasi:http/outgoing-handler` interface to + /// provide the HTTP Response corresponding to the sent Request. + resource future-incoming-response { + /// Returns a pollable which becomes ready when either the Response has + /// been received, or an error has occured. When this pollable is ready, + /// the `get` method will return `some`. + subscribe: func() -> pollable; + + /// Returns the incoming HTTP Response, or an error, once one is ready. + /// + /// The outer `option` represents future readiness. Users can wait on this + /// `option` to become `some` using the `subscribe` method. + /// + /// The outer `result` is used to retrieve the response or error at most + /// once. It will be success on the first call in which the outer option + /// is `some`, and error on subsequent calls. + /// + /// The inner `result` represents that either the incoming HTTP Response + /// status and headers have recieved successfully, or that an error + /// occured. Errors may also occur while consuming the response body, + /// but those will be reported by the `incoming-body` and its + /// `output-stream` child. + get: func() -> option>>; + + } +} diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 31689a6..49a68c9 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -1,6 +1,7 @@ //! Args and commands for interacting with WIT files and dependencies -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use anyhow::Context; use clap::{Args, Subcommand}; use wkg_core::{ lock::LockFile, @@ -85,6 +86,7 @@ pub struct UpdateArgs { impl BuildArgs { pub async fn run(self) -> anyhow::Result<()> { + check_dir(&self.dir).await?; let client = self.common.get_client().await?; let wkg_config = wkg_core::config::Config::load().await?; let mut lock_file = LockFile::load(false).await?; @@ -111,6 +113,7 @@ impl BuildArgs { impl FetchArgs { pub async fn run(self) -> anyhow::Result<()> { + check_dir(&self.dir).await?; let client = self.common.get_client().await?; let wkg_config = wkg_core::config::Config::load().await?; let mut lock_file = LockFile::load(false).await?; @@ -130,6 +133,7 @@ impl FetchArgs { impl UpdateArgs { pub async fn run(self) -> anyhow::Result<()> { + check_dir(&self.dir).await?; let client = self.common.get_client().await?; let wkg_config = wkg_core::config::Config::load().await?; let mut lock_file = LockFile::load(false).await?; @@ -148,3 +152,7 @@ impl UpdateArgs { todo!() } } + +async fn check_dir(dir: impl AsRef) -> anyhow::Result<()> { + tokio::fs::metadata(dir).await.context("Unable to read wit directory. This command should be run from the parent directory of the wit directory or a directory can be overridden with the --wit-dir argument").map(|_|()) +}