diff --git a/Cargo.lock b/Cargo.lock index 1844b75..d646d6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1178,7 +1178,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.4.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -1482,9 +1482,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2158,7 +2158,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.4.0", + "indexmap 2.5.0", ] [[package]] @@ -2946,7 +2946,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.4.0", + "indexmap 2.5.0", "serde 1.0.209", "serde_derive", "serde_json", @@ -2972,7 +2972,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "itoa", "ryu", "serde 1.0.209", @@ -3465,7 +3465,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "toml_datetime", "winnow 0.5.40", ] @@ -3476,7 +3476,7 @@ version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "serde 1.0.209", "serde_spanned", "toml_datetime", @@ -3689,7 +3689,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44b422328c3a86be288f569694aa97df958ade0cd9514ed00bc562952c6778e" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.12.1", "serde 1.0.209", "serde_with", @@ -3712,7 +3712,7 @@ dependencies = [ "dialoguer", "dirs", "futures-util", - "indexmap 2.4.0", + "indexmap 2.5.0", "itertools 0.12.1", "keyring", "libc", @@ -3793,7 +3793,7 @@ dependencies = [ "anyhow", "base64 0.21.7", "hex", - "indexmap 2.4.0", + "indexmap 2.5.0", "pbjson-types", "prost", "prost-types", @@ -3814,7 +3814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b950a71a544b7ac8f5a5e95f43886ac97c3fe5c7080b955b1b534037596d7be" dependencies = [ "anyhow", - "indexmap 2.4.0", + "indexmap 2.5.0", "prost", "thiserror", "warg-crypto", @@ -3903,7 +3903,7 @@ dependencies = [ "anyhow", "heck 0.4.1", "im-rc", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "petgraph", "serde 1.0.209", @@ -3952,7 +3952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6bb07c5576b608f7a2a9baa2294c1a3584a249965d695a9814a496cb6d232f" dependencies = [ "anyhow", - "indexmap 2.4.0", + "indexmap 2.5.0", "serde 1.0.209", "serde_derive", "serde_json", @@ -3968,7 +3968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47c8154d703a6b0e45acf6bd172fa002fc3c7058a9f7615e517220aeca27c638" dependencies = [ "anyhow", - "indexmap 2.4.0", + "indexmap 2.5.0", "serde 1.0.209", "serde_derive", "serde_json", @@ -4050,7 +4050,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbe55c8f9d0dbd25d9447a5a889ff90c0cc3feaa7395310d3d826b2c703eaab" dependencies = [ "bitflags 2.6.0", - "indexmap 2.4.0", + "indexmap 2.5.0", "semver", ] @@ -4063,7 +4063,7 @@ dependencies = [ "ahash", "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap 2.4.0", + "indexmap 2.5.0", "semver", ] @@ -4076,7 +4076,7 @@ dependencies = [ "ahash", "bitflags 2.6.0", "hashbrown 0.14.5", - "indexmap 2.4.0", + "indexmap 2.5.0", "semver", ] @@ -4375,7 +4375,7 @@ checksum = "f725e3885fc5890648be5c5cbc1353b755dc932aa5f1aa7de968b912a3280743" dependencies = [ "anyhow", "bitflags 2.6.0", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "serde 1.0.209", "serde_derive", @@ -4394,7 +4394,7 @@ checksum = "7e2ca3ece38ea2447a9069b43074ba73d96dde1944cba276c54e41371745f9dc" dependencies = [ "anyhow", "bitflags 2.6.0", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "serde 1.0.209", "serde_derive", @@ -4413,7 +4413,7 @@ checksum = "935a97eaffd57c3b413aa510f8f0b550a4a9fe7d59e79cd8b89a83dcb860321f" dependencies = [ "anyhow", "id-arena", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "semver", "serde 1.0.209", @@ -4431,7 +4431,7 @@ checksum = "a4d108165c1167a4ccc8a803dcf5c28e0a51d6739fd228cc7adce768632c764c" dependencies = [ "anyhow", "id-arena", - "indexmap 2.4.0", + "indexmap 2.5.0", "log", "semver", "serde 1.0.209", @@ -4448,6 +4448,7 @@ dependencies = [ "anyhow", "base64 0.22.1", "clap", + "dirs", "docker_credential", "futures-util", "oci-client", @@ -4460,6 +4461,7 @@ dependencies = [ "wasm-pkg-client", "wasm-pkg-common", "wit-component 0.216.0", + "wkg-core", ] [[package]] @@ -4468,17 +4470,22 @@ version = "0.5.0" dependencies = [ "anyhow", "futures-util", + "indexmap 2.5.0", "libc", "semver", "serde 1.0.209", "sha2", "tempfile", "tokio", + "tokio-util", "toml 0.8.19", "tracing", + "wasm-metadata 0.216.0", "wasm-pkg-client", "wasm-pkg-common", "windows-sys 0.52.0", + "wit-component 0.216.0", + "wit-parser 0.216.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 10cd9d0..7442077 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ sha2 = "0.10" tempfile = "3.10.1" thiserror = "1.0" tokio = "1.35.1" +tokio-util = "0.7.10" toml = "0.8.13" tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", default-features = false, features = [ @@ -36,5 +37,7 @@ tracing-subscriber = { version = "0.3.18", default-features = false, features = ] } wasm-pkg-common = { version = "0.5.0", path = "crates/wasm-pkg-common" } wasm-pkg-client = { version = "0.5.0", path = "crates/wasm-pkg-client" } +wasm-metadata = "0.216" wit-component = "0.216" wit-parser = "0.216" +wkg-core = { version = "0.5.0", path = "crates/wkg-core" } diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index 4f1d71a..c336bb2 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -23,7 +23,7 @@ serde_json = { workspace = true } sha2 = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "macros"] } -tokio-util = { version = "0.7.10", features = ["io", "io-util", "codec"] } +tokio-util = { workspace = true, features = ["io", "io-util", "codec"] } toml = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/crates/wasm-pkg-client/src/caching/mod.rs b/crates/wasm-pkg-client/src/caching/mod.rs index e142548..59c0d93 100644 --- a/crates/wasm-pkg-client/src/caching/mod.rs +++ b/crates/wasm-pkg-client/src/caching/mod.rs @@ -1,4 +1,5 @@ use std::future::Future; +use std::sync::Arc; use wasm_pkg_common::{ digest::ContentDigest, @@ -46,7 +47,16 @@ pub trait Cache { /// underlying client to be used as a read-only cache. pub struct CachingClient { client: Option, - cache: T, + cache: Arc, +} + +impl Clone for CachingClient { + fn clone(&self) -> Self { + Self { + client: self.client.clone(), + cache: self.cache.clone(), + } + } } impl CachingClient { @@ -54,7 +64,10 @@ impl CachingClient { /// given, the client will be in offline or read-only mode, meaning it will only be able to return /// things that are already in the cache. pub fn new(client: Option, cache: T) -> Self { - Self { client, cache } + Self { + client, + cache: Arc::new(cache), + } } /// Returns whether or not the client is in read-only mode. diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 3ab4a28..7cbf86f 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -86,23 +86,29 @@ pub struct PublishOpts { } /// A read-only registry client. +#[derive(Clone)] pub struct Client { - config: Config, - sources: RwLock, + config: Arc, + sources: Arc>, } impl Client { /// Returns a new client with the given [`Config`]. pub fn new(config: Config) -> Self { Self { - config, + config: Arc::new(config), sources: Default::default(), } } + /// Returns a reference to the configuration this client was initialized with. + pub fn config(&self) -> &Config { + &self.config + } + /// Returns a new client configured from default global config. - pub fn with_global_defaults() -> Result { - let config = Config::global_defaults()?; + pub async fn with_global_defaults() -> Result { + let config = Config::global_defaults().await?; Ok(Self::new(config)) } diff --git a/crates/wasm-pkg-client/src/oci/mod.rs b/crates/wasm-pkg-client/src/oci/mod.rs index 5c065b1..3ab4258 100644 --- a/crates/wasm-pkg-client/src/oci/mod.rs +++ b/crates/wasm-pkg-client/src/oci/mod.rs @@ -128,6 +128,7 @@ impl OciBackend { CredentialRetrievalError::ConfigNotFound | CredentialRetrievalError::ConfigReadError | CredentialRetrievalError::NoCredentialConfigured + | CredentialRetrievalError::HelperFailure { .. } ) { tracing::debug!("Failed to look up OCI credentials: {err}"); } else { diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index b12abc6..c828177 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -73,21 +73,21 @@ impl Config { /// /// Note: This list is expected to expand in the future to include /// "workspace" config files like `./.wasm-pkg/config.toml`. - pub fn global_defaults() -> Result { + pub async fn global_defaults() -> Result { let mut config = Self::default(); - if let Some(global_config) = Self::read_global_config()? { + if let Some(global_config) = Self::read_global_config().await? { config.merge(global_config); } Ok(config) } /// Reads config from the default global config file location - pub fn read_global_config() -> Result, Error> { + pub async fn read_global_config() -> Result, Error> { let Some(config_dir) = dirs::config_dir() else { return Ok(None); }; let path = config_dir.join("wasm-pkg").join("config.toml"); - let contents = match std::fs::read_to_string(path) { + let contents = match tokio::fs::read_to_string(path).await { Ok(contents) => contents, Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), Err(err) => return Err(Error::ConfigFileIoError(err)), @@ -96,8 +96,10 @@ impl Config { } /// Reads config from a TOML file at the given path. - pub fn from_file(path: impl AsRef) -> Result { - let contents = std::fs::read_to_string(path).map_err(Error::ConfigFileIoError)?; + pub async fn from_file(path: impl AsRef) -> Result { + let contents = tokio::fs::read_to_string(path) + .await + .map_err(Error::ConfigFileIoError)?; Self::from_toml(&contents) } @@ -109,9 +111,11 @@ impl Config { } /// Writes the config to a TOML file at the given path. - pub fn to_file(&self, path: impl AsRef) -> Result<(), Error> { + pub async fn to_file(&self, path: impl AsRef) -> Result<(), Error> { let toml_str = ::toml::to_string(&self).map_err(Error::invalid_config)?; - std::fs::write(path, toml_str).map_err(Error::ConfigFileIoError) + tokio::fs::write(path, toml_str) + .await + .map_err(Error::ConfigFileIoError) } /// Merges the given other config into this one. diff --git a/crates/wkg-core/Cargo.toml b/crates/wkg-core/Cargo.toml index 619f075..2455950 100644 --- a/crates/wkg-core/Cargo.toml +++ b/crates/wkg-core/Cargo.toml @@ -10,13 +10,18 @@ license.workspace = true [dependencies] anyhow = { workspace = true } futures-util = { workspace = true } +indexmap = "2.5" semver = { workspace = true } serde = { workspace = true } tokio = { workspace = true, features = ["macros", "rt"] } +tokio-util = { workspace = true, features = ["io", "io-util", "codec"] } toml = { workspace = true } tracing = { workspace = true } +wasm-metadata = { workspace = true } wasm-pkg-common = { workspace = true } wasm-pkg-client = { workspace = true } +wit-component = { workspace = true } +wit-parser = { workspace = true } [target.'cfg(unix)'.dependencies.libc] version = "0.2.153" diff --git a/crates/wkg-core/src/config.rs b/crates/wkg-core/src/config.rs index 8dfb91f..fec5753 100644 --- a/crates/wkg-core/src/config.rs +++ b/crates/wkg-core/src/config.rs @@ -9,6 +9,7 @@ use anyhow::{Context, Result}; use semver::VersionReq; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; +use wasm_metadata::{Link, LinkType, RegistryMetadata}; /// The default name of the configuration file. pub const CONFIG_FILE_NAME: &str = "wkg.toml"; @@ -73,11 +74,11 @@ pub struct Override { #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] pub struct Metadata { /// The authors of the package. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub authors: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authors: Option>, /// The categories of the package. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub categories: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub categories: Option>, /// The package description. #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, @@ -95,6 +96,37 @@ pub struct Metadata { pub repository: Option, } +impl From for wasm_metadata::RegistryMetadata { + fn from(value: Metadata) -> Self { + let mut meta = RegistryMetadata::default(); + meta.set_authors(value.authors); + meta.set_categories(value.categories); + meta.set_description(value.description); + meta.set_license(value.license); + let mut links = Vec::new(); + if let Some(documentation) = value.documentation { + links.push(Link { + ty: LinkType::Documentation, + value: documentation, + }); + } + if let Some(homepage) = value.homepage { + links.push(Link { + ty: LinkType::Homepage, + value: homepage, + }); + } + if let Some(repository) = value.repository { + links.push(Link { + ty: LinkType::Repository, + value: repository, + }); + } + meta.set_links((!links.is_empty()).then_some(links)); + meta + } +} + #[cfg(test)] mod tests { use super::*; @@ -112,8 +144,8 @@ mod tests { }, )])), metadata: Some(Metadata { - authors: vec!["foo".to_string(), "bar".to_string()], - categories: vec!["foo".to_string(), "bar".to_string()], + authors: Some(vec!["foo".to_string(), "bar".to_string()]), + categories: Some(vec!["foo".to_string(), "bar".to_string()]), description: Some("foo".to_string()), license: Some("foo".to_string()), documentation: Some("foo".to_string()), diff --git a/crates/wkg-core/src/lib.rs b/crates/wkg-core/src/lib.rs index 0c364fd..98bbf90 100644 --- a/crates/wkg-core/src/lib.rs +++ b/crates/wkg-core/src/lib.rs @@ -4,3 +4,5 @@ pub mod config; pub mod lock; +pub mod resolver; +pub mod wit; diff --git a/crates/wkg-core/src/lock.rs b/crates/wkg-core/src/lock.rs index 3ad9ae2..d9c0810 100644 --- a/crates/wkg-core/src/lock.rs +++ b/crates/wkg-core/src/lock.rs @@ -1,6 +1,8 @@ //! Type definitions and functions for working with `wkg.lock` files. use std::{ + cmp::Ordering, + collections::{BTreeSet, HashMap}, ops::{Deref, DerefMut}, path::{Path, PathBuf}, }; @@ -14,6 +16,8 @@ use tokio::{ }; use wasm_pkg_client::{ContentDigest, PackageRef}; +use crate::resolver::{DependencyResolution, DependencyResolutionMap}; + /// The default name of the lock file. pub const LOCK_FILE_NAME: &str = "wkg.lock"; /// The version of the lock file for v1 @@ -33,7 +37,7 @@ pub struct LockFile { /// The locked dependencies in the lock file. /// /// This list is sorted by the name of the locked package. - pub packages: Vec, + pub packages: BTreeSet, locker: Locker, } @@ -63,14 +67,13 @@ impl LockFile { /// file and get an exclusive lock on the file, but will not write the data to the file unless /// [`write`](Self::write) is called. pub async fn new_with_path( - mut packages: Vec, + packages: impl IntoIterator, path: impl AsRef, ) -> Result { - sort_packages(&mut packages); let locker = Locker::open_rw(path.as_ref()).await?; Ok(Self { version: LOCK_FILE_V1, - packages, + packages: packages.into_iter().collect(), locker, }) } @@ -86,7 +89,7 @@ impl LockFile { let contents = tokio::fs::read_to_string(path) .await .context("unable to load lock file from path")?; - let mut lock_file: LockFileIntermediate = + let lock_file: LockFileIntermediate = toml::from_str(&contents).context("unable to parse lock file from path")?; // Ensure version is correct and error if it isn't if lock_file.version != LOCK_FILE_V1 { @@ -95,11 +98,31 @@ impl LockFile { lock_file.version )); } - // Ensure packages are sorted by name - sort_packages(&mut lock_file.packages); Ok(lock_file.into_lock_file(locker)) } + /// Creates a lock file from the dependency map. This will create an empty file (if it doesn't + /// exist) and get an exclusive lock on the file, but will not write the data to the file unless + /// [`write`](Self::write) is called. + pub async fn from_dependencies( + map: &DependencyResolutionMap, + path: impl AsRef, + ) -> Result { + let packages = generate_locked_packages(map); + + LockFile::new_with_path(packages, path).await + } + + /// A helper for updating the current lock file with the given dependency map. This will clear current + /// packages that are not in the dependency map and add new packages that are in the dependency + /// map. + /// + /// This function will not write the data to the file unless [`write`](Self::write) is called. + pub fn update_dependencies(&mut self, map: &DependencyResolutionMap) { + self.packages.clear(); + self.packages.extend(generate_locked_packages(map)); + } + /// Attempts to load the lock file from the current directory. Most of the time, users of this /// crate should use this function. Right now it just checks for a `wkg.lock` file in the /// current directory, but we could add more resolution logic in the future. If the file is not @@ -109,18 +132,14 @@ impl LockFile { let lock_path = PathBuf::from(LOCK_FILE_NAME); if !tokio::fs::try_exists(&lock_path).await? { // Create a new lock file if it doesn't exist so we can then open it readonly if that is set - tokio::fs::write(&lock_path, "") - .await - .context("Unable to create lock file")?; + let mut temp_lock = Self::new_with_path([], &lock_path).await?; + temp_lock.write().await?; } Self::load_from_path(lock_path, readonly).await } /// Serializes and writes the lock file - /// - /// This function requires mutability because it needs to sort the packages before serializing. pub async fn write(&mut self) -> Result<()> { - sort_packages(&mut self.packages); let contents = toml::to_string_pretty(self)?; // Truncate the file before writing to it self.locker.file.set_len(0).await.with_context(|| { @@ -152,6 +171,89 @@ impl LockFile { ) }) } + + /// Resolves a package from the lock file. + /// + /// Returns `Ok(None)` if the package cannot be resolved. + /// + /// Fails if the package cannot be resolved and the lock file is not allowed to be updated. + pub fn resolve( + &self, + registry: Option<&str>, + package_ref: &PackageRef, + requirement: &VersionReq, + ) -> Result> { + // NOTE(thomastaylor312): Using a btree map so we don't have to keep sorting the vec. The + // tradeoff is we have to clone two things here to do the fetch. That tradeoff seems fine to + // me, especially because this is used in CLI commands. + if let Some(pkg) = self.packages.get(&LockedPackage { + name: package_ref.clone(), + registry: registry.map(ToString::to_string), + versions: vec![], + }) { + if let Some(locked) = pkg + .versions + .iter() + .find(|locked| &locked.requirement == requirement) + { + tracing::info!(%package_ref, ?registry, %requirement, resolved_version = %locked.version, "dependency package was resolved by the lock file"); + return Ok(Some(locked)); + } + } + + tracing::info!(%package_ref, ?registry, %requirement, "dependency package was not in the lock file"); + Ok(None) + } +} + +fn generate_locked_packages(map: &DependencyResolutionMap) -> impl Iterator { + type PackageKey = (PackageRef, Option); + type VersionsMap = HashMap; + let mut packages: HashMap = HashMap::new(); + + for resolution in map.values() { + match resolution.key() { + Some((id, registry)) => { + let pkg = match resolution { + DependencyResolution::Registry(pkg) => pkg, + DependencyResolution::Local(_) => unreachable!(), + }; + + let prev = packages + .entry((id.clone(), registry.map(str::to_string))) + .or_default() + .insert( + pkg.requirement.to_string(), + (pkg.version.clone(), pkg.digest.clone()), + ); + + if let Some((prev, _)) = prev { + // The same requirements should resolve to the same version + assert!(prev == pkg.version) + } + } + None => continue, + } + } + + packages.into_iter().map(|((name, registry), versions)| { + let versions: Vec = versions + .into_iter() + .map(|(requirement, (version, digest))| LockedPackageVersion { + requirement: requirement + .parse() + .expect("Version requirement should have been valid. This is programmer error"), + version, + digest, + }) + .collect(); + + LockedPackage { + name, + registry, + versions, + } + }) } /// Represents a locked package in a lock file. @@ -161,7 +263,8 @@ pub struct LockedPackage { pub name: PackageRef, /// The registry the package was resolved from. - #[serde(default, skip_serializing_if = "Option::is_none")] + // NOTE(thomastaylor312): This is a string instead of using the `Registry` type because clippy + // is complaining about it being an interior mutable key type for the btreeset pub registry: Option, /// The locked version of a package. @@ -172,28 +275,39 @@ pub struct LockedPackage { pub versions: Vec, } +impl Ord for LockedPackage { + fn cmp(&self, other: &Self) -> Ordering { + if self.name == other.name { + self.registry.cmp(&other.registry) + } else { + self.name.cmp(&other.name) + } + } +} + +impl PartialOrd for LockedPackage { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// Represents version information for a locked package. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct LockedPackageVersion { - /// The version requirement used to resolve this version (if used). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub requirement: Option, + /// The version requirement used to resolve this version + pub requirement: VersionReq, /// The version the package is locked to. pub version: Version, /// The digest of the package contents. pub digest: ContentDigest, } -fn sort_packages(packages: &mut [LockedPackage]) { - packages.sort_unstable_by(|a, b| a.name.cmp(&b.name)); -} - -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] struct LockFileIntermediate { version: u64, #[serde(alias = "package", default, skip_serializing_if = "Vec::is_empty")] - packages: Vec, + packages: BTreeSet, } impl LockFileIntermediate { @@ -696,13 +810,13 @@ mod tests { let mut fakehasher = sha2::Sha256::new(); fakehasher.update(b"fake"); - let mut expected_deps = vec![ + let mut expected_deps = BTreeSet::from([ LockedPackage { name: "enterprise:holodeck".parse().unwrap(), versions: vec![LockedPackageVersion { version: "0.1.0".parse().unwrap(), digest: fakehasher.clone().into(), - requirement: None, + requirement: VersionReq::parse("=0.1.0").unwrap(), }], registry: None, }, @@ -711,37 +825,29 @@ mod tests { versions: vec![LockedPackageVersion { version: "0.1.0".parse().unwrap(), digest: fakehasher.clone().into(), - requirement: None, + requirement: VersionReq::parse("=0.1.0").unwrap(), }], registry: None, }, - ]; + ]); let mut lock = LockFile::new_with_path(expected_deps.clone(), &path) .await .expect("Shouldn't fail when creating a new lock file"); - // Now sort the expected deps and make sure the lock file deps also got sorted - sort_packages(&mut expected_deps); - assert_eq!( - lock.packages, expected_deps, - "Lock file deps should match expected deps" - ); - // Push one more package onto the lock file before writing it let new_package = LockedPackage { name: "defiant:armor".parse().unwrap(), versions: vec![LockedPackageVersion { version: "0.1.0".parse().unwrap(), digest: fakehasher.into(), - requirement: None, + requirement: VersionReq::parse("=0.1.0").unwrap(), }], registry: None, }; - lock.packages.push(new_package.clone()); - expected_deps.push(new_package); - sort_packages(&mut expected_deps); + lock.packages.insert(new_package.clone()); + expected_deps.insert(new_package); lock.write() .await @@ -750,17 +856,6 @@ mod tests { // Drop the lock file drop(lock); - // Parse out the file into the intermediate format and make sure the vec is in the same - // order as the expected deps - let lock_file = - std::fs::read_to_string(&path).expect("Shouldn't fail when reading lock file"); - let lock_file: LockFileIntermediate = - toml::from_str(&lock_file).expect("Shouldn't fail when parsing lock file"); - assert_eq!( - lock_file.packages, expected_deps, - "Lock file deps should match expected deps" - ); - // Now read the lock file again and make sure everything is correct (and we can lock it // properly) let lock = LockFile::load_from_path(&path, true) diff --git a/crates/wkg-core/src/resolver.rs b/crates/wkg-core/src/resolver.rs new file mode 100644 index 0000000..f027986 --- /dev/null +++ b/crates/wkg-core/src/resolver.rs @@ -0,0 +1,755 @@ +//! A resolver for resolving dependencies from a component registry. +// NOTE(thomastaylor312): This is copied and adapted from the `cargo-component` crate: https://github.com/bytecodealliance/cargo-component/blob/f0be1c7d9917aa97e9102e69e3b838dae38d624b/crates/core/src/registry.rs + +use std::{ + collections::{hash_map, HashMap, HashSet}, + fmt::Debug, + ops::{Deref, DerefMut}, + path::{Path, PathBuf}, + str::FromStr, +}; + +use anyhow::{bail, Context, Result}; +use futures_util::TryStreamExt; +use indexmap::{IndexMap, IndexSet}; +use semver::{Comparator, Op, Version, VersionReq}; +use tokio::io::{AsyncRead, AsyncReadExt}; +use wasm_pkg_client::{ + caching::{CachingClient, FileCache}, + Client, Config, ContentDigest, Error as WasmPkgError, PackageRef, Release, VersionInfo, +}; +use wit_component::DecodedWasm; +use wit_parser::{PackageId, PackageName, Resolve, UnresolvedPackageGroup, WorldId}; + +use crate::lock::LockFile; + +/// The name of the default registry. +pub const DEFAULT_REGISTRY_NAME: &str = "default"; + +// TODO: functions for resolving dependencies from a lock file + +/// Represents a WIT package dependency. +#[derive(Debug, Clone)] +pub enum Dependency { + /// The dependency is a registry package. + Package(RegistryPackage), + + /// The dependency is a path to a local directory or file. + Local(PathBuf), +} + +impl FromStr for Dependency { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(Self::Package(s.parse()?)) + } +} + +/// Represents a reference to a registry package. +#[derive(Debug, Clone)] +pub struct RegistryPackage { + /// The name of the package. + /// + /// If not specified, the name from the mapping will be used. + pub name: Option, + + /// The version requirement of the package. + pub version: VersionReq, + + /// The name of the component registry containing the package. + /// + /// If not specified, the default registry is used. + pub registry: Option, +} + +impl FromStr for RegistryPackage { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(Self { + name: None, + version: s + .parse() + .with_context(|| format!("'{s}' is an invalid registry package version"))?, + registry: None, + }) + } +} + +/// Represents information about a resolution of a registry package. +#[derive(Clone)] +pub struct RegistryResolution { + /// The name of the dependency that was resolved. + /// + /// This may differ from `package` if the dependency was renamed. + pub name: PackageRef, + /// The name of the package from the registry that was resolved. + pub package: PackageRef, + /// The name of the registry used to resolve the package if one was provided + pub registry: Option, + /// The version requirement that was used to resolve the package. + pub requirement: VersionReq, + /// The package version that was resolved. + pub version: Version, + /// The digest of the package contents. + pub digest: ContentDigest, + /// The client to use for fetching the package contents. + client: CachingClient, +} + +impl Debug for RegistryResolution { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("RegistryResolution") + .field("name", &self.name) + .field("package", &self.package) + .field("registry", &self.registry) + .field("requirement", &self.requirement) + .field("version", &self.version) + .field("digest", &self.digest) + .finish() + } +} + +impl RegistryResolution { + /// Fetches the raw package bytes from the registry. Returns an AsyncRead that will stream the + /// package contents + pub async fn fetch(&self) -> Result { + let stream = self + .client + .get_content( + &self.package, + &Release { + version: self.version.clone(), + content_digest: self.digest.clone(), + }, + ) + .await?; + + Ok(tokio_util::io::StreamReader::new(stream.map_err(|e| { + std::io::Error::new(std::io::ErrorKind::Other, e) + }))) + } +} + +/// Represents information about a resolution of a local file. +#[derive(Clone, Debug)] +pub struct LocalResolution { + /// The name of the dependency that was resolved. + pub name: PackageRef, + /// The path to the resolved dependency. + pub path: PathBuf, +} + +/// Represents a resolution of a dependency. +#[derive(Debug, Clone)] +#[allow(clippy::large_enum_variant)] +pub enum DependencyResolution { + /// The dependency is resolved from a registry package. + Registry(RegistryResolution), + /// The dependency is resolved from a local path. + Local(LocalResolution), +} + +impl DependencyResolution { + /// Gets the name of the dependency that was resolved. + pub fn name(&self) -> &PackageRef { + match self { + Self::Registry(res) => &res.name, + Self::Local(res) => &res.name, + } + } + + /// Gets the resolved version. + /// + /// Returns `None` if the dependency is not resolved from a registry package. + pub fn version(&self) -> Option<&Version> { + match self { + Self::Registry(res) => Some(&res.version), + Self::Local(_) => None, + } + } + + /// The key used in sorting and searching the lock file package list. + /// + /// Returns `None` if the dependency is not resolved from a registry package. + pub fn key(&self) -> Option<(&PackageRef, Option<&str>)> { + match self { + DependencyResolution::Registry(pkg) => Some((&pkg.package, pkg.registry.as_deref())), + DependencyResolution::Local(_) => None, + } + } + + /// Decodes the resolved dependency. + pub async fn decode(&self) -> Result { + // If the dependency path is a directory, assume it contains wit to parse as a package. + let bytes = match self { + DependencyResolution::Local(LocalResolution { path, .. }) + if tokio::fs::metadata(path).await?.is_dir() => + { + return Ok(DecodedDependency::Wit { + resolution: self, + package: UnresolvedPackageGroup::parse_dir(path).with_context(|| { + format!("failed to parse dependency `{path}`", path = path.display()) + })?, + }); + } + DependencyResolution::Local(LocalResolution { path, .. }) => { + tokio::fs::read(path).await.with_context(|| { + format!( + "failed to read content of dependency `{name}` at path `{path}`", + name = self.name(), + path = path.display() + ) + })? + } + DependencyResolution::Registry(res) => { + let mut reader = res.fetch().await?; + + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await?; + buf + } + }; + + if &bytes[0..4] != b"\0asm" { + return Ok(DecodedDependency::Wit { + resolution: self, + package: UnresolvedPackageGroup::parse( + // This is fake, but it's needed for the parser to work. + self.name().to_string(), + std::str::from_utf8(&bytes).with_context(|| { + format!( + "dependency `{name}` is not UTF-8 encoded", + name = self.name() + ) + })?, + )?, + }); + } + + Ok(DecodedDependency::Wasm { + resolution: self, + decoded: wit_component::decode(&bytes).with_context(|| { + format!( + "failed to decode content of dependency `{name}`", + name = self.name(), + ) + })?, + }) + } +} + +/// Represents a decoded dependency. +pub enum DecodedDependency<'a> { + /// The dependency decoded from an unresolved WIT package. + Wit { + /// The resolution related to the decoded dependency. + resolution: &'a DependencyResolution, + /// The unresolved WIT package. + package: UnresolvedPackageGroup, + }, + /// The dependency decoded from a Wasm file. + Wasm { + /// The resolution related to the decoded dependency. + resolution: &'a DependencyResolution, + /// The decoded Wasm file. + decoded: DecodedWasm, + }, +} + +impl<'a> DecodedDependency<'a> { + /// Fully resolves the dependency. + /// + /// If the dependency is an unresolved WIT package, it will assume that the + /// package has no foreign dependencies. + pub fn resolve(self) -> Result<(Resolve, PackageId, Vec)> { + match self { + Self::Wit { package, .. } => { + let mut resolve = Resolve::new(); + let source_files = package + .source_map + .source_files() + .map(Path::to_path_buf) + .collect(); + let pkg = resolve.push_group(package)?; + Ok((resolve, pkg, source_files)) + } + Self::Wasm { decoded, .. } => match decoded { + DecodedWasm::WitPackage(resolve, pkg) => Ok((resolve, pkg, Vec::new())), + DecodedWasm::Component(resolve, world) => { + let pkg = resolve.worlds[world].package.unwrap(); + Ok((resolve, pkg, Vec::new())) + } + }, + } + } + + /// Gets the package name of the decoded dependency. + pub fn package_name(&self) -> &PackageName { + match self { + Self::Wit { package, .. } => &package.main.name, + Self::Wasm { decoded, .. } => &decoded.resolve().packages[decoded.package()].name, + } + } + + /// Converts the decoded dependency into a component world. + /// + /// Returns an error if the dependency is not a decoded component. + pub fn into_component_world(self) -> Result<(Resolve, WorldId)> { + match self { + Self::Wasm { + decoded: DecodedWasm::Component(resolve, world), + .. + } => Ok((resolve, world)), + _ => bail!("dependency is not a WebAssembly component"), + } + } +} + +/// Used to resolve dependencies for a WIT package. +pub struct DependencyResolver<'a> { + client: CachingClient, + lock_file: Option<&'a LockFile>, + packages: HashMap>, + dependencies: HashMap, + resolutions: DependencyResolutionMap, +} + +impl<'a> DependencyResolver<'a> { + /// Creates a new dependency resolver. If `config` is `None`, then the resolver will be set to + /// offline mode and a lock file must be given as well. Anything that will require network + /// access will fail in offline mode. + pub fn new( + config: Option, + lock_file: Option<&'a LockFile>, + cache: FileCache, + ) -> anyhow::Result { + if config.is_none() && lock_file.is_none() { + anyhow::bail!("lock file must be provided when offline mode is enabled"); + } + let client = CachingClient::new(config.map(Client::new), cache); + Ok(DependencyResolver { + client, + lock_file, + resolutions: Default::default(), + packages: Default::default(), + dependencies: Default::default(), + }) + } + + /// Creates a new dependency resolver with the given client. This is useful when you already + /// have a client available. If the client is set to offline mode, then a lock file must be + /// given or this will error + pub fn new_with_client( + client: CachingClient, + lock_file: Option<&'a LockFile>, + ) -> anyhow::Result { + if client.is_readonly() && lock_file.is_none() { + anyhow::bail!("lock file must be provided when offline mode is enabled"); + } + Ok(DependencyResolver { + client, + lock_file, + resolutions: Default::default(), + packages: Default::default(), + dependencies: Default::default(), + }) + } + + /// Add a dependency to the resolver. If the dependency already exists, then it will be ignored. + /// To override an existing dependency, use [`override_dependency`](Self::override_dependency). + pub async fn add_dependency( + &mut self, + name: &PackageRef, + dependency: &Dependency, + ) -> Result<()> { + self.add_dependency_internal(name, dependency, false).await + } + + /// Add a dependency to the resolver. If the dependency already exists, then it will be + /// overridden. + pub async fn override_dependency( + &mut self, + name: &PackageRef, + dependency: &Dependency, + ) -> Result<()> { + self.add_dependency_internal(name, dependency, true).await + } + + async fn add_dependency_internal( + &mut self, + name: &PackageRef, + dependency: &Dependency, + force_override: bool, + ) -> Result<()> { + match dependency { + Dependency::Package(package) => { + // Dependency comes from a registry, add a dependency to the resolver + let registry_name = package.registry.as_deref().or_else(|| { + self.client.client().ok().and_then(|client| { + client + .config() + .resolve_registry(name) + .map(|reg| reg.as_ref()) + }) + }); + let package_name = package.name.clone().unwrap_or_else(|| name.clone()); + + // Resolve the version from the lock file if there is one + let locked = match self.lock_file.as_ref().and_then(|resolver| { + resolver + .resolve(registry_name, &package_name, &package.version) + .transpose() + }) { + Some(Ok(locked)) => Some(locked), + Some(Err(e)) => return Err(e), + _ => None, + }; + + if !force_override && self.resolutions.contains_key(name) { + tracing::debug!(%name, "dependency already exists and override is not set, ignoring"); + return Ok(()); + } + self.dependencies.insert( + name.to_owned(), + RegistryDependency { + package: package_name, + version: package.version.clone(), + locked: locked.map(|l| (l.version.clone(), l.digest.clone())), + }, + ); + } + Dependency::Local(p) => { + // A local path dependency, insert a resolution immediately + let res = DependencyResolution::Local(LocalResolution { + name: name.clone(), + path: p.clone(), + }); + + if !force_override && self.resolutions.contains_key(name) { + tracing::debug!(%name, "dependency already exists and override is not set, ignoring"); + return Ok(()); + } + + let prev = self.resolutions.insert(name.clone(), res); + assert!(prev.is_none()); + } + } + + Ok(()) + } + + /// Resolve all dependencies. + /// + /// This will download all dependencies that are not already present in client storage. + /// + /// Returns the dependency resolution map. + pub async fn resolve(mut self) -> Result { + let mut resolutions = self.resolutions; + for (name, dependency) in self.dependencies.into_iter() { + // We need to clone a handle to the client because we mutably borrow self below. Might + // be worth replacing the mutable borrow with a RwLock down the line. + let client = self.client.clone(); + + let (selected_version, digest) = if client.is_readonly() { + dependency + .locked + .as_ref() + .map(|(ver, digest)| (ver, Some(digest))) + .ok_or_else(|| { + anyhow::anyhow!("Couldn't find locked dependency while in offline mode") + })? + } else { + let versions = + load_package(&mut self.packages, &self.client, dependency.package.clone()) + .await? + .with_context(|| { + format!( + "package `{name}` was not found in component registry", + name = dependency.package + ) + })?; + + match &dependency.locked { + Some((version, digest)) => { + // The dependency had a lock file entry, so attempt to do an exact match first + let exact_req = VersionReq { + comparators: vec![Comparator { + op: Op::Exact, + major: version.major, + minor: Some(version.minor), + patch: Some(version.patch), + pre: version.pre.clone(), + }], + }; + + // If an exact match can't be found, fallback to the latest release to satisfy + // the version requirement; this can happen when packages are yanked. If we did + // find an exact match, return the digest for comparison after fetching the + // release + find_latest_release(versions, &exact_req).map(|v| (&v.version, Some(digest))).or_else(|| find_latest_release(versions, &dependency.version).map(|v| (&v.version, None))) + } + None => find_latest_release(versions, &dependency.version).map(|v| (&v.version, None)), + }.with_context(|| format!("component registry package `{name}` has no release matching version requirement `{version}`", name = dependency.package, version = dependency.version))? + }; + + // We need to clone a handle to the client because we mutably borrow self above. Might + // be worth replacing the mutable borrow with a RwLock down the line. + let release = client + .get_release(&dependency.package, selected_version) + .await?; + if let Some(digest) = digest { + if &release.content_digest != digest { + bail!( + "component registry package `{name}` (v`{version}`) has digest `{content}` but the lock file specifies digest `{digest}`", + name = dependency.package, + version = release.version, + content = release.content_digest, + ); + } + } + let resolution = RegistryResolution { + name: name.clone(), + package: dependency.package.clone(), + registry: self.client.client().ok().and_then(|client| { + client + .config() + .resolve_registry(&name) + .map(ToString::to_string) + }), + requirement: dependency.version.clone(), + version: release.version.clone(), + digest: release.content_digest.clone(), + client: self.client.clone(), + }; + resolutions.insert(name, DependencyResolution::Registry(resolution)); + } + + Ok(resolutions) + } +} + +async fn load_package<'b>( + packages: &'b mut HashMap>, + client: &CachingClient, + package: PackageRef, +) -> Result>> { + match packages.entry(package) { + hash_map::Entry::Occupied(e) => Ok(Some(e.into_mut())), + hash_map::Entry::Vacant(e) => match client.list_all_versions(e.key()).await { + Ok(p) => Ok(Some(e.insert(p))), + Err(WasmPkgError::PackageNotFound) => Ok(None), + Err(err) => Err(err.into()), + }, + } +} + +struct RegistryDependency { + /// The canonical package name of the registry package. In most cases, this is the same as the + /// name but could be different if the given package name has been remapped + package: PackageRef, + version: VersionReq, + locked: Option<(Version, ContentDigest)>, +} + +fn find_latest_release<'a>( + versions: &'a [VersionInfo], + req: &VersionReq, +) -> Option<&'a VersionInfo> { + versions + .iter() + .filter(|info| !info.yanked && req.matches(&info.version)) + .max_by(|a, b| a.version.cmp(&b.version)) +} + +// NOTE(thomastaylor312): This is copied from the old wit package in the cargo-component and broken +// out for some reuse. I don't know enough about resolvers to know if there is an easier way to +// write this, so any future people seeing this should feel free to refactor it if they know a +// better way to do it. + +/// Represents a map of dependency resolutions. +/// +/// The key to the map is the package name of the dependency. +#[derive(Debug, Clone, Default)] +pub struct DependencyResolutionMap(HashMap); + +impl AsRef> for DependencyResolutionMap { + fn as_ref(&self) -> &HashMap { + &self.0 + } +} + +impl Deref for DependencyResolutionMap { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for DependencyResolutionMap { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl DependencyResolutionMap { + /// Fetch all dependencies and ensure there are no circular dependencies. Returns the decoded + /// dependencies (sorted topologically), ready to use for output or adding to a [`Resolve`]. + pub async fn decode_dependencies( + &self, + ) -> Result>> { + // Start by decoding all of the dependencies + let mut deps = IndexMap::new(); + for (name, resolution) in self.0.iter() { + let decoded = resolution.decode().await?; + if let Some(prev) = deps.insert(decoded.package_name().clone(), decoded) { + anyhow::bail!( + "duplicate definitions of package `{prev}` found while decoding dependency `{name}`", + prev = prev.package_name() + ); + } + } + + // Do a topological sort of the dependencies + let mut order = IndexSet::new(); + let mut visiting = HashSet::new(); + for dep in deps.values() { + visit(dep, &deps, &mut order, &mut visiting)?; + } + + assert!(visiting.is_empty()); + + // Now that we have the right order, re-order the dependencies to match + deps.sort_by(|name_a, _, name_b, _| { + order.get_index_of(name_a).cmp(&order.get_index_of(name_b)) + }); + + Ok(deps) + } + + /// Given a path to a component or a directory containing wit, use the given dependencies to + /// generate a [`Resolve`] for the root package. + pub async fn generate_resolve(&self, dir: impl AsRef) -> Result<(Resolve, PackageId)> { + let mut merged = Resolve::default(); + + let deps = self.decode_dependencies().await?; + + // Parse the root package itself + let root = UnresolvedPackageGroup::parse_dir(&dir).with_context(|| { + format!( + "failed to parse package from directory `{dir}`", + dir = dir.as_ref().display() + ) + })?; + + let mut source_files: Vec<_> = root + .source_map + .source_files() + .map(Path::to_path_buf) + .collect(); + + // Merge all of the dependencies first + for decoded in deps.into_values() { + match decoded { + DecodedDependency::Wit { + resolution, + package, + } => { + source_files.extend(package.source_map.source_files().map(Path::to_path_buf)); + merged.push_group(package).with_context(|| { + format!( + "failed to merge dependency `{name}`", + name = resolution.name() + ) + })?; + } + DecodedDependency::Wasm { + resolution, + decoded, + } => { + let resolve = match decoded { + DecodedWasm::WitPackage(resolve, _) => resolve, + DecodedWasm::Component(resolve, _) => resolve, + }; + + merged.merge(resolve).with_context(|| { + format!( + "failed to merge world of dependency `{name}`", + name = resolution.name() + ) + })?; + } + }; + } + + let package = merged.push_group(root).with_context(|| { + format!( + "failed to merge package from directory `{dir}`", + dir = dir.as_ref().display() + ) + })?; + + Ok((merged, package)) + } +} + +fn visit<'a>( + dep: &'a DecodedDependency<'a>, + deps: &'a IndexMap, + order: &mut IndexSet, + visiting: &mut HashSet<&'a PackageName>, +) -> Result<()> { + if order.contains(dep.package_name()) { + return Ok(()); + } + + // Visit any unresolved foreign dependencies + match dep { + DecodedDependency::Wit { + package, + resolution, + } => { + for name in package.main.foreign_deps.keys() { + // Only visit known dependencies + // wit-parser will error on unknown foreign dependencies when + // the package is resolved + if let Some(dep) = deps.get(name) { + if !visiting.insert(name) { + anyhow::bail!("foreign dependency `{name}` forms a dependency cycle while parsing dependency `{other}`", other = resolution.name()); + } + + visit(dep, deps, order, visiting)?; + assert!(visiting.remove(name)); + } + } + } + DecodedDependency::Wasm { + decoded, + resolution, + } => { + // Look for foreign packages in the decoded dependency + for (_, package) in &decoded.resolve().packages { + if package.name.namespace == dep.package_name().namespace + && package.name.name == dep.package_name().name + { + continue; + } + + if let Some(dep) = deps.get(&package.name) { + if !visiting.insert(&package.name) { + anyhow::bail!("foreign dependency `{name}` forms a dependency cycle while parsing dependency `{other}`", name = package.name, other = resolution.name()); + } + + visit(dep, deps, order, visiting)?; + assert!(visiting.remove(&package.name)); + } + } + } + } + + assert!(order.insert(dep.package_name().clone())); + + Ok(()) +} diff --git a/crates/wkg-core/src/wit.rs b/crates/wkg-core/src/wit.rs new file mode 100644 index 0000000..0238826 --- /dev/null +++ b/crates/wkg-core/src/wit.rs @@ -0,0 +1,402 @@ +//! Functions for building WIT packages and fetching their dependencies. + +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + str::FromStr, +}; + +use anyhow::{Context, Result}; +use semver::{Version, VersionReq}; +use wasm_pkg_client::{ + caching::{CachingClient, FileCache}, + PackageRef, +}; +use wit_component::{DecodedWasm, WitPrinter}; +use wit_parser::{PackageId, PackageName, Resolve}; + +use crate::{ + config::Config, + lock::LockFile, + resolver::{ + DecodedDependency, Dependency, DependencyResolution, DependencyResolutionMap, + DependencyResolver, LocalResolution, RegistryPackage, + }, +}; + +/// The supported output types for WIT deps +#[derive(Debug, Clone, Copy, Default)] +pub enum OutputType { + /// Output each dependency as a WIT file in the deps directory. + Wit, + /// Output each dependency as a wasm binary file in the deps directory. + #[default] + Wasm, +} + +impl FromStr for OutputType { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result { + let lower_trim = s.trim().to_lowercase(); + match lower_trim.as_str() { + "wit" => Ok(Self::Wit), + "wasm" => Ok(Self::Wasm), + _ => Err(anyhow::anyhow!("Invalid output type: {}", s)), + } + } +} + +/// Builds a WIT package given the configuration and directory to parse. Will update the given lock +/// file with the resolved dependencies but will not write it to disk. +pub async fn build_package( + config: &Config, + wit_dir: impl AsRef, + lock_file: &mut LockFile, + client: CachingClient, +) -> Result<(PackageRef, Option, Vec)> { + let dependencies = resolve_dependencies(config, &wit_dir, Some(lock_file), client) + .await + .context("Unable to resolve dependencies")?; + + lock_file.update_dependencies(&dependencies); + + let (resolve, pkg_id) = dependencies.generate_resolve(wit_dir).await?; + + let pkg = &resolve.packages[pkg_id]; + let name = PackageRef::new( + pkg.name + .namespace + .parse() + .context("Invalid namespace found in package")?, + pkg.name + .name + .parse() + .context("Invalid name found for package")?, + ); + + let bytes = wit_component::encode(Some(true), &resolve, pkg_id)?; + + let mut producers = wasm_metadata::Producers::empty(); + producers.add( + "processed-by", + env!("CARGO_PKG_NAME"), + option_env!("WIT_VERSION_INFO").unwrap_or(env!("CARGO_PKG_VERSION")), + ); + + let mut bytes = producers + .add_to_wasm(&bytes) + .context("failed to add producers metadata to output WIT package")?; + + if let Some(meta) = config.metadata.clone() { + let meta = wasm_metadata::RegistryMetadata::from(meta); + bytes = meta.add_to_wasm(&bytes)?; + } + + Ok((name, pkg.name.version.clone(), bytes)) +} + +/// Fetches and optionally updates all dependencies for the given path and writes them in the +/// specified format. The lock file will be updated with the resolved dependencies but will not be +/// written to disk. +/// +/// This is mostly a convenience wrapper around [`resolve_dependencies`] and [`populate_dependencies`]. +pub async fn fetch_dependencies( + config: &Config, + wit_dir: impl AsRef, + lock_file: &mut LockFile, + client: CachingClient, + output: OutputType, +) -> Result<()> { + // Don't pass lock file if update is true + let dependencies = resolve_dependencies(config, &wit_dir, Some(lock_file), client).await?; + lock_file.update_dependencies(&dependencies); + populate_dependencies(wit_dir, &dependencies, output).await +} + +/// Generate the list of all packages and their version requirement from the given path (a directory +/// or file). +/// +/// This is a lower level function exposed for convenience that is used by higher level functions +/// for resolving dependencies. +pub fn get_packages( + path: impl AsRef, +) -> Result<(PackageRef, HashSet<(PackageRef, VersionReq)>)> { + let group = wit_parser::UnresolvedPackageGroup::parse_path(path)?; + + let name = PackageRef::new( + group + .main + .name + .namespace + .parse() + .context("Invalid namespace found in package")?, + group + .main + .name + .name + .parse() + .context("Invalid name found in package")?, + ); + + // Get all package refs from the main package and then from any nested packages + let packages: HashSet<(PackageRef, VersionReq)> = + packages_from_foreign_deps(group.main.foreign_deps.into_keys()) + .chain( + group + .nested + .into_iter() + .flat_map(|pkg| packages_from_foreign_deps(pkg.foreign_deps.into_keys())), + ) + .collect(); + + Ok((name, packages)) +} + +/// Builds a list of resolved dependencies loaded from the component or path containing the WIT. +/// This will configure the resolver, override any dependencies from configuration and resolve the +/// dependency map. This map can then be used in various other functions for fetching the +/// dependencies and/or building a final resolved package. +pub async fn resolve_dependencies( + config: &Config, + path: impl AsRef, + lock_file: Option<&LockFile>, + client: CachingClient, +) -> Result { + let mut resolver = DependencyResolver::new_with_client(client, lock_file)?; + // add deps from config first in case they're local deps and then add deps from the directory + if let Some(overrides) = config.overrides.as_ref() { + for (pkg, ovride) in overrides.iter() { + let pkg: PackageRef = pkg.parse().context("Unable to parse as a package ref")?; + let dep = match (ovride.path.as_ref(), ovride.version.as_ref()) { + (Some(path), None) => { + let path = tokio::fs::canonicalize(path).await?; + Dependency::Local(path) + } + (Some(path), Some(_)) => { + tracing::warn!("Ignoring version override for local package"); + let path = tokio::fs::canonicalize(path).await?; + Dependency::Local(path) + } + (None, Some(version)) => Dependency::Package(RegistryPackage { + name: Some(pkg.clone()), + version: version.to_owned(), + registry: None, + }), + (None, None) => { + tracing::warn!("Found override without version or path, ignoring"); + continue; + } + }; + resolver + .add_dependency(&pkg, &dep) + .await + .context("Unable to add dependency")?; + } + } + let (_name, packages) = get_packages(path)?; + add_packages_to_resolver(&mut resolver, packages).await?; + resolver.resolve().await +} + +/// Populate a list of dependencies into the given directory. If the directory does not exist it +/// will be created. Any existing files in the directory will be deleted. The dependencies will be +/// put into the `deps` subdirectory within the directory in the format specified by the output +/// type. Please note that if a local dep is encountered when using [`OutputType::Wasm`] and it +/// isn't a wasm binary, it will be copied directly to the directory and not packaged into a wit +/// package first +pub async fn populate_dependencies( + path: impl AsRef, + deps: &DependencyResolutionMap, + output: OutputType, +) -> Result<()> { + // Canonicalizing will error if the path doesn't exist, so we don't need to check for that + let path = tokio::fs::canonicalize(path).await?; + let metadata = tokio::fs::metadata(&path).await?; + if !metadata.is_dir() { + anyhow::bail!("Path is not a directory"); + } + let deps_path = path.join("deps"); + // Remove the whole directory if it already exists and then recreate + if let Err(e) = tokio::fs::remove_dir_all(&deps_path).await { + // If the directory doesn't exist, ignore the error + if e.kind() != std::io::ErrorKind::NotFound { + return Err(anyhow::anyhow!("Unable to remove deps directory: {e}")); + } + } + tokio::fs::create_dir_all(&deps_path).await?; + + let decoded_deps = deps.decode_dependencies().await?; + + for (name, dep) in decoded_deps.iter() { + let mut output_path = deps_path.join(name.to_string()); + + match (dep, output) { + ( + DecodedDependency::Wit { + resolution: DependencyResolution::Local(local), + .. + }, + _, + ) => { + // Local deps always need to be written to a subdirectory of deps so create that here + tokio::fs::create_dir_all(&output_path).await?; + write_local_dep(local, output_path).await?; + } + // This case shouldn't happen because registries only support wit packages. We can't get + // a resolve from the unresolved group, so error out here. Ideally we could print the + // unresolved group, but WitPrinter doesn't support that yet + ( + DecodedDependency::Wit { + resolution: DependencyResolution::Registry(_), + .. + }, + _, + ) => { + anyhow::bail!("Unable to resolve dependency, this is a programmer error"); + } + (DecodedDependency::Wasm { decoded, .. }, OutputType::Wit) => { + tokio::fs::create_dir_all(&output_path).await?; + let (resolve, pkg) = match decoded { + DecodedWasm::WitPackage(r, p) => (r, *p), + DecodedWasm::Component(r, world_id) => { + let pkg_id = r + .worlds + .iter() + .find_map(|(id, w)| { + (id == *world_id).then_some(w).and_then(|w| w.package) + }) + .ok_or_else(|| { + anyhow::anyhow!("Unable to find package for the component's world") + })?; + (r, pkg_id) + } + }; + // Print all the deps + print_wit_deps(resolve, pkg, name, &deps_path).await?; + } + // Right now WIT packages include all of their dependencies, so we don't need to fetch + // those too. In the future, we'll need to look for unsatisfied dependencies and fetch + // them + (DecodedDependency::Wasm { resolution, .. }, OutputType::Wasm) => { + // This is going to be written to a single file, so we don't create a directory here + // NOTE(thomastaylor312): This janky looking thing is to avoid chopping off the + // patch number from the release. Once `add_extension` is stabilized, we can use + // that instead + let mut file_name = output_path.file_name().unwrap().to_owned(); + file_name.push(".wasm"); + output_path.set_file_name(file_name); + match resolution { + DependencyResolution::Local(local) => { + let meta = tokio::fs::metadata(&local.path).await?; + if !meta.is_file() { + anyhow::bail!("Local dependency is not single wit package file"); + } + tokio::fs::copy(&local.path, output_path) + .await + .context("Unable to copy local dependency")?; + } + DependencyResolution::Registry(registry) => { + let mut reader = registry.fetch().await?; + let mut output_file = tokio::fs::File::create(output_path).await?; + tokio::io::copy(&mut reader, &mut output_file).await?; + output_file.sync_all().await?; + } + } + } + } + } + Ok(()) +} + +fn packages_from_foreign_deps( + deps: impl IntoIterator, +) -> impl Iterator { + deps.into_iter().filter_map(|dep| { + let name = PackageRef::new(dep.namespace.parse().ok()?, dep.name.parse().ok()?); + let version = match dep.version { + Some(v) => format!("={v}"), + None => "*".to_string(), + }; + Some(( + name, + version + .parse() + .expect("Unable to parse into version request, this is programmer error"), + )) + }) +} + +async fn add_packages_to_resolver( + resolver: &mut DependencyResolver<'_>, + packages: impl IntoIterator, +) -> Result<()> { + for (package, req) in packages { + resolver + .add_dependency( + &package, + &Dependency::Package(RegistryPackage { + name: Some(package.clone()), + version: req, + registry: None, + }), + ) + .await?; + } + Ok(()) +} + +async fn write_local_dep(local: &LocalResolution, output_path: impl AsRef) -> Result<()> { + let meta = tokio::fs::metadata(&local.path).await?; + if meta.is_file() { + tokio::fs::copy( + &local.path, + output_path.as_ref().join(local.path.file_name().unwrap()), + ) + .await?; + } else { + // For now, don't try to recurse, since most of the tools don't recurse unless + // there is a deps folder anyway, which we don't care about here + let mut dir = tokio::fs::read_dir(&local.path).await?; + while let Some(entry) = dir.next_entry().await? { + if !entry.metadata().await?.is_file() { + continue; + } + let entry_path = entry.path(); + tokio::fs::copy( + &entry_path, + output_path.as_ref().join(entry_path.file_name().unwrap()), + ) + .await?; + } + } + Ok(()) +} + +/// Recursive function that prints the top level dep given and then iterates over each of its dependencies +async fn print_wit_deps( + resolve: &Resolve, + pkg_id: PackageId, + top_level_name: &PackageName, + root_deps_dir: &PathBuf, +) -> Result<()> { + // Print the top level package first, then iterate over all dependencies and print them + let dep_path = root_deps_dir.join(top_level_name.to_string()); + tokio::fs::create_dir_all(&dep_path).await?; + let mut printer = WitPrinter::default(); + let wit = printer + .print(resolve, pkg_id, &[]) + .context("Unable to print wit")?; + tokio::fs::write(dep_path.join("package.wit"), wit).await?; + for pkg in resolve.package_direct_deps(pkg_id) { + // Get the package name + let package_name = resolve + .package_names + .iter() + .find_map(|(name, id)| (*id == pkg).then_some(name)) + .ok_or_else(|| anyhow::anyhow!("Got package ID from world that didn't exist"))?; + // Recurse into this package + Box::pin(print_wit_deps(resolve, pkg, package_name, root_deps_dir)).await?; + } + Ok(()) +} diff --git a/crates/wkg/Cargo.toml b/crates/wkg/Cargo.toml index 1949080..ba6f114 100644 --- a/crates/wkg/Cargo.toml +++ b/crates/wkg/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true [dependencies] anyhow = { workspace = true } clap = { version = "4.5", features = ["derive", "wrap_help", "env"] } +dirs = { workspace = true } docker_credential = { workspace = true } futures-util = { workspace = true, features = ["io"] } oci-client = { workspace = true } @@ -21,6 +22,7 @@ tracing-subscriber = { workspace = true } wasm-pkg-common = { workspace = true } wasm-pkg-client = { workspace = true } wit-component = { workspace = true } +wkg-core = { workspace = true } [dev-dependencies] base64 = { workspace = true } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 908ab59..b577410 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -5,13 +5,18 @@ use clap::{Args, Parser, Subcommand, ValueEnum}; use futures_util::TryStreamExt; use tokio::io::AsyncWriteExt; use tracing::level_filters::LevelFilter; -use wasm_pkg_client::{Client, PublishOpts}; +use wasm_pkg_client::{ + caching::{CachingClient, FileCache}, + Client, PublishOpts, +}; use wasm_pkg_common::{config::Config, package::PackageSpec, registry::Registry}; use wit_component::DecodedWasm; mod oci; +mod wit; use oci::OciCommands; +use wit::WitCommands; #[derive(Parser, Debug)] #[command(version)] @@ -32,6 +37,43 @@ struct Common { /// The path to the configuration file. #[arg(long = "config", value_name = "CONFIG", env = "WKG_CONFIG_FILE")] config: Option, + /// The path to the cache directory. Defaults to the system cache directory. + #[arg(long = "cache", value_name = "CACHE", env = "WKG_CACHE_DIR")] + cache: Option, +} + +impl Common { + /// Helper to load the config from the given path + pub async fn load_config(&self) -> anyhow::Result { + if let Some(config_file) = self.config.as_ref() { + Config::from_file(config_file) + .await + .context(format!("error loading config file {config_file:?}")) + } else { + Config::global_defaults().await.map_err(anyhow::Error::from) + } + } + + /// Helper for loading the [`FileCache`] + pub async fn load_cache(&self) -> anyhow::Result { + let dir = if let Some(dir) = self.cache.as_ref() { + dir.clone() + } else { + dirs::cache_dir().context("unable to find cache directory")? + }; + let dir = dir.join("wkg"); + FileCache::new(dir).await + } + + /// Helper for loading a caching client. This should be the most commonly used method for + /// loading a client, but if you need to modify the config or use your own cache, you can use + /// the [`Common::load_config`] and [`Common::load_cache`] methods. + pub async fn get_client(&self) -> anyhow::Result> { + let config = self.load_config().await?; + let cache = self.load_cache().await?; + let client = Client::new(config); + Ok(CachingClient::new(Some(client), cache)) + } } #[derive(Subcommand, Debug)] @@ -46,6 +88,9 @@ enum Commands { /// Commands for interacting with OCI registries #[clap(subcommand)] Oci(OciCommands), + /// Commands for interacting with WIT files and dependencies + #[clap(subcommand)] + Wit(WitCommands), } #[derive(Args, Debug)] @@ -87,14 +132,14 @@ struct PublishArgs { /// form `:@`. #[arg(long, env = "WKG_PACKAGE")] package: Option, + + #[command(flatten)] + common: Common, } impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { - let client = { - let config = Config::global_defaults()?; - Client::new(config) - }; + let client = self.common.get_client().await?; let package = if let Some(package) = self.package { Some(( @@ -107,6 +152,7 @@ impl PublishArgs { None }; let (package, version) = client + .client()? .publish_release_file( &self.file, PublishOpts { @@ -130,20 +176,14 @@ enum Format { impl GetArgs { pub async fn run(self) -> anyhow::Result<()> { let PackageSpec { package, version } = self.package_spec; - - let client = { - let mut config = if let Some(config_file) = self.common.config { - Config::from_file(&config_file) - .context(format!("error loading config file {config_file:?}"))? - } else { - Config::global_defaults()? - }; - if let Some(registry) = self.registry_args.registry.clone() { - tracing::debug!(%package, %registry, "overriding package registry"); - config.set_package_registry_override(package.clone(), registry); - } - Client::new(config) - }; + let mut config = self.common.load_config().await?; + if let Some(registry) = self.registry_args.registry.clone() { + tracing::debug!(%package, %registry, "overriding package registry"); + config.set_package_registry_override(package.clone(), registry); + } + let client = Client::new(config); + let cache = self.common.load_cache().await?; + let client = CachingClient::new(Some(client), cache); let version = match version { Some(ver) => ver, @@ -179,7 +219,7 @@ impl GetArgs { tempfile::NamedTempFile::with_prefix_in(".wkg-get", parent_dir)?.into_parts(); tracing::debug!(?tmp_path, "Created temporary file"); - let mut content_stream = client.stream_content(&package, &release).await?; + let mut content_stream = client.get_content(&package, &release).await?; let mut file = tokio::fs::File::from_std(tmp_file); while let Some(chunk) = content_stream.try_next().await? { @@ -269,5 +309,6 @@ async fn main() -> anyhow::Result<()> { Commands::Get(args) => args.run().await, Commands::Publish(args) => args.run().await, Commands::Oci(args) => args.run().await, + Commands::Wit(args) => args.run().await, } } diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs new file mode 100644 index 0000000..31689a6 --- /dev/null +++ b/crates/wkg/src/wit.rs @@ -0,0 +1,150 @@ +//! Args and commands for interacting with WIT files and dependencies +use std::path::PathBuf; + +use clap::{Args, Subcommand}; +use wkg_core::{ + lock::LockFile, + wit::{self, OutputType}, +}; + +use crate::Common; + +/// Commands for interacting with wit +#[derive(Debug, Subcommand)] +pub enum WitCommands { + /// Build a WIT package from a directory. By default, this will fetch all dependencies needed + /// and encode them in the WIT package. This will generate a lock file that can be used to fetch + /// the dependencies in the future. + Build(BuildArgs), + /// Fetch dependencies for a component. This will read the package containing the world(s) you + /// have defined in the given wit directory (`wit` by default). It will then fetch the + /// dependencies and write them to the `deps` directory along with a lock file. If no lock file + /// exists, it will fetch all dependencies. If a lock file exists, it will fetch any + /// dependencies that are not in the lock file and update the lock file. To update the lock + /// file, use the `update` command. + Fetch(FetchArgs), + /// Update the lock file with the latest dependencies. This will update all dependencies and + /// generate a new lock file. + Update(UpdateArgs), +} + +impl WitCommands { + pub async fn run(self) -> anyhow::Result<()> { + match self { + WitCommands::Build(args) => args.run().await, + WitCommands::Fetch(args) => args.run().await, + WitCommands::Update(args) => args.run().await, + } + } +} + +#[derive(Debug, Args)] +pub struct BuildArgs { + /// The directory containing the WIT files to build. + #[clap(short = 'd', long = "wit-dir", default_value = "wit")] + pub dir: PathBuf, + + /// The name of the file that should be written. This can also be a full path. Defaults to the + /// current directory with the name of the package + #[clap(short = 'o', long = "output")] + pub output: Option, + + #[clap(flatten)] + pub common: Common, +} + +#[derive(Debug, Args)] +pub struct FetchArgs { + /// The directory containing the WIT files to fetch dependencies for. + #[clap(short = 'd', long = "wit-dir", default_value = "wit")] + pub dir: PathBuf, + + /// The desired output type of the dependencies. Valid options are "wit" or "wasm" (wasm is the + /// WIT package binary format). + #[clap(short = 't', long = "type")] + pub output_type: Option, + + #[clap(flatten)] + pub common: Common, +} + +#[derive(Debug, Args)] +pub struct UpdateArgs { + /// The directory containing the WIT files to update dependencies for. + #[clap(short = 'd', long = "wit-dir", default_value = "wit")] + pub dir: PathBuf, + + /// The desired output type of the dependencies. Valid options are "wit" or "wasm" (wasm is the + /// WIT package binary format). + #[clap(short = 't', long = "type")] + pub output_type: Option, + + #[clap(flatten)] + pub common: Common, +} + +impl BuildArgs { + pub async fn run(self) -> anyhow::Result<()> { + let client = self.common.get_client().await?; + let wkg_config = wkg_core::config::Config::load().await?; + let mut lock_file = LockFile::load(false).await?; + let (pkg_ref, version, bytes) = + wit::build_package(&wkg_config, self.dir, &mut lock_file, client).await?; + let output_path = if let Some(path) = self.output { + path + } else { + let mut file_name = pkg_ref.to_string(); + if let Some(version) = version { + file_name.push_str(&format!("@{version}")); + } + file_name.push_str(".wasm"); + PathBuf::from(file_name) + }; + + tokio::fs::write(&output_path, bytes).await?; + // Now write out the lock file since everything else succeeded + lock_file.write().await?; + println!("WIT package written to {}", output_path.display()); + Ok(()) + } +} + +impl FetchArgs { + pub async fn run(self) -> anyhow::Result<()> { + let client = self.common.get_client().await?; + let wkg_config = wkg_core::config::Config::load().await?; + let mut lock_file = LockFile::load(false).await?; + wit::fetch_dependencies( + &wkg_config, + self.dir, + &mut lock_file, + client, + self.output_type.unwrap_or_default(), + ) + .await?; + // Now write out the lock file since everything else succeeded + lock_file.write().await?; + Ok(()) + } +} + +impl UpdateArgs { + pub async fn run(self) -> anyhow::Result<()> { + let client = self.common.get_client().await?; + let wkg_config = wkg_core::config::Config::load().await?; + let mut lock_file = LockFile::load(false).await?; + // Clear the lock file since we're updating it + lock_file.packages.clear(); + wit::fetch_dependencies( + &wkg_config, + self.dir, + &mut lock_file, + client, + self.output_type.unwrap_or_default(), + ) + .await?; + // Now write out the lock file since everything else succeeded + lock_file.write().await?; + todo!() + } +}