feat(*): Adds support for building and fetching wit packages

This adds a new `wit` subcommand to `wkg` that supports building wit
packages and fetching/populating a deps directory. I'm sure there is much
more we can do here and some obtuse edge cases that aren't supported, but
I did test fetching dependencies for various worlds that used wasi:http
and wasi:cli. In a follow up PR, I'll add some more integration tests

Signed-off-by: Taylor Thomas <taylor@cosmonic.com>

Cargo.toml

@@ -28,6 +28,7 @@ sha2 = "0.10"
 tempfile = "3.10.1"
 thiserror = "1.0"
 tokio = "1.35.1"
+tokio-util = "0.7.10"
 toml = "0.8.13"
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", default-features = false, features = [
@@ -36,5 +37,7 @@ tracing-subscriber = { version = "0.3.18", default-features = false, features =
 ] }
 wasm-pkg-common = { version = "0.5.0", path = "crates/wasm-pkg-common" }
 wasm-pkg-client = { version = "0.5.0", path = "crates/wasm-pkg-client" }
+wasm-metadata = "0.216"
 wit-component = "0.216"
 wit-parser = "0.216"
+wkg-core = { version = "0.5.0", path = "crates/wkg-core" }

crates/wasm-pkg-client/Cargo.toml

@@ -23,7 +23,7 @@ serde_json = { workspace = true }
 sha2 = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt", "macros"] }
-tokio-util = { version = "0.7.10", features = ["io", "io-util", "codec"] }
+tokio-util = { workspace = true, features = ["io", "io-util", "codec"] }
 toml = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }

crates/wasm-pkg-client/src/caching/mod.rs

@@ -1,4 +1,5 @@
 use std::future::Future;
+use std::sync::Arc;
 
 use wasm_pkg_common::{
     digest::ContentDigest,
@@ -46,15 +47,27 @@ pub trait Cache {
 /// underlying client to be used as a read-only cache.
 pub struct CachingClient<T> {
     client: Option<Client>,
-    cache: T,
+    cache: Arc<T>,
+}
+
+impl<T: Cache> Clone for CachingClient<T> {
+    fn clone(&self) -> Self {
+        Self {
+            client: self.client.clone(),
+            cache: self.cache.clone(),
+        }
+    }
 }
 
 impl<T: Cache> CachingClient<T> {
     /// Creates a new caching client from the given client and cache implementation. If no client is
     /// given, the client will be in offline or read-only mode, meaning it will only be able to return
     /// things that are already in the cache.
     pub fn new(client: Option<Client>, cache: T) -> Self {
-        Self { client, cache }
+        Self {
+            client,
+            cache: Arc::new(cache),
+        }
     }
 
     /// Returns whether or not the client is in read-only mode.

crates/wasm-pkg-client/src/lib.rs

@@ -86,23 +86,29 @@ pub struct PublishOpts {
 }
 
 /// A read-only registry client.
+#[derive(Clone)]
 pub struct Client {
-    config: Config,
-    sources: RwLock<RegistrySources>,
+    config: Arc<Config>,
+    sources: Arc<RwLock<RegistrySources>>,
 }
 
 impl Client {
     /// Returns a new client with the given [`Config`].
     pub fn new(config: Config) -> Self {
         Self {
-            config,
+            config: Arc::new(config),
             sources: Default::default(),
         }
     }
 
+    /// Returns a reference to the configuration this client was initialized with.
+    pub fn config(&self) -> &Config {
+        &self.config
+    }
+
     /// Returns a new client configured from default global config.
-    pub fn with_global_defaults() -> Result<Self, Error> {
-        let config = Config::global_defaults()?;
+    pub async fn with_global_defaults() -> Result<Self, Error> {
+        let config = Config::global_defaults().await?;
         Ok(Self::new(config))
     }
 

crates/wasm-pkg-client/src/oci/mod.rs

@@ -128,6 +128,7 @@ impl OciBackend {
                     CredentialRetrievalError::ConfigNotFound
                         | CredentialRetrievalError::ConfigReadError
                         | CredentialRetrievalError::NoCredentialConfigured
+                        | CredentialRetrievalError::HelperFailure { .. }
                 ) {
                     tracing::debug!("Failed to look up OCI credentials: {err}");
                 } else {

crates/wasm-pkg-common/src/config.rs

@@ -73,21 +73,21 @@ impl Config {
     ///
     /// Note: This list is expected to expand in the future to include
     /// "workspace" config files like `./.wasm-pkg/config.toml`.
-    pub fn global_defaults() -> Result<Self, Error> {
+    pub async fn global_defaults() -> Result<Self, Error> {
         let mut config = Self::default();
-        if let Some(global_config) = Self::read_global_config()? {
+        if let Some(global_config) = Self::read_global_config().await? {
             config.merge(global_config);
         }
         Ok(config)
     }
 
     /// Reads config from the default global config file location
-    pub fn read_global_config() -> Result<Option<Self>, Error> {
+    pub async fn read_global_config() -> Result<Option<Self>, Error> {
         let Some(config_dir) = dirs::config_dir() else {
             return Ok(None);
         };
         let path = config_dir.join("wasm-pkg").join("config.toml");
-        let contents = match std::fs::read_to_string(path) {
+        let contents = match tokio::fs::read_to_string(path).await {
             Ok(contents) => contents,
             Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None),
             Err(err) => return Err(Error::ConfigFileIoError(err)),
@@ -96,8 +96,10 @@ impl Config {
     }
 
     /// Reads config from a TOML file at the given path.
-    pub fn from_file(path: impl AsRef<Path>) -> Result<Self, Error> {
-        let contents = std::fs::read_to_string(path).map_err(Error::ConfigFileIoError)?;
+    pub async fn from_file(path: impl AsRef<Path>) -> Result<Self, Error> {
+        let contents = tokio::fs::read_to_string(path)
+            .await
+            .map_err(Error::ConfigFileIoError)?;
         Self::from_toml(&contents)
     }
 
@@ -109,9 +111,11 @@ impl Config {
     }
 
     /// Writes the config to a TOML file at the given path.
-    pub fn to_file(&self, path: impl AsRef<Path>) -> Result<(), Error> {
+    pub async fn to_file(&self, path: impl AsRef<Path>) -> Result<(), Error> {
         let toml_str = ::toml::to_string(&self).map_err(Error::invalid_config)?;
-        std::fs::write(path, toml_str).map_err(Error::ConfigFileIoError)
+        tokio::fs::write(path, toml_str)
+            .await
+            .map_err(Error::ConfigFileIoError)
     }
 
     /// Merges the given other config into this one.