feat(*): Adds support for building and fetching wit packages

This adds a new `wit` subcommand to `wkg` that supports building wit
packages and fetching/populating a deps directory. I'm sure there is much
more we can do here and some obtuse edge cases that aren't supported, but
I did test fetching dependencies for various worlds that used wasi:http
and wasi:cli. In a follow up PR, I'll add some more integration tests

Signed-off-by: Taylor Thomas <taylor@cosmonic.com>

.github/workflows/ci.yml

@@ -8,11 +8,21 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        include:
+          - os: ubuntu-latest
+            additional_test_flags: ""
+          - os: windows-latest
+            additional_test_flags: "--no-default-features"
+          - os: macos-latest
+            additional_test_flags: "--no-default-features"
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
-      - name: Run cargo test
-        run: cargo test --all --workspace
+      # We have to run these separately so we can deactivate a feature for one of the tests
+      - name: Run client tests
+        working-directory: ./crates/wasm-pkg-client
+        run: cargo test ${{ matrix.additional_test_flags }}
+      - name: Run other tests
+        run: cargo test --workspace --exclude wasm-pkg-client
       - name: Run cargo clippy
         run: cargo clippy --all --workspace 

Cargo.toml

@@ -28,6 +28,7 @@ sha2 = "0.10"
 tempfile = "3.10.1"
 thiserror = "1.0"
 tokio = "1.35.1"
+tokio-util = "0.7.10"
 toml = "0.8.13"
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", default-features = false, features = [
@@ -36,5 +37,7 @@ tracing-subscriber = { version = "0.3.18", default-features = false, features =
 ] }
 wasm-pkg-common = { version = "0.5.1", path = "crates/wasm-pkg-common" }
 wasm-pkg-client = { version = "0.5.1", path = "crates/wasm-pkg-client" }
+wasm-metadata = "0.216"
 wit-component = "0.216"
 wit-parser = "0.216"
+wkg-core = { version = "0.5.0", path = "crates/wkg-core" }

crates/wasm-pkg-client/Cargo.toml

@@ -8,6 +8,11 @@ authors.workspace = true
 license.workspace = true
 readme = "../../README.md"
 
+[features]
+default = ["_local"]
+# An internal feature for making sure e2e tests can run locally but not in CI for Mac or Windows
+_local = []
+
 [dependencies]
 anyhow = { workspace = true }
 async-trait = "0.1.77"
@@ -24,7 +29,7 @@ serde_json = { workspace = true }
 sha2 = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt", "macros"] }
-tokio-util = { version = "0.7.10", features = ["io", "io-util", "codec"] }
+tokio-util = { workspace = true, features = ["io", "io-util", "codec"] }
 toml = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }

crates/wasm-pkg-client/src/caching/mod.rs

@@ -1,4 +1,5 @@
 use std::future::Future;
+use std::sync::Arc;
 
 use wasm_pkg_common::{
     digest::ContentDigest,
@@ -46,15 +47,27 @@ pub trait Cache {
 /// underlying client to be used as a read-only cache.
 pub struct CachingClient<T> {
     client: Option<Client>,
-    cache: T,
+    cache: Arc<T>,
+}
+
+impl<T: Cache> Clone for CachingClient<T> {
+    fn clone(&self) -> Self {
+        Self {
+            client: self.client.clone(),
+            cache: self.cache.clone(),
+        }
+    }
 }
 
 impl<T: Cache> CachingClient<T> {
     /// Creates a new caching client from the given client and cache implementation. If no client is
     /// given, the client will be in offline or read-only mode, meaning it will only be able to return
     /// things that are already in the cache.
     pub fn new(client: Option<Client>, cache: T) -> Self {
-        Self { client, cache }
+        Self {
+            client,
+            cache: Arc::new(cache),
+        }
     }
 
     /// Returns whether or not the client is in read-only mode.

crates/wasm-pkg-client/src/lib.rs

@@ -8,7 +8,7 @@
 //! ```no_run
 //! # async fn example() -> anyhow::Result<()> {
 //! // Initialize client from global configuration.
-//! let mut client = wasm_pkg_client::Client::with_global_defaults()?;
+//! let mut client = wasm_pkg_client::Client::with_global_defaults().await?;
 //!
 //! // Get a specific package release version.
 //! let pkg = "example:pkg".parse()?;
@@ -86,23 +86,29 @@ pub struct PublishOpts {
 }
 
 /// A read-only registry client.
+#[derive(Clone)]
 pub struct Client {
-    config: Config,
-    sources: RwLock<RegistrySources>,
+    config: Arc<Config>,
+    sources: Arc<RwLock<RegistrySources>>,
 }
 
 impl Client {
     /// Returns a new client with the given [`Config`].
     pub fn new(config: Config) -> Self {
         Self {
-            config,
+            config: Arc::new(config),
             sources: Default::default(),
         }
     }
 
+    /// Returns a reference to the configuration this client was initialized with.
+    pub fn config(&self) -> &Config {
+        &self.config
+    }
+
     /// Returns a new client configured from default global config.
-    pub fn with_global_defaults() -> Result<Self, Error> {
-        let config = Config::global_defaults()?;
+    pub async fn with_global_defaults() -> Result<Self, Error> {
+        let config = Config::global_defaults().await?;
         Ok(Self::new(config))
     }
 

crates/wasm-pkg-client/src/oci/mod.rs

@@ -128,6 +128,7 @@ impl OciBackend {
                     CredentialRetrievalError::ConfigNotFound
                         | CredentialRetrievalError::ConfigReadError
                         | CredentialRetrievalError::NoCredentialConfigured
+                        | CredentialRetrievalError::HelperFailure { .. }
                 ) {
                     tracing::debug!("Failed to look up OCI credentials: {err}");
                 } else {

crates/wasm-pkg-client/tests/e2e.rs

@@ -8,6 +8,10 @@ use wasm_pkg_client::{Client, Config};
 
 const FIXTURE_WASM: &str = "./tests/testdata/binary_wit.wasm";
 
+#[cfg(any(target_os = "linux", feature = "_local"))]
+// NOTE: These are only run on linux for CI purposes, because they rely on the docker client being
+// available, and for various reasons this has proven to be problematic on both the Windows and
+// MacOS runners due to it not being installed (yay licensing).
 #[tokio::test]
 async fn publish_and_fetch_smoke_test() {
     let _container = GenericImage::new("registry", "2")