Skip to content

Commit

Permalink
feat(wkg)!: Adds support for pushing and pulling OCI wasm artifacts
Browse files Browse the repository at this point in the history
This adds support for pushing and pulling compatible OCI wasm artifacts
from the command line. As part of this refactor, it makes the original
loading feature a separate command for use in debugging. Some of the
features from the `load` subcommand should probably make their way into
the push and pull command, but for now just keeping it simple

Signed-off-by: Taylor Thomas <taylor@cosmonic.com>
  • Loading branch information
thomastaylor312 committed Jun 18, 2024
1 parent 44126e4 commit 2a52302
Show file tree
Hide file tree
Showing 8 changed files with 240 additions and 24 deletions.
11 changes: 5 additions & 6 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@ authors = ["The Wasmtime Project Developers"]
license = "Apache-2.0 WITH LLVM-exception"

[workspace.dependencies]
oci-wasm = "0.0.3"
anyhow = "1"
docker_credential = "1.2.1"
oci-distribution = "0.11.0"
oci-wasm = "0.0.4"
tokio = "1.35.1"
tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", default-features = false, features = [
Expand Down
4 changes: 2 additions & 2 deletions crates/wasm-pkg-common/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ license.workspace = true
metadata-client = ["dep:reqwest"]

[dependencies]
anyhow = "1.0"
anyhow = { workspace = true }
dirs = "5.0.1"
http = "1.1.0"
reqwest = { version = "0.12.0", features = ["json"], optional = true }
Expand All @@ -20,4 +20,4 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
toml = "0.8.13"
thiserror = "1.0"
tracing = "0.1"
tracing = "0.1"
6 changes: 3 additions & 3 deletions crates/wasm-pkg-loader/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ edition.workspace = true
repository = "https://github.com/bytecodealliance/wasm-pkg-tools/tree/main/crates/wasm-pkg-loader"

[dependencies]
anyhow = "1.0.79"
anyhow = { workspace = true }
async-trait = "0.1.77"
base64 = "0.22.0"
bytes = "1.5.0"
dirs = "5.0.1"
docker_credential = "1.2.1"
docker_credential = { workspace = true }
futures-util = { version = "0.3.29", features = ["io"] }
oci-distribution = "0.11.0"
oci-distribution = { workspace = true }
oci-wasm = { workspace = true }
secrecy = { version = "0.8.0", features = ["serde"] }
serde = { version = "1.0.194", features = ["derive"] }
Expand Down
21 changes: 19 additions & 2 deletions crates/wasm-pkg-loader/tests/e2e/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 6 additions & 3 deletions crates/wkg/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@ authors.workspace = true
license.workspace = true

[dependencies]
anyhow = "1.0"
clap = { version = "4.5.4", features = ["derive", "wrap_help"] }
anyhow = { workspace = true }
clap = { version = "4.5", features = ["derive", "wrap_help", "env"] }
docker_credential = { workspace = true }
futures-util = { version = "0.3.29", features = ["io"] }
oci-distribution = { workspace = true }
oci-wasm = { workspace = true }
tempfile = "3.10.1"
tokio = { workspace = true, features = ["macros", "rt"] }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
wasm-pkg-common = { workspace = true }
wasm-pkg-loader = { workspace = true }
wit-component = "0.207"
wit-component = "0.207"
23 changes: 16 additions & 7 deletions crates/wkg/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ use wasm_pkg_common::{config::Config, package::PackageSpec, registry::Registry};
use wasm_pkg_loader::Client;
use wit_component::DecodedWasm;

mod oci;

use oci::OciCommands;

#[derive(Parser, Debug)]
#[command(version)]
struct Cli {
Expand All @@ -19,18 +23,22 @@ struct Cli {
#[derive(Args, Debug)]
struct RegistryArgs {
/// The registry domain to use. Overrides configuration file(s).
#[arg(long = "registry", value_name = "DOMAIN")]
#[arg(long = "registry", value_name = "REGISTRY", env = "WKG_REGISTRY")]
registry: Option<Registry>,
}

#[derive(Subcommand, Debug)]
#[allow(clippy::large_enum_variant)]
enum Commands {
/// Get a package.
Get(GetCommand),
/// Load a package. This is for use in debugging dependency fetching. For pulling a component, use `wit get`
Load(LoadArgs),
/// Commands for interacting with OCI registries
#[clap(subcommand)]
Oci(OciCommands),
}

#[derive(Args, Debug)]
struct GetCommand {
struct LoadArgs {
/// Output path. If this ends with a '/', a filename based on the package
/// name, version, and format will be appended, e.g.
/// `name-space_name@1.0.0.wasm``.
Expand Down Expand Up @@ -61,7 +69,7 @@ enum Format {
Wit,
}

impl GetCommand {
impl LoadArgs {
pub async fn run(self) -> anyhow::Result<()> {
let PackageSpec { package, version } = self.package_spec;

Expand Down Expand Up @@ -105,7 +113,7 @@ impl GetCommand {
};

let (tmp_file, tmp_path) =
tempfile::NamedTempFile::with_prefix_in(".wkg-get", parent_dir)?.into_parts();
tempfile::NamedTempFile::with_prefix_in(".wkg-load", parent_dir)?.into_parts();
tracing::debug!(?tmp_path, "Created temporary file");

let mut content_stream = client.stream_content(&package, &release).await?;
Expand Down Expand Up @@ -195,6 +203,7 @@ async fn main() -> anyhow::Result<()> {
let cli = Cli::parse();

match cli.command {
Commands::Get(cmd) => cmd.run().await,
Commands::Load(args) => args.run().await,
Commands::Oci(args) => args.run().await,
}
}
185 changes: 185 additions & 0 deletions crates/wkg/src/oci.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
use std::path::PathBuf;

use anyhow::Context;
use clap::{Args, Subcommand};
use docker_credential::DockerCredential;
use oci_distribution::{
client::{ClientConfig, ClientProtocol},
secrets::RegistryAuth,
Reference,
};
use oci_wasm::{WasmClient, WasmConfig};

#[derive(Debug, Args)]
pub struct Auth {
/// The username to use for authentication. If no credentials are provided, wkg will load them
/// from a local docker config and credential store and default to anonymous if none are found.
#[clap(
id = "username",
short = 'u',
env = "WKG_OCI_USERNAME",
requires = "password"
)]
pub username: Option<String>,
/// The password to use for authentication. This is required if username is set
#[clap(
id = "password",
short = 'p',
env = "WKG_OCI_PASSWORD",
requires = "username"
)]
pub password: Option<String>,
}

impl Auth {
fn into_auth(self, reference: &Reference) -> anyhow::Result<RegistryAuth> {
match (self.username, self.password) {
(Some(username), Some(password)) => Ok(RegistryAuth::Basic(username, password)),
(None, None) => {
let server_url = format!("https://{}", reference.registry());
match docker_credential::get_credential(&server_url) {
Ok(DockerCredential::UsernamePassword(username, password)) => {
return Ok(RegistryAuth::Basic(username, password));
}
Ok(DockerCredential::IdentityToken(_)) => {
return Err(anyhow::anyhow!("identity tokens not supported"));
}
Err(err) => {
tracing::debug!("Failed to look up OCI credentials: {err}");
}
}
Ok(RegistryAuth::Anonymous)
}
_ => Err(anyhow::anyhow!("Must provide both a username and password")),
}
}
}

#[derive(Debug, Args)]
pub struct Common {
/// A comma delimited list of allowed registries to use for http instead of https
#[clap(
long = "insecure",
default_value = "",
env = "WKG_OCI_INSECURE",
value_delimiter = ','
)]
pub insecure: Vec<String>,
}

/// Commands for interacting with OCI registries
#[derive(Debug, Subcommand)]
pub enum OciCommands {
/// Pull a component from an OCI registry and write it to a file.
Pull(PullArgs),
/// Push a component to an OCI registry.
Push(PushArgs),
}

impl OciCommands {
pub async fn run(self) -> anyhow::Result<()> {
match self {
OciCommands::Pull(args) => args.run().await,
OciCommands::Push(args) => args.run().await,
}
}
}

#[derive(Debug, Args)]
pub struct PullArgs {
#[clap(flatten)]
pub auth: Auth,

#[clap(flatten)]
pub common: Common,

/// The OCI reference to pull
pub reference: Reference,

/// The output path to write the file to
#[clap(short = 'o', long = "output")]
pub output: Option<PathBuf>,
}

#[derive(Debug, Args)]
pub struct PushArgs {
#[clap(flatten)]
pub auth: Auth,

#[clap(flatten)]
pub common: Common,

/// An optional author to set for the pushed component
#[clap(short = 'a', long = "author")]
pub author: Option<String>,

// TODO(thomastaylor312): Add support for custom annotations
/// The OCI reference to push
pub reference: Reference,

/// The path to the file to push
pub file: PathBuf,
}

impl PushArgs {
pub async fn run(self) -> anyhow::Result<()> {
let client = get_client(self.common);
let (conf, layer) = WasmConfig::from_component(&self.file, self.author)
.await
.context("Unable to parse component")?;
let auth = self.auth.into_auth(&self.reference)?;
client
.push(&self.reference, &auth, layer, conf, None)
.await
.context("Unable to push image")?;
println!("Pushed {}", self.reference);
Ok(())
}
}

impl PullArgs {
pub async fn run(self) -> anyhow::Result<()> {
let client = get_client(self.common);
let auth = self.auth.into_auth(&self.reference)?;
let data = client
.pull(&self.reference, &auth)
.await
.context("Unable to pull image")?;
let output_path = match self.output {
Some(output_file) => output_file,
None => PathBuf::from(format!(
"{}.wasm",
self.reference.repository().replace('/', "_")
)),
};
tokio::fs::write(
&output_path,
data.layers
.into_iter()
.next()
.ok_or_else(|| anyhow::anyhow!("No layers found"))?
.data,
)
.await
.context("Unable to write file")?;
println!(
"Successfully wrote {} to {}",
self.reference,
output_path.display()
);
Ok(())
}
}

fn get_client(common: Common) -> WasmClient {
let client = oci_distribution::Client::new(ClientConfig {
protocol: if common.insecure.is_empty() {
ClientProtocol::Https
} else {
ClientProtocol::HttpsExcept(common.insecure)
},
..Default::default()
});

WasmClient::new(client)
}

0 comments on commit 2a52302

Please sign in to comment.