From 967bef1b473e9d2f9de1596b822e14d4cdb41108 Mon Sep 17 00:00:00 2001 From: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> Date: Fri, 9 Aug 2024 08:03:19 -0600 Subject: [PATCH] feat: install extensions using the catalog (#3868) Look up extension.json url from /catalog.json in https://github.com/dfinity/dfx-extensions Fixes https://dfinity.atlassian.net/browse/SDK-1796 --- .github/workflows/update-docs.yml | 1 + CHANGELOG.md | 6 ++ Cargo.lock | 1 + Cargo.toml | 2 +- docs/extension-catalog-schema.json | 8 ++ e2e/tests-dfx/extension.bash | 77 +++++++++++++++++-- src/dfx-core/src/error/extension.rs | 18 +++++ src/dfx-core/src/extension/catalog.rs | 38 +++++++++ src/dfx-core/src/extension/manager/install.rs | 17 +++- src/dfx-core/src/extension/mod.rs | 1 + src/dfx-core/src/json/structure.rs | 19 +++++ src/dfx/src/commands/extension/install.rs | 19 ++--- src/dfx/src/commands/schema.rs | 3 + 13 files changed, 192 insertions(+), 18 deletions(-) create mode 100644 docs/extension-catalog-schema.json create mode 100644 src/dfx-core/src/extension/catalog.rs diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml index 741e7edb0f..260e50cc7b 100644 --- a/.github/workflows/update-docs.yml +++ b/.github/workflows/update-docs.yml @@ -30,6 +30,7 @@ jobs: cargo run -- schema --for dfx-metadata --outfile docs/dfx-metadata-schema.json cargo run -- schema --for extension-manifest --outfile docs/extension-manifest-schema.json cargo run -- schema --for extension-dependencies --outfile docs/extension-dependencies-schema.json + cargo run -- schema --for extension-catalog --outfile docs/extension-catalog-schema.json echo "JSON Schema changes:" if git diff --exit-code ; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 180e93db2a..91cfa7eab4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ A test key id `Ed25519:dfx_test_key` is ready to be used by locally created cani ### feat: Added settings_digest field to the network-id file +### feat: install extensions using the catalog + +`dfx extension install` now locates extensions using the +[extension catalog](https://github.com/dfinity/dfx-extensions/blob/main/catalog.json). +This can be overridden with the `--catalog-url` parameter. + ## Dependencies ### Replica diff --git a/Cargo.lock b/Cargo.lock index 2ca0f2e6d8..54bf18f984 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6510,6 +6510,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d3ca11c042..c15e5e5881 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,7 +72,7 @@ tempfile = "3.3.0" thiserror = "1.0.24" time = "0.3.9" tokio = "1.35" -url = "2.1.0" +url = { version="2.1.0", features=["serde"] } walkdir = "2.3.2" [profile.release] diff --git a/docs/extension-catalog-schema.json b/docs/extension-catalog-schema.json new file mode 100644 index 0000000000..b64b8ff5bf --- /dev/null +++ b/docs/extension-catalog-schema.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ExtensionCatalog", + "type": "object", + "additionalProperties": { + "type": "string" + } +} \ No newline at end of file diff --git a/e2e/tests-dfx/extension.bash b/e2e/tests-dfx/extension.bash index 50f6f5cf65..f6f2916034 100644 --- a/e2e/tests-dfx/extension.bash +++ b/e2e/tests-dfx/extension.bash @@ -179,7 +179,7 @@ EOF dfx_start } -install_extension_from_official_registry() { +install_extension_from_dfx_extensions_repo() { EXTENSION=$1 assert_command_fail dfx snsx @@ -206,12 +206,12 @@ install_extension_from_official_registry() { assert_match 'No extensions installed' } -@test "install extension by name from official registry" { - install_extension_from_official_registry sns +@test "install extension by name from official catalog" { + install_extension_from_dfx_extensions_repo sns } -@test "install extension by url from official registry" { - install_extension_from_official_registry https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/sns/extension.json +@test "install hosted extension by url" { + install_extension_from_dfx_extensions_repo https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/sns/extension.json } get_extension_architecture() { @@ -234,6 +234,73 @@ get_extension_architecture() { echo "$_cputype" } +@test "install extension from catalog" { + start_webserver --directory www + + CATALOG_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary-1/catalog.json" + mkdir -p www/arbitrary-1 + cat > www/arbitrary-1/catalog.json < www/arbitrary-2/foo/extension.json < www/arbitrary-2/foo/foo < www/arbitrary-2/foo/dependencies.json <=0.8.0" + } + } +} +EOF + + arch=$(get_extension_architecture) + + if [ "$(uname)" == "Darwin" ]; then + ARCHIVE_BASENAME="foo-$arch-apple-darwin" + else + ARCHIVE_BASENAME="foo-$arch-unknown-linux-gnu" + fi + + mkdir "$ARCHIVE_BASENAME" + cp www/arbitrary-2/foo/extension.json "$ARCHIVE_BASENAME" + cp www/arbitrary-2/foo/foo "$ARCHIVE_BASENAME" + tar -czf "$ARCHIVE_BASENAME".tar.gz "$ARCHIVE_BASENAME" + rm -rf "$ARCHIVE_BASENAME" + + mkdir -p www/arbitrary/downloads/foo-v0.1.0 + mv "$ARCHIVE_BASENAME".tar.gz www/arbitrary/downloads/foo-v0.1.0/ + + assert_command dfx extension install foo --catalog-url "$CATALOG_URL" +} + + @test "install extension by url from elsewhere" { start_webserver --directory www EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/extension.json" diff --git a/src/dfx-core/src/error/extension.rs b/src/dfx-core/src/error/extension.rs index aea8ba510e..0213a86eb6 100644 --- a/src/dfx-core/src/error/extension.rs +++ b/src/dfx-core/src/error/extension.rs @@ -104,9 +104,15 @@ pub enum DownloadAndInstallExtensionToTempdirError { #[derive(Error, Debug)] pub enum InstallExtensionError { + #[error("extension '{0}' not found in catalog")] + ExtensionNotFound(String), + #[error("Extension '{0}' is already installed at version {1}.")] OtherVersionAlreadyInstalled(String, Version), + #[error(transparent)] + FetchCatalog(#[from] FetchCatalogError), + #[error(transparent)] GetExtensionArchiveName(#[from] GetExtensionArchiveNameError), @@ -216,3 +222,15 @@ pub enum FetchExtensionCompatibilityMatrixError { #[derive(Error, Debug)] #[error(transparent)] pub struct UninstallExtensionError(#[from] RemoveDirectoryAndContentsError); + +#[derive(Error, Debug)] +pub enum FetchCatalogError { + #[error(transparent)] + ParseUrl(#[from] url::ParseError), + + #[error(transparent)] + Get(reqwest::Error), + + #[error(transparent)] + ParseJson(reqwest::Error), +} diff --git a/src/dfx-core/src/extension/catalog.rs b/src/dfx-core/src/extension/catalog.rs new file mode 100644 index 0000000000..fd65b6c2c8 --- /dev/null +++ b/src/dfx-core/src/extension/catalog.rs @@ -0,0 +1,38 @@ +use crate::error::extension::FetchCatalogError; +use crate::extension::url::ExtensionJsonUrl; +use crate::http::get::get_with_retries; +use crate::json::structure::UrlWithJsonSchema; +use backoff::exponential::ExponentialBackoff; +use schemars::JsonSchema; +use serde::Deserialize; +use std::collections::HashMap; +use std::time::Duration; +use url::Url; + +const DEFAULT_CATALOG_URL: &str = + "https://raw.githubusercontent.com/dfinity/dfx-extensions/main/catalog.json"; + +#[derive(Deserialize, Debug, JsonSchema)] +pub struct ExtensionCatalog(pub HashMap); + +impl ExtensionCatalog { + pub async fn fetch(url: Option<&Url>) -> Result { + let url: Option = url.cloned(); + let url = url.unwrap_or_else(|| Url::parse(DEFAULT_CATALOG_URL).unwrap()); + let retry_policy = ExponentialBackoff { + max_elapsed_time: Some(Duration::from_secs(60)), + ..Default::default() + }; + let resp = get_with_retries(url, retry_policy) + .await + .map_err(FetchCatalogError::Get)?; + + resp.json().await.map_err(FetchCatalogError::ParseJson) + } + + pub fn lookup(&self, name: &str) -> Option { + self.0 + .get(name) + .map(|url| ExtensionJsonUrl::new(url.0.clone())) + } +} diff --git a/src/dfx-core/src/extension/manager/install.rs b/src/dfx-core/src/extension/manager/install.rs index 9b5d972e65..852abda024 100644 --- a/src/dfx-core/src/extension/manager/install.rs +++ b/src/dfx-core/src/extension/manager/install.rs @@ -4,6 +4,7 @@ use crate::error::extension::{ GetHighestCompatibleVersionError, GetTopLevelDirectoryError, InstallExtensionError, }; use crate::extension::{ + catalog::ExtensionCatalog, manager::ExtensionManager, manifest::{ExtensionDependencies, ExtensionManifest}, url::ExtensionJsonUrl, @@ -29,11 +30,21 @@ pub enum InstallOutcome { impl ExtensionManager { pub async fn install_extension( &self, - url: &ExtensionJsonUrl, + name: &str, + catalog_url: Option<&Url>, install_as: Option<&str>, version: Option<&Version>, ) -> Result { - let manifest = Self::get_extension_manifest(url).await?; + let url = if let Ok(url) = Url::parse(name) { + ExtensionJsonUrl::new(url) + } else { + ExtensionCatalog::fetch(catalog_url) + .await? + .lookup(name) + .ok_or(InstallExtensionError::ExtensionNotFound(name.to_string()))? + }; + + let manifest = Self::get_extension_manifest(&url).await?; let extension_name: &str = &manifest.name; let effective_extension_name = install_as.unwrap_or(extension_name); @@ -59,7 +70,7 @@ impl ExtensionManager { let extension_version = match version { Some(version) => version.clone(), - None => self.get_highest_compatible_version(url).await?, + None => self.get_highest_compatible_version(&url).await?, }; let github_release_tag = get_git_release_tag(extension_name, &extension_version); let extension_archive = get_extension_archive_name(extension_name)?; diff --git a/src/dfx-core/src/extension/mod.rs b/src/dfx-core/src/extension/mod.rs index 71f15ec03f..b7b61e3955 100644 --- a/src/dfx-core/src/extension/mod.rs +++ b/src/dfx-core/src/extension/mod.rs @@ -1,3 +1,4 @@ +pub mod catalog; pub mod manager; pub mod manifest; pub mod url; diff --git a/src/dfx-core/src/json/structure.rs b/src/dfx-core/src/json/structure.rs index 341a62f5ad..4773377b1c 100644 --- a/src/dfx-core/src/json/structure.rs +++ b/src/dfx-core/src/json/structure.rs @@ -5,6 +5,7 @@ use serde::Serialize; use std::fmt::Display; use std::ops::{Deref, DerefMut}; use std::str::FromStr; +use url::Url; #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, PartialEq, Eq)] #[serde(untagged)] @@ -91,3 +92,21 @@ impl DerefMut for VersionWithJsonSchema { &mut self.0 } } + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(transparent)] +pub struct UrlWithJsonSchema(#[schemars(with = "String")] pub Url); + +impl Deref for UrlWithJsonSchema { + type Target = Url; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for UrlWithJsonSchema { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/src/dfx/src/commands/extension/install.rs b/src/dfx/src/commands/extension/install.rs index 90bda026db..2e01006a5c 100644 --- a/src/dfx/src/commands/extension/install.rs +++ b/src/dfx/src/commands/extension/install.rs @@ -7,7 +7,6 @@ use clap::Parser; use clap::Subcommand; use dfx_core::error::extension::InstallExtensionError::OtherVersionAlreadyInstalled; use dfx_core::extension::manager::InstallOutcome; -use dfx_core::extension::url::ExtensionJsonUrl; use semver::Version; use slog::{error, info, warn}; use tokio::runtime::Runtime; @@ -23,6 +22,9 @@ pub struct InstallOpts { /// Installs a specific version of the extension, bypassing version checks #[clap(long)] version: Option, + /// Specifies the URL of the catalog to use to find the extension. + #[clap(long)] + catalog_url: Option, } pub fn exec(env: &dyn Environment, opts: InstallOpts) -> DfxResult<()> { @@ -36,17 +38,16 @@ pub fn exec(env: &dyn Environment, opts: InstallOpts) -> DfxResult<()> { bail!("Extension '{}' cannot be installed because it conflicts with an existing command. Consider using '--install-as' flag to install this extension under different name.", opts.name) } - let url = if let Ok(url) = Url::parse(&opts.name) { - ExtensionJsonUrl::new(url) - } else { - ExtensionJsonUrl::registered(&opts.name)? - }; - let runtime = Runtime::new().expect("Unable to create a runtime"); let install_outcome = runtime.block_on(async { - mgr.install_extension(&url, opts.install_as.as_deref(), opts.version.as_ref()) - .await + mgr.install_extension( + &opts.name, + opts.catalog_url.as_ref(), + opts.install_as.as_deref(), + opts.version.as_ref(), + ) + .await }); spinner.finish_and_clear(); let logger = env.get_logger(); diff --git a/src/dfx/src/commands/schema.rs b/src/dfx/src/commands/schema.rs index e285149f91..1587851655 100644 --- a/src/dfx/src/commands/schema.rs +++ b/src/dfx/src/commands/schema.rs @@ -2,6 +2,7 @@ use crate::lib::{error::DfxResult, metadata::dfx::DfxMetadata}; use anyhow::Context; use clap::{Parser, ValueEnum}; use dfx_core::config::model::dfinity::{ConfigInterface, TopLevelConfigNetworks}; +use dfx_core::extension::catalog::ExtensionCatalog; use dfx_core::extension::manifest::{ExtensionDependencies, ExtensionManifest}; use schemars::schema_for; use std::path::PathBuf; @@ -13,6 +14,7 @@ enum ForFile { DfxMetadata, ExtensionDependencies, ExtensionManifest, + ExtensionCatalog, } /// Prints the schema for dfx.json. @@ -32,6 +34,7 @@ pub fn exec(opts: SchemaOpts) -> DfxResult { Some(ForFile::DfxMetadata) => schema_for!(DfxMetadata), Some(ForFile::ExtensionDependencies) => schema_for!(ExtensionDependencies), Some(ForFile::ExtensionManifest) => schema_for!(ExtensionManifest), + Some(ForFile::ExtensionCatalog) => schema_for!(ExtensionCatalog), _ => schema_for!(ConfigInterface), }; let nice_schema =