feat: install extensions using the catalog (#3868)

Look up extension.json url from /catalog.json in https://github.com/dfinity/dfx-extensions

Fixes https://dfinity.atlassian.net/browse/SDK-1796

.github/workflows/update-docs.yml

@@ -30,6 +30,7 @@ jobs:
           cargo run -- schema --for dfx-metadata --outfile docs/dfx-metadata-schema.json
           cargo run -- schema --for extension-manifest --outfile docs/extension-manifest-schema.json
           cargo run -- schema --for extension-dependencies --outfile docs/extension-dependencies-schema.json
+          cargo run -- schema --for extension-catalog --outfile docs/extension-catalog-schema.json
 
           echo "JSON Schema changes:"
           if git diff --exit-code ; then

CHANGELOG.md

@@ -17,6 +17,12 @@ A test key id `Ed25519:dfx_test_key` is ready to be used by locally created cani
 
 ### feat: Added settings_digest field to the network-id file
 
+### feat: install extensions using the catalog
+
+`dfx extension install` now locates extensions using the
+[extension catalog](https://github.com/dfinity/dfx-extensions/blob/main/catalog.json).
+This can be overridden with the `--catalog-url` parameter.
+
 ## Dependencies
 
 ### Replica

Cargo.toml

@@ -72,7 +72,7 @@ tempfile = "3.3.0"
 thiserror = "1.0.24"
 time = "0.3.9"
 tokio = "1.35"
-url = "2.1.0"
+url = {  version="2.1.0", features=["serde"] }
 walkdir = "2.3.2"
 
 [profile.release]

docs/extension-catalog-schema.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "ExtensionCatalog",
+  "type": "object",
+  "additionalProperties": {
+    "type": "string"
+  }
+}

e2e/tests-dfx/extension.bash

@@ -179,7 +179,7 @@ EOF
   dfx_start
 }
 
-install_extension_from_official_registry() {
+install_extension_from_dfx_extensions_repo() {
   EXTENSION=$1
 
   assert_command_fail dfx snsx
@@ -206,12 +206,12 @@ install_extension_from_official_registry() {
   assert_match 'No extensions installed'
 }
 
-@test "install extension by name from official registry" {
-  install_extension_from_official_registry sns
+@test "install extension by name from official catalog" {
+  install_extension_from_dfx_extensions_repo sns
 }
 
-@test "install extension by url from official registry" {
-  install_extension_from_official_registry https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/sns/extension.json
+@test "install hosted extension by url" {
+  install_extension_from_dfx_extensions_repo https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/sns/extension.json
 }
 
 get_extension_architecture() {
@@ -234,6 +234,73 @@ get_extension_architecture() {
   echo "$_cputype"
 }
 
+@test "install extension from catalog" {
+  start_webserver --directory www
+
+  CATALOG_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary-1/catalog.json"
+  mkdir -p www/arbitrary-1
+  cat > www/arbitrary-1/catalog.json <<EOF
+{
+  "foo": "http://localhost:$E2E_WEB_SERVER_PORT/arbitrary-2/foo/extension.json",
+  "bar": "http://localhost:$E2E_WEB_SERVER_PORT/arbitrary-2/bar/extension.json"
+}
+EOF
+
+
+  EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary-2/foo/extension.json"
+  mkdir -p  www/arbitrary-2/foo/downloads
+
+  cat > www/arbitrary-2/foo/extension.json <<EOF
+{
+  "name": "foo",
+  "version": "0.1.0",
+  "homepage": "https://github.com/dfinity/dfx-extensions",
+  "authors": "DFINITY",
+  "summary": "Test extension for e2e purposes.",
+  "categories": [],
+  "keywords": [],
+  "download_url_template": "http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/downloads/{{tag}}/{{basename}}.{{archive-format}}"
+}
+EOF
+  cat www/arbitrary-2/foo/extension.json
+  cat > www/arbitrary-2/foo/foo <<EOF
+#!/usr/bin/env bash
+
+echo "the foo output"
+EOF
+  chmod +x www/arbitrary-2/foo/foo
+
+  cat > www/arbitrary-2/foo/dependencies.json <<EOF
+{
+  "0.1.0": {
+    "dfx": {
+      "version": ">=0.8.0"
+    }
+  }
+}
+EOF
+
+  arch=$(get_extension_architecture)
+
+  if [ "$(uname)" == "Darwin" ]; then
+    ARCHIVE_BASENAME="foo-$arch-apple-darwin"
+  else
+    ARCHIVE_BASENAME="foo-$arch-unknown-linux-gnu"
+  fi
+
+  mkdir "$ARCHIVE_BASENAME"
+  cp www/arbitrary-2/foo/extension.json "$ARCHIVE_BASENAME"
+  cp www/arbitrary-2/foo/foo "$ARCHIVE_BASENAME"
+  tar -czf "$ARCHIVE_BASENAME".tar.gz "$ARCHIVE_BASENAME"
+  rm -rf "$ARCHIVE_BASENAME"
+
+  mkdir -p www/arbitrary/downloads/foo-v0.1.0
+  mv "$ARCHIVE_BASENAME".tar.gz www/arbitrary/downloads/foo-v0.1.0/
+
+  assert_command dfx extension install foo --catalog-url "$CATALOG_URL"
+}
+
+
 @test "install extension by url from elsewhere" {
   start_webserver --directory www
   EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/extension.json"

src/dfx-core/src/error/extension.rs

@@ -104,9 +104,15 @@ pub enum DownloadAndInstallExtensionToTempdirError {
 
 #[derive(Error, Debug)]
 pub enum InstallExtensionError {
+    #[error("extension '{0}' not found in catalog")]
+    ExtensionNotFound(String),
+
     #[error("Extension '{0}' is already installed at version {1}.")]
     OtherVersionAlreadyInstalled(String, Version),
 
+    #[error(transparent)]
+    FetchCatalog(#[from] FetchCatalogError),
+
     #[error(transparent)]
     GetExtensionArchiveName(#[from] GetExtensionArchiveNameError),
 
@@ -216,3 +222,15 @@ pub enum FetchExtensionCompatibilityMatrixError {
 #[derive(Error, Debug)]
 #[error(transparent)]
 pub struct UninstallExtensionError(#[from] RemoveDirectoryAndContentsError);
+
+#[derive(Error, Debug)]
+pub enum FetchCatalogError {
+    #[error(transparent)]
+    ParseUrl(#[from] url::ParseError),
+
+    #[error(transparent)]
+    Get(reqwest::Error),
+
+    #[error(transparent)]
+    ParseJson(reqwest::Error),
+}

src/dfx-core/src/extension/catalog.rs

@@ -0,0 +1,38 @@
+use crate::error::extension::FetchCatalogError;
+use crate::extension::url::ExtensionJsonUrl;
+use crate::http::get::get_with_retries;
+use crate::json::structure::UrlWithJsonSchema;
+use backoff::exponential::ExponentialBackoff;
+use schemars::JsonSchema;
+use serde::Deserialize;
+use std::collections::HashMap;
+use std::time::Duration;
+use url::Url;
+
+const DEFAULT_CATALOG_URL: &str =
+    "https://raw.githubusercontent.com/dfinity/dfx-extensions/main/catalog.json";
+
+#[derive(Deserialize, Debug, JsonSchema)]
+pub struct ExtensionCatalog(pub HashMap<String, UrlWithJsonSchema>);
+
+impl ExtensionCatalog {
+    pub async fn fetch(url: Option<&Url>) -> Result<Self, FetchCatalogError> {
+        let url: Option<Url> = url.cloned();
+        let url = url.unwrap_or_else(|| Url::parse(DEFAULT_CATALOG_URL).unwrap());
+        let retry_policy = ExponentialBackoff {
+            max_elapsed_time: Some(Duration::from_secs(60)),
+            ..Default::default()
+        };
+        let resp = get_with_retries(url, retry_policy)
+            .await
+            .map_err(FetchCatalogError::Get)?;
+
+        resp.json().await.map_err(FetchCatalogError::ParseJson)
+    }
+
+    pub fn lookup(&self, name: &str) -> Option<ExtensionJsonUrl> {
+        self.0
+            .get(name)
+            .map(|url| ExtensionJsonUrl::new(url.0.clone()))
+    }
+}

src/dfx-core/src/extension/manager/install.rs

@@ -4,6 +4,7 @@ use crate::error::extension::{
     GetHighestCompatibleVersionError, GetTopLevelDirectoryError, InstallExtensionError,
 };
 use crate::extension::{
+    catalog::ExtensionCatalog,
     manager::ExtensionManager,
     manifest::{ExtensionDependencies, ExtensionManifest},
     url::ExtensionJsonUrl,
@@ -29,11 +30,21 @@ pub enum InstallOutcome {
 impl ExtensionManager {
     pub async fn install_extension(
         &self,
-        url: &ExtensionJsonUrl,
+        name: &str,
+        catalog_url: Option<&Url>,
         install_as: Option<&str>,
         version: Option<&Version>,
     ) -> Result<InstallOutcome, InstallExtensionError> {
-        let manifest = Self::get_extension_manifest(url).await?;
+        let url = if let Ok(url) = Url::parse(name) {
+            ExtensionJsonUrl::new(url)
+        } else {
+            ExtensionCatalog::fetch(catalog_url)
+                .await?
+                .lookup(name)
+                .ok_or(InstallExtensionError::ExtensionNotFound(name.to_string()))?
+        };
+
+        let manifest = Self::get_extension_manifest(&url).await?;
         let extension_name: &str = &manifest.name;
 
         let effective_extension_name = install_as.unwrap_or(extension_name);
@@ -59,7 +70,7 @@ impl ExtensionManager {
 
         let extension_version = match version {
             Some(version) => version.clone(),
-            None => self.get_highest_compatible_version(url).await?,
+            None => self.get_highest_compatible_version(&url).await?,
         };
         let github_release_tag = get_git_release_tag(extension_name, &extension_version);
         let extension_archive = get_extension_archive_name(extension_name)?;

src/dfx-core/src/extension/mod.rs

@@ -1,3 +1,4 @@
+pub mod catalog;
 pub mod manager;
 pub mod manifest;
 pub mod url;

src/dfx-core/src/json/structure.rs

@@ -5,6 +5,7 @@ use serde::Serialize;
 use std::fmt::Display;
 use std::ops::{Deref, DerefMut};
 use std::str::FromStr;
+use url::Url;
 
 #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, PartialEq, Eq)]
 #[serde(untagged)]
@@ -91,3 +92,21 @@ impl DerefMut for VersionWithJsonSchema {
         &mut self.0
     }
 }
+
+#[derive(Debug, Serialize, Deserialize, JsonSchema)]
+#[serde(transparent)]
+pub struct UrlWithJsonSchema(#[schemars(with = "String")] pub Url);
+
+impl Deref for UrlWithJsonSchema {
+    type Target = Url;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for UrlWithJsonSchema {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}

src/dfx/src/commands/extension/install.rs

@@ -7,7 +7,6 @@ use clap::Parser;
 use clap::Subcommand;
 use dfx_core::error::extension::InstallExtensionError::OtherVersionAlreadyInstalled;
 use dfx_core::extension::manager::InstallOutcome;
-use dfx_core::extension::url::ExtensionJsonUrl;
 use semver::Version;
 use slog::{error, info, warn};
 use tokio::runtime::Runtime;
@@ -23,6 +22,9 @@ pub struct InstallOpts {
     /// Installs a specific version of the extension, bypassing version checks
     #[clap(long)]
     version: Option<Version>,
+    /// Specifies the URL of the catalog to use to find the extension.
+    #[clap(long)]
+    catalog_url: Option<Url>,
 }
 
 pub fn exec(env: &dyn Environment, opts: InstallOpts) -> DfxResult<()> {
@@ -36,17 +38,16 @@ pub fn exec(env: &dyn Environment, opts: InstallOpts) -> DfxResult<()> {
         bail!("Extension '{}' cannot be installed because it conflicts with an existing command. Consider using '--install-as' flag to install this extension under different name.", opts.name)
     }
 
-    let url = if let Ok(url) = Url::parse(&opts.name) {
-        ExtensionJsonUrl::new(url)
-    } else {
-        ExtensionJsonUrl::registered(&opts.name)?
-    };
-
     let runtime = Runtime::new().expect("Unable to create a runtime");
 
     let install_outcome = runtime.block_on(async {
-        mgr.install_extension(&url, opts.install_as.as_deref(), opts.version.as_ref())
-            .await
+        mgr.install_extension(
+            &opts.name,
+            opts.catalog_url.as_ref(),
+            opts.install_as.as_deref(),
+            opts.version.as_ref(),
+        )
+        .await
     });
     spinner.finish_and_clear();
     let logger = env.get_logger();

src/dfx/src/commands/schema.rs

@@ -2,6 +2,7 @@ use crate::lib::{error::DfxResult, metadata::dfx::DfxMetadata};
 use anyhow::Context;
 use clap::{Parser, ValueEnum};
 use dfx_core::config::model::dfinity::{ConfigInterface, TopLevelConfigNetworks};
+use dfx_core::extension::catalog::ExtensionCatalog;
 use dfx_core::extension::manifest::{ExtensionDependencies, ExtensionManifest};
 use schemars::schema_for;
 use std::path::PathBuf;
@@ -13,6 +14,7 @@ enum ForFile {
     DfxMetadata,
     ExtensionDependencies,
     ExtensionManifest,
+    ExtensionCatalog,
 }
 
 /// Prints the schema for dfx.json.
@@ -32,6 +34,7 @@ pub fn exec(opts: SchemaOpts) -> DfxResult {
         Some(ForFile::DfxMetadata) => schema_for!(DfxMetadata),
         Some(ForFile::ExtensionDependencies) => schema_for!(ExtensionDependencies),
         Some(ForFile::ExtensionManifest) => schema_for!(ExtensionManifest),
+        Some(ForFile::ExtensionCatalog) => schema_for!(ExtensionCatalog),
         _ => schema_for!(ConfigInterface),
     };
     let nice_schema =