feat: extension install uses dependencies.json (#3816)

CHANGELOG.md

@@ -21,6 +21,8 @@ The [NIST guidelines](https://pages.nist.gov/800-63-3/sp800-63b.html) require pa
 This is now enforced when creating new identities.
 Identities protected by a shorter password can still be decrypted.
 
+### feat: `dfx extension install` now uses the extension's dependencies.json file to pick the highest compatible version
+
 # 0.21.0
 
 ### feat: dfx killall

src/dfx-core/src/error/extension.rs

@@ -105,7 +105,7 @@ pub enum InstallExtensionError {
     GetExtensionArchiveName(#[from] GetExtensionArchiveNameError),
 
     #[error(transparent)]
-    FindLatestExtensionCompatibleVersion(#[from] FindLatestExtensionCompatibleVersionError),
+    GetHighestCompatibleVersion(#[from] GetHighestCompatibleVersionError),
 
     #[error(transparent)]
     GetExtensionDownloadUrl(#[from] GetExtensionDownloadUrlError),
@@ -124,23 +124,33 @@ pub enum GetExtensionArchiveNameError {
 }
 
 #[derive(Error, Debug)]
-pub enum FindLatestExtensionCompatibleVersionError {
-    #[error("DFX version '{0}' is not supported.")]
-    DfxVersionNotFoundInCompatibilityJson(semver::Version),
+pub enum GetHighestCompatibleVersionError {
+    #[error(transparent)]
+    GetDependencies(#[from] GetDependenciesError),
 
-    #[error("Extension '{0}' (version '{1}') not found for DFX version {2}.")]
-    ExtensionVersionNotFoundInRepository(String, semver::Version, String),
+    #[error("No compatible version found.")]
+    NoCompatibleVersionFound(),
 
-    #[error("Cannot parse compatibility.json due to malformed semver '{0}'")]
-    MalformedVersionsEntryForExtensionInCompatibilityMatrix(String, #[source] semver::Error),
+    #[error(transparent)]
+    DfxOnlyPossibleDependency(#[from] DfxOnlySupportedDependency),
+}
 
-    #[error("Cannot find compatible extension for dfx version '{1}': compatibility.json (downloaded from '{0}') has empty list of extension versions.")]
-    ListOfVersionsForExtensionIsEmpty(String, semver::Version),
+#[derive(Error, Debug)]
+pub enum GetDependenciesError {
+    #[error(transparent)]
+    ParseUrl(#[from] url::ParseError),
 
     #[error(transparent)]
-    FetchExtensionCompatibilityMatrix(#[from] FetchExtensionCompatibilityMatrixError),
+    Get(WrappedReqwestError),
+
+    #[error(transparent)]
+    ParseJson(WrappedReqwestError),
 }
 
+#[derive(Error, Debug)]
+#[error("'dfx' is the only supported dependency")]
+pub struct DfxOnlySupportedDependency;
+
 #[derive(Error, Debug)]
 #[error("Failed to parse extension manifest URL '{url}'")]
 pub struct GetExtensionDownloadUrlError {

src/dfx-core/src/extension/manager/install.rs

@@ -1,10 +1,12 @@
 use crate::error::extension::{
     DownloadAndInstallExtensionToTempdirError, FinalizeInstallationError,
-    FindLatestExtensionCompatibleVersionError, GetExtensionArchiveNameError,
-    GetExtensionDownloadUrlError, InstallExtensionError,
+    GetExtensionArchiveNameError, GetExtensionDownloadUrlError, GetHighestCompatibleVersionError,
+    InstallExtensionError,
 };
 use crate::error::reqwest::WrappedReqwestError;
-use crate::extension::{manager::ExtensionManager, manifest::ExtensionCompatibilityMatrix};
+use crate::extension::{
+    manager::ExtensionManager, manifest::ExtensionDependencies, url::ExtensionJsonUrl,
+};
 use crate::http::get::get_with_retries;
 use backoff::exponential::ExponentialBackoff;
 use flate2::read::GzDecoder;
@@ -24,9 +26,10 @@ impl ExtensionManager {
     pub async fn install_extension(
         &self,
         extension_name: &str,
+        url: &ExtensionJsonUrl,
         install_as: Option<&str>,
         version: Option<&Version>,
-    ) -> Result<(), InstallExtensionError> {
+    ) -> Result<Version, InstallExtensionError> {
         let effective_extension_name = install_as.unwrap_or(extension_name);
 
         if self
@@ -40,13 +43,15 @@ impl ExtensionManager {
 
         let extension_version = match version {
             Some(version) => version.clone(),
-            None => self.get_highest_compatible_version(extension_name).await?,
+            None => self.get_highest_compatible_version(url).await?,
         };
         let github_release_tag = get_git_release_tag(extension_name, &extension_version);
         let extension_archive = get_extension_archive_name(extension_name)?;
-        let url = get_extension_download_url(&github_release_tag, &extension_archive)?;
+        let archive_url = get_extension_download_url(&github_release_tag, &extension_archive)?;
 
-        let temp_dir = self.download_and_unpack_extension_to_tempdir(url).await?;
+        let temp_dir = self
+            .download_and_unpack_extension_to_tempdir(archive_url)
+            .await?;
 
         self.finalize_installation(
             extension_name,
@@ -55,7 +60,7 @@ impl ExtensionManager {
             temp_dir,
         )?;
 
-        Ok(())
+        Ok(extension_version)
     }
 
     /// Removing the prerelease tag and build metadata, because they should
@@ -70,11 +75,13 @@ impl ExtensionManager {
 
     async fn get_highest_compatible_version(
         &self,
-        extension_name: &str,
-    ) -> Result<Version, FindLatestExtensionCompatibleVersionError> {
-        let manifest = ExtensionCompatibilityMatrix::fetch().await?;
+        url: &ExtensionJsonUrl,
+    ) -> Result<Version, GetHighestCompatibleVersionError> {
+        let dependencies = ExtensionDependencies::fetch(url).await?;
         let dfx_version = self.dfx_version_strip_semver();
-        manifest.find_latest_compatible_extension_version(extension_name, &dfx_version)
+        dependencies
+            .find_highest_compatible_version(&dfx_version)?
+            .ok_or(GetHighestCompatibleVersionError::NoCompatibleVersionFound())
     }
 
     async fn download_and_unpack_extension_to_tempdir(

src/dfx-core/src/extension/manifest/dependencies.rs

@@ -1,8 +1,14 @@
+use crate::error::extension::{DfxOnlySupportedDependency, GetDependenciesError};
+use crate::error::reqwest::WrappedReqwestError;
+use crate::extension::url::ExtensionJsonUrl;
+use crate::http::get::get_with_retries;
 use crate::json::structure::VersionReqWithJsonSchema;
+use backoff::exponential::ExponentialBackoff;
 use candid::Deserialize;
 use schemars::JsonSchema;
 use semver::Version;
 use std::collections::HashMap;
+use std::time::Duration;
 
 type ExtensionVersion = Version;
 type DependencyName = String;
@@ -19,6 +25,51 @@ pub struct ExtensionDependencies(
     pub HashMap<ExtensionVersion, HashMap<DependencyName, DependencyRequirement>>,
 );
 
+impl ExtensionDependencies {
+    pub async fn fetch(url: &ExtensionJsonUrl) -> Result<Self, GetDependenciesError> {
+        let dependencies_json_url = url.to_dependencies_json()?;
+        let retry_policy = ExponentialBackoff {
+            max_elapsed_time: Some(Duration::from_secs(60)),
+            ..Default::default()
+        };
+        let resp = get_with_retries(dependencies_json_url, retry_policy)
+            .await
+            .map_err(GetDependenciesError::Get)?;
+
+        resp.json()
+            .await
+            .map_err(|e| GetDependenciesError::ParseJson(WrappedReqwestError(e)))
+    }
+
+    pub fn find_highest_compatible_version(
+        &self,
+        dfx_version: &Version,
+    ) -> Result<Option<Version>, DfxOnlySupportedDependency> {
+        let mut keys: Vec<&Version> = self.0.keys().collect();
+        keys.sort();
+        keys.reverse(); // check higher extension versions first
+
+        for key in keys {
+            let dependencies = self.0.get(key).unwrap();
+            for (dependency, requirements) in dependencies {
+                if dependency == "dfx" {
+                    match requirements {
+                        DependencyRequirement::Version(req) => {
+                            if req.matches(dfx_version) {
+                                return Ok(Some(key.clone()));
+                            }
+                        }
+                    }
+                } else {
+                    return Err(DfxOnlySupportedDependency);
+                }
+            }
+        }
+
+        Ok(None)
+    }
+}
+
 #[test]
 fn parse_test_file() {
     let f = r#"
@@ -32,6 +83,11 @@ fn parse_test_file() {
     "dfx": {
       "version": ">=0.9.6"
     }
+  },
+  "0.7.0": {
+    "dfx": {
+      "version": ">=0.9.9"
+    }
   }
 }
 "#;
@@ -40,9 +96,10 @@ fn parse_test_file() {
     let manifest = m.unwrap();
 
     let versions = manifest.0.keys().collect::<Vec<_>>();
-    assert_eq!(versions.len(), 2);
+    assert_eq!(versions.len(), 3);
     assert!(versions.contains(&&Version::new(0, 3, 4)));
     assert!(versions.contains(&&Version::new(0, 6, 2)));
+    assert!(versions.contains(&&Version::new(0, 7, 0)));
 
     let v_3_4 = manifest.0.get(&Version::new(0, 3, 4)).unwrap();
     let dfx = v_3_4.get("dfx").unwrap();
@@ -55,4 +112,23 @@ fn parse_test_file() {
     let DependencyRequirement::Version(req) = dfx;
     assert!(req.matches(&semver::Version::new(0, 9, 6)));
     assert!(!req.matches(&semver::Version::new(0, 9, 5)));
+
+    assert_eq!(
+        manifest
+            .find_highest_compatible_version(&Version::new(0, 8, 5))
+            .unwrap(),
+        Some(Version::new(0, 3, 4))
+    );
+    assert_eq!(
+        manifest
+            .find_highest_compatible_version(&Version::new(0, 9, 6))
+            .unwrap(),
+        Some(Version::new(0, 6, 2))
+    );
+    assert_eq!(
+        manifest
+            .find_highest_compatible_version(&Version::new(0, 9, 10))
+            .unwrap(),
+        Some(Version::new(0, 7, 0))
+    );
 }

src/dfx-core/src/extension/manifest/mod.rs

@@ -1,13 +1,8 @@
 //! Directory contains code that parses the .json files.
 
-pub mod compatibility_matrix;
 pub mod dependencies;
 pub mod extension;
 
-/// `compatibility.json` is a file describing the compatibility
-/// matrix between extensions versions and the dfx version.
-pub use compatibility_matrix::ExtensionCompatibilityMatrix;
-
 /// A file that lists the dependencies of all versions of an extension.
 pub use dependencies::ExtensionDependencies;
 

src/dfx-core/src/extension/mod.rs

@@ -1,5 +1,7 @@
 pub mod manager;
 pub mod manifest;
+pub mod url;
+
 use crate::error::extension::ConvertExtensionIntoClapCommandError;
 use crate::extension::{manager::ExtensionManager, manifest::ExtensionManifest};
 use clap::Command;

src/dfx-core/src/extension/url.rs

@@ -0,0 +1,20 @@
+use url::Url;
+
+pub struct ExtensionJsonUrl(Url);
+
+impl ExtensionJsonUrl {
+    pub fn registered(name: &str) -> Result<Self, url::ParseError> {
+        let s = format!(
+            "https://raw.githubusercontent.com/dfinity/dfx-extensions/main/extensions/{name}/extension.json"
+        );
+        Url::parse(&s).map(ExtensionJsonUrl)
+    }
+
+    pub fn to_dependencies_json(&self) -> Result<Url, url::ParseError> {
+        self.as_url().join("dependencies.json")
+    }
+
+    pub fn as_url(&self) -> &Url {
+        &self.0
+    }
+}