Download newest package version from defined repositories

README.md

@@ -70,6 +70,7 @@ inputPackages:
   - https://raw.githubusercontent.com/insightsengineering/scda/main/DESCRIPTION
   - https://raw.githubusercontent.com/insightsengineering/scda.2022/main/DESCRIPTION
   - https://gitlab.example.com/api/v4/projects/123456/repository/files/DESCRIPTION/raw?ref=main
+  # Forward slashes in 'directory/subdirectory/DESCRIPTION' path are replaced by '%2F' due to URL encoding
   - https://gitlab.example.com/api/v4/projects/234567/repository/files/directory%2Fsubdirectory%2FDESCRIPTION/raw?ref=main
 inputRepositories:
   - Bioconductor.BioCsoft=https://bioconductor.org/packages/release/bioc
@@ -166,7 +167,8 @@ For packages which, according to the input lockfile, should be downloaded from C
 
 The packages can be updated selectively by using the `--updatePackages` flag.
 
-Please note that `renv` might have saved the information in the input lockfile that the package should be downloaded from `CRAN`, `RSPM` or BioConductor repository, but at the same time the definition of that repository in the `renv.lock` header (in the `Repositories` section) might be missing. For such packages `locksmith` will try to check what is the newest available package version at [CRAN](https://cloud.r-project.org).
+Please note that `renv` might have saved the information in the input lockfile that a package `P` should be downloaded from `CRAN`, `RSPM` or BioConductor repository, but at the same time the definition of that repository in the `renv.lock` header (in the `Repositories` section) might be missing.
+In this case, `locksmith` will replicate seemingly undocumented `renv` behavior: the version of package `P` in the lockfile will be updated to the latest version found in any of the repositories **defined** in the lockfile.
 
 Please also note that `locksmith` will not verify whether the dependencies of some packages have changed - this means that the set of package names present in the lockfile will stay the same.
 

cmd/construct.go

@@ -20,7 +20,7 @@ import (
 	"strings"
 )
 
-const lowestPossiblePackageVersion = "0.0.0.0"
+const lowestPossiblePackageVersion = "0.0.0.0.0"
 
 // ConstructOutputPackageList generates a list of all packages and their dependencies
 // which should be included in the output renv.lock file,
@@ -286,8 +286,9 @@ func CheckIfVersionSufficient(availableVersionValue string, versionOperator stri
 	requiredVersion := stringsToInts(requiredVersionStrings)
 
 	available := "="
-	// Compare up to 4 dot- or dash-separated version components.
-	for i := 0; i < 4; i++ {
+	// Compare up to 5 dot- or dash-separated version components.
+	// Examples of packages with 5 version components: RcppEigen, RcppArmadillo.
+	for i := 0; i < 5; i++ {
 		breakLoop := false
 		switch {
 		case availableVersion[i] > requiredVersion[i]:

cmd/renv.go

@@ -206,6 +206,28 @@ func UpdateGitPackages(renvLock *RenvLock, updatePackageRegexp string,
 	}
 }
 
+// GetLatestPackageVersionFromAnyRepository searches for the latest version of soughtPackageName
+// in packagesFiles. It returns the name of the repository (as defined in renv.lock header)
+// where the latest version of that package has been found.
+func GetLatestPackageVersionFromAnyRepository(soughtPackageName string, packagesFiles map[string]PackagesFile) string {
+	latestPackageVersion := lowestPossiblePackageVersion
+	latestPackageVersionRepository := ""
+	for repositoryName, p := range packagesFiles {
+		for _, packageDescription := range p.Packages {
+			if packageDescription.Package == soughtPackageName {
+				log.Trace(soughtPackageName, " version ", packageDescription.Version, " found in ", repositoryName, " repository.")
+				if CheckIfVersionSufficient(packageDescription.Version, ">", latestPackageVersion) {
+					latestPackageVersion = packageDescription.Version
+					latestPackageVersionRepository = repositoryName
+				}
+				break
+			}
+		}
+	}
+	log.Trace("Latest version ", latestPackageVersion, " for package ", soughtPackageName, " found in ", latestPackageVersionRepository, " repository.")
+	return latestPackageVersionRepository
+}
+
 // UpdateRepositoryPackages iterates through the packages in renv.lock and updates the entries
 // corresponding to packages downloaded from CRAN-like repositories. Package version is updated
 // in the renvLock struct. Only packages matching the updatePackageRegexp are updated.
@@ -222,19 +244,21 @@ func UpdateRepositoryPackages(renvLock *RenvLock, updatePackageRegexp string,
 		log.Trace("Package ", k, " matches updated packages regexp ",
 			updatePackageRegexp)
 		var repositoryPackagesFile PackagesFile
+		var notFoundRepositoryName string
 		repositoryName := v.Repository
 		repositoryPackagesFile, ok := packagesFiles[repositoryName]
 		if !ok {
-			log.Error(`Could not retrieve PACKAGES for "`, repositoryName, `" repository `,
-				`(referenced by `, k, `). Attempting to use CRAN's PACKAGES as a fallback.`)
-			repositoryPackagesFile = packagesFiles["CRAN"]
-			repositoryName = "CRAN"
+			// Package coming from a repository not defined in the lockfile.
+			// Check which of the defined repositories has the latest version of that package.
+			notFoundRepositoryName = repositoryName
+			repositoryName = GetLatestPackageVersionFromAnyRepository(k, packagesFiles)
+			repositoryPackagesFile = packagesFiles[repositoryName]
 		}
 		var newPackageVersion string
 		for _, singlePackage := range repositoryPackagesFile.Packages {
 			if singlePackage.Package == k {
 				newPackageVersion = singlePackage.Version
-				continue
+				break
 			}
 		}
 		if newPackageVersion == "" {
@@ -243,6 +267,13 @@ func UpdateRepositoryPackages(renvLock *RenvLock, updatePackageRegexp string,
 		}
 		if entry, ok := renvLock.Packages[k]; ok {
 			if newPackageVersion != entry.Version {
+				if notFoundRepositoryName != "" {
+					log.Warn(
+						"Repository ", notFoundRepositoryName, " referenced by package ", k, " has not ",
+						"been defined in the lockfile and ", k, " will be updated to the latest version ",
+						`found in "`, repositoryName, `" repository.`,
+					)
+				}
 				log.Info("Updating package ", k, " version: ",
 					entry.Version, " → ", newPackageVersion)
 				entry.Version = newPackageVersion
@@ -262,19 +293,6 @@ func GetPackagesFiles(renvLock RenvLock) map[string]PackagesFile {
 		packagesFile := ProcessPackagesFile(packagesFileContent)
 		repositoryPackagesFiles[repository.Name] = packagesFile
 	}
-
-	// Check if the PACKAGES file from a repository named CRAN has been downloaded.
-	_, ok := repositoryPackagesFiles["CRAN"]
-	if !ok {
-		// If not, save CRAN's PACKAGES file to be used as a fallback, for packages which
-		// (according to renv.lock) should be downloaded from a repository not defined in
-		// the renv.lock header.
-		_, _, cranPackagesContent := DownloadTextFile(
-			"https://cloud.r-project.org/src/contrib/PACKAGES", make(map[string]string),
-		)
-		cranPackagesFile := ProcessPackagesFile(cranPackagesContent)
-		repositoryPackagesFiles["CRAN"] = cranPackagesFile
-	}
 	return repositoryPackagesFiles
 }
 

cmd/renv_test.go

@@ -368,6 +368,12 @@ func Test_UpdateRepositoryPackages(t *testing.T) {
 				"", "", []Dependency{},
 				"", "", "", "", "", "", "", []string{},
 			},
+			{
+				"package19",
+				"5.2.1",
+				"", "", []Dependency{},
+				"", "", "", "", "", "", "", []string{},
+			},
 		},
 	}
 	packagesFiles["Repo2"] = PackagesFile{
@@ -378,6 +384,12 @@ func Test_UpdateRepositoryPackages(t *testing.T) {
 				"", "", []Dependency{},
 				"", "", "", "", "", "", "", []string{},
 			},
+			{
+				"package19",
+				"5.2.2",
+				"", "", []Dependency{},
+				"", "", "", "", "", "", "", []string{},
+			},
 		},
 	}
 	packagesFiles["Repo3"] = PackagesFile{
@@ -388,13 +400,9 @@ func Test_UpdateRepositoryPackages(t *testing.T) {
 				"", "", []Dependency{},
 				"", "", "", "", "", "", "", []string{},
 			},
-		},
-	}
-	packagesFiles["CRAN"] = PackagesFile{
-		[]PackageDescription{
 			{
 				"package19",
-				"5.2.3",
+				"5.2.2.4",
 				"", "", []Dependency{},
 				"", "", "", "", "", "", "", []string{},
 			},
@@ -407,6 +415,6 @@ func Test_UpdateRepositoryPackages(t *testing.T) {
 	assert.Equal(t, renvLock.Packages["package16"].Version, "1.2.3")
 	assert.Equal(t, renvLock.Packages["package17"].Version, "1.1.1")
 	assert.Equal(t, renvLock.Packages["package18"].Version, "2.3.2")
-	assert.Equal(t, renvLock.Packages["package19"].Version, "5.2.3")
+	assert.Equal(t, renvLock.Packages["package19"].Version, "5.2.2.4")
 	assert.Equal(t, renvLock.Packages["package21"].Version, "3.8.1")
 }

cmd/root.go

@@ -58,7 +58,7 @@ func setLogLevel() {
 	log.SetFormatter(customFormatter)
 	log.SetReportCaller(false)
 	customFormatter.FullTimestamp = false
-	fmt.Println("logLevel =", logLevel)
+	fmt.Println(`logLevel = "` + logLevel + `"`)
 	switch logLevel {
 	case "trace":
 		log.SetLevel(logrus.TraceLevel)
@@ -77,6 +77,7 @@ func setLogLevel() {
 
 var rootCmd *cobra.Command
 
+//nolint:revive
 func newRootCommand() {
 	rootCmd = &cobra.Command{
 		Use:   "locksmith",
@@ -93,15 +94,15 @@ in an renv.lock-compatible file.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			setLogLevel()
 
-			fmt.Println("config =", cfgFile)
-			fmt.Println("inputPackageList =", inputPackageList)
-			fmt.Println("inputRepositoryList =", inputRepositoryList)
+			fmt.Println(`config = "` + cfgFile + `"`)
+			fmt.Println(`inputPackageList = "` + inputPackageList + `"`)
+			fmt.Println(`inputRepositoryList = "` + inputRepositoryList + `"`)
 			fmt.Println("inputPackages =", inputPackages)
 			fmt.Println("inputRepositories =", inputRepositories)
-			fmt.Println("inputRenvLock =", inputRenvLock)
-			fmt.Println("outputRenvLock =", outputRenvLock)
-			fmt.Println("allowIncompleteRenvLock =", allowIncompleteRenvLock)
-			fmt.Println("updatePackages =", updatePackages)
+			fmt.Println(`inputRenvLock = "` + inputRenvLock + `"`)
+			fmt.Println(`outputRenvLock = "` + outputRenvLock + `"`)
+			fmt.Println(`allowIncompleteRenvLock = "` + allowIncompleteRenvLock + `"`)
+			fmt.Println(`updatePackages = "` + updatePackages + `"`)
 
 			if runtime.GOOS == "windows" {
 				localTempDirectory = os.Getenv("TMP") + `\tmp\locksmith`
@@ -138,7 +139,7 @@ in an renv.lock-compatible file.`,
 		"Token to download non-public files from GitLab.")
 	rootCmd.PersistentFlags().StringVarP(&inputRenvLock, "inputRenvLock", "n", "",
 		"Lockfile which should be read and updated to include the newest versions of the packages.")
-	rootCmd.PersistentFlags().StringVarP(&outputRenvLock, "outputRenvLock", "o", "renv.lock",
+	rootCmd.PersistentFlags().StringVarP(&outputRenvLock, "outputRenvLock", "k", "renv.lock",
 		"File name to save the output renv.lock file.")
 	rootCmd.PersistentFlags().StringVarP(&allowIncompleteRenvLock, "allowIncompleteRenvLock", "i", "",
 		"Locksmith will fail if any of dependencies of input packages cannot be found in the repositories. "+
@@ -173,7 +174,7 @@ func initConfig() {
 		home, err := os.UserHomeDir()
 		cobra.CheckErr(err)
 
-		// Search config in home directory with name ".locksmith" (without extension).
+		// Search for config in home directory.
 		viper.AddConfigPath(home)
 		viper.SetConfigType("yaml")
 		viper.SetConfigName(".locksmith")