diff --git a/cmd/download.go b/cmd/download.go index 90e8909..61968f7 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -19,6 +19,8 @@ package cmd import ( "crypto/tls" "encoding/json" + "errors" + "fmt" "io" "net/http" "regexp" @@ -45,43 +47,48 @@ type GitHubObject struct { Sha string `json:"sha"` } -// DownloadTextFile returns HTTP status code for downloaded file, number of bytes -// in downloaded content, and the downloaded content itself as a string. -func DownloadTextFile(url string, parameters map[string]string) (int, int64, string) { // #nosec G402 +// DownloadTextFile returns number of bytes in downloaded content, +// the downloaded content itself as a string, and error if any occurred. +func DownloadTextFile(url string, parameters map[string]string) (int64, string, error) { // #nosec G402 tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} client := &http.Client{Transport: tr} req, err := http.NewRequest("GET", url, nil) checkError(err) + if err != nil { + return 0, "", err + } + for k, v := range parameters { req.Header.Add(k, v) } resp, err := client.Do(req) - checkError(err) - if err == nil { defer resp.Body.Close() if resp.StatusCode == http.StatusOK { body, err2 := io.ReadAll(resp.Body) - checkError(err2) - return resp.StatusCode, resp.ContentLength, string(body) + if err2 == nil { + return resp.ContentLength, string(body), nil + } + return 0, "", err2 } + return 0, "", errors.New("Received status code " + fmt.Sprint(resp.StatusCode)) } - return -1, 0, "" + return 0, "", err } // GetGitLabProjectAndSha retrieves information about GitLab repository // (project path, repository name and commit SHA) // from projectURL GitLab API endpoint. func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[string]string, - downloadFileFunction func(string, map[string]string) (int, int64, string)) (string, string, string) { + downloadFileFunction func(string, map[string]string) (int64, string, error)) (string, string, string) { var remoteUsername, remoteRepo, remoteSha string log.Trace("Downloading data for GitLab project from ", projectURL) - statusCode, _, projectDataResponse := downloadFileFunction(projectURL, token) - if statusCode == 200 { + _, projectDataResponse, err := downloadFileFunction(projectURL, token) + if err == nil { var projectData GitLabAPIResponse - err := json.Unmarshal([]byte(projectDataResponse), &projectData) - checkError(err) + err2 := json.Unmarshal([]byte(projectDataResponse), &projectData) + checkError(err2) projectPath := strings.Split(projectData.PathWithNamespace, "/") projectPathLength := len(projectPath) remoteUsername = strings.Join(projectPath[:projectPathLength-1], "/") @@ -99,8 +106,8 @@ func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin urlPath = "branches" } tagOrBranchURL := projectURL + "/repository/" + urlPath + "/" + remoteRef - statusCode, _, tagOrBranchDataResponse := downloadFileFunction(tagOrBranchURL, token) - if statusCode == 200 { + _, tagOrBranchDataResponse, err := downloadFileFunction(tagOrBranchURL, token) + if err == nil { var tagOrBranchData GitLabTagOrBranchResponse err := json.Unmarshal([]byte(tagOrBranchDataResponse), &tagOrBranchData) checkError(err) @@ -114,7 +121,7 @@ func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin // GetGitHubSha retrieves SHA of the remoteRef from the remoteUsername/remoteRepo GitHub repository. func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, token map[string]string, - downloadFileFunction func(string, map[string]string) (int, int64, string)) string { + downloadFileFunction func(string, map[string]string) (int64, string, error)) string { var remoteSha string log.Trace("Downloading data for GitHub project ", remoteUsername, "/", remoteRepo) match, errMatch := regexp.MatchString(`v\d+(\.\d+)*`, remoteRef) @@ -128,8 +135,8 @@ func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, to urlPath = "heads" } tagOrBranchURL := "https://api.github.com/repos/" + remoteUsername + "/" + remoteRepo + "/git/ref/" + urlPath + "/" + remoteRef - statusCode, _, tagDataResponse := downloadFileFunction(tagOrBranchURL, token) - if statusCode == 200 { + _, tagDataResponse, err := downloadFileFunction(tagOrBranchURL, token) + if err == nil { var tagOrBranchData GitHubTagOrBranchResponse err := json.Unmarshal([]byte(tagDataResponse), &tagOrBranchData) checkError(err) @@ -144,7 +151,7 @@ func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, to // ProcessDescriptionURL gets information about the git repository in which the package is stored // based on the provided descriptionURL to the package DESCRIPTION file. func ProcessDescriptionURL(descriptionURL string, - downloadFileFunction func(string, map[string]string) (int, int64, string), + downloadFileFunction func(string, map[string]string) (int64, string, error), ) (map[string]string, string, string, string, string, string, string, string, string) { token := make(map[string]string) var remoteType, remoteRef, remoteHost, remoteUsername, remoteRepo string @@ -198,7 +205,7 @@ func ProcessDescriptionURL(descriptionURL string, // It returns a list of structures representing: the contents of DESCRIPTION file // for the packages and various information about git repositories storing the packages. func DownloadDescriptionFiles(packageDescriptionList []string, - downloadFileFunction func(string, map[string]string) (int, int64, string)) []DescriptionFile { + downloadFileFunction func(string, map[string]string) (int64, string, error)) []DescriptionFile { var inputDescriptionFiles []DescriptionFile for _, packageDescriptionURL := range packageDescriptionList { token, remoteType, packageSource, remoteHost, remoteUsername, remoteRepo, remoteSubdir, remoteRef, remoteSha := @@ -208,8 +215,8 @@ func DownloadDescriptionFiles(packageDescriptionList []string, ", remoteUsername = ", remoteUsername, ", remoteRepo = ", remoteRepo, ", remoteSubdir = ", remoteSubdir, ", remoteRef = ", remoteRef, ", remoteSha = ", remoteSha, ) - statusCode, _, descriptionContent := downloadFileFunction(packageDescriptionURL, token) - if statusCode == 200 { + _, descriptionContent, err := downloadFileFunction(packageDescriptionURL, token) + if err == nil { inputDescriptionFiles = append( inputDescriptionFiles, DescriptionFile{ @@ -231,7 +238,7 @@ func DownloadDescriptionFiles(packageDescriptionList []string, // GetPackagesFileContent downloads the PACKAGES file from the repositoryURL using the downloadFileFunction // and returns the contents, or empty string in case of error. func GetPackagesFileContent(repositoryURL string, - downloadFileFunction func(string, map[string]string) (int, int64, string)) string { + downloadFileFunction func(string, map[string]string) (int64, string, error)) string { var packagesFileURL string if strings.Contains(repositoryURL, "/bin/windows/") || strings.Contains(repositoryURL, "/bin/macosx") { // If we're dealing with a repository with binary Windows or macOS packages, @@ -241,8 +248,8 @@ func GetPackagesFileContent(repositoryURL string, packagesFileURL = repositoryURL + "/src/contrib/PACKAGES" } log.Debug("Downloading ", packagesFileURL) - statusCode, _, packagesFileContent := downloadFileFunction(packagesFileURL, map[string]string{}) - if statusCode == 200 { + _, packagesFileContent, err := downloadFileFunction(packagesFileURL, map[string]string{}) + if err == nil { return packagesFileContent } log.Warn("An error occurred while downloading ", packagesFileURL) @@ -253,7 +260,7 @@ func GetPackagesFileContent(repositoryURL string, // Returns a map from repository URL to the string with the contents of PACKAGES file // for that repository. func DownloadPackagesFiles(repositoryList []string, - downloadFileFunction func(string, map[string]string) (int, int64, string)) map[string]string { + downloadFileFunction func(string, map[string]string) (int64, string, error)) map[string]string { inputPackagesFiles := make(map[string]string) for _, repository := range repositoryList { inputPackagesFiles[repository] = GetPackagesFileContent(repository, downloadFileFunction) diff --git a/cmd/download_test.go b/cmd/download_test.go index b34c4f4..e077f04 100644 --- a/cmd/download_test.go +++ b/cmd/download_test.go @@ -17,152 +17,153 @@ limitations under the License. package cmd import ( + "errors" "testing" "github.com/stretchr/testify/assert" ) -func mockedDownloadTextFile(url string, _ map[string]string) (int, int64, string) { // nolint: gocyclo +func mockedDownloadTextFile(url string, _ map[string]string) (int64, string, error) { // nolint: gocyclo switch { case url == "https://gitlab.example.com/api/v4/projects/37706/repository/tags/v1.3.1": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "commit": { "someotherdata": "someotherdata", "id": "aaabbbcccddd111" } - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/37706": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "path_with_namespace": "group1/group2/project1" - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/38706/repository/tags/v1.4.2": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "commit": { "someotherdata": "someotherdata", "id": "aaa222ccc444111" } - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/38706": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "path_with_namespace": "group3/group4/group5/project4" - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/30176/repository/tags/v0.2.0": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "commit": { "someotherdata": "someotherdata", "id": "fff222ccc444eee" } - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/30176": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "path_with_namespace": "group6/project7" - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/39307/repository/branches/main": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "commit": { "someotherdata": "someotherdata", "id": "fff555ddd888eee" } - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/39307": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "path_with_namespace": "group7/project8" - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/39211/repository/branches/main": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "commit": { "someotherdata": "someotherdata", "id": "fffeee999888aaa" } - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/39211": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "path_with_namespace": "group9/subgroup10/subgroup11/project9" - }` + }`, nil case url == "https://api.github.com/repos/insightsengineering/formatters/git/ref/tags/v0.5.4": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "object": { "someotherdata": "someotherdata", "sha": "444eee222111eee" } - }` + }`, nil case url == "https://api.github.com/repos/insightsengineering/rtables/git/ref/tags/v0.6.5": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "object": { "someotherdata": "someotherdata", "sha": "555ddd222111ddd" } - }` + }`, nil case url == "https://api.github.com/repos/insightsengineering/nestcolor/git/ref/heads/main": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "object": { "someotherdata": "someotherdata", "sha": "555333aaabbbddd" } - }` + }`, nil case url == "https://api.github.com/repos/insightsengineering/tern/git/ref/heads/main": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "object": { "someotherdata": "someotherdata", "sha": "555333aaaeeefff" } - }` + }`, nil case url == "https://api.github.com/repos/insightsengineering/rlistings/git/ref/tags/v0.2.6": - return 200, 0, `{ + return 0, `{ "someotherdata": "someotherdata", "object": { "someotherdata": "someotherdata", "sha": "111444999eee222" } - }` + }`, nil case url == "https://gitlab.example.com/api/v4/projects/37706/repository/files/subdirectory%2FDESCRIPTION/raw?ref=v1.3.1": - return 200, 0, "DESCRIPTION contents 1" + return 0, "DESCRIPTION contents 1", nil case url == "https://gitlab.example.com/api/v4/projects/38706/repository/files/subdirectory1%2Fsubdirectory2%2FDESCRIPTION/raw?ref=v1.4.2": - return 200, 0, "DESCRIPTION contents 2" + return 0, "DESCRIPTION contents 2", nil case url == "https://gitlab.example.com/api/v4/projects/30176/repository/files/DESCRIPTION/raw?ref=v0.2.0": - return 200, 0, "DESCRIPTION contents 3" + return 0, "DESCRIPTION contents 3", nil case url == "https://gitlab.example.com/api/v4/projects/39307/repository/files/DESCRIPTION/raw?ref=main": - return 200, 0, "DESCRIPTION contents 4" + return 0, "DESCRIPTION contents 4", nil case url == "https://gitlab.example.com/api/v4/projects/39211/repository/files/subdirectory1%2FDESCRIPTION/raw?ref=main": - return 200, 0, "DESCRIPTION contents 5" + return 0, "DESCRIPTION contents 5", nil case url == "https://raw.githubusercontent.com/insightsengineering/formatters/v0.5.4/subdirectory/DESCRIPTION": - return 200, 0, "DESCRIPTION contents 6" + return 0, "DESCRIPTION contents 6", nil case url == "https://raw.githubusercontent.com/insightsengineering/rtables/v0.6.5/subdirectory1/subdirectory2/DESCRIPTION": - return 200, 0, "DESCRIPTION contents 7" + return 0, "DESCRIPTION contents 7", nil case url == "https://raw.githubusercontent.com/insightsengineering/nestcolor/main/subdirectory/DESCRIPTION": - return 200, 0, "DESCRIPTION contents 8" + return 0, "DESCRIPTION contents 8", nil case url == "https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION": - return 200, 0, "DESCRIPTION contents 9" + return 0, "DESCRIPTION contents 9", nil case url == "https://raw.githubusercontent.com/insightsengineering/rlistings/v0.2.6/DESCRIPTION": - return 200, 0, "DESCRIPTION contents 10" + return 0, "DESCRIPTION contents 10", nil case url == "https://repo1.example.com/repo1/src/contrib/PACKAGES": - return 200, 0, "PACKAGES contents 1" + return 0, "PACKAGES contents 1", nil case url == "https://repo2.example.com/repo2/src/contrib/PACKAGES": - return 200, 0, "PACKAGES contents 2" + return 0, "PACKAGES contents 2", nil case url == "https://repo3.example.com/repo3/src/contrib/PACKAGES": - return 200, 0, "PACKAGES contents 3" + return 0, "PACKAGES contents 3", nil case url == "https://cloud.r-project.org/bin/windows/contrib/4.3/PACKAGES": - return 200, 0, "PACKAGES content bin/windows" + return 0, "PACKAGES content bin/windows", nil case url == "https://cloud.r-project.org/src/contrib/PACKAGES": - return 200, 0, "PACKAGES content src/contrib" + return 0, "PACKAGES content src/contrib", nil case url == "https://example.com/src/contrib/PACKAGES": - return 404, 0, "Not found" + return 0, "", errors.New("Not found") } - return 200, 0, "" + return 0, "", nil } func Test_DownloadDescriptionFiles(t *testing.T) {