Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor DownloadTextFile #25

Merged
merged 4 commits into from
Feb 15, 2024
+82 −74
Merged

Refactor DownloadTextFile #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f3e7a05
Refactor DownloadTextFile
walkowif Feb 15, 2024
e1309b9
Update
walkowif Feb 15, 2024
4c51b33
Update
walkowif Feb 15, 2024
55c3b1c
Update
walkowif Feb 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 33 additions & 26 deletions cmd/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package cmd
import (
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"regexp"
Expand All @@ -45,43 +47,48 @@ type GitHubObject struct {
Sha string `json:"sha"`
}

// DownloadTextFile returns HTTP status code for downloaded file, number of bytes
// in downloaded content, and the downloaded content itself as a string.
func DownloadTextFile(url string, parameters map[string]string) (int, int64, string) { // #nosec G402
// DownloadTextFile returns number of bytes in downloaded content,
// the downloaded content itself as a string, and error if any occurred.
func DownloadTextFile(url string, parameters map[string]string) (int64, string, error) { // #nosec G402
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
req, err := http.NewRequest("GET", url, nil)
checkError(err)
if err != nil {
return 0, "", err
}

for k, v := range parameters {
req.Header.Add(k, v)
}

resp, err := client.Do(req)
checkError(err)

if err == nil {
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
body, err2 := io.ReadAll(resp.Body)
checkError(err2)
return resp.StatusCode, resp.ContentLength, string(body)
if err2 == nil {
return resp.ContentLength, string(body), nil
}
return 0, "", err2
}
return 0, "", errors.New("Received status code " + fmt.Sprint(resp.StatusCode))
}
return -1, 0, ""
return 0, "", err
}

// GetGitLabProjectAndSha retrieves information about GitLab repository
// (project path, repository name and commit SHA)
// from projectURL GitLab API endpoint.
func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[string]string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) (string, string, string) {
downloadFileFunction func(string, map[string]string) (int64, string, error)) (string, string, string) {
var remoteUsername, remoteRepo, remoteSha string
log.Trace("Downloading data for GitLab project from ", projectURL)
statusCode, _, projectDataResponse := downloadFileFunction(projectURL, token)
if statusCode == 200 {
_, projectDataResponse, err := downloadFileFunction(projectURL, token)
if err == nil {
var projectData GitLabAPIResponse
err := json.Unmarshal([]byte(projectDataResponse), &projectData)
checkError(err)
err2 := json.Unmarshal([]byte(projectDataResponse), &projectData)
checkError(err2)
projectPath := strings.Split(projectData.PathWithNamespace, "/")
projectPathLength := len(projectPath)
remoteUsername = strings.Join(projectPath[:projectPathLength-1], "/")
Expand All @@ -99,8 +106,8 @@ func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin
urlPath = "branches"
}
tagOrBranchURL := projectURL + "/repository/" + urlPath + "/" + remoteRef
statusCode, _, tagOrBranchDataResponse := downloadFileFunction(tagOrBranchURL, token)
if statusCode == 200 {
_, tagOrBranchDataResponse, err := downloadFileFunction(tagOrBranchURL, token)
if err == nil {
var tagOrBranchData GitLabTagOrBranchResponse
err := json.Unmarshal([]byte(tagOrBranchDataResponse), &tagOrBranchData)
checkError(err)
Expand All @@ -114,7 +121,7 @@ func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin

// GetGitHubSha retrieves SHA of the remoteRef from the remoteUsername/remoteRepo GitHub repository.
func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, token map[string]string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) string {
downloadFileFunction func(string, map[string]string) (int64, string, error)) string {
var remoteSha string
log.Trace("Downloading data for GitHub project ", remoteUsername, "/", remoteRepo)
match, errMatch := regexp.MatchString(`v\d+(\.\d+)*`, remoteRef)
Expand All @@ -128,8 +135,8 @@ func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, to
urlPath = "heads"
}
tagOrBranchURL := "https://api.github.com/repos/" + remoteUsername + "/" + remoteRepo + "/git/ref/" + urlPath + "/" + remoteRef
statusCode, _, tagDataResponse := downloadFileFunction(tagOrBranchURL, token)
if statusCode == 200 {
_, tagDataResponse, err := downloadFileFunction(tagOrBranchURL, token)
if err == nil {
var tagOrBranchData GitHubTagOrBranchResponse
err := json.Unmarshal([]byte(tagDataResponse), &tagOrBranchData)
checkError(err)
Expand All @@ -144,7 +151,7 @@ func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, to
// ProcessDescriptionURL gets information about the git repository in which the package is stored
// based on the provided descriptionURL to the package DESCRIPTION file.
func ProcessDescriptionURL(descriptionURL string,
downloadFileFunction func(string, map[string]string) (int, int64, string),
downloadFileFunction func(string, map[string]string) (int64, string, error),
) (map[string]string, string, string, string, string, string, string, string, string) {
token := make(map[string]string)
var remoteType, remoteRef, remoteHost, remoteUsername, remoteRepo string
Expand Down Expand Up @@ -198,7 +205,7 @@ func ProcessDescriptionURL(descriptionURL string,
// It returns a list of structures representing: the contents of DESCRIPTION file
// for the packages and various information about git repositories storing the packages.
func DownloadDescriptionFiles(packageDescriptionList []string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) []DescriptionFile {
downloadFileFunction func(string, map[string]string) (int64, string, error)) []DescriptionFile {
var inputDescriptionFiles []DescriptionFile
for _, packageDescriptionURL := range packageDescriptionList {
token, remoteType, packageSource, remoteHost, remoteUsername, remoteRepo, remoteSubdir, remoteRef, remoteSha :=
Expand All @@ -208,8 +215,8 @@ func DownloadDescriptionFiles(packageDescriptionList []string,
", remoteUsername = ", remoteUsername, ", remoteRepo = ", remoteRepo,
", remoteSubdir = ", remoteSubdir, ", remoteRef = ", remoteRef, ", remoteSha = ", remoteSha,
)
statusCode, _, descriptionContent := downloadFileFunction(packageDescriptionURL, token)
if statusCode == 200 {
_, descriptionContent, err := downloadFileFunction(packageDescriptionURL, token)
if err == nil {
inputDescriptionFiles = append(
inputDescriptionFiles,
DescriptionFile{
Expand All @@ -231,7 +238,7 @@ func DownloadDescriptionFiles(packageDescriptionList []string,
// GetPackagesFileContent downloads the PACKAGES file from the repositoryURL using the downloadFileFunction
// and returns the contents, or empty string in case of error.
func GetPackagesFileContent(repositoryURL string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) string {
downloadFileFunction func(string, map[string]string) (int64, string, error)) string {
var packagesFileURL string
if strings.Contains(repositoryURL, "/bin/windows/") || strings.Contains(repositoryURL, "/bin/macosx") {
// If we're dealing with a repository with binary Windows or macOS packages,
Expand All @@ -241,8 +248,8 @@ func GetPackagesFileContent(repositoryURL string,
packagesFileURL = repositoryURL + "/src/contrib/PACKAGES"
}
log.Debug("Downloading ", packagesFileURL)
statusCode, _, packagesFileContent := downloadFileFunction(packagesFileURL, map[string]string{})
if statusCode == 200 {
_, packagesFileContent, err := downloadFileFunction(packagesFileURL, map[string]string{})
if err == nil {
return packagesFileContent
}
log.Warn("An error occurred while downloading ", packagesFileURL)
Expand All @@ -253,7 +260,7 @@ func GetPackagesFileContent(repositoryURL string,
// Returns a map from repository URL to the string with the contents of PACKAGES file
// for that repository.
func DownloadPackagesFiles(repositoryList []string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) map[string]string {
downloadFileFunction func(string, map[string]string) (int64, string, error)) map[string]string {
inputPackagesFiles := make(map[string]string)
for _, repository := range repositoryList {
inputPackagesFiles[repository] = GetPackagesFileContent(repository, downloadFileFunction)
Expand Down
97 changes: 49 additions & 48 deletions cmd/download_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,152 +17,153 @@ limitations under the License.
package cmd

import (
"errors"
"testing"

"github.com/stretchr/testify/assert"
)

func mockedDownloadTextFile(url string, _ map[string]string) (int, int64, string) { // nolint: gocyclo
func mockedDownloadTextFile(url string, _ map[string]string) (int64, string, error) { // nolint: gocyclo
switch {
case url == "https://gitlab.example.com/api/v4/projects/37706/repository/tags/v1.3.1":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "aaabbbcccddd111"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/37706":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group1/group2/project1"
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/38706/repository/tags/v1.4.2":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "aaa222ccc444111"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/38706":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group3/group4/group5/project4"
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/30176/repository/tags/v0.2.0":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "fff222ccc444eee"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/30176":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group6/project7"
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/39307/repository/branches/main":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "fff555ddd888eee"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/39307":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group7/project8"
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/39211/repository/branches/main":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "fffeee999888aaa"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/39211":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group9/subgroup10/subgroup11/project9"
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/formatters/git/ref/tags/v0.5.4":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "444eee222111eee"
}
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/rtables/git/ref/tags/v0.6.5":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "555ddd222111ddd"
}
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/nestcolor/git/ref/heads/main":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "555333aaabbbddd"
}
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/tern/git/ref/heads/main":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "555333aaaeeefff"
}
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/rlistings/git/ref/tags/v0.2.6":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "111444999eee222"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/37706/repository/files/subdirectory%2FDESCRIPTION/raw?ref=v1.3.1":
return 200, 0, "DESCRIPTION contents 1"
return 0, "DESCRIPTION contents 1", nil
case url == "https://gitlab.example.com/api/v4/projects/38706/repository/files/subdirectory1%2Fsubdirectory2%2FDESCRIPTION/raw?ref=v1.4.2":
return 200, 0, "DESCRIPTION contents 2"
return 0, "DESCRIPTION contents 2", nil
case url == "https://gitlab.example.com/api/v4/projects/30176/repository/files/DESCRIPTION/raw?ref=v0.2.0":
return 200, 0, "DESCRIPTION contents 3"
return 0, "DESCRIPTION contents 3", nil
case url == "https://gitlab.example.com/api/v4/projects/39307/repository/files/DESCRIPTION/raw?ref=main":
return 200, 0, "DESCRIPTION contents 4"
return 0, "DESCRIPTION contents 4", nil
case url == "https://gitlab.example.com/api/v4/projects/39211/repository/files/subdirectory1%2FDESCRIPTION/raw?ref=main":
return 200, 0, "DESCRIPTION contents 5"
return 0, "DESCRIPTION contents 5", nil
case url == "https://raw.githubusercontent.com/insightsengineering/formatters/v0.5.4/subdirectory/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 6"
return 0, "DESCRIPTION contents 6", nil
case url == "https://raw.githubusercontent.com/insightsengineering/rtables/v0.6.5/subdirectory1/subdirectory2/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 7"
return 0, "DESCRIPTION contents 7", nil
case url == "https://raw.githubusercontent.com/insightsengineering/nestcolor/main/subdirectory/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 8"
return 0, "DESCRIPTION contents 8", nil
case url == "https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 9"
return 0, "DESCRIPTION contents 9", nil
case url == "https://raw.githubusercontent.com/insightsengineering/rlistings/v0.2.6/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 10"
return 0, "DESCRIPTION contents 10", nil
case url == "https://repo1.example.com/repo1/src/contrib/PACKAGES":
return 200, 0, "PACKAGES contents 1"
return 0, "PACKAGES contents 1", nil
case url == "https://repo2.example.com/repo2/src/contrib/PACKAGES":
return 200, 0, "PACKAGES contents 2"
return 0, "PACKAGES contents 2", nil
case url == "https://repo3.example.com/repo3/src/contrib/PACKAGES":
return 200, 0, "PACKAGES contents 3"
return 0, "PACKAGES contents 3", nil
case url == "https://cloud.r-project.org/bin/windows/contrib/4.3/PACKAGES":
return 200, 0, "PACKAGES content bin/windows"
return 0, "PACKAGES content bin/windows", nil
case url == "https://cloud.r-project.org/src/contrib/PACKAGES":
return 200, 0, "PACKAGES content src/contrib"
return 0, "PACKAGES content src/contrib", nil
case url == "https://example.com/src/contrib/PACKAGES":
return 404, 0, "Not found"
return 0, "", errors.New("Not found")
}
return 200, 0, ""
return 0, "", nil
}

func Test_DownloadDescriptionFiles(t *testing.T) {
Expand Down
Loading