From abaa6aeedab055f2f3a760bbbaa35e0e90e02457 Mon Sep 17 00:00:00 2001 From: David Lesner Date: Thu, 30 May 2024 13:31:36 +0300 Subject: [PATCH 1/6] Support evidence service (#1188) --- artifactory/utils/utils.go | 22 ++++++++++++++++++++++ go.mod | 6 +++--- go.sum | 12 ++++++------ utils/config/config.go | 12 ++++++++++++ 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/artifactory/utils/utils.go b/artifactory/utils/utils.go index 1699a4d43..3879559cc 100644 --- a/artifactory/utils/utils.go +++ b/artifactory/utils/utils.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" ioutils "github.com/jfrog/gofrog/io" + "github.com/jfrog/jfrog-client-go/evidence" "io" "net/http" "net/url" @@ -214,6 +215,27 @@ func CreateLifecycleServiceManager(serviceDetails *config.ServerDetails, isDryRu return lifecycle.New(serviceConfig) } +func CreateEvidenceServiceManager(serviceDetails *config.ServerDetails, isDryRun bool) (*evidence.EvidenceServicesManager, error) { + certsPath, err := coreutils.GetJfrogCertsDir() + if err != nil { + return nil, err + } + evdAuth, err := serviceDetails.CreateEvidenceAuthConfig() + if err != nil { + return nil, err + } + serviceConfig, err := clientConfig.NewConfigBuilder(). + SetServiceDetails(evdAuth). + SetCertificatesPath(certsPath). + SetInsecureTls(serviceDetails.InsecureTls). + SetDryRun(isDryRun). + Build() + if err != nil { + return nil, err + } + return evidence.New(serviceConfig) +} + // This error indicates that the build was scanned by Xray, but Xray found issues with the build. // If Xray failed to scan the build, for example due to a networking issue, a regular error should be returned. var errBuildScan = errors.New("issues found during xray build scan") diff --git a/go.mod b/go.mod index c9a5b3c7a..66709f03e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/gookit/color v1.5.4 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/jfrog/build-info-go v1.9.27 - github.com/jfrog/gofrog v1.7.1 + github.com/jfrog/gofrog v1.7.2 github.com/jfrog/jfrog-client-go v1.40.2 github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 @@ -71,7 +71,7 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/term v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rivo/uniseg v0.4.3 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -96,7 +96,7 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240509093347-62649bc00e43 +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04 // replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go dev diff --git a/go.sum b/go.sum index cdf89edac..a7e03a5e7 100644 --- a/go.sum +++ b/go.sum @@ -85,10 +85,10 @@ github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= github.com/jfrog/build-info-go v1.9.27 h1:7RWJcajqtNNbGHuYkgOLUIG7mmRKF0yxC7mvYAbdVlU= github.com/jfrog/build-info-go v1.9.27/go.mod h1:8T7/ajM9aGshvgpwCtXwIFpyF/R6CEn4W+/FLryNXWw= -github.com/jfrog/gofrog v1.7.1 h1:ME1Meg4hukAT/7X6HUQCVSe4DNjMZACCP8aCY37EW/w= -github.com/jfrog/gofrog v1.7.1/go.mod h1:X7bjfWoQDN0Z4FQGbE91j3gbPP7Urwzm4Z8tkvrlbRI= -github.com/jfrog/jfrog-client-go v1.40.2 h1:zdCWPPT11r0bMGnAXGhZPb3RrIINhiTFCceQABhguZ4= -github.com/jfrog/jfrog-client-go v1.40.2/go.mod h1:m3hIn12eFWk5nJH1swPRtFrjXbiiCscOpX+v/vCdmNI= +github.com/jfrog/gofrog v1.7.2 h1:VkAaA/9tmbw27IqgUOmaZWnO6ATUqL3vRzDnsROKATw= +github.com/jfrog/gofrog v1.7.2/go.mod h1:WJFk88SR9Sr9mKl1bQBig7DmSdXiBGKV3WhL9O6jL9w= +github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04 h1:ERLE/L7YPr6aCUTeAnE8SXU5VOZHd5/XK16rM1TEpts= +github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04/go.mod h1:37RR4pYgXZM4w7tywyfRu8t2wagt0qf5wBtpDILWBsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -148,8 +148,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= -github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/utils/config/config.go b/utils/config/config.go index 2a073da63..287085cb2 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -12,6 +12,7 @@ import ( artifactoryAuth "github.com/jfrog/jfrog-client-go/artifactory/auth" "github.com/jfrog/jfrog-client-go/auth" distributionAuth "github.com/jfrog/jfrog-client-go/distribution/auth" + evidenceAuth "github.com/jfrog/jfrog-client-go/evidence/auth" lifecycleAuth "github.com/jfrog/jfrog-client-go/lifecycle/auth" pipelinesAuth "github.com/jfrog/jfrog-client-go/pipelines/auth" "github.com/jfrog/jfrog-client-go/utils" @@ -578,6 +579,7 @@ type ServerDetails struct { PipelinesUrl string `json:"pipelinesUrl,omitempty"` AccessUrl string `json:"accessUrl,omitempty"` LifecycleUrl string `json:"-"` + EvidenceUrl string `json:"-"` User string `json:"user,omitempty"` Password string `json:"password,omitempty"` SshKeyPath string `json:"sshKeyPath,omitempty"` @@ -668,6 +670,10 @@ func (serverDetails *ServerDetails) GetLifecycleUrl() string { return serverDetails.LifecycleUrl } +func (serverDetails *ServerDetails) GetEvidenceUrl() string { + return serverDetails.EvidenceUrl +} + func (serverDetails *ServerDetails) GetUser() string { return serverDetails.User } @@ -741,6 +747,12 @@ func (serverDetails *ServerDetails) CreateLifecycleAuthConfig() (auth.ServiceDet return serverDetails.createAuthConfig(lcAuth) } +func (serverDetails *ServerDetails) CreateEvidenceAuthConfig() (auth.ServiceDetails, error) { + evdAuth := evidenceAuth.NewEvidenceDetails() + evdAuth.SetUrl(serverDetails.EvidenceUrl) + return serverDetails.createAuthConfig(evdAuth) +} + func (serverDetails *ServerDetails) createAuthConfig(details auth.ServiceDetails) (auth.ServiceDetails, error) { details.SetSshUrl(serverDetails.SshUrl) details.SetAccessToken(serverDetails.AccessToken) From 539f1c60160f7d044a8a29f1e2e9c81191048a8a Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Mon, 3 Jun 2024 09:24:21 +0300 Subject: [PATCH 2/6] Remove the unused `project-init` and `ci-setup` commands (#1190) --- general/cisetup/cisetup.go | 68 --- general/cisetup/githubactionsfilegenerator.go | 69 --- general/cisetup/jenkinsfiledslgenerator.go | 214 --------- general/cisetup/jenkinsfilegenerator.go | 105 ---- general/cisetup/pipelinesconfig.go | 189 -------- general/cisetup/pipelinesyaml.go | 452 ------------------ general/cisetup/utils.go | 147 ------ general/cisetup/utils_test.go | 40 -- general/project/projectinit.go | 257 ---------- general/project/repoutils.go | 186 ------- 10 files changed, 1727 deletions(-) delete mode 100644 general/cisetup/cisetup.go delete mode 100644 general/cisetup/githubactionsfilegenerator.go delete mode 100644 general/cisetup/jenkinsfiledslgenerator.go delete mode 100644 general/cisetup/jenkinsfilegenerator.go delete mode 100644 general/cisetup/pipelinesconfig.go delete mode 100644 general/cisetup/pipelinesyaml.go delete mode 100644 general/cisetup/utils.go delete mode 100644 general/cisetup/utils_test.go delete mode 100644 general/project/projectinit.go delete mode 100644 general/project/repoutils.go diff --git a/general/cisetup/cisetup.go b/general/cisetup/cisetup.go deleted file mode 100644 index adc8f7479..000000000 --- a/general/cisetup/cisetup.go +++ /dev/null @@ -1,68 +0,0 @@ -package cisetup - -import ( - "strings" - - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" -) - -const ConfigServerId = "jfrog-instance" - -type CiSetupData struct { - RepositoryName string - ProjectDomain string - VcsBaseUrl string - LocalDirPath string - GitBranch string - BuildName string - CiType CiType - // A collection of the technologies that were detected in the project. - DetectedTechnologies map[coreutils.Technology]bool - // The chosen build technology stored with all the necessary information. - BuiltTechnology *TechnologyInfo - VcsCredentials VcsServerDetails - GitProvider GitProvider -} - -type TechnologyInfo struct { - Type coreutils.Technology - VirtualRepo string - LocalSnapshotsRepo string - LocalReleasesRepo string - BuildCmd string -} - -func (sd *CiSetupData) GetRepoFullName() string { - return sd.ProjectDomain + "/" + sd.RepositoryName -} - -// Trim technology name from command prefix. (example: mvn clean install >> clean install) -func (sd *CiSetupData) GetBuildCmdForNativeStep() string { - // Remove exec name. - return strings.TrimPrefix(strings.TrimSpace(sd.BuiltTechnology.BuildCmd), sd.BuiltTechnology.Type.GetExecCommandName()+" ") -} - -type VcsServerDetails struct { - Url string `json:"url,omitempty"` - User string `json:"user,omitempty"` - Password string `json:"-"` - AccessToken string `json:"-"` -} - -type GitProvider string - -const ( - Github = "GitHub" - GithubEnterprise = "GitHub Enterprise" - Bitbucket = "Bitbucket" - BitbucketServer = "Bitbucket Server" - Gitlab = "GitLab" -) - -type CiType string - -const ( - Jenkins = "Jenkins" - GithubActions = "GitHub Actions" - Pipelines = "JFrog Pipelines" -) diff --git a/general/cisetup/githubactionsfilegenerator.go b/general/cisetup/githubactionsfilegenerator.go deleted file mode 100644 index 69750c2d0..000000000 --- a/general/cisetup/githubactionsfilegenerator.go +++ /dev/null @@ -1,69 +0,0 @@ -package cisetup - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" -) - -var GithubActionsDir = filepath.Join(".github", "workflows") -var GithubActionsFilePath = filepath.Join(GithubActionsDir, GithubActionsFileName) - -const GithubActionsFileName = "build.yml" -const githubActionsTemplate = ` -name: 'JFrog CI Integration' -on: [push] -jobs: - jfrog-ci-integration: - runs-on: ubuntu-latest - env: - JF_ARTIFACTORY_1: ${{ secrets.JF_ARTIFACTORY_SECRET_1 }} - JFROG_BUILD_STATUS: PASS - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup JFrog CLI - uses: jfrog/setup-jfrog-cli@v1 - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - java-version: '11' - distribution: 'adopt' - - name: Build - run: | - # Configure the project - %s - # Build the project using JFrog CLI - %s - - name: Failure check - run: | - echo "JFROG_BUILD_STATUS=FAIL" >> $GITHUB_ENV - if: failure() - - name: Publish build - run: | - # Collect and store environment variables in the build-info - jfrog rt bce - # Collect and store VCS details in the build-info - jfrog rt bag - # Publish the build-info to Artifactory - jfrog rt bp - # Scan the published build-info with Xray - jfrog rt bs - if: always()` - -type GithubActionsGenerator struct { - SetupData *CiSetupData -} - -func (gg *GithubActionsGenerator) Generate() (githubActionsBytes []byte, githubActionsName string, err error) { - // setM2 env variable if maven is used. - setM2 := gg.SetupData.BuiltTechnology.Type == coreutils.Maven - buildToolsConfigCommands := strings.Join(getTechConfigsCommands(ConfigServerId, setM2, gg.SetupData), "\n ") - buildCommand, err := convertBuildCmd(gg.SetupData) - if err != nil { - return nil, "", err - } - return []byte(fmt.Sprintf(githubActionsTemplate, buildToolsConfigCommands, buildCommand)), GithubActionsFileName, nil -} diff --git a/general/cisetup/jenkinsfiledslgenerator.go b/general/cisetup/jenkinsfiledslgenerator.go deleted file mode 100644 index 3734aec36..000000000 --- a/general/cisetup/jenkinsfiledslgenerator.go +++ /dev/null @@ -1,214 +0,0 @@ -package cisetup - -import ( - "fmt" - "golang.org/x/text/cases" - "golang.org/x/text/language" - "strings" - - "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" -) - -const ( - JenkinsDslFileName = "Jenkinsfile" - resolverIdTemplate = "%s_RESOLVER" - deployerIdTemplate = "%s_DEPLOYER" - homeEnv = "%[1]s_HOME = '/full/path/to/%[1]s' // Set to the local %[1]s installation path." - - jenkinsfileTemplate2 = `pipeline { - - // More info about the Declarative Pipeline Syntax on https://www.jfrog.com/confluence/display/JFROG/Declarative+Pipeline+Syntax - // Declarative syntax is available from version 3.0.0 of the Jenkins Artifactory Plugin. - - agent any - - %s - - %s -}` - - environmentsTemplate = ` - environment { - %s - }` - - allStagesTemplate = ` - stages { - %s - }` - - stageTemplate = ` - stage (%q) { - steps {%s - } - } -` - - cloneStepsTemplate = ` - git branch: %q, - url: %q - // credentialsId: 'git_cred_id'. If cloning the code requires credentials, set the credentials to your git in Jenkins > Configure System > credentials > "username with password" > ID: "git-cred-id"` - - rtConfigServerStepTemplate = ` - rtServer ( - id: %[1]q, - url: %[2]q, - credentialsId: 'rt-cred-id', // Set the credentials to your JFrog instance in Jenkins > Configure System > credentials > "username with password" > ID: "rt-cred-id" - - // bypassProxy: true, (If Jenkins is configured to use an http proxy, you can bypass the proxy when using this Artifactory server) - // timeout: 300 , (Configure the connection timeout (in seconds). The default value (if not configured) is 300 seconds) - ) - rt%[3]sDeployer ( - id: %[4]q, - serverId: %[1]q, - %[5]s, - - // threads: 6, (Optional - Attach custom properties to the published artifacts) - // properties: ['key1=value1', 'key2=value2'], (Optional - Attach custom properties to the published artifacts) - ) - rt%[3]sResolver ( - id: %[6]q, - serverId: %[1]q, - %[7]s - )` - - mavenRepoTemplate = `releaseRepo: %q, - snapshotRepo: %q` - - singleRepoTemplate = `repo: %q` - - mavenRunStepTemplate = ` - rtMavenRun ( - pom: 'pom.xml', // path to pom.xml file - goals: %q, - resolverId: %q, - deployerId: %q, - - // tool: {build installation name}, (Maven tool installation from jenkins from : Jenkins > Manage jenkins > Global Tool Configuration > Maven installations) - // useWrapper: true, (Set to true if you'd like the build to use the Maven Wrapper.) - // opts: '-Xms1024m -Xmx4096m', (Optional - Maven options) - )` - - gradleRunStepTemplate = ` - rtGradleRun ( - buildFile: 'build.gradle', - tasks: %q, - rootDir: "", - resolverId: %q, - deployerId: %q, - - // tool: {build installation name}, // Jenkins > Gradle jenkins > Global Tool Configuration > Gradle installations - )` - - npmInstallStepTemplate = ` - rtNpmInstall ( - resolverId: %q, - - // tool: {build installation name}, (Npm tool installation from jenkins from : Jenkins > Manage jenkins > Global Tool Configuration > NodeJS installations - )` - - npmPublishStepTemplate = ` - rtNpmPublish ( - deployerId: %q, - - // tool: {build installation name}, (Npm tool installation from jenkins from : Jenkins > Manage jenkins > Global Tool Configuration > NodeJS installations - // path: '', (Optional path to the project root. If not set, the root of the workspace is assumed as the root project path.) - // javaArgs: '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005' , (Jenkins spawns a new java process during this step's execution. You have the option of passing any java args to this new process.) - )` - - publishBuildInfoStepsTemplate = ` - rtPublishBuildInfo ( - serverId: %q, - )` -) - -type JenkinsfileDslGenerator struct { - SetupData *CiSetupData -} - -func (jg *JenkinsfileDslGenerator) GenerateDsl() (jenkinsfileBytes []byte, jenkinsfileName string, err error) { - serviceDetails, err := config.GetSpecificConfig(ConfigServerId, false, false) - if err != nil { - return nil, "", err - } - // Generate environments sections - environments := generateEnvironments(jg.SetupData.BuiltTechnology.Type) - // Generate Stages Section - cloneStage := generateStage("Clone", fmt.Sprintf(cloneStepsTemplate, jg.SetupData.GitBranch, jg.SetupData.VcsCredentials.Url)) - rtConfigStage := generateStage("Artifactory configuration", generateRtConfigSteps(jg.SetupData.BuiltTechnology, serviceDetails.ArtifactoryUrl)) - execBuildStage := generateBuildStages(jg.SetupData.GetBuildCmdForNativeStep(), jg.SetupData.BuiltTechnology.Type) - publishBuildInfoStage := generateStage("Publish build info", fmt.Sprintf(publishBuildInfoStepsTemplate, ConfigServerId)) - // Combine all stages together - stagesString := generateAllStages(cloneStage, rtConfigStage, execBuildStage, publishBuildInfoStage) - - return []byte(fmt.Sprintf(jenkinsfileTemplate2, environments, stagesString)), JenkinsDslFileName, nil -} - -func generateStage(stageName, steps string) (stageString string) { - return fmt.Sprintf(stageTemplate, stageName, steps) -} - -func generateAllStages(stages ...string) (allStagesString string) { - allStagesString = "" - for _, stage := range stages { - allStagesString += stage - } - return fmt.Sprintf(allStagesTemplate, allStagesString) -} - -func generateEnvironments(buildType coreutils.Technology) string { - envs := "" - switch buildType { - case coreutils.Maven: - fallthrough - case coreutils.Gradle: - envs += fmt.Sprintf(homeEnv, strings.ToUpper(buildType.String())) - default: - envs += "" - } - if envs == "" { - return "" - } - return fmt.Sprintf(environmentsTemplate, envs) -} - -func generateRtConfigSteps(techInfo *TechnologyInfo, rtUrl string) string { - var deployerRepos string - var resolverRepos string - switch techInfo.Type { - case coreutils.Maven: - deployerRepos = fmt.Sprintf(mavenRepoTemplate, techInfo.LocalReleasesRepo, techInfo.LocalSnapshotsRepo) - resolverRepos = fmt.Sprintf(mavenRepoTemplate, techInfo.VirtualRepo, techInfo.VirtualRepo) - case coreutils.Gradle: - fallthrough - case coreutils.Npm: - deployerRepos = fmt.Sprintf(singleRepoTemplate, techInfo.LocalReleasesRepo) - resolverRepos = fmt.Sprintf(singleRepoTemplate, techInfo.VirtualRepo) - default: - deployerRepos = "//Build type is not supported at the moment" - resolverRepos = "//Build type is not supported at the moment" - } - buildType := string(techInfo.Type) - resolverId := fmt.Sprintf(resolverIdTemplate, strings.ToUpper(buildType)) - deployerId := fmt.Sprintf(deployerIdTemplate, strings.ToUpper(buildType)) - return fmt.Sprintf(rtConfigServerStepTemplate, ConfigServerId, rtUrl, cases.Title(language.Und, cases.NoLower).String(buildType), deployerId, deployerRepos, resolverId, resolverRepos) -} - -func generateBuildStages(buildCmd string, buildType coreutils.Technology) (buildStages string) { - buildStages = "" - resolverId := fmt.Sprintf(resolverIdTemplate, strings.ToUpper(buildType.String())) - deployerId := fmt.Sprintf(deployerIdTemplate, strings.ToUpper(buildType.String())) - switch buildType { - case coreutils.Maven: - buildStages += generateStage("Exec Maven", fmt.Sprintf(mavenRunStepTemplate, buildCmd, resolverId, deployerId)) - case coreutils.Gradle: - buildStages += generateStage("Exec Gradle", fmt.Sprintf(gradleRunStepTemplate, buildCmd, resolverId, deployerId)) - case coreutils.Npm: - buildStages += generateStage("Exec Npm install", fmt.Sprintf(npmInstallStepTemplate, resolverId)) - buildStages += generateStage("Exec Npm publish", fmt.Sprintf(npmPublishStepTemplate, deployerId)) - default: - buildStages = "//Build type is not supported at the moment" - } - return buildStages -} diff --git a/general/cisetup/jenkinsfilegenerator.go b/general/cisetup/jenkinsfilegenerator.go deleted file mode 100644 index 81cbb09a7..000000000 --- a/general/cisetup/jenkinsfilegenerator.go +++ /dev/null @@ -1,105 +0,0 @@ -package cisetup - -import ( - "fmt" - "strings" - - "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" -) - -const JenkinsfileName = "Jenkinsfile" -const m2HomeSet = ` - // The M2_HOME environment variable should be set to the local maven installation path. - M2_HOME = ""` -const jenkinsfileTemplate = `pipeline { - agent any - environment { - %s - - JFROG_CLI_BUILD_NAME = "${JOB_NAME}" - JFROG_CLI_BUILD_NUMBER = "${BUILD_NUMBER}" - - // Sets the CI server build URL in the build-info. - JFROG_CLI_BUILD_URL = "https:////${BUILD_NUMBER}/console" - - } - stages { - stage ('Clone') { - steps { - // If cloning the code requires credentials, follow these steps: - // 1. Uncomment the ending of the below 'git' step. - // 2. Create the 'git_cred_id' credentials as described here - https://www.jenkins.io/doc/book/using/using-credentials/ - git branch: %q, url: %q //, credentialsId: 'git_cred_id' - } - } - - stage ('Config') { - steps { - // Download JFrog CLI. - sh 'curl -fL https://getcli.jfrog.io | sh && chmod +x jfrog' - - // Configure JFrog CLI - withCredentials([string(credentialsId: 'rt-password', variable: 'RT_PASSWORD')]) { - sh '''./jfrog c add %s --url %s --user ${RT_USERNAME} --password ${RT_PASSWORD} - ./%s - ''' - } - } - } - - stage ('Build') { - steps { - dir('%s') { - sh '''./%s''' - } - } - } - } - - post { - success { - script { - env.JFROG_BUILD_STATUS="PASS" - } - } - - failure { - script { - env.JFROG_BUILD_STATUS="FAIL" - } - } - - cleanup { - // Collect and store environment variables in the build-info - sh './jfrog rt bce' - // Collect and store VCS details in the build-info - sh './jfrog rt bag' - // Publish the build-info to Artifactory - sh './jfrog rt bp' - sh './jfrog c remove %s --quiet' - } - } - }` - -type JenkinsfileGenerator struct { - SetupData *CiSetupData -} - -func (jg *JenkinsfileGenerator) Generate() (jenkinsfileBytes []byte, jenkinsfileName string, err error) { - serviceDetails, err := config.GetSpecificConfig(ConfigServerId, false, false) - if err != nil { - return nil, "", err - } - buildToolsConfigCommands := strings.Join(getTechConfigsCommands(ConfigServerId, false, jg.SetupData), cmdAndOperator) - buildCommand, err := convertBuildCmd(jg.SetupData) - if err != nil { - return nil, "", err - } - var envSet string - // Set the M2_HOME env variable if maven is used. - if jg.SetupData.BuiltTechnology.Type == coreutils.Maven { - envSet = m2HomeSet - } - return []byte(fmt.Sprintf(jenkinsfileTemplate, envSet, jg.SetupData.GitBranch, jg.SetupData.VcsCredentials.Url, ConfigServerId, serviceDetails.Url, buildToolsConfigCommands, jg.SetupData.RepositoryName, buildCommand, ConfigServerId)), JenkinsfileName, nil -} diff --git a/general/cisetup/pipelinesconfig.go b/general/cisetup/pipelinesconfig.go deleted file mode 100644 index 34f685f01..000000000 --- a/general/cisetup/pipelinesconfig.go +++ /dev/null @@ -1,189 +0,0 @@ -package cisetup - -import ( - "strings" - - "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "github.com/jfrog/jfrog-client-go/artifactory" - clientConfig "github.com/jfrog/jfrog-client-go/config" - "github.com/jfrog/jfrog-client-go/pipelines" - "github.com/jfrog/jfrog-client-go/pipelines/services" - "github.com/jfrog/jfrog-client-go/utils/errorutils" - "github.com/jfrog/jfrog-client-go/utils/log" -) - -const ( - PipelinesYamlName = "pipelines.yml" -) - -type JFrogPipelinesConfigurator struct { - SetupData *CiSetupData - PipelinesToken string -} - -func (pc *JFrogPipelinesConfigurator) Config() (vcsIntName, rtIntName string, err error) { - log.Info("Configuring JFrog Pipelines...") - serviceDetails, err := config.GetSpecificConfig(ConfigServerId, false, false) - if err != nil { - return "", "", err - } - - psm, err := pc.createPipelinesServiceManager(serviceDetails) - if err != nil { - return "", "", err - } - - vcsIntName, vcsIntId, err := pc.createVcsIntegration(psm) - if err != nil { - return "", "", err - } - - rtIntName, err = pc.createArtifactoryIntegration(psm, serviceDetails) - if err != nil { - return "", "", err - } - - err = pc.doAddPipelineSource(psm, vcsIntId) - if err != nil { - return "", "", err - } - return vcsIntName, rtIntName, nil -} - -func (pc *JFrogPipelinesConfigurator) doAddPipelineSource(psm *pipelines.PipelinesServicesManager, projectIntegrationId int) (err error) { - _, err = psm.AddPipelineSource(projectIntegrationId, pc.SetupData.GetRepoFullName(), pc.SetupData.GitBranch, PipelinesYamlName) - // If source already exists, ignore error. - if _, ok := err.(*services.SourceAlreadyExistsError); ok { - log.Debug("Pipeline Source for repo '" + pc.SetupData.GetRepoFullName() + "' and branch '" + pc.SetupData.GitBranch + "' already exists and will be used.") - err = nil - } - return -} - -func (pc *JFrogPipelinesConfigurator) createPipelinesServiceManager(details *config.ServerDetails) (*pipelines.PipelinesServicesManager, error) { - // Create new details with pipelines token. - pipelinesDetails := *details - pipelinesDetails.AccessToken = pc.PipelinesToken - pipelinesDetails.User = "" - pipelinesDetails.Password = "" - - certsPath, err := coreutils.GetJfrogCertsDir() - if err != nil { - return nil, err - } - pAuth, err := pipelinesDetails.CreatePipelinesAuthConfig() - if err != nil { - return nil, err - } - serviceConfig, err := clientConfig.NewConfigBuilder(). - SetServiceDetails(pAuth). - SetCertificatesPath(certsPath). - SetInsecureTls(pipelinesDetails.InsecureTls). - SetDryRun(false). - Build() - if err != nil { - return nil, err - } - return pipelines.New(serviceConfig) -} - -func (pc *JFrogPipelinesConfigurator) createVcsIntegration(psm *pipelines.PipelinesServicesManager) (integrationName string, integrationId int, err error) { - switch pc.SetupData.GitProvider { - case Github: - integrationName = pc.createIntegrationName(services.GithubName) - integrationId, err = psm.CreateGithubIntegration(integrationName, pc.SetupData.VcsCredentials.AccessToken) - case GithubEnterprise: - integrationName = pc.createIntegrationName(services.GithubEnterpriseName) - integrationId, err = psm.CreateGithubEnterpriseIntegration(integrationName, pc.SetupData.VcsBaseUrl, pc.SetupData.VcsCredentials.AccessToken) - case Bitbucket: - integrationName = pc.createIntegrationName(services.BitbucketName) - integrationId, err = psm.CreateBitbucketIntegration(integrationName, pc.SetupData.VcsCredentials.User, pc.SetupData.VcsCredentials.AccessToken) - case BitbucketServer: - integrationName = pc.createIntegrationName(services.BitbucketServerName) - cred := pc.SetupData.VcsCredentials.AccessToken - if cred == "" { - cred = pc.SetupData.VcsCredentials.Password - } - integrationId, err = psm.CreateBitbucketServerIntegration(integrationName, pc.SetupData.VcsBaseUrl, pc.SetupData.VcsCredentials.User, cred) - case Gitlab: - integrationName = pc.createIntegrationName(services.GitlabName) - integrationId, err = psm.CreateGitlabIntegration(integrationName, pc.SetupData.VcsBaseUrl, pc.SetupData.VcsCredentials.AccessToken) - default: - return "", -1, errorutils.CheckErrorf("vcs type is not supported at the moment") - } - if _, ok := err.(*services.IntegrationAlreadyExistsError); ok { - // If integration already exists, get the id from pipelines. - log.Debug("Integration '" + integrationName + "' already exists and will be used. Fetching id from pipelines...") - var integration *services.Integration - integration, err = psm.GetIntegrationByName(integrationName) - if err != nil { - return - } - integrationId = integration.Id - } - return -} - -func (pc *JFrogPipelinesConfigurator) createArtifactoryIntegration(psm *pipelines.PipelinesServicesManager, details *config.ServerDetails) (string, error) { - integrationName := pc.createIntegrationName("rt") - apiKey, err := pc.getApiKey(details) - if err != nil { - return "", err - } - user := details.User - _, err = psm.CreateArtifactoryIntegration(integrationName, details.ArtifactoryUrl, user, apiKey) - // If integration already exists, ignore error. - if _, ok := err.(*services.IntegrationAlreadyExistsError); ok { - log.Debug("Integration '" + integrationName + "' already exists and will be used.") - err = nil - } - return integrationName, err -} - -// Get API Key if exists, generate one if needed. -func (pc *JFrogPipelinesConfigurator) getApiKey(details *config.ServerDetails) (string, error) { - // Try getting API Key for the user if exists. - asm, err := pc.createRtServiceManager(details) - if err != nil { - return "", err - } - apiKey, err := asm.GetAPIKey() - if err != nil || apiKey != "" { - return apiKey, err - } - - // Generate API Key for the user. - return asm.CreateAPIKey() -} - -func (pc *JFrogPipelinesConfigurator) createIntegrationName(intType string) string { - return intType + "_" + createPipelinesSuitableName(pc.SetupData, "integration") -} - -func (pc *JFrogPipelinesConfigurator) createRtServiceManager(artDetails *config.ServerDetails) (artifactory.ArtifactoryServicesManager, error) { - certsPath, err := coreutils.GetJfrogCertsDir() - if err != nil { - return nil, err - } - artAuth, err := artDetails.CreateArtAuthConfig() - if err != nil { - return nil, err - } - serviceConfig, err := clientConfig.NewConfigBuilder(). - SetServiceDetails(artAuth). - SetCertificatesPath(certsPath). - SetInsecureTls(artDetails.InsecureTls). - SetDryRun(false). - Build() - if err != nil { - return nil, err - } - return artifactory.New(serviceConfig) -} - -func createPipelinesSuitableName(data *CiSetupData, suffix string) string { - name := strings.Join([]string{data.ProjectDomain, data.RepositoryName, suffix}, "_") - // Pipelines does not allow "-" which might exist in repo names. - return strings.ReplaceAll(name, "-", "_") -} diff --git a/general/cisetup/pipelinesyaml.go b/general/cisetup/pipelinesyaml.go deleted file mode 100644 index 1302b754f..000000000 --- a/general/cisetup/pipelinesyaml.go +++ /dev/null @@ -1,452 +0,0 @@ -package cisetup - -import ( - "encoding/json" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "github.com/jfrog/jfrog-client-go/utils" - "github.com/jfrog/jfrog-client-go/utils/errorutils" - "gopkg.in/yaml.v3" - "strconv" -) - -const addRunFilesCmd = "add_run_files /tmp/jfrog/. jfrog" - -type JFrogPipelinesYamlGenerator struct { - VcsIntName string - RtIntName string - SetupData *CiSetupData -} - -func (yg *JFrogPipelinesYamlGenerator) Generate() (pipelineBytes []byte, pipelineName string, err error) { - pipelineName = yg.createPipelineName() - gitResourceName := yg.createGitResourceName() - biResourceName := yg.createBuildInfoResourceName() - gitResource := yg.createGitResource(gitResourceName) - biResource := yg.createBuildInfoResource(biResourceName) - pipeline, err := yg.createPipeline(pipelineName, gitResourceName, biResourceName) - if err != nil { - return nil, "", err - } - pipelineYaml := PipelineYml{ - Resources: []Resource{gitResource, biResource}, - Pipelines: []Pipeline{pipeline}, - } - pipelineBytes, err = yaml.Marshal(&pipelineYaml) - return pipelineBytes, pipelineName, errorutils.CheckError(err) -} - -func (yg *JFrogPipelinesYamlGenerator) getNpmBashCommands(serverId, gitResourceName, convertedBuildCmd string) []string { - var commandsArray []string - commandsArray = append(commandsArray, getCdToResourceCmd(gitResourceName)) - commandsArray = append(commandsArray, getJfrogCliConfigCmd(yg.RtIntName, serverId, true)) - commandsArray = append(commandsArray, getBuildToolConfigCmd(npmConfigCmdName, serverId, yg.SetupData.BuiltTechnology.VirtualRepo)) - commandsArray = append(commandsArray, convertedBuildCmd) - commandsArray = append(commandsArray, jfrogCliBag) - commandsArray = append(commandsArray, jfrogCliBce) - return commandsArray -} - -// Converts build tools commands to run via JFrog CLI. -func (yg *JFrogPipelinesYamlGenerator) convertNpmBuildCmd() (string, error) { - // Replace npm-i. - converted, err := replaceCmdWithRegexp(yg.SetupData.BuiltTechnology.BuildCmd, npmInstallRegexp, npmInstallRegexpReplacement) - if err != nil { - return "", err - } - // Replace npm-ci. - return replaceCmdWithRegexp(converted, npmCiRegexp, npmCiRegexpReplacement) -} - -func replaceCmdWithRegexp(buildCmd, cmdRegexp, replacement string) (string, error) { - regexp, err := utils.GetRegExp(cmdRegexp) - if err != nil { - return "", err - } - return regexp.ReplaceAllString(buildCmd, replacement), nil -} - -func (yg *JFrogPipelinesYamlGenerator) getPipelineEnvVars() map[string]string { - return map[string]string{ - coreutils.CI: strconv.FormatBool(true), - buildNameEnvVar: yg.SetupData.BuildName, - buildNumberEnvVar: runNumberEnvVar, - buildUrlEnvVar: stepUrlEnvVar, - } -} - -func (yg *JFrogPipelinesYamlGenerator) createGitResource(gitResourceName string) Resource { - return Resource{ - Name: gitResourceName, - ResourceType: GitRepo, - ResourceConfiguration: GitRepoResourceConfiguration{ - Path: yg.SetupData.GetRepoFullName(), - GitProvider: yg.VcsIntName, - BuildOn: BuildOn{ - PullRequestCreate: true, - }, - Branches: IncludeExclude{Include: yg.SetupData.GitBranch}, - }, - } -} - -func (yg *JFrogPipelinesYamlGenerator) createBuildInfoResource(buildInfoResourceName string) Resource { - return Resource{ - Name: buildInfoResourceName, - ResourceType: BuildInfo, - ResourceConfiguration: BuildInfoResourceConfiguration{ - SourceArtifactoryIntegration: yg.RtIntName, - BuildName: yg.SetupData.BuildName, - BuildNumber: runNumberEnvVar, - }, - } -} - -func (yg *JFrogPipelinesYamlGenerator) createPipeline(pipelineName, gitResourceName, buildInfoResourceName string) (Pipeline, error) { - steps, err := yg.createSteps(gitResourceName, buildInfoResourceName) - if err != nil { - return Pipeline{}, err - } - return Pipeline{ - Name: pipelineName, - Steps: steps, - Configuration: PipelineConfiguration{ - PipelineEnvVars: PipelineEnvVars{ - ReadOnlyEnvVars: yg.getPipelineEnvVars(), - }, - }, - }, nil -} - -func (yg *JFrogPipelinesYamlGenerator) createSteps(gitResourceName, buildInfoResourceName string) (steps []PipelineStep, err error) { - var step PipelineStep - - switch yg.SetupData.BuiltTechnology.Type { - case coreutils.Maven: - step = yg.createMavenStep(gitResourceName) - case coreutils.Gradle: - step = yg.createGradleStep(gitResourceName) - case coreutils.Npm: - step, err = yg.createNpmStep(gitResourceName) - if err != nil { - return nil, err - } - } - - return []PipelineStep{step, yg.createBuildInfoStep(gitResourceName, step.Name, buildInfoResourceName)}, nil -} - -func (yg *JFrogPipelinesYamlGenerator) createMavenStep(gitResourceName string) PipelineStep { - return PipelineStep{ - Name: createTechStepName(MvnBuild), - StepType: MvnBuild, - Configuration: &MavenStepConfiguration{ - NativeStepConfiguration: yg.getDefaultNativeStepConfiguration(gitResourceName), - MvnCommand: yg.SetupData.GetBuildCmdForNativeStep(), - ResolverSnapshotRepo: yg.SetupData.BuiltTechnology.VirtualRepo, - ResolverReleaseRepo: yg.SetupData.BuiltTechnology.VirtualRepo, - }, - Execution: StepExecution{ - OnFailure: yg.getOnFailureCommands(), - }, - } -} - -func (yg *JFrogPipelinesYamlGenerator) getDefaultNativeStepConfiguration(gitResourceName string) NativeStepConfiguration { - step := NativeStepConfiguration{ - BaseStepConfiguration: BaseStepConfiguration{ - EnvironmentVariables: map[string]string{ - buildStatusEnvVar: passResult, - }, - Integrations: []StepIntegration{ - { - Name: yg.RtIntName, - }, - }, - InputResources: []StepResource{ - { - Name: gitResourceName, - }, - }, - }, - AutoPublishBuildInfo: false, - ForceXrayScan: false, - } - return step -} - -func (yg *JFrogPipelinesYamlGenerator) createGradleStep(gitResourceName string) PipelineStep { - return PipelineStep{ - Name: createTechStepName(GradleBuild), - StepType: GradleBuild, - Configuration: &GradleStepConfiguration{ - NativeStepConfiguration: yg.getDefaultNativeStepConfiguration(gitResourceName), - GradleCommand: yg.SetupData.GetBuildCmdForNativeStep(), - ResolverRepo: yg.SetupData.BuiltTechnology.VirtualRepo, - }, - Execution: StepExecution{ - OnFailure: yg.getOnFailureCommands(), - }, - } -} - -func (yg *JFrogPipelinesYamlGenerator) createNpmStep(gitResourceName string) (PipelineStep, error) { - serverId := yg.createServerIdName() - - converted, err := yg.convertNpmBuildCmd() - if err != nil { - return PipelineStep{}, err - } - - commands := yg.getNpmBashCommands(serverId, gitResourceName, converted) - - step := PipelineStep{ - Name: createTechStepName(NpmBuild), - StepType: Bash, - Configuration: &BaseStepConfiguration{ - EnvironmentVariables: map[string]string{ - buildStatusEnvVar: passResult, - }, - InputResources: []StepResource{ - { - Name: gitResourceName, - }, - }, - Integrations: []StepIntegration{ - { - Name: yg.RtIntName, - }, - }, - }, - Execution: StepExecution{ - OnExecute: commands, - OnComplete: []string{addRunFilesCmd}, - OnFailure: yg.getOnFailureCommands(), - }, - } - return step, nil -} - -func (yg *JFrogPipelinesYamlGenerator) createBuildInfoStep(gitResourceName, previousStepName, buildInfoResourceName string) PipelineStep { - return PipelineStep{ - Name: createTechStepName(PublishBuildInfo), - StepType: PublishBuildInfo, - Configuration: &NativeStepConfiguration{ - BaseStepConfiguration: BaseStepConfiguration{ - InputSteps: []InputStep{ - { - Name: previousStepName, - }, - }, - InputResources: []StepResource{ - { - Name: gitResourceName, - }, - }, - OutputResources: []StepResource{ - { - Name: buildInfoResourceName, - }, - }, - }, - ForceXrayScan: true, - }, - Execution: StepExecution{ - OnComplete: []string{yg.getUpdateCommitStatusCmd(gitResourceName)}, - }, - } -} - -type PipelineYml struct { - Resources []Resource `yaml:"resources,omitempty"` - Pipelines []Pipeline `yaml:"pipelines,omitempty"` -} - -type ResourceType string - -const ( - GitRepo ResourceType = "GitRepo" - BuildInfo ResourceType = "BuildInfo" -) - -type Resource struct { - Name string `yaml:"name,omitempty"` - ResourceType `yaml:"type,omitempty"` - ResourceConfiguration `yaml:"configuration,omitempty"` -} - -type ResourceConfiguration interface { - ResourceConfigurationMarkerFunction() -} - -type GitRepoResourceConfiguration struct { - Path string `yaml:"path,omitempty"` - GitProvider string `yaml:"gitProvider,omitempty"` - BuildOn `yaml:"buildOn,omitempty"` - Branches IncludeExclude `yaml:"branches,omitempty"` -} - -func (g GitRepoResourceConfiguration) ResourceConfigurationMarkerFunction() {} - -type BuildInfoResourceConfiguration struct { - SourceArtifactoryIntegration string `yaml:"sourceArtifactory,omitempty"` - BuildName string `yaml:"buildName,omitempty"` - BuildNumber json.Number `yaml:"buildNumber,omitempty"` -} - -func (b BuildInfoResourceConfiguration) ResourceConfigurationMarkerFunction() {} - -type IncludeExclude struct { - Include string `yaml:"include,omitempty"` - Exclude string `yaml:"exclude,omitempty"` -} - -type BuildOn struct { - PullRequestCreate bool `yaml:"pullRequestCreate,omitempty"` - Commit bool `yaml:"commit,omitempty"` -} - -type Pipeline struct { - Name string `yaml:"name,omitempty"` - Configuration PipelineConfiguration `yaml:"configuration,omitempty"` - Steps []PipelineStep `yaml:"steps,omitempty"` -} - -type PipelineConfiguration struct { - Runtime `yaml:"runtime,omitempty"` - PipelineEnvVars `yaml:"environmentVariables,omitempty"` -} - -type PipelineEnvVars struct { - ReadOnlyEnvVars map[string]string `yaml:"readOnly,omitempty"` -} - -type RuntimeType string - -const ( - Image RuntimeType = "image" -) - -type Runtime struct { - RuntimeType `yaml:"type,omitempty"` - Image RuntimeImage `yaml:"image,omitempty"` -} - -type RuntimeImage struct { - Custom CustomImage `yaml:"custom,omitempty"` -} - -type CustomImage struct { - Name string `yaml:"name,omitempty"` - Tag string `yaml:"tag,omitempty"` - Options string `yaml:"options,omitempty"` - Registry string `yaml:"registry,omitempty"` - SourceRepository string `yaml:"sourceRepository,omitempty"` - Region string `yaml:"region,omitempty"` -} - -type StepType string - -const ( - MvnBuild StepType = "MvnBuild" - GradleBuild StepType = "GradleBuild" - NpmBuild StepType = "NpmBuild" - Bash StepType = "Bash" - PublishBuildInfo StepType = "PublishBuildInfo" -) - -type PipelineStep struct { - Name string `yaml:"name,omitempty"` - StepType `yaml:"type,omitempty"` - Configuration StepConfiguration `yaml:"configuration,omitempty"` - Execution StepExecution `yaml:"execution,omitempty"` -} - -type StepConfiguration interface { - appendInputSteps([]InputStep) -} - -type BaseStepConfiguration struct { - EnvironmentVariables map[string]string `yaml:"environmentVariables,omitempty"` - Integrations []StepIntegration `yaml:"integrations,omitempty"` - InputResources []StepResource `yaml:"inputResources,omitempty"` - OutputResources []StepResource `yaml:"outputResources,omitempty"` - InputSteps []InputStep `yaml:"inputSteps,omitempty"` -} - -func (b *BaseStepConfiguration) appendInputSteps(steps []InputStep) { - b.InputSteps = append(b.InputSteps, steps...) -} - -type NativeStepConfiguration struct { - BaseStepConfiguration `yaml:",inline"` - ForceXrayScan bool `yaml:"forceXrayScan,omitempty"` - FailOnScan bool `yaml:"failOnScan,omitempty"` - AutoPublishBuildInfo bool `yaml:"autoPublishBuildInfo,omitempty"` -} - -type MavenStepConfiguration struct { - NativeStepConfiguration `yaml:",inline"` - MvnCommand string `yaml:"mvnCommand,omitempty"` - ResolverSnapshotRepo string `yaml:"resolverSnapshotRepo,omitempty"` - ResolverReleaseRepo string `yaml:"resolverReleaseRepo,omitempty"` - DeployerSnapshotRepo string `yaml:"deployerSnapshotRepo,omitempty"` - DeployerReleaseRepo string `yaml:"deployerReleaseRepo,omitempty"` -} - -type GradleStepConfiguration struct { - NativeStepConfiguration `yaml:",inline"` - GradleCommand string `yaml:"gradleCommand,omitempty"` - ResolverRepo string `yaml:"resolverRepo,omitempty"` - UsesPlugin bool `yaml:"usesPlugin,omitempty"` - UseWrapper bool `yaml:"useWrapper,omitempty"` -} - -type StepIntegration struct { - Name string `yaml:"name,omitempty"` -} - -type StepResource struct { - Name string `yaml:"name,omitempty"` -} - -type InputStep struct { - Name string `yaml:"name,omitempty"` -} - -type StepExecution struct { - OnStart []string `yaml:"onStart,omitempty"` - OnExecute []string `yaml:"onExecute,omitempty"` - OnComplete []string `yaml:"onComplete,omitempty"` - OnSuccess []string `yaml:"onSuccess,omitempty"` - OnFailure []string `yaml:"onFailure,omitempty"` -} - -func (yg *JFrogPipelinesYamlGenerator) getOnFailureCommands() []string { - return []string{getExportCmd(buildStatusEnvVar, failResult), - jfrogCliBce, - jfrogCliBp} -} - -func (yg *JFrogPipelinesYamlGenerator) getUpdateCommitStatusCmd(gitResourceName string) string { - return updateCommitStatusCmd + " " + gitResourceName -} - -func (yg *JFrogPipelinesYamlGenerator) createGitResourceName() string { - return createPipelinesSuitableName(yg.SetupData, "gitResource") -} - -func (yg *JFrogPipelinesYamlGenerator) createBuildInfoResourceName() string { - return createPipelinesSuitableName(yg.SetupData, "buildInfoResource") -} - -func (yg *JFrogPipelinesYamlGenerator) createPipelineName() string { - return createPipelinesSuitableName(yg.SetupData, "pipeline") -} - -func (yg *JFrogPipelinesYamlGenerator) createServerIdName() string { - return createPipelinesSuitableName(yg.SetupData, "serverId") -} - -func createTechStepName(stepType StepType) string { - return string(stepType) + "Step" -} diff --git a/general/cisetup/utils.go b/general/cisetup/utils.go deleted file mode 100644 index 089c496c6..000000000 --- a/general/cisetup/utils.go +++ /dev/null @@ -1,147 +0,0 @@ -package cisetup - -import ( - "fmt" - "strings" - - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" -) - -const ( - m2pathCmd = "MVN_PATH=`which mvn` && export M2_HOME=`readlink -f $MVN_PATH | xargs dirname | xargs dirname`" - jfrogCliRtPrefix = "jfrog rt" - jfrogCliConfig = "jfrog c add" - jfrogCliOldConfig = "jfrog rt c" - jfrogCliBce = "jfrog rt bce" - jfrogCliBag = "jfrog rt bag" - jfrogCliBp = "jfrog rt bp" - buildNameEnvVar = "JFROG_CLI_BUILD_NAME" - buildNumberEnvVar = "JFROG_CLI_BUILD_NUMBER" - buildUrlEnvVar = "JFROG_CLI_BUILD_URL" - buildStatusEnvVar = "JFROG_BUILD_STATUS" - runNumberEnvVar = "$run_number" - stepUrlEnvVar = "$step_url" - updateCommitStatusCmd = "update_commit_status" - - gradleConfigCmdName = "gradle-config" - npmConfigCmdName = "npm-config" - mvnConfigCmdName = "mvn-config" - serverIdResolve = "server-id-resolve" - repoResolveReleases = "repo-resolve-releases" - repoResolveSnapshots = "repo-resolve-snapshots" - repoResolve = "repo-resolve" - - passResult = "PASS" - failResult = "FAIL" - - urlFlag = "url" - rtUrlFlag = "artifactory-url" - userFlag = "user" - - // Replace exe (group 2) with "jfrog rt exe" while maintaining preceding (if any) and succeeding spaces. - mvnGradleRegexp = `(^|\s)(mvn|gradle)(\s)` - mvnGradleRegexpReplacement = `${1}jfrog rt ${2}${3}` - npmInstallRegexp = `(^|\s)(npm i|npm install)(\s|$)` - npmInstallRegexpReplacement = `${1}jfrog rt npmi${3}` - npmCiRegexp = `(^|\s)(npm ci)(\s|$)` - npmCiRegexpReplacement = `${1}jfrog rt npmci${3}` - - cmdAndOperator = " &&\n" -) - -func getFlagSyntax(flagName string) string { - return fmt.Sprintf("--%s", flagName) -} - -func getCdToResourceCmd(gitResourceName string) string { - return fmt.Sprintf("cd $res_%s_resourcePath", gitResourceName) -} - -func getIntDetailForCmd(intName, detail string) string { - return fmt.Sprintf("$int_%s_%s", intName, detail) -} - -// Returns the JFrog CLI config command according to the given server details. -func getJfrogCliConfigCmd(rtIntName, serverId string, useOld bool) string { - usedConfigCmd := jfrogCliConfig - usedUrlFlag := rtUrlFlag - if useOld { - usedConfigCmd = jfrogCliOldConfig - usedUrlFlag = urlFlag - } - return strings.Join([]string{ - usedConfigCmd, serverId, - getFlagSyntax(usedUrlFlag), getIntDetailForCmd(rtIntName, urlFlag), - getFlagSyntax(userFlag), getIntDetailForCmd(rtIntName, userFlag), - "--enc-password=false", - }, " ") -} - -// Returns an array of JFrog CLI config commands according to the given CiSetupData. -func getTechConfigsCommands(serverId string, setM2ForMaven bool, data *CiSetupData) []string { - var configs []string - switch data.BuiltTechnology.Type { - case coreutils.Maven: - if setM2ForMaven { - configs = append(configs, m2pathCmd) - } - configs = append(configs, getMavenConfigCmd(serverId, data.BuiltTechnology.VirtualRepo)) - - case coreutils.Gradle: - configs = append(configs, getBuildToolConfigCmd(gradleConfigCmdName, serverId, data.BuiltTechnology.VirtualRepo)) - - case coreutils.Npm: - configs = append(configs, getBuildToolConfigCmd(npmConfigCmdName, serverId, data.BuiltTechnology.VirtualRepo)) - - } - return configs -} - -// Converts build tools commands to run via JFrog CLI. -func convertBuildCmd(data *CiSetupData) (buildCmd string, err error) { - commandsArray := []string{} - switch data.BuiltTechnology.Type { - case coreutils.Npm: - buildCmd, err = replaceCmdWithRegexp(data.BuiltTechnology.BuildCmd, npmInstallRegexp, npmInstallRegexpReplacement) - if err != nil { - return "", err - } - buildCmd, err = replaceCmdWithRegexp(buildCmd, npmCiRegexp, npmCiRegexpReplacement) - if err != nil { - return "", err - } - case coreutils.Maven, coreutils.Gradle: - buildCmd, err = replaceCmdWithRegexp(data.BuiltTechnology.BuildCmd, mvnGradleRegexp, mvnGradleRegexpReplacement) - if err != nil { - return "", err - } - } - commandsArray = append(commandsArray, buildCmd) - return strings.Join(commandsArray, cmdAndOperator), nil -} - -// Returns Maven's config command according to given server and repo information. -func getMavenConfigCmd(serverId, repo string) string { - return strings.Join([]string{ - jfrogCliRtPrefix, mvnConfigCmdName, - getFlagSyntax(serverIdResolve), serverId, - getFlagSyntax(repoResolveReleases), repo, - getFlagSyntax(repoResolveSnapshots), repo, - }, " ") -} - -// Returns build tool's (except Maven) config command according to given server and repo information. -func getBuildToolConfigCmd(configCmd, serverId, repo string) string { - return strings.Join([]string{ - jfrogCliRtPrefix, configCmd, - getFlagSyntax(serverIdResolve), serverId, - getFlagSyntax(repoResolve), repo, - }, " ") -} - -// Returns a string of environment variable export command. -// key - The variable name. -// value - the value to be set. -func getExportCmd(key, value string) string { - return fmt.Sprintf("export %s=%s", key, value) -} diff --git a/general/cisetup/utils_test.go b/general/cisetup/utils_test.go deleted file mode 100644 index 42d01e238..000000000 --- a/general/cisetup/utils_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package cisetup - -import ( - "testing" - - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "github.com/stretchr/testify/assert" -) - -func TestConvertBuildCmd(t *testing.T) { - tests := []buildCmd{ - {"simpleMvn", coreutils.Maven, "mvn clean install", "jfrog rt mvn clean install"}, - {"simpleGradle", coreutils.Gradle, "gradle clean build", "jfrog rt gradle clean build"}, - {"simpleNpmInstall", coreutils.Npm, "npm install", "jfrog rt npmi"}, - {"simpleNpmI", coreutils.Npm, "npm i", "jfrog rt npmi"}, - {"simpleNpmCi", coreutils.Npm, "npm ci", "jfrog rt npmci"}, - {"hiddenMvn", coreutils.Npm, "npm i FOLDERmvnHERE", "jfrog rt npmi FOLDERmvnHERE"}, - {"hiddenNpm", coreutils.Maven, "mvn clean install -f \"HIDDENnpm/pom.xml\"", "jfrog rt mvn clean install -f \"HIDDENnpm/pom.xml\""}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - data := &CiSetupData{} - data.BuiltTechnology = &TechnologyInfo{Type: test.tech, BuildCmd: test.original} - converted, err := convertBuildCmd(data) - if err != nil { - assert.NoError(t, err) - return - } - assert.Equal(t, test.expected, converted) - }) - } -} - -type buildCmd struct { - name string - tech coreutils.Technology - original string - expected string -} diff --git a/general/project/projectinit.go b/general/project/projectinit.go deleted file mode 100644 index aa133e601..000000000 --- a/general/project/projectinit.go +++ /dev/null @@ -1,257 +0,0 @@ -package project - -import ( - "os" - "os/exec" - "path" - "path/filepath" - "strings" - - commonCommands "github.com/jfrog/jfrog-cli-core/v2/common/commands" - "github.com/jfrog/jfrog-cli-core/v2/common/project" - "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "github.com/jfrog/jfrog-client-go/utils/errorutils" - "github.com/jfrog/jfrog-client-go/utils/io/fileutils" - "gopkg.in/yaml.v3" -) - -const ( - buildFileName = "build.yaml" -) - -type ProjectInitCommand struct { - projectPath string - serverId string - serverUrl string -} - -func NewProjectInitCommand() *ProjectInitCommand { - return &ProjectInitCommand{} -} - -func (pic *ProjectInitCommand) SetProjectPath(path string) *ProjectInitCommand { - pic.projectPath = path - return pic -} - -func (pic *ProjectInitCommand) SetServerId(id string) *ProjectInitCommand { - pic.serverId = id - return pic -} - -func (pic *ProjectInitCommand) Run() (err error) { - if pic.serverId == "" { - defaultServer, err := config.GetSpecificConfig("", true, false) - if err != nil { - return err - } - pic.serverId = defaultServer.ServerId - pic.serverUrl = defaultServer.Url - } - technologiesMap, err := pic.detectTechnologies() - if err != nil { - return err - } - if _, errNotFound := exec.LookPath("docker"); errNotFound == nil { - technologiesMap[coreutils.Docker] = true - } - // First create repositories for the detected technologies. - for techName := range technologiesMap { - // First create repositories for the detected technology. - err = createDefaultReposIfNeeded(techName, pic.serverId) - if err != nil { - return err - } - err = createProjectBuildConfigs(techName, pic.projectPath, pic.serverId) - if err != nil { - return err - } - } - // Create build config - if err = pic.createBuildConfig(); err != nil { - return - } - - err = coreutils.PrintTable("", "", pic.createSummarizeMessage(technologiesMap), false) - return -} - -func (pic *ProjectInitCommand) createSummarizeMessage(technologiesMap map[coreutils.Technology]bool) string { - return coreutils.PrintBold("This project is initialized!\n") + - coreutils.PrintBold("The project config is stored inside the .jfrog directory.") + - "\n\n" + - coreutils.PrintTitle("🔍 Scan the dependencies of this project for security vulnerabilities by running") + - "\n" + - "jf audit\n\n" + - coreutils.PrintTitle("📦 Scan any software package on you machine for security vulnerabilities by running") + - "\n" + - "jf scan path/to/dir/or/package\n\n" + - coreutils.PrintTitle("🐳 Scan any local docker image on you machine for security vulnerabilities by running") + - "\n" + - "jf docker scan :\n\n" + - coreutils.PrintTitle("💻 If you're using VS Code, IntelliJ IDEA, WebStorm, PyCharm, Android Studio or GoLand") + - "\n" + - "Open the IDE 👉 Install the JFrog extension or plugin 👉 View the JFrog panel" + - "\n\n" + - pic.createBuildMessage(technologiesMap) + - coreutils.PrintTitle("📚 Read more using this link:") + - "\n" + - coreutils.PrintLink(coreutils.GettingStartedGuideUrl) -} - -// Return a string message, which includes all the build and deployment commands, matching the technologiesMap sent. -func (pic *ProjectInitCommand) createBuildMessage(technologiesMap map[coreutils.Technology]bool) string { - message := "" - for tech := range technologiesMap { - switch tech { - case coreutils.Maven: - message += "jf mvn install deploy\n" - case coreutils.Gradle: - message += "jf gradle artifactoryP\n" - case coreutils.Npm: - message += "jf npm install\n" - message += "jf npm publish\n" - case coreutils.Go: - message += - "jf go build\n" + - "jf go-publish v1.0.0\n" - case coreutils.Pip, coreutils.Pipenv: - message += - "jf " + string(tech) + " install\n" + - "jf rt upload path/to/package/file default-pypi-local" + - coreutils.PrintComment(" #Publish your "+string(tech)+" package") + - "\n" - case coreutils.Dotnet: - executableName := coreutils.Nuget - _, errNotFound := exec.LookPath("dotnet") - if errNotFound == nil { - // dotnet exists in path, So use it in the instruction message. - executableName = coreutils.Dotnet - } - message += - "jf " + string(executableName) + " restore\n" + - "jf rt upload '*.nupkg'" + RepoDefaultName[tech][Virtual] + "\n" - } - } - if message != "" { - message = coreutils.PrintTitle("🚧 Build the code & deploy the packages by running") + - "\n" + - message + - "\n" - } - if ok := technologiesMap[coreutils.Docker]; ok { - baseurl := strings.TrimPrefix(strings.TrimSpace(pic.serverUrl), "https://") - baseurl = strings.TrimPrefix(baseurl, "http://") - imageUrl := path.Join(baseurl, DockerVirtualDefaultName, ":") - message += coreutils.PrintTitle("🐳 Pull and push any docker image using Artifactory") + - "\n" + - "jf docker tag : " + imageUrl + "\n" + - "jf docker push " + imageUrl + "\n" + - "jf docker pull " + imageUrl + "\n\n" - } - - if message != "" { - message += coreutils.PrintTitle("📤 Publish the build-info to Artifactory") + - "\n" + - "jf rt build-publish\n\n" - } - return message -} - -// Returns all detected technologies found in the project directory. -// First, try to return only the technologies that detected according to files in the root directory. -// In case no indication found in the root directory, the search continue recursively. -func (pic *ProjectInitCommand) detectTechnologies() (technologiesMap map[coreutils.Technology]bool, err error) { - technologiesMap, err = coreutils.DetectTechnologies(pic.projectPath, false, false) - if err != nil { - return - } - // In case no technologies were detected in the root directory, try again recursively. - if len(technologiesMap) == 0 { - technologiesMap, err = coreutils.DetectTechnologies(pic.projectPath, false, true) - if err != nil { - return - } - } - return -} - -type BuildConfigFile struct { - Version int `yaml:"version,omitempty"` - ConfigType string `yaml:"type,omitempty"` - BuildName string `yaml:"name,omitempty"` -} - -func (pic *ProjectInitCommand) createBuildConfig() error { - jfrogProjectDir := filepath.Join(pic.projectPath, ".jfrog", "projects") - if err := fileutils.CreateDirIfNotExist(jfrogProjectDir); err != nil { - return errorutils.CheckError(err) - } - configFilePath := filepath.Join(jfrogProjectDir, buildFileName) - projectDirName := filepath.Base(filepath.Dir(pic.projectPath)) - buildConfigFile := &BuildConfigFile{Version: 1, ConfigType: "build", BuildName: projectDirName} - resBytes, err := yaml.Marshal(&buildConfigFile) - if err != nil { - return errorutils.CheckError(err) - } - return errorutils.CheckError(os.WriteFile(configFilePath, resBytes, 0644)) -} - -func createDefaultReposIfNeeded(tech coreutils.Technology, serverId string) error { - err := CreateDefaultLocalRepo(tech, serverId) - if err != nil { - return err - } - err = CreateDefaultRemoteRepo(tech, serverId) - if err != nil { - return err - } - - return CreateDefaultVirtualRepo(tech, serverId) -} - -func createProjectBuildConfigs(tech coreutils.Technology, projectPath string, serverId string) error { - jfrogProjectDir := filepath.Join(projectPath, ".jfrog", "projects") - if err := fileutils.CreateDirIfNotExist(jfrogProjectDir); err != nil { - return errorutils.CheckError(err) - } - techName := strings.ToLower(string(tech)) - configFilePath := filepath.Join(jfrogProjectDir, techName+".yaml") - configFile := commonCommands.ConfigFile{ - Version: commonCommands.BuildConfVersion, - ConfigType: techName, - } - configFile.Resolver = project.Repository{ServerId: serverId} - configFile.Deployer = project.Repository{ServerId: serverId} - switch tech { - case coreutils.Maven: - configFile.Resolver.ReleaseRepo = MavenVirtualDefaultName - configFile.Resolver.SnapshotRepo = MavenVirtualDefaultName - configFile.Deployer.ReleaseRepo = MavenVirtualDefaultName - configFile.Deployer.SnapshotRepo = MavenVirtualDefaultName - case coreutils.Dotnet: - fallthrough - case coreutils.Nuget: - configFile.Resolver.NugetV2 = true - fallthrough - default: - configFile.Resolver.Repo = RepoDefaultName[tech][Virtual] - configFile.Deployer.Repo = RepoDefaultName[tech][Virtual] - - } - resBytes, err := yaml.Marshal(&configFile) - if err != nil { - return errorutils.CheckError(err) - } - - return errorutils.CheckError(os.WriteFile(configFilePath, resBytes, 0644)) -} - -func (pic *ProjectInitCommand) CommandName() string { - return "project_init" -} - -func (pic *ProjectInitCommand) ServerDetails() (*config.ServerDetails, error) { - return config.GetSpecificConfig("", true, false) -} diff --git a/general/project/repoutils.go b/general/project/repoutils.go deleted file mode 100644 index 0d2ad0d07..000000000 --- a/general/project/repoutils.go +++ /dev/null @@ -1,186 +0,0 @@ -package project - -import ( - rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" - "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - - "github.com/jfrog/jfrog-client-go/artifactory" - "github.com/jfrog/jfrog-client-go/artifactory/services" -) - -const ( - Local = "local" - Remote = "remote" - Virtual = "virtual" - RemoteUrl = "url" - - // Defaults Repositories - MavenLocalDefaultName = "default-maven-local" - MavenRemoteDefaultName = "default-maven-remote" - MavenRemoteDefaultUrl = "https://repo.maven.apache.org/maven2" - MavenVirtualDefaultName = "default-maven-virtual" - GradleLocalDefaultName = "default-gradle-local" - GradleRemoteDefaultName = "default-gradle-remote" - GradleRemoteDefaultUrl = "https://repo.maven.apache.org/maven2" - GradleVirtualDefaultName = "default-gradle-virtual" - NpmLocalDefaultName = "default-npm-local" - NpmRemoteDefaultName = "default-npm-remote" - NpmRemoteDefaultUrl = "https://registry.npmjs.org" - NpmVirtualDefaultName = "default-npm-virtual" - GoLocalDefaultName = "default-go-local" - GoRemoteDefaultName = "default-go-remote" - GoRemoteDefaultUrl = "https://gocenter.io/" - GoVirtualDefaultName = "default-go-virtual" - PypiLocalDefaultName = "default-pypi-local" - PypiRemoteDefaultName = "default-pypi-remote" - PypiRemoteDefaultUrl = "https://files.pythonhosted.org" - PypiVirtualDefaultName = "default-pypi-virtual" - NugetLocalDefaultName = "default-nuget-local" - NugetRemoteDefaultName = "default-nuget-remote" - NugetRemoteDefaultUrl = "https://www.nuget.org/" - NugetVirtualDefaultName = "default-nuget-virtual" - DockerLocalDefaultName = "default-docker-local" - DockerRemoteDefaultName = "default-docker-remote" - DockerRemoteDefaultUrl = "https://registry-1.docker.io" - DockerVirtualDefaultName = "default-docker-virtual" -) - -var RepoDefaultName = map[coreutils.Technology]map[string]string{ - coreutils.Maven: { - Local: MavenLocalDefaultName, - Remote: MavenRemoteDefaultName, - RemoteUrl: MavenRemoteDefaultUrl, - Virtual: MavenVirtualDefaultName, - }, - coreutils.Gradle: { - Local: GradleLocalDefaultName, - Remote: GradleRemoteDefaultName, - RemoteUrl: GradleRemoteDefaultUrl, - Virtual: GradleVirtualDefaultName, - }, - coreutils.Npm: { - Local: NpmLocalDefaultName, - Remote: NpmRemoteDefaultName, - RemoteUrl: NpmRemoteDefaultUrl, - Virtual: NpmVirtualDefaultName, - }, - coreutils.Go: { - Local: GoLocalDefaultName, - Remote: GoRemoteDefaultName, - RemoteUrl: GoRemoteDefaultUrl, - Virtual: GoVirtualDefaultName, - }, - coreutils.Pip: { - Local: PypiLocalDefaultName, - Remote: PypiRemoteDefaultName, - RemoteUrl: PypiRemoteDefaultUrl, - Virtual: PypiVirtualDefaultName, - }, - coreutils.Pipenv: { - Local: PypiLocalDefaultName, - Remote: PypiRemoteDefaultName, - RemoteUrl: PypiRemoteDefaultUrl, - Virtual: PypiVirtualDefaultName, - }, - coreutils.Nuget: { - Local: NugetLocalDefaultName, - Remote: NugetRemoteDefaultName, - RemoteUrl: NugetRemoteDefaultUrl, - Virtual: NugetVirtualDefaultName, - }, - coreutils.Dotnet: { - Local: NugetLocalDefaultName, - Remote: NugetRemoteDefaultName, - RemoteUrl: NugetRemoteDefaultUrl, - Virtual: NugetVirtualDefaultName, - }, - coreutils.Docker: { - Local: DockerLocalDefaultName, - Remote: DockerRemoteDefaultName, - RemoteUrl: DockerRemoteDefaultUrl, - Virtual: DockerVirtualDefaultName, - }, -} - -func CreateDefaultLocalRepo(technologyType coreutils.Technology, serverId string) error { - servicesManager, err := getServiceManager(serverId) - if err != nil { - return err - } - params := services.NewLocalRepositoryBaseParams() - params.PackageType = technologyType.GetPackageType() - params.Key = RepoDefaultName[technologyType][Local] - // Check if default repository already exists - if exists, err := servicesManager.IsRepoExists(params.Key); exists { - return err - } - if err != nil { - return err - } - return servicesManager.CreateLocalRepositoryWithParams(params) -} - -func createDefaultRemoteNugetRepo(serverId string, baseParams services.RemoteRepositoryBaseParams) error { - servicesManager, err := getServiceManager(serverId) - if err != nil { - return err - } - params := services.NewNugetRemoteRepositoryParams() - params.RemoteRepositoryBaseParams = baseParams - params.DownloadContextPath = "api/v2/package" - params.FeedContextPath = "api/v2" - return servicesManager.CreateRemoteRepository().Nuget(params) -} - -func CreateDefaultRemoteRepo(technologyType coreutils.Technology, serverId string) error { - servicesManager, err := getServiceManager(serverId) - if err != nil { - return err - } - params := services.NewRemoteRepositoryBaseParams() - params.PackageType = technologyType.GetPackageType() - params.Key = RepoDefaultName[technologyType][Remote] - params.Url = RepoDefaultName[technologyType][RemoteUrl] - // Check if default repository already exists - if exists, err := servicesManager.IsRepoExists(params.Key); exists { - return err - } - if err != nil { - return err - } - // NuGet specific case, due to required DownloadContextPath param by Artifactory - if technologyType == coreutils.Nuget || technologyType == coreutils.Dotnet { - return createDefaultRemoteNugetRepo(serverId, params) - } - return servicesManager.CreateRemoteRepositoryWithParams(params) -} - -func CreateDefaultVirtualRepo(technologyType coreutils.Technology, serverId string) error { - servicesManager, err := getServiceManager(serverId) - if err != nil { - return err - } - params := services.NewVirtualRepositoryBaseParams() - params.PackageType = technologyType.GetPackageType() - params.Key = RepoDefaultName[technologyType][Virtual] - params.Repositories = []string{RepoDefaultName[technologyType][Local], RepoDefaultName[technologyType][Remote]} - params.DefaultDeploymentRepo = RepoDefaultName[technologyType][Local] - // Check if default repository already exists - if exists, err := servicesManager.IsRepoExists(params.Key); exists { - return err - } - if err != nil { - return err - } - return servicesManager.CreateVirtualRepositoryWithParams(params) -} - -func getServiceManager(serverId string) (artifactory.ArtifactoryServicesManager, error) { - serviceDetails, err := config.GetSpecificConfig(serverId, true, false) - if err != nil { - return nil, err - } - return rtUtils.CreateServiceManager(serviceDetails, -1, 0, false) - -} From c15cde9842c703c8b9ec51d1e695ce9bfb230e36 Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Mon, 3 Jun 2024 18:32:34 +0300 Subject: [PATCH 3/6] Update Go to 1.22.3 (#1191) --- .github/workflows/analysis.yml | 4 ++-- .github/workflows/frogbot-scan-pull-request.yml | 6 ++++++ .github/workflows/frogbot-scan-repository.yml | 6 ++++++ .github/workflows/test.yml | 2 +- go.mod | 2 +- go.sum | 17 +++++++++++++++++ 6 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 86487ff15..858ab05c2 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -16,7 +16,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.20.x + go-version: 1.22.x cache: false - name: Static Code Analysis @@ -35,7 +35,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.20.x + go-version: 1.22.x cache: false - name: Run Gosec Security Scanner diff --git a/.github/workflows/frogbot-scan-pull-request.yml b/.github/workflows/frogbot-scan-pull-request.yml index 998c8c91f..e6fc7ef30 100644 --- a/.github/workflows/frogbot-scan-pull-request.yml +++ b/.github/workflows/frogbot-scan-pull-request.yml @@ -12,6 +12,12 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.22.x + cache: false + - uses: jfrog/frogbot@v2 env: JFROG_CLI_LOG_LEVEL: "DEBUG" diff --git a/.github/workflows/frogbot-scan-repository.yml b/.github/workflows/frogbot-scan-repository.yml index 01b568f67..0921b2459 100644 --- a/.github/workflows/frogbot-scan-repository.yml +++ b/.github/workflows/frogbot-scan-repository.yml @@ -16,6 +16,12 @@ jobs: # The repository scanning will be triggered periodically on the following branches. branch: [ "dev" ] steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.22.x + cache: false + - uses: jfrog/frogbot@v2 env: JFROG_CLI_LOG_LEVEL: "DEBUG" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea9a56db8..5c3853066 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v5 with: - go-version: 1.20.x + go-version: 1.22.x cache: false - name: Install NuGet diff --git a/go.mod b/go.mod index 66709f03e..f5b9ece9b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/jfrog/jfrog-cli-core/v2 -go 1.20 +go 1.22.3 require github.com/c-bata/go-prompt v0.2.5 // Should not be updated to 0.2.6 due to a bug (https://github.com/jfrog/jfrog-cli-core/pull/372) diff --git a/go.sum b/go.sum index a7e03a5e7..bb180023f 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,11 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat6 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -47,19 +50,23 @@ github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/forPelevin/gomoji v1.2.0 h1:9k4WVSSkE1ARO/BWywxgEUBvR/jMnao6EZzrql5nxJ8= github.com/forPelevin/gomoji v1.2.0/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/gocarina/gocsv v0.0.0-20231116093920-b87c2d0e983a h1:RYfmiM0zluBJOiPDJseKLEN4BapJ42uSi9SZBQ2YyiA= @@ -71,6 +78,7 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= @@ -101,9 +109,11 @@ github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= @@ -132,6 +142,7 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -151,6 +162,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -187,6 +199,7 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= +github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -197,8 +210,11 @@ github.com/vbauerster/mpb/v7 v7.5.3/go.mod h1:i+h4QY6lmLvBNK2ah1fSreiw3ajskRlBp9 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= @@ -290,6 +306,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= From fdc77c0dcf9d259f31040441f22ad8464d5fa71f Mon Sep 17 00:00:00 2001 From: Assaf Attias <49212512+attiasas@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:16:42 +0300 Subject: [PATCH 4/6] Move techutils to the jfrog-cli-security module (#1189) --- utils/coreutils/techutils.go | 496 ----------------------------- utils/coreutils/techutils_test.go | 509 ------------------------------ 2 files changed, 1005 deletions(-) delete mode 100644 utils/coreutils/techutils.go delete mode 100644 utils/coreutils/techutils_test.go diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go deleted file mode 100644 index cc7e6ae8b..000000000 --- a/utils/coreutils/techutils.go +++ /dev/null @@ -1,496 +0,0 @@ -package coreutils - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/jfrog/gofrog/datastructures" - "github.com/jfrog/jfrog-client-go/artifactory/services/fspatterns" - "github.com/jfrog/jfrog-client-go/utils/errorutils" - "github.com/jfrog/jfrog-client-go/utils/log" - - "golang.org/x/exp/maps" - "golang.org/x/text/cases" - "golang.org/x/text/language" - - "github.com/jfrog/jfrog-client-go/utils/io/fileutils" -) - -type Technology string - -const ( - Maven Technology = "maven" - Gradle Technology = "gradle" - Npm Technology = "npm" - Pnpm Technology = "pnpm" - Yarn Technology = "yarn" - Go Technology = "go" - Pip Technology = "pip" - Pipenv Technology = "pipenv" - Poetry Technology = "poetry" - Nuget Technology = "nuget" - Dotnet Technology = "dotnet" - Docker Technology = "docker" - Oci Technology = "oci" -) - -const Pypi = "pypi" - -type TechData struct { - // The name of the package type used in this technology. - packageType string - // Suffixes of file/directory names that indicate if a project uses this technology. - // The name of at least one of the files/directories in the project's directory must end with one of these suffixes. - indicators []string - // Suffixes of file/directory names that indicate if a project does not use this technology. - // The names of all the files/directories in the project's directory must NOT end with any of these suffixes. - exclude []string - // Whether this technology is supported by the 'jf ci-setup' command. - ciSetupSupport bool - // Whether Contextual Analysis supported in this technology. - applicabilityScannable bool - // The files that handle the project's dependencies. - packageDescriptors []string - // Formal name of the technology - formal string - // The executable name of the technology - execCommand string - // The operator for package versioning - packageVersionOperator string - // The package installation command of a package - packageInstallationCommand string -} - -var technologiesData = map[Technology]TechData{ - Maven: { - indicators: []string{"pom.xml"}, - ciSetupSupport: true, - packageDescriptors: []string{"pom.xml"}, - execCommand: "mvn", - applicabilityScannable: true, - }, - Gradle: { - indicators: []string{"build.gradle", "build.gradle.kts"}, - ciSetupSupport: true, - packageDescriptors: []string{"build.gradle", "build.gradle.kts"}, - applicabilityScannable: true, - }, - Npm: { - indicators: []string{"package.json", "package-lock.json", "npm-shrinkwrap.json"}, - exclude: []string{"pnpm-lock.yaml", ".yarnrc.yml", "yarn.lock", ".yarn"}, - ciSetupSupport: true, - packageDescriptors: []string{"package.json"}, - formal: string(Npm), - packageVersionOperator: "@", - packageInstallationCommand: "install", - applicabilityScannable: true, - }, - Pnpm: { - indicators: []string{"pnpm-lock.yaml"}, - exclude: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, - packageDescriptors: []string{"package.json"}, - packageVersionOperator: "@", - packageInstallationCommand: "update", - applicabilityScannable: true, - }, - Yarn: { - indicators: []string{".yarnrc.yml", "yarn.lock", ".yarn", ".yarnrc"}, - exclude: []string{"pnpm-lock.yaml"}, - packageDescriptors: []string{"package.json"}, - packageVersionOperator: "@", - applicabilityScannable: true, - }, - Go: { - indicators: []string{"go.mod"}, - packageDescriptors: []string{"go.mod"}, - packageVersionOperator: "@v", - packageInstallationCommand: "get", - }, - Pip: { - packageType: Pypi, - indicators: []string{"setup.py", "requirements.txt"}, - packageDescriptors: []string{"setup.py", "requirements.txt"}, - exclude: []string{"Pipfile", "Pipfile.lock", "pyproject.toml", "poetry.lock"}, - applicabilityScannable: true, - }, - Pipenv: { - packageType: Pypi, - indicators: []string{"Pipfile", "Pipfile.lock"}, - packageDescriptors: []string{"Pipfile"}, - packageVersionOperator: "==", - packageInstallationCommand: "install", - applicabilityScannable: true, - }, - Poetry: { - packageType: Pypi, - indicators: []string{"pyproject.toml", "poetry.lock"}, - packageDescriptors: []string{"pyproject.toml"}, - packageInstallationCommand: "add", - packageVersionOperator: "==", - applicabilityScannable: true, - }, - Nuget: { - indicators: []string{".sln", ".csproj"}, - packageDescriptors: []string{".sln", ".csproj"}, - formal: "NuGet", - // .NET CLI is used for NuGet projects - execCommand: "dotnet", - packageInstallationCommand: "add", - // packageName -v packageVersion - packageVersionOperator: " -v ", - }, - Dotnet: { - indicators: []string{".sln", ".csproj"}, - packageDescriptors: []string{".sln", ".csproj"}, - formal: ".NET", - }, - Docker: { - applicabilityScannable: true, - }, - Oci: { - applicabilityScannable: true, - }, -} - -func (tech Technology) ToFormal() string { - if technologiesData[tech].formal == "" { - return cases.Title(language.Und).String(tech.String()) - } - return technologiesData[tech].formal -} - -func (tech Technology) String() string { - return string(tech) -} - -func (tech Technology) GetExecCommandName() string { - if technologiesData[tech].execCommand == "" { - return tech.String() - } - return technologiesData[tech].execCommand -} - -func (tech Technology) GetPackageType() string { - if technologiesData[tech].packageType == "" { - return tech.String() - } - return technologiesData[tech].packageType -} - -func (tech Technology) GetPackageDescriptor() []string { - return technologiesData[tech].packageDescriptors -} - -func (tech Technology) IsCiSetup() bool { - return technologiesData[tech].ciSetupSupport -} - -func (tech Technology) GetPackageVersionOperator() string { - return technologiesData[tech].packageVersionOperator -} - -func (tech Technology) GetPackageInstallationCommand() string { - return technologiesData[tech].packageInstallationCommand -} - -func (tech Technology) ApplicabilityScannable() bool { - return technologiesData[tech].applicabilityScannable -} - -func DetectedTechnologiesList() (technologies []string) { - wd, err := os.Getwd() - if errorutils.CheckError(err) != nil { - return - } - return detectedTechnologiesListInPath(wd, false) -} - -func detectedTechnologiesListInPath(path string, recursive bool) (technologies []string) { - detectedTechnologies, err := DetectTechnologies(path, false, recursive) - if err != nil { - return - } - if len(detectedTechnologies) == 0 { - return - } - techStringsList := DetectedTechnologiesToSlice(detectedTechnologies) - log.Info(fmt.Sprintf("Detected: %s.", strings.Join(techStringsList, ", "))) - return techStringsList -} - -// If recursive is true, the search will not be limited to files in the root path. -// If requestedTechs is empty, all technologies will be checked. -// If excludePathPattern is not empty, files/directories that match the wildcard pattern will be excluded from the search. -func DetectTechnologiesDescriptors(path string, recursive bool, requestedTechs []string, requestedDescriptors map[Technology][]string, excludePathPattern string) (technologiesDetected map[Technology]map[string][]string, err error) { - filesList, err := fspatterns.ListFiles(path, recursive, false, true, true, excludePathPattern) - if err != nil { - return - } - workingDirectoryToIndicators, excludedTechAtWorkingDir := mapFilesToRelevantWorkingDirectories(filesList, requestedDescriptors) - var strJson string - if strJson, err = GetJsonIndent(workingDirectoryToIndicators); err != nil { - return - } else if len(workingDirectoryToIndicators) > 0 { - log.Debug(fmt.Sprintf("mapped %d working directories with indicators/descriptors:\n%s", len(workingDirectoryToIndicators), strJson)) - } - technologiesDetected = mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators, excludedTechAtWorkingDir, ToTechnologies(requestedTechs), requestedDescriptors) - if len(technologiesDetected) > 0 { - log.Debug(fmt.Sprintf("Detected %d technologies at %s: %s.", len(technologiesDetected), path, maps.Keys(technologiesDetected))) - } - return -} - -// Map files to relevant working directories according to the technologies' indicators/descriptors and requested descriptors. -// files: The file paths to map. -// requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology. -// Returns: -// 1. workingDirectoryToIndicators: A map of working directories to the files that are relevant to the technologies. -// wd1: [wd1/indicator, wd1/descriptor] -// wd/wd2: [wd/wd2/indicator] -// 2. excludedTechAtWorkingDir: A map of working directories to the technologies that are excluded from the working directory. -// wd1: [tech1, tech2] -// wd/wd2: [tech1] -func mapFilesToRelevantWorkingDirectories(files []string, requestedDescriptors map[Technology][]string) (workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology) { - workingDirectoryToIndicatorsSet := make(map[string]*datastructures.Set[string]) - excludedTechAtWorkingDir = make(map[string][]Technology) - for _, path := range files { - directory := filepath.Dir(path) - - for tech, techData := range technologiesData { - // Check if the working directory contains indicators/descriptors for the technology - relevant := isIndicator(path, techData) || isDescriptor(path, techData) || isRequestedDescriptor(path, requestedDescriptors[tech]) - if relevant { - if _, exist := workingDirectoryToIndicatorsSet[directory]; !exist { - workingDirectoryToIndicatorsSet[directory] = datastructures.MakeSet[string]() - } - workingDirectoryToIndicatorsSet[directory].Add(path) - } - // Check if the working directory contains a file/directory with a name that ends with an excluded suffix - if isExclude(path, techData) { - excludedTechAtWorkingDir[directory] = append(excludedTechAtWorkingDir[directory], tech) - } - } - } - workingDirectoryToIndicators = make(map[string][]string) - for wd, indicators := range workingDirectoryToIndicatorsSet { - workingDirectoryToIndicators[wd] = indicators.ToSlice() - } - return -} - -func isDescriptor(path string, techData TechData) bool { - for _, descriptor := range techData.packageDescriptors { - if strings.HasSuffix(path, descriptor) { - return true - } - } - return false -} - -func isRequestedDescriptor(path string, requestedDescriptors []string) bool { - for _, requestedDescriptor := range requestedDescriptors { - if strings.HasSuffix(path, requestedDescriptor) { - return true - } - } - return false -} - -func isIndicator(path string, techData TechData) bool { - for _, indicator := range techData.indicators { - if strings.HasSuffix(path, indicator) { - return true - } - } - return false -} - -func isExclude(path string, techData TechData) bool { - for _, exclude := range techData.exclude { - if strings.HasSuffix(path, exclude) { - return true - } - } - return false -} - -// Map working directories to technologies according to the given workingDirectoryToIndicators map files. -// workingDirectoryToIndicators: A map of working directories to the files inside the directory that are relevant to the technologies. -// excludedTechAtWorkingDir: A map of working directories to the technologies that are excluded from the working directory. -// requestedTechs: The technologies to check, if empty all technologies will be checked. -// requestedDescriptors: Special requested descriptors (for example in Pip requirement.txt can have different path) for each technology to detect. -func mapWorkingDirectoriesToTechnologies(workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedTechs []Technology, requestedDescriptors map[Technology][]string) (technologiesDetected map[Technology]map[string][]string) { - // Get the relevant technologies to check - technologies := requestedTechs - if len(technologies) == 0 { - technologies = GetAllTechnologiesList() - } - technologiesDetected = make(map[Technology]map[string][]string) - // Map working directories to technologies - for _, tech := range technologies { - techWorkingDirs := getTechInformationFromWorkingDir(tech, workingDirectoryToIndicators, excludedTechAtWorkingDir, requestedDescriptors) - if len(techWorkingDirs) > 0 { - // Found indicators of the technology, add to detected. - technologiesDetected[tech] = techWorkingDirs - } - } - for _, tech := range requestedTechs { - if _, exist := technologiesDetected[tech]; !exist { - // Requested (forced with flag) technology and not found any indicators/descriptors in detection, add as detected. - log.Warn(fmt.Sprintf("Requested technology %s but not found any indicators/descriptors in detection.", tech)) - technologiesDetected[tech] = map[string][]string{} - } - } - return -} - -func getTechInformationFromWorkingDir(tech Technology, workingDirectoryToIndicators map[string][]string, excludedTechAtWorkingDir map[string][]Technology, requestedDescriptors map[Technology][]string) (techWorkingDirs map[string][]string) { - techWorkingDirs = make(map[string][]string) - for wd, indicators := range workingDirectoryToIndicators { - descriptorsAtWd := []string{} - foundIndicator := false - if isTechExcludedInWorkingDir(tech, wd, excludedTechAtWorkingDir) { - // Exclude this technology from this working directory - continue - } - // Check if the working directory contains indicators/descriptors for the technology - for _, path := range indicators { - if isDescriptor(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { - descriptorsAtWd = append(descriptorsAtWd, path) - } - if isIndicator(path, technologiesData[tech]) || isRequestedDescriptor(path, requestedDescriptors[tech]) { - foundIndicator = true - } - } - if foundIndicator { - // Found indicators of the technology in the current working directory, add to detected. - techWorkingDirs[wd] = descriptorsAtWd - } - } - // Don't allow working directory if sub directory already exists as key for the same technology - techWorkingDirs = cleanSubDirectories(techWorkingDirs) - return -} - -func isTechExcludedInWorkingDir(tech Technology, wd string, excludedTechAtWorkingDir map[string][]Technology) bool { - if excludedTechs, exist := excludedTechAtWorkingDir[wd]; exist { - for _, excludedTech := range excludedTechs { - if excludedTech == tech { - return true - } - } - } - return false -} - -// Remove sub directories keys from the given workingDirectoryToFiles map. -// Keys: [dir/dir, dir/directory] -> [dir/dir, dir/directory] -// Keys: [dir, directory] -> [dir, directory] -// Keys: [dir/dir2, dir/dir2/dir3, dir/dir2/dir3/dir4] -> [dir/dir2] -// Values of removed sub directories will be added to the root directory. -func cleanSubDirectories(workingDirectoryToFiles map[string][]string) (result map[string][]string) { - result = make(map[string][]string) - for wd, files := range workingDirectoryToFiles { - root := getExistingRootDir(wd, workingDirectoryToFiles) - result[root] = append(result[root], files...) - } - return -} - -// Get the root directory of the given path according to the given workingDirectoryToIndicators map. -func getExistingRootDir(path string, workingDirectoryToIndicators map[string][]string) (root string) { - root = path - for wd := range workingDirectoryToIndicators { - parentWd := filepath.Dir(wd) - parentRoot := filepath.Dir(root) - if parentRoot != parentWd && strings.HasPrefix(root, wd) { - root = wd - } - } - return -} - -// DetectTechnologies tries to detect all technologies types according to the files in the given path. -// 'isCiSetup' will limit the search of possible techs to Maven, Gradle, and npm. -// 'recursive' will determine if the search will be limited to files in the root path or not. -func DetectTechnologies(path string, isCiSetup, recursive bool) (map[Technology]bool, error) { - var filesList []string - var err error - if recursive { - filesList, err = fileutils.ListFilesRecursiveWalkIntoDirSymlink(path, false) - } else { - filesList, err = fileutils.ListFiles(path, true) - } - if err != nil { - return nil, err - } - log.Info(fmt.Sprintf("Scanning %d file(s):%s", len(filesList), filesList)) - detectedTechnologies := detectTechnologiesByFilePaths(filesList, isCiSetup) - return detectedTechnologies, nil -} - -func detectTechnologiesByFilePaths(paths []string, isCiSetup bool) (detected map[Technology]bool) { - detected = make(map[Technology]bool) - exclude := make(map[Technology]bool) - for _, path := range paths { - for techName, techData := range technologiesData { - // If the detection is in a 'jf ci-setup' command, then the checked technology must be supported. - if !isCiSetup || (isCiSetup && techData.ciSetupSupport) { - // If the project contains a file/directory with a name that ends with an excluded suffix, then this technology is excluded. - for _, excludeFile := range techData.exclude { - if strings.HasSuffix(path, excludeFile) { - exclude[techName] = true - } - } - // If this technology was already excluded, there's no need to look for indicator files/directories. - if _, exist := exclude[techName]; !exist { - // If the project contains a file/directory with a name that ends with the indicator suffix, then the project probably uses this technology. - for _, indicator := range techData.indicators { - if strings.HasSuffix(path, indicator) { - detected[techName] = true - } - } - } - } - } - } - // Remove excluded technologies. - for excludeTech := range exclude { - delete(detected, excludeTech) - } - return detected -} - -// DetectedTechnologiesToSlice returns a string slice that includes all the names of the detected technologies. -func DetectedTechnologiesToSlice(detected map[Technology]bool) []string { - keys := make([]string, 0, len(detected)) - for tech := range detected { - keys = append(keys, string(tech)) - } - return keys -} - -func ToTechnologies(args []string) (technologies []Technology) { - for _, argument := range args { - technologies = append(technologies, Technology(argument)) - } - return -} - -func GetAllTechnologiesList() (technologies []Technology) { - for tech := range technologiesData { - technologies = append(technologies, tech) - } - return -} - -func ContainsApplicabilityScannableTech(technologies []Technology) bool { - for _, technology := range technologies { - if technology.ApplicabilityScannable() { - return true - } - } - return false -} diff --git a/utils/coreutils/techutils_test.go b/utils/coreutils/techutils_test.go deleted file mode 100644 index c69d58ebe..000000000 --- a/utils/coreutils/techutils_test.go +++ /dev/null @@ -1,509 +0,0 @@ -package coreutils - -import ( - "path/filepath" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "golang.org/x/exp/maps" -) - -func TestDetectTechnologiesByFilePaths(t *testing.T) { - tests := []struct { - name string - paths []string - expected map[Technology]bool - }{ - {"simpleMavenTest", []string{"pom.xml"}, map[Technology]bool{Maven: true}}, - {"npmTest", []string{"../package.json"}, map[Technology]bool{Npm: true}}, - {"pnpmTest", []string{"../package.json", "pnpm-lock.yaml"}, map[Technology]bool{Pnpm: true}}, - {"yarnTest", []string{"./package.json", "./.yarn"}, map[Technology]bool{Yarn: true}}, - {"windowsGradleTest", []string{"c:\\users\\test\\package\\build.gradle"}, map[Technology]bool{Gradle: true}}, - {"windowsPipTest", []string{"c:\\users\\test\\package\\setup.py"}, map[Technology]bool{Pip: true}}, - {"windowsPipenvTest", []string{"c:\\users\\test\\package\\Pipfile"}, map[Technology]bool{Pipenv: true}}, - {"golangTest", []string{"/Users/eco/dev/jfrog-cli-core/go.mod"}, map[Technology]bool{Go: true}}, - {"windowsNugetTest", []string{"c:\\users\\test\\package\\project.sln"}, map[Technology]bool{Nuget: true, Dotnet: true}}, - {"noTechTest", []string{"pomxml"}, map[Technology]bool{}}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - detectedTech := detectTechnologiesByFilePaths(test.paths, false) - assert.True(t, reflect.DeepEqual(test.expected, detectedTech), "expected: %s, actual: %s", test.expected, detectedTech) - }) - } -} - -func TestMapFilesToRelevantWorkingDirectories(t *testing.T) { - noRequest := map[Technology][]string{} - noExclude := map[string][]Technology{} - - tests := []struct { - name string - paths []string - requestedDescriptors map[Technology][]string - expectedWorkingDir map[string][]string - expectedExcluded map[string][]Technology - }{ - { - name: "noTechTest", - paths: []string{"pomxml", filepath.Join("sub1", "file"), filepath.Join("sub", "sub", "file")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{}, - expectedExcluded: noExclude, - }, - { - name: "mavenTest", - paths: []string{"pom.xml", filepath.Join("sub1", "pom.xml"), filepath.Join("sub2", "pom.xml")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{ - ".": {"pom.xml"}, - "sub1": {filepath.Join("sub1", "pom.xml")}, - "sub2": {filepath.Join("sub2", "pom.xml")}, - }, - expectedExcluded: noExclude, - }, - { - name: "npmTest", - paths: []string{filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json"), filepath.Join("dir2", "npm-shrinkwrap.json")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{ - "dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json")}, - "dir2": {filepath.Join("dir2", "npm-shrinkwrap.json")}, - }, - expectedExcluded: noExclude, - }, - { - name: "pnpmTest", - paths: []string{filepath.Join("dir", "package.json"), filepath.Join("dir", "pnpm-lock.yaml")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{"dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", "pnpm-lock.yaml")}}, - expectedExcluded: map[string][]Technology{"dir": {Npm, Yarn}}, - }, - { - name: "yarnTest", - paths: []string{filepath.Join("dir", "package.json"), filepath.Join("dir", ".yarn")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{"dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", ".yarn")}}, - expectedExcluded: map[string][]Technology{"dir": {Npm, Pnpm}}, - }, - { - name: "golangTest", - paths: []string{filepath.Join("dir", "dir2", "go.mod")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}}, - expectedExcluded: noExclude, - }, - { - name: "pipTest", - paths: []string{ - filepath.Join("users_dir", "test", "package", "setup.py"), - filepath.Join("users_dir", "test", "package", "blabla.txt"), - filepath.Join("users_dir", "test", "package2", "requirements.txt"), - }, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{ - filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, - filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}}, - expectedExcluded: noExclude, - }, - { - name: "pipRequestedDescriptorTest", - paths: []string{filepath.Join("dir", "blabla.txt"), filepath.Join("dir", "somefile")}, - requestedDescriptors: map[Technology][]string{Pip: {"blabla.txt"}}, - expectedWorkingDir: map[string][]string{"dir": {filepath.Join("dir", "blabla.txt")}}, - expectedExcluded: noExclude, - }, - { - name: "pipenvTest", - paths: []string{filepath.Join("users", "test", "package", "Pipfile")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile")}}, - expectedExcluded: map[string][]Technology{filepath.Join("users", "test", "package"): {Pip}}, - }, - { - name: "gradleTest", - paths: []string{filepath.Join("users", "test", "package", "build.gradle"), filepath.Join("dir", "build.gradle.kts"), filepath.Join("dir", "file")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{ - filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "build.gradle")}, - "dir": {filepath.Join("dir", "build.gradle.kts")}, - }, - expectedExcluded: noExclude, - }, - { - name: "nugetTest", - paths: []string{filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj"), filepath.Join("dir", "file")}, - requestedDescriptors: noRequest, - expectedWorkingDir: map[string][]string{ - "dir": {filepath.Join("dir", "project.sln")}, - filepath.Join("dir", "sub1"): {filepath.Join("dir", "sub1", "project.csproj")}, - }, - expectedExcluded: noExclude, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - detectedWd, detectedExcluded := mapFilesToRelevantWorkingDirectories(test.paths, test.requestedDescriptors) - // Assert working directories - expectedKeys := maps.Keys(test.expectedWorkingDir) - actualKeys := maps.Keys(detectedWd) - assert.ElementsMatch(t, expectedKeys, actualKeys, "expected: %s, actual: %s", expectedKeys, actualKeys) - for key, value := range test.expectedWorkingDir { - assert.ElementsMatch(t, value, detectedWd[key], "expected: %s, actual: %s", value, detectedWd[key]) - } - // Assert excluded - expectedKeys = maps.Keys(test.expectedExcluded) - actualKeys = maps.Keys(detectedExcluded) - assert.ElementsMatch(t, expectedKeys, actualKeys, "expected: %s, actual: %s", expectedKeys, actualKeys) - for key, value := range test.expectedExcluded { - assert.ElementsMatch(t, value, detectedExcluded[key], "expected: %s, actual: %s", value, detectedExcluded[key]) - } - }) - } -} - -func TestMapWorkingDirectoriesToTechnologies(t *testing.T) { - noRequestSpecialDescriptors := map[Technology][]string{} - noRequestTech := []Technology{} - tests := []struct { - name string - workingDirectoryToIndicators map[string][]string - excludedTechAtWorkingDir map[string][]Technology - requestedTechs []Technology - requestedDescriptors map[Technology][]string - - expected map[Technology]map[string][]string - }{ - { - name: "noTechTest", - workingDirectoryToIndicators: map[string][]string{}, - excludedTechAtWorkingDir: map[string][]Technology{}, - requestedTechs: noRequestTech, - requestedDescriptors: noRequestSpecialDescriptors, - expected: map[Technology]map[string][]string{}, - }, - { - name: "all techs test", - workingDirectoryToIndicators: map[string][]string{ - "folder": {filepath.Join("folder", "pom.xml")}, - filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, - filepath.Join("folder", "sub2"): {filepath.Join("folder", "sub2", "pom.xml")}, - "dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json"), filepath.Join("dir", "build.gradle.kts"), filepath.Join("dir", "project.sln")}, - "directory": {filepath.Join("directory", "npm-shrinkwrap.json")}, - "dir3": {filepath.Join("dir3", "package.json"), filepath.Join("dir3", ".yarn")}, - filepath.Join("dir3", "dir"): {filepath.Join("dir3", "dir", "package.json"), filepath.Join("dir3", "dir", "pnpm-lock.yaml")}, - filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}, - filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, - filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, - filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile"), filepath.Join("users", "test", "package", "build.gradle")}, - filepath.Join("dir", "sub1"): {filepath.Join("dir", "sub1", "project.csproj")}, - }, - excludedTechAtWorkingDir: map[string][]Technology{ - filepath.Join("users", "test", "package"): {Pip}, - "dir3": {Npm}, - filepath.Join("dir3", "dir"): {Npm, Yarn}, - }, - requestedTechs: noRequestTech, - requestedDescriptors: noRequestSpecialDescriptors, - expected: map[Technology]map[string][]string{ - Maven: {"folder": {filepath.Join("folder", "pom.xml"), filepath.Join("folder", "sub1", "pom.xml"), filepath.Join("folder", "sub2", "pom.xml")}}, - Npm: { - "dir": {filepath.Join("dir", "package.json")}, - "directory": {}, - }, - Pnpm: {filepath.Join("dir3", "dir"): {filepath.Join("dir3", "dir", "package.json")}}, - Yarn: {"dir3": {filepath.Join("dir3", "package.json")}}, - Go: {filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}}, - Pip: { - filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, - filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, - }, - Pipenv: {filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile")}}, - Gradle: { - "dir": {filepath.Join("dir", "build.gradle.kts")}, - filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "build.gradle")}, - }, - Nuget: {"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, - Dotnet: {"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - detectedTech := mapWorkingDirectoriesToTechnologies(test.workingDirectoryToIndicators, test.excludedTechAtWorkingDir, test.requestedTechs, test.requestedDescriptors) - expectedKeys := maps.Keys(test.expected) - detectedKeys := maps.Keys(detectedTech) - assert.ElementsMatch(t, expectedKeys, detectedKeys, "expected: %s, actual: %s", expectedKeys, detectedKeys) - for key, value := range test.expected { - actualKeys := maps.Keys(detectedTech[key]) - expectedKeys := maps.Keys(value) - assert.ElementsMatch(t, expectedKeys, actualKeys, "for tech %s, expected: %s, actual: %s", key, expectedKeys, actualKeys) - for innerKey, innerValue := range value { - assert.ElementsMatch(t, innerValue, detectedTech[key][innerKey], "expected: %s, actual: %s", innerValue, detectedTech[key][innerKey]) - } - } - }) - } -} - -func TestGetExistingRootDir(t *testing.T) { - tests := []struct { - name string - path string - workingDirectoryToIndicators map[string][]string - expected string - }{ - { - name: "empty", - path: "", - workingDirectoryToIndicators: map[string][]string{}, - expected: "", - }, - { - name: "no match", - path: "dir", - workingDirectoryToIndicators: map[string][]string{ - filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, - "dir2": {filepath.Join("dir2", "go.mod")}, - "dir3": {}, - filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")}, - }, - expected: "dir", - }, - { - name: "match root", - path: filepath.Join("directory", "dir2"), - workingDirectoryToIndicators: map[string][]string{ - filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, - "dir2": {filepath.Join("dir2", "go.mod")}, - "dir3": {}, - filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")}, - }, - expected: filepath.Join("directory", "dir2"), - }, - { - name: "match sub", - path: filepath.Join("directory", "dir2"), - workingDirectoryToIndicators: map[string][]string{ - filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, - "dir2": {filepath.Join("dir2", "go.mod")}, - "directory": {}, - filepath.Join("directory", "dir2"): {filepath.Join("directory", "dir2", "go.mod")}, - }, - expected: "directory", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.expected, getExistingRootDir(test.path, test.workingDirectoryToIndicators)) - }) - } -} - -func TestCleanSubDirectories(t *testing.T) { - tests := []struct { - name string - workingDirectoryToFiles map[string][]string - expected map[string][]string - }{ - { - name: "empty", - workingDirectoryToFiles: map[string][]string{}, - expected: map[string][]string{}, - }, - { - name: "no sub directories", - workingDirectoryToFiles: map[string][]string{ - "directory": {filepath.Join("directory", "file")}, - filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")}, - filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")}, - }, - expected: map[string][]string{ - "directory": {filepath.Join("directory", "file")}, - filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")}, - filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")}, - }, - }, - { - name: "sub directories", - workingDirectoryToFiles: map[string][]string{ - filepath.Join("dir", "dir"): {filepath.Join("dir", "dir", "file")}, - filepath.Join("dir", "directory"): {filepath.Join("dir", "directory", "file")}, - "dir": {filepath.Join("dir", "file")}, - "directory": {filepath.Join("directory", "file")}, - filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "file")}, - filepath.Join("dir", "dir2", "dir3"): {filepath.Join("dir", "dir2", "dir3", "file")}, - filepath.Join("dir", "dir2", "dir3", "dir4"): {filepath.Join("dir", "dir2", "dir3", "dir4", "file")}, - }, - expected: map[string][]string{ - "directory": {filepath.Join("directory", "file")}, - "dir": { - filepath.Join("dir", "file"), - filepath.Join("dir", "dir", "file"), - filepath.Join("dir", "directory", "file"), - filepath.Join("dir", "dir2", "file"), - filepath.Join("dir", "dir2", "dir3", "file"), - filepath.Join("dir", "dir2", "dir3", "dir4", "file"), - }, - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cleaned := cleanSubDirectories(test.workingDirectoryToFiles) - cleanedKeys := maps.Keys(cleaned) - expectedKeys := maps.Keys(test.expected) - assert.ElementsMatch(t, expectedKeys, cleanedKeys, "expected: %s, actual: %s", expectedKeys, cleanedKeys) - for key, value := range test.expected { - assert.ElementsMatch(t, value, cleaned[key], "expected: %s, actual: %s", value, cleaned[key]) - } - }) - } -} - -func TestGetTechInformationFromWorkingDir(t *testing.T) { - workingDirectoryToIndicators := map[string][]string{ - "folder": {filepath.Join("folder", "pom.xml")}, - filepath.Join("folder", "sub1"): {filepath.Join("folder", "sub1", "pom.xml")}, - filepath.Join("folder", "sub2"): {filepath.Join("folder", "sub2", "pom.xml")}, - "dir": {filepath.Join("dir", "package.json"), filepath.Join("dir", "package-lock.json"), filepath.Join("dir", "build.gradle.kts"), filepath.Join("dir", "project.sln"), filepath.Join("dir", "blabla.txt")}, - "directory": {filepath.Join("directory", "npm-shrinkwrap.json")}, - "dir3": {filepath.Join("dir3", "package.json"), filepath.Join("dir3", ".yarn")}, - filepath.Join("dir3", "dir"): {filepath.Join("dir3", "dir", "package.json"), filepath.Join("dir3", "dir", "pnpm-lock.yaml")}, - filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}, - filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, - filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, - filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile"), filepath.Join("users", "test", "package", "build.gradle")}, - filepath.Join("dir", "sub1"): {filepath.Join("dir", "sub1", "project.csproj")}, - } - excludedTechAtWorkingDir := map[string][]Technology{ - filepath.Join("users", "test", "package"): {Pip}, - "dir3": {Npm, Pnpm}, - filepath.Join("dir3", "dir"): {Npm, Yarn}, - } - - tests := []struct { - name string - tech Technology - requestedDescriptors map[Technology][]string - expected map[string][]string - }{ - { - name: "mavenTest", - tech: Maven, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{ - "folder": { - filepath.Join("folder", "pom.xml"), - filepath.Join("folder", "sub1", "pom.xml"), - filepath.Join("folder", "sub2", "pom.xml"), - }, - }, - }, - { - name: "npmTest", - tech: Npm, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{ - "dir": {filepath.Join("dir", "package.json")}, - "directory": {}, - }, - }, - { - name: "pnpmTest", - tech: Pnpm, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{filepath.Join("dir3", "dir"): {filepath.Join("dir3", "dir", "package.json")}}, - }, - { - name: "yarnTest", - tech: Yarn, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{"dir3": {filepath.Join("dir3", "package.json")}}, - }, - { - name: "golangTest", - tech: Go, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{filepath.Join("dir", "dir2"): {filepath.Join("dir", "dir2", "go.mod")}}, - }, - { - name: "pipTest", - tech: Pip, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{ - filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, - filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, - }, - }, - { - name: "pipRequestedDescriptorTest", - tech: Pip, - requestedDescriptors: map[Technology][]string{Pip: {"blabla.txt"}}, - expected: map[string][]string{ - "dir": {filepath.Join("dir", "blabla.txt")}, - filepath.Join("users_dir", "test", "package"): {filepath.Join("users_dir", "test", "package", "setup.py")}, - filepath.Join("users_dir", "test", "package2"): {filepath.Join("users_dir", "test", "package2", "requirements.txt")}, - }, - }, - { - name: "pipenvTest", - tech: Pipenv, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "Pipfile")}}, - }, - { - name: "gradleTest", - tech: Gradle, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{ - filepath.Join("users", "test", "package"): {filepath.Join("users", "test", "package", "build.gradle")}, - "dir": {filepath.Join("dir", "build.gradle.kts")}, - }, - }, - { - name: "nugetTest", - tech: Nuget, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, - }, - { - name: "dotnetTest", - tech: Dotnet, - requestedDescriptors: map[Technology][]string{}, - expected: map[string][]string{"dir": {filepath.Join("dir", "project.sln"), filepath.Join("dir", "sub1", "project.csproj")}}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - techInformation := getTechInformationFromWorkingDir(test.tech, workingDirectoryToIndicators, excludedTechAtWorkingDir, test.requestedDescriptors) - expectedKeys := maps.Keys(test.expected) - actualKeys := maps.Keys(techInformation) - assert.ElementsMatch(t, expectedKeys, actualKeys, "expected: %s, actual: %s", expectedKeys, actualKeys) - for key, value := range test.expected { - assert.ElementsMatch(t, value, techInformation[key], "expected: %s, actual: %s", value, techInformation[key]) - } - }) - } -} - -func TestContainsApplicabilityScannableTech(t *testing.T) { - tests := []struct { - name string - technologies []Technology - want bool - }{ - {name: "contains supported and unsupported techs", technologies: []Technology{Nuget, Go, Npm}, want: true}, - {name: "contains supported techs only", technologies: []Technology{Maven, Yarn, Npm}, want: true}, - {name: "contains unsupported techs only", technologies: []Technology{Dotnet, Nuget, Go}, want: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, ContainsApplicabilityScannableTech(tt.technologies)) - }) - } -} From f80a1af64bfeeb2f8d6ca50261044de8a39c3a76 Mon Sep 17 00:00:00 2001 From: Asaf Ambar Date: Sun, 9 Jun 2024 22:22:15 +0300 Subject: [PATCH 5/6] Utils enhancements to allow go support for JFrog Curation (#1187) --- artifactory/commands/golang/go.go | 22 +++++----- artifactory/commands/golang/go_test.go | 12 +++++- go.mod | 2 +- go.sum | 4 +- utils/golang/utils.go | 42 +++++++++++-------- utils/golang/utils_test.go | 56 ++++++++++++++++++++++++-- 6 files changed, 103 insertions(+), 35 deletions(-) diff --git a/artifactory/commands/golang/go.go b/artifactory/commands/golang/go.go index b6950c50a..4847855a5 100644 --- a/artifactory/commands/golang/go.go +++ b/artifactory/commands/golang/go.go @@ -153,14 +153,12 @@ func (gc *GoCommand) run() (err error) { if err != nil { return } - repoUrl, err := goutils.GetArtifactoryRemoteRepoUrl(resolverDetails, gc.resolverParams.TargetRepo()) + // If noFallback=false, missing packages will be fetched directly from VCS + repoUrl, err := goutils.GetArtifactoryRemoteRepoUrl(resolverDetails, gc.resolverParams.TargetRepo(), goutils.GoProxyUrlParams{Direct: !gc.noFallback}) if err != nil { return } - // If noFallback=false, missing packages will be fetched directly from VCS - if !gc.noFallback { - repoUrl += "|direct" - } + err = biutils.RunGo(gc.goArg, repoUrl) if errorutils.CheckError(err) != nil { err = coreutils.ConvertExitCodeError(err) @@ -330,19 +328,21 @@ func buildPackageVersionRequest(name, branchName string) string { return path.Join(packageVersionRequest, "latest.info") } -func SetArtifactoryAsResolutionServer(serverDetails *config.ServerDetails, depsRepo string) (err error) { - err = setGoProxy(serverDetails, depsRepo) - if err != nil { +func SetArtifactoryAsResolutionServer(serverDetails *config.ServerDetails, depsRepo string, goProxyParams goutils.GoProxyUrlParams) (err error) { + if err = setGoProxy(serverDetails, depsRepo, goProxyParams); err != nil { err = fmt.Errorf("failed while setting Artifactory as a dependencies resolution registry: %s", err.Error()) } return } -func setGoProxy(server *config.ServerDetails, remoteGoRepo string) error { - repoUrl, err := goutils.GetArtifactoryRemoteRepoUrl(server, remoteGoRepo) +func setGoProxy(server *config.ServerDetails, remoteGoRepo string, goProxyParams goutils.GoProxyUrlParams) error { + repoUrl, err := goutils.GetArtifactoryRemoteRepoUrl(server, remoteGoRepo, goProxyParams) if err != nil { return err } - repoUrl += "|direct" return os.Setenv("GOPROXY", repoUrl) } + +func SetGoModCache(cacheFolder string) error { + return os.Setenv("GOMODCACHE", cacheFolder) +} diff --git a/artifactory/commands/golang/go_test.go b/artifactory/commands/golang/go_test.go index ed899aad3..6aaf77d26 100644 --- a/artifactory/commands/golang/go_test.go +++ b/artifactory/commands/golang/go_test.go @@ -3,6 +3,7 @@ package golang import ( "fmt" "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" goutils "github.com/jfrog/jfrog-cli-core/v2/utils/golang" testsutils "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/stretchr/testify/assert" @@ -60,9 +61,18 @@ func TestSetArtifactoryAsResolutionServer(t *testing.T) { cleanup := testsutils.SetEnvWithCallbackAndAssert(t, "GOPROXY", "") defer cleanup() - assert.NoError(t, SetArtifactoryAsResolutionServer(server, repo)) + assert.NoError(t, SetArtifactoryAsResolutionServer(server, repo, goutils.GoProxyUrlParams{Direct: true})) serverUrlWithoutHttp := strings.TrimPrefix(server.ArtifactoryUrl, "http://") expectedGoProxy := fmt.Sprintf("http://%s:%s@%sapi/go/%s|direct", server.User, server.Password, serverUrlWithoutHttp, repo) assert.Equal(t, expectedGoProxy, os.Getenv("GOPROXY")) + + // Verify that the EndpointPrefix value is correctly added to the GOPROXY. + // In this test case, the endpoint prefix is set to api/curation/audit/. + // This parameter allows downloading dependencies from a custom API instead of the default one. + assert.NoError(t, SetArtifactoryAsResolutionServer(server, repo, goutils.GoProxyUrlParams{Direct: true, EndpointPrefix: coreutils.CurationPassThroughApi})) + + serverUrlWithoutHttp = strings.TrimPrefix(server.ArtifactoryUrl, "http://") + expectedGoProxy = fmt.Sprintf("http://%s:%s@%sapi/curation/audit/api/go/%s|direct", server.User, server.Password, serverUrlWithoutHttp, repo) + assert.Equal(t, expectedGoProxy, os.Getenv("GOPROXY")) } diff --git a/go.mod b/go.mod index f5b9ece9b..a321d6552 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,6 @@ require ( replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04 -// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go dev +replace github.com/jfrog/build-info-go => github.com/asafambar/build-info-go v1.8.9-0.20240530151827-93c25df23371 // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.3.3-0.20231223133729-ef57bd08cedc diff --git a/go.sum b/go.sum index bb180023f..e468dfe92 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer5 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/asafambar/build-info-go v1.8.9-0.20240530151827-93c25df23371 h1:U7bqNQ40AfeC55m9v9r8vy/Iza4xrUJiJ2DYcE031Oo= +github.com/asafambar/build-info-go v1.8.9-0.20240530151827-93c25df23371/go.mod h1:5TH6zBwOpQxGXgn+WX4OlH4q5P4k1zE1sFksU5VDhbE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= @@ -91,8 +93,6 @@ github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+ github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w= github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= -github.com/jfrog/build-info-go v1.9.27 h1:7RWJcajqtNNbGHuYkgOLUIG7mmRKF0yxC7mvYAbdVlU= -github.com/jfrog/build-info-go v1.9.27/go.mod h1:8T7/ajM9aGshvgpwCtXwIFpyF/R6CEn4W+/FLryNXWw= github.com/jfrog/gofrog v1.7.2 h1:VkAaA/9tmbw27IqgUOmaZWnO6ATUqL3vRzDnsROKATw= github.com/jfrog/gofrog v1.7.2/go.mod h1:WJFk88SR9Sr9mKl1bQBig7DmSdXiBGKV3WhL9O6jL9w= github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04 h1:ERLE/L7YPr6aCUTeAnE8SXU5VOZHd5/XK16rM1TEpts= diff --git a/utils/golang/utils.go b/utils/golang/utils.go index ab4176111..645b23bad 100644 --- a/utils/golang/utils.go +++ b/utils/golang/utils.go @@ -9,6 +9,8 @@ import ( "io" "net/url" "os/exec" + "path" + "strings" ) type GoCmdConfig struct { @@ -83,33 +85,41 @@ func GetModuleName(projectDir string) (string, error) { return path, nil } -func GetDependenciesList(projectDir string) (map[string]bool, error) { - deps, err := utils.GetDependenciesList(projectDir, log.Logger) - if err != nil { - return nil, errorutils.CheckError(err) - } - return deps, nil +type GoProxyUrlParams struct { + // Fallback to retrieve the modules directly from the source if + // the module failed to be retrieved from the proxy. + // add |direct to the end of the url. + // example: https://gocenter.io|direct + Direct bool + // The path from baseUrl to the standard Go repository path + // URL structure: //api/go/ + EndpointPrefix string } -func GetDependenciesGraph(projectDir string) (map[string][]string, error) { - deps, err := utils.GetDependenciesGraph(projectDir, log.Logger) - if err != nil { - return nil, errorutils.CheckError(err) +func (gdu *GoProxyUrlParams) BuildUrl(url *url.URL, repoName string) string { + url.Path = path.Join(url.Path, gdu.EndpointPrefix, "api/go/", repoName) + + return gdu.addDirect(url.String()) +} + +func (gdu *GoProxyUrlParams) addDirect(url string) string { + if gdu.Direct && !strings.HasSuffix(url, "|direct") { + return url + "|direct" } - return deps, nil + return url } -func GetArtifactoryRemoteRepoUrl(serverDetails *config.ServerDetails, repo string) (string, error) { +func GetArtifactoryRemoteRepoUrl(serverDetails *config.ServerDetails, repo string, goProxyParams GoProxyUrlParams) (string, error) { authServerDetails, err := serverDetails.CreateArtAuthConfig() if err != nil { return "", err } - return getArtifactoryApiUrl(repo, authServerDetails) + return getArtifactoryApiUrl(repo, authServerDetails, goProxyParams) } // Gets the URL of the specified repository Go API in Artifactory. // The URL contains credentials (username and access token or password). -func getArtifactoryApiUrl(repoName string, details auth.ServiceDetails) (string, error) { +func getArtifactoryApiUrl(repoName string, details auth.ServiceDetails, goProxyParams GoProxyUrlParams) (string, error) { rtUrl, err := url.Parse(details.GetUrl()) if err != nil { return "", errorutils.CheckError(err) @@ -129,6 +139,6 @@ func getArtifactoryApiUrl(repoName string, details auth.ServiceDetails) (string, if password != "" { rtUrl.User = url.UserPassword(username, password) } - rtUrl.Path += "api/go/" + repoName - return rtUrl.String(), nil + + return goProxyParams.BuildUrl(rtUrl, repoName), nil } diff --git a/utils/golang/utils_test.go b/utils/golang/utils_test.go index c2326b21b..e6ec32041 100644 --- a/utils/golang/utils_test.go +++ b/utils/golang/utils_test.go @@ -4,6 +4,8 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-client-go/artifactory/auth" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "net/url" "testing" ) @@ -13,7 +15,7 @@ func TestGetArtifactoryRemoteRepoUrl(t *testing.T) { AccessToken: "eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA", } repoName := "test-repo" - repoUrl, err := GetArtifactoryRemoteRepoUrl(server, repoName) + repoUrl, err := GetArtifactoryRemoteRepoUrl(server, repoName, GoProxyUrlParams{}) assert.NoError(t, err) assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@server.com/artifactory/api/go/test-repo", repoUrl) } @@ -25,15 +27,22 @@ func TestGetArtifactoryApiUrl(t *testing.T) { // Test username and password details.SetUser("frog") details.SetPassword("passfrog") - url, err := getArtifactoryApiUrl("test-repo", details) + url, err := getArtifactoryApiUrl("test-repo", details, GoProxyUrlParams{}) assert.NoError(t, err) assert.Equal(t, "https://frog:passfrog@test.com/artifactory/api/go/test-repo", url) + // Test username and password with EndpointPrefix and direct + details.SetUser("frog") + details.SetPassword("passfrog") + url, err = getArtifactoryApiUrl("test-repo", details, GoProxyUrlParams{EndpointPrefix: "test", Direct: true}) + assert.NoError(t, err) + assert.Equal(t, "https://frog:passfrog@test.com/artifactory/test/api/go/test-repo|direct", url) + // Test access token // Set fake access token with username "test" details.SetUser("") details.SetAccessToken("eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA") - url, err = getArtifactoryApiUrl("test-repo", details) + url, err = getArtifactoryApiUrl("test-repo", details, GoProxyUrlParams{}) assert.NoError(t, err) assert.Equal(t, "https://test:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@test.com/artifactory/api/go/test-repo", url) @@ -42,7 +51,46 @@ func TestGetArtifactoryApiUrl(t *testing.T) { // Expect username to be "frog" details.SetUser("frog") details.SetAccessToken("eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA") - url, err = getArtifactoryApiUrl("test-repo", details) + url, err = getArtifactoryApiUrl("test-repo", details, GoProxyUrlParams{}) assert.NoError(t, err) assert.Equal(t, "https://frog:eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA@test.com/artifactory/api/go/test-repo", url) } + +func TestGoProxyUrlParams_BuildUrl(t *testing.T) { + tests := []struct { + name string + RepoName string + Direct bool + EndpointPrefix string + ExpectedUrl string + }{ + { + name: "Url Without direct or Prefix", + RepoName: "go", + ExpectedUrl: "https://test/api/go/go", + }, + { + name: "Url With direct", + RepoName: "go", + Direct: true, + ExpectedUrl: "https://test/api/go/go|direct", + }, + { + name: "Url With Prefix", + RepoName: "go", + EndpointPrefix: "prefix", + ExpectedUrl: "https://test/prefix/api/go/go", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + remoteUrl, err := url.Parse("https://test") + require.NoError(t, err) + gdu := &GoProxyUrlParams{ + Direct: tt.Direct, + EndpointPrefix: tt.EndpointPrefix, + } + assert.Equalf(t, tt.ExpectedUrl, gdu.BuildUrl(remoteUrl, tt.RepoName), "BuildUrl(%v, %v)", remoteUrl, tt.RepoName) + }) + } +} From d8879506936bba9f601b8326af4e337508004be8 Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Mon, 10 Jun 2024 11:00:23 +0300 Subject: [PATCH 6/6] Upgrade depedencies (#1192) --- go.mod | 8 ++++---- go.sum | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index a321d6552..9dfff0a10 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,9 @@ require ( github.com/google/uuid v1.6.0 github.com/gookit/color v1.5.4 github.com/jedib0t/go-pretty/v6 v6.5.9 - github.com/jfrog/build-info-go v1.9.27 + github.com/jfrog/build-info-go v1.9.29 github.com/jfrog/gofrog v1.7.2 - github.com/jfrog/jfrog-client-go v1.40.2 + github.com/jfrog/jfrog-client-go v1.41.0 github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c @@ -96,8 +96,8 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04 +// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04 -replace github.com/jfrog/build-info-go => github.com/asafambar/build-info-go v1.8.9-0.20240530151827-93c25df23371 +// replace github.com/jfrog/build-info-go => github.com/asafambar/build-info-go v1.8.9-0.20240530151827-93c25df23371 // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.3.3-0.20231223133729-ef57bd08cedc diff --git a/go.sum b/go.sum index e468dfe92..1cc7cde52 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer5 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/asafambar/build-info-go v1.8.9-0.20240530151827-93c25df23371 h1:U7bqNQ40AfeC55m9v9r8vy/Iza4xrUJiJ2DYcE031Oo= -github.com/asafambar/build-info-go v1.8.9-0.20240530151827-93c25df23371/go.mod h1:5TH6zBwOpQxGXgn+WX4OlH4q5P4k1zE1sFksU5VDhbE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= @@ -93,10 +91,12 @@ github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+ github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E= github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w= github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= +github.com/jfrog/build-info-go v1.9.29 h1:3vJ+kbk9PpU6wjisXi9c4qISNpYkISh/NmB5mq1ZlSY= +github.com/jfrog/build-info-go v1.9.29/go.mod h1:AzFJlN/yKfKuKcSBaGy5nNmKN1xzx6+XcRWAswCTLTA= github.com/jfrog/gofrog v1.7.2 h1:VkAaA/9tmbw27IqgUOmaZWnO6ATUqL3vRzDnsROKATw= github.com/jfrog/gofrog v1.7.2/go.mod h1:WJFk88SR9Sr9mKl1bQBig7DmSdXiBGKV3WhL9O6jL9w= -github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04 h1:ERLE/L7YPr6aCUTeAnE8SXU5VOZHd5/XK16rM1TEpts= -github.com/jfrog/jfrog-client-go v1.28.1-0.20240530101935-539b5837ce04/go.mod h1:37RR4pYgXZM4w7tywyfRu8t2wagt0qf5wBtpDILWBsk= +github.com/jfrog/jfrog-client-go v1.41.0 h1:g5OTFvreOVQ6U/5LUXFJfA3Bc+AZCo2PO/EzCLxLbLE= +github.com/jfrog/jfrog-client-go v1.41.0/go.mod h1:AN+/mT2DIBE4oRZicJojqND2BEKLfA7f73i5rT3Lfcc= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=