Merge pull request #1389 from merico-dev/refactor-scm-config-use-env

feat: use config field instead of env

examples/apps.yaml

@@ -19,13 +19,15 @@ apps:
     framework: django
   repo:
     url: github.com/[[ GITHUB_USER ]]/myapp1
+    token: [[ env GITHUB_TOKEN ]]
   repoTemplate:
     url: github.com/devstream-io/dtm-repo-scaffolding-python-flask
   ci:
   - type: github-actions
     options:
       imageRepo:
         user: [[ DOCKERHUB_USER ]]
+        password: [[ env IMAGE_REPO_PASSWORD ]]
   cd:
   - type: argocdapp
 - name: myapp2
@@ -34,12 +36,14 @@ apps:
     framework: gin
   repo:
     url: github.com/[[ GITHUB_USER ]]/myapp2
+    token: [[ env GITHUB_TOKEN ]]
   repoTemplate:
     url: github.com/devstream-io/dtm-repo-scaffolding-golang-gin
   ci:
   - type: github-actions
     options:
       imageRepo:
         user: [[ DOCKERHUB_USER ]]
+        password: [[ env IMAGE_REPO_PASSWORD ]]
   cd:
   - type: argocdapp

examples/gitops.yaml

@@ -17,6 +17,7 @@ tools:
       name: [[ app ]]
       branch: main
       scmType: github
+      token: [[ env GITHUB_TOKEN ]]
     sourceRepo:
       org: devstream-io
       name: dtm-repo-scaffolding-python-flask
@@ -29,13 +30,15 @@ tools:
       owner: [[ githubUser ]]
       name:  [[ app ]]
       scmType: github
+      token: [[ env GITHUB_TOKEN ]]
     pipeline:
       configLocation: https://raw.githubusercontent.com/devstream-io/dtm-pipeline-templates/main/github-actions/workflows/main.yml
       language:
         name: python
         framework: flask
       imageRepo:
         user: [[ dockerUser ]]
+        password: [[ env IMAGE_REPO_PASSWORD ]]
 - name: helm-installer
   instanceID: argocd
 - name: argocdapp
@@ -52,5 +55,6 @@ tools:
       valuefile: values.yaml
       path: helm/[[ app ]]
       repoURL: ${{repo-scaffolding.myapp.outputs.repoURL}}
+      token: [[ env GITHUB_TOKEN ]]
     imageRepo:
       user: [[ dockerUser ]]

examples/quickstart.yaml

@@ -17,6 +17,7 @@ tools:
       name: go-webapp-devstream-demo
       branch: main
       scmType: github
+      token: [[ env GITHUB_TOKEN ]]
     sourceRepo:
       org: devstream-io
       name: dtm-repo-scaffolding-golang-gin
@@ -30,10 +31,12 @@ tools:
       name: go-webapp-devstream-demo
       branch: main
       scmType: github
+      token: [[ env GITHUB_TOKEN ]]
     pipeline:
       configLocation: https://raw.githubusercontent.com/devstream-io/dtm-pipeline-templates/main/github-actions/workflows/main.yml
       language:
         name: go
         framework: gin
       imageRepo:
         user: [[ ImageRepoUser ]]
+        password: [[ env IMAGE_REPO_PASSWORD ]]

internal/pkg/configmanager/pipelineDefault.go

@@ -84,6 +84,7 @@ func pipelineArgocdAppGenerator(options RawOptions, globalVars *pipelineGlobalOp
 		if _, repoURLExist := sourceMap["repoURL"]; !repoURLExist {
 			sourceMap["repoURL"] = globalVars.Repo.GetCloneURL()
 			sourceMap["repoBranch"] = globalVars.Repo.Branch
+			sourceMap["token"] = globalVars.Repo.Token
 		}
 		options["source"] = sourceMap
 	} else {
@@ -92,6 +93,7 @@ func pipelineArgocdAppGenerator(options RawOptions, globalVars *pipelineGlobalOp
 			"path":       fmt.Sprintf("helm/%s", globalVars.AppName),
 			"repoURL":    string(globalVars.Repo.GetCloneURL()),
 			"repoBranch": globalVars.Repo.Branch,
+			"token":      globalVars.Repo.Token,
 		}
 	}
 	// config imageRepo default options

internal/pkg/configmanager/pipelineDefault_test.go

@@ -90,6 +90,7 @@ var _ = Describe("pipelineArgocdAppGenerator func", func() {
 				"path":       "helm/test_app",
 				"repoURL":    "https://scm.test.com",
 				"repoBranch": "testBranch",
+				"token":      "",
 			},
 			"imageRepo": RawOptions{
 				"owner": "test_user",

internal/pkg/plugin/argocdapp/argocdapp.go

@@ -25,6 +25,7 @@ func pushArgocdConfigFiles(rawOptions configmanager.RawOptions) error {
 	repoInfo := &git.RepoInfo{
 		CloneURL: git.ScmURL(opts.Source.RepoURL),
 		Branch:   opts.Source.RepoBranch,
+		Token:    opts.Source.Token,
 	}
 	scmClient, err := scm.NewClientWithAuth(repoInfo)
 	if err != nil {

internal/pkg/plugin/argocdapp/options.go

@@ -47,6 +47,7 @@ type source struct {
 	Path       string `mapstructure:"path" validate:"required"`
 	RepoURL    string `mapstructure:"repoURL" validate:"required"`
 	RepoBranch string `mapstructure:"repoBranch"`
+	Token      string `mapstructure:"token"`
 }
 
 // / newOptions create options by raw options

internal/pkg/plugin/gitlabci/options_test.go

@@ -54,8 +54,9 @@ var _ = Describe("action struct", func() {
 				"ImageRepoDockerSecret": "image-repo-auth",
 				"RepoType":              "gitlab",
 				"imageRepo": map[string]interface{}{
-					"url":  "exmaple.com",
-					"user": "test_user",
+					"url":      "exmaple.com",
+					"user":     "test_user",
+					"password": "",
 				},
 				"dingTalk":            nilDingTalkConfig,
 				"DingTalkSecretKey":   "DINGTALK_SECURITY_VALUE",

internal/pkg/plugin/installer/ci/ci_test.go

@@ -52,8 +52,9 @@ var _ = Describe("PipelineConfig struct", func() {
 				"ImageRepoSecret":       "IMAGE_REPO_SECRET",
 				"ImageRepoDockerSecret": "image-repo-auth",
 				"imageRepo": map[string]interface{}{
-					"url":  "exmaple.com",
-					"user": "test_user",
+					"url":      "exmaple.com",
+					"user":     "test_user",
+					"password": "",
 				},
 				"dingTalk":            nilDingTalkConfig,
 				"DingTalkSecretKey":   "DINGTALK_SECURITY_VALUE",
@@ -94,8 +95,9 @@ var _ = Describe("PipelineConfig struct", func() {
 			"StepGlobalVars":        "",
 			"RepoType":              "gitlab",
 			"imageRepo": map[string]interface{}{
-				"url":  "exmaple.com",
-				"user": "test_user",
+				"url":      "exmaple.com",
+				"user":     "test_user",
+				"password": "",
 			},
 			"dingTalk":           emptyDingtalk,
 			"sonarqube":          emptySonar,

internal/pkg/plugin/installer/ci/step/github.go

@@ -1,12 +1,9 @@
 package step
 
 import (
-	"os"
-
 	"github.com/devstream-io/devstream/pkg/util/jenkins"
 	"github.com/devstream-io/devstream/pkg/util/log"
 	"github.com/devstream-io/devstream/pkg/util/scm"
-	"github.com/devstream-io/devstream/pkg/util/scm/github"
 )
 
 const (
@@ -15,6 +12,7 @@ const (
 
 type GithubStepConfig struct {
 	RepoOwner string `mapstructure:"repoOwner"`
+	Token     string `mapstructure:"token"`
 }
 
 func (g *GithubStepConfig) GetJenkinsPlugins() []*jenkins.JenkinsPlugin {
@@ -31,7 +29,7 @@ func (g *GithubStepConfig) ConfigJenkins(jenkinsClient jenkins.JenkinsAPI) (*jen
 	err := jenkinsClient.CreatePasswordCredential(
 		githubCredentialName,
 		g.RepoOwner,
-		os.Getenv(github.TokenEnvKey),
+		g.Token,
 	)
 	if err != nil {
 		log.Debugf("jenkins preinstall github credentials failed: %s", err)
@@ -52,5 +50,6 @@ func (g *GithubStepConfig) ConfigSCM(client scm.ClientOperation) error {
 func newGithubStep(config *StepGlobalOption) *GithubStepConfig {
 	return &GithubStepConfig{
 		RepoOwner: config.RepoInfo.GetRepoOwner(),
+		Token:     config.RepoInfo.Token,
 	}
 }

internal/pkg/plugin/installer/ci/step/gitlab.go

@@ -1,8 +1,6 @@
 package step
 
 import (
-	"os"
-
 	"github.com/devstream-io/devstream/pkg/util/jenkins"
 	"github.com/devstream-io/devstream/pkg/util/log"
 	"github.com/devstream-io/devstream/pkg/util/scm"
@@ -18,6 +16,7 @@ type GitlabStepConfig struct {
 	SSHPrivateKey string `mapstructure:"sshPrivateKey"`
 	RepoOwner     string `mapstructure:"repoOwner"`
 	BaseURL       string `mapstructure:"baseURL"`
+	Token         string `mapstructure:"token"`
 }
 
 func (g *GitlabStepConfig) GetJenkinsPlugins() []*jenkins.JenkinsPlugin {
@@ -42,7 +41,7 @@ func (g *GitlabStepConfig) ConfigJenkins(jenkinsClient jenkins.JenkinsAPI) (*jen
 		}
 	}
 	// 2. create gitlab connection by casc
-	err := jenkinsClient.CreateGiltabCredential(gitlabCredentialName, os.Getenv("GITLAB_TOKEN"))
+	err := jenkinsClient.CreateGiltabCredential(gitlabCredentialName, g.Token)
 	if err != nil {
 		log.Debugf("jenkins preinstall gitlab credentials failed: %s", err)
 		return nil, err
@@ -65,5 +64,6 @@ func newGitlabStep(config *StepGlobalOption) *GitlabStepConfig {
 		SSHPrivateKey: config.RepoInfo.SSHPrivateKey,
 		RepoOwner:     config.RepoInfo.GetRepoOwner(),
 		BaseURL:       config.RepoInfo.BaseURL,
+		Token:         config.RepoInfo.Token,
 	}
 }

internal/pkg/plugin/installer/ci/step/imagerepo.go

@@ -3,7 +3,6 @@ package step
 import (
 	"encoding/base64"
 	"fmt"
-	"os"
 
 	"github.com/devstream-io/devstream/pkg/util/jenkins"
 	"github.com/devstream-io/devstream/pkg/util/k8s"
@@ -12,7 +11,6 @@ import (
 )
 
 const (
-	imageRepoPasswordEnv = "IMAGE_REPO_PASSWORD"
 	// imageRepoDockerSecretName is used for creating k8s secret
 	// and it will be used by jenkins for mount
 	imageRepoDockerSecretName = "image-repo-auth"
@@ -22,8 +20,9 @@ const (
 )
 
 type ImageRepoStepConfig struct {
-	URL  string `mapstructure:"url"`
-	User string `mapstructure:"user"`
+	URL      string `mapstructure:"url"`
+	User     string `mapstructure:"user"`
+	Password string `mapstructure:"password"`
 }
 
 func (g *ImageRepoStepConfig) GetJenkinsPlugins() []*jenkins.JenkinsPlugin {
@@ -50,23 +49,21 @@ func (g *ImageRepoStepConfig) ConfigJenkins(jenkinsClient jenkins.JenkinsAPI) (*
 }
 
 func (g *ImageRepoStepConfig) ConfigSCM(client scm.ClientOperation) error {
-	imageRepoPasswd := os.Getenv(imageRepoPasswordEnv)
-	if imageRepoPasswd == "" {
-		return fmt.Errorf("the environment variable IMAGE_REPO_PASSWORD is not set")
+	if g.Password == "" {
+		return fmt.Errorf("config field password is not set")
 	}
 
 	if err := client.AddRepoSecret(imageRepoUserName, g.User); err != nil {
 		return err
 	}
-	return client.AddRepoSecret(imageRepoSecretName, imageRepoPasswd)
+	return client.AddRepoSecret(imageRepoSecretName, g.Password)
 }
 
 func (g *ImageRepoStepConfig) generateDockerAuthSecretData() (map[string][]byte, error) {
-	imageRepoPasswd := os.Getenv(imageRepoPasswordEnv)
-	if imageRepoPasswd == "" {
-		return nil, fmt.Errorf("the environment variable IMAGE_REPO_PASSWORD is not set")
+	if g.Password == "" {
+		return nil, fmt.Errorf("config field password is not set")
 	}
-	tmpStr := fmt.Sprintf("%s:%s", g.User, imageRepoPasswd)
+	tmpStr := fmt.Sprintf("%s:%s", g.User, g.Password)
 	authStr := base64.StdEncoding.EncodeToString([]byte(tmpStr))
 	authURL := g.GetImageRepoURL()
 	log.Debugf("jenkins plugin imageRepo auth string: %s.", authStr)

internal/pkg/plugin/installer/ci/step/imagerepo_test.go

@@ -2,7 +2,6 @@ package step
 
 import (
 	"fmt"
-	"os"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -32,29 +31,26 @@ var _ = Describe("ImageRepoStepConfig", func() {
 	})
 
 	Context("generateDockerAuthSecretData method", func() {
-		var (
-			existImageEnv   string
-			testImageEnvVar string
-		)
-		BeforeEach(func() {
-			existImageEnv = os.Getenv("IMAGE_REPO_PASSWORD")
-			if existImageEnv != "" {
-				err := os.Unsetenv("IMAGE_REPO_PASSWORD")
-				Expect(err).Error().ShouldNot(HaveOccurred())
-			}
-		})
 		When("image env is not set", func() {
+			BeforeEach(func() {
+				c = &ImageRepoStepConfig{
+					URL:  url,
+					User: user,
+				}
+			})
 			It("should return error", func() {
 				_, err := c.generateDockerAuthSecretData()
 				Expect(err).Error().Should(HaveOccurred())
-				Expect(err.Error()).Should(Equal("the environment variable IMAGE_REPO_PASSWORD is not set"))
+				Expect(err.Error()).Should(Equal("config field password is not set"))
 			})
 		})
 		When("image env is set", func() {
 			BeforeEach(func() {
-				testImageEnvVar = "test_image_repo_env"
-				err := os.Setenv("IMAGE_REPO_PASSWORD", testImageEnvVar)
-				Expect(err).Error().ShouldNot(HaveOccurred())
+				c = &ImageRepoStepConfig{
+					URL:      url,
+					User:     user,
+					Password: "test",
+				}
 			})
 			It("should return image auth data", func() {
 				d, err := c.generateDockerAuthSecretData()
@@ -69,48 +65,43 @@ var _ = Describe("ImageRepoStepConfig", func() {
       "auth": "%s"
     }
   }
-}`, c.URL, "dGVzdF91c2VyOnRlc3RfaW1hZ2VfcmVwb19lbnY=")
+}`, c.URL, "dGVzdF91c2VyOnRlc3Q=")
 				Expect(string(configJson)).Should(Equal(expectStr))
 			})
 		})
 	})
 
 	Context("ConfigSCM method", func() {
 		var (
-			scmClient             *scm.MockScmClient
-			errMsg, existImageEnv string
+			scmClient *scm.MockScmClient
 		)
-		BeforeEach(func() {
-			existImageEnv = os.Getenv("IMAGE_REPO_PASSWORD")
-			if existImageEnv != "" {
-				err := os.Unsetenv("IMAGE_REPO_PASSWORD")
-				Expect(err).Error().ShouldNot(HaveOccurred())
-			}
-		})
 		When("imageRepoPassword is not valid", func() {
 			BeforeEach(func() {
-				errMsg = "the environment variable IMAGE_REPO_PASSWORD is not set"
+				c = &ImageRepoStepConfig{
+					URL:  url,
+					User: user,
+				}
 				scmClient = &scm.MockScmClient{}
 			})
 			It("should return error", func() {
 				err := c.ConfigSCM(scmClient)
 				Expect(err).Error().Should(HaveOccurred())
-				Expect(err.Error()).Should(Equal(errMsg))
+				Expect(err.Error()).Should(Equal("config field password is not set"))
 			})
 		})
 		When("all valid", func() {
 			BeforeEach(func() {
 				scmClient = &scm.MockScmClient{}
-				os.Setenv("IMAGE_REPO_PASSWORD", "test")
+				c = &ImageRepoStepConfig{
+					URL:      url,
+					User:     user,
+					Password: "test",
+				}
 			})
 			It("should return nil", func() {
 				err := c.ConfigSCM(scmClient)
 				Expect(err).Error().ShouldNot(HaveOccurred())
 			})
 		})
-		AfterEach(func() {
-			os.Setenv("IMAGE_REPO_PASSWORD", existImageEnv)
-		})
 	})
-
 })