feat: jenkins-pipeline support dingtalk and sonar

Signed-off-by: Meng JiaFeng <jiafeng.meng@merico.dev>

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/google/go-cmp v0.5.8
 	github.com/google/go-github/v42 v42.0.0
+	github.com/imdario/mergo v0.3.12
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/mittwald/go-helm-client v0.8.4
@@ -126,7 +127,6 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmoiron/sqlx v1.3.1 // indirect

internal/pkg/plugin/jenkinspipeline/create.go

@@ -12,7 +12,6 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) {
 	operator := &plugininstaller.Operator{
 		PreExecuteOperations: plugininstaller.PreExecuteOperations{
 			jenkins.SetJobDefaultConfig,
-			jenkins.SetHarborAuth,
 			jenkins.ValidateJobConfig,
 		},
 		ExecuteOperations: plugininstaller.ExecuteOperations{

internal/pkg/plugininstaller/jenkins/option.go

@@ -3,8 +3,6 @@ package jenkins
 import (
 	"context"
 	"fmt"
-	"net/url"
-	"path"
 
 	"github.com/mitchellh/mapstructure"
 
@@ -14,6 +12,7 @@ import (
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/jenkins/plugins"
 	"github.com/devstream-io/devstream/pkg/util/jenkins"
 	"github.com/devstream-io/devstream/pkg/util/log"
+	"github.com/devstream-io/devstream/pkg/util/mapz"
 	"github.com/devstream-io/devstream/pkg/util/scm/git"
 )
 
@@ -23,9 +22,9 @@ type JobOptions struct {
 	Pipeline Pipeline `mapstructure:"pipeline"`
 
 	// used in package
+	CIConfig    *ci.CIConfig       `mapstructure:"ci"`
 	BasicAuth   *jenkins.BasicAuth `mapstructure:"basicAuth"`
 	ProjectRepo *common.Repo       `mapstructure:"projectRepo"`
-	CIConfig    *ci.CIConfig       `mapstructure:"ci"`
 	SecretToken string             `mapstructure:"secretToken"`
 }
 
@@ -46,20 +45,6 @@ type SCM struct {
 	SSHprivateKey string `mapstructure:"sshPrivateKey"`
 }
 
-type jobScriptRenderInfo struct {
-	RepoType          string
-	JobName           string
-	RepositoryURL     string
-	Branch            string
-	SecretToken       string
-	FolderName        string
-	GitlabConnection  string
-	RepoCredentialsId string
-	RepoURL           string
-	RepoName          string
-	RepoOwner         string
-}
-
 func newJobOptions(options plugininstaller.RawOptions) (*JobOptions, error) {
 	var opts JobOptions
 	if err := mapstructure.Decode(options, &opts); err != nil {
@@ -84,7 +69,7 @@ func (j *JobOptions) buildWebhookInfo() *git.WebhookConfig {
 	var webHookURL string
 	switch j.ProjectRepo.RepoType {
 	case "gitlab":
-		webHookURL = fmt.Sprintf("%s/project/%s", j.Jenkins.URL, j.Pipeline.getJobPath())
+		webHookURL = fmt.Sprintf("%s/project/%s", j.Jenkins.URL, j.Pipeline.JobName)
 	case "github":
 		webHookURL = fmt.Sprintf("%s/github-webhook/", j.Jenkins.URL)
 	}
@@ -106,40 +91,13 @@ func (j *JobOptions) deleteJob(client jenkins.JenkinsAPI) error {
 	}
 	isDeleted, err := job.Delete(context.Background())
 	if err != nil {
-		log.Debugf("jenkins delete job %s failed: %s", j.Pipeline.getJobPath(), err)
+		log.Debugf("jenkins delete job %s failed: %s", j.Pipeline.JobName, err)
 		return err
 	}
-	log.Debugf("jenkins delete job %s status: %v", j.Pipeline.getJobPath(), isDeleted)
+	log.Debugf("jenkins delete job %s status: %v", j.Pipeline.JobName, isDeleted)
 	return nil
 }
 
-func (j *JobOptions) buildCIConfig() {
-	jenkinsFilePath := j.Pipeline.JenkinsfilePath
-	ciConfig := &ci.CIConfig{
-		Type: "jenkins",
-	}
-	// config CIConfig
-	jenkinsfileURL, err := url.ParseRequestURI(jenkinsFilePath)
-	// if path is url, download from remote
-	if err != nil || jenkinsfileURL.Host == "" {
-		ciConfig.LocalPath = jenkinsFilePath
-	} else {
-		ciConfig.RemoteURL = jenkinsFilePath
-	}
-	var imageName string
-	if j.ProjectRepo != nil {
-		imageName = j.ProjectRepo.Repo
-	} else {
-		imageName = j.Pipeline.JobName
-	}
-	harborURLHost := path.Join(j.Pipeline.getImageHost(), defaultImageProject)
-	ciConfig.Vars = map[string]interface{}{
-		"ImageName":        imageName,
-		"ImageRepoAddress": harborURLHost,
-	}
-	j.CIConfig = ciConfig
-}
-
 func (j *JobOptions) extractJenkinsPlugins() []pluginConfigAPI {
 	var pluginsConfigs []pluginConfigAPI
 	switch j.ProjectRepo.RepoType {
@@ -152,14 +110,16 @@ func (j *JobOptions) extractJenkinsPlugins() []pluginConfigAPI {
 	case "github":
 		pluginsConfigs = append(pluginsConfigs, &plugins.GithubJenkinsConfig{
 			JenkinsURL: j.Jenkins.URL,
+			RepoOwner:  j.ProjectRepo.Owner,
 		})
 	}
+	pluginsConfigs = append(pluginsConfigs, j.Pipeline.extractPipelinePlugins(j.Jenkins.Namespace)...)
 	return pluginsConfigs
 }
 
 func (j *JobOptions) createOrUpdateJob(jenkinsClient jenkins.JenkinsAPI) error {
 	// 1. render groovy script
-	jobRenderInfo := &jobScriptRenderInfo{
+	jobRenderInfo := &jenkins.JobScriptRenderInfo{
 		RepoType:         j.ProjectRepo.RepoType,
 		JobName:          j.Pipeline.getJobName(),
 		RepositoryURL:    j.SCM.CloneURL,
@@ -193,3 +153,22 @@ func (j *JobOptions) createOrUpdateJob(jenkinsClient jenkins.JenkinsAPI) error {
 	}
 	return nil
 }
+
+func (j *JobOptions) buildCIConfig() (*ci.CIConfig, error) {
+	ciConfig := j.Pipeline.buildCIConfig()
+	// get render variables
+	plugins := j.extractJenkinsPlugins()
+	configVars := &jenkins.JenkinsFileRenderInfo{
+		AppName: j.Pipeline.JobName,
+	}
+	for _, p := range plugins {
+		p.UpdateJenkinsFileRenderVars(configVars)
+	}
+	rawConfigVars, err := mapz.DecodeStructToMap(configVars)
+	if err != nil {
+		log.Debugf("jenkins config Jenkinsfile variables failed => %+v", err)
+		return nil, err
+	}
+	ciConfig.Vars = rawConfigVars
+	return ciConfig, nil
+}

internal/pkg/plugininstaller/jenkins/option_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci"
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/common"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/jenkins/plugins"
 	"github.com/devstream-io/devstream/pkg/util/jenkins"
 	"github.com/devstream-io/devstream/pkg/util/jenkins/dingtalk"
 )
@@ -135,7 +136,7 @@ var _ = Describe("JobOptions struct", func() {
 			Pipeline: Pipeline{
 				JobName:         jobName,
 				JenkinsfilePath: jenkinsFilePath,
-				ImageRepo:       ImageRepo{},
+				ImageRepo:       &plugins.ImageRepoJenkinsConfig{},
 			},
 			BasicAuth:   basicAuth,
 			ProjectRepo: projectRepo,
@@ -172,7 +173,7 @@ var _ = Describe("JobOptions struct", func() {
 	Context("buildWebhookInfo method", func() {
 		It("should work normal", func() {
 			webHookInfo := jobOptions.buildWebhookInfo()
-			Expect(webHookInfo.Address).Should(Equal(fmt.Sprintf("%s/project/%s", jobOptions.Jenkins.URL, jobOptions.Pipeline.getJobPath())))
+			Expect(webHookInfo.Address).Should(Equal(fmt.Sprintf("%s/project/%s", jobOptions.Jenkins.URL, jobOptions.Pipeline.JobName)))
 			Expect(webHookInfo.SecretToken).Should(Equal(secretToken))
 		})
 	})
@@ -206,22 +207,39 @@ var _ = Describe("JobOptions struct", func() {
 				jobOptions.Pipeline.JenkinsfilePath = "test/local"
 			})
 			It("should use localPath", func() {
-				jobOptions.buildCIConfig()
-				Expect(jobOptions.CIConfig.LocalPath).Should(Equal(jobOptions.Pipeline.JenkinsfilePath))
-				Expect(jobOptions.CIConfig.RemoteURL).Should(BeEmpty())
+				ciConfig, err := jobOptions.buildCIConfig()
+				Expect(err).Error().ShouldNot(HaveOccurred())
+				Expect(ciConfig.LocalPath).Should(Equal(jobOptions.Pipeline.JenkinsfilePath))
+				Expect(ciConfig.RemoteURL).Should(BeEmpty())
 			})
 		})
 		When("jenkinsfilePath is remote url", func() {
 			BeforeEach(func() {
 				jobOptions.Pipeline.JenkinsfilePath = "http://www.test.com/Jenkinsfile"
 			})
 			It("should use remote url", func() {
-				jobOptions.buildCIConfig()
-				Expect(jobOptions.CIConfig.LocalPath).Should(BeEmpty())
-				Expect(jobOptions.CIConfig.RemoteURL).Should(Equal(jobOptions.CIConfig.RemoteURL))
-				Expect(string(jobOptions.CIConfig.Type)).Should(Equal("jenkins"))
+				ciConfig, err := jobOptions.buildCIConfig()
+				Expect(err).Error().ShouldNot(HaveOccurred())
+				Expect(ciConfig.LocalPath).Should(BeEmpty())
+				Expect(ciConfig.RemoteURL).Should(Equal(jobOptions.Pipeline.JenkinsfilePath))
+				Expect(string(ciConfig.Type)).Should(Equal("jenkins"))
 			})
 		})
 	})
 
+	Context("extractJenkinsPlugins method", func() {
+		When("params is right", func() {
+			BeforeEach(func() {
+				jobOptions.ProjectRepo.RepoType = "gitlab"
+				jobOptions.Pipeline = Pipeline{
+					JobName:         jobName,
+					JenkinsfilePath: jenkinsFilePath,
+				}
+			})
+			It("should return pluginConfig", func() {
+				configs := jobOptions.extractJenkinsPlugins()
+				Expect(len(configs)).Should(Equal(1))
+			})
+		})
+	})
 })

internal/pkg/plugininstaller/jenkins/pipeline.go

@@ -2,27 +2,19 @@ package jenkins
 
 import (
 	"net/url"
+	"reflect"
 	"strings"
+
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/ci"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/jenkins/plugins"
 )
 
 type Pipeline struct {
-	JobName         string    `mapstructure:"jobName" validate:"required"`
-	JenkinsfilePath string    `mapstructure:"jenkinsfilePath" validate:"required"`
-	ImageRepo       ImageRepo `mapstructure:"imageRepo"`
-}
-
-type ImageRepo struct {
-	URL  string `mapstructure:"url" validate:"url"`
-	User string `mapstructure:"user"`
-}
-
-func (p *Pipeline) getImageHost() string {
-	harborAddress := p.ImageRepo.URL
-	harborURL, err := url.ParseRequestURI(harborAddress)
-	if err != nil {
-		return harborAddress
-	}
-	return harborURL.Host
+	JobName         string                          `mapstructure:"jobName" validate:"required"`
+	JenkinsfilePath string                          `mapstructure:"jenkinsfilePath" validate:"required"`
+	ImageRepo       *plugins.ImageRepoJenkinsConfig `mapstructure:"imageRepo"`
+	Dingtalk        *plugins.DingtalkJenkinsConfig  `mapstructure:"dingTalk"`
+	Sonarqube       *plugins.SonarQubeJenkinsConfig `mapstructure:"sonarqube"`
 }
 
 func (p *Pipeline) getJobName() string {
@@ -32,13 +24,48 @@ func (p *Pipeline) getJobName() string {
 	return p.JobName
 }
 
-func (p *Pipeline) getJobPath() string {
-	return p.JobName
-}
-
 func (p *Pipeline) getJobFolder() string {
 	if strings.Contains(p.JobName, "/") {
 		return strings.Split(p.JobName, "/")[0]
 	}
 	return ""
 }
+
+func (p *Pipeline) extractPipelinePlugins(jenkinsNamespace string) []pluginConfigAPI {
+	var pluginsConfigs []pluginConfigAPI
+	v := reflect.ValueOf(p).Elem()
+	for i := 0; i < v.NumField(); i++ {
+		valueField := v.Field(i)
+		if valueField.Kind() == reflect.Ptr && !valueField.IsNil() {
+			fieldVal := valueField.Interface().(pluginConfigAPI)
+			pluginsConfigs = append(pluginsConfigs, fieldVal)
+		}
+
+	}
+	return pluginsConfigs
+}
+
+func (p *Pipeline) setDefaultValue(repoName, jenkinsNamespace string) {
+	if p.JobName == "" {
+		p.JobName = repoName
+	}
+	if p.ImageRepo != nil && p.ImageRepo.AuthNamespace == "" {
+		p.ImageRepo.AuthNamespace = jenkinsNamespace
+	}
+}
+
+func (p *Pipeline) buildCIConfig() *ci.CIConfig {
+	// config CIConfig
+	jenkinsFilePath := p.JenkinsfilePath
+	ciConfig := &ci.CIConfig{
+		Type: "jenkins",
+	}
+	jenkinsfileURL, err := url.ParseRequestURI(jenkinsFilePath)
+	// if path is url, download from remote
+	if err != nil || jenkinsfileURL.Host == "" {
+		ciConfig.LocalPath = jenkinsFilePath
+	} else {
+		ciConfig.RemoteURL = jenkinsFilePath
+	}
+	return ciConfig
+}