feat: Jenkins repo support private repo

docs/best-practices/gitlab-jenkins-harbor.zh.md

@@ -485,8 +485,8 @@ jenkins-pipeline 插件的配置如下：
       user: admin
       enableRestart: true
     scm:
-      projectURL: http://gitlab.example.com:30080/root/spring-demo
-      projectBranch: master
+      cloneURL: http://gitlab.example.com:30080/root/spring-demo
+      branch: master
     pipeline:
       jobName: test-job
       jenkinsfilePath: https://raw.githubusercontent.com/devstream-io/dtm-jenkins-pipeline-example/main/springboot/Jenkinsfile
@@ -547,8 +547,8 @@ tools:
       user: admin
       enableRestart: true
     scm:
-      projectURL: http://gitlab.example.com:30080/root/spring-demo
-      projectBranch: master
+      cloneURL: http://gitlab.example.com:30080/root/spring-demo
+      branch: master
     pipeline:
       jobName: test-job
       jenkinsfilePath: https://raw.githubusercontent.com/devstream-io/dtm-jenkins-pipeline-example/main/springboot/Jenkinsfile

docs/plugins/jenkins-pipeline.zh.md

@@ -35,6 +35,7 @@
 ```shell
 export IMAGE_REPO_PASSWORD=YOUR_IMAGE_REPO_PASSWORD
 export GITLAB_TOKEN=YOUR_GITLAB_TOKEN
+export GITLAB_SSHKEY=YOUR_REPO_PRIVATE_KEY
 ```
 
 然后准备 DevStream 插件配置：
@@ -64,8 +65,9 @@ tools:
       user: admin
       enableRestart: true
     scm:
-      projectURL: YOUR_JENKINS_ADDR/root/spring-demo
-      projectBranch: master
+      cloneURL: git@YOUR_REPO_CLONE_ADDRESS/root/spring-demo
+      branch: master
+      apiURL: YOUR_JENKINS_ADDR
     pipeline:
       jobName: test-job
       jenkinsfilePath: https://raw.githubusercontent.com/devstream-io/dtm-jenkins-pipeline-example/main/springboot/Jenkinsfile
@@ -77,6 +79,7 @@ tools:
 上述配置文件中使用的 GitLab、Jenkins 和 Harbor 访问地址需要替换成你的环境中实际地址。例如：
 
 - **YOUR_GITLAB_ADDR**: http://54.71.232.26:30080
+- **YOUR_REPO_CLONE_ADDRESS**: http://54.71.232.26:30022
 - **YOUR_JENKINS_ADDR**: http://54.71.232.26:32000
 - **YOUR_HARBOR_ADDR**: http://harbor.example.com:80
 

internal/pkg/plugininstaller/common/repo.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/devstream-io/devstream/pkg/util/file"
 	"github.com/devstream-io/devstream/pkg/util/log"
+	"github.com/devstream-io/devstream/pkg/util/scm"
 	"github.com/devstream-io/devstream/pkg/util/scm/git"
 	"github.com/devstream-io/devstream/pkg/util/scm/github"
 	"github.com/devstream-io/devstream/pkg/util/scm/gitlab"
@@ -78,52 +79,30 @@ func (d *Repo) BuildRepoInfo() *git.RepoInfo {
 	}
 }
 
-// BuildURL return url build from repo struct
-func (d *Repo) BuildURL() string {
-	repoInfo := d.BuildRepoInfo()
-	switch d.RepoType {
-	case "github":
-		return fmt.Sprintf("https://github.com/%s/%s.git", repoInfo.GetRepoOwner(), d.Repo)
-	case "gitlab":
-		var gitlabURL string
-		if d.BaseURL != "" {
-			gitlabURL = d.BaseURL
-		} else {
-			gitlabURL = gitlab.DefaultGitlabHost
-		}
-		return fmt.Sprintf("%s/%s/%s.git", gitlabURL, repoInfo.GetRepoOwner(), d.Repo)
-	default:
-		return ""
-	}
-}
-
 // NewRepoFromURL build repo struct from scm url
-func NewRepoFromURL(repoURL, branch string) (*Repo, error) {
+func NewRepoFromURL(repoType, apiURL, cloneURL, branch string) (*Repo, error) {
 	repo := &Repo{
 		Branch: branch,
 	}
-	u, err := url.ParseRequestURI(repoURL)
-	if err != nil {
-		return nil, err
-	}
-	//config repo type
-	if strings.Contains(u.Host, "github") {
+
+	if scm.IsGithubRepo(repoType, cloneURL) {
 		repo.RepoType = "github"
-	} else if strings.Contains(u.Host, "gitlab.com") {
-		repo.RepoType = "gitlab"
-		repo.BaseURL = gitlab.DefaultGitlabHost
 	} else {
 		repo.RepoType = "gitlab"
-		repo.BaseURL = fmt.Sprintf("%s://%s", u.Scheme, u.Host)
+		// extract gitlab baseURL from url string
+		if apiURL == "" {
+			apiURL = cloneURL
+		}
+		gitlabBaseURL, err := gitlab.ExtractBaseURLfromRaw(apiURL)
+		if err != nil {
+			return nil, fmt.Errorf("gitlab repo extract baseURL failed: %w", err)
+		}
+		repo.BaseURL = gitlabBaseURL
 	}
-	path := u.Path
-	// config repo owner org and
-	pathPart := strings.Split(strings.TrimPrefix(path, "/"), "/")
-	if len(pathPart) != 2 {
-		return nil, fmt.Errorf("git repo path is not valid")
+
+	if err := repo.updateRepoPathByCloneURL(cloneURL); err != nil {
+		return nil, fmt.Errorf("git extract repo info failed: %w", err)
 	}
-	repo.Owner = pathPart[0]
-	repo.Repo = pathPart[1]
 	repo.Branch = repo.getBranch()
 	return repo, nil
 }
@@ -150,3 +129,28 @@ func (d *Repo) getRepoDownloadURL() string {
 	log.Debugf("LatestCodeZipfileDownloadUrl: %s.", latestCodeZipfileDownloadURL)
 	return latestCodeZipfileDownloadURL
 }
+
+func (d *Repo) updateRepoPathByCloneURL(cloneURL string) error {
+	var paths string
+	c, err := url.ParseRequestURI(cloneURL)
+	if err != nil {
+		if strings.Contains(cloneURL, "git@") {
+			gitSSHLastIndex := strings.LastIndex(cloneURL, ":")
+			if gitSSHLastIndex == -1 {
+				return fmt.Errorf("git ssh repo not valid")
+			}
+			paths = strings.Trim(cloneURL[gitSSHLastIndex:], ":")
+		} else {
+			return fmt.Errorf("git repo transport not support for now")
+		}
+	} else {
+		paths = c.Path
+	}
+	projectPaths := strings.Split(strings.Trim(paths, "/"), "/")
+	if len(projectPaths) != 2 {
+		return fmt.Errorf("git repo path is not valid")
+	}
+	d.Owner = projectPaths[0]
+	d.Repo = strings.TrimSuffix(projectPaths[1], ".git")
+	return nil
+}

internal/pkg/plugininstaller/common/repo_test.go

@@ -29,7 +29,6 @@ var _ = Describe("Repo Struct", func() {
 			Visibility:        visibility,
 		}
 	})
-
 	Context("BuildRepoRenderConfig method", func() {
 		It("should return map", func() {
 			config := repo.BuildRepoRenderConfig()
@@ -48,3 +47,81 @@ var _ = Describe("Repo Struct", func() {
 		})
 	})
 })
+
+var _ = Describe("NewRepoFromURL func", func() {
+	var (
+		repoType, apiURL, cloneURL, branch string
+	)
+	When("is github repo", func() {
+		BeforeEach(func() {
+			cloneURL = "git@github.com:test/dtm-test.git"
+			branch = "test"
+		})
+		It("should return github repo info", func() {
+			repo, err := NewRepoFromURL(repoType, apiURL, cloneURL, branch)
+			Expect(err).Error().ShouldNot(HaveOccurred())
+			Expect(repo).ShouldNot(BeNil())
+			Expect(repo.Repo).Should(Equal("dtm-test"))
+			Expect(repo.Owner).Should(Equal("test"))
+			Expect(repo.RepoType).Should(Equal("github"))
+			Expect(repo.Branch).Should(Equal("test"))
+		})
+	})
+	When("clone url is not valid", func() {
+		BeforeEach(func() {
+			cloneURL = "git@gh.neting.cctest/dtm-test.git"
+			branch = "test"
+		})
+		It("should return error", func() {
+			_, err := NewRepoFromURL(repoType, apiURL, cloneURL, branch)
+			Expect(err).Error().Should(HaveOccurred())
+		})
+	})
+	When("is gitlab repo", func() {
+		BeforeEach(func() {
+			repoType = "gitlab"
+		})
+		When("apiURL is not set, cloneURL is ssh format", func() {
+			BeforeEach(func() {
+				cloneURL = "git@gitlab.test.com:root/test-demo.git"
+				apiURL = ""
+				branch = "test"
+			})
+			It("should return error", func() {
+				_, err := NewRepoFromURL(repoType, apiURL, cloneURL, branch)
+				Expect(err).Error().Should(HaveOccurred())
+			})
+		})
+		When("apiURL is not set, cloneURL is http format", func() {
+			BeforeEach(func() {
+				cloneURL = "http://gitlab.test.com:3000/root/test-demo.git"
+				apiURL = ""
+				branch = "test"
+			})
+			It("should return error", func() {
+				repo, err := NewRepoFromURL(repoType, apiURL, cloneURL, branch)
+				Expect(err).Error().ShouldNot(HaveOccurred())
+				Expect(repo.BaseURL).Should(Equal("http://gitlab.test.com:3000"))
+				Expect(repo.Owner).Should(Equal("root"))
+				Expect(repo.Repo).Should(Equal("test-demo"))
+				Expect(repo.Branch).Should(Equal("test"))
+			})
+		})
+		When("apiURL is set", func() {
+			BeforeEach(func() {
+				cloneURL = "git@gitlab.test.com:root/test-demo.git"
+				apiURL = "http://gitlab.http.com"
+				branch = "test"
+			})
+			It("should set apiURL as BaseURL", func() {
+				repo, err := NewRepoFromURL(repoType, apiURL, cloneURL, branch)
+				Expect(err).Error().ShouldNot(HaveOccurred())
+				Expect(repo.BaseURL).Should(Equal("http://gitlab.http.com"))
+				Expect(repo.Owner).Should(Equal("root"))
+				Expect(repo.Repo).Should(Equal("test-demo"))
+				Expect(repo.Branch).Should(Equal("test"))
+			})
+
+		})
+	})
+})

internal/pkg/plugininstaller/jenkins/installer.go

@@ -75,6 +75,10 @@ func PreInstall(plugins []string, cascTemplate string) plugininstaller.BaseOpera
 		switch opts.ProjectRepo.RepoType {
 		case "gitlab":
 			// 3. create gitlab connection for gitlab
+			err := opts.createGitlabSSHPrivateKey(jenkinsClient)
+			if err != nil {
+				return err
+			}
 			return opts.createGitlabConnection(jenkinsClient, cascTemplate)
 		default:
 			log.Debugf("jenkins preinstall only support gitlab for now")

internal/pkg/plugininstaller/jenkins/option.go

@@ -22,6 +22,7 @@ import (
 const (
 	jenkinsGitlabCredentialName = "jenkinsGitlabCredential"
 	jenkinsGitlabConnectionName = "jenkinsGitlabConnection"
+	jenkinsSSHkeyCredentialName = "jenkinsSSHKeyCredential"
 )
 
 type JobOptions struct {
@@ -44,8 +45,11 @@ type Jenkins struct {
 }
 
 type SCM struct {
-	ProjectURL    string `mapstructure:"projectURL" validate:"required"`
-	ProjectBranch string `mapstructure:"projectBranch"`
+	CloneURL      string `mapstructure:"cloneURL" validate:"required"`
+	APIURL        string `mapstructure:"apiURL"`
+	Branch        string `mapstructure:"branch"`
+	Type          string `mapstructure:"type"`
+	SSHprivateKey string `mapstructure:"sshPrivateKey"`
 }
 
 type Pipeline struct {
@@ -60,13 +64,14 @@ type ImageRepo struct {
 }
 
 type jobScriptRenderInfo struct {
-	RepoType         string
-	JobName          string
-	RepositoryURL    string
-	Branch           string
-	SecretToken      string
-	FolderName       string
-	GitlabConnection string
+	RepoType          string
+	JobName           string
+	RepositoryURL     string
+	Branch            string
+	SecretToken       string
+	FolderName        string
+	GitlabConnection  string
+	RepoCredentialsId string
 }
 
 func newJobOptions(options plugininstaller.RawOptions) (*JobOptions, error) {
@@ -91,15 +96,19 @@ func (j *JobOptions) newJenkinsClient() (jenkins.JenkinsAPI, error) {
 
 func (j *JobOptions) createOrUpdateJob(jenkinsClient jenkins.JenkinsAPI) error {
 	// 1. render groovy script
-	jobScript, err := jenkins.BuildRenderedScript(&jobScriptRenderInfo{
+	jobRenderInfo := &jobScriptRenderInfo{
 		RepoType:         j.ProjectRepo.RepoType,
 		JobName:          j.getJobName(),
-		RepositoryURL:    j.ProjectRepo.BuildURL(),
+		RepositoryURL:    j.SCM.CloneURL,
 		Branch:           j.ProjectRepo.Branch,
 		SecretToken:      j.SecretToken,
 		FolderName:       j.getJobFolder(),
 		GitlabConnection: jenkinsGitlabConnectionName,
-	})
+	}
+	if j.SCM.SSHprivateKey != "" {
+		jobRenderInfo.RepoCredentialsId = jenkinsSSHkeyCredentialName
+	}
+	jobScript, err := jenkins.BuildRenderedScript(jobRenderInfo)
 	if err != nil {
 		log.Debugf("jenkins redner template failed: %s", err)
 		return err
@@ -210,3 +219,13 @@ func (j *JobOptions) buildCIConfig() {
 	}
 	j.CIConfig = ciConfig
 }
+
+func (j *JobOptions) createGitlabSSHPrivateKey(jenkinsClient jenkins.JenkinsAPI) error {
+	if j.SCM.SSHprivateKey == "" {
+		log.Warnf("jenkins gittlab ssh key not config, private repo can't be clone")
+		return nil
+	}
+	return jenkinsClient.CreateSSHKeyCredential(
+		jenkinsSSHkeyCredentialName, j.ProjectRepo.Owner, j.SCM.SSHprivateKey,
+	)
+}

internal/pkg/plugininstaller/jenkins/option_test.go

@@ -40,7 +40,12 @@ func (m *mockSuccessJenkinsClient) InstallPluginsIfNotExists([]string, bool) err
 func (m *mockSuccessJenkinsClient) CreateGiltabCredential(string, string) error {
 	return nil
 }
-func (m *mockSuccessJenkinsClient) ConfigCasc(string) error {
+
+func (m *mockSuccessJenkinsClient) CreateSSHKeyCredential(id, userName, privateKey string) error {
+	return nil
+}
+
+func (m *mockSuccessJenkinsClient) ConfigCasc(cascScript string) error {
 	return nil
 }
 
@@ -78,6 +83,10 @@ func (m *mockErrorJenkinsClient) ApplyDingTalkBot(dingtalk.BotConfig) error {
 	return testError
 }
 
+func (m *mockErrorJenkinsClient) CreateSSHKeyCredential(id, userName, privateKey string) error {
+	return testError
+}
+
 var _ = Describe("JobOptions struct", func() {
 	var (
 		jenkinsURL, secretToken, jobName, projectURL, jenkinsFilePath, userName, password, repoOwner, repoName string
@@ -119,8 +128,8 @@ var _ = Describe("JobOptions struct", func() {
 				EnableRestart: false,
 			},
 			SCM: SCM{
-				ProjectURL:    projectURL,
-				ProjectBranch: "test",
+				CloneURL: projectURL,
+				Branch:   "test",
 			},
 			Pipeline: Pipeline{
 				JobName:         jobName,