feat: new plugin jenkins-pipeline-kubernetes && jenkins plugin enhancement

cmd/plugin/jenkins-pipeline-kubernetes/main.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"github.com/devstream-io/devstream/internal/pkg/plugin/jenkinspipelinekubernetes"
+	"github.com/devstream-io/devstream/pkg/util/log"
+)
+
+// NAME is the name of this DevStream plugin.
+const NAME = "jenkins-pipeline-kubernetes"
+
+// Plugin is the type used by DevStream core. It's a string.
+type Plugin string
+
+// Create implements the create of jenkins-pipeline-kubernetes.
+func (p Plugin) Create(options map[string]interface{}) (map[string]interface{}, error) {
+	return jenkinspipelinekubernetes.Create(options)
+}
+
+// Update implements the update of jenkins-pipeline-kubernetes.
+func (p Plugin) Update(options map[string]interface{}) (map[string]interface{}, error) {
+	return jenkinspipelinekubernetes.Update(options)
+}
+
+// Delete implements the delete of jenkins-pipeline-kubernetes.
+func (p Plugin) Delete(options map[string]interface{}) (bool, error) {
+	return jenkinspipelinekubernetes.Delete(options)
+}
+
+// Read implements the read of jenkins-pipeline-kubernetes.
+func (p Plugin) Read(options map[string]interface{}) (map[string]interface{}, error) {
+	return jenkinspipelinekubernetes.Read(options)
+}
+
+// DevStreamPlugin is the exported variable used by the DevStream core.
+var DevStreamPlugin Plugin
+
+func main() {
+	log.Infof("%T: %s is a plugin for DevStream. Use it with DevStream.\n", DevStreamPlugin, NAME)
+}

docs/plugins/jenkins-pipeline-kubernetes.md

@@ -0,0 +1,10 @@
+# jenkins-pipeline-kubernetes plugin
+
+TODO(aFlyBird0): Add your document here.
+## Usage
+
+```yaml
+
+--8<-- "jenkins-pipeline-kubernetes.yaml"
+
+```

docs/plugins/jenkins-pipeline-kubernetes.zh.md

@@ -0,0 +1,21 @@
+# jenkins-pipeline-kubernetes 插件
+
+这个插件在已有的 Jenkins 上建立 Jenkins job, 将 GitHub 作为 SCM。
+
+步骤：
+
+1. 访问 Jenkins web UI，创建 token。步骤：People -> admin ->Configure -> API Token -> Add new Token。
+2. 按需修改配置项，其中 `githubRepoUrl` 为 GitHub 仓库地址，应预先建立一个 GitHub 仓库，并创建一个名为 "Jenkinsfile" 的文件放至仓库根目录。
+3. 设置环境变量
+    - `GITHUB_TOKEN`
+    - `JENKINS_TOKEN`
+
+## 用例
+
+```yaml
+
+--8<-- "jenkins-pipeline-kubernetes.yaml"
+
+```
+
+目前，所有选项均为必填项。

docs/plugins/jenkins.md

@@ -2,6 +2,8 @@
 
 This plugin installs [Jenkins](https://jenkins.io) in an existing Kubernetes cluster using the Helm chart.
 
+It also installs [GitHub Pull Request Builder(ghprb)](https://plugins.jenkins.io/ghprb/) and [OWASP Markup Formatter](https://plugins.jenkins.io/antisamy-markup-formatter/) plugins. Then enable HTML parsing using OWASP Markup Formatter Plugin , useful with ghprb plugin.
+
 ## Usage
 
 ### Production Environment

docs/plugins/jenkins.zh.md

@@ -2,6 +2,8 @@
 
 本插件使用 helm 在已有的 k8s 集群上安装 [Jenkins](https://jenkins.io)。
 
+并且安装 [GitHub Pull Request Builder](https://plugins.jenkins.io/ghprb/) 插件和 [OWASP Markup Formatter](https://plugins.jenkins.io/antisamy-markup-formatter/) 插件；同时利用 OWASP Markup Formatter 插件激活 HTML 渲染模式。
+
 ## 使用方法
 
 ### 生产环境

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/mittwald/go-helm-client v0.8.4
 	github.com/onsi/ginkgo/v2 v2.1.4
 	github.com/onsi/gomega v1.19.0
+	github.com/parnurzeal/gorequest v0.2.16
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.8.1
@@ -207,6 +208,7 @@ require (
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	k8s.io/kubectl v0.22.4 // indirect
 	k8s.io/kubernetes v1.22.4 // indirect
+	moul.io/http2curl v1.0.0 // indirect
 	oras.land/oras-go v0.4.0 // indirect
 	sigs.k8s.io/kustomize/api v0.8.11 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.11.0 // indirect

go.sum

@@ -1057,6 +1057,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
+github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ=
+github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
@@ -1979,6 +1981,8 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8=
+moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE=
 oras.land/oras-go v0.4.0 h1:u6+7D+raZDYHwlz/uOwNANiRmyYDSSMW7A9E1xXycUQ=
 oras.land/oras-go v0.4.0/go.mod h1:VJcU+VE4rkclUbum5C0O7deEZbBYnsnpbGSACwTjOcg=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

internal/pkg/plugin/jenkinspipelinekubernetes/client.go

@@ -0,0 +1,153 @@
+package jenkinspipelinekubernetes
+
+import (
+	_ "embed"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/parnurzeal/gorequest"
+
+	"github.com/devstream-io/devstream/pkg/util/log"
+)
+
+// ClientInf represents the client abstraction for jenkins
+type ClientInf interface {
+	CreateCredentialSecretText() error
+	CreateCredentialUsernamePassword() error
+	GetCrumb() (string, error)
+	GetCrumbHeader() (headerKey, headerValue string, err error)
+	CreateItem(jobXmlContent string) error
+}
+
+// Client is the client for jenkins
+type Client struct {
+	Opts *Options
+}
+
+func NewClient(options *Options) *Client {
+	return &Client{
+		Opts: options,
+	}
+}
+
+//func (c *Client) ifCredentialExists() bool {
+//	// todo
+//	return false
+//}
+
+// CreateCredentialSecretText creates a credential in the type of "Secret text"
+func (c *Client) CreateCredentialSecretText() error {
+
+	accessURL := c.Opts.GetJenkinsAccessURL()
+	crumb, err := c.GetCrumb()
+	if err != nil {
+		return fmt.Errorf("failed to create credential secret: %s", err)
+	}
+
+	// TODO(aFlyBird0): use gorequest to do the request
+	cmdCreateCredential := fmt.Sprintf(`
+curl -H %s -X POST '%s/credentials/store/system/domain/_/createCredentials' \
+--data-urlencode 'json={
+  "": "0",
+   "credentials": {
+	   "scope": "GLOBAL",
+	   "id": "%s",
+	   "secret": "%s",
+	   "description": "%s",
+	   "$class": "org.jenkinsci.plugins.plaincredentials.impl.StringCredentialsImpl"
+  }
+}'`, crumb, accessURL, jenkinsCredentialID, c.Opts.GitHubToken, jenkinsCredentialDesc)
+
+	cmd := exec.Command("sh", "-c", cmdCreateCredential)
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to create credential secret: %s", err)
+	}
+
+	return nil
+}
+
+// CreateCredentialUsernamePassword creates a credential in the type of "Username with password"
+func (c *Client) CreateCredentialUsernamePassword() error {
+
+	accessURL := c.Opts.GetJenkinsAccessURL()
+	crumb, err := c.GetCrumb()
+	if err != nil {
+		return fmt.Errorf("failed to create credential secret: %s", err)
+	}
+
+	// TODO(aFlyBird0): use gorequest to do the request
+	cmdCreateCredential := fmt.Sprintf(`
+curl -H %s -X POST '%s/credentials/store/system/domain/_/createCredentials' \
+--data-urlencode 'json={
+	"": "0",
+	"credentials": {
+		"scope": "GLOBAL",
+		"id": "%s",
+		"username": "foo-useless-username",
+		"password": "%s",
+		"description": "%s",
+		"$class": "com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl"
+  }
+}'`, crumb, accessURL, jenkinsCredentialID, c.Opts.GitHubToken, jenkinsCredentialDesc)
+
+	cmd := exec.Command("sh", "-c", cmdCreateCredential)
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to create credential secret: %s", err)
+	}
+
+	return nil
+}
+
+// GetCrumb returns the crumb for jenkins,
+// jenkins uses crumb to prevent CSRF(cross-site request forgery),
+// format: "Jenkins-Crumb:a70290b6423777f0a4c771d4805637ac36d5fd78336a20d48d72167ef5f13b9a"
+// ref: https://www.jenkins.io/doc/upgrade-guide/2.176/#upgrading-to-jenkins-lts-2-176-3
+// ref: https://stackoverflow.com/questions/44711696/jenkins-403-no-valid-crumb-was-included-in-the-request
+func (c *Client) GetCrumb() (string, error) {
+	request := gorequest.New()
+	getCrumbURL := c.Opts.GetJenkinsAccessURL() + `/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)`
+	resp, body, errs := request.Get(getCrumbURL).End()
+	log.Debugf("GetCrumb url: %s", getCrumbURL)
+	if len(errs) != 0 {
+		return "", fmt.Errorf("failed to get crumb: %s", errs)
+	}
+	if resp.StatusCode != 200 {
+		return "", fmt.Errorf("failed to get crumb, here is response: %s", body)
+	}
+
+	return strings.TrimSpace(body), nil
+}
+
+// GetCrumbHeader behaves like GetCrumb, but it returns the header key and value
+func (c *Client) GetCrumbHeader() (headerKey, headerValue string, err error) {
+	// crumb format: "Jenkins-Crumb:a70290b6423777f0a4c771d4805637ac36d5fd78336a20d48d72167ef5f13b9a"
+	crumb, err := c.GetCrumb()
+	if err != nil {
+		return "", "", err
+	}
+	crumbMap := strings.Split(crumb, ":")
+	if len(crumbMap) != 2 {
+		return "", "", fmt.Errorf("failed to get crumb, here is response: %s", crumb)
+	}
+	return crumbMap[0], crumbMap[1], nil
+}
+
+//go:embed job-template.xml
+var jobTemplate string
+
+// CreateItem creates a job in jenkins with the given job xml
+func (c *Client) CreateItem(jobXmlContent string) error {
+	request := gorequest.New()
+	resp, body, errs := request.Post(c.Opts.GetJenkinsAccessURL()+"/createItem").
+		Set("Content-Type", "application/xml").
+		Query("name=" + c.Opts.JenkinsJobName).Send(jobXmlContent).End()
+
+	if len(errs) != 0 {
+		return fmt.Errorf("failed to create job: %s", errs)
+	} else if resp.StatusCode != 200 {
+		return fmt.Errorf("failed to create job, here is response: %s", body)
+	}
+
+	return nil
+}

internal/pkg/plugin/jenkinspipelinekubernetes/create.go

@@ -0,0 +1,98 @@
+package jenkinspipelinekubernetes
+
+import (
+	_ "embed"
+	"fmt"
+	"strings"
+
+	"github.com/mitchellh/mapstructure"
+
+	"github.com/devstream-io/devstream/pkg/util/log"
+)
+
+const (
+	jenkinsCredentialID   = "credential-jenkins-pipeline-kubernetes-by-devstream"
+	jenkinsCredentialDesc = "Jenkins Pipeline secret, created by devstream/jenkins-pipeline-kubernetes"
+)
+
+func Create(options map[string]interface{}) (map[string]interface{}, error) {
+	var opts Options
+	if err := mapstructure.Decode(options, &opts); err != nil {
+		return nil, err
+	}
+
+	if errs := validate(&opts); len(errs) != 0 {
+		for _, e := range errs {
+			log.Errorf("Options error: %s.", e)
+		}
+		return nil, fmt.Errorf("opts are illegal")
+	}
+
+	client := NewClient(&opts)
+
+	// always try to create credential, there will be no error if it already exists
+	if err := client.CreateCredentialUsernamePassword(); err != nil {
+		return nil, err
+	}
+
+	jobXmlContent := renderJobXml(jobTemplate, &opts)
+
+	// TODO(aFlyBird0): check if the job already exists
+
+	if err := client.CreateItem(jobXmlContent); err != nil {
+		return nil, fmt.Errorf("failed to create job: %s", err)
+	}
+
+	// TODO(aFlyBird0): use JCasC to configure job creation
+	// configuration as code: update jenkins config
+	//        + configure system -> GitHub Pull Request Builder -> Jenkins URL override
+	//        + configure system -> GitHub Pull Request Builder -> git personal access token
+	// - job creation:
+	//        + pr builder
+	//        + main builder
+	//        + GitHub project Project url
+	//        + select GitHub Pull Request Builder under Build Triggers
+	//        + Trigger phrase: optional
+	//        + Pipeline Definition: pipeline script from SCM
+	//        + Branch Specifier & Refspec: PR/main
+	//        + unselect Lightweight checkout
+	//        + Jenkinsfile template -> github repo (https://github.com/IronCore864/jenkins-test/blob/main/Jenkinsfile, https://plugins.jenkins.io/kubernetes/)
+
+	// TODO(aFlyBird0): about how to create an new config yaml in JCasC:
+	// refer: https://plugins.jenkins.io/configuration-as-code/
+	// refer: https://github.com/jenkinsci/helm-charts/blob/e4242561e5ea205bfa3405064cf5fe39b5a22d93/charts/jenkins/values.yaml#L341
+	// refer: https://github.com/jenkinsci/helm-charts/blob/e4242561e5ea205bfa3405064cf5fe39b5a22d93/charts/jenkins/templates/jcasc-config.yaml#L1-L25
+	// example: If we want to create a new config yaml in JCasC, we can use the following code:
+	// 			1. render this ConfigMap by yourself: https://github.com/jenkinsci/helm-charts/blob/e4242561e5ea205bfa3405064cf5fe39b5a22d93/charts/jenkins/templates/jcasc-config.yaml#L6-L26
+	//          2. $key should be your config file name(without .yaml extension), value should be the real config file content
+	//			3. once you want to put a new config yaml into $JenkinsHome/config-as-code/ folder, just use k8s sdk to create a new ConfigMap
+	//			4. don't forget to add a label to the ConfigMap, such as "author: devstream". So that we could easy to filter the ConfigMap created by devstream to delete them.
+	// 			5. if you want to update an existing config yaml, just use k8s sdk to update the ConfigMap
+	//			6. it seems that once you create/update a ConfigMap, Jenkins(installed by helm) will automatically reload the config yaml,
+	//				if not, you can use the following api to reload the config yaml: "POST JENKINS_URL/configuration-as-code/reload"（ remember to add jenkins crumb in http header)
+	//				see here for info:https://github.com/jenkinsci/configuration-as-code-plugin/blob/master/docs/features/configurationReload.md
+	// there are many things to do:
+	// 1. figure out the JCasC content we want to create
+	// 2. create a new ConfigMap according to the explanation above
+	// 3. handle update of ConfigMap
+	// 4. add "read" functions to the ConfigMap to get the content of the ConfigMap and check if the resource is drifted
+	// 5. maybe we also should consider to expose some config key in ConfigMap to the user
+
+	// TODO(aFlyBird0): build dtm resource
+
+	// TODO(aFlyBird0): what if user doesn't use helm to install jenkins? then the JCasC may not be automatically reloaded.
+
+	return (&resource{}).toMap(), nil
+}
+
+// TODO(aFlyBird0): unit test
+// TODO(aFlyBird0): now jenkins script path is hardcoded to "Jenkinsfile", it should be configurable
+func renderJobXml(jobTemplate string, opts *Options) string {
+	// note: maybe it is better to use html/template to generate the job template,
+	// but that way is complex and this is the simplest way to do it
+	jobXml := strings.Replace(jobTemplate, "{{.GitHubRepoURL}}", opts.GitHubRepoURL, 1)
+	jobXml = strings.Replace(jobXml, "{{.CredentialsID}}", jenkinsCredentialID, 1)
+
+	log.Debugf("job xml rendered: %s", jobXml)
+	return jobXml
+}