support image pull auth

.github/workflows/main.yaml

@@ -21,13 +21,13 @@ jobs:
       - name: Install Dependencies
         run: go get -v -t -d ./...
 
-#      - name: Lint
-#        uses: golangci/golangci-lint-action@v3
-#        with:
-#          version: latest
+      - name: Lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: latest
 
       - name: Test
-        run: go test $(go list ./... | grep -v github.com/zc2638/ink/test)
+        run: make tests
 
   build:
     runs-on: ubuntu-20.04

Makefile

@@ -1,4 +1,5 @@
 tag ?= latest
+packages = `go list ./... | grep -v github.com/zc2638/ink/test`
 
 build-%:
 	@CGO_ENABLED=0 go build -ldflags="-s -w" -installsuffix cgo -o output/$* ./cmd/$*
@@ -22,5 +23,8 @@ clean:
 	@rm -rf output
 	@echo "clean complete"
 
+tests:
+	@go test $(packages)
+
 e2e:
 	@ginkgo -v test/e2e/suite

README.md

@@ -1,13 +1,13 @@
 # ink
+
 ![LICENSE](https://img.shields.io/github/license/zc2638/swag.svg?style=flat-square&color=blue)
 [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/zc2638/ink/main.yaml?branch=main&style=flat-square)](https://github.com/zc2638/ink/actions/workflows/main.yaml)
 
-Controllable CICD service
+Controllable CICD workflow service
 
 ## TODO
 
 - Add the worker for kubernetes.
-- Support docker image pull auth for registry.
 - Add more storage backend support, like MySQL and Postgres.
 - Support setting mode.
 
@@ -108,3 +108,20 @@ data:
   secret1: secret1abc123
   secret2: this is secret2
 ```
+
+#### For imagePullSecrets
+
+```yaml
+kind: Secret
+name: image-pull-auth-secret
+namespace: default
+data:
+  .dockerconfigjson: |
+    {
+      "auths": {
+        "index.docker.io": {
+          "auth": "bmFtZTpwd2Q="
+        }
+      }
+    }
+```

core/command/ctl.go

@@ -59,8 +59,8 @@ func NewCtl() *cobra.Command {
 	)
 
 	secretCmd := &cobra.Command{Use: "secret", Short: "secret operation"}
-	Register(secretCmd, "list", "list secrets", secretList)
-	Register(secretCmd, "delete", "delete secret", secretDelete)
+	Register(secretCmd, "list", "list secrets", secretList, secretListExample)
+	Register(secretCmd, "delete", "delete secret", secretDelete, secretDeleteExample)
 
 	workflowCmd := &cobra.Command{Use: "workflow", Short: "workflow operation"}
 	Register(workflowCmd, "get", "get workflow info", workflowGet, workflowGetExample)

core/command/example.go

@@ -20,6 +20,7 @@ func (s Example) String() string {
 	return string(s)
 }
 
+//nolint:gosec
 const secretListExample Example = `
 # List secrets
 inkctl secret list

core/service/secret/secret.go

@@ -18,11 +18,10 @@ import (
 	"context"
 
 	"github.com/zc2638/ink/core/constant"
-	storageV1 "github.com/zc2638/ink/pkg/api/storage/v1"
-	"github.com/zc2638/ink/pkg/database"
-
 	"github.com/zc2638/ink/core/service"
 	v1 "github.com/zc2638/ink/pkg/api/core/v1"
+	storageV1 "github.com/zc2638/ink/pkg/api/storage/v1"
+	"github.com/zc2638/ink/pkg/database"
 )
 
 func New() service.Secret {
@@ -31,7 +30,7 @@ func New() service.Secret {
 
 type srv struct{}
 
-func (s *srv) List(ctx context.Context, namespace string) ([]*v1.Secret, error) {
+func (s *srv) List(ctx context.Context, _ string) ([]*v1.Secret, error) {
 	db := database.FromContext(ctx)
 
 	var list []storageV1.Secret

core/worker/hooks/docker/convert.go

@@ -58,9 +58,6 @@ func toContainerConfig(spec *worker.Workflow, step *worker.Step) *container.Conf
 		cfg.Cmd = shell.EchoEnvCommand("INK_SCRIPT", cmdName)
 	}
 
-	for _, sec := range step.Secrets {
-		cfg.Env = append(cfg.Env, sec.Name+"="+sec.Data)
-	}
 	if len(step.VolumeMounts) != 0 {
 		cfg.Volumes = toVolumeSet(spec, step)
 	}

core/worker/hooks/docker/docker.go

@@ -161,16 +161,9 @@ func (h *docker) Step(ctx context.Context, spec *worker.Workflow, step *worker.S
 
 	image := ImageExpand(step.Image)
 	isLatest := strings.HasSuffix(image, ":latest")
-	pullOpts := types.ImagePullOptions{}
-
-	// TODO docker registry auth
-	//authMap := map[string]string{
-	//	"username": "",
-	//	"password": "",
-	//}
-	//buf, _ := json.Marshal(&authMap)
-	//authStr := base64.URLEncoding.EncodeToString(buf)
-	//pullopts.RegistryAuth = authStr
+	pullOpts := types.ImagePullOptions{
+		RegistryAuth: step.ImagePullAuth,
+	}
 
 	if step.ImagePullPolicy == v1.PullIfNotPresent {
 		var imageExist bool

core/worker/types.go

@@ -15,6 +15,7 @@
 package worker
 
 import (
+	"encoding/json"
 	"fmt"
 	"maps"
 	"path/filepath"
@@ -55,11 +56,6 @@ func (s *Workflow) GetStep(name string) *Step {
 }
 
 type (
-	Secret struct {
-		Name string
-		Data string
-	}
-
 	Volume struct {
 		v1.Volume
 
@@ -77,11 +73,11 @@ type Step struct {
 	Name            string
 	Image           string
 	ImagePullPolicy v1.PullPolicy
+	ImagePullAuth   string
 	Privileged      bool
 	WorkingDir      string
 	Network         string
 	Env             map[string]string
-	Secrets         []Secret
 	DNS             []string
 	DNSSearch       []string
 	ExtraHosts      []string
@@ -108,10 +104,6 @@ func (s *Step) CombineEnv(env ...any) map[string]string {
 			}
 		}
 	}
-
-	for _, secret := range s.Secrets {
-		out[secret.Name] = secret.Data
-	}
 	return out
 }
 
@@ -135,6 +127,16 @@ func Convert(in *v1.Workflow, status *v1.Stage, secrets []*v1.Secret) (*Workflow
 		out.Volumes = append(out.Volumes, Volume{Volume: v})
 	}
 
+	imagePullSecrets := make([]*v1.Secret, 0)
+	for _, v := range in.Spec.ImagePullSecrets {
+		for _, sv := range secrets {
+			if v == sv.Name {
+				imagePullSecrets = append(imagePullSecrets, sv)
+				break
+			}
+		}
+	}
+
 	for _, v := range in.Spec.Steps {
 		var id string
 		for _, vv := range status.Steps {
@@ -147,30 +149,13 @@ func Convert(in *v1.Workflow, status *v1.Stage, secrets []*v1.Secret) (*Workflow
 			return nil, fmt.Errorf("step not found: %s", v.Name)
 		}
 
-		env := make(map[string]string)
-		for _, ev := range v.Env {
-			if ev.Name == "" {
-				continue
-			}
-			if ev.Value != "" {
-				env[ev.Name] = ev.Value
-				continue
-			}
-			// secret to env
-			if ev.ValueFrom != nil && ev.ValueFrom.SecretKeyRef != nil {
-				secValue := ev.ValueFrom.SecretKeyRef.Find(secrets)
-				env[ev.Name] = secValue
-			}
-		}
-
-		out.Steps = append(out.Steps, &Step{
+		step := &Step{
 			ID:              completeID(id),
 			Name:            v.Name,
 			Image:           v.Image,
 			ImagePullPolicy: v.ImagePullPolicy,
 			Privileged:      v.Privileged,
 			WorkingDir:      v.WorkingDir,
-			Env:             env,
 			Entrypoint:      v.Entrypoint,
 			Shell:           v.Shell,
 			Command:         v.Command,
@@ -180,7 +165,43 @@ func Convert(in *v1.Workflow, status *v1.Stage, secrets []*v1.Secret) (*Workflow
 			DNS:             v.DNS,
 			DNSSearch:       v.DNSSearch,
 			ExtraHosts:      v.ExtraHosts,
-		})
+		}
+
+		// image registry auth
+		for _, sv := range imagePullSecrets {
+			_ = sv.Decrypt()
+			dockerAuthsData, ok := sv.Data[v1.DockerConfigJSONKey]
+			if !ok {
+				continue
+			}
+
+			var dockerAuths v1.DockerAuths
+			if err := json.Unmarshal([]byte(dockerAuthsData), &dockerAuths); err != nil {
+				continue
+			}
+			step.ImagePullAuth = dockerAuths.Match(v.Image)
+			break
+		}
+
+		env := make(map[string]string)
+		for _, ev := range v.Env {
+			if ev.Name == "" {
+				continue
+			}
+			if ev.Value != "" {
+				env[ev.Name] = ev.Value
+				continue
+			}
+			// secret to env
+			if ev.ValueFrom != nil && ev.ValueFrom.SecretKeyRef != nil {
+				_, secData := ev.ValueFrom.SecretKeyRef.Find(secrets)
+				env[ev.Name] = secData
+			}
+		}
+		if len(env) > 0 {
+			step.Env = env
+		}
+		out.Steps = append(out.Steps, step)
 	}
 
 	Compile(out)

pkg/api/core/v1/secret.go

@@ -16,7 +16,13 @@ package v1
 
 import (
 	"encoding/base64"
+	"encoding/json"
+	"errors"
 	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/docker/distribution/reference"
 )
 
 type Secret struct {
@@ -47,5 +53,81 @@ func (s *Secret) Decrypt() error {
 		}
 		s.Data[k] = string(valBytes)
 	}
+
+	return nil
+}
+
+const DockerConfigJSONKey = ".dockerconfigjson"
+
+type DockerAuths struct {
+	Auths map[string]DockerAuth `json:"auths" yaml:"auths"`
+}
+
+func (das *DockerAuths) Match(image string) string {
+	for k, v := range das.Auths {
+		if !das.matchHostname(image, k) {
+			continue
+		}
+
+		authMap := map[string]string{
+			"username": v.Username,
+			"password": v.Password,
+		}
+		bs, err := json.Marshal(&authMap)
+		if err != nil {
+			return ""
+		}
+		return base64.URLEncoding.EncodeToString(bs)
+	}
+	return ""
+}
+
+func (das *DockerAuths) matchHostname(image, hostname string) bool {
+	ref, err := reference.ParseAnyReference(image)
+	if err != nil {
+		return false
+	}
+	named, err := reference.ParseNamed(ref.String())
+	if err != nil {
+		return false
+	}
+	if hostname == "index.docker.io" {
+		hostname = "docker.io"
+	}
+	// the auth address could be a fully qualified url in which case,
+	// we should parse so we can extract the domain name.
+	if strings.HasPrefix(hostname, "http://") ||
+		strings.HasPrefix(hostname, "https://") {
+		parsed, err := url.Parse(hostname)
+		if err == nil {
+			hostname = parsed.Host
+		}
+	}
+	return reference.Domain(named) == hostname
+}
+
+type DockerAuth struct {
+	Auth     string `json:"auth" yaml:"auth"`
+	Username string `json:"username,omitempty" yaml:"username,omitempty"`
+	Password string `json:"password,omitempty" yaml:"password,omitempty"`
+}
+
+func (da *DockerAuth) Encrypt() {
+	authStr := da.Username + ":" + da.Password
+	da.Auth = base64.StdEncoding.EncodeToString([]byte(authStr))
+}
+
+func (da *DockerAuth) Decrypt() error {
+	b, err := base64.StdEncoding.DecodeString(da.Auth)
+	if err != nil {
+		return err
+	}
+
+	parts := strings.SplitN(string(b), ":", 2)
+	if len(parts) < 2 {
+		return errors.New("invalid auth")
+	}
+	da.Username = parts[0]
+	da.Password = parts[1]
 	return nil
 }