feat(TKC-2217): include Toolkit operations in other containers (#5740)

* feat: include Init Process in the Toolkit image
* feat: copy the Toolkit binary to the volume
* feat: copy only the Busybox binaries from Toolkit image
* chore: clean up actions optimization code a bit
* chore: don't process setup step in action optimizations
* feat: add `pure` flag for the ContainerStage to define if it can be combined with other container
* feat: fit pure operations in TestWorkflow into other containers
* fix: ensure timestamps for step results in case of aborting
* feat: make `fetch` operation pure too
* chore: delete obsolete comments
* fix: unit tests and bugfixes
* fix: use proper image for setup step

.github/workflows/docker-build-api-executors-tag.yaml

@@ -128,7 +128,7 @@ jobs:
         with:
           distribution: goreleaser
           version: latest
-          args: release -f goreleaser_files/.goreleaser-docker-build-testworkflow.yml
+          args: release -f goreleaser_files/.goreleaser-docker-build-${{ matrix.service }}.yml
         env:
           GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
           ANALYTICS_TRACKING_ID: ${{secrets.TESTKUBE_API_GA_MEASUREMENT_ID}}

.github/workflows/docker-build-develop.yaml

@@ -114,7 +114,7 @@ jobs:
         with:
           distribution: goreleaser
           version: latest
-          args: release -f goreleaser_files/.goreleaser-docker-build-testworkflow.yml --snapshot
+          args: release -f goreleaser_files/.goreleaser-docker-build-${{ matrix.service }}.yml --snapshot
         env:
           GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
           ANALYTICS_TRACKING_ID: ${{secrets.TESTKUBE_API_GA_MEASUREMENT_ID}}

.github/workflows/docker-build-release.yaml

@@ -115,7 +115,7 @@ jobs:
         with:
           distribution: goreleaser
           version: latest
-          args: release -f goreleaser_files/.goreleaser-docker-build-testworkflow.yml --snapshot
+          args: release -f goreleaser_files/.goreleaser-docker-build-${{ matrix.service }}.yml --snapshot
         env:
           GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
           ANALYTICS_TRACKING_ID: ${{secrets.TESTKUBE_API_GA_MEASUREMENT_ID}}

.github/workflows/sandbox.yaml

@@ -137,7 +137,7 @@ jobs:
         with:
           distribution: goreleaser
           version: latest
-          args: release -f goreleaser_files/.goreleaser-docker-build-testworkflow.yml --snapshot
+          args: release -f goreleaser_files/.goreleaser-docker-build-${{ matrix.service }}.yml --snapshot
         env:
           GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
           ANALYTICS_TRACKING_ID: ${{secrets.TESTKUBE_API_GA_MEASUREMENT_ID}}

build/testworkflow-init/Dockerfile

@@ -1,6 +1,7 @@
 # syntax=docker/dockerfile:1
 ARG BUSYBOX_IMAGE
 FROM ${BUSYBOX_IMAGE}
+RUN cp -rf /bin /.tktw-bin
 COPY testworkflow-init /init
 USER 1001
 ENTRYPOINT ["/init"]

build/testworkflow-toolkit/Dockerfile

@@ -1,8 +1,13 @@
 # syntax=docker/dockerfile:1
+ARG BUSYBOX_IMAGE
 ARG ALPINE_IMAGE
+
+FROM ${BUSYBOX_IMAGE} AS busybox
 FROM ${ALPINE_IMAGE}
 RUN apk --no-cache add ca-certificates libssl3 git openssh-client
+COPY --from=busybox /bin /.tktw-bin
 COPY testworkflow-toolkit /toolkit
+COPY testworkflow-init /init
 RUN adduser --disabled-password --home / --no-create-home --uid 1001 default
 USER 1001
 ENTRYPOINT ["/toolkit"]

cmd/tcl/testworkflow-toolkit/commands/services.go

@@ -179,7 +179,7 @@ func NewServicesCmd() *cobra.Command {
 						v, err := expressions.EvalTemplate(svcSpec.Timeout, machines...)
 						ui.ExitOnError(fmt.Sprintf("%s: %d: error: timeout expression", commontcl.ServiceLabel(name), index), err)
 						d, err := time.ParseDuration(strings.ReplaceAll(v, " ", ""))
-						ui.ExitOnError(fmt.Sprintf("%s: %d: error: invalid timeout: %s:", commontcl.ServiceLabel(name), index, v), err)
+						ui.ExitOnError(fmt.Sprintf("%s: %d: error: invalid timeout: %s", commontcl.ServiceLabel(name), index, v), err)
 						svcInstances[index].Timeout = &d
 					}
 				}

cmd/tcl/testworkflow-toolkit/spawn/utils.go

@@ -170,11 +170,12 @@ func ProcessFetch(transferSrv transfer.Server, fetch []testworkflowsv1.StepParal
 				ContainerConfig: testworkflowsv1.ContainerConfig{
 					Image:           env.Config().Images.Toolkit,
 					ImagePullPolicy: corev1.PullIfNotPresent,
-					Command:         common.Ptr([]string{"/toolkit", "transfer"}),
+					Command:         common.Ptr([]string{constants.DefaultToolkitPath, "transfer"}),
 					Env: []corev1.EnvVar{
 						{Name: "TK_NS", Value: env.Namespace()},
 						{Name: "TK_REF", Value: env.Ref()},
 						stage.BypassToolkitCheck,
+						stage.BypassPure,
 					},
 					Args: &result,
 				},

cmd/testworkflow-init/commands/setup.go

@@ -30,12 +30,26 @@ func Setup(config lite.ActionSetup) error {
 		stdoutUnsafe.Print(" skipped\n")
 	}
 
+	// Copy the toolkit
+	stdoutUnsafe.Print("Configuring toolkit...")
+	if config.CopyToolkit {
+		err := exec.Command("cp", "/toolkit", data.ToolkitPath).Run()
+		if err != nil {
+			stdoutUnsafe.Error(" error\n")
+			stdoutUnsafe.Errorf("  failed to copy the /toolkit utilities: %s\n", err.Error())
+			return err
+		}
+		stdoutUnsafe.Print(" done\n")
+	} else {
+		stdoutUnsafe.Print(" skipped\n")
+	}
+
 	// Copy the shell and useful libraries
 	stdoutUnsafe.Print("Configuring shell...")
 	if config.CopyBinaries {
 		// Use `cp` on the whole directory, as it has plenty of files, which lead to the same FS block.
 		// Copying individual files will lead to high FS usage
-		err := exec.Command("cp", "-rf", "/bin", data.InternalBinPath).Run()
+		err := exec.Command("cp", "-rf", "/.tktw-bin", data.InternalBinPath).Run()
 		if err != nil {
 			stdoutUnsafe.Error(" error\n")
 			stdoutUnsafe.Errorf("  failed to copy the binaries: %s\n", err.Error())

cmd/testworkflow-init/data/constants.go

@@ -11,6 +11,7 @@ const (
 var (
 	InternalBinPath = filepath.Join(InternalPath, "bin")
 	InitPath        = filepath.Join(InternalPath, "init")
+	ToolkitPath     = filepath.Join(InternalPath, "toolkit")
 	StatePath       = filepath.Join(InternalPath, "state")
 )
 

cmd/testworkflow-init/orchestration/setup.go

@@ -220,7 +220,7 @@ func (c *setup) SetWorkingDir(workingDir string) {
 		wd = workingDir
 		_ = os.MkdirAll(wd, 0755)
 	} else {
-		err = os.MkdirAll(wd, 0755)
+		_ = os.MkdirAll(wd, 0755)
 	}
 	err = os.Chdir(wd)
 

goreleaser_files/.goreleaser-docker-build-testworkflow.yml → goreleaser_files/.goreleaser-docker-build-testworkflow-init.yml

@@ -19,8 +19,8 @@ env:
   - DOCKER_IMAGE_URL={{ if index .Env "SANDBOX_IMAGE" }}https://hub.docker.com/r/kubeshop/testkube-sandbox{{ else }}https://hub.docker.com/r/kubeshop/{{ .Env.REPOSITORY }}{{ end }}
 builds:
   - id: "linux"
-    main: "./cmd/{{ .Env.SERVICE }}"
-    binary: "{{ .Env.SERVICE }}"
+    main: "./cmd/testworkflow-init"
+    binary: "testworkflow-init"
     env:
       - CGO_ENABLED=0
     goos:
@@ -34,7 +34,7 @@ builds:
       -X github.com/kubeshop/testkube/pkg/version.Commit={{ .FullCommit }}
       -s -w
 dockers:
-  - dockerfile: ./build/{{ .Env.SERVICE }}/Dockerfile
+  - dockerfile: ./build/testworkflow-init/Dockerfile
     use: buildx
     goos: linux
     goarch: amd64
@@ -55,7 +55,7 @@ dockers:
       - "--build-arg=ALPINE_IMAGE={{ .Env.ALPINE_IMAGE }}"
       - "--build-arg=BUSYBOX_IMAGE={{ .Env.BUSYBOX_IMAGE }}"
 
-  - dockerfile: ./build/{{ .Env.SERVICE }}/Dockerfile
+  - dockerfile: ./build/testworkflow-init/Dockerfile
     use: buildx
     goos: linux
     goarch: arm64

goreleaser_files/.goreleaser-docker-build-testworkflow-toolkit.yml

@@ -0,0 +1,112 @@
+version: 2
+env:
+  # Goreleaser always uses the docker buildx builder with name "default"; see
+  # https://github.com/goreleaser/goreleaser/pull/3199
+  # To use a builder other than "default", set this variable.
+  # Necessary for, e.g., GitHub actions cache integration.
+  - DOCKER_REPO={{ if index .Env "DOCKER_REPO"  }}{{ .Env.DOCKER_REPO }}{{ else }}kubeshop{{ end }}
+  - DOCKER_BUILDX_BUILDER={{ if index .Env "DOCKER_BUILDX_BUILDER"  }}{{ .Env.DOCKER_BUILDX_BUILDER }}{{ else }}default{{ end }}
+  # Setup to enable Docker to use, e.g., the GitHub actions cache; see
+  # https://docs.docker.com/build/building/cache/backends/
+  # https://github.com/moby/buildkit#export-cache
+  - DOCKER_BUILDX_CACHE_FROM={{ if index .Env "DOCKER_BUILDX_CACHE_FROM"  }}{{ .Env.DOCKER_BUILDX_CACHE_FROM }}{{ else }}type=registry{{ end }}
+  - DOCKER_BUILDX_CACHE_TO={{ if index .Env "DOCKER_BUILDX_CACHE_TO"  }}{{ .Env.DOCKER_BUILDX_CACHE_TO }}{{ else }}type=inline{{ end }}
+  # Build image with commit sha tag
+  - IMAGE_TAG_SHA={{ if index .Env "IMAGE_TAG_SHA"  }}{{ .Env.IMAGE_TAG_SHA }}{{ else }}{{ end }}
+  # Build Sandbox Image
+  - SANDBOX_IMAGE={{ if index .Env "SANDBOX_IMAGE"  }}{{ .Env.SANDBOX_IMAGE }}{{ else }}{{ end }}
+  - DOCKER_IMAGE_TITLE={{ if index .Env "SANDBOX_IMAGE" }}testkube-sandbox-{{ .Env.SERVICE }}{{ else }}{{ .Env.REPOSITORY }}{{ end }}
+  - DOCKER_IMAGE_URL={{ if index .Env "SANDBOX_IMAGE" }}https://hub.docker.com/r/kubeshop/testkube-sandbox{{ else }}https://hub.docker.com/r/kubeshop/{{ .Env.REPOSITORY }}{{ end }}
+builds:
+  - id: "linux-init"
+    main: "./cmd/testworkflow-init"
+    binary: "testworkflow-init"
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+    mod_timestamp: "{{ .CommitTimestamp }}"
+    ldflags:
+      -X github.com/kubeshop/testkube/pkg/version.Version={{ .Version }}
+      -X github.com/kubeshop/testkube/pkg/version.Commit={{ .FullCommit }}
+      -s -w
+  - id: "linux-toolkit"
+    main: "./cmd/testworkflow-toolkit"
+    binary: "testworkflow-toolkit"
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+    mod_timestamp: "{{ .CommitTimestamp }}"
+    ldflags:
+      -X github.com/kubeshop/testkube/pkg/version.Version={{ .Version }}
+      -X github.com/kubeshop/testkube/pkg/version.Commit={{ .FullCommit }}
+      -s -w
+dockers:
+  - dockerfile: ./build/testworkflow-toolkit/Dockerfile
+    use: buildx
+    goos: linux
+    goarch: amd64
+    image_templates:
+      - "{{ if .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:{{ .ShortCommit }}{{ end }}"
+      - "{{ if not .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:{{ .Version }}-amd64{{ end }}"
+      - "{{ if .Env.SANDBOX_IMAGE }}{{ .Env.DOCKER_REPO }}/testkube-sandbox:{{ .Env.SERVICE }}-{{ .Env.BRANCH_IDENTIFIER }}-{{ .ShortCommit }}{{ end }}"
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--label=org.opencontainers.image.title={{ .Env.DOCKER_IMAGE_TITLE }}"
+      - "--label=org.opencontainers.image.url={{ .Env.DOCKER_IMAGE_URL }}"
+      - "--label=org.opencontainers.image.created={{ .Date}}"
+      - "--label=org.opencontainers.image.revision={{ .FullCommit }}"
+      - "--label=org.opencontainers.image.version={{ .Version }}"
+      - "--builder={{ .Env.DOCKER_BUILDX_BUILDER }}"
+      - "--cache-to={{ .Env.DOCKER_BUILDX_CACHE_TO }}"
+      - "--cache-from={{ .Env.DOCKER_BUILDX_CACHE_FROM }}"
+      - "--build-arg=ALPINE_IMAGE={{ .Env.ALPINE_IMAGE }}"
+      - "--build-arg=BUSYBOX_IMAGE={{ .Env.BUSYBOX_IMAGE }}"
+
+  - dockerfile: ./build/testworkflow-toolkit/Dockerfile
+    use: buildx
+    goos: linux
+    goarch: arm64
+    image_templates:
+      - "{{ if not .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:{{ .Version }}-arm64v8{{ end }}"
+    build_flag_templates:
+      - "--platform=linux/arm64/v8"
+      - "--label=org.opencontainers.image.created={{ .Date }}"
+      - "--label=org.opencontainers.image.title={{ .ProjectName }}"
+      - "--label=org.opencontainers.image.revision={{ .FullCommit }}"
+      - "--label=org.opencontainers.image.version={{ .Version }}"
+      - "--builder={{ .Env.DOCKER_BUILDX_BUILDER }}"
+      - "--cache-to={{ .Env.DOCKER_BUILDX_CACHE_TO }}"
+      - "--cache-from={{ .Env.DOCKER_BUILDX_CACHE_FROM }}"
+      - "--build-arg=ALPINE_IMAGE={{ .Env.ALPINE_IMAGE }}"
+      - "--build-arg=BUSYBOX_IMAGE={{ .Env.BUSYBOX_IMAGE }}"
+
+docker_manifests:
+  - name_template: "{{ if not .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:{{ .Version }}{{ end }}"
+    image_templates:
+      - "{{ if not .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:{{ .Version }}-amd64{{ end }}"
+      - "{{ if not .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:{{ .Version }}-arm64v8{{ end }}"
+  - name_template: "{{ if not .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:latest{{ end }}"
+    image_templates:
+      - "{{ if not .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:{{ .Version }}-amd64{{ end }}"
+      - "{{ if not .Env.IMAGE_TAG_SHA }}{{ .Env.DOCKER_REPO }}/{{ .Env.REPOSITORY }}:{{ .Version }}-arm64v8{{ end }}"
+
+
+release:
+  disable: true
+
+docker_signs:
+  - cmd: cosign
+    artifacts: all
+    output: true
+    args:
+      - "sign"
+      - "${artifact}"
+      - "--yes"

pkg/api/v1/testkube/model_test_workflow_result_extended.go

@@ -492,6 +492,21 @@ func predictTestWorkflowStepStatus(v TestWorkflowStepResult, sig TestWorkflowSig
 }
 
 func recomputeTestWorkflowStepResult(v TestWorkflowStepResult, sig TestWorkflowSignature, r *TestWorkflowResult) TestWorkflowStepResult {
+	// Ensure there is a queue time if the step is already started
+	if v.QueuedAt.IsZero() {
+		if !v.StartedAt.IsZero() {
+			v.QueuedAt = v.StartedAt
+		} else if !v.FinishedAt.IsZero() {
+			v.QueuedAt = v.FinishedAt
+		}
+	}
+
+	// Ensure there is a start time if the step is already finished
+	if v.StartedAt.IsZero() && !v.FinishedAt.IsZero() {
+		v.StartedAt = v.QueuedAt
+	}
+
+	// Compute children
 	children := sig.Children
 	if len(children) == 0 {
 		return v

pkg/tcl/testworkflowstcl/testworkflowprocessor/operations.go

@@ -36,6 +36,9 @@ func ProcessExecute(_ testworkflowprocessor.InternalProcessor, layer testworkflo
 	hasWorkflows := len(step.Execute.Workflows) > 0
 	hasTests := len(step.Execute.Tests) > 0
 
+	// Allow to combine it within other containers
+	stage.SetPure(true)
+
 	// Fail if there is nothing to run
 	if !hasTests && !hasWorkflows {
 		return nil, errors.New("no test workflows and tests provided to the 'execute' step")
@@ -44,7 +47,7 @@ func ProcessExecute(_ testworkflowprocessor.InternalProcessor, layer testworkflo
 	container.
 		SetImage(constants.DefaultToolkitImage).
 		SetImagePullPolicy(corev1.PullIfNotPresent).
-		SetCommand("/toolkit", "execute").
+		SetCommand(constants.DefaultToolkitPath, "execute").
 		EnableToolkit(stage.Ref()).
 		AppendVolumeMounts(layer.AddEmptyDirVolume(nil, constants.DefaultTransferDirPath))
 	args := make([]string, 0)
@@ -91,6 +94,9 @@ func ProcessParallel(_ testworkflowprocessor.InternalProcessor, layer testworkfl
 	stage := stage.NewContainerStage(layer.NextRef(), container.CreateChild())
 	stage.SetCategory("Run in parallel")
 
+	// Allow to combine it within other containers
+	stage.SetPure(true)
+
 	// Inherit container defaults
 	inherited := common.Ptr(stage.Container().ToContainerConfig())
 	inherited.VolumeMounts = nil
@@ -99,7 +105,7 @@ func ProcessParallel(_ testworkflowprocessor.InternalProcessor, layer testworkfl
 	stage.Container().
 		SetImage(constants.DefaultToolkitImage).
 		SetImagePullPolicy(corev1.PullIfNotPresent).
-		SetCommand("/toolkit", "parallel").
+		SetCommand(constants.DefaultToolkitPath, "parallel").
 		EnableToolkit(stage.Ref()).
 		AppendVolumeMounts(layer.AddEmptyDirVolume(nil, constants.DefaultTransferDirPath))
 
@@ -124,10 +130,13 @@ func ProcessServicesStart(_ testworkflowprocessor.InternalProcessor, layer testw
 	stage := stage.NewContainerStage(layer.NextRef(), container.CreateChild())
 	stage.SetCategory("Start services")
 
+	// Allow to combine it within other containers
+	stage.SetPure(true)
+
 	stage.Container().
 		SetImage(constants.DefaultToolkitImage).
 		SetImagePullPolicy(corev1.PullIfNotPresent).
-		SetCommand("/toolkit", "services", "-g", "{{env.TK_SVC_REF}}").
+		SetCommand(constants.DefaultToolkitPath, "services", "-g", "{{env.TK_SVC_REF}}").
 		EnableToolkit(stage.Ref()).
 		AppendVolumeMounts(layer.AddEmptyDirVolume(nil, constants.DefaultTransferDirPath))
 
@@ -154,10 +163,13 @@ func ProcessServicesStop(_ testworkflowprocessor.InternalProcessor, layer testwo
 	stage.SetOptional(true)
 	stage.SetCategory("Stop services")
 
+	// Allow to combine it within other containers
+	stage.SetPure(true)
+
 	stage.Container().
 		SetImage(constants.DefaultToolkitImage).
 		SetImagePullPolicy(corev1.PullIfNotPresent).
-		SetCommand("/toolkit", "kill", "{{env.TK_SVC_REF}}").
+		SetCommand(constants.DefaultToolkitPath, "kill", "{{env.TK_SVC_REF}}").
 		EnableToolkit(stage.Ref()).
 		AppendVolumeMounts(layer.AddEmptyDirVolume(nil, constants.DefaultTransferDirPath))