From 2e55bec5dbf2afad231ea15620d5857dcc310279 Mon Sep 17 00:00:00 2001 From: Priya Modali Date: Thu, 4 Mar 2021 12:43:03 -0800 Subject: [PATCH] Implement custom tester functionality in Skaffold (#5451) * Adding new test runner for custom tester. * Adding new custom tester config to Skaffold comfig. * Adding usage example for newly added custon tester. * Extracting structure tests run logic into a seperate method. * Moving structure test's dependency file extraction logic to NewTester() to reduce code redundancy. * Revert "Moving structure test's dependency file extraction logic to NewTester() to reduce code redundancy." This reverts commit 5e1d8591eb4f48a73dab9d0544da269acd1ce830. * Moving structure test's dependency file extraction logic to NewTester() to reduce code redundancy. * Making the custom command a required field.[D * Adding new error codes for custom tester. * Updating custom test example. * Removing errors file. * Updating custom test example. * Adding generated schema files. * Added custom test example to `integration/run_test.go`. * Added custom test example in `integration/examples directory`. * Updating the apiVersion in example skaffold.yaml. * Updating the apiVersion in example skaffold.yaml. * Removing the custom-test form examples folder as the newly added custom test fields are not available in the current version of the skaffold config. * Updates per review feedback. * Added schema validation & updated example. * Adding tests for error scenarios. * Updated the test exit code. * Special casing error tests for Windows platform. * Updating the container name for custom test example. * Updated custom-test example container pods to continue running. * Added tests for TestDependencies. * Adding tests for custom test schema validation. * Updated unit tests per review feedback. * Updated unit tests to use mock by overriding util.RunCmd method. * Updating windows specific unit test. * Added another instance of custom command to the integration test. * Updated custom test dependencies. * Updating custom test example with command dependencies. --- docs/content/en/schemas/v2beta13.json | 81 +++++- integration/examples/custom-tests/Dockerfile | 12 + integration/examples/custom-tests/README.md | 32 +++ .../examples/custom-tests/k8s-pod.yaml | 8 + integration/examples/custom-tests/main.go | 39 +++ .../examples/custom-tests/main_test.go | 53 ++++ .../examples/custom-tests/skaffold.yaml | 21 ++ integration/examples/custom-tests/test.sh | 21 ++ integration/run_test.go | 5 + pkg/skaffold/schema/latest/config.go | 35 ++- pkg/skaffold/schema/validation/validation.go | 26 ++ .../schema/validation/validation_test.go | 60 +++++ pkg/skaffold/test/custom/custom.go | 149 +++++++++++ pkg/skaffold/test/custom/custom_test.go | 240 ++++++++++++++++++ pkg/skaffold/test/test_factory.go | 20 +- 15 files changed, 794 insertions(+), 8 deletions(-) create mode 100644 integration/examples/custom-tests/Dockerfile create mode 100644 integration/examples/custom-tests/README.md create mode 100644 integration/examples/custom-tests/k8s-pod.yaml create mode 100644 integration/examples/custom-tests/main.go create mode 100644 integration/examples/custom-tests/main_test.go create mode 100644 integration/examples/custom-tests/skaffold.yaml create mode 100755 integration/examples/custom-tests/test.sh create mode 100644 pkg/skaffold/test/custom/custom.go create mode 100644 pkg/skaffold/test/custom/custom_test.go diff --git a/docs/content/en/schemas/v2beta13.json b/docs/content/en/schemas/v2beta13.json index 54c3025c35d..7259ba2ad87 100755 --- a/docs/content/en/schemas/v2beta13.json +++ b/docs/content/en/schemas/v2beta13.json @@ -922,6 +922,74 @@ "description": "*beta* tags images with a configurable template string.", "x-intellij-html-description": "beta tags images with a configurable template string." }, + "CustomTest": { + "required": [ + "command" + ], + "properties": { + "command": { + "type": "string", + "description": "custom command to be executed. If the command exits with a non-zero return code, the test will be considered to have failed.", + "x-intellij-html-description": "custom command to be executed. If the command exits with a non-zero return code, the test will be considered to have failed." + }, + "dependencies": { + "$ref": "#/definitions/CustomTestDependencies", + "description": "additional test-specific file dependencies; changes to these files will re-run this test.", + "x-intellij-html-description": "additional test-specific file dependencies; changes to these files will re-run this test." + }, + "timeoutSeconds": { + "type": "integer", + "description": "sets the wait time for skaffold for the command to complete. If unset or 0, Skaffold will wait until the command completes.", + "x-intellij-html-description": "sets the wait time for skaffold for the command to complete. If unset or 0, Skaffold will wait until the command completes." + } + }, + "preferredOrder": [ + "command", + "timeoutSeconds", + "dependencies" + ], + "additionalProperties": false, + "description": "describes the custom test command provided by the user. Custom tests are run after an image build whenever build or test dependencies are changed.", + "x-intellij-html-description": "describes the custom test command provided by the user. Custom tests are run after an image build whenever build or test dependencies are changed." + }, + "CustomTestDependencies": { + "properties": { + "command": { + "type": "string", + "description": "represents a command that skaffold executes to obtain dependencies. The output of this command *must* be a valid JSON array.", + "x-intellij-html-description": "represents a command that skaffold executes to obtain dependencies. The output of this command must be a valid JSON array." + }, + "ignore": { + "items": { + "type": "string" + }, + "type": "array", + "description": "specifies the paths that should be ignored by skaffold's file watcher. If a file exists in both `paths` and in `ignore`, it will be ignored, and will be excluded from both retest and file synchronization. Will only work in conjunction with `paths`.", + "x-intellij-html-description": "specifies the paths that should be ignored by skaffold's file watcher. If a file exists in both paths and in ignore, it will be ignored, and will be excluded from both retest and file synchronization. Will only work in conjunction with paths.", + "default": "[]" + }, + "paths": { + "items": { + "type": "string" + }, + "type": "array", + "description": "should be set to the file dependencies for this command, so that the skaffold file watcher knows when to retest and perform file synchronization.", + "x-intellij-html-description": "should be set to the file dependencies for this command, so that the skaffold file watcher knows when to retest and perform file synchronization.", + "default": "[]", + "examples": [ + "[\"src/test/**\"]" + ] + } + }, + "preferredOrder": [ + "command", + "paths", + "ignore" + ], + "additionalProperties": false, + "description": "used to specify dependencies for custom test command. `paths` should be specified for file watching to work as expected.", + "x-intellij-html-description": "used to specify dependencies for custom test command. paths should be specified for file watching to work as expected." + }, "DateTimeTagger": { "properties": { "format": { @@ -2910,6 +2978,14 @@ "image" ], "properties": { + "custom": { + "items": { + "$ref": "#/definitions/CustomTest" + }, + "type": "array", + "description": "the set of custom tests to run after an artifact is built.", + "x-intellij-html-description": "the set of custom tests to run after an artifact is built." + }, "image": { "type": "string", "description": "artifact on which to run those tests.", @@ -2933,11 +3009,12 @@ }, "preferredOrder": [ "image", + "custom", "structureTests" ], "additionalProperties": false, - "description": "a list of structure tests to run on images that Skaffold builds.", - "x-intellij-html-description": "a list of structure tests to run on images that Skaffold builds." + "description": "a list of tests to run on images that Skaffold builds.", + "x-intellij-html-description": "a list of tests to run on images that Skaffold builds." } } } diff --git a/integration/examples/custom-tests/Dockerfile b/integration/examples/custom-tests/Dockerfile new file mode 100644 index 00000000000..415a8456f21 --- /dev/null +++ b/integration/examples/custom-tests/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.15 as builder +COPY main.go . +# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations +ARG SKAFFOLD_GO_GCFLAGS +RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o /app main.go + +FROM alpine:3 +# Define GOTRACEBACK to mark this container as using the Go language runtime +# for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/). +ENV GOTRACEBACK=single +CMD ["./app"] +COPY --from=builder /app . diff --git a/integration/examples/custom-tests/README.md b/integration/examples/custom-tests/README.md new file mode 100644 index 00000000000..370376b23ba --- /dev/null +++ b/integration/examples/custom-tests/README.md @@ -0,0 +1,32 @@ +### Example: Running custom tests on built images + +This example shows how to run _custom tests_ on newly built images in the skaffold dev loop. + +Custom tests are associated with single image artifacts. When test dependencies change, no build will happen but tests would get re-run. Tests are configured in the `skaffold.yaml` in the `test` stanza, e.g. + +```yaml +test: + - image: skaffold-example + custom: + - command: + timeoutSeconds: + dependencies: + paths: + - +``` + +As tests take time, you might prefer to configure tests using [profiles](https://skaffold.dev/docs/https://skaffold.dev/docs/environment/profiles/) so that they can be automatically enabled or disabled, e.g. +If the `command` exits with a non-zero return code then the test will have failed, and deployment will not continue. + +```yaml +profiles: + - name: test + test: + - image: skaffold-example + custom: + - command: + timeoutSeconds: + dependencies: + paths: + - +``` \ No newline at end of file diff --git a/integration/examples/custom-tests/k8s-pod.yaml b/integration/examples/custom-tests/k8s-pod.yaml new file mode 100644 index 00000000000..0b3ede01644 --- /dev/null +++ b/integration/examples/custom-tests/k8s-pod.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: custom-test +spec: + containers: + - name: custom-test + image: custom-test-example \ No newline at end of file diff --git a/integration/examples/custom-tests/main.go b/integration/examples/custom-tests/main.go new file mode 100644 index 00000000000..256b20a9399 --- /dev/null +++ b/integration/examples/custom-tests/main.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The Skaffold Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "math/rand" + "time" +) + +func MinInt(a, b int) int { + if a < b { + return a + } + return b +} + +func main() { + rand.Seed(time.Now().UnixNano()) + for { + x := rand.Intn(100) + y := rand.Intn(100) + + min := MinInt(x, y) + fmt.Println("Min of ", x, " and ", y, " is: ", min) + time.Sleep(time.Second * 1) + } +} diff --git a/integration/examples/custom-tests/main_test.go b/integration/examples/custom-tests/main_test.go new file mode 100644 index 00000000000..575879d30e6 --- /dev/null +++ b/integration/examples/custom-tests/main_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2021 The Skaffold Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "testing" +) + +func TestMinIntBasic(tb *testing.T) { + fmt.Println("Running Basic test.") + min := MinInt(5, -5) + if min != -5 { + tb.Errorf("MinInt(5, -5) returned %d; expecting -5", min) + } +} + +func TestMinIntTableDriven(tdt *testing.T) { + var tests = []struct { + x, y int + want int + }{ + {0, 0, 0}, + {1, 0, 0}, + {0, 1, 0}, + {0, -1, -1}, + {-1, 0, -1}, + {-2, -5, -5}, + {-5, -2, -5}, + } + + fmt.Println("Running Table driven test.") + for _, t := range tests { + testname := fmt.Sprintf("TestMinInt(): %d,%d", t.x, t.y) + tdt.Run(testname, func(tdt *testing.T) { + min := MinInt(t.x, t.y) + if min != t.want { + tdt.Errorf("MinInt(%d, %d) returned %d; expecting %d", t.x, t.y, min, t.want) + } + }) + } +} diff --git a/integration/examples/custom-tests/skaffold.yaml b/integration/examples/custom-tests/skaffold.yaml new file mode 100644 index 00000000000..92aaa5f917e --- /dev/null +++ b/integration/examples/custom-tests/skaffold.yaml @@ -0,0 +1,21 @@ +apiVersion: skaffold/v2beta13 +kind: Config +build: + artifacts: + - image: custom-test-example +test: + - image: custom-test-example + custom: + - command: ./test.sh + timeoutSeconds: 60 + dependencies: + paths: + - "*_test.go" + - "test.sh" + - command: echo Hello world!! + dependencies: + command: echo [\"main_test.go\"] +deploy: + kubectl: + manifests: + - k8s-* \ No newline at end of file diff --git a/integration/examples/custom-tests/test.sh b/integration/examples/custom-tests/test.sh new file mode 100755 index 00000000000..efd097bc091 --- /dev/null +++ b/integration/examples/custom-tests/test.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2021 The Skaffold Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +echo "go custom test $@" + +go test . \ No newline at end of file diff --git a/integration/run_test.go b/integration/run_test.go index 0f069a33311..e97ac9def2c 100644 --- a/integration/run_test.go +++ b/integration/run_test.go @@ -51,6 +51,11 @@ func TestRun(t *testing.T) { dir: "examples/structure-tests", pods: []string{"getting-started"}, }, + { + description: "custom-tests", + dir: "examples/custom-tests", + pods: []string{"custom-test"}, + }, { description: "microservices", dir: "examples/microservices", diff --git a/pkg/skaffold/schema/latest/config.go b/pkg/skaffold/schema/latest/config.go index ef443c5c89c..1d8152489ce 100644 --- a/pkg/skaffold/schema/latest/config.go +++ b/pkg/skaffold/schema/latest/config.go @@ -468,12 +468,15 @@ type ResourceRequirement struct { ResourceStorage string `yaml:"resourceStorage,omitempty"` } -// TestCase is a list of structure tests to run on images that Skaffold builds. +// TestCase is a list of tests to run on images that Skaffold builds. type TestCase struct { // ImageName is the artifact on which to run those tests. // For example: `gcr.io/k8s-skaffold/example`. ImageName string `yaml:"image" yamltags:"required"` + // CustomTests lists the set of custom tests to run after an artifact is built. + CustomTests []CustomTest `yaml:"custom,omitempty"` + // StructureTests lists the [Container Structure Tests](https://github.com/GoogleContainerTools/container-structure-test) // to run on that artifact. // For example: `["./test/*"]`. @@ -1016,6 +1019,36 @@ type CustomDependencies struct { Ignore []string `yaml:"ignore,omitempty"` } +// CustomTest describes the custom test command provided by the user. +// Custom tests are run after an image build whenever build or test dependencies are changed. +type CustomTest struct { + // Command is the custom command to be executed. If the command exits with a non-zero return + // code, the test will be considered to have failed. + Command string `yaml:"command" yamltags:"required"` + + // TimeoutSeconds sets the wait time for skaffold for the command to complete. + // If unset or 0, Skaffold will wait until the command completes. + TimeoutSeconds int `yaml:"timeoutSeconds,omitempty"` + + // Dependencies are additional test-specific file dependencies; changes to these files will re-run this test. + Dependencies *CustomTestDependencies `yaml:"dependencies,omitempty"` +} + +// CustomTestDependencies is used to specify dependencies for custom test command. +// `paths` should be specified for file watching to work as expected. +type CustomTestDependencies struct { + // Command represents a command that skaffold executes to obtain dependencies. The output of this command *must* be a valid JSON array. + Command string `yaml:"command,omitempty" yamltags:"oneOf=dependency"` + + // Paths should be set to the file dependencies for this command, so that the skaffold file watcher knows when to retest and perform file synchronization. + // For example: `["src/test/**"]` + Paths []string `yaml:"paths,omitempty" yamltags:"oneOf=dependency" skaffold:"filepath"` + + // Ignore specifies the paths that should be ignored by skaffold's file watcher. If a file exists in both `paths` and in `ignore`, it will be ignored, and will be excluded from both retest and file synchronization. + // Will only work in conjunction with `paths`. + Ignore []string `yaml:"ignore,omitempty" skaffold:"filepath"` +} + // DockerfileDependency *beta* is used to specify a custom build artifact that is built from a Dockerfile. This allows skaffold to determine dependencies from the Dockerfile. type DockerfileDependency struct { // Path locates the Dockerfile relative to workspace. diff --git a/pkg/skaffold/schema/validation/validation.go b/pkg/skaffold/schema/validation/validation.go index 74a806d98ea..a47b3a167f5 100644 --- a/pkg/skaffold/schema/validation/validation.go +++ b/pkg/skaffold/schema/validation/validation.go @@ -56,6 +56,7 @@ func Process(configs []*latest.SkaffoldConfig) error { errs = append(errs, validateLogPrefix(config.Deploy.Logs)...) errs = append(errs, validateArtifactTypes(config.Build)...) errs = append(errs, validateTaggingPolicy(config.Build)...) + errs = append(errs, validateCustomTest(config.Test)...) } errs = append(errs, validateArtifactDependencies(configs)...) errs = append(errs, validateSingleKubeContext(configs)...) @@ -479,3 +480,28 @@ func validateSingleKubeContext(configs []*latest.SkaffoldConfig) []error { } return nil } + +// validateCustomTest +// - makes sure that command is not empty +// - makes sure that dependencies.ignore is only used in conjunction with dependencies.paths +func validateCustomTest(tcs []*latest.TestCase) (errs []error) { + for _, tc := range tcs { + for _, ct := range tc.CustomTests { + if ct.Command == "" { + errs = append(errs, fmt.Errorf("custom test command must not be empty;")) + return + } + + if ct.Dependencies == nil { + continue + } + if ct.Dependencies.Command != "" && ct.Dependencies.Paths != nil { + errs = append(errs, fmt.Errorf("dependencies can use either command or paths, but not both")) + } + if ct.Dependencies.Paths == nil && ct.Dependencies.Ignore != nil { + errs = append(errs, fmt.Errorf("customTest has invalid dependencies; dependencies.ignore can only be used in conjunction with dependencies.paths")) + } + } + } + return +} diff --git a/pkg/skaffold/schema/validation/validation_test.go b/pkg/skaffold/schema/validation/validation_test.go index 33d51c174eb..b8c68d3898a 100644 --- a/pkg/skaffold/schema/validation/validation_test.go +++ b/pkg/skaffold/schema/validation/validation_test.go @@ -1276,3 +1276,63 @@ func TestValidateTaggingPolicy(t *testing.T) { }) } } + +func TestValidateCustomTest(t *testing.T) { + tests := []struct { + description string + command string + dependencies *latest.CustomTestDependencies + expectedErrors int + }{ + { + description: "no errors", + command: "echo Hello!", + dependencies: &latest.CustomTestDependencies{ + Paths: []string{"somepath"}, + Ignore: []string{"anotherpath"}, + }, + }, { + description: "empty command", + command: "", + dependencies: &latest.CustomTestDependencies{ + Paths: []string{"somepath"}, + Ignore: []string{"anotherpath"}, + }, + expectedErrors: 1, + }, { + description: "use both path and command", + command: "echo Hello!", + dependencies: &latest.CustomTestDependencies{ + Command: "bazel query deps", + Paths: []string{"somepath"}, + }, + expectedErrors: 1, + }, { + description: "ignore in conjunction with command", + command: "echo Hello!", + dependencies: &latest.CustomTestDependencies{ + Command: "bazel query deps", + Ignore: []string{"ignoreme"}, + }, + expectedErrors: 1, + }, { + command: "echo Hello!", + description: "nil dependencies", + dependencies: nil, + }, + } + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + testCase := &latest.TestCase{ + ImageName: "image", + CustomTests: []latest.CustomTest{{ + Command: test.command, + Dependencies: test.dependencies, + }}, + } + + errs := validateCustomTest([]*latest.TestCase{testCase}) + t.CheckDeepEqual(test.expectedErrors, len(errs)) + }) + } +} diff --git a/pkg/skaffold/test/custom/custom.go b/pkg/skaffold/test/custom/custom.go new file mode 100644 index 00000000000..3377d92adca --- /dev/null +++ b/pkg/skaffold/test/custom/custom.go @@ -0,0 +1,149 @@ +/* +Copyright 2021 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os/exec" + "runtime" + "time" + + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/list" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/color" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" +) + +// for tests +var doRunCustomCommand = runCustomCommand + +const Windows string = "windows" + +type Runner struct { + customTest latest.CustomTest + testWorkingDir string +} + +// New creates a new custom.Runner. +func New(cfg docker.Config, wd string, ct latest.CustomTest) (*Runner, error) { + return &Runner{ + customTest: ct, + testWorkingDir: wd, + }, nil +} + +// Test is the entrypoint for running custom tests +func (ct *Runner) Test(ctx context.Context, out io.Writer, _ []build.Artifact) error { + if err := doRunCustomCommand(ctx, out, ct.customTest); err != nil { + return fmt.Errorf("running custom test command: %w", err) + } + + return nil +} + +func runCustomCommand(ctx context.Context, out io.Writer, test latest.CustomTest) error { + // Expand command + command, err := util.ExpandEnvTemplate(test.Command, nil) + if err != nil { + return fmt.Errorf("unable to parse test command %q: %w", test.Command, err) + } + + if test.TimeoutSeconds <= 0 { + color.Default.Fprintf(out, "Running custom test command: %q\n", command) + } else { + color.Default.Fprintf(out, "Running custom test command: %q with timeout %d s\n", command, test.TimeoutSeconds) + newCtx, cancel := context.WithTimeout(ctx, time.Duration(test.TimeoutSeconds)*time.Second) + + defer cancel() + ctx = newCtx + } + + var cmd *exec.Cmd + // We evaluate the command with a shell so that it can contain env variables. + if runtime.GOOS == Windows { + cmd = exec.CommandContext(ctx, "cmd.exe", "/C", command) + } else { + cmd = exec.CommandContext(ctx, "sh", "-c", command) + } + + cmd.Stdout = out + cmd.Stderr = out + + if err := util.RunCmd(cmd); err != nil { + if e, ok := err.(*exec.ExitError); ok { + // If the process exited by itself, just return the error + if e.Exited() { + color.Red.Fprintf(out, "Command finished with non-0 exit code.\n") + return fmt.Errorf("command finished with non-0 exit code: %w", e) + } + // If the context is done, it has been killed by the exec.Command + select { + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + color.Red.Fprintf(out, "Command timed out\n") + } else if ctx.Err() == context.Canceled { + color.Red.Fprintf(out, "Command cancelled\n") + } + return ctx.Err() + default: + return e + } + } + return err + } + color.Green.Fprintf(out, "Command finished successfully\n") + + return nil +} + +// TestDependencies returns dependencies listed for a custom test +func (ct *Runner) TestDependencies() ([]string, error) { + test := ct.customTest + // var set orderedFileSet + + if test.Dependencies != nil { + switch { + case test.Dependencies.Command != "": + var cmd *exec.Cmd + // We evaluate the command with a shell so that it can contain env variables. + if runtime.GOOS == Windows { + cmd = exec.CommandContext(context.Background(), "cmd.exe", "/C", test.Dependencies.Command) + } else { + cmd = exec.CommandContext(context.Background(), "sh", "-c", test.Dependencies.Command) + } + + output, err := util.RunCmdOut(cmd) + if err != nil { + return nil, fmt.Errorf("getting dependencies from command: %q: %w", test.Dependencies.Command, err) + } + var deps []string + if err := json.Unmarshal(output, &deps); err != nil { + return nil, fmt.Errorf("unmarshalling dependency output into string array: %w", err) + } + return deps, nil + + case test.Dependencies.Paths != nil: + return list.Files(ct.testWorkingDir, test.Dependencies.Paths, test.Dependencies.Ignore) + } + } + return nil, nil +} diff --git a/pkg/skaffold/test/custom/custom_test.go b/pkg/skaffold/test/custom/custom_test.go new file mode 100644 index 00000000000..bcc83ee317d --- /dev/null +++ b/pkg/skaffold/test/custom/custom_test.go @@ -0,0 +1,240 @@ +/* +Copyright 2021 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package custom + +import ( + "context" + "fmt" + "io/ioutil" + "path/filepath" + "runtime" + "testing" + + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/runcontext" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" + "github.com/GoogleContainerTools/skaffold/testutil" +) + +func TestNewCustomTestRunner(t *testing.T) { + testutil.Run(t, "Testing new custom test runner", func(t *testutil.T) { + if runtime.GOOS == Windows { + t.Override(&util.DefaultExecCommand, testutil.CmdRun("cmd.exe /C echo Running Custom Test command.")) + } else { + t.Override(&util.DefaultExecCommand, testutil.CmdRun("sh -c echo Running Custom Test command.")) + } + tmpDir := t.NewTempDir().Touch("test.yaml") + + custom := latest.CustomTest{ + Command: "echo Running Custom Test command.", + TimeoutSeconds: 10, + Dependencies: &latest.CustomTestDependencies{ + Paths: []string{"**"}, + Ignore: []string{"b*"}, + }, + } + + cfg := &mockConfig{ + workingDir: tmpDir.Root(), + tests: []*latest.TestCase{{ + ImageName: "image", + CustomTests: []latest.CustomTest{custom}, + }}, + } + + testRunner, err := New(cfg, cfg.workingDir, custom) + t.CheckNoError(err) + err = testRunner.Test(context.Background(), ioutil.Discard, nil) + + t.CheckNoError(err) + }) +} + +func TestCustomCommandError(t *testing.T) { + tests := []struct { + description string + custom latest.CustomTest + shouldErr bool + expectedCmd string + expectedWindowsCmd string + expectedError string + }{ + { + description: "Non zero exit", + custom: latest.CustomTest{ + Command: "exit 20", + }, + shouldErr: true, + expectedCmd: "sh -c exit 20", + expectedWindowsCmd: "cmd.exe /C exit 20", + expectedError: "exit status 20", + }, + { + description: "Command timed out", + custom: latest.CustomTest{ + Command: "sleep 20", + TimeoutSeconds: 2, + }, + shouldErr: true, + expectedCmd: "sh -c sleep 20", + expectedWindowsCmd: "cmd.exe /C sleep 20", + expectedError: "context deadline exceeded", + }, + } + for _, test := range tests { + testutil.Run(t, "Testing new custom test runner", func(t *testutil.T) { + tmpDir := t.NewTempDir().Touch("test.yaml") + command := test.expectedCmd + if runtime.GOOS == Windows { + command = test.expectedWindowsCmd + } + t.Override(&util.DefaultExecCommand, testutil.CmdRunErr(command, fmt.Errorf(test.expectedError))) + + cfg := &mockConfig{ + workingDir: tmpDir.Root(), + tests: []*latest.TestCase{{ + ImageName: "image", + CustomTests: []latest.CustomTest{test.custom}, + }}, + } + + testRunner, err := New(cfg, cfg.workingDir, test.custom) + t.CheckNoError(err) + err = testRunner.Test(context.Background(), ioutil.Discard, nil) + + t.CheckError(test.shouldErr, err) + if test.expectedError != "" { + t.CheckErrorContains(test.expectedError, err) + } + }) + } +} + +func TestTestDependenciesCommand(t *testing.T) { + testutil.Run(t, "Testing new custom test runner", func(t *testutil.T) { + tmpDir := t.NewTempDir().Touch("test.yaml") + + custom := latest.CustomTest{ + Command: "echo Hello!", + Dependencies: &latest.CustomTestDependencies{ + Command: "echo [\"file1\",\"file2\",\"file3\"]", + }, + } + + cfg := &mockConfig{ + workingDir: tmpDir.Root(), + tests: []*latest.TestCase{{ + ImageName: "image", + CustomTests: []latest.CustomTest{custom}, + }}, + } + + if runtime.GOOS == Windows { + t.Override(&util.DefaultExecCommand, testutil.CmdRunOut( + "cmd.exe /C echo [\"file1\",\"file2\",\"file3\"]", + "[\"file1\",\"file2\",\"file3\"]", + )) + } else { + t.Override(&util.DefaultExecCommand, testutil.CmdRunOut( + "sh -c echo [\"file1\",\"file2\",\"file3\"]", + "[\"file1\",\"file2\",\"file3\"]", + )) + } + + expected := []string{"file1", "file2", "file3"} + testRunner, err := New(cfg, cfg.workingDir, custom) + t.CheckNoError(err) + deps, err := testRunner.TestDependencies() + + t.CheckNoError(err) + t.CheckDeepEqual(expected, deps) + }) +} + +func TestTestDependenciesPaths(t *testing.T) { + tests := []struct { + description string + ignore []string + paths []string + expected []string + shouldErr bool + }{ + { + description: "watch everything", + paths: []string{"."}, + expected: []string{"bar", filepath.FromSlash("baz/file"), "foo"}, + }, + { + description: "watch nothing", + }, + { + description: "ignore some paths", + paths: []string{"."}, + ignore: []string{"b*"}, + expected: []string{"foo"}, + }, + { + description: "glob", + paths: []string{"**"}, + expected: []string{"bar", filepath.FromSlash("baz/file"), "foo"}, + }, + { + description: "error", + paths: []string{"unknown"}, + shouldErr: true, + }, + } + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + // Directory structure: + // foo + // bar + // - baz + // file + tmpDir := t.NewTempDir(). + Touch("foo", "bar", "baz/file") + + custom := latest.CustomTest{ + Command: "echo Hello!", + Dependencies: &latest.CustomTestDependencies{ + Paths: test.paths, + Ignore: test.ignore, + }, + } + + cfg := &mockConfig{ + workingDir: tmpDir.Root(), + tests: []*latest.TestCase{{ + ImageName: "image", + CustomTests: []latest.CustomTest{custom}, + }}, + } + + testRunner, err := New(cfg, cfg.workingDir, custom) + t.CheckNoError(err) + deps, err := testRunner.TestDependencies() + + t.CheckErrorAndDeepEqual(test.shouldErr, err, test.expected, deps) + }) + } +} + +type mockConfig struct { + runcontext.RunContext // Embedded to provide the default values. + workingDir string + tests []*latest.TestCase +} diff --git a/pkg/skaffold/test/test_factory.go b/pkg/skaffold/test/test_factory.go index 5fe44c16ad5..55e98944869 100644 --- a/pkg/skaffold/test/test_factory.go +++ b/pkg/skaffold/test/test_factory.go @@ -28,6 +28,7 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/logfile" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/custom" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure" ) @@ -49,7 +50,6 @@ func NewTester(cfg Config, imagesAreLocal func(imageName string) (bool, error)) } return FullTester{ - // runners: getRunner(cfg, imagesAreLocal, cfg.TestCases()), runners: runner, muted: cfg.Muted(), }, nil @@ -115,11 +115,21 @@ func (t FullTester) runTests(ctx context.Context, out io.Writer, bRes []build.Ar func getRunner(cfg Config, imagesAreLocal func(imageName string) (bool, error), tcs []*latest.TestCase) ([]runner, error) { var runners []runner for _, tc := range tcs { - structureRunner, err := structure.New(cfg, cfg.GetWorkingDir(), tc, imagesAreLocal) - if err != nil { - return nil, err + if len(tc.StructureTests) != 0 { + structureRunner, err := structure.New(cfg, cfg.GetWorkingDir(), tc, imagesAreLocal) + if err != nil { + return nil, err + } + runners = append(runners, structureRunner) + } + + for _, customTest := range tc.CustomTests { + customRunner, err := custom.New(cfg, cfg.GetWorkingDir(), customTest) + if err != nil { + return nil, err + } + runners = append(runners, customRunner) } - runners = append(runners, structureRunner) } return runners, nil }