From 23510054a1d94cc7d943e4a9025756bcac3a5b3c Mon Sep 17 00:00:00 2001 From: Olivier Thomann Date: Tue, 14 Jan 2020 12:14:52 -0500 Subject: [PATCH] Changes to add results to a task --- cmd/entrypoint/main.go | 19 ++- docs/developers/README.md | 2 +- pkg/apis/pipeline/paths.go | 2 + pkg/apis/pipeline/v1alpha1/resource_types.go | 4 + pkg/apis/pipeline/v1alpha1/task_types.go | 12 ++ .../v1alpha1/zz_generated.deepcopy.go | 21 +++ pkg/entrypoint/entrypointer.go | 39 +++++ pkg/pod/entrypoint.go | 23 ++- pkg/pod/entrypoint_test.go | 152 +++++++++++++++++- pkg/pod/pod.go | 2 +- 10 files changed, 262 insertions(+), 14 deletions(-) diff --git a/cmd/entrypoint/main.go b/cmd/entrypoint/main.go index 3392e07917c..82b02b2e330 100644 --- a/cmd/entrypoint/main.go +++ b/cmd/entrypoint/main.go @@ -25,16 +25,17 @@ import ( "syscall" "time" + "github.com/tektoncd/pipeline/pkg/apis/pipeline" "github.com/tektoncd/pipeline/pkg/entrypoint" ) var ( - ep = flag.String("entrypoint", "", "Original specified entrypoint to execute") - waitFiles = flag.String("wait_file", "", "Comma-separated list of paths to wait for") - waitFileContent = flag.Bool("wait_file_content", false, "If specified, expect wait_file to have content") - postFile = flag.String("post_file", "", "If specified, file to write upon completion") - terminationPath = flag.String("termination_path", "/tekton/termination", "If specified, file to write upon termination") - + ep = flag.String("entrypoint", "", "Original specified entrypoint to execute") + waitFiles = flag.String("wait_file", "", "Comma-separated list of paths to wait for") + waitFileContent = flag.Bool("wait_file_content", false, "If specified, expect wait_file to have content") + postFile = flag.String("post_file", "", "If specified, file to write upon completion") + terminationPath = flag.String("termination_path", "/tekton/termination", "If specified, file to write upon termination") + results = flag.String("results", "", "If specified, list of file names that might contain task results") waitPollingInterval = time.Second ) @@ -51,6 +52,12 @@ func main() { Waiter: &realWaiter{}, Runner: &realRunner{}, PostWriter: &realPostWriter{}, + Results: strings.Split(*results, ","), + } + if len(e.Results) != 0 { + if err := os.MkdirAll(pipeline.DefaultResultPath, 0755); err != nil { + log.Fatalf("Error creating the results directory: %v", err) + } } if err := e.Go(); err != nil { switch t := err.(type) { diff --git a/docs/developers/README.md b/docs/developers/README.md index 4db81961d49..e38ad8baea3 100644 --- a/docs/developers/README.md +++ b/docs/developers/README.md @@ -185,7 +185,7 @@ There are known issues with the existing implementation of sidecars: - When the `nop` image does provide the sidecar's command, the sidecar will continue to run even after `nop` has been swapped into the sidecar container's image -field. See https://github.com/tektoncd/pipeline/issues/1347 for the issue +field. See for the issue tracking this bug. Until this issue is resolved the best way to avoid it is to avoid overriding the `nop` image when deploying the tekton controller, or ensuring that the overridden `nop` image contains as few commands as possible. diff --git a/pkg/apis/pipeline/paths.go b/pkg/apis/pipeline/paths.go index 1016faab10a..37d8305aa7e 100644 --- a/pkg/apis/pipeline/paths.go +++ b/pkg/apis/pipeline/paths.go @@ -19,4 +19,6 @@ package pipeline const ( // WorkspaceDir is the root directory used for PipelineResources and (by default) Workspaces WorkspaceDir = "/workspace" + // DefaultResultPath is the path for task result + DefaultResultPath = "/tekton/results" ) diff --git a/pkg/apis/pipeline/v1alpha1/resource_types.go b/pkg/apis/pipeline/v1alpha1/resource_types.go index 0c9d2ec9e32..f32704bf552 100644 --- a/pkg/apis/pipeline/v1alpha1/resource_types.go +++ b/pkg/apis/pipeline/v1alpha1/resource_types.go @@ -123,8 +123,12 @@ type PipelineResourceResult struct { Key string `json:"key"` Value string `json:"value"` ResourceRef PipelineResourceRef `json:"resourceRef,omitempty"` + ResultType ResultType `json:"type,omitempty"` } +// ResultType used to find out whether a PipelineResourceResult is from a task result or not +type ResultType string + // ResourceFromType returns an instance of the correct PipelineResource object type which can be // used to add input and output containers as well as volumes to a TaskRun's pod in order to realize // a PipelineResource in a pod. diff --git a/pkg/apis/pipeline/v1alpha1/task_types.go b/pkg/apis/pipeline/v1alpha1/task_types.go index e2cdf5eca22..69123d0a6cf 100644 --- a/pkg/apis/pipeline/v1alpha1/task_types.go +++ b/pkg/apis/pipeline/v1alpha1/task_types.go @@ -64,6 +64,18 @@ type TaskSpec struct { // Workspaces are the volumes that this Task requires. Workspaces []WorkspaceDeclaration `json:"workspaces,omitempty"` + + // Results are values that this Task can output + Results []TaskResult `json:"results,omitempty"` +} + +// TaskResult used to describe the results of a task +type TaskResult struct { + // Name the given name + Name string `json:"name"` + + // Description the given description + Description string `json:"description"` } // Step embeds the Container type, which allows it to include fields not diff --git a/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go index 42aa8ba2a60..7c808b7f570 100644 --- a/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pipeline/v1alpha1/zz_generated.deepcopy.go @@ -1378,6 +1378,22 @@ func (in *TaskResourceBinding) DeepCopy() *TaskResourceBinding { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaskResult) DeepCopyInto(out *TaskResult) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskResult. +func (in *TaskResult) DeepCopy() *TaskResult { + if in == nil { + return nil + } + out := new(TaskResult) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TaskRun) DeepCopyInto(out *TaskRun) { *out = *in @@ -1650,6 +1666,11 @@ func (in *TaskSpec) DeepCopyInto(out *TaskSpec) { *out = make([]v1alpha2.WorkspaceDeclaration, len(*in)) copy(*out, *in) } + if in.Results != nil { + in, out := &in.Results, &out.Results + *out = make([]TaskResult, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/entrypoint/entrypointer.go b/pkg/entrypoint/entrypointer.go index 39cbc248e8b..e70dddc83aa 100644 --- a/pkg/entrypoint/entrypointer.go +++ b/pkg/entrypoint/entrypointer.go @@ -18,8 +18,12 @@ package entrypoint import ( "fmt" + "io/ioutil" + "os" + "path/filepath" "time" + "github.com/tektoncd/pipeline/pkg/apis/pipeline" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" "github.com/tektoncd/pipeline/pkg/logging" "github.com/tektoncd/pipeline/pkg/termination" @@ -51,6 +55,9 @@ type Entrypointer struct { Runner Runner // PostWriter encapsulates writing files when complete. PostWriter PostWriter + + // Results is the set of files that might contain task results + Results []string } // Waiter encapsulates waiting for files to exist. @@ -100,12 +107,44 @@ func (e Entrypointer) Go() error { // Write the post file *no matter what* e.WritePostFile(e.PostFile, err) + if e.Results != nil { + if err := e.readResultsFromDisk(); err != nil { + logger.Fatalf("Error while handling results: %s", err) + } + } if wErr := termination.WriteMessage(e.TerminationPath, output); wErr != nil { logger.Fatalf("Error while writing message: %s", wErr) } return err } +func (e Entrypointer) readResultsFromDisk() error { + output := []v1alpha1.PipelineResourceResult{} + for _, resultFile := range e.Results { + // check result file + fileContents, err := ioutil.ReadFile(filepath.Join(pipeline.DefaultResultPath, resultFile)) + if os.IsNotExist(err) { + continue + } else if err != nil { + return err + } + // if the file doesn't exist, ignore it + output = append(output, v1alpha1.PipelineResourceResult{ + Key: resultFile, + Value: string(fileContents), + ResultType: "result", + }) + } + // push output to termination path + if len(output) != 0 { + if err := termination.WriteMessage(e.TerminationPath, output); err != nil { + return err + } + } + return nil +} + +// WritePostFile write the postfile func (e Entrypointer) WritePostFile(postFile string, err error) { if err != nil && postFile != "" { postFile = fmt.Sprintf("%s.err", postFile) diff --git a/pkg/pod/entrypoint.go b/pkg/pod/entrypoint.go index 8686ff2627d..16d5afa49b2 100644 --- a/pkg/pod/entrypoint.go +++ b/pkg/pod/entrypoint.go @@ -22,6 +22,7 @@ import ( "path/filepath" "strings" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -85,8 +86,8 @@ var ( // method, using entrypoint_lookup.go. // // TODO(#1605): Also use entrypoint injection to order sidecar start/stop. -func orderContainers(entrypointImage string, steps []corev1.Container) (corev1.Container, []corev1.Container, error) { - toolsInit := corev1.Container{ +func orderContainers(entrypointImage string, steps []corev1.Container, results []v1alpha1.TaskResult) (corev1.Container, []corev1.Container, error) { + initContainer := corev1.Container{ Name: "place-tools", Image: entrypointImage, Command: []string{"cp", "/ko-app/entrypoint", entrypointBinary}, @@ -117,6 +118,7 @@ func orderContainers(entrypointImage string, steps []corev1.Container) (corev1.C "-termination_path", terminationPath, } } + argsForEntrypoint = append(argsForEntrypoint, resultArgument(steps, results)...) cmd, args := s.Command, s.Args if len(cmd) == 0 { @@ -137,7 +139,22 @@ func orderContainers(entrypointImage string, steps []corev1.Container) (corev1.C // Mount the Downward volume into the first step container. steps[0].VolumeMounts = append(steps[0].VolumeMounts, downwardMount) - return toolsInit, steps, nil + return initContainer, steps, nil +} + +func resultArgument(steps []corev1.Container, results []v1alpha1.TaskResult) []string { + if results == nil || len(results) == 0 { + return nil + } + return []string{"-results", collectResultsName(results)} +} + +func collectResultsName(results []v1alpha1.TaskResult) string { + var resultNames []string + for _, r := range results { + resultNames = append(resultNames, r.Name) + } + return strings.Join(resultNames, ",") } // UpdateReady updates the Pod's annotations to signal the first step to start diff --git a/pkg/pod/entrypoint_test.go b/pkg/pod/entrypoint_test.go index 2b5d150ccd8..4a726e384c9 100644 --- a/pkg/pod/entrypoint_test.go +++ b/pkg/pod/entrypoint_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" fakek8s "k8s.io/client-go/kubernetes/fake" @@ -85,7 +86,7 @@ func TestOrderContainers(t *testing.T) { VolumeMounts: []corev1.VolumeMount{toolsMount}, TerminationMessagePath: "/tekton/termination", }} - gotInit, got, err := orderContainers(images.EntrypointImage, steps) + gotInit, got, err := orderContainers(images.EntrypointImage, steps, nil) if err != nil { t.Fatalf("orderContainers: %v", err) } @@ -93,17 +94,162 @@ func TestOrderContainers(t *testing.T) { t.Errorf("Diff (-want, +got): %s", d) } - wantInit := corev1.Container{ + wantInit := []corev1.Container{{ Name: "place-tools", Image: images.EntrypointImage, Command: []string{"cp", "/ko-app/entrypoint", entrypointBinary}, VolumeMounts: []corev1.VolumeMount{toolsMount}, - } + }} if d := cmp.Diff(wantInit, gotInit); d != "" { t.Errorf("Init Container Diff (-want, +got): %s", d) } } +func TestEntryPointResults(t *testing.T) { + results := []v1alpha1.TaskResult{{ + Name: "sum", + Description: "This is the sum result of the task", + }, { + Name: "sub", + Description: "This is the sub result of the task", + }} + + steps := []corev1.Container{{ + Image: "step-1", + Command: []string{"cmd"}, + Args: []string{"arg1", "arg2"}, + }, { + Image: "step-2", + Command: []string{"cmd1", "cmd2", "cmd3"}, // multiple cmd elements + Args: []string{"arg1", "arg2"}, + VolumeMounts: []corev1.VolumeMount{volumeMount}, // pre-existing volumeMount + }, { + Image: "step-3", + Command: []string{"cmd"}, + Args: []string{"arg1", "arg2"}, + }} + want := []corev1.Container{{ + Image: "step-1", + Command: []string{entrypointBinary}, + Args: []string{ + "-wait_file", "/tekton/downward/ready", + "-wait_file_content", + "-post_file", "/tekton/tools/0", + "-termination_path", "/tekton/termination", + "-results", "sum,sub", + "-entrypoint", "cmd", "--", + "arg1", "arg2", + }, + VolumeMounts: []corev1.VolumeMount{toolsMount, downwardMount}, + TerminationMessagePath: "/tekton/termination", + }, { + Image: "step-2", + Command: []string{entrypointBinary}, + Args: []string{ + "-wait_file", "/tekton/tools/0", + "-post_file", "/tekton/tools/1", + "-termination_path", "/tekton/termination", + "-results", "sum,sub", + "-entrypoint", "cmd1", "--", + "cmd2", "cmd3", + "arg1", "arg2", + }, + VolumeMounts: []corev1.VolumeMount{volumeMount, toolsMount}, + TerminationMessagePath: "/tekton/termination", + }, { + Image: "step-3", + Command: []string{entrypointBinary}, + Args: []string{ + "-wait_file", "/tekton/tools/1", + "-post_file", "/tekton/tools/2", + "-termination_path", "/tekton/termination", + "-results", "sum,sub", + "-entrypoint", "cmd", "--", + "arg1", "arg2", + }, + VolumeMounts: []corev1.VolumeMount{toolsMount}, + TerminationMessagePath: "/tekton/termination", + }} + _, got, err := orderContainers(images.EntrypointImage, steps, results) + if err != nil { + t.Fatalf("orderContainers: %v", err) + } + if d := cmp.Diff(want, got); d != "" { + t.Errorf("Diff (-want, +got): %s", d) + } +} + +func TestEntryPointResultsSingleStep(t *testing.T) { + results := []v1alpha1.TaskResult{{ + Name: "sum", + Description: "This is the sum result of the task", + }, { + Name: "sub", + Description: "This is the sub result of the task", + }} + + steps := []corev1.Container{{ + Image: "step-1", + Command: []string{"cmd"}, + Args: []string{"arg1", "arg2"}, + }} + want := []corev1.Container{{ + Image: "step-1", + Command: []string{entrypointBinary}, + Args: []string{ + "-wait_file", "/tekton/downward/ready", + "-wait_file_content", + "-post_file", "/tekton/tools/0", + "-termination_path", "/tekton/termination", + "-results", "sum,sub", + "-entrypoint", "cmd", "--", + "arg1", "arg2", + }, + VolumeMounts: []corev1.VolumeMount{toolsMount, downwardMount}, + TerminationMessagePath: "/tekton/termination", + }} + _, got, err := orderContainers(images.EntrypointImage, steps, results) + if err != nil { + t.Fatalf("orderContainers: %v", err) + } + if d := cmp.Diff(want, got); d != "" { + t.Errorf("Diff (-want, +got): %s", d) + } +} +func TestEntryPointSingleResultsSingleStep(t *testing.T) { + results := []v1alpha1.TaskResult{{ + Name: "sum", + Description: "This is the sum result of the task", + }} + + steps := []corev1.Container{{ + Image: "step-1", + Command: []string{"cmd"}, + Args: []string{"arg1", "arg2"}, + }} + want := []corev1.Container{{ + Image: "step-1", + Command: []string{entrypointBinary}, + Args: []string{ + "-wait_file", "/tekton/downward/ready", + "-wait_file_content", + "-post_file", "/tekton/tools/0", + "-termination_path", "/tekton/termination", + "-results", "sum", + "-entrypoint", "cmd", "--", + "arg1", "arg2", + }, + VolumeMounts: []corev1.VolumeMount{toolsMount, downwardMount}, + TerminationMessagePath: "/tekton/termination", + }} + _, got, err := orderContainers(images.EntrypointImage, steps, results) + if err != nil { + t.Fatalf("orderContainers: %v", err) + } + if d := cmp.Diff(want, got); d != "" { + t.Errorf("Diff (-want, +got): %s", d) + } +} func TestUpdateReady(t *testing.T) { for _, c := range []struct { desc string diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go index bd3d058a3bc..8b6b2886e3d 100644 --- a/pkg/pod/pod.go +++ b/pkg/pod/pod.go @@ -112,7 +112,7 @@ func MakePod(images pipeline.Images, taskRun *v1alpha1.TaskRun, taskSpec v1alpha // Rewrite steps with entrypoint binary. Append the entrypoint init // container to place the entrypoint binary. - entrypointInit, stepContainers, err := orderContainers(images.EntrypointImage, stepContainers) + entrypointInit, stepContainers, err := orderContainers(images.EntrypointImage, stepContainers, taskSpec.Results) if err != nil { return nil, err }