TEP-0125: Add credentials filter to entrypoint logger

This adds a filter for secret values to the logging mechanism in the pipeline entrypoint.

The secrets locations in the form of environment variables and files are detected by the controller
creating the pipeline step pod. The information about secrets attached to that pod are given
to the entrypoint as a json file.

From there the entrypoint will read the secret values from the given locations and redact all
occurences from the output log.

cmd/entrypoint/runner.go

@@ -30,6 +30,8 @@ import (
 	"sync"
 	"syscall"
 
+	"github.com/tektoncd/pipeline/pkg/apis/config"
+	"github.com/tektoncd/pipeline/pkg/credentials/filter"
 	"github.com/tektoncd/pipeline/pkg/entrypoint"
 	"github.com/tektoncd/pipeline/pkg/pod"
 )
@@ -82,30 +84,70 @@ func (rr *realRunner) Run(ctx context.Context, args ...string) error {
 	signal.Notify(rr.signals)
 	defer signal.Reset()
 
+	var err error
+
+	// determine std or file output
+	writeStdoutToFile := rr.stdoutPath != ""
+	writeStderrToFile := rr.stderrPath != ""
+	var stdoutWriter io.WriteCloser = os.Stdout
+	if writeStdoutToFile {
+		stdoutWriter, err = getFileWriter(rr.stdoutPath)
+		if err != nil {
+			return err
+		}
+	}
+	var stderrWriter io.WriteCloser = os.Stderr
+	if writeStderrToFile {
+		stderrWriter, err = getFileWriter(rr.stderrPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	// created filtered writers if credential filtering is active
+	isCredentialsFilterActive := os.Getenv(config.EnvEnableLoggingCredentialsFilter) == "true"
+	if isCredentialsFilterActive {
+		secretLocations, err := filter.NewSecretLocationsFromFile()
+		if err != nil {
+			return err
+		}
+
+		detectedSecrets, err := filter.DetectSecretsFromLocations(secretLocations)
+		if err != nil {
+			return err
+		}
+
+		stdoutWriter = filter.NewFilteredCredentialsWriter(detectedSecrets, stdoutWriter)
+		defer stdoutWriter.Close()
+		stderrWriter = filter.NewFilteredCredentialsWriter(detectedSecrets, stderrWriter)
+		defer stderrWriter.Close()
+	}
+
 	cmd := exec.CommandContext(ctx, name, args...)
 
 	// Build a list of tee readers that we'll read from after the command is
 	// is started. If we are not configured to tee stdout/stderr this will be
 	// empty and contents will not be copied.
 	var readers []*namedReader
-	if rr.stdoutPath != "" {
-		stdout, err := newTeeReader(cmd.StdoutPipe, rr.stdoutPath)
+	if writeStdoutToFile {
+		stdout, err := newTeeReader(cmd.StdoutPipe, rr.stdoutPath, stdoutWriter)
 		if err != nil {
 			return err
 		}
 		readers = append(readers, stdout)
 	} else {
 		// This needs to be set in an else since StdoutPipe will fail if cmd.Stdout is already set.
-		cmd.Stdout = os.Stdout
+		cmd.Stdout = stdoutWriter
 	}
-	if rr.stderrPath != "" {
-		stderr, err := newTeeReader(cmd.StderrPipe, rr.stderrPath)
+	if writeStderrToFile {
+		stderr, err := newTeeReader(cmd.StderrPipe, rr.stderrPath, stderrWriter)
 		if err != nil {
 			return err
 		}
 		readers = append(readers, stderr)
 	} else {
-		cmd.Stderr = os.Stderr
+		// This needs to be set in an else since StdoutPipe will fail if cmd.Stdout is already set.
+		cmd.Stderr = stderrWriter
 	}
 
 	// dedicated PID group used to forward signals to
@@ -168,12 +210,19 @@ func (rr *realRunner) Run(ctx context.Context, args ...string) error {
 // be used for multiple streams if desired.
 // The behavior of the Reader is the same as io.TeeReader - reads from the pipe
 // will be written to the file.
-func newTeeReader(pipe func() (io.ReadCloser, error), path string) (*namedReader, error) {
+func newTeeReader(pipe func() (io.ReadCloser, error), path string, writer io.WriteCloser) (*namedReader, error) {
 	in, err := pipe()
 	if err != nil {
 		return nil, fmt.Errorf("error creating pipe: %w", err)
 	}
 
+	return &namedReader{
+		name:   path,
+		Reader: io.TeeReader(in, writer),
+	}, nil
+}
+
+func getFileWriter(path string) (io.WriteCloser, error) {
 	if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
 		return nil, fmt.Errorf("error creating parent directory: %w", err)
 	}
@@ -182,10 +231,7 @@ func newTeeReader(pipe func() (io.ReadCloser, error), path string) (*namedReader
 		return nil, fmt.Errorf("error opening %s: %w", path, err)
 	}
 
-	return &namedReader{
-		name:   path,
-		Reader: io.TeeReader(in, f),
-	}, nil
+	return f, nil
 }
 
 // namedReader is just a helper struct that lets us give a reader a name for

cmd/entrypoint/runner_test.go

@@ -11,6 +11,9 @@ import (
 	"syscall"
 	"testing"
 	"time"
+
+	"github.com/tektoncd/pipeline/pkg/apis/config"
+	"github.com/tektoncd/pipeline/pkg/credentials/filter"
 )
 
 // TestRealRunnerSignalForwarding will artificially put an interrupt signal (SIGINT) in the rr.signals chan.
@@ -134,3 +137,41 @@ func TestRealRunnerTimeout(t *testing.T) {
 		t.Fatalf("step didn't timeout")
 	}
 }
+
+// TestRealRunnerCredentialsFilter tests whether the credentials filter is correctly applied.
+func TestRealRunnerCredentialsFilter(t *testing.T) {
+	rr := realRunner{
+		stdoutPath: "stdoutfile",
+	}
+	defer os.Remove("stdoutfile")
+	os.Setenv(config.EnvEnableLoggingCredentialsFilter, "true")
+	defer os.Unsetenv(config.EnvEnableLoggingCredentialsFilter)
+	os.Setenv("SECRET", "supersecret")
+	defer os.Unsetenv(config.EnvEnableLoggingCredentialsFilter)
+
+	secretLocations := filter.SecretLocations{
+		EnvironmentVariables: []string{"SECRET"},
+	}
+
+	cleanup, err := filter.WriteSecretLocationsToFile(&secretLocations)
+	defer cleanup()
+	if err != nil {
+		t.Fatalf("could not write secret locations to file: %v", err)
+	}
+
+	err = rr.Run(context.Background(), "echo", "this writes a supersecret value to stdout")
+	if err != nil {
+		t.Fatalf("runner returned unexpected error: %v", err)
+	}
+
+	bytes, err := os.ReadFile("stdoutfile")
+	if err != nil {
+		t.Fatalf("could not read output stream from file: %v", err)
+	}
+
+	stdoutContent := strings.TrimRight(string(bytes), " \n")
+	expectedStdout := "this writes a [REDACTED:SECRET] value to stdout"
+	if stdoutContent != expectedStdout {
+		t.Errorf("expected stdout '%s' but got '%s'", expectedStdout, stdoutContent)
+	}
+}

config/config-feature-flags.yaml

@@ -90,3 +90,10 @@ data:
   # in the TaskRun/PipelineRun such as the source from where a remote Task/Pipeline
   # definition was fetched.
   enable-provenance-in-status: "false"
+  # Setting this flag to "true" will enable filtering credentials from the log output
+  # of pipeline runs. This will redact all secret data attached to the pod.
+  # For a detailed description of what that means see the docs section about credential
+  # filtering at https://github.com/tektoncd/pipeline/blob/main/docs/logs.md.
+  # This is an experimental feature and thus should still be considered
+  # an alpha feature.
+  enable-logging-credentials-filter: "false"

docs/developers/README.md

@@ -20,3 +20,4 @@ channel for training and tutorials on Tekton!
 - How specific features are implemented:
   - [PipelineResources (deprecated)](./pipelineresources.md)
   - [Results](./results-lifecycle.md)
+  - [Credential Filtering](./credential-filtering.md)

docs/developers/credential-filtering.md

@@ -0,0 +1,64 @@
+# Logs Credential Filter (TEP-0125)
+
+Often it is the case that secret values will sneak into the ouput logs of your pipelines. You can either handle this manually by avoiding to print them in the output or use the feature for filtering secret values from the output log. To enable this feature set the `enable-logging-credentials-filter` value in the `feature-flags` config map to `"true"` (see [Customizing the Pipelines Controller behavior](install.md#customizing-the-pipelines-controller-behavior)).
+
+The tekton controller looks for secret refs in volumes and env and also for CSI volumes referencing secrets in the definition of the pipeline pod. It saves the information in form of the names of environment variables and paths to files with secrets. The pod can then read those values and redact them.
+
+This design does not transmit actual secrets and the pod does not need access to the API sever to retrieve them. It tells the pipeline pod only the places where it can find secrets attached to it. And those are the secrets that will be redacted. Because only those can be used in the corresponding pipeline step.
+
+The credential filter will redact values contained in those secrets from the output log stream and replace them with `[REDACTED:<secret-name>]`.
+
+## Secret Detection Logic
+
+While creating the pod for the pipeline step, the controller will try to detect all secrets attached to the pod. Usually secrets are attached to the pod either by setting environment variables or mounting files into it. These detected secret locations are then transmitted to the pipeline pod for credential filtering.
+
+### Secret Locations
+
+The secret locations detected by the controller are transmitted to the entrypoint as environment variable in json format with the following format:
+
+```json
+{
+  "environmentVariables": ["ENV1", "ENV2"],
+  "files": ["/path/to/secre1", "/path/to/secret2"]
+}
+```
+
+The entrypoint can then read the environment variables and file contents and redact them from the output log stream.
+
+### Secrets stored in Environment Variables
+
+The controller detects secrets stored as environment variables from the following pod syntax:
+
+```yaml
+env:
+- name: MY_SECRET_VALUE_IN_ENV
+  valueFrom:
+    secretKeyRef:
+      name: my-k8s-secret
+      key: secret_value_key
+```
+
+The secret key reference is the trigger to detect an environment variable that contains a secret and that needs to be redacted.
+
+### Secrets mounted as Files 
+
+Secrets can also be mounted into pods via files in different ways. Currently the following pod syntax is supported in the secret detection logic.
+
+```yaml
+volumes:
+- name: secret-volume
+  secret:
+    secretName: my-k8s-secret
+```
+
+```yaml
+volumes:
+- name: secret-volume-csi
+  csi:
+    driver: secrets-store.csi.k8s.io
+    readOnly: true
+    volumeAttributes:
+      secretProviderClass: secret-provider-class
+```
+
+Currently classic secret volumes and csi secret volumes with the driver `secrets-store.csi.k8s.io` are supported. In both cases the volume directories or items in that volume will be added to the detected secret locations.

docs/developers/taskruns.md

@@ -93,8 +93,10 @@ Here is an example of a directory layout for a simple Task with 2 script steps:
 |-- downward
 |   |-- ..2021_09_16_18_31_06.270542700
 |   |   `-- ready
+|   |   `-- secret-locations.json
 |   |-- ..data -> ..2021_09_16_18_31_06.270542700
 |   `-- ready -> ..data/ready
+|   `-- secret-locations.json -> ..data/secret-locations.json
 |-- home
 |-- results
 |-- run

docs/install.md

@@ -9,28 +9,27 @@ weight: 100
 
 This guide explains how to install Tekton Pipelines. It covers the following topics:
 
-- [Before you begin](#before-you-begin)
-- [Installing Tekton Pipelines on Kubernetes](#installing-tekton-pipelines-on-kubernetes)
+- [Installing Tekton Pipelines](#installing-tekton-pipelines)
+  - [Before you begin](#before-you-begin)
+  - [Installing Tekton Pipelines on Kubernetes](#installing-tekton-pipelines-on-kubernetes)
     - [Installing Tekton Pipelines on OpenShift](#installing-tekton-pipelines-on-openshift)
-- [Configuring PipelineResource storage](#configuring-pipelineresource-storage)
+  - [Configuring PipelineResource storage](#configuring-pipelineresource-storage)
     - [Configuring a persistent volume](#configuring-a-persistent-volume)
     - [Configuring a cloud storage bucket](#configuring-a-cloud-storage-bucket)
         - [Example configuration for an S3 bucket](#example-configuration-for-an-s3-bucket)
         - [Example configuration for a GCS bucket](#example-configuration-for-a-gcs-bucket)
 - [Configuring CloudEvents notifications](#configuring-cloudevents-notifications)
-- [Installing and configuring remote Task and Pipeline resolution](#installing-and-configuring-remote-task-and-pipeline-resolution)
 - [Configuring self-signed cert for private registry](#configuring-self-signed-cert-for-private-registry)
 - [Customizing basic execution parameters](#customizing-basic-execution-parameters)
     - [Customizing the Pipelines Controller behavior](#customizing-the-pipelines-controller-behavior)
     - [Alpha Features](#alpha-features)
-    - [Beta Features](#beta-features)
 - [Configuring High Availability](#configuring-high-availability)
 - [Configuring tekton pipeline controller performance](#configuring-tekton-pipeline-controller-performance)
 - [Creating a custom release of Tekton Pipelines](#creating-a-custom-release-of-tekton-pipelines)
 - [Verify Tekton Pipelines release](#verify-tekton-pipelines-release)
     - [Verify signatures using `cosign`](#verify-signatures-using-cosign)
-    - [Verify the tansparency logs using `rekor-cli`](#verify-the-transparency-logs-using-rekor-cli)
-- [Next steps](#next-steps)
+    - [Verify the transparency logs using `rekor-cli`](#verify-the-transparency-logs-using-rekor-cli)
+  - [Next steps](#next-steps)
 
 ## Before you begin
 
@@ -427,6 +426,8 @@ features](#alpha-features) to be used.
   field contains metadata about resources used in the TaskRun/PipelineRun such as the 
   source from where a remote Task/Pipeline definition was fetched.
 
+- `enable-logging-credentials-filter`: set this flag to "true" to enable filtering credentials from the log output of pipeline runs. This will redact all secrets attached to the pod from the output stream. See the section about [Logs Credential Filter](logs.md) for more details.
+
 For example:
 
 ```yaml

docs/logs.md

@@ -34,3 +34,76 @@ You can get execution logs using one of the following methods:
 - Get the logs using [Tekton Dashboard](https://github.com/tektoncd/dashboard).
 
 - Configure an external service to consume and display the logs. For example, [ElasticSearch, Beats, and Kibana](https://github.com/mgreau/tekton-pipelines-elastic-tutorials).
+
+
+# Logs Credential Filter
+
+Often it is the case that secret values will sneak into the ouput logs of your pipelines. You can either handle this manually by avoiding to print them in the output or use the feature for filtering secret values from the output log. To enable this feature set the `enable-logging-credentials-filter` value in the `feature-flags` config map to `"true"` (see [Customizing the Pipelines Controller behavior](install.md#customizing-the-pipelines-controller-behavior)).
+
+This design does not transmit actual secrets and the pod does not need access to the API sever to retrieve them. It tells the pipeline pod only the places where it can find secrets attached to it. And those are the secrets that will be redacted. Because only those can be used in the corresponding pipeline step.
+
+The credential filter will redact values contained in those secrets from the output log stream and replace them with `[REDACTED:<secret-name>]`.
+
+## Secret Detection Logic
+
+While creating the pod for the pipeline step, the controller will try to detect all secrets attached to the pod. Usually secrets are attached to the pod either by setting environment variables or mounting files into it. These detected secret locations are then transmitted to the pipeline pod for credential filtering.
+
+### Secrets stored in Environment Variables
+
+The controller detects secrets stored as environment variables from the following pod syntax:
+
+```yaml
+env:
+- name: MY_SECRET_VALUE_IN_ENV
+  valueFrom:
+    secretKeyRef:
+      name: my-k8s-secret
+      key: secret_value_key
+```
+
+The secret key reference is the trigger to detect an environment variable that contains a secret and that needs to be redacted.
+
+### Secrets mounted as Files 
+
+Secrets can also be mounted into pods via files in different ways. Currently the following pod syntax is supported in the secret detection logic.
+
+```yaml
+volumes:
+- name: secret-volume
+  secret:
+    secretName: my-k8s-secret
+```
+
+```yaml
+volumes:
+- name: secret-volume-projected
+  projected:
+    sources:
+    - secret:
+        name: my-k8s-secret
+```
+
+```yaml
+volumes:
+- name: secret-volume-csi
+  csi:
+    driver: secrets-store.csi.k8s.io
+    readOnly: true
+    volumeAttributes:
+      secretProviderClass: secret-provider-class
+```
+
+Currently classic secret volumes and csi secret volumes with the driver `secrets-store.csi.k8s.io` are supported. In both cases the volume directories or items in that volume will be added to the detected secret locations.
+
+### Secrets mounted through workspaces
+
+Workspaces are creating the same volume and volume mount structure as the syntax above. Therefore they are supported in the same way.
+
+For example the secret from the following workspace will also be redacted:
+
+```yaml
+workspaces:
+  - name: my-secret-workspace
+    secret:
+      secretName: my-k8s-secret
+```