From ff8d74fc7a2074eaf423347c3c37a5249b1660ef Mon Sep 17 00:00:00 2001 From: Khurram Baig Date: Wed, 23 Aug 2023 14:05:25 +0530 Subject: [PATCH] Add Google Creds for Results Enables passing of GCS creds using k8s secret. --- docs/TektonResult.md | 19 ++++ .../operator/v1alpha1/tektonresult_types.go | 8 +- .../testdata/api-deployment-gcs.yaml | 83 ++++++++++++++ .../kubernetes/tektonresult/transform.go | 107 ++++++++++++++++-- .../kubernetes/tektonresult/transform_test.go | 41 +++++++ 5 files changed, 249 insertions(+), 9 deletions(-) create mode 100644 pkg/reconciler/kubernetes/tektonresult/testdata/api-deployment-gcs.yaml diff --git a/docs/TektonResult.md b/docs/TektonResult.md index f82a3f9c3b..b9b67c0115 100644 --- a/docs/TektonResult.md +++ b/docs/TektonResult.md @@ -148,3 +148,22 @@ stringData: S3_MULTI_PART_SIZE: "5242880" ``` + +### GCS specific Property +The follow keys are needed for GCS: +* `gcs_creds_secret_name` +* `gcc_creds_secret_key` +* `gcs_bucket_name` + +We need to create a secret with google application creds for a bucket `foo-bar` like below: + +``` +kubectl create secret generic gcs-credentials --from-file=creds.json +``` + +In the above example, our properties are: +``` +gcs_creds_secret_name: gcs-credentials +gcc_creds_secret_key: creds.json +gcs_bucket_name: foo-bar +``` diff --git a/pkg/apis/operator/v1alpha1/tektonresult_types.go b/pkg/apis/operator/v1alpha1/tektonresult_types.go index 49837e0d33..2bb191ba20 100644 --- a/pkg/apis/operator/v1alpha1/tektonresult_types.go +++ b/pkg/apis/operator/v1alpha1/tektonresult_types.go @@ -82,10 +82,14 @@ type ResultsAPIProperties struct { S3AccessKeyID string `json:"s3_access_key_id,omitempty"` S3SecretAccessKey string `json:"s3_secret_access_key,omitempty"` S3MultiPartSize int64 `json:"s3_multi_part_size,omitempty"` - LoggingPVCName string `json:"logging_pvc_name"` + LoggingPVCName string `json:"logging_pvc_name,omitempty"` + GcsBucketName string `json:"gcs_bucket_name,omitempty"` + StorageEmulatorHost string `json:"storage_emulator_host,omitempty"` // name of the secret used to get S3 credentials and // pass it as environment variables to the "tekton-results-api" deployment under "api" container - SecretName string `json:"secret_name,omitempty"` + SecretName string `json:"secret_name,omitempty"` + GCSCredsSecretName string `json:"gcs_creds_secret_name,omitempty"` + GCSCredsSecretKey string `json:"gcs_creds_secret_key,omitempty"` } // TektonResultStatus defines the observed state of TektonResult diff --git a/pkg/reconciler/kubernetes/tektonresult/testdata/api-deployment-gcs.yaml b/pkg/reconciler/kubernetes/tektonresult/testdata/api-deployment-gcs.yaml new file mode 100644 index 0000000000..53ff1bd3ba --- /dev/null +++ b/pkg/reconciler/kubernetes/tektonresult/testdata/api-deployment-gcs.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: tekton-results-api + app.kubernetes.io/part-of: tekton-results + app.kubernetes.io/version: 9f84a1f + name: tekton-results-api + namespace: tekton-pipelines +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: tekton-results-api + app.kubernetes.io/version: 9f84a1f + template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "false" + labels: + app.kubernetes.io/name: tekton-results-api + app.kubernetes.io/version: 9f84a1f + spec: + containers: + - env: + - name: test + value: tekton-results-postgres-service.tekton-pipelines.svc.cluster.local + image: gcr.io/tekton-releases/github.com/tektoncd/results/cmd/api:9f84a1f@sha256:606816e51ebecb58fccc28f5a95699255ed8742470df673294ce25f69ffc451c + name: test + - env: + - name: DB_HOST + value: tekton-results-postgres-service.tekton-pipelines.svc.cluster.local + - name: DB_USER + valueFrom: + secretKeyRef: + key: POSTGRES_USER + name: tekton-results-postgres + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + key: POSTGRES_PASSWORD + name: tekton-results-postgres + - name: DB_NAME + value: tekton-results + - name: S3_BUCKET_NAME # used to verify that, this value replaced by transformer + value: foo + - name: GOOGLE_APPLICATION_CREDENTIALS + value: creds.json + image: gcr.io/tekton-releases/github.com/tektoncd/results/cmd/api:9f84a1f@sha256:606816e51ebecb58fccc28f5a95699255ed8742470df673294ce25f69ffc451c + name: api + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /etc/tekton/results + name: config + readOnly: true + - mountPath: /etc/tls + name: tls + readOnly: true + - mountPath: /creds/google + name: google-creds + serviceAccountName: tekton-results-api + volumes: + - configMap: + name: tekton-results-api-config + name: config + - name: tls + secret: + secretName: tekton-results-tls + - name: google-creds + secret: + secretName: foo + items: + - key: bar + path: bar diff --git a/pkg/reconciler/kubernetes/tektonresult/transform.go b/pkg/reconciler/kubernetes/tektonresult/transform.go index 14b935e960..732ed8d96f 100644 --- a/pkg/reconciler/kubernetes/tektonresult/transform.go +++ b/pkg/reconciler/kubernetes/tektonresult/transform.go @@ -36,13 +36,16 @@ import ( const ( // Results ConfigMap - configAPI = "tekton-results-api-config" - deploymentAPI = "tekton-results-api" - configINFO = "tekton-results-info" - configMetrics = "tekton-results-config-observability" - configPostgresDB = "tekton-results-postgres" - pvcLoggingVolume = "tekton-logs" - apiContainerName = "api" + configAPI = "tekton-results-api-config" + deploymentAPI = "tekton-results-api" + configINFO = "tekton-results-info" + configMetrics = "tekton-results-config-observability" + configPostgresDB = "tekton-results-postgres" + pvcLoggingVolume = "tekton-logs" + apiContainerName = "api" + googleAPPCredsEnvName = "GOOGLE_APPLICATION_CREDENTIALS" + googleCredsVolName = "google-creds" + googleCredsPath = "/creds/google" ) var ( @@ -73,6 +76,7 @@ func (r *Reconciler) transform(ctx context.Context, manifest *mf.Manifest, comp updateApiConfig(instance.Spec.ResultsAPIProperties), enablePVCLogging(instance.Spec.ResultsAPIProperties), updateEnvWithSecretName(instance.Spec.ResultsAPIProperties), + populateGoogleCreds(instance.Spec.ResultsAPIProperties), common.AddDeploymentRestrictedPSA(), common.AddStatefulSetRestrictedPSA(), common.DeploymentImages(resultImgs), @@ -227,6 +231,95 @@ func updateApiConfig(p interface{}) mf.Transformer { } } +func populateGoogleCreds(props v1alpha1.ResultsAPIProperties) mf.Transformer { + return func(u *unstructured.Unstructured) error { + if props.LogsType != "GCS" || props.GCSCredsSecretName == "" || + props.GCSCredsSecretKey == "" || !props.LogsAPI || + u.GetKind() != "Deployment" || u.GetName() != deploymentAPI { + return nil + } + + d := &appsv1.Deployment{} + err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured(u.Object, d) + if err != nil { + return err + } + + // find the matching container and add env and secret name object + for i, container := range d.Spec.Template.Spec.Containers { + if container.Name != apiContainerName { + continue + } + add := true + vol := corev1.Volume{ + Name: googleCredsVolName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: props.GCSCredsSecretName, + Items: []corev1.KeyToPath{{ + Key: props.GCSCredsSecretKey, + Path: props.GCSCredsSecretKey, + }}, + }, + }, + } + for k := 0; k < len(d.Spec.Template.Spec.Volumes); k++ { + if d.Spec.Template.Spec.Volumes[k].Name == googleCredsVolName { + d.Spec.Template.Spec.Volumes[k] = vol + add = false + } + } + if add { + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, vol) + } + + volMount := corev1.VolumeMount{ + Name: googleCredsVolName, + MountPath: googleCredsPath, + } + + add = true + for k := 0; k < len(d.Spec.Template.Spec.Containers[i].VolumeMounts); k++ { + if d.Spec.Template.Spec.Containers[i].VolumeMounts[k].Name == googleCredsVolName { + d.Spec.Template.Spec.Containers[i].VolumeMounts[k] = volMount + add = false + } + } + if add { + d.Spec.Template.Spec.Containers[i].VolumeMounts = append( + d.Spec.Template.Spec.Containers[i].VolumeMounts, volMount) + } + + path := googleCredsPath + "/" + props.GCSCredsSecretKey + newEnv := corev1.EnvVar{ + Name: googleAPPCredsEnvName, + Value: path, + } + add = true + for k, env := range d.Spec.Template.Spec.Containers[i].Env { + if env.Name == googleAPPCredsEnvName { + d.Spec.Template.Spec.Containers[i].Env[k] = newEnv + add = false + break + } + } + if add { + d.Spec.Template.Spec.Containers[i].Env = append( + d.Spec.Template.Spec.Containers[i].Env, newEnv) + } + + break + } + + uObj, err := k8sruntime.DefaultUnstructuredConverter.ToUnstructured(d) + if err != nil { + return err + } + u.SetUnstructuredContent(uObj) + return nil + } +} + // updates env keys with the secret name into "tekton-results-api" deployment in "api" container func updateEnvWithSecretName(props v1alpha1.ResultsAPIProperties) mf.Transformer { return func(u *unstructured.Unstructured) error { diff --git a/pkg/reconciler/kubernetes/tektonresult/transform_test.go b/pkg/reconciler/kubernetes/tektonresult/transform_test.go index bab717a8e4..998b3bd0e1 100644 --- a/pkg/reconciler/kubernetes/tektonresult/transform_test.go +++ b/pkg/reconciler/kubernetes/tektonresult/transform_test.go @@ -123,6 +123,47 @@ S3_SECRET_ACCESS_KEY=secret S3_MULTI_PART_SIZE=123`) } +func Test_GoogleCred(t *testing.T) { + testData := []string{path.Join("testdata", "api-deployment-gcs.yaml"), path.Join("testdata", "api-deployment.yaml")} + for i := range testData { + manifest, err := mf.ManifestFrom(mf.Recursive(testData[i])) + assert.NilError(t, err) + + deployment := &appsv1.Deployment{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Resources()[0].Object, deployment) + assert.NilError(t, err) + prop := v1alpha1.ResultsAPIProperties{ + LogsAPI: true, + LogsType: "GCS", + GCSCredsSecretName: "foo-test", + GCSCredsSecretKey: "bar-test", + } + + manifest, err = manifest.Transform(populateGoogleCreds(prop)) + assert.NilError(t, err) + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(manifest.Resources()[0].Object, deployment) + assert.NilError(t, err) + + path := googleCredsPath + "/" + prop.GCSCredsSecretKey + newEnv := corev1.EnvVar{ + Name: googleAPPCredsEnvName, + Value: path, + } + + var i int + for i = range deployment.Spec.Template.Spec.Containers { + if deployment.Spec.Template.Spec.Containers[i].Name != apiContainerName { + continue + } + } + + assert.Equal(t, deployment.Spec.Template.Spec.Volumes[2].Name, googleCredsVolName) + assert.Equal(t, deployment.Spec.Template.Spec.Containers[i].VolumeMounts[2].Name, googleCredsVolName) + assert.Equal(t, deployment.Spec.Template.Spec.Containers[i].Env[5], newEnv) + } +} + func TestUpdateEnvWithSecretName(t *testing.T) { testData := path.Join("testdata", "api-deployment.yaml") manifest, err := mf.ManifestFrom(mf.Recursive(testData))