Add AWS Telemetry (#109)

Merge in AWS Telemetry files from v1.3 branch to main
awslabs · Feb 28, 2022 · 2e8bc14 · 2e8bc14
1 parent cbf3a86
commit 2e8bc14
Show file tree

Hide file tree

Showing 10 changed files with 192 additions and 1 deletion.
diff --git a/awsconfigs/common/aws-telemetry/cronjob.yaml b/awsconfigs/common/aws-telemetry/cronjob.yaml
@@ -0,0 +1,84 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: aws-kubeflow-telemetry
+spec:
+  schedule: "0 0 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 0
+  failedJobsHistoryLimit: 0
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 0
+      backoffLimit: 3
+      template:
+        metadata:
+          annotations:
+            # istio sidecar is not neeeded since there is no inbound or outbound traffic from the service mesh
+            sidecar.istio.io/inject: "false"
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: amazonlinux
+            image: public.ecr.aws/amazonlinux/amazonlinux:2
+            command:
+              - /bin/sh
+              - -c
+              - |
+                # Following code uses IMDS service. See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+
+                get_instance_id() {
+                  # use IMDSv2 if enabled else fallback to IMDSv1
+                  local _token
+                  _token=$(curl -s --retry 3 --max-time 3 -X PUT http://169.254.169.254/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+                  if [[ -n ${_token+x} ]]; then
+                    IMDSV2_HEADER=(-H "X-aws-ec2-metadata-token: ${_token}")
+                  fi
+                  
+                  INSTANCE_ID=$(curl -s --retry 3 "${IMDSV2_HEADER[@]}" http://169.254.169.254/latest/meta-data/instance-id)
+
+                  local _instance_id_regex="^(i-\S{17})"
+                  if [[ -z ${INSTANCE_ID+x} || ! ${INSTANCE_ID} =~ ${_instance_id_regex} ]]; then
+                    exit 0
+                  fi
+                }
+
+                get_region() {
+                  # regions where S3 buckets have been created
+                  local _valid_regions=(
+                    "us-east-1"
+                    "us-east-2"
+                    "us-west-1"
+                    "us-west-2"
+                    "af-south-1"
+                    "ap-east-1"
+                    "ap-southeast-1"
+                    "ap-southeast-2"
+                    "ap-southeast-3"
+                    "ap-south-1"
+                    "ap-northeast-1"
+                    "ap-northeast-2"
+                    "ap-northeast-3"
+                    "ca-central-1"
+                    "eu-central-1"
+                    "eu-north-1"
+                    "eu-west-1"
+                    "eu-west-2"
+                    "eu-west-3"
+                    "eu-south-1"
+                    "me-south-1"
+                    "sa-east-1"
+                  )
+                  REGION=$(curl -s --retry 3 "${IMDSV2_HEADER[@]}" http://169.254.169.254/latest/meta-data/placement/availability-zone | awk '{print substr($1, 1, length($1)-1)}')
+
+                  if [[ -z ${REGION+x} || ! ${_valid_regions[${REGION}]+x} ]]; then
+                    exit 0
+                  fi
+                }
+
+                sleep $((1 + $RANDOM % 300))
+                get_instance_id
+                get_region
+
+                # send a GET request to S3 access point
+                curl -s -o /dev/null "https://kubeflow-on-aws-usage-tracking-${REGION}.s3.${REGION}.amazonaws.com/instance-${INSTANCE_ID}.log?x-instance-id=${INSTANCE_ID}"
diff --git a/awsconfigs/common/aws-telemetry/job.yaml b/awsconfigs/common/aws-telemetry/job.yaml
@@ -0,0 +1,78 @@
+# Since CronJob operates on fix schedule, this job is in place for one off tracking at the time Kubeflow is deployed.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: aws-kubelow-telemetry
+spec:
+  ttlSecondsAfterFinished: 0
+  backoffLimit: 3
+  template:
+    metadata:
+      annotations:
+        # istio sidecar is not neeeded since there is no inbound or outbound traffic from the service mesh
+        sidecar.istio.io/inject: "false"
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: amazonlinux
+        image: public.ecr.aws/amazonlinux/amazonlinux:2
+        command:
+          - /bin/sh
+          - -c
+          - |
+            # Following code uses IMDS service. See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+
+            get_instance_id() {
+              # use IMDSv2 if enabled else fallback to IMDSv1
+              local _token
+              _token=$(curl -s --retry 3 --max-time 3 -X PUT http://169.254.169.254/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+              if [[ -n ${_token+x} ]]; then
+                IMDSV2_HEADER=(-H "X-aws-ec2-metadata-token: ${_token}")
+              fi
+
+              INSTANCE_ID=$(curl -s --retry 3 "${IMDSV2_HEADER[@]}" http://169.254.169.254/latest/meta-data/instance-id)
+
+              local _instance_id_regex="^(i-\S{17})"
+              if [[ -z ${INSTANCE_ID+x} || ! ${INSTANCE_ID} =~ ${_instance_id_regex} ]]; then
+                exit 0
+              fi
+            }
+
+            get_region() {
+              # regions where S3 buckets have been created
+              local _valid_regions=(
+                "us-east-1"
+                "us-east-2"
+                "us-west-1"
+                "us-west-2"
+                "af-south-1"
+                "ap-east-1"
+                "ap-southeast-1"
+                "ap-southeast-2"
+                "ap-southeast-3"
+                "ap-south-1"
+                "ap-northeast-1"
+                "ap-northeast-2"
+                "ap-northeast-3"
+                "ca-central-1"
+                "eu-central-1"
+                "eu-north-1"
+                "eu-west-1"
+                "eu-west-2"
+                "eu-west-3"
+                "eu-south-1"
+                "me-south-1"
+                "sa-east-1"
+              )
+              REGION=$(curl -s --retry 3 "${IMDSV2_HEADER[@]}" http://169.254.169.254/latest/meta-data/placement/availability-zone | awk '{print substr($1, 1, length($1)-1)}')
+
+              if [[ -z ${REGION+x} || ! ${_valid_regions[${REGION}]+x} ]]; then
+                exit 0
+              fi
+            }
+
+            get_instance_id
+            get_region
+
+            # send a GET request to S3 access point
+            curl -s -o /dev/null "https://kubeflow-on-aws-usage-tracking-${REGION}.s3.${REGION}.amazonaws.com/instance-${INSTANCE_ID}.log?x-instance-id=${INSTANCE_ID}"
diff --git a/awsconfigs/common/aws-telemetry/kustomization.yaml b/awsconfigs/common/aws-telemetry/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+commonLabels:
+  app: aws-telemetry
+resources:
+- cronjob.yaml
+- job.yaml
diff --git a/docs/deployment/cognito-rds-s3/README.md b/docs/deployment/cognito-rds-s3/README.md
@@ -83,6 +83,8 @@ Follow the [Configure Kubeflow Pipelines](../rds-s3/README.md#2-configure-kubefl
         # Training Operator
         kustomize build upstream/apps/training-operator/upstream/overlays/kubeflow | kubectl apply -f -
 
+        # AWS Telemetry - This is an optional component. See usage tracking documentation for more information
+        kustomize build awsconfigs/common/aws-telemetry | kubectl apply -f -
 
         # AWS Secret Manager
         kustomize build awsconfigs/common/aws-secrets-manager/base | kubectl apply -f -

diff --git a/docs/deployment/cognito-rds-s3/kustomization.yaml b/docs/deployment/cognito-rds-s3/kustomization.yaml
@@ -40,6 +40,8 @@ resources:
 - ../../../upstream/apps/training-operator/upstream/overlays/kubeflow
 # MPI Operator
 - ../../../upstream/apps/mpi-job/upstream/overlays/kubeflow
+# AWS Telemetry - This is an optional component. See usage tracking documentation for more information
+- ../../../awsconfigs/common/aws-telemetry
 
 # Configured for AWS Cognito
 

diff --git a/docs/deployment/cognito/README.md b/docs/deployment/cognito/README.md
@@ -277,6 +277,9 @@ In this section, we will be creating certificate to enable TLS authentication at
         # Training Operator
         kustomize build upstream/apps/training-operator/upstream/overlays/kubeflow | kubectl apply -f -
 
+        # AWS Telemetry - This is an optional component. See usage tracking documentation for more information
+        kustomize build awsconfigs/common/aws-telemetry | kubectl apply -f -
+
         # Ingress
         kustomize build awsconfigs/common/istio-ingress/overlays/cognito | kubectl apply -f -
 

diff --git a/docs/deployment/cognito/kustomization.yaml b/docs/deployment/cognito/kustomization.yaml
@@ -44,6 +44,8 @@ resources:
 - ../../../upstream/apps/training-operator/upstream/overlays/kubeflow
 # MPI Operator
 - ../../../upstream/apps/mpi-job/upstream/overlays/kubeflow
+# AWS Telemetry - This is an optional component. See usage tracking documentation for more information
+- ../../../awsconfigs/common/aws-telemetry
 
 # Configured for AWS Cognito
 

diff --git a/docs/deployment/rds-s3/kustomization.yaml b/docs/deployment/rds-s3/kustomization.yaml
@@ -47,6 +47,8 @@ resources:
 - ../../../upstream/apps/training-operator/upstream/overlays/kubeflow
 # MPI Operator
 - ../../../upstream/apps/mpi-job/upstream/overlays/kubeflow
+# AWS Telemetry - This is an optional component. See usage tracking documentation for more information
+- ../../../awsconfigs/common/aws-telemetry
 
 # User namespace
 - ../../../upstream/common/user-namespace/base

diff --git a/docs/deployment/vanilla/README.md b/docs/deployment/vanilla/README.md
@@ -282,6 +282,14 @@ Install the MPI Operator official Kubeflow component:
 kustomize build apps/mpi-job/upstream/overlays/kubeflow | kubectl apply -f -
 ```
 
+#### AWS Telemetry
+
+Install the AWS Kubeflow telemetry component. This is an optional component. See the [usage tracking documentation](TBD) for more information
+
+```sh
+kustomize build awsconfigs/common/aws-telemetry | kubectl apply -f -
+```
+
 #### User Namespace
 
 Finally, create a new namespace for the the default user (named `kubeflow-user-example-com`).

diff --git a/docs/deployment/vanilla/kustomization.yaml b/docs/deployment/vanilla/kustomization.yaml
@@ -52,4 +52,6 @@ resources:
 # MPI Operator
 - ../../../upstream/apps/mpi-job/upstream/overlays/kubeflow
 # User namespace
-- ../../../upstream/common/user-namespace/base
+- ../../../upstream/common/user-namespace/base
+# AWS Telemetry - This is an optional component. See usage tracking documentation for more information
+- ../../../awsconfigs/common/aws-telemetry