Experimental GCP config for kubeflow setup (#971)

* kubeflow asm config * exp config for GCP setup * update tests * exp kfdef should use master * resolve comments * remove exp DM entry * add reeadme; rename folder
kubeflow · Mar 14, 2020 · 5b2ecb7 · 5b2ecb7
1 parent 700c099
commit 5b2ecb7
Show file tree

Hide file tree

Showing 20 changed files with 1,003 additions and 22 deletions.
diff --git a/gcp/cnrm/README.md b/gcp/cnrm/README.md
@@ -0,0 +1,4 @@
+###Experimental config directory in CNRM format
+
+* CNRM resources converted from current kuebflow DM entry.
+* IstioControlPlane CR from ASM
diff --git a/gcp/cnrm/cluster/cluster.yaml b/gcp/cnrm/cluster/cluster.yaml
@@ -0,0 +1,40 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO(kunming): kustomize this config to include all the options we currently offer through DM
+
+apiVersion: identity.cnrm.cloud.google.com/v1alpha2
+kind: IdentityNamespace
+metadata:
+  name: default
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec: {}
+---
+apiVersion: container.cnrm.cloud.google.com/v1alpha2
+kind: ContainerCluster
+metadata:
+  clusterName: "test-project/us-central1-a/test-cluster" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"test-project"},{"name":"cluster-name","value":"test-cluster"},{"name":"gcloud.compute.zone","value":"us-central1-a"}]}}
+  name: test-cluster # {"type":"string","x-kustomize":{"setter":{"name":"cluster-name","value":"test-cluster"}}}
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  minMasterVersion: 1.14.10-gke.24
+  location: us-central1-a # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.compute.zone","value":"us-central1-a"}}}
+  workloadIdentity:
+    identityNamespace: default
+  labels:
+    mesh_id: "test-project_us-central1-a_test-cluster" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"test-project"},{"name":"cluster-name","value":"test-cluster"},{"name":"gcloud.compute.zone","value":"us-central1-a"}]}}
+  loggingService: logging.googleapis.com/kubernetes
+  monitoringService: monitoring.googleapis.com/kubernetes
+  network: default
+  subnetwork: default
diff --git a/gcp/cnrm/cluster/istio-operator.yaml b/gcp/cnrm/cluster/istio-operator.yaml
@@ -0,0 +1,35 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: install.istio.io/v1alpha2
+kind: IstioControlPlane
+metadata:
+  clusterName: "test-project/us-central1-a/test-cluster" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"test-project"},{"name":"cluster-name","value":"test-cluster"},{"name":"gcloud.compute.zone","value":"us-central1-a"}]}}
+spec:
+  profile: asm
+  hub: gcr.io/asm-testing
+  tag: 1.4.5-asm-test.7
+  values:
+    gateways:
+      istio-ingressgateway:
+        type: NodePort
+    global:
+      meshID: "test-project_us-central1-a_test-cluster" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"test-project"},{"name":"cluster-name","value":"test-cluster"},{"name":"gcloud.compute.zone","value":"us-central1-a"}]}}
+      trustDomain: "test-project.svc.id.goog" # {"type":"string","x-kustomize":{"partialSetters":[{"name":"gcloud.core.project","value":"test-project"}]}}
+      sds:
+        token:
+          aud: "test-project.svc.id.goog" # {"type":"string","x-kustomize":{"partialSetters":[{"name":"gcloud.core.project","value":"test-project"}]}}
+    nodeagent:
+      env:
+        GKE_CLUSTER_URL: "https://container.googleapis.com/v1/projects/test-project/locations/us-central1-a/clusters/test-cluster" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"test-project"},{"name":"cluster-name","value":"test-cluster"},{"name":"gcloud.compute.zone","value":"us-central1-a"}]}}
diff --git a/gcp/cnrm/cluster/nodepool.yaml b/gcp/cnrm/cluster/nodepool.yaml
@@ -0,0 +1,35 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: container.cnrm.cloud.google.com/v1alpha2
+kind: ContainerNodePool
+metadata:
+  clusterName: "test-project/us-central1-a/test-cluster" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"test-project"},{"name":"cluster-name","value":"test-cluster"},{"name":"gcloud.compute.zone","value":"us-central1-a"}]}}
+  name: test-cluster-cpu-pool-v1 # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"cluster-name","value":"test-cluster"}]}}
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  initialNodeCount: 2
+  autoscaling:
+    minNodeCount: 2
+    maxNodeCount: 8 # {"type":"integer","x-kustomize":{"setter":{"name":"max-nodes","value":"8"}}}
+  nodeConfig:
+    machineType: n1-standard-8
+    minCpuPlatform: 'Intel Broadwell'
+    metadata:
+      disable-legacy-endpoints: "true"
+    serviceAccount: test-cluster-vm@test-project.iam.gserviceaccount.com # {"type":"string","x-kustomize":{"partialSetters":[{"name":"cluster-name","value":"test-cluster",{"name":"gcloud.core.project","value":"test-project"}]}}
+    workloadMetadataConfig:
+      nodeMetadata: GKE_METADATA_SERVER
+  clusterRef:
+    name: test-cluster # {"type":"string","x-kustomize":{"setter":{"name":"cluster-name","value":"test-cluster"}}}
diff --git a/gcp/cnrm/project/enable-services.yaml b/gcp/cnrm/project/enable-services.yaml
@@ -0,0 +1,149 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stackdriver
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: stackdriver
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: stackdriver.googleapis.com
+---
+# GCE
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: compute
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: compute.googleapis.com
+---
+# GKE
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: gke
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: container.googleapis.com
+---
+# Short lived iam credentials for workload identity
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: iamcredentials
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: iamcredentials.googleapis.com
+---
+# Mesh Certificate Authority
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: meshca
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: meshca.googleapis.com
+---
+# Mesh Telemetry
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: meshtelemetry
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: meshtelemetry.googleapis.com
+---
+# Mesh Config
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: meshconfig
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: meshconfig.googleapis.com
+---
+# Anthos
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: anthos
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: anthos.googleapis.com
+---
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: deploymentmanager
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: deploymentmanager.googleapis.com
+---
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: servicemanagement
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: servicemanagement.googleapis.com
+---
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: cloudresourcemanager
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: cloudresourcemanager.googleapis.com
+---
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: endpoints
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: endpoints.googleapis.com
+---
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: file
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: file.googleapis.com
+---
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: ml
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: ml.googleapis.com
+---
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: iam
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: iam.googleapis.com
+---
+apiVersion: cnrm.cloud.google.com/v1alpha1
+kind: CloudService
+metadata:
+  name: sqladmin
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  service: sqladmin.googleapis.com
diff --git a/gcp/cnrm/project/kf-admin-sa.yaml b/gcp/cnrm/project/kf-admin-sa.yaml
@@ -0,0 +1,36 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: iam.cnrm.cloud.google.com/v1alpha1
+kind: IAMServiceAccount
+metadata:
+  name: test-cluster-admin # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"cluster-name","value":"test-cluster"}]}}
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  displayName: kubeflow admin service account
+  projectRoles:
+  - roles/source.admin
+  - roles/servicemanagement.admin
+  - roles/compute.networkAdmin
+  - roles/cloudbuild.builds.editor
+  - roles/viewer
+  - roles/storage.admin
+  - roles/bigquery.admin
+  - roles/dataflow.admin
+  - roles/ml.admin
+  - roles/dataproc.editor
+  - roles/cloudsql.admin
+  - roles/logging.logWriter
+  - roles/monitoring.metricWriter
+  - roles/monitoring.viewer
diff --git a/gcp/cnrm/project/kf-user-sa.yaml b/gcp/cnrm/project/kf-user-sa.yaml
@@ -0,0 +1,34 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: iam.cnrm.cloud.google.com/v1alpha1
+kind: IAMServiceAccount
+metadata:
+  name: test-cluster-user # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"cluster-name","value":"test-cluster"}]}}
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  displayName: kubeflow user service account
+  projectRoles:
+  - roles/cloudbuild.builds.editor
+  - roles/viewer
+  - roles/source.admin
+  - roles/storage.admin
+  - roles/bigquery.admin
+  - roles/dataflow.admin
+  - roles/ml.admin
+  - roles/dataproc.editor
+  - roles/cloudsql.admin
+  - roles/logging.logWriter
+  - roles/monitoring.metricWriter
+  - roles/monitoring.viewer
diff --git a/gcp/cnrm/project/kf-vm-sa.yaml b/gcp/cnrm/project/kf-vm-sa.yaml
@@ -0,0 +1,28 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: iam.cnrm.cloud.google.com/v1alpha1
+kind: IAMServiceAccount
+metadata:
+  name: test-cluster-vm # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"cluster-name","value":"test-cluster"}]}}
+  namespace: "test-project" # {"type":"string","x-kustomize":{"setBy":"kpt","setter":{"name":"gcloud.core.project","value":"test-project"}}}
+spec:
+  displayName: Anthos Service Mesh credentials
+  projectRoles:
+  - roles/logging.logWriter
+  - roles/monitoring.metricWriter
+  - roles/meshtelemetry.reporter
+  - roles/cloudtrace.agent
+  - roles/monitoring.viewer
+  - roles/storage.objectViewer