diff --git a/config/hubs/openscapes.cluster.yaml b/config/hubs/openscapes.cluster.yaml new file mode 100644 index 0000000000..e39c017cbe --- /dev/null +++ b/config/hubs/openscapes.cluster.yaml @@ -0,0 +1,184 @@ +name: openscapes +provider: kubeconfig +kubeconfig: + file: secrets/openscapes.yaml +hubs: + - name: staging + domain: staging.openscapes.2i2c.cloud + template: daskhub + auth0: + connection: github + config: &openscapesHubConfig + scratchBucket: + enabled: false + basehub: + nfsPVC: + nfs: + # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html + mountOptions: + - rsize=1048576 + - wsize=1048576 + - timeo=600 + - soft # We pick soft over hard, so NFS lockups don't lead to hung processes + - retrans=2 + - noresvport + serverIP: fs-b25253b5.efs.us-west-2.amazonaws.com + baseShareName: / + shareCreator: + tolerations: + - key: node-role.kubernetes.io/master + operator: "Exists" + effect: "NoSchedule" + jupyterhub: + homepage: + templateVars: + org: + name: Openscapes + logo_url: https://www.openscapes.org/img/logo.png + url: https://www.openscapes.org/ + designed_by: + name: 2i2c + url: https://2i2c.org + operated_by: + name: 2i2c + url: https://2i2c.org + funded_by: + name: Openscapes + url: https://www.openscapes.org/ + singleuser: + defaultUrl: /lab + initContainers: + # Need to explicitly fix ownership here, since EFS doesn't do anonuid + - name: volume-mount-ownership-fix + image: busybox + command: ["sh", "-c", "id && chown 1000:1000 /home/jovyan && ls -lhd /home/jovyan"] + securityContext: + runAsUser: 0 + volumeMounts: + - name: home + mountPath: /home/jovyan + subPath: "{username}" + + image: + name: 783616723547.dkr.ecr.us-west-2.amazonaws.com/user-image + tag: "d78bb6c" + profileList: + # The mem-guarantees are here so k8s doesn't schedule other pods + # on these nodes. + - display_name: "Small: m5.large" + description: "~2 CPU, ~8G RAM" + kubespawner_override: + # Expllicitly unset mem_limit, so it overrides the default memory limit we set in + # basehub/values.yaml + mem_limit: null + mem_guarantee: 7G + node_selector: + node.kubernetes.io/instance-type: m5.large + - display_name: "Medium: m5.xlarge" + description: "~4 CPU, ~15G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 13G + node_selector: + node.kubernetes.io/instance-type: m5.xlarge + - display_name: "Large: m5.2xlarge" + description: "~8 CPU, ~30G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 28G + node_selector: + node.kubernetes.io/instance-type: m5.2xlarge + - display_name: "Huge: m5.8xlarge" + description: "~32 CPU, ~128G RAM" + kubespawner_override: + mem_limit: null + mem_guarantee: 120G + node_selector: + node.kubernetes.io/instance-type: m5.8xlarge + scheduling: + userPlaceholder: + enabled: false + replicas: 0 + userScheduler: + enabled: false + proxy: + service: + type: LoadBalancer + https: + enabled: true + chp: + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + traefik: + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + hub: + allowNamedServers: true + networkPolicy: + # FIXME: For dask gateway + enabled: false + readinessProbe: + enabled: false + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + dask-gateway: + traefik: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + controller: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + gateway: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + # TODO: figure out a replacement for userLimits. + extraConfig: + optionHandler: | + from dask_gateway_server.options import Options, Integer, Float, String + def cluster_options(user): + def option_handler(options): + if ":" not in options.image: + raise ValueError("When specifying an image you must also provide a tag") + extra_annotations = { + "hub.jupyter.org/username": user.name, + "prometheus.io/scrape": "true", + "prometheus.io/port": "8787", + } + extra_labels = { + "hub.jupyter.org/username": user.name, + } + return { + "worker_cores_limit": options.worker_cores, + "worker_cores": min(options.worker_cores / 2, 1), + "worker_memory": "%fG" % options.worker_memory, + "image": options.image, + "scheduler_extra_pod_annotations": extra_annotations, + "worker_extra_pod_annotations": extra_annotations, + "scheduler_extra_pod_labels": extra_labels, + "worker_extra_pod_labels": extra_labels, + } + return Options( + Integer("worker_cores", 2, min=1, max=16, label="Worker Cores"), + Float("worker_memory", 4, min=1, max=32, label="Worker Memory (GiB)"), + String("image", default="pangeo/pangeo-notebook:latest", label="Image"), + handler=option_handler, + ) + c.Backend.cluster_options = cluster_options + idle: | + # timeout after 30 minutes of inactivity + c.KubeClusterConfig.idle_timeout = 1800 + - name: prod + domain: openscapes.2i2c.cloud + template: daskhub + auth0: + connection: github + config: *openscapesHubConfig \ No newline at end of file diff --git a/kops/openscapes.jsonnet b/kops/openscapes.jsonnet new file mode 100644 index 0000000000..eda9dcd66e --- /dev/null +++ b/kops/openscapes.jsonnet @@ -0,0 +1,96 @@ +local cluster = import "./libsonnet/cluster.jsonnet"; +local ig = import "./libsonnet/instancegroup.jsonnet"; + +local zone = "us-west-2b"; +local nodes = [ + {spec+: { machineType: "m5.large" }}, // 2CPU, 8G RAM + {spec+: { machineType: "m5.xlarge" }}, // 4CPU, 16G RAM + {spec+: { machineType: "m5.2xlarge" }}, // 8CPU, 32G RAM + {spec+: { machineType: "m5.8xlarge" }} // 32CPU, 128G RAM +]; + +local data = { + cluster: cluster { + metadata+: { + name: "openscapeshub.k8s.local" + }, + _config+:: { + zone: zone, + masterInstanceGroupName: data.master.metadata.name + } + }, + master: ig { + metadata+: { + labels+: { + "kops.k8s.io/cluster": data.cluster.metadata.name + }, + name: "master" + }, + spec+: { + machineType: "m5.medium", + subnets: [zone], + nodeLabels+: { + "hub.jupyter.org/node-purpose": "core", + "k8s.dask.org/node-purpose": "core" + }, + // Needs to be at least 1 + minSize: 1, + maxSize: 3, + role: "Master" + }, + }, + notebookNodes: [ + ig { + local thisIg = self, + metadata+: { + labels+: { + "kops.k8s.io/cluster": data.cluster.metadata.name + }, + name: "notebook-%s" % std.strReplace(thisIg.spec.machineType, ".", "-") + }, + spec+: { + machineType: n.machineType, + subnets: [zone], + maxSize: 20, + role: "Node", + nodeLabels+: { + "hub.jupyter.org/node-purpose": "user", + "k8s.dask.org/node-purpose": "scheduler" + }, + taints: [ + "hub.jupyter.org_dedicated=user:NoSchedule", + "hub.jupyter.org/dedicated=user:NoSchedule" + ], + }, + } + n for n in nodes + ], + daskNodes: [ + ig { + local thisIg = self, + metadata+: { + labels+: { + "kops.k8s.io/cluster": data.cluster.metadata.name + }, + name: "dask-%s" % std.strReplace(thisIg.spec.machineType, ".", "-") + }, + spec+: { + machineType: n.machineType, + subnets: [zone], + maxSize: 20, + role: "Node", + nodeLabels+: { + "k8s.dask.org/node-purpose": "worker" + }, + taints: [ + "k8s.dask.org_dedicated=worker:NoSchedule", + "k8s.dask.org/dedicated=worker:NoSchedule" + ], + }, + } + n for n in nodes + ] +}; + +{ 'kops.yaml': [ + data.cluster, + data.master +] + data.notebookNodes + data.daskNodes} \ No newline at end of file diff --git a/kops/ssh-keys/openscapes.key b/kops/ssh-keys/openscapes.key new file mode 100644 index 0000000000..24fb66893d --- /dev/null +++ b/kops/ssh-keys/openscapes.key @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:4iq33Uz3gjWLsQstL+TiR9A46nV3O1/xqOpObd0Tm5e0ErwufWdn5/YiEBV3oa2/XTAkyS+XE5WoFwRCMh+FHT/unvjMWXNq+ewTvQ/AnvQoYUI4l9s+4x36F/7tpKl0UOWoHnEBhjabXP06jUHNOz4jsFTlaXAvEv5VJq4fIkbW5jBQatOIl1qoFA4XYUCnHMPKBRDb1JqexwKOMFUCRTkrl8k2u/nwrhSihz7OOXCYB/IzRST5dLqB3o0sOtpPqxBlQRzeyOnCNf6dgQBrn5PnHkAa43QelXxCUtZPXE5XS9PkMzbQNUehtPvDzVGjMHfmS6Oaq/4ScsNn2DS2b13DPVka+tgd6IjS6CNlFDJ9lyhQ0/+o16ydOCJIrg4ySS68SrC7cl43FfY8LbtSs+9e6T1FC7scOhODXKTKYoVfTAIgyK77+SeDgsbnNkRyYzw56miLxR37VJOk9KCce2Ig4lB3ZlV+iFUWZw9MHdvWl61DpmcF7dHxuLajfwkDDE65KAqdJ7laF5GgxYSaIMuTXHZflp43kqGy/bD/AMMvnPaRUMkzTpUxYaCm8aUetMotL08X08U/zl7M0gPVMaU71v2Y+tkrV/tefr+nw6NbqTWLVHah/9CdDhN+JwS3XQisKcqEDOvSIr/4pJ8xIdtncUmIjmShNwXBHSbU7ONpm1L8lEIVom1rEb9rd6NSd75StOHBINi2TpYa8v5M1fpme6pV73lEqMPoCiFvuOwG4k3+s9ImeZ8Ro2lTeSkT+jhIvuytHK6VRaaTOKtIZ3tLESyGqL7wy2fN+IaJ7ymn6pAnSUIqy2q2JPLKqPSeSFKhJOudW6mTzA2WJTwq+mxmHAWBFZnTyMnKN15ez8pAWXEW2mzqC3r15GAvewSJeb63B5HDpaXU2XoSbk2OHG8vZc45noRo4sH+HTepq8YNiAwnbwwi6ISjHvzXFn3tKnmVwUk+2Syj61JKaJowBdndtAUrbptt/xLrvEwRTCdY5amlTl2kFqdJ6w5ttT6DxQcuJtniCzPf716hHuIMWMf6TBmpRQcqcHHBnBlvPSVHUxtJICuCHGt54O8MX72LExVQRxeXhCA5lr/XvImVDo36m4NIzFASSJssnaDvvJa5sQe2i5aGftl/1wOZlv2RViq+SjhjA7YsdfOcpeiH+pdJg6ZCVpLQzw/mIg35J/lVNbp8FZCHcS3w0w0YL/QKQ2/oqkYBef+X9wb+hgSzNBTqm2lpXVH7dMj62zs58FKmAo3aRDWZVNeOazeKM+P/ASTG/1sttzb/QbL/IB+UX9nAxBtDLvEY8w5ArooJaXEfT4HZcCgQHrvNgysHfrPRxQcpYxV527DHdO/y7vDx27XVRTyDjvqOt4lCIjfp5TLwlV9SpwK/e1FyWJ0O+PhO1lZj0YQnNyCNHH9Ck9uiCUDPLuV5S3Q/pEzndArXAR+PLzVROhmS8hCVzAqokn+tITpb5xKXm7TCMhHok92hsOtIrF0xRSAa3pdSLVjKZKhJzRaAbqa/OmfJhZsftukNGYw2NrTN4WmbKnFPo5aJT8eDm2RCStpfwnuvCVOQri4Ykh7nUTEwf1xX86vcxZyC4ZTi/uYaQA3xEED89KQorIrFeIENPIEtJni515ZoNmKKwD9h9SDMcYzcmFRwXC1TeALuumzVJJu57Rl3Qgc/uMOg9UDwzMAFxG09QGduq2wFxF72td2e1ETVH021kJhZwS3lDwWEvpM8+bz1wF1hHy+0b3f7aGia3n1MMPXWPPtwFNHFN6ptV7k6+h2I9+i4MUS6QBiqP6L67NHe8pxvhLMO7qS6HMUNRMBXRgY4t8XP/cc1MW3G+D07rWRMKjRHoorY2sdpeo/kVEuYdnl3Ij/NgDk+yfhT+zMlm+2Z8ks0FUmp3ck4YnxsQyT2qlnB/FcUDheFzG7u5RpgXpO3pATy2wDvncNbCahQhA1S1pKeGGE8gGRvKoLylnIEqMHIj2zzUBxgD1oX/Sz8PJvaSise/57wpgyVyK3e/jOZ3DB+CViQA1p2K+f+GPAr+CsBCoMKLSUFdZKaqW8aYTeNl1KpezF+XHcClD4tF1dgpp/CbPyBPN+RSDphkvAKL1KSVMf6ca9Wr6RiOvN4rg0wOiugoaSSdD7n+owMq09FbcdtT6nRDzYAYWNtgbfpT8i8Bvjm63qYr/aY3hNQiSoxLyJg9LK7P/K/XKClQlStCpi5TBsZ2hI9Culm5viSQ9Ezw5S6P+aqHekGoR6HkOkWZoThvbJoOjg/aN1xTXq/XFtG6M2sR5cYjM9aIJelxbfTYNZXA5aKeBjdvbQq5HJpNx/0f4w/RWSyme80VMkOKOVOqXzQsEIgW0u9oXaYtFMMSICD/QQRSBKEpTutfj2/JQFEoNN6o3sjnqGdK/kkeacmaUDdQ1Ma0WDPzDO3rRisEf4inpiJbwar4CjZWx3dlG0mAgekSiLgjU54X0X+4NZBqyI31z+UipD1/oZB666tr9ZPrUElBvYyOAqcjTiujR/7VyObdBJ8avA2fQHtEgTCvlQhlQrHNNs5PlGzxXeloOcC2oyQQelarA2PNtlYFZ6OQdg1ZFtzqwF3RjW6ikZhMV02MWTU/03NadL8UgCIUgO5KEfGeRAyzVrU+Iq/jKN9XdQDHW5c1vWtaevpn/rzQ9xFkXoxTB0Meeow2nJEh33fOYnVY4K/uGVNZJ59FaM9lntX+GD+hNePoBQ+rcBmDmgEKZjf78Lw2qvELDPGGDJtCabgBVZ/PMF0x63eYQXISMXeHCELzT63BD9ES2bmW/h6C6HENn1wC2i+zAXJGtLVxpOvETotjJ35MvHToFW6h9YKxur7NOKzFn6Y/JHBOWkv3rIDosKBdT+s+6Nu4YEZ9QrdrlmjBccBvJBUTWlug5R0bcB5mrBhnN20le1aznMATTNmIOzAiMHFzsWy6XBuSiMt9mvefRlOgdXwKVl5oFwuiPgMiVmWyJw8TpKo/y34arepoy43F8aOStR0Ryr0KMVn5tQ1MB62UpoTpEvz7lI5irTHMZgH5FbRfOOkZSlPLmdXmn8sbCu6qxCm07xckx5QjlwU/HVsrGniHbJEluYaH/51u3KDK3ydZ3z/ZbLrta03orHQrzxFuEe1+nbVmLIr767ilJLcj95pM/VcXxz2JfNxmMfXvkMYPhgGrEQjfGN3kiLxcnBvs+tqReyUhEkYo5WXw49MvelOvGm4jkUm9+2gj4h8fK9XibC4dtzm3hkhw5ni+EXFDNQvuCemJUvwnSqHw2Ygi7s5bvReWe6VmS4zwdvjrJCs+m7uy85ISz5XcDTiVHkvt3u9vYwfch8VxULhrfaw2B0dmOizXRrW0Vwq8I6jKvy2qK8F7Jo0BZdmNO6Z/rGlHq4V88CUQ+uVN5gF878NMMpOIkRRsLwJtE1c/zRNy/c+LBRCDNI0QGMvkaRa//B9v6rVNMjCBz9l1C2OSMP3uz01xwBg,iv:q00QVhwHmehRy3vtCCG1j8s1q78BUGqOeHjGtqOZL2s=,tag:lyiQv6BDlNu4dID1VzRxiQ==,type:str]", + "sops": { + "kms": null, + "gcp_kms": [ + { + "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs", + "created_at": "2021-05-14T23:09:54Z", + "enc": "CiQA4OM7eD7URjoKU5bdHCbRjbPcLAPV155BS3sSDH92Ck77V9USSQBy9hCYsxuj3UIz1RzywUtsR4z7J4g0O8oYyMmZk2iw0B/uUszHkoMKvym5j4GNV3nAZeceEbZVi9QRz/Lmd79vTs4wq16q5EA=" + } + ], + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2021-05-14T23:09:57Z", + "mac": "ENC[AES256_GCM,data:Vf6ZNPZIQRK/ItLJqarWl1qMaiT+4Ls6pZLHyRHWE12UYcmwaTYIb9umZHi/UaOTzC/xBdoSGNNnBz9lr2VmQq6GwkYbZ6QXuHWtEcrNpwwbHfRVF18aFeEyNraI2FGdVuebKyEAEoXptI/S5+LJ6BTjqetoswpl2S35tPa/M/E=,iv:w1lY8WY9OcLWSKmlCkV8y6VKM1cl0RsKxn/m0ZnhmPI=,tag:ySmQvTTj83Nc9+TwByhqcA==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.1" + } +} \ No newline at end of file diff --git a/kops/ssh-keys/openscapes.key.pub b/kops/ssh-keys/openscapes.key.pub new file mode 100644 index 0000000000..df5252f4aa --- /dev/null +++ b/kops/ssh-keys/openscapes.key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCqnr+uN1faeBKDqkRocJr1v6tLhGvmOaPQku8B+ZvZV1K3a0HfdIPh1/TFff1lAvbmCBBvgbkiScwoN3DUZBxf4TfU9HnRIMDReWX72alJoh1XokY9wOuanyc+Hbs9RIDpQbwD/6VTBwQuP+AUezVQXHdFxD6iIFgOf8JdjAYgUC+b2s2oVEZOo2Y2Z4lLRTPDU0ohkFbIZEvt1fUuXRluw+XbdnsSexKa76ZTQnL+NBJSpPnGvwohsjkaY5xdodGKe9rXe63H/UBtRgxVaZe7vR8JD0+xzxQURfEfEQTb6n3u4uLIQoKyNOsFStDgERo39HzM4DUsZcYZw0D251kaQy+sORm54gqWDe/dkCv2T7ztzKB90Krv75LuSQxMpfBobIh5ujM5QU8UQfgNwjftJ5MLV9qyq6yF5IYowOfCT4vALUdO7gAyoXlkogfCi7B+NU2mlUkxewSVnz0lTZRX8OSX0C9eoGf3kgubB3d2waiRFkcvic+Mz2p3VjlRoRk= yuvipanda@do-the-work.local