From a74f5d18074acc1f3492eaec331c2f3239812f48 Mon Sep 17 00:00:00 2001 From: Cyril TOVENA Date: Tue, 25 Dec 2018 20:53:04 -0400 Subject: [PATCH] Adds Kind local cluster support with documentation --- build/Makefile | 75 ++++++++++++++++++++++++++++++++++++++++++++---- build/README.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 5 deletions(-) diff --git a/build/Makefile b/build/Makefile index 2a82eabbd3..e21a4c324b 100644 --- a/build/Makefile +++ b/build/Makefile @@ -48,6 +48,10 @@ GCP_BUCKET_CHARTS ?= agones-chart MINIKUBE_PROFILE ?= agones GO_BUILD_TAGS ?= none +# kind cluster name to use +KIND_PROFILE ?= agones +KIND_CONTAINER_NAME=kind-$(KIND_PROFILE)-control-plane + # Game Server image to use while doing end-to-end tests GS_TEST_IMAGE ?= gcr.io/agones-images/udp-server:0.5 @@ -346,8 +350,9 @@ push-build-image: # useful for pprof and stats viewing, etc controller-portforward: PORT ?= 6060 controller-portforward: - docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) -p $(PORT):$(PORT) $(build_tag) \ - kubectl port-forward deployments/agones-controller -n agones-system $(PORT) + docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) \ + -e "KUBECONFIG=/root/.kube/$(kubeconfig_file)" -p $(PORT):$(PORT) $(build_tag) \ + kubectl port-forward deployments/agones-controller -n agones-system $(PORT) # start pprof with a web ui pprof-web: @@ -391,8 +396,8 @@ do-release: @echo "Now go make the $(RELEASE_VERSION) release on Github!" setup-test-cluster: $(ensure-build-image) - docker run --rm -it $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) kubectl apply -f $(mount_path)/build/helm.yaml - docker run --rm $(common_mounts) $(DOCKER_RUN_ARGS) $(build_tag) helm init --service-account helm + $(DOCKER_RUN) kubectl apply -f $(mount_path)/build/helm.yaml + $(DOCKER_RUN) helm init --wait --service-account helm clean-test-cluster: $(ensure-build-image) $(uninstall) docker run --rm -it $(common_mounts) -e "KUBECONFIG=/root/.kube/$(kubeconfig_file)" $(DOCKER_RUN_ARGS) $(build_tag) helm reset @@ -540,4 +545,64 @@ minikube-test-e2e: minikube-agones-profile test-e2e # minikube port forwarding minikube-controller-portforward: - $(MAKE) controller-portforward DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)" \ No newline at end of file + $(MAKE) controller-portforward DOCKER_RUN_ARGS="--network=host -v $(minikube_cert_mount)" + + +# _ ___ _ +# | |/ (_)_ __ __| | +# | ' /| | '_ \ / _` | +# | . \| | | | | (_| | +# |_|\_\_|_| |_|\__,_| + +# creates a kind cluster for use with agones +# Kind stand for Kubernetes IN Docker +# You can change the cluster name using KIND_PROFILE env var +kind-test-cluster: DOCKER_RUN_ARGS+=--network=host +kind-test-cluster: $(ensure-build-image) + @if [ -z $$(kind get clusters | grep $(KIND_PROFILE)) ]; then\ + echo "Could not find $(KIND_PROFILE) cluster. Creating...";\ + kind create cluster --name $(KIND_PROFILE) --image kindest/node:v1.11.3 --wait 5m;\ + fi + $(MAKE) setup-test-cluster KUBECONFIG="$(shell kind get kubeconfig-path --name="$(KIND_PROFILE)")" DOCKER_RUN_ARGS="$(DOCKER_RUN_ARGS)" + +# deletes the kind agones cluster +# useful if you're want to start from scratch +kind-delete-cluster: + kind delete cluster --name $(KIND_PROFILE) + +# start an interactive shell with kubectl configured to target the kind cluster + kind-shell: $(ensure-build-image) + $(MAKE) shell KUBECONFIG="$(shell kind get kubeconfig-path --name="$(KIND_PROFILE)")" \ + DOCKER_RUN_ARGS="--network=host $(DOCKER_RUN_ARGS)" + +# installs the current dev version of agones +# you should build-images and kind-push first. +kind-install: + $(MAKE) install DOCKER_RUN_ARGS="--network=host" ALWAYS_PULL_SIDECAR=false \ + IMAGE_PULL_POLICY=IfNotPresent PING_SERVICE_TYPE=NodePort \ + KUBECONFIG="$(shell kind get kubeconfig-path --name="$(KIND_PROFILE)")" + +# pushses the current dev version of agones to the kind single node cluster. +kind-push: + BUNDLE_FILE=$$(mktemp -d)/agones.tar.gz; \ + docker save \ + $(sidecar_tag) \ + $(controller_tag) \ + $(ping_tag) \ + -o $$BUNDLE_FILE; \ + docker cp $$BUNDLE_FILE $(KIND_CONTAINER_NAME):/agones.tar.gz; \ + docker exec $(KIND_CONTAINER_NAME) docker load -i /agones.tar.gz; \ + rm -f $$BUNDLE_FILE + +# Runs e2e tests against our kind cluster +kind-test-e2e: +kind-test-e2e: + $(MAKE) KUBECONFIG="$(shell kind get kubeconfig-path --name="$(KIND_PROFILE)")" \ + DOCKER_RUN_ARGS=--network=host \ + test-e2e + +# kind port forwarding +kind-controller-portforward: + $(MAKE) controller-portforward \ + KUBECONFIG="$(shell kind get kubeconfig-path --name="$(KIND_PROFILE)")" \ + DOCKER_RUN_ARGS="--network=host" diff --git a/build/README.md b/build/README.md index ac07984a14..0e65d724bf 100644 --- a/build/README.md +++ b/build/README.md @@ -18,6 +18,7 @@ Table of Contents * [Testing and Building](#testing-and-building) * [Running a Test Google Kubernetes Engine Cluster](#running-a-test-google-kubernetes-engine-cluster) * [Running a Test Minikube cluster](#running-a-test-minikube-cluster) + * [Running a Test Kind cluster](#running-a-test-kind-cluster) * [Running a Custom Test Environment](#running-a-custom-test-environment) * [Next Steps](#next-steps) * [Make Variable Reference](#make-variable-reference) @@ -64,6 +65,13 @@ Table of Contents * [make minikube-shell](#make-minikube-shell) * [make minikube-transfer-image](#make-minikube-transfer-image) * [make minikube-controller-portforward](#make-minikube-controller-portforward) + * [Kind](#Kind) + * [make kind-test-cluster](#make-kind-test-cluster) + * [make kind-push](#make-kind-push) + * [make kind-install](#make-kind-install) + * [make kind-test-e2e](#make-kind-test-e2e) + * [make kind-shell](#make-kind-shell) + * [make kind-controller-portforward](#make-kind-controller-portforward) * [Custom Environment](#custom-environment) * [make setup-custom-test-cluster](#make-setup-custom-test-cluster) * [make clean-custom-test-cluster](#make-clean-custom-test-cluster) @@ -268,6 +276,42 @@ $ make minikube-transfer-image TAG=myimage:0.1 Running end-to-end tests on minikube is done via the `make minikube-test-e2e` target. This target use the same `make test-e2e` but also setup some prerequisites for use with a minikube cluster. +### Running a Test Kind cluster + This will setup a [Kubernetes IN Docker](https://github.com/kubernetes-sigs/kind) cluster named agones by default. + +Because Kind runs on a docker on the host, some of the standard build and development Make targets +need to be replaced by kind specific targets. + +If you have [go](https://golang.org/) and [docker](https://www.docker.com/) installed `go get sigs.k8s.io/kind` is all you need ! + +Next we will create the Agones Kind cluster. Run `make kind-test-cluster` to create the `agones` Kubernetes cluster. + +This will also setup helm and a kubeconfig, the kubeconfig location can found using the following command `kind get kubeconfig-path --name=agones` assuming you're using the default `KIND_PROFILE`. + +You can verify that your new cluster information by running (if you don't have kubectl you can skip to the shell section): + +``` +KUBECONFIG=$(kind get kubeconfig-path --name=agones) kubectl cluster-info +``` + +Great! Now we are setup, we also provide a development shell with a handful set of tools like kubectl and helm. + +Run `make kind-shell` to enter the development shell. You should see a bash shell that has you as the root user. +Enter `kubectl get pods` and press enter. You should see that you have no resources currently, but otherwise see no errors. +Assuming that all works, let's exit the shell by typing `exit` and hitting enter, and look at a building, pushing and installing Agones on Kind next. + +You may remember in the first part of this walkthrough, we ran `make build`, which created all the images and binaries +we needed to work with Agones locally. We can push these images them straight into kind very easily! + +Run `make kind-push` which will send all of Agones's docker images from your local Docker into the Agones Kind container. + +Now that the images are pushed, to install the development version, +run `make kind-install` and Agones will install the images that you built and pushed to the Agones Kind cluster. + +Running end-to-end tests on Kind is done via the `make kind-test-e2e` target. This target use the same `make test-e2e` but also setup some prerequisites for use with a Kind cluster. + +If you are having performance issues, check out these docs [here](https://github.com/kubernetes-sigs/kind/tree/master/docs/user#creating-a-cluster) + ### Running a Custom Test Environment This section is addressed to developers using a custom Kubernetes provider, a custom image repository and/or multiple test clusters. @@ -471,6 +515,38 @@ Use TAG to specify the image to transfer into minikube The minikube version of [`make controller-portforward`](#make-controller-portforward) to setup port forwarding to the controller deployment. +### Kind + +[Kind - kubernetes in docker](https://github.com/kubernetes-sigs/kind) is a tool for running local Kubernetes clusters using Docker container "nodes". +Kind is primarily designed for testing Kubernetes 1.11+, initially targeting the [conformance tests](https://github.com/kubernetes/community/blob/master/contributors/devel/conformance-tests.md). + +Since Kind runs locally, there are some targets that need to be used instead of the standard ones above. + +#### `make kind-test-cluster` +Starts a local kubernetes cluster, you can delete it with `make kind-delete-cluster` + +Use KIND_PROFILE variable to change the name of the cluster. + +#### `make kind-push` +Push the local Agones Docker images that have already been built +via `make build` or `make build-images` into the "agones" Kind cluster. + +#### `make kind-install` +Installs the current development version of Agones into the Kubernetes cluster. +Use this instead of `make install`, as it disables PullAlways on the install.yaml + +### `make kind-test-e2e` +Runs end-to-end tests on the previously installed version of Agones. +These tests validate Agones flow from start to finish. + +#### `make kind-shell` +Connecting to Kind requires so enhanced permissions, so use this target +instead of `make shell` to start an interactive shell for development on Kind. + +#### `make kind-controller-portforward` +The Kind version of [`make controller-portforward`](#make-controller-portforward) to setup +port forwarding to the controller deployment. + ### Custom Environment #### `make setup-custom-test-cluster`