feat(*): automate policy generation

app/kumactl/cmd/root.go

@@ -21,6 +21,7 @@ import (
 	kumactl_errors "github.com/kumahq/kuma/app/kumactl/pkg/errors"
 	kuma_cmd "github.com/kumahq/kuma/pkg/cmd"
 	"github.com/kumahq/kuma/pkg/core"
+	_ "github.com/kumahq/kuma/pkg/core/bootstrap"
 	kuma_log "github.com/kumahq/kuma/pkg/log"
 	// Register gateway resources.
 	_ "github.com/kumahq/kuma/pkg/plugins/runtime/gateway/register"

mk/generate.mk

@@ -1,6 +1,8 @@
 ENVOY_IMPORTS := ./pkg/xds/envoy/imports.go
 PROTO_DIRS := ./pkg/config ./api
 
+CONTROLLER_GEN := go run -mod=mod sigs.k8s.io/controller-tools/cmd/controller-gen@master
+
 .PHONY: clean/proto
 clean/proto: ## Dev: Remove auto-generated Protobuf files
 	find $(PROTO_DIRS) -name '*.pb.go' -delete
@@ -28,6 +30,52 @@ protoc/plugins:
 	$(PROTOC_GO) --proto_path=./api pkg/plugins/ca/provided/config/*.proto
 	$(PROTOC_GO) --proto_path=./api pkg/plugins/ca/builtin/config/*.proto
 
+POLICIES_DIR := pkg/plugins/policies
+
+policies = $(foreach dir,$(wildcard $(POLICIES_DIR)/*),$(notdir $(dir)))
+generate_policy_targets = $(addprefix generate/policy/,$(policies))
+cleanup_policy_targets = $(addprefix cleanup/policy/,$(policies))
+
+generate/policies: $(cleanup_policy_targets) $(generate_policy_targets)
+
+# deletes all files in policy directory except *.proto and validator.go
+cleanup/policy/%:
+	$(shell find pkg/plugins/policies/$* -not -name '*.proto' -not -name 'validator.go' -type f -delete)
+
+generate/policy/%: generate/schema/%
+	@echo "Policy $* successfully generated"
+	@echo "Don't forget to update Helm chart with 'make generate/helm/$*'"
+
+generate/schema/%: generate/controller-gen/%
+	tools/policy-gen/crd-extract-openapi.sh $*
+
+generate/controller-gen/%: generate/kumapolicy-gen/%
+	$(CONTROLLER_GEN) "crd:crdVersions=v1,ignoreUnexportedFields=true" paths="./$(POLICIES_DIR)/$*/k8s/..." output:crd:artifacts:config=$(POLICIES_DIR)/$*/k8s/crd
+	$(CONTROLLER_GEN) object paths=$(POLICIES_DIR)/$*/k8s/v1alpha1/zz_generated.types.go
+
+generate/kumapolicy-gen/%: generate/dirs/%
+	@cd tools/policy-gen/protoc-gen-kumapolicy && go build && cd -
+	$(PROTOC) \
+		--proto_path=./api \
+		--kumapolicy_opt=endpoints-template=tools/policy-gen/templates/endpoints.yaml \
+		--kumapolicy_out=$(POLICIES_DIR)/$* \
+		--go_opt=paths=source_relative \
+		--go_out=plugins=grpc,$(go_mapping):. \
+		--plugin=protoc-gen-kumapolicy=tools/policy-gen/protoc-gen-kumapolicy/protoc-gen-kumapolicy \
+		$(POLICIES_DIR)/$*/api/v1alpha1/*.proto
+	@rm tools/policy-gen/protoc-gen-kumapolicy/protoc-gen-kumapolicy
+
+generate/dirs/%:
+	@cd $(POLICIES_DIR)/$* && \
+	mkdir -p api/v1alpha1 && \
+	mkdir -p k8s/v1alpha1 && \
+	mkdir -p k8s/crd && \
+	cd -
+
+generate/helm/%:
+	tools/policy-gen/crd-helm-copy.sh $*
+
+
 KUMA_GUI_GIT_URL=https://github.com/kumahq/kuma-gui.git
 KUMA_GUI_VERSION=master
 KUMA_GUI_FOLDER=app/kuma-ui/pkg/resources/data

pkg/plugins/policies/donothingpolicy/api/v1alpha1/donothingpolicy.proto

@@ -0,0 +1,40 @@
+syntax = "proto3";
+
+package kuma.plugins.policies.donothingpolicy.v1alpha1;
+
+import "mesh/options.proto";
+option go_package = "github.com/kumahq/kuma/pkg/plugins/policies/donothingpolicy/api/v1alpha1";
+
+import "mesh/v1alpha1/selector.proto";
+import "config.proto";
+
+option (doc.config) = {
+  type : Policy,
+  name : "DoNothingPolicy",
+  file_name : "donothingpolicy"
+};
+
+// DoNothingPolicy defines permission for traffic between dataplanes.
+message DoNothingPolicy {
+
+  option (kuma.mesh.resource).name = "DoNothingPolicyResource";
+  option (kuma.mesh.resource).type = "DoNothingPolicy";
+  option (kuma.mesh.resource).package = "mesh";
+  option (kuma.mesh.resource).kds.send_to_zone = true;
+  option (kuma.mesh.resource).ws.name = "donothingpolicy";
+  option (kuma.mesh.resource).ws.plural = "donothingpolicies";
+  option (kuma.mesh.resource).allow_to_inspect = true;
+
+  // List of selectors to match dataplanes that are sources of traffic.
+  repeated kuma.mesh.v1alpha1.Selector sources = 1 [ (doc.required) = true ];
+  // List of selectors to match services that are destinations of traffic.
+  repeated kuma.mesh.v1alpha1.Selector destinations = 2
+      [ (doc.required) = true ];
+
+  message Conf {
+    // Set true in case of doing nothing
+    bool enableDoNothing = 1;
+  }
+
+  Conf conf = 3;
+}

tools/policy-gen/README.md

@@ -0,0 +1,83 @@
+# How to generate a new Kuma policy
+
+1. Create a new directory for the policy in `pkg/plugins/policies`. Example:
+    ```shell
+    mkdir -p pkg/plugins/policies/donothingpolicy
+    ```
+
+2. Create a proto file for new policy in `pkg/plugins/policies/donothingpolicy/api/v1alpha1`. For example 
+donothingpolicy.proto:
+    ```protobuf
+    syntax = "proto3";
+
+    package kuma.plugins.policies.donothingpolicy.v1alpha1;
+
+    import "mesh/options.proto";
+    option go_package = "github.com/kumahq/kuma/pkg/plugins/policies/donothingpolicy/api/v1alpha1";
+
+    import "mesh/v1alpha1/selector.proto";
+    import "config.proto";
+
+    option (doc.config) = {
+    type : Policy,
+    name : "DoNothingPolicy",
+    file_name : "donothingpolicy"
+    };
+
+    // DoNothingPolicy defines permission for traffic between dataplanes.
+    message DoNothingPolicy {
+
+    option (kuma.mesh.resource).name = "DoNothingPolicyResource";
+    option (kuma.mesh.resource).type = "DoNothingPolicy";
+    option (kuma.mesh.resource).package = "mesh";
+    option (kuma.mesh.resource).kds.send_to_zone = true;
+    option (kuma.mesh.resource).ws.name = "donothingpolicy";
+    option (kuma.mesh.resource).ws.plural = "donothingpolicies";
+    option (kuma.mesh.resource).allow_to_inspect = true;
+
+    // List of selectors to match dataplanes that are sources of traffic.
+    repeated kuma.mesh.v1alpha1.Selector sources = 1 [ (doc.required) = true ];
+    // List of selectors to match services that are destinations of traffic.
+    repeated kuma.mesh.v1alpha1.Selector destinations = 2 [ (doc.required) = true ];
+
+    message Conf {
+      bool enableDoNothing = 1;
+    }
+
+    Conf conf = 3;
+
+    }
+    ```
+
+3. Call `make generate/policy/<POLICY_NAME>`. Example:
+   ```shell
+   make generate/policy/donothingpolicy
+   ```
+
+4. **Optional.** Add validation. Create file `validator.go`, file with such name won't be cleaned up
+by `make cleanup/policy/donothingpolicy`. Implement method `validate() error`:
+   ```go
+   package v1alpha1
+
+   func (t *DoNothingPolicyResource) validate() error { 
+       // validate resource here
+       return nil
+   }
+   ```
+
+6. Add import to `pkg/core/bootstrap/plugins.go`:
+   ```go
+   _ "github.com/kumahq/kuma/pkg/plugins/policies/donothingpolicy"
+   ```
+
+7. Update Helm chart with a new CRD:
+   ```shell
+   make generate/helm/donothingpolicy
+   ```
+   Also, today it's required to update `cp-rbac.yaml` manually, automation is yet to come.
+
+Now you can check swagger-ui for this policy:
+
+```shell
+docker run -p 80:8080 -e SWAGGER_JSON=/policy/rest.yaml -v $PWD/pkg/plugins/policies/donothingpolicy/api/v1alpha1:/policy swaggerapi/swagger-ui
+```

tools/policy-gen/crd-extract-openapi.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+POLICY=$1
+VERSION=${2:-"v1alpha1"}
+
+POLICIES_DIR=pkg/plugins/policies
+POLICIES_API_DIR="${POLICIES_DIR}/${POLICY}/api/${VERSION}"
+POLICIES_CRD_DIR="${POLICIES_DIR}/${POLICY}/k8s/crd"
+
+SCHEMA_TEMPLATE=tools/policy-gen/templates/schema.yaml
+
+# 1. Copy file ${SCHEMA_TEMPLATE} to ${POLICIES_API_DIR}/schema.yaml. It contains
+#    information about fields that are equal for all resources 'type', 'mesh' and 'name'.
+#
+# 2. Using yq extract item from the list '.spec.version[]' that has ${VERSION} and
+#    take '.schema.openAPIV3Schema.properties.spec'.
+#
+# 3. Delete 'type' and 'description' for the extracted item, because these are 'type'
+#    and 'description' for the 'spec' field.
+#
+# 4. Using yq eval-all with ireduce merge the file from Step 1 and output from Step 3,
+#    placing the result into the file from Step 1
+
+echo "Generating schema for ${POLICY}/${VERSION} based on CRD"
+
+function cleanupOnError() {
+    rm "${POLICIES_API_DIR}"/schema.yaml
+    echo "Script failed, schema.yaml wasn't generated"
+}
+trap cleanupOnError ERR
+
+cp "${SCHEMA_TEMPLATE}" "${POLICIES_API_DIR}"/schema.yaml
+
+if [ "$(find "${POLICIES_CRD_DIR}" -type f | wc -l | xargs echo)" != 1 ]; then
+  echo "Exactly 1 file is expected in ${POLICIES_CRD_DIR}"
+  exit 1
+fi
+
+CRD_FILE=$(find "${POLICIES_CRD_DIR}" -type f)
+
+# we don't want expressions to be expanded with yq, that's why we're intentionally using single quotes
+# shellcheck disable=SC2016
+yq e '.spec.versions[] | select (.name == "'"${VERSION}"'") | .schema.openAPIV3Schema.properties.spec | del(.type) | del(.description)' \
+"${CRD_FILE}" | yq eval-all -i '. as $item ireduce ({}; . * $item )' \
+"${POLICIES_API_DIR}"/schema.yaml -

tools/policy-gen/crd-helm-copy.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+POLICY=$1
+
+POLICIES_DIR=pkg/plugins/policies
+POLICIES_CRD_DIR="${POLICIES_DIR}/${POLICY}/k8s/crd"
+
+if [ "$(find "${POLICIES_CRD_DIR}" -type f | wc -l | xargs echo)" != 1 ]; then
+  echo "More than 1 file in crd directory"
+  exit 1
+fi
+
+CRD_FILE="$(find "${POLICIES_CRD_DIR}" -type f)"
+
+HELM_CRD_DIR=deployments/charts/kuma/crds
+
+cp "${CRD_FILE}" "${HELM_CRD_DIR}"