Add annotate-apply-time-mutations function (#556)

* Add annotate-mutations fn

* Fixed doc

* Test cleanup

* Fix sample readme

* Add build line

* Fix ci errors

* Review changes

* Rename annotate-apply-time-mutations, refactor

* Continue rename updates

* Further refactor

* Update tests

* Presubmit fixes

* Review comment

* Formatting

* Fix sample test diff

* e2e fix

* Relocate function to contrib

* Update readme to new gcr

* update pkg path

* More path fix

* makefile fix

* add Makefile for contrib/go

* update root-level Makefile

* Review comments

* go mod up

* readme fix

* readme fix

Co-authored-by: Michael Thacker <thackem@google.com>
Co-authored-by: Mengqi Yu <mengqiy@google.com>

Makefile

@@ -25,6 +25,7 @@ unit-test: unit-test-go unit-test-ts ## Run unit tests for all functions
 
 unit-test-go: ## Run unit tests for Go functions
 	cd functions/go && $(MAKE) test
+	cd functions/contrib/go && $(MAKE) test
 
 unit-test-ts: ## Run unit tests for TS functions
 	cd functions/ts && $(MAKE) test
@@ -38,6 +39,7 @@ test: unit-test e2e-test ## Run all unit tests and e2e tests
 check-licenses:
 	cd functions/ts && $(MAKE) check-licenses
 	cd functions/go && $(MAKE) check-licenses
+	cd functions/contrib/go && $(MAKE) check-licenses
 	cd functions/contrib/ts && $(MAKE) check-licenses
 
 verify-docs:
@@ -47,11 +49,13 @@ verify-docs:
 build: ## Build all function images. Variable 'TAG' is used to specify tag. 'dev' will be used if not set.
 	cd functions/go && $(MAKE) TAG=$(TAG) build
 	cd functions/ts && $(MAKE) TAG=$(TAG) build
+	cd functions/contrib/go && $(MAKE) TAG=$(TAG) build
 	cd functions/contrib/ts && $(MAKE) TAG=$(TAG) build
 
 push: ## Push images to registry. WARN: This operation should only be done in CI environment.
 	cd functions/go && $(MAKE) push
 	cd functions/ts && $(MAKE) push
+	cd functions/contrib/go && $(MAKE) push
 	cd functions/contrib/ts && $(MAKE) push
 
 site-generate: ## Collect function branches and generate a catalog of their examples and documentation using kpt next.

examples/contrib/annotate-apply-time-mutations-simple/.expected/config.yaml

@@ -0,0 +1,2 @@
+testType: eval
+image: gcr.io/kpt-fn-contrib/annotate-apply-time-mutations:unstable

examples/contrib/annotate-apply-time-mutations-simple/.expected/diff.patch

@@ -0,0 +1,40 @@
+diff --git a/resources.yaml b/resources.yaml
+index 76ca35b..8325565 100644
+--- a/resources.yaml
++++ b/resources.yaml
+@@ -5,11 +5,34 @@ metadata:
+   namespace: example-namespace
+   annotations:
+     unmodified-key: foobarbaz
++    config.kubernetes.io/apply-time-mutation: |
++      - sourceRef:
++          group: foo.cloud.google.com
++          kind: RefKind
++          name: example-name1
++          namespace: example-namespace
++        sourcePath: $.status.number
++        targetPath: $.spec.fieldList[0]
++      - sourceRef:
++          apiVersion: bar.cloud.google.com/v1beta1
++          kind: OtherKind
++          name: example-name2
++          namespace: example-namespace
++        sourcePath: $.status.number2
++        targetPath: $.spec.numberMap.a
++      - sourceRef:
++          group: resourcemanager.cnrm.cloud.google.com
++          kind: Project
++          name: example-name
++          namespace: example-namespace
++        sourcePath: $.status.number
++        targetPath: $.spec.stringField
++        token: $ref2
+ spec:
+   fieldList:
+     - 5 # apply-time-mutation: ${foo.cloud.google.com/namespaces/example-namespace/RefKind/example-name1:$.status.number}
+     - 6
+   numberMap:
+     a: 0 # apply-time-mutation: ${bar.cloud.google.com/v1beta1/namespaces/example-namespace/OtherKind/example-name2:$.status.number2}
+-  stringField: placeholder # apply-time-mutation: serviceAccount:service-${resourcemanager.cnrm.cloud.google.com/namespaces/example-namespace/Project/example-name:$.status.number}@service.gserviceaccount.com
++  stringField: serviceAccount:service-$ref2@service.gserviceaccount.com # apply-time-mutation: serviceAccount:service-${resourcemanager.cnrm.cloud.google.com/namespaces/example-namespace/Project/example-name:$.status.number}@service.gserviceaccount.com
+   unmodifiedField: "no-diff"

examples/contrib/annotate-apply-time-mutations-simple/.krmignore

@@ -0,0 +1,2 @@
+.expected
+

examples/contrib/annotate-apply-time-mutations-simple/Kptfile

@@ -0,0 +1,4 @@
+apiVersion: kpt.dev/v1
+kind: Kptfile
+metadata:
+  name: test-mutation

examples/contrib/annotate-apply-time-mutations-simple/README.md

@@ -0,0 +1,32 @@
+# annotate-apply-time-mutations: Simple Example
+
+### Overview
+
+This example shows how the [annotate-apply-time-mutations] function works.
+
+Running `annotate-apply-time-mutations` function on the example package will:
+
+1.  Set `config.k8s.io/apply-time-mutation` annotation on resources with `apply-time-mutation` inline comments.
+
+### Fetch the example package
+
+Get the example package by running the following commands:
+
+```shell
+kpt pkg get https://github.com/GoogleContainerTools/kpt-functions-catalog.git/examples/contrib/annotate-apply-time-mutations-simple
+```
+
+### Function invocation
+
+Invoke the function with the following command:
+
+```shell
+kpt fn eval annotate-apply-time-mutations-simple --image gcr.io/kpt-fn-contrib/annotate-apply-time-mutations:unstable
+```
+
+### Expected result
+
+1.  File resources.yaml will include `config.k8s.io/apply-time-mutation` annotation matching the comment markups.
+2.  Commented fields with templated values will be updated with the template and replacement tokens.
+
+[annotate-apply-time-mutations] https://catalog.kpt.dev/annotate-apply-time-mutations/v0.1/?id=definitions

examples/contrib/annotate-apply-time-mutations-simple/resources.yaml

@@ -0,0 +1,15 @@
+apiVersion: foo.cloud.google.com/v1beta1
+kind: MyTestKind
+metadata:
+  name: my-test-resource
+  namespace: example-namespace
+  annotations:
+    unmodified-key: foobarbaz
+spec:
+  fieldList:
+    - 5 # apply-time-mutation: ${foo.cloud.google.com/namespaces/example-namespace/RefKind/example-name1:$.status.number}
+    - 6
+  numberMap:
+    a: 0 # apply-time-mutation: ${bar.cloud.google.com/v1beta1/namespaces/example-namespace/OtherKind/example-name2:$.status.number2}
+  stringField: placeholder # apply-time-mutation: serviceAccount:service-${resourcemanager.cnrm.cloud.google.com/namespaces/example-namespace/Project/example-name:$.status.number}@service.gserviceaccount.com
+  unmodifiedField: "no-diff"

functions/contrib/go/Makefile

@@ -0,0 +1,133 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+SHELL=/bin/bash
+GOPATH := $(shell go env GOPATH)
+TAG ?= unstable
+GCR = gcr.io/kpt-fn-contrib
+GOBIN := $(shell go env GOPATH)/bin
+
+# Edit this list to contain all go functions
+FUNCTIONS := \
+	annotate-apply-time-mutations
+
+# Targets for running all function tests
+FUNCTION_TESTS := $(patsubst %,%-TEST,$(FUNCTIONS))
+# Targets for generating all functions docs
+FUNCTION_GENERATE_DOCS := $(patsubst %,%-GENERATE,$(FUNCTIONS))
+
+FUNCTION_CHECKLICENSES := $(patsubst %,%-CHECKLICENSES,$(FUNCTIONS))
+# Targets to build functions
+FUNCTION_BUILDS := $(patsubst %,%-BUILD,$(FUNCTIONS))
+# Targets to push images
+FUNCTION_PUSH := $(patsubst %,%-PUSH,$(FUNCTIONS))
+# Current function name used by individual function targets
+CURRENT_FUNCTION ?= unknown
+
+.DEFAULT_GOAL := help
+.PHONY: help
+help: ## Print this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: verify
+verify: generate $(FUNCTIONS) ## Verify all functions
+
+.PHONY: $(FUNCTIONS)
+$(FUNCTIONS):
+	$(MAKE) CURRENT_FUNCTION=$@ func-verify
+
+.PHONY: test
+test: generate $(FUNCTION_TESTS) ## Run unit tests for all functions
+
+.PHONY: $(FUNCTION_TESTS)
+$(FUNCTION_TESTS):
+	$(MAKE) CURRENT_FUNCTION=$(subst -TEST,,$@) func-test
+
+generate: install-mdtogo generate-docs
+
+.PHONY: generate-docs
+generate-docs: $(FUNCTION_GENERATE_DOCS) ## Generate docs for all functions
+
+.PHONY: $(FUNCTION_GENERATE_DOCS)
+$(FUNCTION_GENERATE_DOCS):
+	$(MAKE) CURRENT_FUNCTION=$(subst -GENERATE,,$@) docs-generate
+
+.PHONY: build
+build: generate $(FUNCTION_BUILDS) ## Build all function images. 'TAG' env is used to specify tag. 'dev' will be used if not set.
+
+.PHONY: $(FUNCTION_BUILDS)
+$(FUNCTION_BUILDS):
+	$(MAKE) CURRENT_FUNCTION=$(subst -BUILD,,$@) TAG=$(TAG) DEFAULT_GCR=$(GCR) func-build
+
+.PHONY: push
+push: $(FUNCTION_PUSH) ## Push images to registry. WARN: This operation should only be done in CI environment.
+
+.PHONY: $(FUNCTION_PUSH)
+$(FUNCTION_PUSH):
+	$(MAKE) CURRENT_FUNCTION=$(subst -PUSH,,$@) TAG=$(TAG) DEFAULT_GCR=$(GCR) func-push
+
+.PHONY: check-licenses
+check-licenses: install-go-licenses $(FUNCTION_CHECKLICENSES)
+
+.PHONY: $(FUNCTION_CHECKLICENSES)
+$(FUNCTION_CHECKLICENSES):
+	$(MAKE) CURRENT_FUNCTION=$(subst -CHECKLICENSES,,$@) func-check-licenses
+
+install-go-licenses:
+ifeq (, $(shell which go-licenses))
+	../../scripts/install-go-licenses.sh
+GO_LICENSES=$(GOPATH)/bin/go-licenses
+else
+GO_LICENSES=$(shell which go-licenses)
+endif
+
+install-mdtogo:
+	GO111MODULE=on go get -v github.com/GoogleContainerTools/kpt/mdtogo@main
+
+# Recipes for individual function
+.PHONY: func-fix func-vet func-fmt func-test func-lint \
+	func-build func-push func-verify func-check-licenses docs-generate
+func-verify: docs-generate func-fix func-vet func-fmt func-test func-lint func-check-licenses
+
+func-fix:
+	cd $(CURRENT_FUNCTION) && go fix ./...
+
+func-fmt:
+	cd $(CURRENT_FUNCTION) && go fmt ./...
+
+func-lint:
+	(which $(GOPATH)/bin/golangci-lint || \
+		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.41.1)
+	cd $(CURRENT_FUNCTION) && time $(GOPATH)/bin/golangci-lint run --timeout=10m ./...
+
+func-test:
+	cd $(CURRENT_FUNCTION) && go test -cover ./...
+
+func-vet:
+	cd $(CURRENT_FUNCTION) && go vet ./...
+
+func-check-licenses: install-go-licenses
+	@echo Checking licenses for $(CURRENT_FUNCTION)...
+	cd $(CURRENT_FUNCTION) && $(GO_LICENSES) check .
+
+func-build: func-verify
+	@echo Building image for $(CURRENT_FUNCTION)...
+	../../../scripts/go-function-release.sh build contrib
+
+func-push:
+	@echo Pushing image $(CURRENT_FUNCTION)...
+	../../../scripts/go-function-release.sh push contrib
+
+docs-generate:
+	$(GOBIN)/mdtogo $(CURRENT_FUNCTION) $(CURRENT_FUNCTION)/generated --license=none --strategy=cmdDocs
+

functions/contrib/go/annotate-apply-time-mutations/Dockerfile

@@ -0,0 +1,16 @@
+FROM golang:1.16-alpine3.13
+ENV CGO_ENABLED=0
+WORKDIR /go/src/
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+RUN go build -o /usr/local/bin/function ./
+
+#############################################
+
+FROM alpine:3.13
+COPY --from=0 /usr/local/bin/function /usr/local/bin/function
+ENTRYPOINT ["function"]
+

functions/contrib/go/annotate-apply-time-mutations/README.md

@@ -0,0 +1,104 @@
+# annotate-apply-time-mutations
+
+## Overview
+
+<!--mdtogo:Short-->
+
+The `annotate-apply-time-mutations` function enables authors to use inline comments,
+rather than annotations, to specify field replacements using the apply time mutation feature.
+
+It works by reading `# apply-time-mutation:` comments on resource YAML and adds the equivalent
+`config.kubernetes.io/apply-time-mutation` annotation to the resource.
+
+The `config.kubernetes.io/apply-time-mutation` annotation is read by the [apply time mutation]
+functionality which patches the resource config at apply time, during `kpt live apply`, with
+the referenced resource's live value.
+
+<!--mdtogo-->
+
+<!--mdtogo:Long-->
+
+## Usage
+
+The annotate-apply-time-mutations function can be executed declaratively as part of `kpt fn render`
+
+```yaml
+apiVersion: kpt.dev/v1
+kind: Kptfile
+pipeline:
+  mutators:
+    - image: gcr.io/kpt-fn-contrib/annotate-apply-time-mutations:unstable
+```
+
+or imperatively like:
+
+```shell
+kpt fn eval --image gcr.io/kpt-fn-contrib/annotate-apply-time-mutations:unstable
+```
+
+
+The `annotate-apply-time-mutations` function does the following:
+
+1.  Scans the package for `apply-time-mutation` comment markup.
+2.  Appends the equivalent `config.k8s.io/apply-time-mutation` annotation to the same.
+
+The expected `apply-time-mutation` comment format is:
+
+`# apply-time-mutation: [prefix]${[group]/[version]/namespaces/[source namespace]/[kind]/[source name]:[source field path]}[suffix]`
+
+Prefix, version, and suffix are optional fields.
+
+For fields with a substitution as well as a constant prefix and/or suffix, this function will insert a replacement token in the field, matched in the annotation.
+`field: [prefix]replaceme[suffix] # apply-time-mutation: ...`
+
+<!--mdtogo-->
+
+## Examples
+
+<!--mdtogo:Examples-->
+
+Appending an `config.kubernetes.io/apply-time-mutation` annotation based on a comment.
+
+Let's start with a sample resource.
+
+```yaml
+apiVersion: iam.cnrm.cloud.google.com/v1beta1
+kind: IAMPolicyMember
+metadata:
+  name: my-policy
+  namespace: example-namespace
+spec:
+  member: placeholder # apply-time-mutation: "serviceAccount:service-${resourcemanager.cnrm.cloud.google.com/namespaces/example-namespace/Project/example-name:$.status.number}@container-engine-robot.iam.gserviceaccount.com"
+```
+
+Invoke the function:
+
+```shell
+kpt fn eval --image gcr.io/kpt-fn-contrib/annotate-apply-time-mutations:unstable
+```
+
+Resource will be updated to the following:
+
+```yaml
+apiVersion: iam.cnrm.cloud.google.com/v1beta1
+kind: IAMPolicyMember
+metadata:
+  name: my-policy
+  namespace: example-namespace
+  annotations:
+    config.kubernetes.io/apply-time-mutation: |
+      - sourceRef:
+          group: resourcemanager.cnrm.cloud.google.com
+          kind: Project
+          name: example-name
+          namespace: example-namespace
+        sourcePath: $.status.number
+        targetPath: $.spec.member
+        token: $ref0
+spec:
+  member: serviceAccount:service-$ref0@container-engine-robot.iam.gserviceaccount.com # apply-time-mutation: ...
+```
+
+<!--mdtogo-->
+
+[apply time mutation] https://kpt.dev/reference/cli/live/apply/