From 36d39547f78738ec5bc8536d8d0953ddd26a01cb Mon Sep 17 00:00:00 2001 From: Matthew Christopher Date: Thu, 10 Aug 2023 15:22:51 -0700 Subject: [PATCH] Add two new labels - Add the app.kubernetes.io/name and app.kubernetes.io/version labels. - Remove serviceoperator.azure.com/version in favor of the standard app.kubernetes.io/version label. - Update taskfile to use new labels in various places. This fixes #3165. --- Taskfile.yml | 7 ++++-- scripts/v2/wait-for-operator-ready.sh | 19 ++++++++++---- v2/cmd/controller/app/setup.go | 25 ++++++++++--------- v2/config/crd/labels.yaml | 3 ++- v2/config/manager/manager.yaml | 7 +++++- .../manager/manager_metrics_service.yaml | 2 ++ v2/config/webhook/service.yaml | 4 ++- v2/internal/crdmanagement/manager.go | 10 +++++--- v2/internal/crdmanagement/manager_test.go | 3 ++- 9 files changed, 54 insertions(+), 26 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 835022ed179..e095099edcc 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -32,7 +32,8 @@ vars: # Version tags for the v2 controller must start with "v2", e.g. "v2.0.0-alpha.0". sh: "{{.SCRIPTS_ROOT}}/build_version.py v2" - LATEST_VERSION_TAG: $(git describe --tags $(git rev-list --tags=v2* --max-count=1)) + LATEST_VERSION_TAG: + sh: git describe --tags $(git rev-list --tags=v2* --max-count=1) VERSION_FLAGS: -ldflags "-X {{.PACKAGE}}/internal/version.BuildVersion={{.VERSION}}" @@ -341,6 +342,8 @@ tasks: --set crdPattern=*" - "kubectl create namespace pre-release" - task: controller:wait-for-operator-ready + vars: + ARGS: "-o" - "go test -timeout 15m -count=1 -v -run Test_Pre_Release_ResourceCanBeCreated_BeforeUpgrade ./test/pre-release" controller:test-upgrade-post: @@ -615,7 +618,7 @@ tasks: dir: "v2/" cmds: - mkdir -p bin # in case it doesn't exist - - "{{.SCRIPTS_ROOT}}/kustomize-build.sh -k operator -v {{.VERSION}} | sed -e 's@localhost:5000/azureserviceoperator:latest@{{.PUBLIC_REGISTRY}}{{.CONTROLLER_DOCKER_IMAGE}}@g' > bin/azureserviceoperator_{{.VERSION}}.yaml" + - "{{.SCRIPTS_ROOT}}/kustomize-build.sh -k operator -v {{.VERSION}} | sed -e 's@{{.LOCAL_REGISTRY_CONTROLLER_DOCKER_IMAGE}}@{{.PUBLIC_REGISTRY}}{{.CONTROLLER_DOCKER_IMAGE}}@g' > bin/azureserviceoperator_{{.VERSION}}.yaml" - "{{.SCRIPTS_ROOT}}/kustomize-build.sh -k crd -v {{.VERSION}} > bin/azureserviceoperator_customresourcedefinitions_{{.VERSION}}.yaml" controller:make-multitenant-files: diff --git a/scripts/v2/wait-for-operator-ready.sh b/scripts/v2/wait-for-operator-ready.sh index 252a249718c..97ce7740c19 100755 --- a/scripts/v2/wait-for-operator-ready.sh +++ b/scripts/v2/wait-for-operator-ready.sh @@ -10,18 +10,20 @@ set -o pipefail print_usage() { echo "Usage: wait-for-operator-ready.sh [-c]" echo " -c: Do NOT wait for CRDs to reach established state" + echo " -o: Use old label - can be removed after 2.3.0 release" } CHECK_ESTABLISHED=1 -while getopts 'c' flag; do +OLD_LABEL=0 +while getopts 'co' flag; do case "${flag}" in c) CHECK_ESTABLISHED=0 ;; + o) OLD_LABEL=1 ;; *) print_usage exit 1 ;; esac done - function all_crds_have_cabundle() { for crd in $(kubectl api-resources -o name | grep '\.azure\.com'); do cabundle=$(kubectl get crd "$crd" -o jsonpath='{.spec.conversion.webhook.clientConfig.caBundle}') @@ -40,15 +42,22 @@ function wait_for_crds_cabundle() { } function wait_for_crds_established() { - until kubectl wait --for=condition=established --timeout=5s crd -l 'serviceoperator.azure.com/version'; do - sleep 5 - done + if [[ "$OLD_LABEL" -eq 1 ]]; then + until kubectl wait --for=condition=established --timeout=5s crd -l 'serviceoperator.azure.com/version'; do + sleep 5 + done + else + until kubectl wait --for=condition=established --timeout=5s crd -l 'app.kubernetes.io/name == azure-service-operator'; do + sleep 5 + done + fi } if [[ "$CHECK_ESTABLISHED" -eq 1 ]]; then echo "Waiting for CRDs established..." # This has to be a timeout wrapping kubectl wait as we're racing with CRDs being added, and kubectl wait will fail if nothing matches the -l filter export -f wait_for_crds_established + export OLD_LABEL=${OLD_LABEL} timeout 1m bash -c wait_for_crds_established fi diff --git a/v2/cmd/controller/app/setup.go b/v2/cmd/controller/app/setup.go index c206cb24f54..b336c1c881a 100644 --- a/v2/cmd/controller/app/setup.go +++ b/v2/cmd/controller/app/setup.go @@ -13,18 +13,16 @@ import ( "regexp" "time" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kerrors "k8s.io/apimachinery/pkg/util/errors" - "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/benbjohnson/clock" "github.com/go-logr/logr" "github.com/pkg/errors" apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -32,22 +30,22 @@ import ( clientconfig "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - . "github.com/Azure/azure-service-operator/v2/internal/logging" - asometrics "github.com/Azure/azure-service-operator/v2/internal/metrics" - armreconciler "github.com/Azure/azure-service-operator/v2/internal/reconcilers/arm" - common "github.com/Azure/azure-service-operator/v2/pkg/common/config" - "github.com/Azure/azure-service-operator/v2/api" "github.com/Azure/azure-service-operator/v2/internal/config" "github.com/Azure/azure-service-operator/v2/internal/controllers" "github.com/Azure/azure-service-operator/v2/internal/crdmanagement" "github.com/Azure/azure-service-operator/v2/internal/identity" + . "github.com/Azure/azure-service-operator/v2/internal/logging" + asometrics "github.com/Azure/azure-service-operator/v2/internal/metrics" + armreconciler "github.com/Azure/azure-service-operator/v2/internal/reconcilers/arm" "github.com/Azure/azure-service-operator/v2/internal/reconcilers/generic" "github.com/Azure/azure-service-operator/v2/internal/util/interval" "github.com/Azure/azure-service-operator/v2/internal/util/kubeclient" "github.com/Azure/azure-service-operator/v2/internal/util/lockedrand" + common "github.com/Azure/azure-service-operator/v2/pkg/common/config" "github.com/Azure/azure-service-operator/v2/pkg/genruntime" "github.com/Azure/azure-service-operator/v2/pkg/genruntime/conditions" ) @@ -84,8 +82,11 @@ func SetupPreUpgradeCheck(ctx context.Context) error { } // If this CRD is annotated with "serviceoperator.azure.com/version", it must be >=2.0.0 and so safe - // as we didn't start using this label until 2.0.0 - if _, ok := crd.Labels[crdmanagement.ServiceOperatorVersionLabel]; ok { + // as we didn't start using this label until 2.0.0. Same with "app.kubernetes.io/version" which was added in 2.3.0 + // in favor of our custom serviceoperator.azure.com + _, hasOldLabel := crd.Labels[crdmanagement.ServiceOperatorVersionLabelOld] + _, hasNewLabel := crd.Labels[crdmanagement.ServiceOperatorVersionLabel] + if hasOldLabel || hasNewLabel { continue } diff --git a/v2/config/crd/labels.yaml b/v2/config/crd/labels.yaml index a463a65852e..c3da5a9760d 100644 --- a/v2/config/crd/labels.yaml +++ b/v2/config/crd/labels.yaml @@ -3,7 +3,8 @@ kind: LabelTransformer metadata: name: global-labels labels: - serviceoperator.azure.com/version: ${VERSION} + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: ${VERSION} fieldSpecs: - path: metadata/labels create: true diff --git a/v2/config/manager/manager.yaml b/v2/config/manager/manager.yaml index 7485765eb28..9f61bdba568 100644 --- a/v2/config/manager/manager.yaml +++ b/v2/config/manager/manager.yaml @@ -8,6 +8,8 @@ kind: ServiceAccount metadata: name: default namespace: system + labels: + app.kubernetes.io/name: azure-service-operator --- apiVersion: apps/v1 kind: Deployment @@ -15,8 +17,9 @@ metadata: name: controller-manager namespace: system labels: - app: azure-service-operator-v2 control-plane: controller-manager + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: ${VERSION} spec: selector: matchLabels: @@ -27,6 +30,8 @@ spec: labels: aadpodidbinding: aso-manager-binding control-plane: controller-manager + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: ${VERSION} annotations: kubectl.kubernetes.io/default-container: manager spec: diff --git a/v2/config/manager/manager_metrics_service.yaml b/v2/config/manager/manager_metrics_service.yaml index b6c286ac254..067cb5804e4 100644 --- a/v2/config/manager/manager_metrics_service.yaml +++ b/v2/config/manager/manager_metrics_service.yaml @@ -3,6 +3,8 @@ kind: Service metadata: labels: control-plane: controller-manager + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: ${VERSION} name: controller-manager-metrics-service namespace: system spec: diff --git a/v2/config/webhook/service.yaml b/v2/config/webhook/service.yaml index 31e0f829591..fe7026d6c65 100644 --- a/v2/config/webhook/service.yaml +++ b/v2/config/webhook/service.yaml @@ -1,9 +1,11 @@ - apiVersion: v1 kind: Service metadata: name: webhook-service namespace: system + labels: + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: ${VERSION} spec: ports: - port: 443 diff --git a/v2/internal/crdmanagement/manager.go b/v2/internal/crdmanagement/manager.go index cb8bdce3a1a..cfd3b6bf7b7 100644 --- a/v2/internal/crdmanagement/manager.go +++ b/v2/internal/crdmanagement/manager.go @@ -27,9 +27,13 @@ import ( "github.com/Azure/azure-service-operator/v2/internal/util/match" ) -// ServiceOperatorVersionLabel is the label the CRDs have on them containing the ASO version. This value must match the value +// ServiceOperatorVersionLabelOld is the label the CRDs have on them containing the ASO version. This value must match the value // injected by config/crd/labels.yaml -const ServiceOperatorVersionLabel = "serviceoperator.azure.com/version" +const ServiceOperatorVersionLabelOld = "serviceoperator.azure.com/version" +const ServiceOperatorVersionLabel = "app.kubernetes.io/version" +const ServiceOperatorAppLabel = "app.kubernetes.io/name" +const ServiceOperatorAppValue = "azure-service-operator" + const CRDLocation = "crds" const certMgrInjectCAFromAnnotation = "cert-manager.io/inject-ca-from" @@ -52,7 +56,7 @@ func (m *Manager) ListOperatorCRDs(ctx context.Context) ([]apiextensions.CustomR list := apiextensions.CustomResourceDefinitionList{} selector := labels.NewSelector() - requirement, err := labels.NewRequirement(ServiceOperatorVersionLabel, selection.Exists, nil) + requirement, err := labels.NewRequirement(ServiceOperatorAppLabel, selection.Equals, []string{ServiceOperatorAppValue}) if err != nil { return nil, err } diff --git a/v2/internal/crdmanagement/manager_test.go b/v2/internal/crdmanagement/manager_test.go index cd6005115cf..7696ed03268 100644 --- a/v2/internal/crdmanagement/manager_test.go +++ b/v2/internal/crdmanagement/manager_test.go @@ -404,6 +404,7 @@ func Test_ListCRDs_ListsOnlyCRDsMatchingLabel(t *testing.T) { crd3.Labels = map[string]string{ crdmanagement.ServiceOperatorVersionLabel: "123", + crdmanagement.ServiceOperatorAppLabel: crdmanagement.ServiceOperatorAppValue, } g.Expect(kubeClient.Create(ctx, &crd1)).To(Succeed()) @@ -509,7 +510,7 @@ func makeBasicCRD(name string) apiextensions.CustomResourceDefinition { func makeBasicCRDWithVersion(name string, version string) apiextensions.CustomResourceDefinition { crd := makeBasicCRD(name) crd.Labels = map[string]string{ - crdmanagement.ServiceOperatorVersionLabel: version, + crdmanagement.ServiceOperatorVersionLabelOld: version, } return crd