fix: move versions to annotations in control plane static pods

Labels shouldn't be used, as this is not supposed to be used for filtering pods. Use proper annotation prefix private for Talos. Add config-version annotation to track how static pod propagates up to API server (it will be used in control plane upgrade). Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
siderolabs · Feb 16, 2021 · 9205870 · 9205870
1 parent ecd0921
commit 9205870
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 17 deletions.
diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go b/internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go
@@ -189,10 +189,13 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "kube-apiserver",
 				Namespace: "kube-system",
+				Annotations: map[string]string{
+					constants.AnnotationStaticPodSecretsVersion: secretsVersion,
+					constants.AnnotationStaticPodConfigVersion:  configResource.Metadata().Version().String(),
+				},
 				Labels: map[string]string{
-					"tier":            "control-plane",
-					"k8s-app":         "kube-apiserver",
-					"secrets-version": secretsVersion,
+					"tier":    "control-plane",
+					"k8s-app": "kube-apiserver",
 				},
 			},
 			Spec: v1.PodSpec{
@@ -277,10 +280,13 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "kube-controller-manager",
 				Namespace: "kube-system",
+				Annotations: map[string]string{
+					constants.AnnotationStaticPodSecretsVersion: secretsVersion,
+					constants.AnnotationStaticPodConfigVersion:  configResource.Metadata().Version().String(),
+				},
 				Labels: map[string]string{
-					"tier":            "control-plane",
-					"k8s-app":         "kube-controller-manager",
-					"secrets-version": secretsVersion,
+					"tier":    "control-plane",
+					"k8s-app": "kube-controller-manager",
 				},
 			},
 			Spec: v1.PodSpec{
@@ -356,10 +362,13 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "kube-scheduler",
 				Namespace: "kube-system",
+				Annotations: map[string]string{
+					constants.AnnotationStaticPodSecretsVersion: secretsVersion,
+					constants.AnnotationStaticPodConfigVersion:  configResource.Metadata().Version().String(),
+				},
 				Labels: map[string]string{
-					"tier":            "control-plane",
-					"k8s-app":         "kube-scheduler",
-					"secrets-version": secretsVersion,
+					"tier":    "control-plane",
+					"k8s-app": "kube-scheduler",
 				},
 			},
 			Spec: v1.PodSpec{

diff --git a/pkg/kubernetes/kubernetes.go b/pkg/kubernetes/kubernetes.go
@@ -287,11 +287,6 @@ func (h *Client) CordonAndDrain(node string) (err error) {
 	return h.Drain(node)
 }
 
-const (
-	talosCordonedAnnotationName  = "talos.dev/cordoned"
-	talosCordonedAnnotationValue = "true"
-)
-
 // Cordon marks a node as unschedulable.
 func (h *Client) Cordon(name string) error {
 	err := retry.Exponential(30*time.Second, retry.WithUnits(250*time.Millisecond), retry.WithJitter(50*time.Millisecond)).Retry(func() error {
@@ -304,7 +299,7 @@ func (h *Client) Cordon(name string) error {
 			return nil
 		}
 
-		node.Annotations[talosCordonedAnnotationName] = talosCordonedAnnotationValue
+		node.Annotations[constants.AnnotationCordonedKey] = constants.AnnotationCordonedValue
 		node.Spec.Unschedulable = true
 
 		if _, err := h.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{}); err != nil {
@@ -333,14 +328,14 @@ func (h *Client) Uncordon(name string, force bool) error {
 			return retry.UnexpectedError(err)
 		}
 
-		if !force && node.Annotations[talosCordonedAnnotationName] != talosCordonedAnnotationValue {
+		if !force && node.Annotations[constants.AnnotationCordonedKey] != constants.AnnotationCordonedValue {
 			// not cordoned by Talos, skip it
 			return nil
 		}
 
 		if node.Spec.Unschedulable {
 			node.Spec.Unschedulable = false
-			delete(node.Annotations, talosCordonedAnnotationName)
+			delete(node.Annotations, constants.AnnotationCordonedKey)
 
 			if _, err := h.CoreV1().Nodes().Update(attemptCtx, node, metav1.UpdateOptions{}); err != nil {
 				return retry.ExpectedError(err)

diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go
@@ -387,6 +387,18 @@ const (
 	// NodeReadyTimeout is the timeout to wait for the node to be ready (CNI to be running).
 	// For bootstrap API, this includes time to run bootstrap.
 	NodeReadyTimeout = BootTimeout
+
+	// AnnotationCordonedKey is the annotation key for the nodes cordoned by Talos.
+	AnnotationCordonedKey = "talos.dev/cordoned"
+
+	// AnnotationCordonedValue is the annotation key for the nodes cordoned by Talos.
+	AnnotationCordonedValue = "true"
+
+	// AnnotationStaticPodSecretsVersion is the annotation key for the static pod secret version.
+	AnnotationStaticPodSecretsVersion = "talos.dev/secrets-version"
+
+	// AnnotationStaticPodConfigVersion is the annotation key for the static pod config version.
+	AnnotationStaticPodConfigVersion = "talos.dev/config-version"
 )
 
 // See https://linux.die.net/man/3/klogctl