diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go index 2843928f31..e444aa40b3 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go @@ -13,6 +13,7 @@ import ( "log" "os" "path/filepath" + "strings" "time" "github.com/AlekSi/pointer" @@ -121,6 +122,10 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru secretsResources, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { + if err = ctrl.cleanupPods(logger, nil); err != nil { + return fmt.Errorf("error cleaning up static pods: %w", err) + } + continue } @@ -141,6 +146,10 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru if bootstrapStatus.(*v1alpha1.BootstrapStatus).Status().SelfHostedControlPlane { logger.Print("skipped as running self-hosted control plane") + if err = ctrl.cleanupPods(logger, nil); err != nil { + return fmt.Errorf("error cleaning up static pods: %w", err) + } + continue } @@ -162,19 +171,31 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru for _, staticPod := range staticPods.Items { switch staticPod.Metadata().Phase() { case resource.PhaseRunning: - if err = ctrl.runPod(ctx, r, logger, staticPod.(*k8s.StaticPod)); err != nil { + if err = ctrl.writePod(ctx, r, logger, staticPod); err != nil { return fmt.Errorf("error running pod: %w", err) } case resource.PhaseTearingDown: - if err = ctrl.teardownPod(logger, staticPod.(*k8s.StaticPod)); err != nil { + if err = ctrl.teardownPod(logger, staticPod); err != nil { return fmt.Errorf("error tearing down pod: %w", err) } } } + + if err = ctrl.cleanupPods(logger, staticPods.Items); err != nil { + return fmt.Errorf("error cleaning up static pods: %w", err) + } } } -func (ctrl *KubeletStaticPodController) runPod(ctx context.Context, r controller.Runtime, logger *log.Logger, staticPod *k8s.StaticPod) error { +func (ctrl *KubeletStaticPodController) podPath(staticPod resource.Resource) string { + return filepath.Join(constants.ManifestsDirectory, ctrl.podFilename(staticPod)) +} + +func (ctrl *KubeletStaticPodController) podFilename(staticPod resource.Resource) string { + return fmt.Sprintf("%s%s.yaml", constants.TalosManifestPrefix, staticPod.Metadata().ID()) +} + +func (ctrl *KubeletStaticPodController) writePod(ctx context.Context, r controller.Runtime, logger *log.Logger, staticPod resource.Resource) error { staticPodStatus := k8s.NewStaticPodStatus(staticPod.Metadata().Namespace(), staticPod.Metadata().ID()) if err := r.AddFinalizer(ctx, staticPod.Metadata(), staticPodStatus.String()); err != nil { @@ -186,7 +207,7 @@ func (ctrl *KubeletStaticPodController) runPod(ctx context.Context, r controller return nil } - podPath := filepath.Join(constants.ManifestsDirectory, fmt.Sprintf("%s.yaml", staticPod.Metadata().ID())) + podPath := ctrl.podPath(staticPod) existingPod, err := ioutil.ReadFile(podPath) if err != nil { @@ -204,8 +225,8 @@ func (ctrl *KubeletStaticPodController) runPod(ctx context.Context, r controller return ioutil.WriteFile(podPath, renderedPod, 0o600) } -func (ctrl *KubeletStaticPodController) teardownPod(logger *log.Logger, staticPod *k8s.StaticPod) error { - podPath := filepath.Join(constants.ManifestsDirectory, fmt.Sprintf("%s.yaml", staticPod.Metadata().ID())) +func (ctrl *KubeletStaticPodController) teardownPod(logger *log.Logger, staticPod resource.Resource) error { + podPath := ctrl.podPath(staticPod) _, err := os.Stat(podPath) if err != nil { @@ -225,6 +246,47 @@ func (ctrl *KubeletStaticPodController) teardownPod(logger *log.Logger, staticPo return nil } +func (ctrl *KubeletStaticPodController) cleanupPods(logger *log.Logger, staticPods []resource.Resource) error { + manifestDir, err := os.Open(constants.ManifestsDirectory) + if err != nil { + return fmt.Errorf("error opening manifests directory: %w", err) + } + + defer manifestDir.Close() //nolint: errcheck + + manifests, err := manifestDir.Readdirnames(0) + if err != nil { + return fmt.Errorf("error listing manifests: %w", err) + } + + expectedManifests := map[string]struct{}{} + + for _, staticPod := range staticPods { + expectedManifests[ctrl.podFilename(staticPod)] = struct{}{} + } + + for _, manifest := range manifests { + // skip manifests + if !strings.HasPrefix(manifest, constants.TalosManifestPrefix) { + continue + } + + if _, expected := expectedManifests[manifest]; expected { + continue + } + + podPath := filepath.Join(constants.ManifestsDirectory, manifest) + + logger.Printf("cleaning up static pod %q", podPath) + + if err = os.Remove(podPath); err != nil { + return fmt.Errorf("error cleaning up static pod: %w", err) + } + } + + return nil +} + func (ctrl *KubeletStaticPodController) teardownStatuses(ctx context.Context, r controller.Runtime) error { statuses, err := r.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.StaticPodStatusType, "", resource.VersionUndefined)) if err != nil { diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go index 8799cfd83b..7c3b89d60d 100644 --- a/pkg/machinery/constants/constants.go +++ b/pkg/machinery/constants/constants.go @@ -223,6 +223,9 @@ const ( // ManifestsDirectory is the directory that contains all static manifests. ManifestsDirectory = "/etc/kubernetes/manifests" + // TalosManifestPrefix is the prefix for static pod files created in ManifestsDirectory by Talos. + TalosManifestPrefix = "talos-" + // KubeletKubeconfig is the generated kubeconfig for kubelet. KubeletKubeconfig = "/etc/kubernetes/kubeconfig-kubelet"