diff --git a/.drone.yml b/.drone.yml index 3e75e0201947..447d09358e81 100644 --- a/.drone.yml +++ b/.drone.yml @@ -644,6 +644,10 @@ steps: UPGRADE_CHANNEL="latest" else UPGRADE_CHANNEL=$(echo $DRONE_BRANCH | sed 's/release-/v/') + # Check if the UPGRADE_CHANNEL exists, in the case of new minor releases it won't + if ! curl --head --silent --fail https://update.k3s.io/v1-release/channels/$UPGRADE_CHANNEL; then + UPGRADE_CHANNEL="latest" + fi fi E2E_RELEASE_CHANNEL=$UPGRADE_CHANNEL go test -v -timeout=45m ./upgradecluster_test.go -ci -local cp ./coverage.out /tmp/artifacts/upgrade-coverage.out diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 80f4af12cc8a..684051d199c1 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -52,7 +52,7 @@ jobs: with: path: | ~/.vagrant.d/boxes - key: vagrant-box-ubuntu-2204 + key: vagrant-box-ubuntu-2404 - name: "Vagrant Plugin(s)" run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml index 94a19cec247f..d9a4bf7b8a1d 100644 --- a/.github/workflows/install.yaml +++ b/.github/workflows/install.yaml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - vm: [centos-7, rocky-8, rocky-9, fedora, opensuse-leap, ubuntu-2204] + vm: [centos-9, rocky-8, rocky-9, fedora, opensuse-leap, ubuntu-2404] max-parallel: 3 defaults: run: @@ -65,7 +65,7 @@ jobs: vagrant ssh -c "sudo mv /tmp/k3s /usr/local/bin/k3s" vagrant provision --provision-with=k3s-upload - name: Add binary to PATH - if: matrix.vm == 'centos-7' || matrix.vm == 'rocky-8' || matrix.vm == 'rocky-9' || matrix.vm == 'opensuse-leap' + if: matrix.vm == 'centos-9' || matrix.vm == 'rocky-8' || matrix.vm == 'rocky-9' || matrix.vm == 'opensuse-leap' run: vagrant provision --provision-with=add-bin-path - name: "⏩ Install K3s" run: | diff --git a/.github/workflows/nightly-install.yaml b/.github/workflows/nightly-install.yaml index c318fb0b3b0a..a721678bf442 100644 --- a/.github/workflows/nightly-install.yaml +++ b/.github/workflows/nightly-install.yaml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: channel: [stable, latest] - vm: [rocky-8, fedora, opensuse-leap, ubuntu-2204] + vm: [rocky-9, fedora, opensuse-leap, ubuntu-2404] max-parallel: 4 defaults: run: diff --git a/Dockerfile.test b/Dockerfile.test index 7bbc78536b52..607b04725174 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -40,7 +40,7 @@ FROM vagrantlibvirt/vagrant-libvirt:0.12.1 AS test-e2e RUN apt-get update && apt-get install -y docker.io ENV VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 RUN vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp -RUN vagrant box add generic/ubuntu2204 --provider libvirt --force +RUN vagrant box add bento/ubuntu-24.04 --provider libvirt --force RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \ chmod +x ./kubectl; \ mv ./kubectl /usr/local/bin/kubectl diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index d0d0b0252f4e..d12d6063ca96 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -227,8 +227,6 @@ type Control struct { ClusterInit bool ClusterReset bool ClusterResetRestorePath string - EncryptForce bool - EncryptSkip bool MinTLSVersion string CipherSuites []string TLSMinVersion uint16 `json:"-"` diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 7dd5ae64fa87..993bb2cfc591 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -14,7 +14,6 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/control/deps" "github.com/k3s-io/k3s/pkg/daemons/executor" - "github.com/k3s-io/k3s/pkg/secretsencrypt" "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" @@ -61,18 +60,6 @@ func Server(ctx context.Context, cfg *config.Control) error { if err := apiServer(ctx, cfg); err != nil { return err } - if cfg.EncryptSecrets { - controllerName := "reencrypt-secrets" - cfg.Runtime.ClusterControllerStarts[controllerName] = func(ctx context.Context) { - // cfg.Runtime.Core is populated before this callback is triggered - if err := secretsencrypt.Register(ctx, - controllerName, - cfg, - cfg.Runtime.Core.Core().V1().Node()); err != nil { - logrus.Errorf("Failed to register %s controller: %v", controllerName, err) - } - } - } } // Wait for an apiserver to become available before starting additional controllers, diff --git a/pkg/secretsencrypt/config.go b/pkg/secretsencrypt/config.go index 382a66731142..aae309d8fbad 100644 --- a/pkg/secretsencrypt/config.go +++ b/pkg/secretsencrypt/config.go @@ -27,13 +27,19 @@ import ( ) const ( - EncryptionStart string = "start" - EncryptionPrepare string = "prepare" - EncryptionRotate string = "rotate" - EncryptionRotateKeys string = "rotate_keys" - EncryptionReencryptRequest string = "reencrypt_request" - EncryptionReencryptActive string = "reencrypt_active" - EncryptionReencryptFinished string = "reencrypt_finished" + EncryptionStart string = "start" + EncryptionPrepare string = "prepare" + EncryptionRotate string = "rotate" + EncryptionRotateKeys string = "rotate_keys" + EncryptionReencryptRequest string = "reencrypt_request" + EncryptionReencryptActive string = "reencrypt_active" + EncryptionReencryptFinished string = "reencrypt_finished" + SecretListPageSize int64 = 20 + SecretQPS float32 = 200 + SecretBurst int = 200 + SecretsUpdateErrorEvent string = "SecretsUpdateError" + SecretsProgressEvent string = "SecretsProgress" + SecretsUpdateCompleteEvent string = "SecretsUpdateComplete" ) var EncryptionHashAnnotation = version.Program + ".io/encryption-config-hash" @@ -178,7 +184,9 @@ func BootstrapEncryptionHashAnnotation(node *corev1.Node, runtime *config.Contro return nil } -func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, stage string) error { +// WriteEncryptionHashAnnotation writes the encryption hash to the node annotation and optionally to a file. +// The file is used to track the last stage of the reencryption process. +func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1.Node, skipFile bool, stage string) error { encryptionConfigHash, err := GenEncryptionConfigHash(runtime) if err != nil { return err @@ -192,6 +200,9 @@ func WriteEncryptionHashAnnotation(runtime *config.ControlRuntime, node *corev1. return err } logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name) + if skipFile { + return nil + } return os.WriteFile(runtime.EncryptionHash, []byte(ann), 0600) } @@ -253,10 +264,11 @@ func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics b return true, err } tsMetric := mf["apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds"] - successMetric := mf["apiserver_encryption_config_controller_automatic_reload_success_total"] + // Potentially multiple metrics with different success/failure labels + totalMetrics := mf["apiserver_encryption_config_controller_automatic_reloads_total"] // First time, no metrics exist, so return zeros - if tsMetric == nil && successMetric == nil && initialMetrics { + if tsMetric == nil && totalMetrics == nil && initialMetrics { return true, nil } @@ -265,8 +277,8 @@ func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics b return false, nil } - if successMetric == nil { - lastFailure = "encryption config success metric not found" + if totalMetrics == nil { + lastFailure = "encryption config total metric not found" return false, nil } @@ -275,8 +287,14 @@ func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics b return true, fmt.Errorf("encryption reload time is incorrectly ahead of current time") } - reloadSuccessCounter = int64(successMetric.GetMetric()[0].GetCounter().GetValue()) - + for _, totalMetric := range totalMetrics.GetMetric() { + logrus.Debugf("totalMetric: %+v", totalMetric) + for _, label := range totalMetric.GetLabel() { + if label.GetName() == "status" && label.GetValue() == "success" { + reloadSuccessCounter = int64(totalMetric.GetCounter().GetValue()) + } + } + } return true, nil }) diff --git a/pkg/secretsencrypt/controller.go b/pkg/secretsencrypt/controller.go deleted file mode 100644 index ac820fdb798f..000000000000 --- a/pkg/secretsencrypt/controller.go +++ /dev/null @@ -1,246 +0,0 @@ -package secretsencrypt - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/k3s-io/k3s/pkg/cluster" - "github.com/k3s-io/k3s/pkg/daemons/config" - "github.com/k3s-io/k3s/pkg/util" - coreclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1" - "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/pager" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/retry" -) - -const ( - controllerAgentName string = "reencrypt-controller" - secretsUpdateStartEvent string = "SecretsUpdateStart" - secretsProgressEvent string = "SecretsProgress" - secretsUpdateCompleteEvent string = "SecretsUpdateComplete" - secretsUpdateErrorEvent string = "SecretsUpdateError" - - secretListPageSize = 20 -) - -type handler struct { - ctx context.Context - controlConfig *config.Control - nodes coreclient.NodeController - k8s *kubernetes.Clientset - recorder record.EventRecorder -} - -func Register( - ctx context.Context, - controllerName string, - controlConfig *config.Control, - nodes coreclient.NodeController, -) error { - restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor) - if err != nil { - return err - } - // For secrets we need a much higher QPS than what wrangler provides, so we create a new clientset - restConfig.QPS = 200 - restConfig.Burst = 200 - k8s, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return err - } - - h := &handler{ - ctx: ctx, - controlConfig: controlConfig, - nodes: nodes, - k8s: k8s, - recorder: util.BuildControllerEventRecorder(k8s, controllerAgentName, metav1.NamespaceDefault), - } - - nodes.OnChange(ctx, controllerName, h.onChangeNode) - return nil -} - -// onChangeNode handles changes to Nodes. We are looking for a specific annotation change -func (h *handler) onChangeNode(nodeName string, node *corev1.Node) (*corev1.Node, error) { - if node == nil { - return nil, nil - } - - ann, ok := node.Annotations[EncryptionHashAnnotation] - if !ok { - return node, nil - } - - // This is consistent with events attached to the node generated by the kubelet - // https://github.com/kubernetes/kubernetes/blob/612130dd2f4188db839ea5c2dea07a96b0ad8d1c/pkg/kubelet/kubelet.go#L479-L485 - nodeRef := &corev1.ObjectReference{ - Kind: "Node", - Name: node.Name, - UID: types.UID(node.Name), - Namespace: "", - } - - if valid, err := h.validateReencryptStage(node, ann); err != nil { - h.recorder.Event(nodeRef, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) - return node, err - } else if !valid { - return node, nil - } - - reencryptHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive) - if err != nil { - h.recorder.Event(nodeRef, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) - return node, err - } - ann = EncryptionReencryptActive + "-" + reencryptHash - - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - node, err = h.nodes.Get(nodeName, metav1.GetOptions{}) - if err != nil { - return err - } - node.Annotations[EncryptionHashAnnotation] = ann - _, err = h.nodes.Update(node) - return err - }) - if err != nil { - h.recorder.Event(nodeRef, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) - return node, err - } - - if err := h.updateSecrets(nodeRef); err != nil { - h.recorder.Event(nodeRef, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) - return node, err - } - - // If skipping, revert back to the previous stage - if h.controlConfig.EncryptSkip { - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - node, err = h.nodes.Get(nodeName, metav1.GetOptions{}) - if err != nil { - return err - } - BootstrapEncryptionHashAnnotation(node, h.controlConfig.Runtime) - _, err = h.nodes.Update(node) - return err - }) - return node, err - } - - // Remove last key - curKeys, err := GetEncryptionKeys(h.controlConfig.Runtime, false) - if err != nil { - h.recorder.Event(nodeRef, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) - return node, err - } - - logrus.Infoln("Removing key: ", curKeys[len(curKeys)-1]) - curKeys = curKeys[:len(curKeys)-1] - if err = WriteEncryptionConfig(h.controlConfig.Runtime, curKeys, true); err != nil { - h.recorder.Event(nodeRef, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) - return node, err - } - - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - node, err = h.nodes.Get(nodeName, metav1.GetOptions{}) - if err != nil { - return err - } - return WriteEncryptionHashAnnotation(h.controlConfig.Runtime, node, EncryptionReencryptFinished) - }) - if err != nil { - h.recorder.Event(nodeRef, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) - return node, err - } - if err := cluster.Save(h.ctx, h.controlConfig, true); err != nil { - h.recorder.Event(nodeRef, corev1.EventTypeWarning, secretsUpdateErrorEvent, err.Error()) - return node, err - } - return node, nil -} - -// validateReencryptStage ensures that the request for reencryption is valid and -// that there is only one active reencryption at a time -func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (bool, error) { - split := strings.Split(annotation, "-") - if len(split) != 2 { - err := fmt.Errorf("invalid annotation %s found on node %s", annotation, node.ObjectMeta.Name) - return false, err - } - stage := split[0] - hash := split[1] - - // Validate the specific stage and the request via sha256 hash - if stage != EncryptionReencryptRequest { - return false, nil - } - if reencryptRequestHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptRequest); err != nil { - return false, err - } else if reencryptRequestHash != hash { - err = fmt.Errorf("invalid hash: %s found on node %s", hash, node.ObjectMeta.Name) - return false, err - } - reencryptActiveHash, err := GenReencryptHash(h.controlConfig.Runtime, EncryptionReencryptActive) - if err != nil { - return false, err - } - labelSelector := labels.Set{util.ControlPlaneRoleLabelKey: "true"}.String() - nodes, err := h.nodes.List(metav1.ListOptions{LabelSelector: labelSelector}) - if err != nil { - return false, err - } - for _, node := range nodes.Items { - if ann, ok := node.Annotations[EncryptionHashAnnotation]; ok { - split := strings.Split(ann, "-") - if len(split) != 2 { - return false, fmt.Errorf("invalid annotation %s found on node %s", ann, node.ObjectMeta.Name) - } - stage := split[0] - hash := split[1] - if stage == EncryptionReencryptActive && hash == reencryptActiveHash { - return false, fmt.Errorf("another reencrypt is already active") - } - } - } - return true, nil -} - -func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error { - secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { - return h.k8s.CoreV1().Secrets(metav1.NamespaceAll).List(h.ctx, opts) - })) - secretPager.PageSize = secretListPageSize - - i := 0 - if err := secretPager.EachListItem(h.ctx, metav1.ListOptions{}, func(obj runtime.Object) error { - secret, ok := obj.(*corev1.Secret) - if !ok { - return errors.New("failed to convert object to Secret") - } - if _, err := h.k8s.CoreV1().Secrets(secret.Namespace).Update(h.ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) { - return fmt.Errorf("failed to update secret: %v", err) - } - if i != 0 && i%50 == 0 { - h.recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i) - } - i++ - return nil - }); err != nil { - return err - } - - h.recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsUpdateCompleteEvent, "completed reencrypt of %d secrets", i) - return nil -} diff --git a/pkg/server/secrets-encrypt.go b/pkg/server/secrets-encrypt.go index 3172ae8970c3..9436ca086691 100644 --- a/pkg/server/secrets-encrypt.go +++ b/pkg/server/secrets-encrypt.go @@ -17,11 +17,19 @@ import ( "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/secretsencrypt" "github.com/k3s-io/k3s/pkg/util" + "github.com/pkg/errors" "github.com/rancher/wrangler/v3/pkg/generated/controllers/core" "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" apiserverconfigv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/pager" "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" ) @@ -150,8 +158,7 @@ func encryptionEnable(ctx context.Context, server *config.Control, enable bool) if err := cluster.Save(ctx, server, true); err != nil { return err } - server.EncryptSkip = true - return setReencryptAnnotation(server) + return reencryptAndRemoveKey(ctx, server, true, os.Getenv("NODE_NAME")) } func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler { @@ -219,7 +226,7 @@ func encryptionPrepare(ctx context.Context, server *config.Control, force bool) if err != nil { return err } - return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionPrepare) + return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, false, secretsencrypt.EncryptionPrepare) }) if err != nil { return err @@ -250,7 +257,7 @@ func encryptionRotate(ctx context.Context, server *config.Control, force bool) e if err != nil { return err } - return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, secretsencrypt.EncryptionRotate) + return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, false, secretsencrypt.EncryptionRotate) }) if err != nil { return err @@ -262,25 +269,20 @@ func encryptionReencrypt(ctx context.Context, server *config.Control, force bool if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), secretsencrypt.EncryptionRotate); err != nil && !force { return err } - server.EncryptForce = force - server.EncryptSkip = skip + // Set the reencrypt-active annotation so other nodes know we are in the process of reencrypting. + // As this stage is not persisted, we do not write the annotation to file nodeName := os.Getenv("NODE_NAME") - node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) - if err != nil { + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, true, secretsencrypt.EncryptionReencryptActive) + }); err != nil { return err } - reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest) - if err != nil { - return err - } - ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash - node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann - if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil { - return err - } - logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name) - return nil + return reencryptAndRemoveKey(ctx, server, skip, nodeName) } func addAndRotateKeys(server *config.Control) error { @@ -321,6 +323,19 @@ func encryptionRotateKeys(ctx context.Context, server *config.Control) error { return err } + // Set the reencrypt-active annotation so other nodes know we are in the process of reencrypting. + // As this stage is not persisted, we do not write the annotation to file + nodeName := os.Getenv("NODE_NAME") + if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, true, secretsencrypt.EncryptionReencryptActive) + }); err != nil { + return err + } + if err := addAndRotateKeys(server); err != nil { return err } @@ -329,26 +344,100 @@ func encryptionRotateKeys(ctx context.Context, server *config.Control) error { return err } - return setReencryptAnnotation(server) + return reencryptAndRemoveKey(ctx, server, false, nodeName) } -func setReencryptAnnotation(server *config.Control) error { - nodeName := os.Getenv("NODE_NAME") - node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) +func reencryptAndRemoveKey(ctx context.Context, server *config.Control, skip bool, nodeName string) error { + if err := updateSecrets(ctx, server, nodeName); err != nil { + return err + } + + // If skipping, revert back to the previous stage and do not remove the key + if skip { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + secretsencrypt.BootstrapEncryptionHashAnnotation(node, server.Runtime) + _, err = server.Runtime.Core.Core().V1().Node().Update(node) + return err + }) + return err + } + + // Remove last key + curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime, false) if err != nil { return err } - reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest) + logrus.Infoln("Removing key: ", curKeys[len(curKeys)-1]) + curKeys = curKeys[:len(curKeys)-1] + if err = secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil { + return err + } + + if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + return secretsencrypt.WriteEncryptionHashAnnotation(server.Runtime, node, false, secretsencrypt.EncryptionReencryptFinished) + }); err != nil { + return err + } + + return cluster.Save(ctx, server, true) +} + +func updateSecrets(ctx context.Context, server *config.Control, nodeName string) error { + restConfig, err := clientcmd.BuildConfigFromFlags("", server.Runtime.KubeConfigSupervisor) + if err != nil { + return err + } + // For secrets we need a much higher QPS than default + restConfig.QPS = secretsencrypt.SecretQPS + restConfig.Burst = secretsencrypt.SecretBurst + k8s, err := kubernetes.NewForConfig(restConfig) if err != nil { return err } - ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash - node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann - if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil { + + nodeRef := &corev1.ObjectReference{ + Kind: "Node", + Name: nodeName, + UID: types.UID(nodeName), + Namespace: "", + } + + // For backwards compatibility with the old controller, we use an event recorder instead of logrus + recorder := util.BuildControllerEventRecorder(k8s, "secrets-reencrypt", metav1.NamespaceDefault) + + secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { + return k8s.CoreV1().Secrets(metav1.NamespaceAll).List(ctx, opts) + })) + secretPager.PageSize = secretsencrypt.SecretListPageSize + + i := 0 + if err := secretPager.EachListItem(ctx, metav1.ListOptions{}, func(obj runtime.Object) error { + secret, ok := obj.(*corev1.Secret) + if !ok { + return errors.New("failed to convert object to Secret") + } + if _, err := k8s.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) { + recorder.Eventf(nodeRef, corev1.EventTypeWarning, secretsencrypt.SecretsUpdateErrorEvent, "failed to update secret: %v", err) + return fmt.Errorf("failed to update secret: %v", err) + } + if i != 0 && i%50 == 0 { + recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsencrypt.SecretsProgressEvent, "reencrypted %d secrets", i) + } + i++ + return nil + }); err != nil { return err } - logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name) + recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsencrypt.SecretsUpdateCompleteEvent, "reencrypted %d secrets", i) return nil } diff --git a/tests/TESTING.md b/tests/TESTING.md index a49921922827..467d756e5f3a 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -71,12 +71,12 @@ ___ Install tests are a collection of tests defined under the [tests/install](./tests/install). These tests are used to validate the installation and operation of K3s on a variety of operating systems. The test themselves are Vagrantfiles describing single-node installations that are easily spun up with Vagrant for the `libvirt` and `virtualbox` providers: - [Install Script](install) :arrow_right: scheduled nightly and on an install script change - - [CentOS 7](install/centos-7) (stand-in for RHEL 7) + - [CentOS 9 Stream](install/centos-stream) - [Rocky Linux 8](install/rocky-8) (stand-in for RHEL 8) - [Rocky Linux 9](install/rocky-9) (stand-in for RHEL 9) - - [Fedora 37](install/fedora) - - [Leap 15.5](install/opensuse-leap) (stand-in for SLES) - - [Ubuntu 22.04](install/ubuntu-2204) + - [Fedora 40](install/fedora) + - [Leap 15.6](install/opensuse-leap) (stand-in for SLES) + - [Ubuntu 24.04](install/ubuntu-2404) ## Format When adding new installer test(s) please copy the prevalent style for the `Vagrantfile`. diff --git a/tests/e2e/dualstack/Vagrantfile b/tests/e2e/dualstack/Vagrantfile index 1c419af67acb..73fc39506f1e 100644 --- a/tests/e2e/dualstack/Vagrantfile +++ b/tests/e2e/dualstack/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0" ]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go index f6019e063870..c9612f9b7142 100644 --- a/tests/e2e/dualstack/dualstack_test.go +++ b/tests/e2e/dualstack/dualstack_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") diff --git a/tests/e2e/embeddedmirror/Vagrantfile b/tests/e2e/embeddedmirror/Vagrantfile index 67bc1709f128..76f0f6204bbc 100644 --- a/tests/e2e/embeddedmirror/Vagrantfile +++ b/tests/e2e/embeddedmirror/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/embeddedmirror/embeddedmirror_test.go b/tests/e2e/embeddedmirror/embeddedmirror_test.go index 6eb0e9284413..7188b552b988 100644 --- a/tests/e2e/embeddedmirror/embeddedmirror_test.go +++ b/tests/e2e/embeddedmirror/embeddedmirror_test.go @@ -13,9 +13,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2310, generic/centos7, generic/rocky8, -// opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +// eurolinux-vagrant/rocky-8, eurolinux-vagrant/rocky-9, +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") diff --git a/tests/e2e/externalip/Vagrantfile b/tests/e2e/externalip/Vagrantfile index 742922e35cdf..b378f7fe74ce 100644 --- a/tests/e2e/externalip/Vagrantfile +++ b/tests/e2e/externalip/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/externalip/externalip_test.go b/tests/e2e/externalip/externalip_test.go index 47e15530b15c..524bb8340276 100644 --- a/tests/e2e/externalip/externalip_test.go +++ b/tests/e2e/externalip/externalip_test.go @@ -17,8 +17,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") diff --git a/tests/e2e/privateregistry/Vagrantfile b/tests/e2e/privateregistry/Vagrantfile index 09d7c2d70634..8300f81b30ab 100644 --- a/tests/e2e/privateregistry/Vagrantfile +++ b/tests/e2e/privateregistry/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/privateregistry/privateregistry_test.go b/tests/e2e/privateregistry/privateregistry_test.go index 5c65807a952b..88a357a24abe 100644 --- a/tests/e2e/privateregistry/privateregistry_test.go +++ b/tests/e2e/privateregistry/privateregistry_test.go @@ -13,9 +13,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2310, generic/centos7, generic/rocky8, -// opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +// eurolinux-vagrant/rocky-8, eurolinux-vagrant/rocky-9, +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") diff --git a/tests/e2e/rootless/Vagrantfile b/tests/e2e/rootless/Vagrantfile index 905958d55818..04f095d7828f 100644 --- a/tests/e2e/rootless/Vagrantfile +++ b/tests/e2e/rootless/Vagrantfile @@ -1,6 +1,6 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0"]) -NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2310']) +NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/rootless/rootless_test.go b/tests/e2e/rootless/rootless_test.go index 6528130ea4a4..361778c72db7 100644 --- a/tests/e2e/rootless/rootless_test.go +++ b/tests/e2e/rootless/rootless_test.go @@ -14,8 +14,8 @@ import ( // Rootless is only valid on a single node, but requires node/kernel configuration, requiring a E2E test environment. -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var ci = flag.Bool("ci", false, "running on CI") var local = flag.Bool("local", false, "deploy a locally built K3s binary") diff --git a/tests/e2e/rotateca/Vagrantfile b/tests/e2e/rotateca/Vagrantfile index 89cc90e64080..7f5fd10f83d8 100644 --- a/tests/e2e/rotateca/Vagrantfile +++ b/tests/e2e/rotateca/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/rotateca/rotateca_test.go b/tests/e2e/rotateca/rotateca_test.go index 9bf0f1430b37..c43ab4d10899 100644 --- a/tests/e2e/rotateca/rotateca_test.go +++ b/tests/e2e/rotateca/rotateca_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") diff --git a/tests/e2e/s3/Vagrantfile b/tests/e2e/s3/Vagrantfile index 75c44426607f..8297ad645228 100644 --- a/tests/e2e/s3/Vagrantfile +++ b/tests/e2e/s3/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310']) + ['bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/s3/s3_test.go b/tests/e2e/s3/s3_test.go index ac203f63d4e0..fc3be6a5fde4 100644 --- a/tests/e2e/s3/s3_test.go +++ b/tests/e2e/s3/s3_test.go @@ -14,9 +14,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2310, generic/centos7, generic/rocky8, -// opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +// eurolinux-vagrant/rocky-8, eurolinux-vagrant/rocky-9, +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var ci = flag.Bool("ci", false, "running on CI") var local = flag.Bool("local", false, "deploy a locally built K3s binary") diff --git a/tests/e2e/scripts/run_tests.sh b/tests/e2e/scripts/run_tests.sh index 87238ac69578..d26b732599aa 100755 --- a/tests/e2e/scripts/run_tests.sh +++ b/tests/e2e/scripts/run_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -nodeOS=${1:-"generic/ubuntu2310"} +nodeOS=${1:-"bento/ubuntu-24.04"} servercount=${2:-3} agentcount=${3:-1} db=${4:-"etcd"} diff --git a/tests/e2e/secretsencryption/Vagrantfile b/tests/e2e/secretsencryption/Vagrantfile index 628310df7609..f73a4ad60d7d 100644 --- a/tests/e2e/secretsencryption/Vagrantfile +++ b/tests/e2e/secretsencryption/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go index 992deba77e0d..187dcedba2fc 100644 --- a/tests/e2e/secretsencryption/secretsencryption_test.go +++ b/tests/e2e/secretsencryption/secretsencryption_test.go @@ -15,8 +15,8 @@ import ( // This test is desigened for the new secrets-encrypt rotate-keys command, // Added in v1.28.0+k3s1 -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var hardened = flag.Bool("hardened", false, "true or false") var ci = flag.Bool("ci", false, "running on CI") diff --git a/tests/e2e/secretsencryption_old/Vagrantfile b/tests/e2e/secretsencryption_old/Vagrantfile index 28ef2628c0c1..954ef232e525 100644 --- a/tests/e2e/secretsencryption_old/Vagrantfile +++ b/tests/e2e/secretsencryption_old/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/secretsencryption_old/secretsencryption_test.go b/tests/e2e/secretsencryption_old/secretsencryption_test.go index 940d42ccd498..c7b1f3c98870 100644 --- a/tests/e2e/secretsencryption_old/secretsencryption_test.go +++ b/tests/e2e/secretsencryption_old/secretsencryption_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var hardened = flag.Bool("hardened", false, "true or false") var ci = flag.Bool("ci", false, "running on CI") diff --git a/tests/e2e/snapshotrestore/Vagrantfile b/tests/e2e/snapshotrestore/Vagrantfile index 504aa3e8af67..6e9cac5f9613 100644 --- a/tests/e2e/snapshotrestore/Vagrantfile +++ b/tests/e2e/snapshotrestore/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/snapshotrestore/snapshotrestore_test.go b/tests/e2e/snapshotrestore/snapshotrestore_test.go index a5ce53a2da2d..dc47907f78c7 100644 --- a/tests/e2e/snapshotrestore/snapshotrestore_test.go +++ b/tests/e2e/snapshotrestore/snapshotrestore_test.go @@ -14,10 +14,10 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2310, generic/centos7, generic/rocky8, -// opensuse/Leap-15.3.x86_64 +// bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +// eurolinux-vagrant/rocky-8, eurolinux-vagrant/rocky-9, -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") diff --git a/tests/e2e/splitserver/Vagrantfile b/tests/e2e/splitserver/Vagrantfile index 118538e3dfb5..73a65904fe68 100644 --- a/tests/e2e/splitserver/Vagrantfile +++ b/tests/e2e/splitserver/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-etcd-0", "server-etcd-1", "server-etcd-2", "server-cp-0", "server-cp-1", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/splitserver/splitserver_test.go b/tests/e2e/splitserver/splitserver_test.go index c386bad219e1..c78520d67b41 100644 --- a/tests/e2e/splitserver/splitserver_test.go +++ b/tests/e2e/splitserver/splitserver_test.go @@ -16,8 +16,8 @@ import ( "golang.org/x/sync/errgroup" ) -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var etcdCount = flag.Int("etcdCount", 3, "number of server nodes only deploying etcd") var controlPlaneCount = flag.Int("controlPlaneCount", 1, "number of server nodes acting as control plane") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") diff --git a/tests/e2e/startup/Vagrantfile b/tests/e2e/startup/Vagrantfile index eb48e393f4af..94d8a3f203c7 100644 --- a/tests/e2e/startup/Vagrantfile +++ b/tests/e2e/startup/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index 8b7fa01f7edf..c926164fac14 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var ci = flag.Bool("ci", false, "running on CI") var local = flag.Bool("local", false, "deploy a locally built K3s binary") diff --git a/tests/e2e/tailscale/Vagrantfile b/tests/e2e/tailscale/Vagrantfile index e93fc0d90be2..31b2b22a8a83 100644 --- a/tests/e2e/tailscale/Vagrantfile +++ b/tests/e2e/tailscale/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0", "agent-1" ]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/tailscale/tailscale_test.go b/tests/e2e/tailscale/tailscale_test.go index 576be7dab219..3def1ac41ab5 100644 --- a/tests/e2e/tailscale/tailscale_test.go +++ b/tests/e2e/tailscale/tailscale_test.go @@ -11,8 +11,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") diff --git a/tests/e2e/token/Vagrantfile b/tests/e2e/token/Vagrantfile index f17a9be79bac..d0530b8b9c15 100644 --- a/tests/e2e/token/Vagrantfile +++ b/tests/e2e/token/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 2790714b6f0d..bd0cc38a1fc8 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -14,9 +14,10 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2310, generic/centos7, generic/rocky8, opensuse/Leap-15.6.x86_64 +// bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +// eurolinux-vagrant/rocky-8, eurolinux-vagrant/rocky-9, -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") diff --git a/tests/e2e/upgradecluster/Vagrantfile b/tests/e2e/upgradecluster/Vagrantfile index ec2275802c16..214cc5cdcd16 100644 --- a/tests/e2e/upgradecluster/Vagrantfile +++ b/tests/e2e/upgradecluster/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) RELEASE_CHANNEL = (ENV['E2E_RELEASE_CHANNEL'] || "latest") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd") diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index c3516a8fa287..18bd1cbee7b1 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -13,9 +13,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2310, generic/centos7, generic/rocky8 -// opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// bento/ubuntu-24.04, eurolinux-vagrant/rocky-8, eurolinux-vagrant/rocky-9 +// opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") diff --git a/tests/e2e/vagrantdefaults.rb b/tests/e2e/vagrantdefaults.rb index f773fd6d4ce4..6841db377003 100644 --- a/tests/e2e/vagrantdefaults.rb +++ b/tests/e2e/vagrantdefaults.rb @@ -1,12 +1,10 @@ def defaultOSConfigure(vm) box = vm.box.to_s - if box.include?("generic/ubuntu") + if box.include?("ubuntu") vm.provision "Set DNS", type: "shell", inline: "netplan set ethernets.eth0.nameservers.addresses=[8.8.8.8,1.1.1.1]; netplan apply", run: 'once' elsif box.include?("Leap") || box.include?("Tumbleweed") vm.provision "Install apparmor-parser", type: "shell", inline: "zypper install -y apparmor-parser" - elsif box.include?("rocky8") || box.include?("rocky9") - vm.provision "Disable firewall", type: "shell", inline: "systemctl stop firewalld" - elsif box.include?("centos7") + elsif box.include?("rocky") || box.include?("centos") vm.provision "Disable firewall", type: "shell", inline: "systemctl stop firewalld" elsif box.include?("alpine") vm.provision "Install tools", type: "shell", inline: "apk add coreutils" @@ -78,7 +76,7 @@ def getHardenedArg(vm, hardened, scripts_location) puts "Invalid E2E_HARDENED option" exit 1 end - if vm.box.to_s.include?("generic/ubuntu") + if vm.box.to_s.include?("ubuntu") vm.provision "Install kube-bench", type: "shell", inline: <<-SHELL export KBV=0.8.0 curl -L "https://github.com/aquasecurity/kube-bench/releases/download/v${KBV}/kube-bench_${KBV}_linux_amd64.deb" -o "kube-bench_${KBV}_linux_amd64.deb" @@ -90,13 +88,13 @@ def getHardenedArg(vm, hardened, scripts_location) def jqInstall(vm) box = vm.box.to_s - if box.include?("generic/ubuntu") + if box.include?("ubuntu") vm.provision "Install jq", type: "shell", inline: "apt install -y jq" elsif box.include?("Leap") || box.include?("Tumbleweed") vm.provision "Install jq", type: "shell", inline: "zypper install -y jq" - elsif box.include?("rocky8") || box.include?("rocky9") + elsif box.include?("rocky") vm.provision "Install jq", type: "shell", inline: "dnf install -y jq" - elsif box.include?("centos7") + elsif box.include?("centos") vm.provision "Install jq", type: "shell", inline: "yum install -y jq" elsif box.include?("alpine") vm.provision "Install jq", type: "shell", inline: "apk add coreutils" @@ -122,7 +120,7 @@ def dockerInstall(vm) vm.provision "shell", inline: "transactional-update pkg install -y docker apparmor-parser" vm.provision 'docker-reload', type: 'reload', run: 'once' vm.provision "shell", inline: "systemctl enable --now docker" - elsif box.include?("rocky8") || box.include?("rocky9") + elsif box.include?("rocky") vm.provision "shell", inline: "dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo" vm.provision "shell", inline: "dnf install -y docker-ce" end diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index 63bd28a780c8..80d99d67afdb 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310', 'generic/ubuntu2310']) + ['bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04', 'bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd") diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 8853e9f42565..accae34dadf8 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -14,9 +14,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2310, generic/centos7, generic/rocky8, -// opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +// eurolinux-vagrant/rocky-8, eurolinux-vagrant/rocky-9, +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") diff --git a/tests/e2e/wasm/Vagrantfile b/tests/e2e/wasm/Vagrantfile index 405d4e1050e3..eea5355b3cd7 100644 --- a/tests/e2e/wasm/Vagrantfile +++ b/tests/e2e/wasm/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2310']) + ['bento/ubuntu-24.04']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") diff --git a/tests/e2e/wasm/wasm_test.go b/tests/e2e/wasm/wasm_test.go index 5ae84525296e..1e887a086a29 100644 --- a/tests/e2e/wasm/wasm_test.go +++ b/tests/e2e/wasm/wasm_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2310, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2310", "VM operating system") +// Valid nodeOS: bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64 +var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 0, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") diff --git a/tests/install/centos-7/Vagrantfile b/tests/install/centos-9/Vagrantfile similarity index 93% rename from tests/install/centos-7/Vagrantfile rename to tests/install/centos-9/Vagrantfile index f7be36b3a138..b2a5080ced78 100644 --- a/tests/install/centos-7/Vagrantfile +++ b/tests/install/centos-9/Vagrantfile @@ -6,14 +6,14 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-k3s"] - config.vm.box = "generic/centos7" + config.vm.box = "eurolinux-vagrant/centos-stream-9" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true # Load in helper functions load "../install_util.rb" - config.vm.define 'install-centos-7', primary: true do |test| + config.vm.define 'install-centos-9', primary: true do |test| test.vm.hostname = 'smoke' test.vm.provision "add-bin-path", type: "shell", inline: "echo \"export PATH=/usr/local/bin:\$PATH\" >> ~/.bashrc" test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' diff --git a/tests/install/fedora/Vagrantfile b/tests/install/fedora/Vagrantfile index 09bf843dcfca..67d4074e4399 100644 --- a/tests/install/fedora/Vagrantfile +++ b/tests/install/fedora/Vagrantfile @@ -6,7 +6,7 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-k3s"] - config.vm.box = 'generic/fedora37' + config.vm.box = 'bento/fedora-latest' config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true diff --git a/tests/install/rocky-8/Vagrantfile b/tests/install/rocky-8/Vagrantfile index cc755ec89579..3015ee5400ee 100644 --- a/tests/install/rocky-8/Vagrantfile +++ b/tests/install/rocky-8/Vagrantfile @@ -6,7 +6,7 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-k3s"] - config.vm.box = "generic/rocky8" + config.vm.box = "bento/rockylinux-8" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true diff --git a/tests/install/rocky-9/Vagrantfile b/tests/install/rocky-9/Vagrantfile index 9ec4bba82f0c..4b70fa4e12bd 100644 --- a/tests/install/rocky-9/Vagrantfile +++ b/tests/install/rocky-9/Vagrantfile @@ -7,7 +7,7 @@ ENV['INSTALL_K3S_CHANNEL'] ||= 'testing' Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-k3s"] - config.vm.box = "generic/rocky9" + config.vm.box = "eurolinux-vagrant/rocky-9" config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true diff --git a/tests/install/ubuntu-2204/Vagrantfile b/tests/install/ubuntu-2404/Vagrantfile similarity index 93% rename from tests/install/ubuntu-2204/Vagrantfile rename to tests/install/ubuntu-2404/Vagrantfile index 7b23ebeff657..c09c4d419b85 100644 --- a/tests/install/ubuntu-2204/Vagrantfile +++ b/tests/install/ubuntu-2404/Vagrantfile @@ -6,14 +6,14 @@ ENV['TEST_INSTALL_SH'] ||= '../../../install.sh' Vagrant.configure("2") do |config| config.vagrant.plugins = ["vagrant-k3s"] - config.vm.box = 'generic/ubuntu2204' + config.vm.box = 'bento/ubuntu-24.04' config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds config.vm.synced_folder '.', '/vagrant', disabled: true # Load in helper functions load "../install_util.rb" - config.vm.define 'install-ubuntu-2204', primary: true do |test| + config.vm.define 'install-ubuntu-2404', primary: true do |test| test.vm.hostname = 'smoke' test.vm.provision 'k3s-upload', type: 'file', run: 'always', source: ENV['TEST_INSTALL_SH'], destination: 'install.sh' test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s|