From e5b773833120fdd7ca4d57e0a0a4fe781495bf7e Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Thu, 9 Sep 2021 10:56:04 +0000 Subject: [PATCH] test: add more tests More to come. Signed-off-by: Alexey Palazhchenko --- Makefile | 1 + controllers/talosconfig_controller.go | 1 + go.mod | 1 + go.sum | 1 + internal/integration/helpers_test.go | 150 +++++++++++++++++++++++ internal/integration/integration_test.go | 143 +++++++++++---------- internal/integration/setup_test.go | 45 ++----- 7 files changed, 238 insertions(+), 104 deletions(-) create mode 100644 internal/integration/helpers_test.go diff --git a/Makefile b/Makefile index 41e5453..58a394a 100644 --- a/Makefile +++ b/Makefile @@ -135,6 +135,7 @@ env-up: talosctl ## Start development environment. --name=cabpt-env \ --kubernetes-version=$(K8S_VERSION) \ --mtu=1450 \ + --skip-kubeconfig \ --crashdump ./talosctl kubeconfig kubeconfig \ --talosconfig=talosconfig \ diff --git a/controllers/talosconfig_controller.go b/controllers/talosconfig_controller.go index f7abcd9..2b2c3af 100644 --- a/controllers/talosconfig_controller.go +++ b/controllers/talosconfig_controller.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" bootstrapv1alpha3 "github.com/talos-systems/cluster-api-bootstrap-provider-talos/api/v1alpha3" + // +kubebuilder:scaffold:imports ) const ( diff --git a/go.mod b/go.mod index 7286958..80925fa 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/talos-systems/cluster-api-bootstrap-provider-talos go 1.16 require ( + github.com/AlekSi/pointer v1.1.0 github.com/evanphx/json-patch v4.11.0+incompatible github.com/go-logr/logr v0.1.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 7f2b014..56ab15b 100644 --- a/go.sum +++ b/go.sum @@ -578,6 +578,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/integration/helpers_test.go b/internal/integration/helpers_test.go new file mode 100644 index 0000000..25ff8a7 --- /dev/null +++ b/internal/integration/helpers_test.go @@ -0,0 +1,150 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package integration + +import ( + "context" + "flag" + "fmt" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/talos-systems/talos/pkg/machinery/config" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + capiv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + bootstrapv1alpha3 "github.com/talos-systems/cluster-api-bootstrap-provider-talos/api/v1alpha3" + // +kubebuilder:scaffold:imports +) + +var skipCleanup bool + +func init() { + const env = "INTEGRATION_SKIP_CLEANUP" + def, _ := strconv.ParseBool(os.Getenv(env)) + flag.BoolVar(&skipCleanup, "skip-cleanup", def, fmt.Sprintf("Cleanup after tests [%s]", env)) +} + +// sleepCtx blocks until ctx is canceled or timeout passed. +func sleepCtx(ctx context.Context, timeout time.Duration) { + sCtx, sCancel := context.WithTimeout(ctx, timeout) + defer sCancel() + <-sCtx.Done() +} + +// generateName generates a unique name. +func generateName(t *testing.T, kind string) string { + // use milliseconds since UTC midnight: unique enough, short enough, ordered + now := time.Now().UTC() + clock := time.Duration(now.Hour())*time.Hour + + time.Duration(now.Minute())*time.Minute + + time.Duration(now.Second())*time.Second + + time.Duration(now.Nanosecond()) + n := clock / time.Microsecond + + return fmt.Sprintf("%s-%s-%d", strings.ReplaceAll(strings.ToLower(t.Name()), "/", "-"), kind, n) +} + +// createCluster creates a Cluster with "ready" infrastructure. +func createCluster(ctx context.Context, t *testing.T, c client.Client, namespaceName string) *capiv1.Cluster { + t.Helper() + + clusterName := generateName(t, "cluster") + cluster := &capiv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaceName, + Name: clusterName, + }, + Spec: capiv1.ClusterSpec{ + ClusterNetwork: &capiv1.ClusterNetwork{}, + }, + } + + require.NoError(t, c.Create(ctx, cluster), "can't create a cluster") + + cluster.Status.InfrastructureReady = true + require.NoError(t, c.Status().Update(ctx, cluster)) + + return cluster +} + +// createMachine creates a Machine owned by the Cluster. +func createMachine(ctx context.Context, t *testing.T, c client.Client, cluster *capiv1.Cluster) *capiv1.Machine { + t.Helper() + + machineName := generateName(t, "machine") + dataSecretName := "my-test-secret" + machine := &capiv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: machineName, + }, + Spec: capiv1.MachineSpec{ + ClusterName: cluster.Name, + Bootstrap: capiv1.Bootstrap{ + DataSecretName: &dataSecretName, // TODO + }, + }, + } + + require.NoError(t, controllerutil.SetOwnerReference(cluster, machine, scheme.Scheme)) + + require.NoError(t, c.Create(ctx, machine)) + + return machine +} + +// createTalosConfig creates a TalosConfig owned by the Machine. +func createTalosConfig(ctx context.Context, t *testing.T, c client.Client, machine *capiv1.Machine) *bootstrapv1alpha3.TalosConfig { + t.Helper() + + talosConfigName := generateName(t, "talosconfig") + talosConfig := &bootstrapv1alpha3.TalosConfig{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: machine.Namespace, + Name: talosConfigName, + }, + Spec: bootstrapv1alpha3.TalosConfigSpec{ + GenerateType: "init", + }, + } + + require.NoError(t, controllerutil.SetOwnerReference(machine, talosConfig, scheme.Scheme)) + + require.NoError(t, c.Create(ctx, talosConfig)) + + // TODO that should not be needed + if !skipCleanup { + t.Cleanup(func() { + t.Logf("Deleting TalosConfig %q ...", talosConfigName) + assert.NoError(t, c.Delete(context.Background(), talosConfig)) // not ctx because it can be already canceled + }) + } + + return talosConfig +} + +type runtimeMode struct { + requiresInstall bool +} + +func (m runtimeMode) String() string { + return fmt.Sprintf("runtimeMode(%v)", m.requiresInstall) +} + +func (m runtimeMode) RequiresInstall() bool { + return m.requiresInstall +} + +// check interface +var _ config.RuntimeMode = runtimeMode{} diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index 322b2a1..733a35f 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -8,99 +8,98 @@ import ( "testing" "time" + "github.com/AlekSi/pointer" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/talos-systems/talos/pkg/machinery/client" + clientconfig "github.com/talos-systems/talos/pkg/machinery/client/config" + "github.com/talos-systems/talos/pkg/machinery/config" + "github.com/talos-systems/talos/pkg/machinery/config/configloader" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - capiv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - bootstrapv1alpha3 "github.com/talos-systems/cluster-api-bootstrap-provider-talos/api/v1alpha3" - // +kubebuilder:scaffold:imports ) func TestIntegration(t *testing.T) { ctx, c := setupSuite(t) - // namespaced objects - var ( - clusterName = "test-cluster" - machineName = "test-machine" - dataSecretName = "test-secret" - talosConfigName = "test-config" - ) - t.Run("Basic", func(t *testing.T) { t.Parallel() - namespaceName := setupTest(ctx, t, c) - - cluster := &capiv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespaceName, - Name: clusterName, - }, - Spec: capiv1.ClusterSpec{ - ClusterNetwork: &capiv1.ClusterNetwork{ - Pods: &capiv1.NetworkRanges{ - CIDRBlocks: []string{"192.168.0.0/16"}, - }, - ServiceDomain: "cluster.local", - Services: &capiv1.NetworkRanges{ - CIDRBlocks: []string{"10.128.0.0/12"}, - }, - }, - }, - } - require.NoError(t, c.Create(ctx, cluster), "can't create a cluster") - - cluster.Status.InfrastructureReady = true - require.NoError(t, c.Status().Update(ctx, cluster)) - - machine := &capiv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespaceName, - Name: machineName, - }, - Spec: capiv1.MachineSpec{ - ClusterName: cluster.Name, - Bootstrap: capiv1.Bootstrap{ - DataSecretName: &dataSecretName, - }, - }, - } - - require.NoError(t, controllerutil.SetOwnerReference(cluster, machine, scheme.Scheme)) - require.NoError(t, c.Create(ctx, machine)) - - config := &bootstrapv1alpha3.TalosConfig{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespaceName, - Name: talosConfigName, - }, - Spec: bootstrapv1alpha3.TalosConfigSpec{ - GenerateType: "init", - }, - } - require.NoError(t, controllerutil.SetOwnerReference(machine, config, scheme.Scheme)) - err := c.Create(ctx, config) - require.NoError(t, err) + namespaceName := setupTest(ctx, t, c) + cluster := createCluster(ctx, t, c, namespaceName) + machine := createMachine(ctx, t, c, cluster) + talosConfig := createTalosConfig(ctx, t, c, machine) + // wait for TalosConfig to be reconciled for ctx.Err() == nil { key := types.NamespacedName{ Namespace: namespaceName, - Name: talosConfigName, + Name: talosConfig.Name, } - err = c.Get(ctx, key, config) + err := c.Get(ctx, key, talosConfig) require.NoError(t, err) - if config.Status.Ready { + if talosConfig.Status.Ready { break } - t.Logf("Config: %+v", config) - time.Sleep(5 * time.Second) + t.Log("Waiting ...") + sleepCtx(ctx, 5*time.Second) + } + + assert.Equal(t, machine.Name+"-bootstrap-data", pointer.GetString(talosConfig.Status.DataSecretName), "%+v", talosConfig) + + clientConfig, err := clientconfig.FromString(talosConfig.Status.TalosConfig) + require.NoError(t, err) + assert.Len(t, clientConfig.Contexts, 1) + assert.NotEmpty(t, clientConfig.Context) + context := clientConfig.Contexts[clientConfig.Context] + require.NotNil(t, context) + + assert.Empty(t, context.Endpoints) + assert.Empty(t, context.Nodes) + creds, err := client.CredentialsFromConfigContext(context) + require.NoError(t, err) + assert.NotEmpty(t, creds.CA) + + var caSecret corev1.Secret + key := types.NamespacedName{ + Namespace: namespaceName, + Name: cluster.Name + "-ca", + } + require.NoError(t, c.Get(ctx, key, &caSecret)) + assert.Len(t, caSecret.Data, 2) + assert.Equal(t, corev1.SecretTypeOpaque, caSecret.Type) // TODO why not SecretTypeTLS? + assert.NotEmpty(t, creds.Crt.Certificate, caSecret.Data[corev1.TLSCertKey]) // TODO decode and load + assert.NotEmpty(t, caSecret.Data[corev1.TLSPrivateKeyKey]) + + var talosSecret corev1.Secret + key = types.NamespacedName{ + Namespace: namespaceName, + Name: cluster.Name + "-talos", } + require.NoError(t, c.Get(ctx, key, &talosSecret)) + assert.Len(t, talosSecret.Data, 3) + assert.NotEmpty(t, talosSecret.Data["certs"]) // TODO more tests + assert.NotEmpty(t, talosSecret.Data["kubeSecrets"]) + assert.NotEmpty(t, talosSecret.Data["trustdInfo"]) + + var bootstrapDataSecret corev1.Secret + key = types.NamespacedName{ + Namespace: namespaceName, + Name: machine.Name + "-bootstrap-data", + } + require.NoError(t, c.Get(ctx, key, &bootstrapDataSecret)) + assert.Len(t, bootstrapDataSecret.Data, 1) + provider, err := configloader.NewFromBytes(bootstrapDataSecret.Data["value"]) + require.NoError(t, err) + + provider.(*v1alpha1.Config).ClusterConfig.ControlPlane.Endpoint.Host = "FIXME" + + // TODO more tests + _, err = provider.Validate(runtimeMode{false}, config.WithStrict()) + require.NoError(t, err) }) } diff --git a/internal/integration/setup_test.go b/internal/integration/setup_test.go index 29b8a91..78a5365 100644 --- a/internal/integration/setup_test.go +++ b/internal/integration/setup_test.go @@ -6,16 +6,13 @@ package integration import ( "context" - "flag" - "fmt" - "os" "os/signal" "path/filepath" - "strconv" "strings" "testing" "time" + "github.com/AlekSi/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" @@ -23,7 +20,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" - "k8s.io/utils/pointer" clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client" "sigs.k8s.io/cluster-api/util/patch" ctrl "sigs.k8s.io/controller-runtime" @@ -36,14 +32,6 @@ import ( // +kubebuilder:scaffold:imports ) -var skipCleanupF bool - -func init() { - const env = "INTEGRATION_SKIP_CLEANUP" - def, _ := strconv.ParseBool(os.Getenv(env)) - flag.BoolVar(&skipCleanupF, "skip-cleanup", def, fmt.Sprintf("Cleanup after tests [%s]", env)) -} - // setupSuite setups the whole test suite. func setupSuite(t *testing.T) (context.Context, client.Client) { t.Helper() @@ -54,7 +42,7 @@ func setupSuite(t *testing.T) (context.Context, client.Client) { ctx := context.Background() - if !skipCleanupF { + if !skipCleanup { // cancel context on first Ctrl+C, kill on second var stop context.CancelFunc ctx, stop = signal.NotifyContext(context.Background(), unix.SIGTERM, unix.SIGINT) @@ -113,7 +101,7 @@ func setupSuite(t *testing.T) (context.Context, client.Client) { func setupTest(ctx context.Context, t *testing.T, c client.Client) string { t.Helper() - namespace := fmt.Sprintf("%s-%d", strings.ToLower(strings.ReplaceAll(t.Name(), "/", "-")), time.Now().Unix()) + namespace := generateName(t, "ns") ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -123,15 +111,14 @@ func setupTest(ctx context.Context, t *testing.T, c client.Client) string { err := c.Create(ctx, ns) require.NoError(t, err) - if !skipCleanupF { + if !skipCleanup { t.Cleanup(func() { opts := &client.DeleteOptions{ - GracePeriodSeconds: pointer.Int64Ptr(0), + GracePeriodSeconds: pointer.ToInt64(0), } t.Logf("Deleting namespace %q ...", namespace) assert.NoError(t, c.Delete(context.Background(), ns, opts)) // not ctx because it can be already canceled - t.Logf("Namespace %q deleted.", namespace) }) } @@ -197,7 +184,7 @@ func startTestEnv(ctx context.Context, t *testing.T) *rest.Config { ErrorIfPathMissing: true, MaxTime: 20 * time.Second, PollInterval: time.Second, - CleanUpAfterUse: !skipCleanupF, + CleanUpAfterUse: !skipCleanup, }, WebhookInstallOptions: envtest.WebhookInstallOptions{ // TODO paths? @@ -206,7 +193,7 @@ func startTestEnv(ctx context.Context, t *testing.T) *rest.Config { PollInterval: time.Second, }, ErrorIfCRDPathMissing: true, - UseExistingCluster: pointer.BoolPtr(true), + UseExistingCluster: pointer.ToBool(true), } // Run Start in the goroutine to handle context cancelation. @@ -217,7 +204,7 @@ func startTestEnv(ctx context.Context, t *testing.T) *rest.Config { } startErr := make(chan result, 1) go func() { - if !skipCleanupF { + if !skipCleanup { t.Cleanup(func() { t.Log("Stopping test-env ...") @@ -265,7 +252,7 @@ func stopCAPI(ctx context.Context, t *testing.T, c client.Client) { patchHelper, err := patch.NewHelper(&deployment, c) require.NoError(t, err) - deployment.Spec.Replicas = pointer.Int32Ptr(0) + deployment.Spec.Replicas = pointer.ToInt32(0) require.NoError(t, patchHelper.Patch(ctx, &deployment)) @@ -279,11 +266,8 @@ func stopCAPI(ctx context.Context, t *testing.T, c client.Client) { } t.Logf("Waiting: %+v ...", deployment.Status) - - select { - case <-time.After(5 * time.Second): - // nothing, continue - case <-ctx.Done(): + sleepCtx(ctx, 5*time.Second) + if ctx.Err() != nil { t.Fatalf("Failed to stop CAPI components: %s.", ctx.Err()) } } @@ -322,11 +306,8 @@ func waitForCAPIAvailability(ctx context.Context, t *testing.T, c client.Client) } t.Logf("Waiting: %+v ...", deployment.Status) - - select { - case <-time.After(5 * time.Second): - // nothing, continue - case <-ctx.Done(): + sleepCtx(ctx, 5*time.Second) + if ctx.Err() != nil { t.Fatalf("Failed to wait for CAPI availability: %s.", ctx.Err()) } }