Skip to content

Commit

Permalink
test: add test for the second machine
Browse files Browse the repository at this point in the history 
More to come.

Signed-off-by: Alexey Palazhchenko <alexey.palazhchenko@talos-systems.com>
  • Loading branch information
@AlekSi
AlekSi committed Sep 9, 2021
1 parent e5b7738 commit 56fb73b
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 52 deletions.
63 changes: 61 additions & 2 deletions internal/integration/helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ package integration

import (
"context"
"crypto/x509"
"encoding/pem"
"flag"
"fmt"
"os"
Expand All @@ -16,8 +18,11 @@ import (

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/talos-systems/talos/pkg/machinery/config"
talosclient "github.com/talos-systems/talos/pkg/machinery/client"
talosclientconfig "github.com/talos-systems/talos/pkg/machinery/client/config"
machineconfig "github.com/talos-systems/talos/pkg/machinery/config"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/scheme"
capiv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -67,6 +72,10 @@ func createCluster(ctx context.Context, t *testing.T, c client.Client, namespace
},
Spec: capiv1.ClusterSpec{
ClusterNetwork: &capiv1.ClusterNetwork{},
ControlPlaneEndpoint: capiv1.APIEndpoint{
Host: clusterName + ".host",
Port: 12345,
},
},
}

Expand Down Expand Up @@ -134,6 +143,56 @@ func createTalosConfig(ctx context.Context, t *testing.T, c client.Client, machi
return talosConfig
}

// waitForReady waits for TalosConfig to be reconciled (ready).
func waitForReady(ctx context.Context, t *testing.T, c client.Client, talosConfig *bootstrapv1alpha3.TalosConfig) {
t.Helper()

for ctx.Err() == nil {
key := types.NamespacedName{
Namespace: talosConfig.Namespace,
Name: talosConfig.Name,
}

err := c.Get(ctx, key, talosConfig)
require.NoError(t, err)

if talosConfig.Status.Ready {
break
}

t.Log("Waiting ...")
sleepCtx(ctx, 3*time.Second)
}
}

// parsePEMCertificate parses PEM-encoded x509 certificate.
func parsePEMCertificate(t *testing.T, b []byte) *x509.Certificate {
block, rest := pem.Decode(b)
assert.Empty(t, rest)
require.NotEmpty(t, block.Bytes)
cert, err := x509.ParseCertificate(block.Bytes)
require.NoError(t, err)
return cert
}

// validateClientConfig validates talosctl configuration.
func validateClientConfig(t *testing.T, config *talosclientconfig.Config) *talosclient.Credentials {
t.Helper()

assert.Len(t, config.Contexts, 1)
assert.NotEmpty(t, config.Context)
context := config.Contexts[config.Context]
require.NotNil(t, context)

assert.Empty(t, context.Endpoints)
assert.Empty(t, context.Nodes)
creds, err := talosclient.CredentialsFromConfigContext(context)
require.NoError(t, err)
assert.NotEmpty(t, creds.CA)

return creds
}

type runtimeMode struct {
requiresInstall bool
}
Expand All @@ -147,4 +206,4 @@ func (m runtimeMode) RequiresInstall() bool {
}

// check interface
var _ config.RuntimeMode = runtimeMode{}
var _ machineconfig.RuntimeMode = runtimeMode{}
129 changes: 79 additions & 50 deletions internal/integration/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,15 @@ package integration

import (
"testing"
"time"

"github.com/AlekSi/pointer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/talos-systems/talos/pkg/machinery/client"
clientconfig "github.com/talos-systems/talos/pkg/machinery/client/config"
"github.com/talos-systems/talos/pkg/machinery/config"
talosclientconfig "github.com/talos-systems/talos/pkg/machinery/client/config"
machineconfig "github.com/talos-systems/talos/pkg/machinery/config"
"github.com/talos-systems/talos/pkg/machinery/config/configloader"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/generate"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
)
Expand All @@ -30,76 +29,106 @@ func TestIntegration(t *testing.T) {
cluster := createCluster(ctx, t, c, namespaceName)
machine := createMachine(ctx, t, c, cluster)
talosConfig := createTalosConfig(ctx, t, c, machine)
waitForReady(ctx, t, c, talosConfig)

// wait for TalosConfig to be reconciled
for ctx.Err() == nil {
key := types.NamespacedName{
Namespace: namespaceName,
Name: talosConfig.Name,
}

err := c.Get(ctx, key, talosConfig)
// check talosConfig
{
assert.Equal(t, machine.Name+"-bootstrap-data", pointer.GetString(talosConfig.Status.DataSecretName), "%+v", talosConfig)
clientConfig, err := talosclientconfig.FromString(talosConfig.Status.TalosConfig)
require.NoError(t, err)

if talosConfig.Status.Ready {
break
}

t.Log("Waiting ...")
sleepCtx(ctx, 5*time.Second)
creds := validateClientConfig(t, clientConfig)
talosCA := parsePEMCertificate(t, creds.CA)
_ = talosCA
// t.Logf("Talos CA:\n%s", spew.Sdump(talosCA))
}

assert.Equal(t, machine.Name+"-bootstrap-data", pointer.GetString(talosConfig.Status.DataSecretName), "%+v", talosConfig)

clientConfig, err := clientconfig.FromString(talosConfig.Status.TalosConfig)
require.NoError(t, err)
assert.Len(t, clientConfig.Contexts, 1)
assert.NotEmpty(t, clientConfig.Context)
context := clientConfig.Contexts[clientConfig.Context]
require.NotNil(t, context)

assert.Empty(t, context.Endpoints)
assert.Empty(t, context.Nodes)
creds, err := client.CredentialsFromConfigContext(context)
require.NoError(t, err)
assert.NotEmpty(t, creds.CA)

// get <cluster>-ca secret
var caSecret corev1.Secret
key := types.NamespacedName{
Namespace: namespaceName,
Name: cluster.Name + "-ca",
}
require.NoError(t, c.Get(ctx, key, &caSecret))
assert.Len(t, caSecret.Data, 2)
assert.Equal(t, corev1.SecretTypeOpaque, caSecret.Type) // TODO why not SecretTypeTLS?
assert.NotEmpty(t, creds.Crt.Certificate, caSecret.Data[corev1.TLSCertKey]) // TODO decode and load
assert.NotEmpty(t, caSecret.Data[corev1.TLSPrivateKeyKey])

// check <cluster>-ca secret
{
assert.Len(t, caSecret.Data, 2)
assert.Equal(t, corev1.SecretTypeOpaque, caSecret.Type) // TODO why not SecretTypeTLS?
assert.NotEmpty(t, caSecret.Data[corev1.TLSCertKey])
assert.NotEmpty(t, caSecret.Data[corev1.TLSPrivateKeyKey])
kubeCA := parsePEMCertificate(t, caSecret.Data[corev1.TLSCertKey])
_ = kubeCA
// t.Logf("kubeCA:\n%s", spew.Sdump(kubeCA))
}

// get <cluster>-talos secret
var talosSecret corev1.Secret
key = types.NamespacedName{
Namespace: namespaceName,
Name: cluster.Name + "-talos",
}
require.NoError(t, c.Get(ctx, key, &talosSecret))
assert.Len(t, talosSecret.Data, 3)
assert.NotEmpty(t, talosSecret.Data["certs"]) // TODO more tests
assert.NotEmpty(t, talosSecret.Data["kubeSecrets"])
assert.NotEmpty(t, talosSecret.Data["trustdInfo"])

// check <cluster>-talos secret
{
assert.Len(t, talosSecret.Data, 3)
assert.NotEmpty(t, talosSecret.Data["certs"])
assert.NotEmpty(t, talosSecret.Data["kubeSecrets"])
assert.NotEmpty(t, talosSecret.Data["trustdInfo"])
}

// get <machine>-bootstrap-data secret
var bootstrapDataSecret corev1.Secret
key = types.NamespacedName{
Namespace: namespaceName,
Name: machine.Name + "-bootstrap-data",
}
require.NoError(t, c.Get(ctx, key, &bootstrapDataSecret))
assert.Len(t, bootstrapDataSecret.Data, 1)
provider, err := configloader.NewFromBytes(bootstrapDataSecret.Data["value"])
require.NoError(t, err)

provider.(*v1alpha1.Config).ClusterConfig.ControlPlane.Endpoint.Host = "FIXME"
// check <machine>-bootstrap-data secret
var provider machineconfig.Provider
{
assert.Len(t, bootstrapDataSecret.Data, 1)
var err error
provider, err = configloader.NewFromBytes(bootstrapDataSecret.Data["value"])
require.NoError(t, err)
_, err = provider.Validate(runtimeMode{false}, machineconfig.WithStrict())
require.NoError(t, err)
}

// cross-checks
{
secretsBundle := generate.NewSecretsBundleFromConfig(generate.NewClock(), provider)

var certs generate.Certs
require.NoError(t, yaml.Unmarshal(talosSecret.Data["certs"], &certs))
assert.NotEmpty(t, certs.Admin)
certs.Admin = nil
assert.Equal(t, secretsBundle.Certs, &certs)
assert.Equal(t, caSecret.Data[corev1.TLSCertKey], certs.K8s.Crt)

var kubeSecrets generate.Secrets
require.NoError(t, yaml.Unmarshal(talosSecret.Data["kubeSecrets"], &kubeSecrets))
assert.Equal(t, secretsBundle.Secrets, &kubeSecrets)

var trustdInfo generate.TrustdInfo
require.NoError(t, yaml.Unmarshal(talosSecret.Data["trustdInfo"], &trustdInfo))
assert.Equal(t, secretsBundle.TrustdInfo, &trustdInfo)
}

// create the second machine
machine2 := createMachine(ctx, t, c, cluster)
talosConfig2 := createTalosConfig(ctx, t, c, machine2)
waitForReady(ctx, t, c, talosConfig2)

// get <machine>-bootstrap-data secret
var bootstrapDataSecret2 corev1.Secret
key = types.NamespacedName{
Namespace: namespaceName,
Name: machine2.Name + "-bootstrap-data",
}
require.NoError(t, c.Get(ctx, key, &bootstrapDataSecret2))

// TODO more tests
_, err = provider.Validate(runtimeMode{false}, config.WithStrict())
require.NoError(t, err)
assert.Equal(t, bootstrapDataSecret.Data, bootstrapDataSecret2.Data) // ?!
})
}

0 comments on commit 56fb73b

Please sign in to comment.