From b914398154e21d41bf301abde0272e49cde331f4 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 18 Feb 2021 21:31:40 +0300 Subject: [PATCH] refactor: split kubernetes/etcd resource generation into subresources Fixes #3062 There's no user-visible change in this PR. It carefully separates generated secrets (e.g. certs) from source secrets from the config (e.g. CAs), so that certs are generated on config changes which actually affect cert input. And same way separates etcd and Kubernetes PKI, so if etcd CA got changed, only etcd certs will be regenerated. This should have noticeable impact with RSA-based PKI as it reduces number of times PKI gets generated. Signed-off-by: Andrey Smirnov --- .../k8s/kubelet_static_pod_controller.go | 2 +- .../machined/pkg/controllers/k8s/manifest.go | 14 +- .../pkg/controllers/k8s/manifest_apply.go | 2 +- .../k8s/render_secrets_static_pod.go | 79 ++++++-- .../machined/pkg/controllers/k8s/templates.go | 2 +- .../machined/pkg/controllers/secrets/etcd.go | 146 +++++++++++++++ .../pkg/controllers/secrets/kubernetes.go | 144 ++++++--------- .../machined/pkg/controllers/secrets/root.go | 172 ++++++++++++++++++ .../runtime/v1alpha2/v1alpha2_controller.go | 2 + .../pkg/runtime/v1alpha2/v1alpha2_state.go | 2 + internal/pkg/kubeconfig/admin.go | 13 +- pkg/resources/secrets/etcd.go | 78 ++++++++ pkg/resources/secrets/kubernetes.go | 33 ++-- pkg/resources/secrets/root.go | 112 ++++++++++++ 14 files changed, 662 insertions(+), 139 deletions(-) create mode 100644 internal/app/machined/pkg/controllers/secrets/etcd.go create mode 100644 internal/app/machined/pkg/controllers/secrets/root.go create mode 100644 pkg/resources/secrets/etcd.go create mode 100644 pkg/resources/secrets/root.go diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go index 870409547a..3ebb7064de 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go @@ -132,7 +132,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru return err } - secrets := secretsResources.(*secrets.Kubernetes).Secrets() + secrets := secretsResources.(*secrets.Kubernetes).Certs() bootstrapStatus, err := r.Get(ctx, v1alpha1.NewBootstrapStatus().Metadata()) if err != nil { diff --git a/internal/app/machined/pkg/controllers/k8s/manifest.go b/internal/app/machined/pkg/controllers/k8s/manifest.go index f3950b75b2..e42336d645 100644 --- a/internal/app/machined/pkg/controllers/k8s/manifest.go +++ b/internal/app/machined/pkg/controllers/k8s/manifest.go @@ -47,8 +47,8 @@ func (ctrl *ManifestController) Run(ctx context.Context, r controller.Runtime, l }, { Namespace: secrets.NamespaceName, - Type: secrets.KubernetesType, - ID: pointer.ToString(secrets.KubernetesID), + Type: secrets.RootType, + ID: pointer.ToString(secrets.RootKubernetesID), Kind: controller.DependencyWeak, }, }); err != nil { @@ -77,7 +77,7 @@ func (ctrl *ManifestController) Run(ctx context.Context, r controller.Runtime, l config := configResource.(*config.K8sControlPlane).Manifests() - secretsResources, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined)) + secretsResources, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootKubernetesID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { if err = ctrl.teardownAll(ctx, r); err != nil { @@ -90,9 +90,9 @@ func (ctrl *ManifestController) Run(ctx context.Context, r controller.Runtime, l return err } - secrets := secretsResources.(*secrets.Kubernetes).Secrets() + secrets := secretsResources.(*secrets.Root).KubernetesSpec() - renderedManifests, err := ctrl.render(config, *secrets) + renderedManifests, err := ctrl.render(config, secrets) if err != nil { return err } @@ -137,11 +137,11 @@ type renderedManifest struct { data []byte } -func (ctrl *ManifestController) render(cfg config.K8sManifestsSpec, scrt secrets.KubernetesSpec) ([]renderedManifest, error) { +func (ctrl *ManifestController) render(cfg config.K8sManifestsSpec, scrt *secrets.RootKubernetesSpec) ([]renderedManifest, error) { templateConfig := struct { config.K8sManifestsSpec - Secrets secrets.KubernetesSpec + Secrets *secrets.RootKubernetesSpec }{ K8sManifestsSpec: cfg, Secrets: scrt, diff --git a/internal/app/machined/pkg/controllers/k8s/manifest_apply.go b/internal/app/machined/pkg/controllers/k8s/manifest_apply.go index bab6be6959..57ed26f695 100644 --- a/internal/app/machined/pkg/controllers/k8s/manifest_apply.go +++ b/internal/app/machined/pkg/controllers/k8s/manifest_apply.go @@ -95,7 +95,7 @@ func (ctrl *ManifestApplyController) Run(ctx context.Context, r controller.Runti return err } - secrets := secretsResources.(*secrets.Kubernetes).Secrets() + secrets := secretsResources.(*secrets.Kubernetes).Certs() bootstrapStatus, err := r.Get(ctx, v1alpha1.NewBootstrapStatus().Metadata()) if err != nil { diff --git a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go index 06b86d9fd6..b776f7cdeb 100644 --- a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go +++ b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go @@ -14,6 +14,7 @@ import ( "path/filepath" stdlibtemplate "text/template" + "github.com/AlekSi/pointer" "github.com/talos-systems/crypto/x509" "github.com/talos-systems/os-runtime/pkg/controller" "github.com/talos-systems/os-runtime/pkg/resource" @@ -42,9 +43,21 @@ func (ctrl *RenderSecretsStaticPodController) ManagedResources() (resource.Names //nolint: gocyclo func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r controller.Runtime, logger *log.Logger) error { if err := r.UpdateDependencies([]controller.Dependency{ + { + Namespace: secrets.NamespaceName, + Type: secrets.RootType, + Kind: controller.DependencyWeak, + }, { Namespace: secrets.NamespaceName, Type: secrets.KubernetesType, + ID: pointer.ToString(secrets.KubernetesID), + Kind: controller.DependencyWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.EtcdType, + ID: pointer.ToString(secrets.EtcdID), Kind: controller.DependencyWeak, }, }); err != nil { @@ -67,9 +80,39 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control return fmt.Errorf("error getting secrets resource: %w", err) } - secrets := secretsRes.(*secrets.Kubernetes).Secrets() + etcdRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootEtcdRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootEtcdID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootK8sRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootKubernetesID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootEtcdSecrets := rootEtcdRes.(*secrets.Root).EtcdSpec() + rootK8sSecrets := rootK8sRes.(*secrets.Root).KubernetesSpec() + etcdSecrets := etcdRes.(*secrets.Etcd).Certs() + k8sSecrets := secretsRes.(*secrets.Kubernetes).Certs() - serviceAccountKey, err := secrets.ServiceAccount.GetKey() + serviceAccountKey, err := rootK8sSecrets.ServiceAccount.GetKey() if err != nil { return fmt.Errorf("error parsing service account key: %w", err) } @@ -96,25 +139,25 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control directory: constants.KubernetesAPIServerSecretsDir, secrets: []secret{ { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.EtcdCA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootEtcdSecrets.EtcdCA }, certFilename: "etcd-client-ca.crt", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.EtcdPeer }, + getter: func() *x509.PEMEncodedCertificateAndKey { return etcdSecrets.EtcdPeer }, certFilename: "etcd-client.crt", keyFilename: "etcd-client.key", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.CA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.CA }, certFilename: "ca.crt", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.APIServer }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.APIServer }, certFilename: "apiserver.crt", keyFilename: "apiserver.key", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.APIServerKubeletClient }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.APIServerKubeletClient }, certFilename: "apiserver-kubelet-client.crt", keyFilename: "apiserver-kubelet-client.key", }, @@ -129,11 +172,11 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control keyFilename: "service-account.key", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.AggregatorCA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.AggregatorCA }, certFilename: "aggregator-ca.crt", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.FrontProxy }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.FrontProxy }, certFilename: "front-proxy-client.crt", keyFilename: "front-proxy-client.key", }, @@ -154,7 +197,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control directory: constants.KubernetesControllerManagerSecretsDir, secrets: []secret{ { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.CA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.CA }, certFilename: "ca.crt", keyFilename: "ca.key", }, @@ -171,7 +214,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control templates: []template{ { filename: "kubeconfig", - template: []byte("{{ .AdminKubeconfig }}"), + template: []byte("{{ .Secrets.AdminKubeconfig }}"), }, }, }, @@ -181,7 +224,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control templates: []template{ { filename: "kubeconfig", - template: []byte("{{ .AdminKubeconfig }}"), + template: []byte("{{ .Secrets.AdminKubeconfig }}"), }, }, }, @@ -214,6 +257,16 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control } } + type templateParams struct { + Root *secrets.RootKubernetesSpec + Secrets *secrets.KubernetesCertsSpec + } + + params := templateParams{ + Root: rootK8sSecrets, + Secrets: k8sSecrets, + } + for _, templ := range pod.templates { var t *stdlibtemplate.Template @@ -224,7 +277,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control var buf bytes.Buffer - if err = t.Execute(&buf, secrets); err != nil { + if err = t.Execute(&buf, ¶ms); err != nil { return fmt.Errorf("error executing template %q: %w", templ.filename, err) } diff --git a/internal/app/machined/pkg/controllers/k8s/templates.go b/internal/app/machined/pkg/controllers/k8s/templates.go index fdde37dbec..04a8f7bc14 100644 --- a/internal/app/machined/pkg/controllers/k8s/templates.go +++ b/internal/app/machined/pkg/controllers/k8s/templates.go @@ -15,7 +15,7 @@ resources: - aescbc: keys: - name: key1 - secret: {{ .AESCBCEncryptionSecret }} + secret: {{ .Root.AESCBCEncryptionSecret }} - identity: {} `) diff --git a/internal/app/machined/pkg/controllers/secrets/etcd.go b/internal/app/machined/pkg/controllers/secrets/etcd.go new file mode 100644 index 0000000000..153cc1a293 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/etcd.go @@ -0,0 +1,146 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + "log" + + "github.com/AlekSi/pointer" + "github.com/talos-systems/os-runtime/pkg/controller" + "github.com/talos-systems/os-runtime/pkg/resource" + "github.com/talos-systems/os-runtime/pkg/state" + + "github.com/talos-systems/talos/internal/pkg/etcd" + "github.com/talos-systems/talos/pkg/resources/secrets" + "github.com/talos-systems/talos/pkg/resources/v1alpha1" +) + +// EtcdController manages secrets.Etcd based on configuration. +type EtcdController struct{} + +// Name implements controller.Controller interface. +func (ctrl *EtcdController) Name() string { + return "secrets.EtcdController" +} + +// ManagedResources implements controller.Controller interface. +func (ctrl *EtcdController) ManagedResources() (resource.Namespace, resource.Type) { + return secrets.NamespaceName, secrets.EtcdType +} + +// Run implements controller.Controller interface. +// +//nolint: gocyclo, dupl +func (ctrl *EtcdController) Run(ctx context.Context, r controller.Runtime, logger *log.Logger) error { + if err := r.UpdateDependencies([]controller.Dependency{ + { + Namespace: secrets.NamespaceName, + Type: secrets.RootType, + ID: pointer.ToString(secrets.RootEtcdID), + Kind: controller.DependencyWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: pointer.ToString("networkd"), + Kind: controller.DependencyWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.TimeSyncType, + ID: pointer.ToString(v1alpha1.TimeSyncID), + Kind: controller.DependencyWeak, + }, + }); err != nil { + return fmt.Errorf("error setting up dependencies: %w", err) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + etcdRootRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootEtcdID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting etcd root secrets: %w", err) + } + + etcdRoot := etcdRootRes.(*secrets.Root).EtcdSpec() + + // wait for networkd to be healthy as it might change IPs/hostname + networkdResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "networkd", resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !networkdResource.(*v1alpha1.Service).Healthy() { + continue + } + + // wait for time sync as certs depend on current time + timeSyncResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.TimeSyncType, v1alpha1.TimeSyncID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !timeSyncResource.(*v1alpha1.TimeSync).Sync() { + continue + } + + if err = r.Update(ctx, secrets.NewEtcd(), func(r resource.Resource) error { + return ctrl.updateSecrets(etcdRoot, r.(*secrets.Etcd).Certs()) + }); err != nil { + return err + } + } +} + +func (ctrl *EtcdController) updateSecrets(etcdRoot *secrets.RootEtcdSpec, etcdCerts *secrets.EtcdCertsSpec) error { + var err error + + etcdCerts.EtcdPeer, err = etcd.GeneratePeerCert(etcdRoot.EtcdCA) + if err != nil { + return fmt.Errorf("error generating etcd certs: %w", err) + } + + return nil +} + +func (ctrl *EtcdController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes.go b/internal/app/machined/pkg/controllers/secrets/kubernetes.go index ec80995e68..3abf9d06b9 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubernetes.go +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes.go @@ -10,6 +10,7 @@ import ( "fmt" "log" "net" + "net/url" "time" "github.com/AlekSi/pointer" @@ -18,19 +19,15 @@ import ( "github.com/talos-systems/os-runtime/pkg/resource" "github.com/talos-systems/os-runtime/pkg/state" - "github.com/talos-systems/talos/internal/pkg/etcd" "github.com/talos-systems/talos/internal/pkg/kubeconfig" - talosconfig "github.com/talos-systems/talos/pkg/machinery/config" - "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" + "github.com/talos-systems/talos/pkg/machinery/config" "github.com/talos-systems/talos/pkg/machinery/constants" - "github.com/talos-systems/talos/pkg/resources/config" "github.com/talos-systems/talos/pkg/resources/secrets" "github.com/talos-systems/talos/pkg/resources/v1alpha1" ) // KubernetesController manages secrets.Kubernetes based on configuration. -type KubernetesController struct { -} +type KubernetesController struct{} // Name implements controller.Controller interface. func (ctrl *KubernetesController) Name() string { @@ -44,13 +41,13 @@ func (ctrl *KubernetesController) ManagedResources() (resource.Namespace, resour // Run implements controller.Controller interface. // -//nolint: gocyclo +//nolint: gocyclo, dupl func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, logger *log.Logger) error { if err := r.UpdateDependencies([]controller.Dependency{ - { // TODO: should render config for kubernetes secrets controller - Namespace: config.NamespaceName, - Type: config.V1Alpha1Type, - ID: pointer.ToString(config.V1Alpha1ID), + { + Namespace: secrets.NamespaceName, + Type: secrets.RootType, + ID: pointer.ToString(secrets.RootKubernetesID), Kind: controller.DependencyWeak, }, { @@ -65,12 +62,6 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, ID: pointer.ToString(v1alpha1.TimeSyncID), Kind: controller.DependencyWeak, }, - { - Namespace: config.NamespaceName, - Type: config.MachineTypeType, - ID: pointer.ToString(config.MachineTypeID), - Kind: controller.DependencyWeak, - }, }); err != nil { return fmt.Errorf("error setting up dependencies: %w", err) } @@ -82,39 +73,20 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, case <-r.EventCh(): } - cfg, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.V1Alpha1Type, config.V1Alpha1ID, resource.VersionUndefined)) + k8sRootRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootKubernetesID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { if err = ctrl.teardownAll(ctx, r); err != nil { - return fmt.Errorf("error destroying static pods: %w", err) + return fmt.Errorf("error destroying resources: %w", err) } continue } - return fmt.Errorf("error getting config: %w", err) - } - - cfgProvider := cfg.(*config.V1Alpha1).Config() - - machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) - if err != nil { - if state.IsNotFoundError(err) { - continue - } - - return fmt.Errorf("error getting machine type: %w", err) + return fmt.Errorf("error getting root k8s secrets: %w", err) } - machineType := machineTypeRes.(*config.MachineType).MachineType() - - if machineType != machine.TypeControlPlane && machineType != machine.TypeInit { - if err = ctrl.teardownAll(ctx, r); err != nil { - return fmt.Errorf("error destroying static pods: %w", err) - } - - continue - } + k8sRoot := k8sRootRes.(*secrets.Root).KubernetesSpec() // wait for networkd to be healthy as it might change IPs/hostname networkdResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "networkd", resource.VersionUndefined)) @@ -145,62 +117,29 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, } if err = r.Update(ctx, secrets.NewKubernetes(), func(r resource.Resource) error { - k8sSecrets := r.(*secrets.Kubernetes) //nolint: errcheck - - return ctrl.updateSecrets(cfgProvider, k8sSecrets) + return ctrl.updateSecrets(k8sRoot, r.(*secrets.Kubernetes).Certs()) }); err != nil { return err } } } -//nolint: gocyclo -func (ctrl *KubernetesController) updateSecrets(cfgProvider talosconfig.Provider, k8sSecrets *secrets.Kubernetes) error { - k8sSecrets.Secrets().EtcdCA = cfgProvider.Cluster().Etcd().CA() - - if k8sSecrets.Secrets().EtcdCA == nil { - return fmt.Errorf("missing cluster.etcdCA secret") - } - - k8sSecrets.Secrets().AggregatorCA = cfgProvider.Cluster().AggregatorCA() - - if k8sSecrets.Secrets().AggregatorCA == nil { - return fmt.Errorf("missing cluster.aggregatorCA secret") - } - - k8sSecrets.Secrets().CA = cfgProvider.Cluster().CA() - - if k8sSecrets.Secrets().CA == nil { - return fmt.Errorf("missing cluster.CA secret") - } - - var err error - - k8sSecrets.Secrets().EtcdPeer, err = etcd.GeneratePeerCert(cfgProvider.Cluster().Etcd().CA()) - if err != nil { - return err - } - - urls := []string{cfgProvider.Cluster().Endpoint().Hostname()} - urls = append(urls, cfgProvider.Cluster().CertSANs()...) +func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.RootKubernetesSpec, k8sSecrets *secrets.KubernetesCertsSpec) error { + urls := []string{k8sRoot.Endpoint.Hostname()} + urls = append(urls, k8sRoot.CertSANs...) altNames := altNamesFromURLs(urls) - apiServiceIPs, err := cfgProvider.Cluster().Network().APIServerIPs() - if err != nil { - return fmt.Errorf("failed to calculate API service IP: %w", err) - } - - altNames.IPs = append(altNames.IPs, apiServiceIPs...) + altNames.IPs = append(altNames.IPs, k8sRoot.APIServerIPs...) // Add kubernetes default svc with cluster domain to AltNames altNames.DNSNames = append(altNames.DNSNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", - "kubernetes.default.svc."+cfgProvider.Cluster().Network().DNSDomain(), + "kubernetes.default.svc."+k8sRoot.DNSDomain, ) - ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sSecrets.Secrets().CA) + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.CA) if err != nil { return fmt.Errorf("failed to parse CA certificate: %w", err) } @@ -216,7 +155,7 @@ func (ctrl *KubernetesController) updateSecrets(cfgProvider talosconfig.Provider return fmt.Errorf("failed to generate api-server cert: %w", err) } - k8sSecrets.Secrets().APIServer = x509.NewCertificateAndKeyFromKeyPair(apiServer) + k8sSecrets.APIServer = x509.NewCertificateAndKeyFromKeyPair(apiServer) apiServerKubeletClient, err := x509.NewKeyPair(ca, x509.CommonName(constants.KubernetesAdminCertCommonName), @@ -227,11 +166,9 @@ func (ctrl *KubernetesController) updateSecrets(cfgProvider talosconfig.Provider return fmt.Errorf("failed to generate api-server cert: %w", err) } - k8sSecrets.Secrets().APIServerKubeletClient = x509.NewCertificateAndKeyFromKeyPair(apiServerKubeletClient) + k8sSecrets.APIServerKubeletClient = x509.NewCertificateAndKeyFromKeyPair(apiServerKubeletClient) - k8sSecrets.Secrets().ServiceAccount = cfgProvider.Cluster().ServiceAccount() - - aggregatorCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sSecrets.Secrets().AggregatorCA) + aggregatorCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.AggregatorCA) if err != nil { return fmt.Errorf("failed to parse aggregator CA: %w", err) } @@ -244,20 +181,15 @@ func (ctrl *KubernetesController) updateSecrets(cfgProvider talosconfig.Provider return fmt.Errorf("failed to generate aggregator cert: %w", err) } - k8sSecrets.Secrets().FrontProxy = x509.NewCertificateAndKeyFromKeyPair(frontProxy) - - k8sSecrets.Secrets().AESCBCEncryptionSecret = cfgProvider.Cluster().AESCBCEncryptionSecret() + k8sSecrets.FrontProxy = x509.NewCertificateAndKeyFromKeyPair(frontProxy) var buf bytes.Buffer - if err = kubeconfig.GenerateAdmin(cfgProvider.Cluster(), &buf); err != nil { + if err = kubeconfig.GenerateAdmin(&generateAdminAdapter{k8sRoot: k8sRoot}, &buf); err != nil { return fmt.Errorf("failed to generate admin kubeconfig: %w", err) } - k8sSecrets.Secrets().AdminKubeconfig = buf.String() - - k8sSecrets.Secrets().BootstrapTokenID = cfgProvider.Cluster().Token().ID() - k8sSecrets.Secrets().BootstrapTokenSecret = cfgProvider.Cluster().Token().Secret() + k8sSecrets.AdminKubeconfig = buf.String() return nil } @@ -301,3 +233,29 @@ func altNamesFromURLs(urls []string) *AltNames { return &an } + +// generateAdminAdapter allows to translate input config into GenerateAdmin input. +type generateAdminAdapter struct { + k8sRoot *secrets.RootKubernetesSpec +} + +func (adapter *generateAdminAdapter) Name() string { + return adapter.k8sRoot.Name +} + +func (adapter *generateAdminAdapter) Endpoint() *url.URL { + return adapter.k8sRoot.Endpoint +} + +func (adapter *generateAdminAdapter) CA() *x509.PEMEncodedCertificateAndKey { + return adapter.k8sRoot.CA +} + +func (adapter *generateAdminAdapter) AdminKubeconfig() config.AdminKubeconfig { + return adapter +} + +func (adapter *generateAdminAdapter) CertLifetime() time.Duration { + // this certificate is not delivered to the user, it's used only internally by Talos + return x509.DefaultCertificateValidityDuration +} diff --git a/internal/app/machined/pkg/controllers/secrets/root.go b/internal/app/machined/pkg/controllers/secrets/root.go new file mode 100644 index 0000000000..1bf7be08a8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/root.go @@ -0,0 +1,172 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + "log" + + "github.com/AlekSi/pointer" + "github.com/talos-systems/os-runtime/pkg/controller" + "github.com/talos-systems/os-runtime/pkg/resource" + "github.com/talos-systems/os-runtime/pkg/state" + + talosconfig "github.com/talos-systems/talos/pkg/machinery/config" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" + "github.com/talos-systems/talos/pkg/resources/config" + "github.com/talos-systems/talos/pkg/resources/secrets" +) + +// RootController manages secrets.Root based on configuration. +type RootController struct{} + +// Name implements controller.Controller interface. +func (ctrl *RootController) Name() string { + return "secrets.RootController" +} + +// ManagedResources implements controller.Controller interface. +func (ctrl *RootController) ManagedResources() (resource.Namespace, resource.Type) { + return secrets.NamespaceName, secrets.RootType +} + +// Run implements controller.Controller interface. +// +//nolint: gocyclo +func (ctrl *RootController) Run(ctx context.Context, r controller.Runtime, logger *log.Logger) error { + if err := r.UpdateDependencies([]controller.Dependency{ + { + Namespace: config.NamespaceName, + Type: config.V1Alpha1Type, + ID: pointer.ToString(config.V1Alpha1ID), + Kind: controller.DependencyWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: pointer.ToString(config.MachineTypeID), + Kind: controller.DependencyWeak, + }, + }); err != nil { + return fmt.Errorf("error setting up dependencies: %w", err) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.V1Alpha1Type, config.V1Alpha1ID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying static pods: %w", err) + } + + continue + } + + return fmt.Errorf("error getting config: %w", err) + } + + cfgProvider := cfg.(*config.V1Alpha1).Config() + + machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.(*config.MachineType).MachineType() + + if machineType != machine.TypeControlPlane && machineType != machine.TypeInit { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying secrets: %w", err) + } + + continue + } + + if err = r.Update(ctx, secrets.NewRoot(secrets.RootEtcdID), func(r resource.Resource) error { + return ctrl.updateEtcdSecrets(cfgProvider, r.(*secrets.Root).EtcdSpec()) + }); err != nil { + return err + } + + if err = r.Update(ctx, secrets.NewRoot(secrets.RootKubernetesID), func(r resource.Resource) error { + return ctrl.updateK8sSecrets(cfgProvider, r.(*secrets.Root).KubernetesSpec()) + }); err != nil { + return err + } + } +} + +func (ctrl *RootController) updateEtcdSecrets(cfgProvider talosconfig.Provider, etcdSecrets *secrets.RootEtcdSpec) error { + etcdSecrets.EtcdCA = cfgProvider.Cluster().Etcd().CA() + + if etcdSecrets.EtcdCA == nil { + return fmt.Errorf("missing cluster.etcdCA secret") + } + + return nil +} + +func (ctrl *RootController) updateK8sSecrets(cfgProvider talosconfig.Provider, k8sSecrets *secrets.RootKubernetesSpec) error { + k8sSecrets.Name = cfgProvider.Cluster().Name() + k8sSecrets.Endpoint = cfgProvider.Cluster().Endpoint() + k8sSecrets.CertSANs = cfgProvider.Cluster().CertSANs() + k8sSecrets.DNSDomain = cfgProvider.Cluster().Network().DNSDomain() + + var err error + + k8sSecrets.APIServerIPs, err = cfgProvider.Cluster().Network().APIServerIPs() + if err != nil { + return fmt.Errorf("error building API service IPs: %w", err) + } + + k8sSecrets.AggregatorCA = cfgProvider.Cluster().AggregatorCA() + + if k8sSecrets.AggregatorCA == nil { + return fmt.Errorf("missing cluster.aggregatorCA secret") + } + + k8sSecrets.CA = cfgProvider.Cluster().CA() + + if k8sSecrets.CA == nil { + return fmt.Errorf("missing cluster.CA secret") + } + + k8sSecrets.ServiceAccount = cfgProvider.Cluster().ServiceAccount() + + k8sSecrets.AESCBCEncryptionSecret = cfgProvider.Cluster().AESCBCEncryptionSecret() + + k8sSecrets.BootstrapTokenID = cfgProvider.Cluster().Token().ID() + k8sSecrets.BootstrapTokenSecret = cfgProvider.Cluster().Token().Secret() + + return nil +} + +func (ctrl *RootController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go index 87dff5b948..4919761f5b 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go @@ -62,7 +62,9 @@ func (ctrl *Controller) Run(ctx context.Context) error { &k8s.ManifestController{}, &k8s.ManifestApplyController{}, &k8s.RenderSecretsStaticPodController{}, + &secrets.EtcdController{}, &secrets.KubernetesController{}, + &secrets.RootController{}, } { if err := ctrl.controllerRuntime.RegisterController(c); err != nil { return err diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go index 01e49cb4fb..87312381a7 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go @@ -80,7 +80,9 @@ func NewState() (*State, error) { &k8s.StaticPod{}, &k8s.StaticPodStatus{}, &k8s.SecretsStatus{}, + &secrets.Etcd{}, &secrets.Kubernetes{}, + &secrets.Root{}, } { if err := s.resourceRegistry.Register(ctx, r); err != nil { return nil, err diff --git a/internal/pkg/kubeconfig/admin.go b/internal/pkg/kubeconfig/admin.go index 6bb8f858b3..b559a161be 100644 --- a/internal/pkg/kubeconfig/admin.go +++ b/internal/pkg/kubeconfig/admin.go @@ -8,6 +8,7 @@ import ( "encoding/base64" "fmt" "io" + "net/url" "text/template" "time" @@ -38,8 +39,18 @@ contexts: current-context: admin@{{ .Cluster }} ` +// GenerateAdminInput is the interface for the GenerateAdmin function. +// +// This interface is implemented by config.Cluster(). +type GenerateAdminInput interface { + Name() string + Endpoint() *url.URL + CA() *x509.PEMEncodedCertificateAndKey + AdminKubeconfig() config.AdminKubeconfig +} + // GenerateAdmin generates admin kubeconfig for the cluster. -func GenerateAdmin(config config.ClusterConfig, out io.Writer) error { +func GenerateAdmin(config GenerateAdminInput, out io.Writer) error { tpl, err := template.New("kubeconfig").Parse(adminKubeConfigTemplate) if err != nil { return fmt.Errorf("error parsing kubeconfig template: %w", err) diff --git a/pkg/resources/secrets/etcd.go b/pkg/resources/secrets/etcd.go new file mode 100644 index 0000000000..9b94f71ba7 --- /dev/null +++ b/pkg/resources/secrets/etcd.go @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "fmt" + + "github.com/talos-systems/crypto/x509" + "github.com/talos-systems/os-runtime/pkg/resource" + "github.com/talos-systems/os-runtime/pkg/resource/core" +) + +// EtcdType is type of Etcd resource. +const EtcdType = resource.Type("secrets/etcd") + +// EtcdID is a resource ID of singletone instance. +const EtcdID = resource.ID("etcd") + +// Etcd contains etcd generated secrets. +type Etcd struct { + md resource.Metadata + spec interface{} +} + +// EtcdCertsSpec describes etcd certs secrets. +type EtcdCertsSpec struct { + EtcdPeer *x509.PEMEncodedCertificateAndKey `yaml:"etcdPeer"` +} + +// NewEtcd initializes a Etc resource. +func NewEtcd() *Etcd { + r := &Etcd{ + md: resource.NewMetadata(NamespaceName, EtcdType, EtcdID, resource.VersionUndefined), + spec: &EtcdCertsSpec{}, + } + + r.md.BumpVersion() + + return r +} + +// Metadata implements resource.Resource. +func (r *Etcd) Metadata() *resource.Metadata { + return &r.md +} + +// Spec implements resource.Resource. +func (r *Etcd) Spec() interface{} { + return r.spec +} + +func (r *Etcd) String() string { + return fmt.Sprintf("secrets.Etcd(%q)", r.md.ID()) +} + +// DeepCopy implements resource.Resource. +func (r *Etcd) DeepCopy() resource.Resource { + return &Etcd{ + md: r.md, + spec: r.spec, + } +} + +// ResourceDefinition implements core.ResourceDefinitionProvider interface. +func (r *Etcd) ResourceDefinition() core.ResourceDefinitionSpec { + return core.ResourceDefinitionSpec{ + Type: EtcdType, + Aliases: []resource.Type{"etcdSecrets", "etcdSecret"}, + DefaultNamespace: NamespaceName, + } +} + +// Certs returns .spec. +func (r *Etcd) Certs() *EtcdCertsSpec { + return r.spec.(*EtcdCertsSpec) +} diff --git a/pkg/resources/secrets/kubernetes.go b/pkg/resources/secrets/kubernetes.go index 03ed538253..c0723b88c0 100644 --- a/pkg/resources/secrets/kubernetes.go +++ b/pkg/resources/secrets/kubernetes.go @@ -15,40 +15,29 @@ import ( // KubernetesType is type of Kubernetes resource. const KubernetesType = resource.Type("secrets/kubernetes") -// KubernetesID is ID of the singleton instance. -const KubernetesID = resource.ID("kubernetes") +// KubernetesID is a resource ID of singleton instance. +const KubernetesID = resource.ID("k8s-certs") -// Kubernetes contains K8s secrets. +// Kubernetes contains K8s generated secrets. type Kubernetes struct { md resource.Metadata - spec KubernetesSpec + spec interface{} } -// KubernetesSpec describes Kubernetes resources. -type KubernetesSpec struct { - EtcdCA *x509.PEMEncodedCertificateAndKey `yaml:"etcdCA"` - EtcdPeer *x509.PEMEncodedCertificateAndKey `yaml:"etcdPeer"` - - CA *x509.PEMEncodedCertificateAndKey `yaml:"ca"` +// KubernetesCertsSpec describes generated Kubernetes certificates. +type KubernetesCertsSpec struct { APIServer *x509.PEMEncodedCertificateAndKey `yaml:"apiServer"` APIServerKubeletClient *x509.PEMEncodedCertificateAndKey `yaml:"apiServerKubeletClient"` - ServiceAccount *x509.PEMEncodedKey `yaml:"serviceAccount"` - AggregatorCA *x509.PEMEncodedCertificateAndKey `yaml:"aggregatorCA"` FrontProxy *x509.PEMEncodedCertificateAndKey `yaml:"frontProxy"` - AESCBCEncryptionSecret string `yaml:"aesCBCEncryptionSecret"` - AdminKubeconfig string `yaml:"adminKubeconfig"` - - BootstrapTokenID string `yaml:"bootstrapTokenID"` - BootstrapTokenSecret string `yaml:"bootstrapTokenSecret"` } // NewKubernetes initializes a Kubernetes resource. func NewKubernetes() *Kubernetes { r := &Kubernetes{ md: resource.NewMetadata(NamespaceName, KubernetesType, KubernetesID, resource.VersionUndefined), - spec: KubernetesSpec{}, + spec: &KubernetesCertsSpec{}, } r.md.BumpVersion() @@ -82,12 +71,12 @@ func (r *Kubernetes) DeepCopy() resource.Resource { func (r *Kubernetes) ResourceDefinition() core.ResourceDefinitionSpec { return core.ResourceDefinitionSpec{ Type: KubernetesType, - Aliases: []resource.Type{"secrets", "secret"}, + Aliases: []resource.Type{"k8sSecret", "k8sSecrets"}, DefaultNamespace: NamespaceName, } } -// Secrets returns .spec. -func (r *Kubernetes) Secrets() *KubernetesSpec { - return &r.spec +// Certs returns .spec. +func (r *Kubernetes) Certs() *KubernetesCertsSpec { + return r.spec.(*KubernetesCertsSpec) } diff --git a/pkg/resources/secrets/root.go b/pkg/resources/secrets/root.go new file mode 100644 index 0000000000..78e4984caf --- /dev/null +++ b/pkg/resources/secrets/root.go @@ -0,0 +1,112 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "fmt" + "net" + "net/url" + + "github.com/talos-systems/crypto/x509" + "github.com/talos-systems/os-runtime/pkg/resource" + "github.com/talos-systems/os-runtime/pkg/resource/core" +) + +// RootType is type of Root secret resource. +const RootType = resource.Type("secrets/root") + +// IDs of various resources of RootType. +const ( + RootEtcdID = resource.ID("etcd-root") + RootKubernetesID = resource.ID("k8s-root") +) + +// Root contains root (not generated) secrets. +type Root struct { + md resource.Metadata + spec interface{} +} + +// RootEtcdSpec describes etcd CA secrets. +type RootEtcdSpec struct { + EtcdCA *x509.PEMEncodedCertificateAndKey `yaml:"etcdCA"` +} + +// RootKubernetesSpec describes root Kubernetes secrets. +type RootKubernetesSpec struct { + Name string `yaml:"name"` + Endpoint *url.URL `yaml:"endpoint"` + CertSANs []string `yaml:"certSANs"` + APIServerIPs []net.IP `yaml:"apiServerIPs"` + DNSDomain string `yaml:"dnsDomain"` + + CA *x509.PEMEncodedCertificateAndKey `yaml:"ca"` + ServiceAccount *x509.PEMEncodedKey `yaml:"serviceAccount"` + AggregatorCA *x509.PEMEncodedCertificateAndKey `yaml:"aggregatorCA"` + + AESCBCEncryptionSecret string `yaml:"aesCBCEncryptionSecret"` + + BootstrapTokenID string `yaml:"bootstrapTokenID"` + BootstrapTokenSecret string `yaml:"bootstrapTokenSecret"` +} + +// NewRoot initializes a Root resource. +func NewRoot(id resource.ID) *Root { + r := &Root{ + md: resource.NewMetadata(NamespaceName, RootType, id, resource.VersionUndefined), + } + + switch id { + case RootEtcdID: + r.spec = &RootEtcdSpec{} + case RootKubernetesID: + r.spec = &RootKubernetesSpec{} + } + + r.md.BumpVersion() + + return r +} + +// Metadata implements resource.Resource. +func (r *Root) Metadata() *resource.Metadata { + return &r.md +} + +// Spec implements resource.Resource. +func (r *Root) Spec() interface{} { + return r.spec +} + +func (r *Root) String() string { + return fmt.Sprintf("secrets.Root(%q)", r.md.ID()) +} + +// DeepCopy implements resource.Resource. +func (r *Root) DeepCopy() resource.Resource { + return &Root{ + md: r.md, + spec: r.spec, + } +} + +// ResourceDefinition implements core.ResourceDefinitionProvider interface. +func (r *Root) ResourceDefinition() core.ResourceDefinitionSpec { + return core.ResourceDefinitionSpec{ + Type: RootType, + Aliases: []resource.Type{"rootSecret", "rootSecrets"}, + DefaultNamespace: NamespaceName, + } +} + +// EtcdSpec returns .spec. +func (r *Root) EtcdSpec() *RootEtcdSpec { + return r.spec.(*RootEtcdSpec) +} + +// KubernetesSpec returns .spec. +func (r *Root) KubernetesSpec() *RootKubernetesSpec { + return r.spec.(*RootKubernetesSpec) +}