diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go index 870409547a..3ebb7064de 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod_controller.go @@ -132,7 +132,7 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru return err } - secrets := secretsResources.(*secrets.Kubernetes).Secrets() + secrets := secretsResources.(*secrets.Kubernetes).Certs() bootstrapStatus, err := r.Get(ctx, v1alpha1.NewBootstrapStatus().Metadata()) if err != nil { diff --git a/internal/app/machined/pkg/controllers/k8s/manifest.go b/internal/app/machined/pkg/controllers/k8s/manifest.go index f3950b75b2..e42336d645 100644 --- a/internal/app/machined/pkg/controllers/k8s/manifest.go +++ b/internal/app/machined/pkg/controllers/k8s/manifest.go @@ -47,8 +47,8 @@ func (ctrl *ManifestController) Run(ctx context.Context, r controller.Runtime, l }, { Namespace: secrets.NamespaceName, - Type: secrets.KubernetesType, - ID: pointer.ToString(secrets.KubernetesID), + Type: secrets.RootType, + ID: pointer.ToString(secrets.RootKubernetesID), Kind: controller.DependencyWeak, }, }); err != nil { @@ -77,7 +77,7 @@ func (ctrl *ManifestController) Run(ctx context.Context, r controller.Runtime, l config := configResource.(*config.K8sControlPlane).Manifests() - secretsResources, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.KubernetesType, secrets.KubernetesID, resource.VersionUndefined)) + secretsResources, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootKubernetesID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { if err = ctrl.teardownAll(ctx, r); err != nil { @@ -90,9 +90,9 @@ func (ctrl *ManifestController) Run(ctx context.Context, r controller.Runtime, l return err } - secrets := secretsResources.(*secrets.Kubernetes).Secrets() + secrets := secretsResources.(*secrets.Root).KubernetesSpec() - renderedManifests, err := ctrl.render(config, *secrets) + renderedManifests, err := ctrl.render(config, secrets) if err != nil { return err } @@ -137,11 +137,11 @@ type renderedManifest struct { data []byte } -func (ctrl *ManifestController) render(cfg config.K8sManifestsSpec, scrt secrets.KubernetesSpec) ([]renderedManifest, error) { +func (ctrl *ManifestController) render(cfg config.K8sManifestsSpec, scrt *secrets.RootKubernetesSpec) ([]renderedManifest, error) { templateConfig := struct { config.K8sManifestsSpec - Secrets secrets.KubernetesSpec + Secrets *secrets.RootKubernetesSpec }{ K8sManifestsSpec: cfg, Secrets: scrt, diff --git a/internal/app/machined/pkg/controllers/k8s/manifest_apply.go b/internal/app/machined/pkg/controllers/k8s/manifest_apply.go index bab6be6959..57ed26f695 100644 --- a/internal/app/machined/pkg/controllers/k8s/manifest_apply.go +++ b/internal/app/machined/pkg/controllers/k8s/manifest_apply.go @@ -95,7 +95,7 @@ func (ctrl *ManifestApplyController) Run(ctx context.Context, r controller.Runti return err } - secrets := secretsResources.(*secrets.Kubernetes).Secrets() + secrets := secretsResources.(*secrets.Kubernetes).Certs() bootstrapStatus, err := r.Get(ctx, v1alpha1.NewBootstrapStatus().Metadata()) if err != nil { diff --git a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go index 06b86d9fd6..b776f7cdeb 100644 --- a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go +++ b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go @@ -14,6 +14,7 @@ import ( "path/filepath" stdlibtemplate "text/template" + "github.com/AlekSi/pointer" "github.com/talos-systems/crypto/x509" "github.com/talos-systems/os-runtime/pkg/controller" "github.com/talos-systems/os-runtime/pkg/resource" @@ -42,9 +43,21 @@ func (ctrl *RenderSecretsStaticPodController) ManagedResources() (resource.Names //nolint: gocyclo func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r controller.Runtime, logger *log.Logger) error { if err := r.UpdateDependencies([]controller.Dependency{ + { + Namespace: secrets.NamespaceName, + Type: secrets.RootType, + Kind: controller.DependencyWeak, + }, { Namespace: secrets.NamespaceName, Type: secrets.KubernetesType, + ID: pointer.ToString(secrets.KubernetesID), + Kind: controller.DependencyWeak, + }, + { + Namespace: secrets.NamespaceName, + Type: secrets.EtcdType, + ID: pointer.ToString(secrets.EtcdID), Kind: controller.DependencyWeak, }, }); err != nil { @@ -67,9 +80,39 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control return fmt.Errorf("error getting secrets resource: %w", err) } - secrets := secretsRes.(*secrets.Kubernetes).Secrets() + etcdRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootEtcdRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootEtcdID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootK8sRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootKubernetesID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting secrets resource: %w", err) + } + + rootEtcdSecrets := rootEtcdRes.(*secrets.Root).EtcdSpec() + rootK8sSecrets := rootK8sRes.(*secrets.Root).KubernetesSpec() + etcdSecrets := etcdRes.(*secrets.Etcd).Certs() + k8sSecrets := secretsRes.(*secrets.Kubernetes).Certs() - serviceAccountKey, err := secrets.ServiceAccount.GetKey() + serviceAccountKey, err := rootK8sSecrets.ServiceAccount.GetKey() if err != nil { return fmt.Errorf("error parsing service account key: %w", err) } @@ -96,25 +139,25 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control directory: constants.KubernetesAPIServerSecretsDir, secrets: []secret{ { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.EtcdCA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootEtcdSecrets.EtcdCA }, certFilename: "etcd-client-ca.crt", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.EtcdPeer }, + getter: func() *x509.PEMEncodedCertificateAndKey { return etcdSecrets.EtcdPeer }, certFilename: "etcd-client.crt", keyFilename: "etcd-client.key", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.CA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.CA }, certFilename: "ca.crt", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.APIServer }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.APIServer }, certFilename: "apiserver.crt", keyFilename: "apiserver.key", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.APIServerKubeletClient }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.APIServerKubeletClient }, certFilename: "apiserver-kubelet-client.crt", keyFilename: "apiserver-kubelet-client.key", }, @@ -129,11 +172,11 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control keyFilename: "service-account.key", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.AggregatorCA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.AggregatorCA }, certFilename: "aggregator-ca.crt", }, { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.FrontProxy }, + getter: func() *x509.PEMEncodedCertificateAndKey { return k8sSecrets.FrontProxy }, certFilename: "front-proxy-client.crt", keyFilename: "front-proxy-client.key", }, @@ -154,7 +197,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control directory: constants.KubernetesControllerManagerSecretsDir, secrets: []secret{ { - getter: func() *x509.PEMEncodedCertificateAndKey { return secrets.CA }, + getter: func() *x509.PEMEncodedCertificateAndKey { return rootK8sSecrets.CA }, certFilename: "ca.crt", keyFilename: "ca.key", }, @@ -171,7 +214,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control templates: []template{ { filename: "kubeconfig", - template: []byte("{{ .AdminKubeconfig }}"), + template: []byte("{{ .Secrets.AdminKubeconfig }}"), }, }, }, @@ -181,7 +224,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control templates: []template{ { filename: "kubeconfig", - template: []byte("{{ .AdminKubeconfig }}"), + template: []byte("{{ .Secrets.AdminKubeconfig }}"), }, }, }, @@ -214,6 +257,16 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control } } + type templateParams struct { + Root *secrets.RootKubernetesSpec + Secrets *secrets.KubernetesCertsSpec + } + + params := templateParams{ + Root: rootK8sSecrets, + Secrets: k8sSecrets, + } + for _, templ := range pod.templates { var t *stdlibtemplate.Template @@ -224,7 +277,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control var buf bytes.Buffer - if err = t.Execute(&buf, secrets); err != nil { + if err = t.Execute(&buf, ¶ms); err != nil { return fmt.Errorf("error executing template %q: %w", templ.filename, err) } diff --git a/internal/app/machined/pkg/controllers/k8s/templates.go b/internal/app/machined/pkg/controllers/k8s/templates.go index fdde37dbec..04a8f7bc14 100644 --- a/internal/app/machined/pkg/controllers/k8s/templates.go +++ b/internal/app/machined/pkg/controllers/k8s/templates.go @@ -15,7 +15,7 @@ resources: - aescbc: keys: - name: key1 - secret: {{ .AESCBCEncryptionSecret }} + secret: {{ .Root.AESCBCEncryptionSecret }} - identity: {} `) diff --git a/internal/app/machined/pkg/controllers/secrets/etcd.go b/internal/app/machined/pkg/controllers/secrets/etcd.go new file mode 100644 index 0000000000..153cc1a293 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/etcd.go @@ -0,0 +1,146 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + "log" + + "github.com/AlekSi/pointer" + "github.com/talos-systems/os-runtime/pkg/controller" + "github.com/talos-systems/os-runtime/pkg/resource" + "github.com/talos-systems/os-runtime/pkg/state" + + "github.com/talos-systems/talos/internal/pkg/etcd" + "github.com/talos-systems/talos/pkg/resources/secrets" + "github.com/talos-systems/talos/pkg/resources/v1alpha1" +) + +// EtcdController manages secrets.Etcd based on configuration. +type EtcdController struct{} + +// Name implements controller.Controller interface. +func (ctrl *EtcdController) Name() string { + return "secrets.EtcdController" +} + +// ManagedResources implements controller.Controller interface. +func (ctrl *EtcdController) ManagedResources() (resource.Namespace, resource.Type) { + return secrets.NamespaceName, secrets.EtcdType +} + +// Run implements controller.Controller interface. +// +//nolint: gocyclo, dupl +func (ctrl *EtcdController) Run(ctx context.Context, r controller.Runtime, logger *log.Logger) error { + if err := r.UpdateDependencies([]controller.Dependency{ + { + Namespace: secrets.NamespaceName, + Type: secrets.RootType, + ID: pointer.ToString(secrets.RootEtcdID), + Kind: controller.DependencyWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.ServiceType, + ID: pointer.ToString("networkd"), + Kind: controller.DependencyWeak, + }, + { + Namespace: v1alpha1.NamespaceName, + Type: v1alpha1.TimeSyncType, + ID: pointer.ToString(v1alpha1.TimeSyncID), + Kind: controller.DependencyWeak, + }, + }); err != nil { + return fmt.Errorf("error setting up dependencies: %w", err) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + etcdRootRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootEtcdID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying resources: %w", err) + } + + continue + } + + return fmt.Errorf("error getting etcd root secrets: %w", err) + } + + etcdRoot := etcdRootRes.(*secrets.Root).EtcdSpec() + + // wait for networkd to be healthy as it might change IPs/hostname + networkdResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "networkd", resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !networkdResource.(*v1alpha1.Service).Healthy() { + continue + } + + // wait for time sync as certs depend on current time + timeSyncResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.TimeSyncType, v1alpha1.TimeSyncID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return err + } + + if !timeSyncResource.(*v1alpha1.TimeSync).Sync() { + continue + } + + if err = r.Update(ctx, secrets.NewEtcd(), func(r resource.Resource) error { + return ctrl.updateSecrets(etcdRoot, r.(*secrets.Etcd).Certs()) + }); err != nil { + return err + } + } +} + +func (ctrl *EtcdController) updateSecrets(etcdRoot *secrets.RootEtcdSpec, etcdCerts *secrets.EtcdCertsSpec) error { + var err error + + etcdCerts.EtcdPeer, err = etcd.GeneratePeerCert(etcdRoot.EtcdCA) + if err != nil { + return fmt.Errorf("error generating etcd certs: %w", err) + } + + return nil +} + +func (ctrl *EtcdController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/controllers/secrets/kubernetes.go b/internal/app/machined/pkg/controllers/secrets/kubernetes.go index ec80995e68..3abf9d06b9 100644 --- a/internal/app/machined/pkg/controllers/secrets/kubernetes.go +++ b/internal/app/machined/pkg/controllers/secrets/kubernetes.go @@ -10,6 +10,7 @@ import ( "fmt" "log" "net" + "net/url" "time" "github.com/AlekSi/pointer" @@ -18,19 +19,15 @@ import ( "github.com/talos-systems/os-runtime/pkg/resource" "github.com/talos-systems/os-runtime/pkg/state" - "github.com/talos-systems/talos/internal/pkg/etcd" "github.com/talos-systems/talos/internal/pkg/kubeconfig" - talosconfig "github.com/talos-systems/talos/pkg/machinery/config" - "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" + "github.com/talos-systems/talos/pkg/machinery/config" "github.com/talos-systems/talos/pkg/machinery/constants" - "github.com/talos-systems/talos/pkg/resources/config" "github.com/talos-systems/talos/pkg/resources/secrets" "github.com/talos-systems/talos/pkg/resources/v1alpha1" ) // KubernetesController manages secrets.Kubernetes based on configuration. -type KubernetesController struct { -} +type KubernetesController struct{} // Name implements controller.Controller interface. func (ctrl *KubernetesController) Name() string { @@ -44,13 +41,13 @@ func (ctrl *KubernetesController) ManagedResources() (resource.Namespace, resour // Run implements controller.Controller interface. // -//nolint: gocyclo +//nolint: gocyclo, dupl func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, logger *log.Logger) error { if err := r.UpdateDependencies([]controller.Dependency{ - { // TODO: should render config for kubernetes secrets controller - Namespace: config.NamespaceName, - Type: config.V1Alpha1Type, - ID: pointer.ToString(config.V1Alpha1ID), + { + Namespace: secrets.NamespaceName, + Type: secrets.RootType, + ID: pointer.ToString(secrets.RootKubernetesID), Kind: controller.DependencyWeak, }, { @@ -65,12 +62,6 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, ID: pointer.ToString(v1alpha1.TimeSyncID), Kind: controller.DependencyWeak, }, - { - Namespace: config.NamespaceName, - Type: config.MachineTypeType, - ID: pointer.ToString(config.MachineTypeID), - Kind: controller.DependencyWeak, - }, }); err != nil { return fmt.Errorf("error setting up dependencies: %w", err) } @@ -82,39 +73,20 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, case <-r.EventCh(): } - cfg, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.V1Alpha1Type, config.V1Alpha1ID, resource.VersionUndefined)) + k8sRootRes, err := r.Get(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, secrets.RootKubernetesID, resource.VersionUndefined)) if err != nil { if state.IsNotFoundError(err) { if err = ctrl.teardownAll(ctx, r); err != nil { - return fmt.Errorf("error destroying static pods: %w", err) + return fmt.Errorf("error destroying resources: %w", err) } continue } - return fmt.Errorf("error getting config: %w", err) - } - - cfgProvider := cfg.(*config.V1Alpha1).Config() - - machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) - if err != nil { - if state.IsNotFoundError(err) { - continue - } - - return fmt.Errorf("error getting machine type: %w", err) + return fmt.Errorf("error getting root k8s secrets: %w", err) } - machineType := machineTypeRes.(*config.MachineType).MachineType() - - if machineType != machine.TypeControlPlane && machineType != machine.TypeInit { - if err = ctrl.teardownAll(ctx, r); err != nil { - return fmt.Errorf("error destroying static pods: %w", err) - } - - continue - } + k8sRoot := k8sRootRes.(*secrets.Root).KubernetesSpec() // wait for networkd to be healthy as it might change IPs/hostname networkdResource, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "networkd", resource.VersionUndefined)) @@ -145,62 +117,29 @@ func (ctrl *KubernetesController) Run(ctx context.Context, r controller.Runtime, } if err = r.Update(ctx, secrets.NewKubernetes(), func(r resource.Resource) error { - k8sSecrets := r.(*secrets.Kubernetes) //nolint: errcheck - - return ctrl.updateSecrets(cfgProvider, k8sSecrets) + return ctrl.updateSecrets(k8sRoot, r.(*secrets.Kubernetes).Certs()) }); err != nil { return err } } } -//nolint: gocyclo -func (ctrl *KubernetesController) updateSecrets(cfgProvider talosconfig.Provider, k8sSecrets *secrets.Kubernetes) error { - k8sSecrets.Secrets().EtcdCA = cfgProvider.Cluster().Etcd().CA() - - if k8sSecrets.Secrets().EtcdCA == nil { - return fmt.Errorf("missing cluster.etcdCA secret") - } - - k8sSecrets.Secrets().AggregatorCA = cfgProvider.Cluster().AggregatorCA() - - if k8sSecrets.Secrets().AggregatorCA == nil { - return fmt.Errorf("missing cluster.aggregatorCA secret") - } - - k8sSecrets.Secrets().CA = cfgProvider.Cluster().CA() - - if k8sSecrets.Secrets().CA == nil { - return fmt.Errorf("missing cluster.CA secret") - } - - var err error - - k8sSecrets.Secrets().EtcdPeer, err = etcd.GeneratePeerCert(cfgProvider.Cluster().Etcd().CA()) - if err != nil { - return err - } - - urls := []string{cfgProvider.Cluster().Endpoint().Hostname()} - urls = append(urls, cfgProvider.Cluster().CertSANs()...) +func (ctrl *KubernetesController) updateSecrets(k8sRoot *secrets.RootKubernetesSpec, k8sSecrets *secrets.KubernetesCertsSpec) error { + urls := []string{k8sRoot.Endpoint.Hostname()} + urls = append(urls, k8sRoot.CertSANs...) altNames := altNamesFromURLs(urls) - apiServiceIPs, err := cfgProvider.Cluster().Network().APIServerIPs() - if err != nil { - return fmt.Errorf("failed to calculate API service IP: %w", err) - } - - altNames.IPs = append(altNames.IPs, apiServiceIPs...) + altNames.IPs = append(altNames.IPs, k8sRoot.APIServerIPs...) // Add kubernetes default svc with cluster domain to AltNames altNames.DNSNames = append(altNames.DNSNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc", - "kubernetes.default.svc."+cfgProvider.Cluster().Network().DNSDomain(), + "kubernetes.default.svc."+k8sRoot.DNSDomain, ) - ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sSecrets.Secrets().CA) + ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.CA) if err != nil { return fmt.Errorf("failed to parse CA certificate: %w", err) } @@ -216,7 +155,7 @@ func (ctrl *KubernetesController) updateSecrets(cfgProvider talosconfig.Provider return fmt.Errorf("failed to generate api-server cert: %w", err) } - k8sSecrets.Secrets().APIServer = x509.NewCertificateAndKeyFromKeyPair(apiServer) + k8sSecrets.APIServer = x509.NewCertificateAndKeyFromKeyPair(apiServer) apiServerKubeletClient, err := x509.NewKeyPair(ca, x509.CommonName(constants.KubernetesAdminCertCommonName), @@ -227,11 +166,9 @@ func (ctrl *KubernetesController) updateSecrets(cfgProvider talosconfig.Provider return fmt.Errorf("failed to generate api-server cert: %w", err) } - k8sSecrets.Secrets().APIServerKubeletClient = x509.NewCertificateAndKeyFromKeyPair(apiServerKubeletClient) + k8sSecrets.APIServerKubeletClient = x509.NewCertificateAndKeyFromKeyPair(apiServerKubeletClient) - k8sSecrets.Secrets().ServiceAccount = cfgProvider.Cluster().ServiceAccount() - - aggregatorCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sSecrets.Secrets().AggregatorCA) + aggregatorCA, err := x509.NewCertificateAuthorityFromCertificateAndKey(k8sRoot.AggregatorCA) if err != nil { return fmt.Errorf("failed to parse aggregator CA: %w", err) } @@ -244,20 +181,15 @@ func (ctrl *KubernetesController) updateSecrets(cfgProvider talosconfig.Provider return fmt.Errorf("failed to generate aggregator cert: %w", err) } - k8sSecrets.Secrets().FrontProxy = x509.NewCertificateAndKeyFromKeyPair(frontProxy) - - k8sSecrets.Secrets().AESCBCEncryptionSecret = cfgProvider.Cluster().AESCBCEncryptionSecret() + k8sSecrets.FrontProxy = x509.NewCertificateAndKeyFromKeyPair(frontProxy) var buf bytes.Buffer - if err = kubeconfig.GenerateAdmin(cfgProvider.Cluster(), &buf); err != nil { + if err = kubeconfig.GenerateAdmin(&generateAdminAdapter{k8sRoot: k8sRoot}, &buf); err != nil { return fmt.Errorf("failed to generate admin kubeconfig: %w", err) } - k8sSecrets.Secrets().AdminKubeconfig = buf.String() - - k8sSecrets.Secrets().BootstrapTokenID = cfgProvider.Cluster().Token().ID() - k8sSecrets.Secrets().BootstrapTokenSecret = cfgProvider.Cluster().Token().Secret() + k8sSecrets.AdminKubeconfig = buf.String() return nil } @@ -301,3 +233,29 @@ func altNamesFromURLs(urls []string) *AltNames { return &an } + +// generateAdminAdapter allows to translate input config into GenerateAdmin input. +type generateAdminAdapter struct { + k8sRoot *secrets.RootKubernetesSpec +} + +func (adapter *generateAdminAdapter) Name() string { + return adapter.k8sRoot.Name +} + +func (adapter *generateAdminAdapter) Endpoint() *url.URL { + return adapter.k8sRoot.Endpoint +} + +func (adapter *generateAdminAdapter) CA() *x509.PEMEncodedCertificateAndKey { + return adapter.k8sRoot.CA +} + +func (adapter *generateAdminAdapter) AdminKubeconfig() config.AdminKubeconfig { + return adapter +} + +func (adapter *generateAdminAdapter) CertLifetime() time.Duration { + // this certificate is not delivered to the user, it's used only internally by Talos + return x509.DefaultCertificateValidityDuration +} diff --git a/internal/app/machined/pkg/controllers/secrets/root.go b/internal/app/machined/pkg/controllers/secrets/root.go new file mode 100644 index 0000000000..1bf7be08a8 --- /dev/null +++ b/internal/app/machined/pkg/controllers/secrets/root.go @@ -0,0 +1,172 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "context" + "fmt" + "log" + + "github.com/AlekSi/pointer" + "github.com/talos-systems/os-runtime/pkg/controller" + "github.com/talos-systems/os-runtime/pkg/resource" + "github.com/talos-systems/os-runtime/pkg/state" + + talosconfig "github.com/talos-systems/talos/pkg/machinery/config" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" + "github.com/talos-systems/talos/pkg/resources/config" + "github.com/talos-systems/talos/pkg/resources/secrets" +) + +// RootController manages secrets.Root based on configuration. +type RootController struct{} + +// Name implements controller.Controller interface. +func (ctrl *RootController) Name() string { + return "secrets.RootController" +} + +// ManagedResources implements controller.Controller interface. +func (ctrl *RootController) ManagedResources() (resource.Namespace, resource.Type) { + return secrets.NamespaceName, secrets.RootType +} + +// Run implements controller.Controller interface. +// +//nolint: gocyclo +func (ctrl *RootController) Run(ctx context.Context, r controller.Runtime, logger *log.Logger) error { + if err := r.UpdateDependencies([]controller.Dependency{ + { + Namespace: config.NamespaceName, + Type: config.V1Alpha1Type, + ID: pointer.ToString(config.V1Alpha1ID), + Kind: controller.DependencyWeak, + }, + { + Namespace: config.NamespaceName, + Type: config.MachineTypeType, + ID: pointer.ToString(config.MachineTypeID), + Kind: controller.DependencyWeak, + }, + }); err != nil { + return fmt.Errorf("error setting up dependencies: %w", err) + } + + for { + select { + case <-ctx.Done(): + return nil + case <-r.EventCh(): + } + + cfg, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.V1Alpha1Type, config.V1Alpha1ID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying static pods: %w", err) + } + + continue + } + + return fmt.Errorf("error getting config: %w", err) + } + + cfgProvider := cfg.(*config.V1Alpha1).Config() + + machineTypeRes, err := r.Get(ctx, resource.NewMetadata(config.NamespaceName, config.MachineTypeType, config.MachineTypeID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + continue + } + + return fmt.Errorf("error getting machine type: %w", err) + } + + machineType := machineTypeRes.(*config.MachineType).MachineType() + + if machineType != machine.TypeControlPlane && machineType != machine.TypeInit { + if err = ctrl.teardownAll(ctx, r); err != nil { + return fmt.Errorf("error destroying secrets: %w", err) + } + + continue + } + + if err = r.Update(ctx, secrets.NewRoot(secrets.RootEtcdID), func(r resource.Resource) error { + return ctrl.updateEtcdSecrets(cfgProvider, r.(*secrets.Root).EtcdSpec()) + }); err != nil { + return err + } + + if err = r.Update(ctx, secrets.NewRoot(secrets.RootKubernetesID), func(r resource.Resource) error { + return ctrl.updateK8sSecrets(cfgProvider, r.(*secrets.Root).KubernetesSpec()) + }); err != nil { + return err + } + } +} + +func (ctrl *RootController) updateEtcdSecrets(cfgProvider talosconfig.Provider, etcdSecrets *secrets.RootEtcdSpec) error { + etcdSecrets.EtcdCA = cfgProvider.Cluster().Etcd().CA() + + if etcdSecrets.EtcdCA == nil { + return fmt.Errorf("missing cluster.etcdCA secret") + } + + return nil +} + +func (ctrl *RootController) updateK8sSecrets(cfgProvider talosconfig.Provider, k8sSecrets *secrets.RootKubernetesSpec) error { + k8sSecrets.Name = cfgProvider.Cluster().Name() + k8sSecrets.Endpoint = cfgProvider.Cluster().Endpoint() + k8sSecrets.CertSANs = cfgProvider.Cluster().CertSANs() + k8sSecrets.DNSDomain = cfgProvider.Cluster().Network().DNSDomain() + + var err error + + k8sSecrets.APIServerIPs, err = cfgProvider.Cluster().Network().APIServerIPs() + if err != nil { + return fmt.Errorf("error building API service IPs: %w", err) + } + + k8sSecrets.AggregatorCA = cfgProvider.Cluster().AggregatorCA() + + if k8sSecrets.AggregatorCA == nil { + return fmt.Errorf("missing cluster.aggregatorCA secret") + } + + k8sSecrets.CA = cfgProvider.Cluster().CA() + + if k8sSecrets.CA == nil { + return fmt.Errorf("missing cluster.CA secret") + } + + k8sSecrets.ServiceAccount = cfgProvider.Cluster().ServiceAccount() + + k8sSecrets.AESCBCEncryptionSecret = cfgProvider.Cluster().AESCBCEncryptionSecret() + + k8sSecrets.BootstrapTokenID = cfgProvider.Cluster().Token().ID() + k8sSecrets.BootstrapTokenSecret = cfgProvider.Cluster().Token().Secret() + + return nil +} + +func (ctrl *RootController) teardownAll(ctx context.Context, r controller.Runtime) error { + list, err := r.List(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.RootType, "", resource.VersionUndefined)) + if err != nil { + return err + } + + // TODO: change this to proper teardown sequence + + for _, res := range list.Items { + if err = r.Destroy(ctx, res.Metadata()); err != nil { + return err + } + } + + return nil +} diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go index 87dff5b948..4919761f5b 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go @@ -62,7 +62,9 @@ func (ctrl *Controller) Run(ctx context.Context) error { &k8s.ManifestController{}, &k8s.ManifestApplyController{}, &k8s.RenderSecretsStaticPodController{}, + &secrets.EtcdController{}, &secrets.KubernetesController{}, + &secrets.RootController{}, } { if err := ctrl.controllerRuntime.RegisterController(c); err != nil { return err diff --git a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go index 01e49cb4fb..87312381a7 100644 --- a/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go +++ b/internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_state.go @@ -80,7 +80,9 @@ func NewState() (*State, error) { &k8s.StaticPod{}, &k8s.StaticPodStatus{}, &k8s.SecretsStatus{}, + &secrets.Etcd{}, &secrets.Kubernetes{}, + &secrets.Root{}, } { if err := s.resourceRegistry.Register(ctx, r); err != nil { return nil, err diff --git a/internal/pkg/kubeconfig/admin.go b/internal/pkg/kubeconfig/admin.go index 6bb8f858b3..b559a161be 100644 --- a/internal/pkg/kubeconfig/admin.go +++ b/internal/pkg/kubeconfig/admin.go @@ -8,6 +8,7 @@ import ( "encoding/base64" "fmt" "io" + "net/url" "text/template" "time" @@ -38,8 +39,18 @@ contexts: current-context: admin@{{ .Cluster }} ` +// GenerateAdminInput is the interface for the GenerateAdmin function. +// +// This interface is implemented by config.Cluster(). +type GenerateAdminInput interface { + Name() string + Endpoint() *url.URL + CA() *x509.PEMEncodedCertificateAndKey + AdminKubeconfig() config.AdminKubeconfig +} + // GenerateAdmin generates admin kubeconfig for the cluster. -func GenerateAdmin(config config.ClusterConfig, out io.Writer) error { +func GenerateAdmin(config GenerateAdminInput, out io.Writer) error { tpl, err := template.New("kubeconfig").Parse(adminKubeConfigTemplate) if err != nil { return fmt.Errorf("error parsing kubeconfig template: %w", err) diff --git a/pkg/resources/secrets/etcd.go b/pkg/resources/secrets/etcd.go new file mode 100644 index 0000000000..9b94f71ba7 --- /dev/null +++ b/pkg/resources/secrets/etcd.go @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "fmt" + + "github.com/talos-systems/crypto/x509" + "github.com/talos-systems/os-runtime/pkg/resource" + "github.com/talos-systems/os-runtime/pkg/resource/core" +) + +// EtcdType is type of Etcd resource. +const EtcdType = resource.Type("secrets/etcd") + +// EtcdID is a resource ID of singletone instance. +const EtcdID = resource.ID("etcd") + +// Etcd contains etcd generated secrets. +type Etcd struct { + md resource.Metadata + spec interface{} +} + +// EtcdCertsSpec describes etcd certs secrets. +type EtcdCertsSpec struct { + EtcdPeer *x509.PEMEncodedCertificateAndKey `yaml:"etcdPeer"` +} + +// NewEtcd initializes a Etc resource. +func NewEtcd() *Etcd { + r := &Etcd{ + md: resource.NewMetadata(NamespaceName, EtcdType, EtcdID, resource.VersionUndefined), + spec: &EtcdCertsSpec{}, + } + + r.md.BumpVersion() + + return r +} + +// Metadata implements resource.Resource. +func (r *Etcd) Metadata() *resource.Metadata { + return &r.md +} + +// Spec implements resource.Resource. +func (r *Etcd) Spec() interface{} { + return r.spec +} + +func (r *Etcd) String() string { + return fmt.Sprintf("secrets.Etcd(%q)", r.md.ID()) +} + +// DeepCopy implements resource.Resource. +func (r *Etcd) DeepCopy() resource.Resource { + return &Etcd{ + md: r.md, + spec: r.spec, + } +} + +// ResourceDefinition implements core.ResourceDefinitionProvider interface. +func (r *Etcd) ResourceDefinition() core.ResourceDefinitionSpec { + return core.ResourceDefinitionSpec{ + Type: EtcdType, + Aliases: []resource.Type{"etcdSecrets", "etcdSecret"}, + DefaultNamespace: NamespaceName, + } +} + +// Certs returns .spec. +func (r *Etcd) Certs() *EtcdCertsSpec { + return r.spec.(*EtcdCertsSpec) +} diff --git a/pkg/resources/secrets/kubernetes.go b/pkg/resources/secrets/kubernetes.go index 03ed538253..c0723b88c0 100644 --- a/pkg/resources/secrets/kubernetes.go +++ b/pkg/resources/secrets/kubernetes.go @@ -15,40 +15,29 @@ import ( // KubernetesType is type of Kubernetes resource. const KubernetesType = resource.Type("secrets/kubernetes") -// KubernetesID is ID of the singleton instance. -const KubernetesID = resource.ID("kubernetes") +// KubernetesID is a resource ID of singleton instance. +const KubernetesID = resource.ID("k8s-certs") -// Kubernetes contains K8s secrets. +// Kubernetes contains K8s generated secrets. type Kubernetes struct { md resource.Metadata - spec KubernetesSpec + spec interface{} } -// KubernetesSpec describes Kubernetes resources. -type KubernetesSpec struct { - EtcdCA *x509.PEMEncodedCertificateAndKey `yaml:"etcdCA"` - EtcdPeer *x509.PEMEncodedCertificateAndKey `yaml:"etcdPeer"` - - CA *x509.PEMEncodedCertificateAndKey `yaml:"ca"` +// KubernetesCertsSpec describes generated Kubernetes certificates. +type KubernetesCertsSpec struct { APIServer *x509.PEMEncodedCertificateAndKey `yaml:"apiServer"` APIServerKubeletClient *x509.PEMEncodedCertificateAndKey `yaml:"apiServerKubeletClient"` - ServiceAccount *x509.PEMEncodedKey `yaml:"serviceAccount"` - AggregatorCA *x509.PEMEncodedCertificateAndKey `yaml:"aggregatorCA"` FrontProxy *x509.PEMEncodedCertificateAndKey `yaml:"frontProxy"` - AESCBCEncryptionSecret string `yaml:"aesCBCEncryptionSecret"` - AdminKubeconfig string `yaml:"adminKubeconfig"` - - BootstrapTokenID string `yaml:"bootstrapTokenID"` - BootstrapTokenSecret string `yaml:"bootstrapTokenSecret"` } // NewKubernetes initializes a Kubernetes resource. func NewKubernetes() *Kubernetes { r := &Kubernetes{ md: resource.NewMetadata(NamespaceName, KubernetesType, KubernetesID, resource.VersionUndefined), - spec: KubernetesSpec{}, + spec: &KubernetesCertsSpec{}, } r.md.BumpVersion() @@ -82,12 +71,12 @@ func (r *Kubernetes) DeepCopy() resource.Resource { func (r *Kubernetes) ResourceDefinition() core.ResourceDefinitionSpec { return core.ResourceDefinitionSpec{ Type: KubernetesType, - Aliases: []resource.Type{"secrets", "secret"}, + Aliases: []resource.Type{"k8sSecret", "k8sSecrets"}, DefaultNamespace: NamespaceName, } } -// Secrets returns .spec. -func (r *Kubernetes) Secrets() *KubernetesSpec { - return &r.spec +// Certs returns .spec. +func (r *Kubernetes) Certs() *KubernetesCertsSpec { + return r.spec.(*KubernetesCertsSpec) } diff --git a/pkg/resources/secrets/root.go b/pkg/resources/secrets/root.go new file mode 100644 index 0000000000..78e4984caf --- /dev/null +++ b/pkg/resources/secrets/root.go @@ -0,0 +1,112 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package secrets + +import ( + "fmt" + "net" + "net/url" + + "github.com/talos-systems/crypto/x509" + "github.com/talos-systems/os-runtime/pkg/resource" + "github.com/talos-systems/os-runtime/pkg/resource/core" +) + +// RootType is type of Root secret resource. +const RootType = resource.Type("secrets/root") + +// IDs of various resources of RootType. +const ( + RootEtcdID = resource.ID("etcd-root") + RootKubernetesID = resource.ID("k8s-root") +) + +// Root contains root (not generated) secrets. +type Root struct { + md resource.Metadata + spec interface{} +} + +// RootEtcdSpec describes etcd CA secrets. +type RootEtcdSpec struct { + EtcdCA *x509.PEMEncodedCertificateAndKey `yaml:"etcdCA"` +} + +// RootKubernetesSpec describes root Kubernetes secrets. +type RootKubernetesSpec struct { + Name string `yaml:"name"` + Endpoint *url.URL `yaml:"endpoint"` + CertSANs []string `yaml:"certSANs"` + APIServerIPs []net.IP `yaml:"apiServerIPs"` + DNSDomain string `yaml:"dnsDomain"` + + CA *x509.PEMEncodedCertificateAndKey `yaml:"ca"` + ServiceAccount *x509.PEMEncodedKey `yaml:"serviceAccount"` + AggregatorCA *x509.PEMEncodedCertificateAndKey `yaml:"aggregatorCA"` + + AESCBCEncryptionSecret string `yaml:"aesCBCEncryptionSecret"` + + BootstrapTokenID string `yaml:"bootstrapTokenID"` + BootstrapTokenSecret string `yaml:"bootstrapTokenSecret"` +} + +// NewRoot initializes a Root resource. +func NewRoot(id resource.ID) *Root { + r := &Root{ + md: resource.NewMetadata(NamespaceName, RootType, id, resource.VersionUndefined), + } + + switch id { + case RootEtcdID: + r.spec = &RootEtcdSpec{} + case RootKubernetesID: + r.spec = &RootKubernetesSpec{} + } + + r.md.BumpVersion() + + return r +} + +// Metadata implements resource.Resource. +func (r *Root) Metadata() *resource.Metadata { + return &r.md +} + +// Spec implements resource.Resource. +func (r *Root) Spec() interface{} { + return r.spec +} + +func (r *Root) String() string { + return fmt.Sprintf("secrets.Root(%q)", r.md.ID()) +} + +// DeepCopy implements resource.Resource. +func (r *Root) DeepCopy() resource.Resource { + return &Root{ + md: r.md, + spec: r.spec, + } +} + +// ResourceDefinition implements core.ResourceDefinitionProvider interface. +func (r *Root) ResourceDefinition() core.ResourceDefinitionSpec { + return core.ResourceDefinitionSpec{ + Type: RootType, + Aliases: []resource.Type{"rootSecret", "rootSecrets"}, + DefaultNamespace: NamespaceName, + } +} + +// EtcdSpec returns .spec. +func (r *Root) EtcdSpec() *RootEtcdSpec { + return r.spec.(*RootEtcdSpec) +} + +// KubernetesSpec returns .spec. +func (r *Root) KubernetesSpec() *RootKubernetesSpec { + return r.spec.(*RootKubernetesSpec) +}