Skip to content

Commit

Permalink
feat: add a tool and package to convert self-hosted CP to static pods
Browse files Browse the repository at this point in the history
This is required to upgrade from Talos 0.8.x to 0.9.x. After the cluster
is fully upgraded, control plane is still self-hosted (as it was
bootstrapped with bootkube).

Tool `talosctl convert-k8s` (and library behind it) performs the upgrade
to self-hosted version.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
  • Loading branch information
smira authored and talos-bot committed Feb 18, 2021
1 parent 3a78bfc commit 7751920
Show file tree
Hide file tree
Showing 50 changed files with 2,966 additions and 1,744 deletions.
4 changes: 2 additions & 2 deletions .drone.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -354,9 +354,9 @@ local integration_provision_tests_prepare = Step("provision-tests-prepare", priv
local integration_provision_tests_track_0 = Step("provision-tests-track-0", privileged=true, depends_on=[integration_provision_tests_prepare], environment={"REGISTRY": local_registry});
local integration_provision_tests_track_1 = Step("provision-tests-track-1", privileged=true, depends_on=[integration_provision_tests_prepare], environment={"REGISTRY": local_registry});

local integration_cilium = Step("e2e-cilium-1.8.5", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
local integration_cilium = Step("e2e-cilium-1.9.4", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
"SHORT_INTEGRATION_TEST": "yes",
"CUSTOM_CNI_URL": "https://raw.githubusercontent.com/cilium/cilium/v1.8.5/install/kubernetes/quick-install.yaml",
"CUSTOM_CNI_URL": "https://raw.githubusercontent.com/cilium/cilium/v1.9.4/install/kubernetes/quick-install.yaml",
"REGISTRY": local_registry,
"CLUSTER_CIDR": 2,
});
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ CLUSTERCTL_URL ?= https://github.com/kubernetes-sigs/cluster-api/releases/downlo
SONOBUOY_VERSION ?= 0.19.0
SONOBUOY_URL ?= https://github.com/heptio/sonobuoy/releases/download/v$(SONOBUOY_VERSION)/sonobuoy_$(SONOBUOY_VERSION)_$(OPERATING_SYSTEM)_amd64.tar.gz
TESTPKGS ?= github.com/talos-systems/talos/...
RELEASES ?= v0.7.1 v0.8.0
RELEASES ?= v0.7.1 v0.8.3
SHORT_INTEGRATION_TEST ?=
CUSTOM_CNI_URL ?=

Expand Down
7 changes: 7 additions & 0 deletions api/machine/machine.proto
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ service MachineService {
rpc Rollback(RollbackRequest) returns (RollbackResponse);
rpc Reset(ResetRequest) returns (ResetResponse);
rpc Recover(RecoverRequest) returns (RecoverResponse);
rpc RemoveBootkubeInitializedKey(google.protobuf.Empty)
returns (RemoveBootkubeInitializedKeyResponse);
rpc ServiceList(google.protobuf.Empty) returns (ServiceListResponse);
rpc ServiceRestart(ServiceRestartRequest) returns (ServiceRestartResponse);
rpc ServiceStart(ServiceStartRequest) returns (ServiceStartResponse);
Expand All @@ -64,6 +66,7 @@ service MachineService {
message ApplyConfigurationRequest {
bytes data = 1;
bool on_reboot = 2;
bool immediate = 3;
}

// ApplyConfigurationResponse describes the response to a configuration request.
Expand Down Expand Up @@ -836,3 +839,7 @@ message GenerateConfiguration {
message GenerateConfigurationResponse {
repeated GenerateConfiguration messages = 1;
}

// RemoveBootkubeInitializedKeyResponse describes the response to a RemoveBootkubeInitializedKey request.
message RemoveBootkubeInitializedKey { common.Metadata metadata = 1; }
message RemoveBootkubeInitializedKeyResponse { repeated RemoveBootkubeInitializedKey messages = 1; }
7 changes: 5 additions & 2 deletions cmd/talosctl/cmd/talos/apply-config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ var applyConfigCmdFlags struct {
insecure bool
interactive bool
onReboot bool
immediate bool
}

// applyConfigCmd represents the applyConfiguration command.
Expand Down Expand Up @@ -134,8 +135,9 @@ var applyConfigCmd = &cobra.Command{
}

if _, err := c.ApplyConfiguration(ctx, &machineapi.ApplyConfigurationRequest{
Data: cfgBytes,
OnReboot: applyConfigCmdFlags.onReboot,
Data: cfgBytes,
OnReboot: applyConfigCmdFlags.onReboot,
Immediate: applyConfigCmdFlags.immediate,
}); err != nil {
return fmt.Errorf("error applying new configuration: %s", err)
}
Expand All @@ -151,6 +153,7 @@ func init() {
applyConfigCmd.Flags().StringSliceVar(&applyConfigCmdFlags.certFingerprints, "cert-fingerprint", nil, "list of server certificate fingeprints to accept (defaults to no check)")
applyConfigCmd.Flags().BoolVar(&applyConfigCmdFlags.interactive, "interactive", false, "apply the config using text based interactive mode")
applyConfigCmd.Flags().BoolVar(&applyConfigCmdFlags.onReboot, "on-reboot", false, "apply the config on reboot")
applyConfigCmd.Flags().BoolVar(&applyConfigCmdFlags.immediate, "immediate", false, "apply the config immediately (without a reboot)")

// deprecated, to be removed in 0.10
applyConfigCmd.Flags().BoolVar(&applyConfigCmdFlags.onReboot, "no-reboot", false, "apply the config only after the reboot")
Expand Down
57 changes: 57 additions & 0 deletions cmd/talosctl/cmd/talos/convert-k8s.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package talos

import (
"context"

"github.com/spf13/cobra"

"github.com/talos-systems/talos/pkg/cluster"
k8s "github.com/talos-systems/talos/pkg/cluster/kubernetes"
"github.com/talos-systems/talos/pkg/machinery/client"
)

// convertK8sCmd represents the convert-k8s command.
var convertK8sCmd = &cobra.Command{
Use: "convert-k8s",
Short: "Convert Kubernetes control plane from self-hosted (bootkube) to Talos-managed (static pods).",
Long: `Command converts control plane bootstrapped on Talos <= 0.8 to Talos-managed control plane (Talos >= 0.9).
As part of the conversion process tool reads existing configuration of the control plane, updates
Talos node configuration to reflect changes made since the boostrap time. Once config is updated,
tool releases static pods and deletes self-hosted DaemonSets.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
return WithClient(convertKubernetes)
},
}

var convertOptions k8s.ConvertOptions

func init() {
convertK8sCmd.Flags().StringVar(&convertOptions.ControlPlaneEndpoint, "endpoint", "", "the cluster control plane endpoint")
convertK8sCmd.Flags().BoolVar(&convertOptions.ForceYes, "force", false, "skip prompts, assume yes")
addCommand(convertK8sCmd)
}

func convertKubernetes(ctx context.Context, c *client.Client) error {
clientProvider := &cluster.ConfigClientProvider{
DefaultClient: c,
}
defer clientProvider.Close() //nolint: errcheck

state := struct {
cluster.ClientProvider
cluster.K8sProvider
}{
ClientProvider: clientProvider,
K8sProvider: &cluster.KubernetesClient{
ClientProvider: clientProvider,
ForceEndpoint: convertOptions.ControlPlaneEndpoint,
},
}

return k8s.ConvertToStaticPods(ctx, &state, convertOptions)
}
16 changes: 14 additions & 2 deletions cmd/talosctl/cmd/talos/upgrade-k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package talos

import (
"context"
"fmt"

"github.com/spf13/cobra"

Expand Down Expand Up @@ -45,13 +46,24 @@ func upgradeKubernetes(ctx context.Context, c *client.Client) error {
defer clientProvider.Close() //nolint: errcheck

state := struct {
cluster.ClientProvider
cluster.K8sProvider
}{
ClientProvider: clientProvider,
K8sProvider: &cluster.KubernetesClient{
ClientProvider: clientProvider,
ForceEndpoint: healthCmdFlags.forceEndpoint,
ForceEndpoint: upgradeOptions.ControlPlaneEndpoint,
},
}

return k8s.Upgrade(ctx, &state, upgradeOptions)
selfHosted, err := k8s.IsSelfHostedControlPlane(ctx, &state, Nodes[0])
if err != nil {
return fmt.Errorf("error checking self-hosted status: %w", err)
}

if selfHosted {
return k8s.UpgradeSelfHosted(ctx, &state, upgradeOptions)
}

return k8s.UpgradeTalosManaged(ctx, &state, upgradeOptions)
}
64 changes: 55 additions & 9 deletions internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/prometheus/procfs"
"github.com/rs/xid"
"github.com/talos-systems/go-blockdevice/blockdevice/partition/gpt"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/concurrency"
"golang.org/x/sys/unix"
"google.golang.org/grpc"
Expand Down Expand Up @@ -103,14 +104,33 @@ func (s *Server) Register(obj *grpc.Server) {
}

// ApplyConfiguration implements machine.MachineService.
func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfigurationRequest) (reply *machine.ApplyConfigurationResponse, err error) {
if !in.OnReboot {
if err = s.Controller.Runtime().SetConfig(in.GetData()); err != nil {
//
//nolint: gocyclo
func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfigurationRequest) (*machine.ApplyConfigurationResponse, error) {
log.Printf("apply config request: immediate %v, on reboot %v", in.Immediate, in.OnReboot)

switch {
// --immediate
case in.Immediate:
if err := s.Controller.Runtime().CanApplyImmediate(in.GetData()); err != nil {
return nil, err
}

if err := s.Controller.Runtime().SetConfig(in.GetData()); err != nil {
return nil, err
}

if err := ioutil.WriteFile(constants.ConfigPath, in.GetData(), 0o600); err != nil {
return nil, err
}
// default (no flags)
case !in.OnReboot:
if err := s.Controller.Runtime().SetConfig(in.GetData()); err != nil {
return nil, err
}

go func() {
if err = s.Controller.Run(runtime.SequenceApplyConfiguration, in); err != nil {
if err := s.Controller.Run(runtime.SequenceApplyConfiguration, in); err != nil {
if !runtime.IsRebootError(err) {
log.Println("apply configuration failed:", err)
}
Expand All @@ -120,7 +140,8 @@ func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfig
}
}
}()
} else {
// --no-reboot
case in.OnReboot:
cfg, err := s.Controller.Runtime().ValidateConfig(in.GetData())
if err != nil {
return nil, err
Expand All @@ -143,13 +164,11 @@ func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfig
}
}

reply = &machine.ApplyConfigurationResponse{
return &machine.ApplyConfigurationResponse{
Messages: []*machine.ApplyConfiguration{
{},
},
}

return reply, nil
}, nil
}

// GenerateConfiguration implements the machine.MachineServer interface.
Expand Down Expand Up @@ -1722,6 +1741,33 @@ func (s *Server) EtcdForfeitLeadership(ctx context.Context, in *machine.EtcdForf
return reply, nil
}

// RemoveBootkubeInitializedKey implements machine.MachineService.
//
// Temporary API only used when converting from self-hosted to Talos-managed control plane.
// This API can be removed once the conversion process is no longer needed (Talos 0.11?).
func (s *Server) RemoveBootkubeInitializedKey(ctx context.Context, in *empty.Empty) (*machine.RemoveBootkubeInitializedKeyResponse, error) {
client, err := etcd.NewLocalClient()
if err != nil {
return nil, fmt.Errorf("failed to create etcd client: %w", err)
}

// nolint: errcheck
defer client.Close()

ctx = clientv3.WithRequireLeader(ctx)

_, err = client.Delete(ctx, constants.InitializedKey)
if err != nil {
return nil, fmt.Errorf("error deleting initialized key: %w", err)
}

return &machine.RemoveBootkubeInitializedKeyResponse{
Messages: []*machine.RemoveBootkubeInitializedKey{
{},
},
}, nil
}

func upgradeMutex(c *etcd.Client) (*concurrency.Mutex, error) {
sess, err := concurrency.NewSession(c.Client,
concurrency.WithTTL(MinimumEtcdUpgradeLeaseLockSeconds),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ import (
"github.com/talos-systems/os-runtime/pkg/resource"
"github.com/talos-systems/os-runtime/pkg/state"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
"github.com/talos-systems/talos/pkg/images"
talosconfig "github.com/talos-systems/talos/pkg/machinery/config"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/machinery/constants"
"github.com/talos-systems/talos/pkg/resources/config"
)

// K8sControlPlaneController manages config.K8sControlPlane based on configuration.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ import (
"github.com/talos-systems/os-runtime/pkg/resource"
"github.com/talos-systems/os-runtime/pkg/state"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/resources/config"
)

// MachineTypeController manages config.MachineType based on configuration.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
"github.com/talos-systems/talos/pkg/machinery/constants"
"github.com/talos-systems/talos/pkg/resources/config"
"github.com/talos-systems/talos/pkg/resources/k8s"
)

// ControlPlaneStaticPodController manages k8s.StaticPod based on control plane configuration.
Expand Down
6 changes: 3 additions & 3 deletions internal/app/machined/pkg/controllers/k8s/extra_manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ import (
"github.com/talos-systems/os-runtime/pkg/resource"
"github.com/talos-systems/os-runtime/pkg/state"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/v1alpha1"
"github.com/talos-systems/talos/pkg/resources/config"
"github.com/talos-systems/talos/pkg/resources/k8s"
"github.com/talos-systems/talos/pkg/resources/v1alpha1"
)

// ExtraManifestController renders manifests based on templates and config/secrets.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ import (
"github.com/talos-systems/os-runtime/pkg/state"
"gopkg.in/yaml.v3"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/v1alpha1"
"github.com/talos-systems/talos/pkg/kubernetes/kubelet"
"github.com/talos-systems/talos/pkg/machinery/constants"
"github.com/talos-systems/talos/pkg/resources/k8s"
"github.com/talos-systems/talos/pkg/resources/secrets"
"github.com/talos-systems/talos/pkg/resources/v1alpha1"
)

// KubeletStaticPodController renders static pod definitions and manages k8s.StaticPodStatus.
Expand Down
6 changes: 3 additions & 3 deletions internal/app/machined/pkg/controllers/k8s/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ import (
"github.com/talos-systems/os-runtime/pkg/resource"
"github.com/talos-systems/os-runtime/pkg/state"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
"github.com/talos-systems/talos/pkg/resources/config"
"github.com/talos-systems/talos/pkg/resources/k8s"
"github.com/talos-systems/talos/pkg/resources/secrets"
)

// ManifestController renders manifests based on templates and config/secrets.
Expand Down
6 changes: 3 additions & 3 deletions internal/app/machined/pkg/controllers/k8s/manifest_apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ import (
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/v1alpha1"
"github.com/talos-systems/talos/internal/pkg/etcd"
"github.com/talos-systems/talos/pkg/machinery/constants"
"github.com/talos-systems/talos/pkg/resources/k8s"
"github.com/talos-systems/talos/pkg/resources/secrets"
"github.com/talos-systems/talos/pkg/resources/v1alpha1"
)

// ManifestApplyController applies manifests via control plane endpoint.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ import (
"github.com/talos-systems/os-runtime/pkg/resource"
"github.com/talos-systems/os-runtime/pkg/state"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/k8s"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
"github.com/talos-systems/talos/pkg/machinery/constants"
"github.com/talos-systems/talos/pkg/resources/k8s"
"github.com/talos-systems/talos/pkg/resources/secrets"
)

// RenderSecretsStaticPodController manages k8s.SecretsReady and renders secrets from secrets.Kubernetes.
Expand Down
6 changes: 3 additions & 3 deletions internal/app/machined/pkg/controllers/secrets/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ import (
"github.com/talos-systems/os-runtime/pkg/resource"
"github.com/talos-systems/os-runtime/pkg/state"

"github.com/talos-systems/talos/internal/app/machined/pkg/resources/config"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/secrets"
"github.com/talos-systems/talos/internal/app/machined/pkg/resources/v1alpha1"
"github.com/talos-systems/talos/internal/pkg/etcd"
"github.com/talos-systems/talos/internal/pkg/kubeconfig"
talosconfig "github.com/talos-systems/talos/pkg/machinery/config"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/machinery/constants"
"github.com/talos-systems/talos/pkg/resources/config"
"github.com/talos-systems/talos/pkg/resources/secrets"
"github.com/talos-systems/talos/pkg/resources/v1alpha1"
)

// KubernetesController manages secrets.Kubernetes based on configuration.
Expand Down
Loading

0 comments on commit 7751920

Please sign in to comment.