Skip to content

Commit

Permalink
feat: add support for extra volume mounts for control plane pods
Browse files Browse the repository at this point in the history
This allows to mount extra volumes into Talos-managed control plane
static pods. With additional options like extra files, any additional
content/configuration can be mounted.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
  • Loading branch information
smira authored and talos-bot committed Feb 20, 2021
1 parent 06b8c09 commit 8789849
Show file tree
Hide file tree
Showing 12 changed files with 798 additions and 29 deletions.
22 changes: 20 additions & 2 deletions internal/app/machined/pkg/controllers/config/k8s_control_plane.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,21 @@ func (ctrl *K8sControlPlaneController) Run(ctx context.Context, r controller.Run
}
}

func convertVolumes(volumes []talosconfig.VolumeMount) []config.K8sExtraVolume {
result := make([]config.K8sExtraVolume, 0, len(volumes))

for _, volume := range volumes {
result = append(result, config.K8sExtraVolume{
Name: volume.Name(),
HostPath: volume.HostPath(),
MountPath: volume.MountPath(),
ReadOnly: volume.ReadOnly(),
})
}

return result
}

func (ctrl *K8sControlPlaneController) manageAPIServerConfig(ctx context.Context, r controller.Runtime, logger *log.Logger, cfgProvider talosconfig.Provider) error {
return r.Update(ctx, config.NewK8sControlPlaneAPIServer(), func(r resource.Resource) error {
r.(*config.K8sControlPlane).SetAPIServer(config.K8sControlPlaneAPIServerSpec{
Expand All @@ -123,6 +138,7 @@ func (ctrl *K8sControlPlaneController) manageAPIServerConfig(ctx context.Context
LocalPort: cfgProvider.Cluster().LocalAPIServerPort(),
ServiceCIDR: cfgProvider.Cluster().Network().ServiceCIDR(),
ExtraArgs: cfgProvider.Cluster().APIServer().ExtraArgs(),
ExtraVolumes: convertVolumes(cfgProvider.Cluster().APIServer().ExtraVolumes()),
})

return nil
Expand All @@ -137,6 +153,7 @@ func (ctrl *K8sControlPlaneController) manageControllerManagerConfig(ctx context
PodCIDR: cfgProvider.Cluster().Network().PodCIDR(),
ServiceCIDR: cfgProvider.Cluster().Network().ServiceCIDR(),
ExtraArgs: cfgProvider.Cluster().ControllerManager().ExtraArgs(),
ExtraVolumes: convertVolumes(cfgProvider.Cluster().ControllerManager().ExtraVolumes()),
})

return nil
Expand All @@ -146,8 +163,9 @@ func (ctrl *K8sControlPlaneController) manageControllerManagerConfig(ctx context
func (ctrl *K8sControlPlaneController) manageSchedulerConfig(ctx context.Context, r controller.Runtime, logger *log.Logger, cfgProvider talosconfig.Provider) error {
return r.Update(ctx, config.NewK8sControlPlaneScheduler(), func(r resource.Resource) error {
r.(*config.K8sControlPlane).SetScheduler(config.K8sControlPlaneSchedulerSpec{
Image: cfgProvider.Cluster().Scheduler().Image(),
ExtraArgs: cfgProvider.Cluster().Scheduler().ExtraArgs(),
Image: cfgProvider.Cluster().Scheduler().Image(),
ExtraArgs: cfgProvider.Cluster().Scheduler().ExtraArgs(),
ExtraVolumes: convertVolumes(cfgProvider.Cluster().Scheduler().ExtraVolumes()),
})

return nil
Expand Down
200 changes: 200 additions & 0 deletions internal/app/machined/pkg/controllers/config/k8s_control_plane_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package config_test

import (
"context"
"fmt"
"log"
"net/url"
"reflect"
"sync"
"testing"
"time"

"github.com/stretchr/testify/suite"
"github.com/talos-systems/go-retry/retry"
"github.com/talos-systems/os-runtime/pkg/controller/runtime"
"github.com/talos-systems/os-runtime/pkg/resource"
"github.com/talos-systems/os-runtime/pkg/state"
"github.com/talos-systems/os-runtime/pkg/state/impl/inmem"
"github.com/talos-systems/os-runtime/pkg/state/impl/namespaced"

configctrl "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/config"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/resources/config"
"github.com/talos-systems/talos/pkg/resources/k8s"
)

type K8sControlPlaneSuite struct {
suite.Suite

state state.State

runtime *runtime.Runtime
wg sync.WaitGroup

ctx context.Context
ctxCancel context.CancelFunc
}

func (suite *K8sControlPlaneSuite) SetupTest() {
suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 3*time.Minute)

suite.state = state.WrapCore(namespaced.NewState(inmem.Build))

var err error

logger := log.New(log.Writer(), "controller-runtime: ", log.Flags())

suite.runtime, err = runtime.NewRuntime(suite.state, logger)
suite.Require().NoError(err)

suite.Require().NoError(suite.runtime.RegisterController(&configctrl.K8sControlPlaneController{}))

suite.startRuntime()
}

func (suite *K8sControlPlaneSuite) startRuntime() {
suite.wg.Add(1)

go func() {
defer suite.wg.Done()

suite.Assert().NoError(suite.runtime.Run(suite.ctx))
}()
}

func (suite *K8sControlPlaneSuite) assertK8sControlPlanes(manifests []string) error {
resources, err := suite.state.List(suite.ctx, resource.NewMetadata(config.NamespaceName, config.K8sControlPlaneType, "", resource.VersionUndefined))
if err != nil {
return retry.UnexpectedError(err)
}

ids := make([]string, 0, len(resources.Items))

for _, res := range resources.Items {
ids = append(ids, res.Metadata().ID())
}

if !reflect.DeepEqual(manifests, ids) {
return retry.ExpectedError(fmt.Errorf("expected %q, got %q", manifests, ids))
}

return nil
}

func (suite *K8sControlPlaneSuite) TestReconcileDefaults() {
machineType := config.NewMachineType()
machineType.SetMachineType(machine.TypeControlPlane)

u, err := url.Parse("https://foo:6443")
suite.Require().NoError(err)

cfg := config.NewV1Alpha1(&v1alpha1.Config{
ConfigVersion: "v1alpha1",
MachineConfig: &v1alpha1.MachineConfig{},
ClusterConfig: &v1alpha1.ClusterConfig{
ControlPlane: &v1alpha1.ControlPlaneConfig{
Endpoint: &v1alpha1.Endpoint{
URL: u,
},
},
},
})

suite.Require().NoError(suite.state.Create(suite.ctx, machineType))
suite.Require().NoError(suite.state.Create(suite.ctx, cfg))

suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
func() error {
return suite.assertK8sControlPlanes(
[]string{
config.K8sExtraManifestsID,
config.K8sControlPlaneAPIServerID,
config.K8sControlPlaneControllerManagerID,
config.K8sControlPlaneSchedulerID,
config.K8sManifestsID,
},
)
},
))
}

func (suite *K8sControlPlaneSuite) TestReconcileExtraVolumes() {
machineType := config.NewMachineType()
machineType.SetMachineType(machine.TypeControlPlane)

u, err := url.Parse("https://foo:6443")
suite.Require().NoError(err)

cfg := config.NewV1Alpha1(&v1alpha1.Config{
ConfigVersion: "v1alpha1",
MachineConfig: &v1alpha1.MachineConfig{},
ClusterConfig: &v1alpha1.ClusterConfig{
ControlPlane: &v1alpha1.ControlPlaneConfig{
Endpoint: &v1alpha1.Endpoint{
URL: u,
},
},
APIServerConfig: &v1alpha1.APIServerConfig{
ExtraVolumesConfig: []v1alpha1.VolumeMountConfig{
{
VolumeHostPath: "/var/lib",
VolumeMountPath: "/var/foo",
},
},
},
},
})

suite.Require().NoError(suite.state.Create(suite.ctx, machineType))
suite.Require().NoError(suite.state.Create(suite.ctx, cfg))

suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
func() error {
return suite.assertK8sControlPlanes(
[]string{
config.K8sExtraManifestsID,
config.K8sControlPlaneAPIServerID,
config.K8sControlPlaneControllerManagerID,
config.K8sControlPlaneSchedulerID,
config.K8sManifestsID,
},
)
},
))

r, err := suite.state.Get(suite.ctx, config.NewK8sControlPlaneAPIServer().Metadata())
suite.Require().NoError(err)

apiServerCfg := r.(*config.K8sControlPlane).APIServer()

suite.Assert().Equal([]config.K8sExtraVolume{
{
Name: "-var-foo",
HostPath: "/var/lib",
MountPath: "/var/foo",
ReadOnly: false,
},
}, apiServerCfg.ExtraVolumes)
}

func (suite *K8sControlPlaneSuite) TearDownTest() {
suite.T().Log("tear down")

suite.ctxCancel()

suite.wg.Wait()

// trigger updates in resources to stop watch loops
suite.Assert().NoError(suite.state.Create(context.Background(), k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, "-")))
suite.Assert().NoError(suite.state.Destroy(context.Background(), config.NewK8sControlPlaneAPIServer().Metadata()))
}

func TestK8sControlPlaneSuite(t *testing.T) {
suite.Run(t, new(K8sControlPlaneSuite))
}
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,37 @@ func (ctrl *ControlPlaneStaticPodController) teardownAll(ctx context.Context, r
return nil
}

func volumeMounts(volumes []config.K8sExtraVolume) []v1.VolumeMount {
result := make([]v1.VolumeMount, 0, len(volumes))

for _, volume := range volumes {
result = append(result, v1.VolumeMount{
Name: volume.Name,
MountPath: volume.MountPath,
ReadOnly: volume.ReadOnly,
})
}

return result
}

func volumes(volumes []config.K8sExtraVolume) []v1.Volume {
result := make([]v1.Volume, 0, len(volumes))

for _, volume := range volumes {
result = append(result, v1.Volume{
Name: volume.Name,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: volume.HostPath,
},
},
})
}

return result
}

func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context, r controller.Runtime, logger *log.Logger, configResource *config.K8sControlPlane, secretsVersion string) error {
cfg := configResource.APIServer()

Expand Down Expand Up @@ -214,21 +245,21 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
},
},
},
VolumeMounts: []v1.VolumeMount{
VolumeMounts: append([]v1.VolumeMount{
{
Name: "secrets",
MountPath: constants.KubernetesAPIServerSecretsDir,
ReadOnly: true,
},
},
}, volumeMounts(cfg.ExtraVolumes)...),
},
},
HostNetwork: true,
SecurityContext: &v1.PodSecurityContext{
RunAsNonRoot: pointer.ToBool(true),
RunAsUser: pointer.ToInt64(constants.KubernetesRunUser),
},
Volumes: []v1.Volume{
Volumes: append([]v1.Volume{
{
Name: "secrets",
VolumeSource: v1.VolumeSource{
Expand All @@ -237,7 +268,7 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
},
},
},
},
}, volumes(cfg.ExtraVolumes)...),
},
})

Expand Down Expand Up @@ -295,13 +326,13 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
Name: "kube-controller-manager",
Image: cfg.Image,
Command: args,
VolumeMounts: []v1.VolumeMount{
VolumeMounts: append([]v1.VolumeMount{
{
Name: "secrets",
MountPath: constants.KubernetesControllerManagerSecretsDir,
ReadOnly: true,
},
},
}, volumeMounts(cfg.ExtraVolumes)...),
LivenessProbe: &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Expand All @@ -319,7 +350,7 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
RunAsNonRoot: pointer.ToBool(true),
RunAsUser: pointer.ToInt64(constants.KubernetesRunUser),
},
Volumes: []v1.Volume{
Volumes: append([]v1.Volume{
{
Name: "secrets",
VolumeSource: v1.VolumeSource{
Expand All @@ -328,7 +359,7 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
},
},
},
},
}, volumes(cfg.ExtraVolumes)...),
},
})

Expand Down Expand Up @@ -377,13 +408,13 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
Name: "kube-scheduler",
Image: cfg.Image,
Command: args,
VolumeMounts: []v1.VolumeMount{
VolumeMounts: append([]v1.VolumeMount{
{
Name: "secrets",
MountPath: constants.KubernetesSchedulerSecretsDir,
ReadOnly: true,
},
},
}, volumeMounts(cfg.ExtraVolumes)...),
LivenessProbe: &v1.Probe{
Handler: v1.Handler{
HTTPGet: &v1.HTTPGetAction{
Expand All @@ -401,7 +432,7 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
RunAsNonRoot: pointer.ToBool(true),
RunAsUser: pointer.ToInt64(constants.KubernetesRunUser),
},
Volumes: []v1.Volume{
Volumes: append([]v1.Volume{
{
Name: "secrets",
VolumeSource: v1.VolumeSource{
Expand All @@ -410,7 +441,7 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
},
},
},
},
}, volumes(cfg.ExtraVolumes)...),
},
})

Expand Down
Loading

0 comments on commit 8789849

Please sign in to comment.