Skip to content

Commit

Permalink
Support containerd configuration overrides (#608)
Browse files Browse the repository at this point in the history 
* drop unused containerd config types

* allow passing custom containerd configuration (overrides defaults)

* use mergo v1
  • Loading branch information
@neoaggelos
neoaggelos authored Aug 15, 2024
1 parent f37670a commit 9ad780e
Show file tree
Hide file tree
Showing 11 changed files with 93 additions and 280 deletions.
3 changes: 3 additions & 0 deletions src/k8s/api/v1/bootstrap_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ type BootstrapConfig struct {
ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
ExtraNodeK8sDqliteArgs map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`

// Extra configuration for the containerd config.toml
ExtraNodeContainerdConfig map[string]any `json:"extra-node-containerd-config,omitempty" yaml:"extra-node-containerd-config,omitempty"`
}

func (b *BootstrapConfig) GetDatastoreType() string { return getField(b.DatastoreType) }
Expand Down
6 changes: 6 additions & 0 deletions src/k8s/api/v1/join_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ type ControlPlaneNodeJoinConfig struct {
ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
ExtraNodeK8sDqliteArgs map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`

// Extra configuration for the containerd config.toml
ExtraNodeContainerdConfig map[string]any `json:"extra-node-containerd-config,omitempty" yaml:"extra-node-containerd-config,omitempty"`
}

type WorkerNodeJoinConfig struct {
Expand All @@ -55,6 +58,9 @@ type WorkerNodeJoinConfig struct {
ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
ExtraNodeK8sAPIServerProxyArgs map[string]*string `json:"extra-node-k8s-apiserver-proxy-args,omitempty" yaml:"extra-node-k8s-apiserver-proxy-args,omitempty"`

// Extra configuration for the containerd config.toml
ExtraNodeContainerdConfig map[string]any `json:"extra-node-containerd-config,omitempty" yaml:"extra-node-containerd-config,omitempty"`
}

func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string {
Expand Down
1 change: 1 addition & 0 deletions src/k8s/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/canonical/k8s
go 1.22.5

require (
dario.cat/mergo v1.0.0
github.com/canonical/go-dqlite v1.22.0
github.com/canonical/lxd v0.0.0-20240730172021-8e39e5d4f55f
github.com/canonical/microcluster/v2 v2.0.2
Expand Down
2 changes: 2 additions & 0 deletions src/k8s/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
Expand Down
4 changes: 2 additions & 2 deletions src/k8s/pkg/k8sd/app/hooks_bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ func (a *App) onBootstrapWorkerNode(ctx context.Context, s state.State, encodedT
}

// Worker node services
if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil {
if err := setup.Containerd(snap, joinConfig.ExtraNodeContainerdConfig, joinConfig.ExtraNodeContainerdArgs); err != nil {
return fmt.Errorf("failed to configure containerd: %w", err)
}
if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs); err != nil {
Expand Down Expand Up @@ -394,7 +394,7 @@ func (a *App) onBootstrapControlPlane(ctx context.Context, s state.State, bootst
}

// Configure services
if err := setup.Containerd(snap, nil, bootstrapConfig.ExtraNodeContainerdArgs); err != nil {
if err := setup.Containerd(snap, bootstrapConfig.ExtraNodeContainerdConfig, bootstrapConfig.ExtraNodeContainerdArgs); err != nil {
return fmt.Errorf("failed to configure containerd: %w", err)
}
if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), bootstrapConfig.ExtraNodeKubeletArgs); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion src/k8s/pkg/k8sd/app/hooks_join.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func (a *App) onPostJoin(ctx context.Context, s state.State, initConfig map[stri
}

// Configure services
if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil {
if err := setup.Containerd(snap, joinConfig.ExtraNodeContainerdConfig, joinConfig.ExtraNodeContainerdArgs); err != nil {
return fmt.Errorf("failed to configure containerd: %w", err)
}
if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), joinConfig.ExtraNodeKubeletArgs); err != nil {
Expand Down
203 changes: 75 additions & 128 deletions src/k8s/pkg/k8sd/setup/containerd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ import (
"os"
"path/filepath"

"dario.cat/mergo"
"github.com/canonical/k8s/pkg/k8sd/images"
"github.com/canonical/k8s/pkg/k8sd/types"
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
"github.com/canonical/k8s/pkg/utils"
Expand All @@ -16,119 +16,96 @@ import (

const defaultPauseImage = "ghcr.io/canonical/k8s-snap/pause:3.10"

var (
containerdConfigTomlTemplate = mustTemplate("containerd", "config.toml")
)

type containerdConfigTomlConfig struct {
CNIConfDir string
CNIBinDir string
ImportsDir string
RegistryConfigDir string
PauseImage string
}

type containerdConfig struct {
Version int `toml:"version"`
Plugins containerdConfigPlugins `toml:"plugins,omitempty"`
}

type containerdConfigPlugins struct {
CRI containerdConfigPluginsCRI `toml:"io.containerd.grpc.v1.cri,omitempty"`
}

type containerdConfigPluginsCRI struct {
Registry containerdConfigPluginsCRIRegistry `toml:"registry,omitempty"`
}

type containerdConfigPluginsCRIRegistry struct {
Configs map[string]containerdConfigPluginsCRIRegistryConfig `toml:"configs,omitempty"`
}

type containerdConfigPluginsCRIRegistryConfig struct {
Auth containerdConfigPluginsCRIRegistryConfigAuth `toml:"auth,omitempty"`
}
func defaultContainerdConfig(
cniConfDir string,
cniBinDir string,
importsDir string,
registryConfigDir string,
pauseImage string,
) map[string]any {
return map[string]any{
"version": 2,
"oom_score": 0,
"imports": []string{filepath.Join(importsDir, "*.toml")},

"grpc": map[string]any{
"uid": 0,
"gid": 0,
"max_recv_message_size": 16777216,
"max_send_message_size": 16777216,
},

type containerdConfigPluginsCRIRegistryConfigAuth struct {
Username string `toml:"username,omitempty"`
Password string `toml:"password,omitempty"`
Token string `toml:"token,omitempty"`
}
"debug": map[string]any{
"uid": 0,
"gid": 0,
"address": "",
"level": "",
},

type containerdHostsConfig struct {
Server string `toml:"server,omitempty"`
Host map[string]containerdHostsConfigHost `toml:"hosts,omitempty"`
}
"metrics": map[string]any{
"address": "",
"grpc_histogram": false,
},

type containerdHostsConfigHost struct {
Capabilities []string `toml:"capabilities,omitempty"`
SkipVerify bool `toml:"skip_verify,omitempty"`
OverridePath bool `toml:"override_path,omitempty"`
}
"cgroup": map[string]any{
"path": "",
},

func containerdAuthConfig(registries []types.ContainerdRegistry) containerdConfig {
authConfigs := make(map[string]containerdConfigPluginsCRIRegistryConfig, len(registries))
for _, registry := range registries {
if registry.Username != "" || registry.Password != "" || registry.Token != "" {
for _, url := range registry.URLs {
authConfigs[url] = containerdConfigPluginsCRIRegistryConfig{
Auth: containerdConfigPluginsCRIRegistryConfigAuth{
Username: registry.Username,
Password: registry.Password,
Token: registry.Token,
"plugins": map[string]any{
"io.containerd.grpc.v1.cri": map[string]any{
"stream_server_address": "127.0.0.1",
"stream_server_port": "0",
"enable_selinux": false,
"sandbox_image": pauseImage,
"stats_collect_period": 10,
"enable_tls_streaming": false,
"max_container_log_line_size": 16384,

"containerd": map[string]any{
"no_pivot": false,
"default_runtime_name": "runc",

"runtimes": map[string]any{
"runc": map[string]any{
"runtime_type": "io.containerd.runc.v2",
},
},
}
}
}
}
},

return containerdConfig{
Version: 2,
Plugins: containerdConfigPlugins{
CRI: containerdConfigPluginsCRI{
Registry: containerdConfigPluginsCRIRegistry{
Configs: authConfigs,
"cni": map[string]any{
"bin_dir": cniBinDir,
"conf_dir": cniConfDir,
},

"registry": map[string]any{
"config_path": registryConfigDir,
},
},
},
}
}

func containerdHostConfig(registry types.ContainerdRegistry) containerdHostsConfig {
if len(registry.URLs) == 0 {
return containerdHostsConfig{}
}

hosts := make(map[string]containerdHostsConfigHost, len(registry.URLs))
for _, url := range registry.URLs {
hosts[url] = containerdHostsConfigHost{
Capabilities: []string{"pull", "resolve"},
SkipVerify: registry.SkipVerify,
OverridePath: registry.OverridePath,
}
}

return containerdHostsConfig{
Server: registry.URLs[0],
Host: hosts,
}
}

// Containerd configures configuration and arguments for containerd on the local node.
// Optionally, a number of registry mirrors and auths can be configured.
func Containerd(snap snap.Snap, registries []types.ContainerdRegistry, extraArgs map[string]*string) error {
configToml, err := os.OpenFile(filepath.Join(snap.ContainerdConfigDir(), "config.toml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
func Containerd(snap snap.Snap, extraContainerdConfig map[string]any, extraArgs map[string]*string) error {
configToml := defaultContainerdConfig(
snap.CNIConfDir(),
snap.CNIBinDir(),
snap.ContainerdExtraConfigDir(),
snap.ContainerdRegistryConfigDir(),
defaultPauseImage,
)

if err := mergo.Merge(&configToml, extraContainerdConfig, mergo.WithAppendSlice, mergo.WithOverride); err != nil {
return fmt.Errorf("failed to merge containerd config.toml overrides: %w", err)
}

b, err := toml.Marshal(configToml)
if err != nil {
return fmt.Errorf("failed to open config.toml: %w", err)
return fmt.Errorf("failed to render containerd config.toml: %w", err)
}
defer configToml.Close()
if err := containerdConfigTomlTemplate.Execute(configToml, containerdConfigTomlConfig{
CNIConfDir: snap.CNIConfDir(),
CNIBinDir: snap.CNIBinDir(),
ImportsDir: snap.ContainerdExtraConfigDir(),
RegistryConfigDir: snap.ContainerdRegistryConfigDir(),
PauseImage: defaultPauseImage,
}); err != nil {

if err := os.WriteFile(filepath.Join(snap.ContainerdConfigDir(), "config.toml"), b, 0600); err != nil {
return fmt.Errorf("failed to write config.toml: %w", err)
}

Expand Down Expand Up @@ -179,36 +156,6 @@ func Containerd(snap snap.Snap, registries []types.ContainerdRegistry, extraArgs
}
}

// registry auths
if authConfig := containerdAuthConfig(registries); len(authConfig.Plugins.CRI.Registry.Configs) > 0 {
b, err := toml.Marshal(authConfig)
if err != nil {
return fmt.Errorf("failed to marshal registry auth configurations: %w", err)
}

if err := os.WriteFile(filepath.Join(snap.ContainerdExtraConfigDir(), "k8sd-auths.toml"), b, 0600); err != nil {
return fmt.Errorf("failed to write registry auth configurations: %w", err)
}
}

// registry mirrors
for _, registry := range registries {
if hostConfig := containerdHostConfig(registry); len(hostConfig.Host) > 0 {
b, err := toml.Marshal(hostConfig)
if err != nil {
return fmt.Errorf("failed to render registry mirrors for %s: %w", registry.Host, err)
}

dir := filepath.Join(snap.ContainerdRegistryConfigDir(), registry.Host)
if err := os.Mkdir(dir, 0700); err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to create directory for registry %s: %w", registry.Host, err)
}
if err := os.WriteFile(filepath.Join(dir, "hosts.toml"), b, 0600); err != nil {
return fmt.Errorf("failed to write hosts.toml for registry %s: %w", registry.Host, err)
}
}
}

return nil
}

Expand Down
Loading

0 comments on commit 9ad780e

Please sign in to comment.