Skip to content

Commit

Permalink
Add csrsigning controller for worker node certificate refreshes (#548)
Browse files Browse the repository at this point in the history
* Add controllers/csrsigning controller

* add csrsigning controller in k8sd
  • Loading branch information
neoaggelos authored Jul 19, 2024
1 parent 3d30ad9 commit 2994b87
Show file tree
Hide file tree
Showing 14 changed files with 580 additions and 2 deletions.
3 changes: 3 additions & 0 deletions src/k8s/cmd/k8sd/k8sd.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ var rootCmdOpts struct {
disableControlPlaneConfigController bool
disableFeatureController bool
disableUpdateNodeConfigController bool
disableCSRSigningController bool
}

func NewRootCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
Expand All @@ -40,6 +41,7 @@ func NewRootCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
DisableControlPlaneConfigController: rootCmdOpts.disableControlPlaneConfigController,
DisableUpdateNodeConfigController: rootCmdOpts.disableUpdateNodeConfigController,
DisableFeatureController: rootCmdOpts.disableFeatureController,
DisableCSRSigningController: rootCmdOpts.disableCSRSigningController,
})
if err != nil {
cmd.PrintErrf("Error: Failed to initialize k8sd: %v", err)
Expand Down Expand Up @@ -68,6 +70,7 @@ func NewRootCmd(env cmdutil.ExecutionEnvironment) *cobra.Command {
cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableControlPlaneConfigController, "disable-control-plane-config-controller", false, "Disable the Control Plane Config Controller")
cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableUpdateNodeConfigController, "disable-update-node-config-controller", false, "Disable the Update Node Config Controller")
cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableFeatureController, "disable-feature-controller", false, "Disable the Feature Controller")
cmd.PersistentFlags().BoolVar(&rootCmdOpts.disableCSRSigningController, "disable-csrsigning-controller", false, "Disable the CSR signing controller")

cmd.Flags().Uint("port", 0, "Default port for the HTTP API")
cmd.Flags().MarkDeprecated("port", "this flag does not have any effect, and will be removed in a future version")
Expand Down
6 changes: 5 additions & 1 deletion src/k8s/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ require (
k8s.io/cli-runtime v0.29.0
k8s.io/client-go v0.30.1
k8s.io/klog/v2 v2.120.1
sigs.k8s.io/controller-runtime v0.18.4
)

require (
Expand Down Expand Up @@ -54,6 +55,7 @@ require (
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v5.7.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
Expand All @@ -67,6 +69,7 @@ require (
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
Expand Down Expand Up @@ -113,7 +116,6 @@ require (
github.com/muhlemmer/gu v0.3.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/onsi/ginkgo/v2 v2.17.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
Expand Down Expand Up @@ -143,11 +145,13 @@ require (
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.starlark.net v0.0.0-20240329153429-e6e8e7ce1b7a // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
google.golang.org/grpc v1.62.1 // indirect
Expand Down
16 changes: 16 additions & 0 deletions src/k8s/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
Expand Down Expand Up @@ -197,6 +199,8 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
Expand Down Expand Up @@ -629,8 +633,14 @@ go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw
go.starlark.net v0.0.0-20240329153429-e6e8e7ce1b7a h1:Oe+v9w90BBIxQZ4U39+axR8KxrBbxqnRudPPcBIlP3o=
go.starlark.net v0.0.0-20240329153429-e6e8e7ce1b7a/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand All @@ -654,6 +664,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand Down Expand Up @@ -890,6 +902,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
Expand Down Expand Up @@ -1062,6 +1076,8 @@ oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw=
sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=
Expand Down
8 changes: 7 additions & 1 deletion src/k8s/pkg/client/kubernetes/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ import (

"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)

// Client is a wrapper around the kubernetes.Interface.
type Client struct {
kubernetes.Interface
config *rest.Config
}

func NewClient(restClientGetter genericclioptions.RESTClientGetter) (*Client, error) {
Expand All @@ -21,5 +23,9 @@ func NewClient(restClientGetter genericclioptions.RESTClientGetter) (*Client, er
if err != nil {
return nil, fmt.Errorf("failed to create Kubernetes clientset: %w", err)
}
return &Client{clientset}, nil
return &Client{Interface: clientset, config: config}, nil
}

func (c *Client) RESTConfig() *rest.Config {
return rest.CopyConfig(c.config)
}
15 changes: 15 additions & 0 deletions src/k8s/pkg/k8sd/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/canonical/k8s/pkg/k8sd/api"
"github.com/canonical/k8s/pkg/k8sd/controllers"
"github.com/canonical/k8s/pkg/k8sd/controllers/csrsigning"
"github.com/canonical/k8s/pkg/k8sd/database"
"github.com/canonical/k8s/pkg/log"
"github.com/canonical/k8s/pkg/snap"
Expand Down Expand Up @@ -39,6 +40,8 @@ type Config struct {
DisableUpdateNodeConfigController bool
// DisableFeatureController is a bool flag to disable feature controller
DisableFeatureController bool
// DisableCSRSigningController is a bool flag to disable csrsigning controller.
DisableCSRSigningController bool
}

// App is the k8sd microcluster instance.
Expand All @@ -55,6 +58,7 @@ type App struct {

nodeConfigController *controllers.NodeConfigurationController
controlPlaneConfigController *controllers.ControlPlaneConfigurationController
csrsigningController *csrsigning.Controller

// updateNodeConfigController
triggerUpdateNodeConfigControllerCh chan struct{}
Expand Down Expand Up @@ -151,6 +155,17 @@ func New(cfg Config) (*App, error) {
} else {
log.L().Info("feature-controller disabled via config")
}

if !cfg.DisableCSRSigningController {
app.csrsigningController = csrsigning.New(csrsigning.Options{
Snap: cfg.Snap,
WaitReady: app.readyWg.Wait,
LeaderElection: true,
})
} else {
log.L().Info("csrsigning-controller disabled via config")
}

return app, nil
}

Expand Down
10 changes: 10 additions & 0 deletions src/k8s/pkg/k8sd/app/hooks_start.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,5 +75,15 @@ func (a *App) onStart(s *state.State) error {
)
}

// start csrsigning controller
if a.csrsigningController != nil {
go a.csrsigningController.Run(
s.Context,
func(ctx context.Context) (types.ClusterConfig, error) {
return databaseutil.GetClusterConfig(ctx, s)
},
)
}

return nil
}
15 changes: 15 additions & 0 deletions src/k8s/pkg/k8sd/controllers/csrsigning/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package csrsigning

import "github.com/canonical/k8s/pkg/k8sd/types"

type internalConfig struct {
autoApprove bool
}

func internalConfigFromAnnotations(annotations types.Annotations) internalConfig {
var cfg internalConfig
if v, ok := annotations.Get("k8sd/v1alpha1/csrsigning/auto-approve"); ok && v == "true" {
cfg.autoApprove = true
}
return cfg
}
11 changes: 11 additions & 0 deletions src/k8s/pkg/k8sd/controllers/csrsigning/const.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package csrsigning

import "time"

const (
// requeueAfterSigningFailure is the time to requeue requests when any step of the signing process failed
requeueAfterSigningFailure = 3 * time.Second

// requeueAfterWaitingForApproved is the amount of time to requeue requests if waiting for CSR to be approved
requeueAfterWaitingForApproved = 10 * time.Second
)
105 changes: 105 additions & 0 deletions src/k8s/pkg/k8sd/controllers/csrsigning/controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package csrsigning

import (
"context"
"fmt"
"time"

"github.com/canonical/k8s/pkg/k8sd/types"
"github.com/canonical/k8s/pkg/log"
"github.com/canonical/k8s/pkg/snap"
"github.com/canonical/k8s/pkg/utils"
"k8s.io/client-go/rest"

"sigs.k8s.io/controller-runtime/pkg/cache"
ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/metrics/server"
)

type Controller struct {
snap snap.Snap
waitReady func()

leaderElection bool
}

type Options struct {
Snap snap.Snap
WaitReady func()

LeaderElection bool
}

func New(opts Options) *Controller {
return &Controller{
snap: opts.Snap,
waitReady: opts.WaitReady,

leaderElection: opts.LeaderElection,
}
}

func (c *Controller) getRESTConfig(ctx context.Context) (*rest.Config, error) {
for {
client, err := c.snap.KubernetesClient("")
if err == nil {
return client.RESTConfig(), nil
}

select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(3 * time.Second):
}
}
}

func (c *Controller) Run(ctx context.Context, getClusterConfig func(context.Context) (types.ClusterConfig, error)) error {
ctx = log.NewContext(ctx, log.FromContext(ctx).WithName("csrsigning"))

// TODO(neoaggelos): This should be moved to init() or some other initialization step
ctrllog.SetLogger(log.FromContext(ctx))

c.waitReady()

config, err := c.getRESTConfig(ctx)
if err != nil {
return fmt.Errorf("failed to get Kubernetes REST config: %w", err)
}

// TODO(neoaggelos): In case of more controllers, a single manager object should be created
// and passed here as configuration.
mgr, err := manager.New(config, manager.Options{
Logger: log.FromContext(ctx),
LeaderElection: c.leaderElection,
LeaderElectionID: "a27980c4.k8sd-csrsigning-controller",
LeaderElectionNamespace: "kube-system",
BaseContext: func() context.Context { return ctx },
Cache: cache.Options{
SyncPeriod: utils.Pointer(10 * time.Minute),
},
Metrics: server.Options{
BindAddress: "0",
},
})
if err != nil {
return fmt.Errorf("failed to create controller manager: %w", err)
}

if err := (&csrSigningReconciler{
Manager: mgr,
Logger: mgr.GetLogger(),
Client: mgr.GetClient(),

getClusterConfig: getClusterConfig,
}).SetupWithManager(mgr); err != nil {
return fmt.Errorf("failed to setup csrsigning controller: %w", err)
}

if err := mgr.Start(ctx); err != nil {
return fmt.Errorf("controller manager failed: %w", err)
}

return nil
}
Loading

0 comments on commit 2994b87

Please sign in to comment.