Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Add basic batteries #41

Merged
merged 3 commits into from
Oct 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,29 @@ export KUBECONFIG=.gcp/admin.kubeconfig
kubectl api-resources
```

## Batteries

Example server contains a simple implementation of batteries that can be used to extend the gcp API.

Batteries:
- `leases` - a Kubernetes lease resources from `coordination.k8s.io`
- `authentication` - Kubernetes native authentication using `authentication.k8s.io`
- `authorization` - Kubernetes native authorization using `authorization.k8s.io`
- `admission` - Kubernetes native admission using `admissionregistration.k8s.io`
- `flowcontrol` - Kubernetes native flow control using `flowcontrol.apiserver.k8s.io`


When starting server without any flags, in-memory storage will be used and batteries will be disabled by default.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does in-memory storage mean? You mean you wire through no-op informers I assume?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in-memory etcd - our usual standalone development mode.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ack


Important: In the long run, we plan to move existing apis into batteries on its own, and make default server to be a simple server without any resources.

To start the server with batteries enabled, use the following flags:

```bash
./bin/gcp start --batteries=lease,authentication,authorization,admission,flowcontrol
```


## Contributing

We ❤️ our contributors! If you're interested in helping us out, please check out [contributing to Generic Control Plane](CONTRIBUTING.md).
Expand Down
56 changes: 5 additions & 51 deletions server/admission/plugins.go → server/batteries/admission.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package admission
package batteries

import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
validatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/validating"
"k8s.io/apiserver/pkg/admission/plugin/resourcequota"
Expand All @@ -29,7 +27,6 @@ import (
"k8s.io/kubernetes/plugin/pkg/admission/certificates/ctbattest"
certsigning "k8s.io/kubernetes/plugin/pkg/admission/certificates/signing"
certsubjectrestriction "k8s.io/kubernetes/plugin/pkg/admission/certificates/subjectrestriction"
"k8s.io/kubernetes/plugin/pkg/admission/defaulttolerationseconds"
"k8s.io/kubernetes/plugin/pkg/admission/deny"
"k8s.io/kubernetes/plugin/pkg/admission/eventratelimit"
"k8s.io/kubernetes/plugin/pkg/admission/gc"
Expand All @@ -41,11 +38,10 @@ import (

// AllOrderedPlugins is the list of all the plugins in order.
var AllOrderedPlugins = []string{
admit.PluginName, // AlwaysAdmit
autoprovision.PluginName, // NamespaceAutoProvision
lifecycle.PluginName, // NamespaceLifecycle
exists.PluginName, // NamespaceExists
// limitranger.PluginName, // LimitRanger
admit.PluginName, // AlwaysAdmit
autoprovision.PluginName, // NamespaceAutoProvision
lifecycle.PluginName, // NamespaceLifecycle
exists.PluginName, // NamespaceExists
serviceaccount.PluginName, // ServiceAccount
eventratelimit.PluginName, // EventRateLimit
gc.PluginName, // OwnerReferencesPermissionEnforcement
Expand All @@ -62,45 +58,3 @@ var AllOrderedPlugins = []string{
resourcequota.PluginName, // ResourceQuota
deny.PluginName, // AlwaysDeny
}

// RegisterAllAdmissionPlugins registers all admission plugins.
// The order of registration is irrelevant, see AllOrderedPlugins for execution order.
func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
admit.Register(plugins) // DEPRECATED as no real meaning
autoprovision.Register(plugins)
lifecycle.Register(plugins)
exists.Register(plugins)
// limitranger.Register(plugins)
serviceaccount.Register(plugins)
eventratelimit.Register(plugins)
gc.Register(plugins)
certapproval.Register(plugins)
certsigning.Register(plugins)
ctbattest.Register(plugins)
certsubjectrestriction.Register(plugins)
mutatingwebhook.Register(plugins)
validatingadmissionpolicy.Register(plugins)
validatingwebhook.Register(plugins)
resourcequota.Register(plugins)
deny.Register(plugins)
}

// DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver.
func DefaultOffAdmissionPlugins() sets.Set[string] {
defaultOnPlugins := sets.New(
lifecycle.PluginName, // NamespaceLifecycle
// limitranger.PluginName, // LimitRanger
serviceaccount.PluginName, // ServiceAccount
defaulttolerationseconds.PluginName, // DefaultTolerationSeconds
mutatingwebhook.PluginName, // MutatingAdmissionWebhook
validatingwebhook.PluginName, // ValidatingAdmissionWebhook
resourcequota.PluginName, // ResourceQuota
certapproval.PluginName, // CertificateApproval
certsigning.PluginName, // CertificateSigning
ctbattest.PluginName, // ClusterTrustBundleAttest
certsubjectrestriction.PluginName, // CertificateSubjectRestriction
validatingadmissionpolicy.PluginName, // ValidatingAdmissionPolicy, only active when feature gate ValidatingAdmissionPolicy is enabled
)

return sets.New[string](AllOrderedPlugins...).Difference(defaultOnPlugins)
}
168 changes: 168 additions & 0 deletions server/batteries/battery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
package batteries

import (
"golang.org/x/exp/slices"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
validatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/validating"
"k8s.io/apiserver/pkg/admission/plugin/resourcequota"
mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
"k8s.io/kubernetes/plugin/pkg/admission/admit"
certapproval "k8s.io/kubernetes/plugin/pkg/admission/certificates/approval"
"k8s.io/kubernetes/plugin/pkg/admission/certificates/ctbattest"

certsigning "k8s.io/kubernetes/plugin/pkg/admission/certificates/signing"
certsubjectrestriction "k8s.io/kubernetes/plugin/pkg/admission/certificates/subjectrestriction"
"k8s.io/kubernetes/plugin/pkg/admission/defaulttolerationseconds"
"k8s.io/kubernetes/plugin/pkg/admission/deny"
"k8s.io/kubernetes/plugin/pkg/admission/eventratelimit"
"k8s.io/kubernetes/plugin/pkg/admission/gc"
"k8s.io/kubernetes/plugin/pkg/admission/namespace/autoprovision"
"k8s.io/kubernetes/plugin/pkg/admission/namespace/exists"
"k8s.io/kubernetes/plugin/pkg/admission/serviceaccount"
// Admission policies
)

type Batteries struct {
list BatteriesList

BatteriesArgs []string
}

type Battery string

type BatteriesList map[Battery]BatterySpec

type BatterySpec struct {
// Enabled indicates whether the battery is enabled.
Enabled bool

// GroupNames is the list of group names that the battery is responsible for.
// If disabled, the battery will not be registered for these groups.
GroupNames []string
}

const (
// BatteryLeases is the name of the lease battery.
BatteryLeases Battery = "leases"
// BatteryAuthentication is the name of the authentication battery.
BatteryAuthentication Battery = "authentication"
// BatteryAuthorization is the name of the authorization battery.
BatteryAuthorization Battery = "authorization"
// BatteryAdmission is the name of the admission battery.
BatteryAdmission Battery = "admission"
// BatteryFlowControl is the name of the flow control battery.
BatteryFlowControl Battery = "flowcontrol"
// BatteryCRDs is the name of the CRD battery.
BatteryCRDs Battery = "crds"
)

var (
// The generic features.
defaultBatteries = map[Battery]BatterySpec{
BatteryLeases: {Enabled: false, GroupNames: []string{"coordination.k8s.io"}},
BatteryAuthentication: {Enabled: false, GroupNames: []string{"authentication.k8s.io", "rbac.authentication.k8s.io"}},
BatteryAuthorization: {Enabled: false, GroupNames: []string{"authorization.k8s.io", "rbac.authorization.k8s.io"}},
BatteryAdmission: {Enabled: false, GroupNames: []string{"admissionregistration.k8s.io"}},
BatteryFlowControl: {Enabled: false, GroupNames: []string{"flowcontrol.apiserver.k8s.io"}},
BatteryCRDs: {Enabled: false, GroupNames: []string{"apiextensions.k8s.io"}},
}
)

func (b Battery) String() string {
return string(b)
}

func New() Batteries {
b := Batteries{
list: make(BatteriesList, len(defaultBatteries)),
}
for name, spec := range defaultBatteries {
b.list[name] = spec
}
return b
}

func (b Batteries) Enable(name Battery) {
_b := b.list[name]
_b.Enabled = true
b.list[name] = _b
}

func (b Batteries) Disable(name Battery) {
_b := b.list[name]
_b.Enabled = false
b.list[name] = _b
}

func (b Batteries) IsEnabled(name Battery) bool {
spec, ok := b.list[name]
return ok && spec.Enabled
}

// RegisterAllAdmissionPlugins registers all admission plugins based on the batteries configuration.
func (b Batteries) RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
admit.Register(plugins) // DEPRECATED as no real meaning
autoprovision.Register(plugins)
lifecycle.Register(plugins)
exists.Register(plugins)
serviceaccount.Register(plugins)
eventratelimit.Register(plugins)
gc.Register(plugins)
certapproval.Register(plugins)
certsigning.Register(plugins)
ctbattest.Register(plugins)
certsubjectrestriction.Register(plugins)
mutatingwebhook.Register(plugins)
validatingadmissionpolicy.Register(plugins)
validatingwebhook.Register(plugins)
resourcequota.Register(plugins)
deny.Register(plugins)
}

func (b Batteries) DefaultOffAdmissionPlugins() sets.Set[string] {
defaultOnPlugins := sets.New(
lifecycle.PluginName, // NamespaceLifecycle
// limitranger.PluginName, // LimitRanger
serviceaccount.PluginName, // ServiceAccount
resourcequota.PluginName, // ResourceQuota
certapproval.PluginName, // CertificateApproval
certsigning.PluginName, // CertificateSigning
ctbattest.PluginName, // ClusterTrustBundleAttest
certsubjectrestriction.PluginName, // CertificateSubjectRestriction
defaulttolerationseconds.PluginName, // DefaultTolerationSeconds
)

if b.IsEnabled(BatteryAdmission) {
defaultOnPlugins.Insert(
mutatingwebhook.PluginName, // MutatingAdmissionWebhook
validatingwebhook.PluginName, // ValidatingAdmissionWebhook
validatingadmissionpolicy.PluginName, // ValidatingAdmissionPolicy, only active when feature gate ValidatingAdmissionPolicy is enabled
)
}

return sets.New[string](AllOrderedPlugins...).Difference(defaultOnPlugins)
}

func (b Batteries) containsAndDisabled(name string) bool {
for _, spec := range b.list {
if slices.Contains(spec.GroupNames, name) && !spec.Enabled {
return true
}
}
return false
}

func (b Batteries) FilterStorageProviders(input []controlplaneapiserver.RESTStorageProvider) []controlplaneapiserver.RESTStorageProvider {
var result []controlplaneapiserver.RESTStorageProvider
for _, rest := range input {
if b.containsAndDisabled(rest.GroupName()) {
continue
}
result = append(result, rest)
}
return result
}
62 changes: 62 additions & 0 deletions server/batteries/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
Copyright 2024 The KCP Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package batteries

import (
"fmt"

"github.com/spf13/pflag"

utilruntime "k8s.io/apimachinery/pkg/util/runtime"
genericfeatures "k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
)

// AddFlags adds the flags for the admin authentication to the given FlagSet.
func (s *Batteries) AddFlags(fs *pflag.FlagSet) {
if s == nil {
return
}

fs.StringSliceVar(&s.BatteriesArgs, "batteries", []string{}, "The batteries to enable in the generic control-plane server.")
}

func (b Batteries) Complete() {
// Ensure all some related configuration are configured

for _, name := range b.BatteriesArgs {
if _, ok := b.list[Battery(name)]; ok {
b.Enable(Battery(name))
}
}

// If lease is disabled, we disable APIServerIdentity
if !b.IsEnabled(BatteryLeases) {
utilruntime.Must(utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=false", genericfeatures.APIServerIdentity)))
}
}

func (b Batteries) Validate() []error {
var errs []error
for _, name := range b.BatteriesArgs {
fmt.Println(name)
if _, ok := b.list[Battery(name)]; !ok {
errs = append(errs, fmt.Errorf("invalid battery %q", name))
}
}
return errs
}
9 changes: 9 additions & 0 deletions server/cmd/options/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ import (
"k8s.io/kubernetes/pkg/controlplane"
controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver"
generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"

"github.com/kcp-dev/generic-controlplane/server/batteries"
)

// Config holds the configuration for the generic controlplane server.
Expand All @@ -46,6 +48,8 @@ type Config struct {
type ExtraConfig struct {
// authentication
GcpAdminToken, UserToken string
// Batteries holds the batteries configuration for the generic controlplane server.
Batteries batteries.Batteries
}

type completedConfig struct {
Expand All @@ -68,6 +72,8 @@ type CompletedConfig struct {

// Complete fills in any fields not set that are required to have valid data.
func (c *Config) Complete() (CompletedConfig, error) {
c.Batteries.Complete()

return CompletedConfig{&completedConfig{
Options: c.Options,

Expand All @@ -85,6 +91,9 @@ func (c *Config) Complete() (CompletedConfig, error) {
func NewConfig(opts CompletedOptions) (*Config, error) {
c := &Config{
Options: opts,
ExtraConfig: ExtraConfig{
Batteries: opts.Extra.Batteries,
},
}

if opts.EmbeddedEtcd.Enabled {
Expand Down
Loading