Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(kuma-cp): introduce tag first Virtual Outbound model #7076

Merged
merged 5 commits into from
Jun 21, 2023
Merged

feat(kuma-cp): introduce tag first Virtual Outbound model #7076

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5612a7b
feat(kuma-cp): introduce tag first Virtual Outbound model
Automaat Jun 21, 2023
f3742c7
feat(kuma-cp): revert default
Automaat Jun 21, 2023
9326fee
feat(kuma-cp): code review
Automaat Jun 21, 2023
27effab
feat(kuma-cp): fix check
Automaat Jun 21, 2023
5a74d6a
feat(kuma-cp): fix check
Automaat Jun 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/generated/kuma-cp.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -681,6 +681,10 @@ experimental:
# If true, instead of embedding kubernetes outbounds into Dataplane object, they are persisted next to VIPs in ConfigMap
# This can improve performance, but it should be enabled only after all instances are migrated to version that supports this config
kubeOutboundsAsVIPs: true # ENV: KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS
# Tag first virtual outbound model is compressed version of default Virtual Outbound model
# It is recommended to use tag first model for deployments with more than 2k services
# This is not backward compatible model.
useTagFirstVirtualOutboundModel: false # ENV: KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL

proxy:
gateway:
Expand Down
2 changes: 1 addition & 1 deletion pkg/api-server/api_server_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ func tryStartApiServer(t *testApiServerConfigurer) (*api_server.ApiServer, kuma_
server.MeshResourceTypes(server.HashMeshExcludedResources),
net.LookupIP,
cfg.Multizone.Zone.Name,
vips.NewPersistence(resManager, config_manager.NewConfigManager(t.store)),
vips.NewPersistence(resManager, config_manager.NewConfigManager(t.store), false),
cfg.DNSServer.Domain,
80,
),
Expand Down
2 changes: 1 addition & 1 deletion pkg/api-server/customization/customization_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func createTestApiServer(store store.ResourceStore, config *config_api_server.Ap
server.MeshResourceTypes(server.HashMeshExcludedResources),
net.LookupIP,
cfg.Multizone.Zone.Name,
vips.NewPersistence(resManager, config_manager.NewConfigManager(store)),
vips.NewPersistence(resManager, config_manager.NewConfigManager(store), false),
cfg.DNSServer.Domain,
cfg.DNSServer.ServiceVipPort,
),
Expand Down
2 changes: 1 addition & 1 deletion pkg/api-server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,7 @@ func SetupServer(rt runtime.Runtime) error {
server.MeshResourceTypes(server.HashMeshExcludedResources),
net.LookupIP,
cfg.Multizone.Zone.Name,
vips.NewPersistence(rt.ResourceManager(), rt.ConfigManager()),
vips.NewPersistence(rt.ResourceManager(), rt.ConfigManager(), cfg.Experimental.UseTagFirstVirtualOutboundModel),
cfg.DNSServer.Domain,
cfg.DNSServer.ServiceVipPort,
),
Expand Down
11 changes: 8 additions & 3 deletions pkg/config/app/kuma-cp/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,10 @@ var DefaultConfig = func() Config {
DpServer: dp_server.DefaultDpServerConfig(),
Access: access.DefaultAccessConfig(),
Experimental: ExperimentalConfig{
GatewayAPI: false,
KubeOutboundsAsVIPs: true,
KDSDeltaEnabled: false,
GatewayAPI: false,
KubeOutboundsAsVIPs: true,
KDSDeltaEnabled: false,
UseTagFirstVirtualOutboundModel: true,
Automaat marked this conversation as resolved.
Show resolved Hide resolved
},
Proxy: xds.DefaultProxyConfig(),
InterCp: intercp.DefaultInterCpConfig(),
Expand Down Expand Up @@ -349,6 +350,10 @@ type ExperimentalConfig struct {
KubeOutboundsAsVIPs bool `json:"kubeOutboundsAsVIPs" envconfig:"KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS"`
// KDSDeltaEnabled defines if using KDS Sync with incremental xDS
KDSDeltaEnabled bool `json:"kdsDeltaEnabled" envconfig:"KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED"`
// Tag first virtual outbound model is compressed version of default Virtual Outbound model
// It is recommended to use tag first model for deployments with more than 2k services
// This is not backward compatible model.
UseTagFirstVirtualOutboundModel bool `json:"useTagFirstVirtualOutboundModel" envconfig:"KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL"`
}

func (e ExperimentalConfig) Validate() error {
Expand Down
4 changes: 4 additions & 0 deletions pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -678,6 +678,10 @@ experimental:
# If true, instead of embedding kubernetes outbounds into Dataplane object, they are persisted next to VIPs in ConfigMap
# This can improve performance, but it should be enabled only after all instances are migrated to version that supports this config
kubeOutboundsAsVIPs: true # ENV: KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS
# Tag first virtual outbound model is compressed version of default Virtual Outbound model
# It is recommended to use tag first model for deployments with more than 2k services
# This is not backward compatible model.
useTagFirstVirtualOutboundModel: false # ENV: KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL

proxy:
gateway:
Expand Down
3 changes: 3 additions & 0 deletions pkg/config/loader_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ var _ = Describe("Config loader", func() {
Expect(cfg.Experimental.GatewayAPI).To(BeTrue())
Expect(cfg.Experimental.KubeOutboundsAsVIPs).To(BeTrue())
Expect(cfg.Experimental.KDSDeltaEnabled).To(BeTrue())
Expect(cfg.Experimental.UseTagFirstVirtualOutboundModel).To(BeFalse())

Expect(cfg.Proxy.Gateway.GlobalDownstreamMaxConnections).To(BeNumerically("==", 1))
},
Expand Down Expand Up @@ -648,6 +649,7 @@ experimental:
kubeOutboundsAsVIPs: true
cniApp: "kuma-cni"
kdsDeltaEnabled: true
useTagFirstVirtualOutboundModel: false
proxy:
gateway:
globalDownstreamMaxConnections: 1
Expand Down Expand Up @@ -881,6 +883,7 @@ proxy:
"KUMA_ACCESS_STATIC_VIEW_CLUSTERS_GROUPS": "zt-group1,zt-group2",
"KUMA_EXPERIMENTAL_GATEWAY_API": "true",
"KUMA_EXPERIMENTAL_KUBE_OUTBOUNDS_AS_VIPS": "true",
"KUMA_EXPERIMENTAL_USE_TAG_FIRST_VIRTUAL_OUTBOUND_MODEL": "false",
"KUMA_PROXY_GATEWAY_GLOBAL_DOWNSTREAM_MAX_CONNECTIONS": "1",
},
yamlFileConfig: "",
Expand Down
2 changes: 1 addition & 1 deletion pkg/core/bootstrap/bootstrap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ func initializeMeshCache(builder *core_runtime.Builder) error {
xds_server.MeshResourceTypes(xds_server.HashMeshExcludedResources),
builder.LookupIP(),
builder.Config().Multizone.Zone.Name,
vips.NewPersistence(builder.ReadOnlyResourceManager(), builder.ConfigManager()),
vips.NewPersistence(builder.ReadOnlyResourceManager(), builder.ConfigManager(), builder.Config().Experimental.UseTagFirstVirtualOutboundModel),
builder.Config().DNSServer.Domain,
builder.Config().DNSServer.ServiceVipPort,
)
Expand Down
43 changes: 35 additions & 8 deletions pkg/dns/vips/persistence.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
type Persistence struct {
configManager config_manager.ConfigManager
resourceManager manager.ReadOnlyResourceManager
useTagFirstView bool
}

const template = "kuma-%s-dns-vips"
Expand All @@ -38,10 +39,11 @@ func MeshFromConfigKey(name string) (string, bool) {
return mesh, true
}

func NewPersistence(resourceManager manager.ReadOnlyResourceManager, configManager config_manager.ConfigManager) *Persistence {
func NewPersistence(resourceManager manager.ReadOnlyResourceManager, configManager config_manager.ConfigManager, useTagFirstVirtualOutboundModel bool) *Persistence {
return &Persistence{
resourceManager: resourceManager,
configManager: configManager,
useTagFirstView: useTagFirstVirtualOutboundModel,
}
}

Expand All @@ -60,11 +62,25 @@ func (m *Persistence) GetByMesh(ctx context.Context, mesh string) (*VirtualOutbo
return NewEmptyVirtualOutboundView(), nil
}

res := map[HostnameEntry]VirtualOutbound{}
if err := json.Unmarshal([]byte(resource.Spec.GetConfig()), &res); err != nil {
return nil, err
var virtualOutboundView *VirtualOutboundMeshView
if m.useTagFirstView {
res := NewEmptyTagFirstOutboundView()
if err := json.Unmarshal([]byte(resource.Spec.GetConfig()), &res); err != nil {
return nil, err
}
virtualOutboundView = res.ToVirtualOutboundView()
} else {
res := map[HostnameEntry]VirtualOutbound{}
if err := json.Unmarshal([]byte(resource.Spec.GetConfig()), &res); err != nil {
return nil, err
}
virtualOutboundView, err = NewVirtualOutboundView(res)
if err != nil {
return nil, err
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we're using TagFirstView shouldn't we fallback to reading the old version for future migration to this format?
What we usually do with these changes is to first ship the flag as disable and 2 versions later we make it enabled by default. This way folks transparently adopted the new format.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually feels like the deserialization should just not take into account this feature flag and just try both (because if you rollback you might have the flag off but the persisted version is the old format).

}
return NewVirtualOutboundView(res)

return virtualOutboundView, nil
}

func (m *Persistence) Set(ctx context.Context, mesh string, vips *VirtualOutboundMeshView) error {
Expand All @@ -78,10 +94,21 @@ func (m *Persistence) Set(ctx context.Context, mesh string, vips *VirtualOutboun
create = true
}

jsonBytes, err := json.Marshal(vips.byHostname)
if err != nil {
return errors.Wrap(err, "unable to marshall VIP list")
var jsonBytes []byte
var err error
if m.useTagFirstView {
view := NewTagFirstOutboundView(vips)
jsonBytes, err = json.Marshal(view)
if err != nil {
return errors.Wrap(err, "unable to marshall VIP list")
}
} else {
jsonBytes, err = json.Marshal(vips.byHostname)
Automaat marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return errors.Wrap(err, "unable to marshall VIP list")
}
}

resource.Spec.Config = string(jsonBytes)

if create {
Expand Down
6 changes: 3 additions & 3 deletions pkg/dns/vips/persistence_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ var _ = Describe("Meshed Persistence", func() {
Spec: &system_proto.Config{Config: `{"0:backend_3":{"address":"240.0.2.1","outbounds":[{"TagSet":{"kuma.io/service":"backend_3"}}]},"0:frontend_3":{"address":"240.0.2.3","outbounds":[{"TagSet":{"kuma.io/service":"frontend_3"}}]},"1:host.com":{"address":"240.0.1.4","outbounds":[{"TagSet":{"kuma.io/service":"external-host"}}]}}`},
},
},
})
}, false)
})

It("should return vips for mesh", func() {
Expand All @@ -132,7 +132,7 @@ var _ = Describe("Meshed Persistence", func() {
countingCm = &countingConfigManager{
configs: map[string]*system.ConfigResource{},
}
meshedPersistence = vips.NewPersistence(rm, countingCm)
meshedPersistence = vips.NewPersistence(rm, countingCm, false)
})

It("should create a new config", func() {
Expand Down Expand Up @@ -214,7 +214,7 @@ var _ = Describe("Meshed Persistence", func() {
},
},
}
meshedPersistence = vips.NewPersistence(rm, countingCm)
meshedPersistence = vips.NewPersistence(rm, countingCm, false)
})

It("should return meshed vips", func() {
Expand Down
Loading