Skip to content

Commit

Permalink
Add support for a new plugin to manage blueprint routes and configure…
Browse files Browse the repository at this point in the history
… the

manager appropriately. And some more changes as per review comments.
  • Loading branch information
ramr committed Apr 2, 2018
1 parent 336b00e commit 327ba9b
Show file tree
Hide file tree
Showing 8 changed files with 442 additions and 40 deletions.
7 changes: 7 additions & 0 deletions hack/lib/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,13 @@ function os::start::router() {
oc adm router --config="${ADMIN_KUBECONFIG}" --images="${USE_IMAGES}" --service-account=router
fi

# Note that when the haproxy config manager is set based on router type,
# the env entry may need to be always set or removed (if defaulted).
if [[ -n "${ROUTER_HAPROXY_CONFIG_MANAGER:-}" ]]; then
os::log::debug "Changing the router DC to enable the haproxy config manager"
oc set env dc/router -c router ROUTER_HAPROXY_CONFIG_MANAGER=true
fi

# Set the SYN eater to make router reloads more robust
if [[ -n "${DROP_SYN_DURING_RESTART:-}" ]]; then
# Rewrite the DC for the router to add the environment variable into the pod definition
Expand Down
21 changes: 18 additions & 3 deletions pkg/cmd/infra/router/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ type TemplateRouter struct {
}

type TemplateRouterConfigManager struct {
ConfigManagerName string
UseHAProxyConfigManager bool
CommitInterval time.Duration
BlueprintRouteNamespace string
BlueprintRouteLabelSelector string
Expand Down Expand Up @@ -162,7 +162,7 @@ func (o *TemplateRouter) Bind(flag *pflag.FlagSet) {
flag.StringVar(&o.Ciphers, "ciphers", util.Env("ROUTER_CIPHERS", ""), "Specifies the cipher suites to use. You can choose a predefined cipher set ('modern', 'intermediate', or 'old') or specify exact cipher suites by passing a : separated list.")
flag.BoolVar(&o.StrictSNI, "strict-sni", isTrue(util.Env("ROUTER_STRICT_SNI", "")), "Use strict-sni bind processing (do not use default cert).")
flag.StringVar(&o.MetricsType, "metrics-type", util.Env("ROUTER_METRICS_TYPE", ""), "Specifies the type of metrics to gather. Supports 'haproxy'.")
flag.StringVar(&o.ConfigManagerName, "config-manager", util.Env("ROUTER_CONFIG_MANAGER", ""), "Specifies the manager to use for dynamically configuring changes with the underlying router. Supports 'haproxy-manager'.")
flag.BoolVar(&o.UseHAProxyConfigManager, "haproxy-config-manager", isTrue(util.Env("ROUTER_HAPROXY_CONFIG_MANAGER", "")), "Use the the haproxy config manager (and dynamic configuration API) to configure route and endpoint changes. Reduces the number of haproxy reloads needed on configuration changes.")
flag.DurationVar(&o.CommitInterval, "commit-interval", getIntervalFromEnv("COMMIT_INTERVAL", defaultCommitInterval), "Controls how often to commit (to the actual config) all the changes made using the router specific dynamic configuration manager.")
flag.StringVar(&o.BlueprintRouteNamespace, "blueprint-route-namespace", util.Env("ROUTER_BLUEPRINT_ROUTE_NAMESPACE", ""), "Specifies the namespace which contains the routes that serve as blueprints for the dynamic configuration manager.")
flag.StringVar(&o.BlueprintRouteLabelSelector, "blueprint-route-labels", util.Env("ROUTER_BLUEPRINT_ROUTE_LABELS", ""), "A label selector to apply to the routes in the blueprint route namespace. These selected routes will serve as blueprints for the dynamic dynamic configuration manager.")
Expand Down Expand Up @@ -438,7 +438,8 @@ func (o *TemplateRouterOptions) Run() error {
}

var cfgManager templateplugin.ConfigManager
if o.ConfigManagerName == "haproxy-manager" {
var blueprintPlugin router.Plugin
if o.UseHAProxyConfigManager {
blueprintRoutes, err := o.blueprintRoutes(routeclient)
if err != nil {
return err
Expand All @@ -452,6 +453,9 @@ func (o *TemplateRouterOptions) Run() error {
WildcardRoutesAllowed: o.AllowWildcardRoutes,
}
cfgManager = haproxyconfigmanager.NewHAProxyConfigManager(cmopts)
if len(o.BlueprintRouteNamespace) > 0 {
blueprintPlugin = haproxyconfigmanager.NewBlueprintPlugin(cfgManager)
}
}

pluginCfg := templateplugin.TemplatePluginConfig{
Expand Down Expand Up @@ -505,6 +509,17 @@ func (o *TemplateRouterOptions) Run() error {
controller := factory.Create(plugin, false, o.EnableIngress)
controller.Run()

if blueprintPlugin != nil {
// f is like factory but filters the routes based on the
// blueprint route namespace and label selector (if any).
f := o.RouterSelection.NewFactory(routeclient, projectclient.Project().Projects(), kc)
f.LabelSelector = o.BlueprintRouteLabelSelector
f.Namespace = o.BlueprintRouteNamespace
f.ResyncInterval = o.ResyncInterval
c := f.Create(blueprintPlugin, false, false)
c.Run()
}

proc.StartReaper()

select {}
Expand Down
13 changes: 5 additions & 8 deletions pkg/oc/admin/router/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -684,14 +684,11 @@ func RunCmdRouter(f *clientcmd.Factory, cmd *cobra.Command, out, errout io.Write
env["ROUTER_CANONICAL_HOSTNAME"] = cfg.RouterCanonicalHostname
}
// automatically start the internal metrics agent if we are handling a known type
if cfg.Type == "haproxy-router" {
env["ROUTER_CONFIG_MANAGER"] = "haproxy-manager"
if cfg.StatsPort != 0 {
env["ROUTER_LISTEN_ADDR"] = fmt.Sprintf("0.0.0.0:%d", cfg.StatsPort)
env["ROUTER_METRICS_TYPE"] = "haproxy"
env["ROUTER_METRICS_TLS_CERT_FILE"] = "/etc/pki/tls/metrics/tls.crt"
env["ROUTER_METRICS_TLS_KEY_FILE"] = "/etc/pki/tls/metrics/tls.key"
}
if cfg.Type == "haproxy-router" && cfg.StatsPort != 0 {
env["ROUTER_LISTEN_ADDR"] = fmt.Sprintf("0.0.0.0:%d", cfg.StatsPort)
env["ROUTER_METRICS_TYPE"] = "haproxy"
env["ROUTER_METRICS_TLS_CERT_FILE"] = "/etc/pki/tls/metrics/tls.crt"
env["ROUTER_METRICS_TLS_KEY_FILE"] = "/etc/pki/tls/metrics/tls.key"
}
env.Add(secretEnv)
if len(defaultCert) > 0 {
Expand Down
55 changes: 55 additions & 0 deletions pkg/router/template/configmanager/haproxy/blueprint_plugin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package haproxy

import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
kapi "k8s.io/kubernetes/pkg/apis/core"

routeapi "github.com/openshift/origin/pkg/route/apis/route"
templaterouter "github.com/openshift/origin/pkg/router/template"
)

// BlueprintPlugin implements the router.Plugin interface to process routes
// from the blueprint namespace for the associated config manager.
type BlueprintPlugin struct {
manager templaterouter.ConfigManager
}

// NewBlueprintPlugin returns a new blueprint routes plugin.
func NewBlueprintPlugin(cm templaterouter.ConfigManager) *BlueprintPlugin {
return &BlueprintPlugin{manager: cm}
}

// HandleRoute processes watch events on blueprint routes.
func (p *BlueprintPlugin) HandleRoute(eventType watch.EventType, route *routeapi.Route) error {
switch eventType {
case watch.Added, watch.Modified:
p.manager.AddBlueprint(route)
case watch.Deleted:
p.manager.RemoveBlueprint(route)
}

return nil
}

// HandleNode processes watch events on the Node resource.
func (p *BlueprintPlugin) HandleNode(eventType watch.EventType, node *kapi.Node) error {
return nil
}

// HandleEndpoints processes watch events on the Endpoints resource.
func (p *BlueprintPlugin) HandleEndpoints(eventType watch.EventType, endpoints *kapi.Endpoints) error {
return nil
}

// HandleNamespaces processes watch events on namespaces.
func (p *BlueprintPlugin) HandleNamespaces(namespaces sets.String) error {
return nil
}

// Commit commits the changes made to a watched resource.
func (p *BlueprintPlugin) Commit() error {
// Nothing to do as the config manager does an automatic commit when
// any blueprint routes change.
return nil
}
212 changes: 212 additions & 0 deletions pkg/router/template/configmanager/haproxy/blueprint_plugin_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
package haproxy

import (
"fmt"
"testing"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/watch"
kapi "k8s.io/kubernetes/pkg/apis/core"

routeapi "github.com/openshift/origin/pkg/route/apis/route"
templaterouter "github.com/openshift/origin/pkg/router/template"
)

type fakeConfigManager struct {
blueprints map[string]*routeapi.Route
}

func newFakeConfigManager() *fakeConfigManager {
return &fakeConfigManager{
blueprints: make(map[string]*routeapi.Route),
}
}

func (cm *fakeConfigManager) Initialize(router templaterouter.RouterInterface, certPath string) {
}

func (cm *fakeConfigManager) AddBlueprint(route *routeapi.Route) {
cm.blueprints[routeKey(route)] = route
}

func (cm *fakeConfigManager) RemoveBlueprint(route *routeapi.Route) {
delete(cm.blueprints, routeKey(route))
}

func (cm *fakeConfigManager) FindBlueprint(id string) (*routeapi.Route, bool) {
route, ok := cm.blueprints[id]
return route, ok
}

func (cm *fakeConfigManager) Register(id string, route *routeapi.Route) {
}

func (cm *fakeConfigManager) AddRoute(id string, route *routeapi.Route) error {
return nil
}

func (cm *fakeConfigManager) RemoveRoute(id string, route *routeapi.Route) error {
return nil
}

func (cm *fakeConfigManager) ReplaceRouteEndpoints(id string, oldEndpoints, newEndpoints []templaterouter.Endpoint, weight int32) error {
return nil
}

func (cm *fakeConfigManager) RemoveRouteEndpoints(id string, endpoints []templaterouter.Endpoint) error {
return nil
}

func (cm *fakeConfigManager) Notify(event templaterouter.RouterEventType) {
}

func (cm *fakeConfigManager) ServerTemplateName(id string) string {
return "fakeConfigManager"
}

func (cm *fakeConfigManager) ServerTemplateSize(id string) string {
return "1"
}

func (cm *fakeConfigManager) GenerateDynamicServerNames(id string) []string {
return []string{}
}

func routeKey(route *routeapi.Route) string {
return fmt.Sprintf("%s:%s", route.Name, route.Namespace)
}

// TestHandleRoute test route watch events
func TestHandleRoute(t *testing.T) {
original := metav1.Time{Time: time.Now()}

route := &routeapi.Route{
ObjectMeta: metav1.ObjectMeta{
CreationTimestamp: original,
Namespace: "bp",
Name: "chevron",
},
Spec: routeapi.RouteSpec{
Host: "www.blueprints.org",
To: routeapi.RouteTargetReference{
Name: "TestService",
Weight: new(int32),
},
},
}

cm := newFakeConfigManager()
plugin := NewBlueprintPlugin(cm)
plugin.HandleRoute(watch.Added, route)

id := routeKey(route)
if _, ok := cm.FindBlueprint(id); !ok {
t.Errorf("TestHandleRoute was unable to find a blueprint %s after HandleRoute was called", id)
}

// update a blueprint with a newer time and host
v2route := route.DeepCopy()
v2route.CreationTimestamp = metav1.Time{Time: original.Add(time.Hour)}
v2route.Spec.Host = "updated.blueprint.org"
if err := plugin.HandleRoute(watch.Added, v2route); err != nil {
t.Errorf("TestHandleRoute unexpected error after blueprint update: %v", err)
}

blueprints := []*routeapi.Route{v2route, route}
for _, r := range blueprints {
// delete the blueprint and check that it doesn't exist.
if err := plugin.HandleRoute(watch.Deleted, v2route); err != nil {
t.Errorf("TestHandleRoute unexpected error after blueprint delete: %v", err)
}

routeId := routeKey(r)
if _, ok := cm.FindBlueprint(routeId); ok {
t.Errorf("TestHandleRoute found a blueprint %s after it was deleted", routeId)
}
}
}

func TestHandleNode(t *testing.T) {
node := &kapi.Node{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"design": "blueprint"},
},
}

cm := newFakeConfigManager()
plugin := NewBlueprintPlugin(cm)

if err := plugin.HandleNode(watch.Added, node); err != nil {
t.Errorf("TestHandleNode unexpected error after node add: %v", err)
}

if err := plugin.HandleNode(watch.Modified, node); err != nil {
t.Errorf("TestHandleNode unexpected error after node modify: %v", err)
}

if err := plugin.HandleNode(watch.Deleted, node); err != nil {
t.Errorf("TestHandleNode unexpected error after node delete: %v", err)
}
}

func TestHandleEndpoints(t *testing.T) {
endpoints := &kapi.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Namespace: "bpe",
Name: "shell",
},
Subsets: []kapi.EndpointSubset{{
Addresses: []kapi.EndpointAddress{{IP: "1.1.1.1"}},
Ports: []kapi.EndpointPort{{Port: 9876}},
}},
}

v2Endpoints := &kapi.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Namespace: "bpe",
Name: "shell",
},
Subsets: []kapi.EndpointSubset{{
Addresses: []kapi.EndpointAddress{{IP: "1.1.1.1"}, {IP: "2.2.2.2"}},
Ports: []kapi.EndpointPort{{Port: 9876}, {Port: 8888}},
}},
}

cm := newFakeConfigManager()
plugin := NewBlueprintPlugin(cm)

if err := plugin.HandleEndpoints(watch.Added, endpoints); err != nil {
t.Errorf("TestHandleEndpoints unexpected error after endpoints add: %v", err)
}

if err := plugin.HandleEndpoints(watch.Modified, v2Endpoints); err != nil {
t.Errorf("TestHandleEndpoints unexpected error after endpoints modify: %v", err)
}

if err := plugin.HandleEndpoints(watch.Deleted, v2Endpoints); err != nil {
t.Errorf("TestHandleEndpoints unexpected error after endpoints delete: %v", err)
}
}

func TestHandleNamespaces(t *testing.T) {
cm := newFakeConfigManager()
plugin := NewBlueprintPlugin(cm)

if err := plugin.HandleNamespaces(sets.String{}); err != nil {
t.Errorf("TestHandleNamespaces unexpected error after empty set: %v", err)
}

if err := plugin.HandleNamespaces(sets.NewString("76")); err != nil {
t.Errorf("TestHandleNamespaces unexpected error after set: %v", err)
}

if err := plugin.HandleNamespaces(sets.NewString("76", "711")); err != nil {
t.Errorf("TestHandleNamespaces unexpected error after set multiple: %v", err)
}

if err := plugin.HandleNamespaces(sets.NewString("arco")); err != nil {
t.Errorf("TestHandleNamespaces unexpected error after reset: %v", err)
}
}
Loading

0 comments on commit 327ba9b

Please sign in to comment.