diff --git a/bcs-k8s/bcs-apiserver-proxy/cmd/app.go b/bcs-k8s/bcs-apiserver-proxy/cmd/app.go index 29621b0b87..14b26cacec 100644 --- a/bcs-k8s/bcs-apiserver-proxy/cmd/app.go +++ b/bcs-k8s/bcs-apiserver-proxy/cmd/app.go @@ -17,6 +17,9 @@ import ( "context" "errors" "fmt" + "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/health" + ipvsConfig "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/ipvs/config" + "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/utils" "net/http" "net/http/pprof" "os" @@ -137,71 +140,69 @@ func (pm *ProxyManager) Run() error { blog.Errorf("checkVirtualServerAndCreateVsWhenNotExist failed: %v", err) return } - - adds, deletes, err := pm.getAddOrDeleteRealServers() + err = pm.syncAddLvsRealServers() if err != nil { - blog.Errorf("getAddOrDeleteRealServers failed: %v", err) + blog.Errorf("add lvs real servers failed: %v", err) return } - if len(adds) == 0 && len(deletes) == 0 { - blog.Infof("cluster master endpointIPs equal lvs backend realServers, no need to sync") + err = pm.syncDeleteLvsRealServers() + if err != nil { + blog.Errorf("delete lvs real servers failed: %v", err) + return + } + err = pm.persistLvsConfig() + if err != nil { + blog.Errorf("persist lvs config failed: %v", err) return } - - pm.syncLvsRealServers(adds, deletes) }() } } -func (pm *ProxyManager) syncLvsRealServers(adds, deletes sets.String) error { +func (pm *ProxyManager) syncAddLvsRealServers() error { if pm == nil { return ErrProxyManagerNotInited } - blog.V(5).Infof("syncLvsRealServers, adds: [%v] deletes: [%v]", adds, deletes) + adds, err := pm.getAddRealServers() + if err != nil { + blog.Errorf("getAddRealServers failed: %v", err) + return err + } + if len(adds) == 0 { + return nil + } + + blog.V(5).Infof("syncAddLvsRealServers, adds: [%v]", adds) if len(adds) > 0 { for s := range adds { err := pm.lvsProxy.CreateRealServer(s) if err != nil { - blog.Errorf("syncLvsRealServers CreateRealServer[%s] failed: %v", s, err) - continue - } - - blog.Infof("syncLvsRealServers CreateRealServer[%s] successful", s) - } - } - - if len(deletes) > 0 { - for s := range deletes { - err := pm.lvsProxy.DeleteRealServer(s) - if err != nil { - blog.Errorf("syncLvsRealServers DeleteRealServer[%s] failed: %v", s, err) + blog.Errorf("syncAddLvsRealServers CreateRealServer[%s] failed: %v", s, err) continue } - blog.Infof("syncLvsRealServers DeleteRealServer[%s] successful", s) + blog.Infof("syncAddLvsRealServers CreateRealServer[%s] successful", s) } } - blog.V(5).Infof("syncLvsRealServers, adds: [%v] deletes: [%v] successful", adds, deletes) + blog.V(5).Infof("syncAddLvsRealServers, adds: [%v] successful", adds) return nil } -func (pm *ProxyManager) getAddOrDeleteRealServers() (sets.String, sets.String, error) { +func (pm *ProxyManager) getAddRealServers() (sets.String, error) { if pm == nil { - return nil, nil, ErrProxyManagerNotInited + return nil, ErrProxyManagerNotInited } - var ( - addServers, deleteServers sets.String - ) + var addServers sets.String // get cluster master endpoint IPs clusterEndpoints, err := pm.clusterEndpointsIP.GetClusterEndpoints() if err != nil { - return nil, nil, err + return nil, err } clusterRs := []string{} for _, ep := range clusterEndpoints { @@ -212,15 +213,63 @@ func (pm *ProxyManager) getAddOrDeleteRealServers() (sets.String, sets.String, e // get proxy lvs endpoint real server proxyRs, err := pm.lvsProxy.ListRealServer() if err != nil { - return nil, nil, err + return nil, err } proxyRsMap := sets.NewString(proxyRs...) // diff get add & delete server addServers = clusterRsMap.Difference(proxyRsMap) - deleteServers = proxyRsMap.Difference(clusterRsMap) - return addServers, deleteServers, nil + return addServers, nil +} + +func (pm *ProxyManager) syncDeleteLvsRealServers() error { + if pm == nil { + return ErrProxyManagerNotInited + } + healthCheck, err := health.NewHealthConfig(pm.options.HealthCheck.HealthScheme, pm.options.HealthCheck.HealthPath) + rsList, err := pm.lvsProxy.ListRealServer() + if err != nil { + return err + } + for _, rs := range rsList { + ip, port := utils.SplitServer(rs) + if healthCheck.IsHTTPAPIHealth(ip, port) { + continue + } + err := pm.lvsProxy.DeleteRealServer(rs) + if err != nil { + return err + } + blog.Infof("syncDeleteLvsRealServers delete real server [%s] successful", rs) + } + return nil +} + +func (pm *ProxyManager) persistLvsConfig() error { + vs, err := pm.lvsProxy.GetVirtualServer() + if err != nil { + return err + } + rsList, err := pm.lvsProxy.ListRealServer() + if err != nil { + return err + } + scheduler, err := pm.lvsProxy.GetScheduler() + if err != nil { + return err + } + c := ipvsConfig.IpvsConfig{ + Scheduler: scheduler, + VirtualServer: vs, + RealServer: rsList, + } + err = ipvsConfig.WriteIpvsConfig(pm.options.PersistConfig.IpvsPersistDir, c) + if err != nil { + return nil + } + + return nil } func (pm *ProxyManager) initProxyOptions(options *config.ProxyAPIServerOptions) { @@ -252,7 +301,7 @@ func (pm *ProxyManager) initLvsProxy() error { return ErrProxyManagerNotInited } - lvsProxy := service.NewLvsProxy() + lvsProxy := service.NewLvsProxy(pm.options.ProxyLvs.Scheduler) pm.lvsProxy = lvsProxy // exist lvs @@ -302,17 +351,12 @@ func (pm *ProxyManager) initClusterEndpointsClient() error { KubeConfig: pm.options.K8sConfig.KubeConfig, })) - if pm.options.SystemInterval.EndpointInterval > 0 { - opts = append(opts, endpoint.WithInterval(time.Second*time.Duration(pm.options.SystemInterval.EndpointInterval))) - } - endpointClient, err := endpoint.NewEndpointsClient(opts...) if err != nil { return err } pm.clusterEndpointsIP = endpointClient - go pm.clusterEndpointsIP.SyncClusterEndpoints(pm.ctx) return nil } @@ -404,7 +448,6 @@ func (pm *ProxyManager) close() { return } - pm.clusterEndpointsIP.Stop() pm.lvsProxy.DeleteVirtualServer(pm.options.ProxyLvs.VirtualAddress) pm.cancel() } diff --git a/bcs-k8s/bcs-apiserver-proxy/cmd/config/options.go b/bcs-k8s/bcs-apiserver-proxy/cmd/config/options.go index 7e6e30c415..d4f14d2ee8 100644 --- a/bcs-k8s/bcs-apiserver-proxy/cmd/config/options.go +++ b/bcs-k8s/bcs-apiserver-proxy/cmd/config/options.go @@ -27,6 +27,7 @@ type ProxyAPIServerOptions struct { DebugMode bool `json:"debugMode"` ProxyLvs ProxyLvs `json:"proxyLvs"` + PersistConfig PersistConfig `json:"persistConfig"` HealthCheck HealthCheckOptions `json:"healthCheck"` K8sConfig K8sConfig `json:"k8sConfig"` SystemInterval SystemInterval `json:"systemInterval"` @@ -46,6 +47,11 @@ func (opt ProxyAPIServerOptions) Validate() bool { // ProxyLvs virtual server type ProxyLvs struct { VirtualAddress string `json:"virtualAddress" value:"127.0.0.1:6443" usage:"Proxy lvs address:port"` + Scheduler string `json:"lvsScheduler" value:"sh" usage:"one of rr|wrr|lc|wlc|lblc|lblcr|dh|sh|sed|nq"` +} + +type PersistConfig struct { + IpvsPersistDir string `json:"ipvsPersistDir" value:"/root/.bcs" usage:"persist ipvs rules"` } // HealthCheckOptions health check scheme&path @@ -62,7 +68,6 @@ type K8sConfig struct { // SystemInterval ticker interval type SystemInterval struct { - EndpointInterval int64 `json:"endpointInterval" value:"5" usage:"dynamic update cluster endpointsIP interval"` ManagerInterval int64 `json:"managerInterval" value:"10" usage:"dynamic refresh ipvs rules interval"` } diff --git a/bcs-k8s/bcs-apiserver-proxy/go.mod b/bcs-k8s/bcs-apiserver-proxy/go.mod index 69c97eb8ea..dfa9821f72 100644 --- a/bcs-k8s/bcs-apiserver-proxy/go.mod +++ b/bcs-k8s/bcs-apiserver-proxy/go.mod @@ -9,14 +9,12 @@ replace ( require ( github.com/Tencent/bk-bcs/bcs-common v0.0.0-20210621082136-e7b1aa4848c4 - github.com/google/go-cmp v0.5.5 // indirect github.com/gorilla/mux v1.8.0 github.com/lithammer/dedent v1.1.0 github.com/moby/ipvs v1.0.1 github.com/prometheus/client_golang v1.9.0 + github.com/spf13/viper v1.8.1 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/api v0.18.6 k8s.io/apimachinery v0.18.6 k8s.io/client-go v0.18.6 diff --git a/bcs-k8s/bcs-apiserver-proxy/ipvs_tools/main.go b/bcs-k8s/bcs-apiserver-proxy/ipvs_tools/main.go index a39d1288e3..373a9363a9 100644 --- a/bcs-k8s/bcs-apiserver-proxy/ipvs_tools/main.go +++ b/bcs-k8s/bcs-apiserver-proxy/ipvs_tools/main.go @@ -16,9 +16,46 @@ package main import ( "flag" "fmt" + ipvsConfig "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/ipvs/config" + "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/utils" "log" + "os" ) +const ( + // Init command + Init Operation = "init" + // Reload command + Reload Operation = "reload" + // Add command + Add Operation = "add" + // Delete command + Delete Operation = "delete" +) + +// Operation for operation command +type Operation string + +func (o Operation) validate() bool { + return o == Init || o == Add || o == Delete || o == Reload +} + +func (o Operation) isInitCommand() bool { + return o == Init +} + +func (o Operation) isReloadCommand() bool { + return o == Reload +} + +func (o Operation) isAddCommand() bool { + return o == Add +} + +func (o Operation) isDeleteCommand() bool { + return o == Delete +} + type sliceString []string // String xxx @@ -33,51 +70,139 @@ func (f *sliceString) Set(value string) error { } type options struct { - command string - virtualServer string - realServer sliceString + command string + virtualServer string + realServer sliceString + scheduler string + ipvsPersistDir string + toolPath string + healthScheme string + healthPath string } var opts options func main() { - care, err := NewLvsCare(opts) + operation := Operation(opts.command) + switch operation{ + case Init: + initFunc() + case Reload: + reloadFunc() + case Add: + addFunc() + case Delete: + deleteFunc() + default: + log.Printf("invalid operation command") + } + + return +} + +func initFunc() { + if !validateInitOptions(opts) { + log.Println("validate options failed, check your options") + return + } + care, err := NewLvsCareFromFlag(opts) + if err != nil { + log.Printf("create lvsCare failed: %v", err) + } + err = care.CreateVirtualService() + if err != nil { + log.Printf("lvs[%s] init real servers %v failed: %v", opts.virtualServer, opts.realServer, err) + return + } + scheduler, err := care.lvs.GetScheduler() + vs, err := care.lvs.GetVirtualServer() + rs, err := care.lvs.ListRealServer() + if err != nil { + log.Println("init failed") + } + config := ipvsConfig.IpvsConfig{ + Scheduler: scheduler, + VirtualServer: vs, + RealServer: rs, + } + err = ipvsConfig.WriteIpvsConfig(opts.ipvsPersistDir, config) + if err != nil { + return + } + err = utils.SetIpvsStartup(opts.ipvsPersistDir, opts.toolPath) if err != nil { - log.Printf("NewLvsCare failed: %v", err) + log.Println("set ipvs startup failed") return } + log.Printf("lvs[%s] init real servers %v successful", opts.virtualServer, opts.realServer) + return +} - switch care.GetLvsCommand() { - case Add: - err := care.CreateVirtualService() - if err != nil { - log.Printf("lvs[%s] add real servers %v failed: %v", opts.virtualServer, opts.realServer, err) - return - } +func reloadFunc() { + care, err := NewLvsCareFromConfig(opts) + if err != nil { + log.Printf("reload ipvs failed: %v", err) + } + err = care.CreateVirtualService() +} - log.Printf("lvs[%s] add real servers %v successful", opts.virtualServer, opts.realServer) +func addFunc() { + care, err := NewLvsCareFromFlag(opts) + if err != nil { + log.Printf("create lvsCare failed: %v", err) + } + err = care.CreateVirtualService() + if err != nil { + log.Printf("lvs[%s] add real servers %v failed: %v", opts.virtualServer, opts.realServer, err) return - case Delete: - err := care.DeleteVirtualService() - if err != nil { - log.Printf("lvs[%s] delete failed: %v", opts.virtualServer, err) - return - } + } + + log.Printf("lvs[%s] add real servers %v successful", opts.virtualServer, opts.realServer) + return +} - log.Printf("lvs[%s] delete successful", opts.virtualServer) +func deleteFunc() { + care, err := NewLvsCareFromFlag(opts) + if err != nil { + log.Printf("create lvsCare failed: %v", err) + } + err = care.DeleteVirtualService() + if err != nil { + log.Printf("lvs[%s] delete failed: %v", opts.virtualServer, err) return - default: - log.Printf("invalid operation command, please input add or delete") } + log.Printf("lvs[%s] delete successful", opts.virtualServer) return } +func validateInitOptions(opt options) bool { + tool, err := os.Stat(opt.toolPath) + if err != nil { + if os.IsNotExist(err) { + log.Println("error path, please set the valid absolute path for apiserver-proxy-tools") + return false + }else { + log.Println("error path, please set the valid absolute path for apiserver-proxy-tools") + } + } + if tool.IsDir() { + log.Println("error path, please set the valid absolute path for apiserver-proxy-tools") + return false + } + return true +} + func init() { - flag.StringVar(&opts.command, "cmd", "", "virtual server add or delete") + flag.StringVar(&opts.command, "cmd", "", "one of init|reload|add|delete") flag.StringVar(&opts.virtualServer, "vs", "127.0.0.1:6443", "virtual server") + flag.StringVar(&opts.scheduler, "scheduler", "sh", "lvs scheduler, one of rr|wrr|lc|wlc|lblc|lblcr|dh|sh|sed|nq") flag.Var(&opts.realServer, "rs", "virtual server backend real server, for example: "+ "-rs=127.0.0.1:6443 -rs=127.0.0.2:6443") + flag.StringVar(&opts.ipvsPersistDir, "persistDir", "/root/.bcs", "persistent ipvs rules path") + flag.StringVar(&opts.toolPath, "toolPath", "/root/apiserver-proxy-tools", "absolute path for apiserver-proxy-tools") + flag.StringVar(&opts.healthScheme, "healthScheme", "https", "scheme for health check") + flag.StringVar(&opts.healthPath, "healthPath", "/healthz", "path for health check") flag.Parse() } diff --git a/bcs-k8s/bcs-apiserver-proxy/ipvs_tools/service.go b/bcs-k8s/bcs-apiserver-proxy/ipvs_tools/service.go index 7bd3718581..70c2a36a3b 100644 --- a/bcs-k8s/bcs-apiserver-proxy/ipvs_tools/service.go +++ b/bcs-k8s/bcs-apiserver-proxy/ipvs_tools/service.go @@ -16,6 +16,9 @@ package main import ( "errors" "fmt" + "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/health" + ipvsConfig "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/ipvs/config" + "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/utils" "log" "strings" @@ -25,93 +28,59 @@ import ( var ( // ErrLvsCareNotInited for lvsCare not inited ErrLvsCareNotInited = errors.New("LvsCare not inited") - // ErrNotValidateOperation for invalid command - ErrNotValidateOperation = errors.New("invalid operation command") ) -const ( - // Add command - Add Operation = "add" - // Delete command - Delete Operation = "delete" - // Invalid command - Invalid Operation = "invalid operation" -) - -// Operation for operation command -type Operation string +type Scheduler string -func (o Operation) validate() bool { - return o == Add || o == Delete +func (s Scheduler) validate() bool { + return s == "rr" || s == "wrr" || s == "lc" || s == "wlc" || + s == "lblc" || s == "lblcr" || s == "dh" || s == "sh" || + s == "sed" || s == "nq" } -func (o Operation) isAddCommand() bool { - return o == Add -} - -func (o Operation) isDeleteCommand() bool { - return o == Delete -} - -// NewLvsCare init lvsCare client -func NewLvsCare(opts options) (*LvsCare, error) { +// NewLvsCareFromFlag init lvsCare client +func NewLvsCareFromFlag(opts options) (*LvsCare, error) { care := &LvsCare{ - command: Operation(opts.command), virtualServer: opts.virtualServer, realServer: opts.realServer, - lvs: service.NewLvsProxy(), + lvs: service.NewLvsProxy(opts.scheduler), } - ok := care.validate() - if !ok { - infoMsg := fmt.Errorf("LvsCare validate failed") + schedulerOk := Scheduler(opts.scheduler).validate() + if !schedulerOk { + infoMsg := fmt.Errorf("LvsCare validate failed, invalid scheduler") return nil, infoMsg } return care, nil } -// LvsCare for create or delete vs -type LvsCare struct { - command Operation - virtualServer string - realServer []string - lvs service.LvsProxy -} - -func (lvs *LvsCare) validate() bool { - if lvs == nil { - return false - } - - ok := lvs.command.validate() - if !ok { - log.Println("Command operation only support: add or delete virtual service operation") - return false +// NewLvsCareFromConfig init lvsCare client +func NewLvsCareFromConfig(opts options) (*LvsCare, error) { + config, err := ipvsConfig.ReadIpvsConfig(opts.ipvsPersistDir) + if err != nil { + log.Printf("read ipvs config failed: %v", err) + return nil, nil } - - if len(lvs.virtualServer) == 0 { - log.Println("virtual server is empty") - return false + care := &LvsCare{ + virtualServer: config.VirtualServer, + realServer: config.RealServer, + lvs: service.NewLvsProxy(opts.scheduler), } - - if lvs.command.isAddCommand() { - if len(lvs.realServer) == 0 { - log.Println("real servers is empty") - return false - } + schedulerOk := Scheduler(opts.scheduler).validate() + if !schedulerOk { + infoMsg := fmt.Errorf("LvsCare validate failed, invalid scheduler") + return nil, infoMsg } - - return true + return care, nil } -// GetLvsCommand get operation command -func (lvs *LvsCare) GetLvsCommand() Operation { - if lvs == nil { - return Invalid - } - - return lvs.command +// LvsCare for create or delete vs +type LvsCare struct { + virtualServer string + realServer []string + scheduler Scheduler + lvs service.LvsProxy } // CreateVirtualService create vs @@ -132,7 +101,16 @@ func (lvs *LvsCare) CreateVirtualService() error { } for _, r := range lvs.realServer { - err := lvs.lvs.CreateRealServer(r) + healthCheck, err := health.NewHealthConfig(opts.healthScheme, opts.healthPath) + if err != nil { + log.Printf("build health check client faild: %v", err) + } + ip, port := utils.SplitServer(r) + if !healthCheck.IsHTTPAPIHealth(ip, port) { + log.Printf("create rs [%s] failed, it is not health", r) + continue + } + err = lvs.lvs.CreateRealServer(r) if err != nil { errs = append(errs, fmt.Sprintf("CreateRealServer[%s/%s] failed: %v", lvs.virtualServer, r, err)) } diff --git a/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/endpoint.go b/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/endpoint.go index 99a948ae2b..60708670d8 100644 --- a/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/endpoint.go +++ b/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/endpoint.go @@ -17,6 +17,7 @@ import ( "context" "errors" "fmt" + "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/health" "sync" "time" @@ -48,7 +49,7 @@ type EndpointsHealthOptions struct { } var ( - defaultHealthScheme = schemeHTTPS + defaultHealthScheme = "https" defaultHealthPath = "/healthz" defaultInterval = time.Second * 3 @@ -57,8 +58,6 @@ var ( // ClusterEndpointsIP is a interface for sync kubernetes master endpointIPs type ClusterEndpointsIP interface { GetClusterEndpoints() ([]utils.EndPoint, error) - SyncClusterEndpoints(ctx context.Context) - Stop() } // NewEndpointsClient init endpoints client @@ -153,83 +152,18 @@ type endpointsClient struct { cancel context.CancelFunc } -// SyncClusterEndpoints sync cluster master endpointIPs -func (ec *endpointsClient) SyncClusterEndpoints(ctx context.Context) { - if ec == nil { - blog.Errorf("SyncClusterEndpoints failed; %v", ErrEndpointsClientNotInited) - return - } - - select { - case <-ctx.Done(): - blog.Errorf("external context quit: %v", ctx.Err()) - return - case <-ec.ctx.Done(): - blog.Errorf("ec context quit: %v", ec.ctx.Err()) - return - default: - } - - quitCtx, cancel := context.WithCancel(ctx) - defer cancel() - - go func() { - select { - case <-quitCtx.Done(): - blog.Errorf("external context quit: %v", quitCtx.Err()) - case <-ec.ctx.Done(): - blog.Errorf("EndpointsClient context quit: %v", ec.ctx.Err()) - cancel() - } - }() - - coldStart := make(chan struct{}, 1) - coldStart <- struct{}{} - - ticker := time.NewTicker(ec.interval) - defer ticker.Stop() - - for { - select { - case <-quitCtx.Done(): - blog.Infof("EndpointsClient quit: %v", quitCtx.Err()) - return - case <-coldStart: - case <-ticker.C: - } - - func() { - defer func() { - if r := recover(); r != nil { - blog.Errorf("EndpointsClient panic: %v", r) - } - }() - - // get apiServer Endpoints - clusterEndpoints, err := ec.getAPIServerEndpoints() - if err != nil { - blog.Errorf("getAPIServerEndpoints failed: %v", err) - return - } - - ec.Mutex.Lock() - ec.masterEndpoints = clusterEndpoints - ec.Mutex.Unlock() - }() - } - -} - // GetClusterEndpoints get cluster endpointIPs func (ec *endpointsClient) GetClusterEndpoints() ([]utils.EndPoint, error) { if ec == nil { return nil, ErrEndpointsClientNotInited } - ec.Mutex.Lock() - clusterEndpoints := ec.masterEndpoints - ec.Mutex.Unlock() - + // get apiServer Endpoints + clusterEndpoints, err := ec.getAPIServerEndpoints() + if err != nil { + blog.Errorf("getAPIServerEndpoints failed: %v", err) + return nil, err + } return clusterEndpoints, nil } @@ -278,7 +212,7 @@ func (ec *endpointsClient) getAPIServerEndpoints() ([]utils.EndPoint, error) { ) // healthCheck client - healthCheck, err := NewHealthConfig(ec.healthOptions.Scheme, ec.healthOptions.Path) + healthCheck, err := health.NewHealthConfig(ec.healthOptions.Scheme, ec.healthOptions.Path) if err != nil { blog.Errorf("NewHealthConfig failed: %v", err) return nil, err diff --git a/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/endpoint_test.go b/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/endpoint_test.go index 6b23d10a77..4e1be76a07 100644 --- a/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/endpoint_test.go +++ b/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/endpoint_test.go @@ -14,9 +14,7 @@ package endpoint import ( - "context" "testing" - "time" ) func getEndpointsClient() ClusterEndpointsIP { @@ -34,8 +32,6 @@ func getEndpointsClient() ClusterEndpointsIP { } func TestEndpoints_GetClusterEndpoints(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() client := getEndpointsClient() if client == nil { @@ -45,23 +41,10 @@ func TestEndpoints_GetClusterEndpoints(t *testing.T) { t.Logf("%+v", client) - go client.SyncClusterEndpoints(ctx) - - for { - select { - case <-ctx.Done(): - client.Stop() - t.Logf("SyncClusterEndpoints quit: %v", ctx.Err()) - return - case <-time.After(time.Second * 5): - } - - endpoints, err := client.GetClusterEndpoints() - if err != nil { - t.Fatalf("GetClusterEndpoints failed: %v", err) - return - } - - t.Logf("GetClusterEndpoints %+v", endpoints) + endpoints, err := client.GetClusterEndpoints() + if err != nil { + t.Fatalf("GetClusterEndpoints failed: %v", err) + return } + t.Logf("GetClusterEndpoints %+v", endpoints) } diff --git a/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/health_check.go b/bcs-k8s/bcs-apiserver-proxy/pkg/health/health_check.go similarity index 97% rename from bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/health_check.go rename to bcs-k8s/bcs-apiserver-proxy/pkg/health/health_check.go index 7d3d93f8c2..387c36b97f 100644 --- a/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/health_check.go +++ b/bcs-k8s/bcs-apiserver-proxy/pkg/health/health_check.go @@ -11,7 +11,7 @@ * */ -package endpoint +package health import ( "crypto/tls" @@ -34,6 +34,12 @@ type HealthCheck interface { IsHTTPAPIHealth(addr string, port uint32) bool } +const ( + schemeHTTPS = "https" + schemeHTTP = "http" + +) + func validateScheme(scheme string) error { if scheme != schemeHTTPS && scheme != schemeHTTP { return ErrSchemeInValid diff --git a/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/health_check_test.go b/bcs-k8s/bcs-apiserver-proxy/pkg/health/health_check_test.go similarity index 98% rename from bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/health_check_test.go rename to bcs-k8s/bcs-apiserver-proxy/pkg/health/health_check_test.go index 4ceb6a688c..18eb534bcb 100644 --- a/bcs-k8s/bcs-apiserver-proxy/pkg/endpoint/health_check_test.go +++ b/bcs-k8s/bcs-apiserver-proxy/pkg/health/health_check_test.go @@ -11,7 +11,7 @@ * */ -package endpoint +package health import ( "fmt" diff --git a/bcs-k8s/bcs-apiserver-proxy/pkg/ipvs/config/config.go b/bcs-k8s/bcs-apiserver-proxy/pkg/ipvs/config/config.go new file mode 100644 index 0000000000..57d49a59a6 --- /dev/null +++ b/bcs-k8s/bcs-apiserver-proxy/pkg/ipvs/config/config.go @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making Blueking Container Service available. + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package config + +import ( + "fmt" + "github.com/spf13/viper" + "os" + "path" +) + +const ( + IpvsConfigFileName = "ipvsConfig.yaml" +) + +type IpvsConfig struct { + Scheduler string `json:"scheduler"` + VirtualServer string `json:"vs"` + RealServer []string `json:"rs"` +} + +func WriteIpvsConfig(dir string, config IpvsConfig) error { + _, exist := os.Stat(dir) + if os.IsNotExist(exist) { + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + fmt.Println("create ipvs persist dir failed") + return err + } + } + viper.SetConfigFile(path.Join(dir, IpvsConfigFileName)) + viper.Set("vs", config.VirtualServer) + viper.Set("rs", config.RealServer) + viper.Set("scheduler", config.Scheduler) + err := viper.WriteConfigAs(path.Join(dir,IpvsConfigFileName)) + if err != nil { + fmt.Println("persist ipvs config to file failed") + return err + } + return nil +} + +func ReadIpvsConfig(dir string) (*IpvsConfig, error) { + _, exist := os.Stat(dir) + if os.IsNotExist(exist) { + err := fmt.Errorf("ipvs config persist dir [%s] not exists", dir) + return nil, err + } + viper.SetConfigFile(path.Join(dir, IpvsConfigFileName)) + err := viper.ReadInConfig() + if err != nil { + fmt.Printf("read persist config failed, %v", err) + return nil, err + } + config := &IpvsConfig{ + VirtualServer: viper.GetString("vs"), + RealServer: viper.GetStringSlice("rs"), + Scheduler: viper.GetString("scheduler"), + } + return config, nil +} diff --git a/bcs-k8s/bcs-apiserver-proxy/pkg/service/service.go b/bcs-k8s/bcs-apiserver-proxy/pkg/service/service.go index c5abb12807..8c0133e0a6 100644 --- a/bcs-k8s/bcs-apiserver-proxy/pkg/service/service.go +++ b/bcs-k8s/bcs-apiserver-proxy/pkg/service/service.go @@ -25,10 +25,17 @@ import ( "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/utils" ) +var ( + // ErrEndpointsClientNotInited show endpointsClient not inited + ErrLvsServiceNotInited = errors.New("lvsService not inited") +) + //LvsProxy is lvs virtualServer and realServer operation interface type LvsProxy interface { // CreateVirtualServer create the specified VirtualServer by vs CreateVirtualServer(vs string) error + // GetVirtualServer get virtual server by lvs + GetVirtualServer() (string, error) // IsVirtualServerAvailable check vs available or not IsVirtualServerAvailable(vs string) bool // DeleteVirtualServer delete vs form host @@ -39,21 +46,24 @@ type LvsProxy interface { ListRealServer() ([]string, error) // DeleteRealServer delete real server DeleteRealServer(rs string) error + // GetScheduler get lvs scheduler + GetScheduler() (string, error) } // NewLvsProxy init LvsProxy interface -func NewLvsProxy() LvsProxy { +func NewLvsProxy(scheduler string) LvsProxy { l := &lvsProxy{} l.handle = ipvs.New() - + l.scheduler = scheduler return l } type lvsProxy struct { - vs *utils.EndPoint - lock sync.Mutex - rs []*utils.EndPoint - handle ipvs.Interface + vs *utils.EndPoint + scheduler string + lock sync.Mutex + rs []*utils.EndPoint + handle ipvs.Interface } // CreateVirtualServer create virtual server and set lvsProxy.vs by vs, return err when create fails @@ -66,7 +76,7 @@ func (l *lvsProxy) CreateVirtualServer(vs string) error { // set virtual server l.vs = &utils.EndPoint{IP: virIP, Port: virPort} - vServer := utils.BuildVirtualServer(vs) + vServer := utils.BuildVirtualServer(vs, l.scheduler) err := l.handle.AddVirtualServer(vServer) if errors.Is(err, syscall.EEXIST) { blog.Debug("CreateRealServer exist: ", err) @@ -80,6 +90,13 @@ func (l *lvsProxy) CreateVirtualServer(vs string) error { return nil } +func (l *lvsProxy) GetVirtualServer() (string, error) { + if l.vs == nil || l.vs.String() == "" { + return "", ErrLvsServiceNotInited + } + return l.vs.String(), nil +} + // DeleteVirtualServer delete virtual server if exist func (l *lvsProxy) DeleteVirtualServer(vs string) error { vIP, vPort := utils.SplitServer(vs) @@ -87,7 +104,7 @@ func (l *lvsProxy) DeleteVirtualServer(vs string) error { blog.Error("DeleteVirtualServer error: real server ip and port is empty ") return fmt.Errorf("virtual server ip and port is null") } - virServer := utils.BuildVirtualServer(vs) + virServer := utils.BuildVirtualServer(vs, l.scheduler) err := l.handle.DeleteVirtualServer(virServer) if err != nil { blog.Warn("DeleteVirtualServer error: ", err) @@ -115,9 +132,9 @@ func (l *lvsProxy) IsVirtualServerAvailable(vs string) bool { return isExist } - resultVirServer := utils.BuildVirtualServer(vs) + resultVirServer := utils.BuildVirtualServer(vs, l.scheduler) for _, vir := range virArray { - blog.Infof("IsVirtualServerAvailable debug: check vir ip: %s, port %v ", vir.Address.String(), vir.Port) + blog.Debug("IsVirtualServerAvailable debug: check vir ip: %s, port %v ", vir.Address.String(), vir.Port) if vir.String() == resultVirServer.String() { isExist = true } @@ -155,7 +172,7 @@ func (l *lvsProxy) CreateRealServer(rs string) error { } // virtual server build rs server - vServer := utils.BuildVirtualServer(l.vs.String()) + vServer := utils.BuildVirtualServer(l.vs.String(), l.scheduler) err := l.handle.AddRealServer(vServer, realServer) if errors.Is(err, syscall.EEXIST) { blog.Debug("CreateRealServer exist: ", err) @@ -175,7 +192,7 @@ func (l *lvsProxy) ListRealServer() ([]string, error) { return nil, fmt.Errorf("ListRealServer failed, lvsProxy l.vs is empty") } - vs := utils.BuildVirtualServer(l.vs.String()) + vs := utils.BuildVirtualServer(l.vs.String(), l.scheduler) dstArray, err := l.handle.GetRealServers(vs) if err != nil { blog.Errorf("GetRealServers failed: %s; %v ", vs, err) @@ -197,7 +214,7 @@ func (l *lvsProxy) GetRealServer(rsHost string) (*utils.EndPoint, int) { ip, port := utils.SplitServer(rsHost) // get virtual server backend rs - vs := utils.BuildVirtualServer(l.vs.String()) + vs := utils.BuildVirtualServer(l.vs.String(), l.scheduler) dstArray, err := l.handle.GetRealServers(vs) if err != nil { blog.Error("GetRealServer error[get real server failed]: %s; %d; %v ", ip, port, err) @@ -227,7 +244,7 @@ func (l *lvsProxy) DeleteRealServer(rs string) error { return errors.New("virtual service is empty") } - virServer := utils.BuildVirtualServer(l.vs.String()) + virServer := utils.BuildVirtualServer(l.vs.String(), l.scheduler) realServer := utils.BuildRealServer(rs) err := l.handle.DeleteRealServer(virServer, realServer) if err != nil { @@ -253,3 +270,10 @@ func (l *lvsProxy) DeleteRealServer(rs string) error { return nil } + +func (l *lvsProxy) GetScheduler() (string, error) { + if l.scheduler == "" { + return "", ErrLvsServiceNotInited + } + return l.scheduler, nil +} diff --git a/bcs-k8s/bcs-apiserver-proxy/pkg/service/service_test.go b/bcs-k8s/bcs-apiserver-proxy/pkg/service/service_test.go index 06f9991196..6044986025 100644 --- a/bcs-k8s/bcs-apiserver-proxy/pkg/service/service_test.go +++ b/bcs-k8s/bcs-apiserver-proxy/pkg/service/service_test.go @@ -19,7 +19,7 @@ import ( ) func TestLvsProxy_IsVSAvailable(t *testing.T) { - lvs := NewLvsProxy() + lvs := NewLvsProxy("sh") vsList := []struct { vs string @@ -46,7 +46,7 @@ func TestLvsProxy_IsVSAvailable(t *testing.T) { } func TestLvsProxy_CreateVirtualServer(t *testing.T) { - lvs := NewLvsProxy() + lvs := NewLvsProxy("sh") vsList := []string{"127.0.0.1:6443", "127.0.0.2:6443"} @@ -61,7 +61,7 @@ func TestLvsProxy_CreateVirtualServer(t *testing.T) { } func TestLvsProxy_DeleteVirtualServer(t *testing.T) { - lvs := NewLvsProxy() + lvs := NewLvsProxy("sh") vsList := []string{"127.0.0.1:6443", "127.0.0.2:6443"} for _, server := range vsList { @@ -75,7 +75,7 @@ func TestLvsProxy_DeleteVirtualServer(t *testing.T) { } func TestLvsProxy_CreateRealServer(t *testing.T) { - lvs := NewLvsProxy() + lvs := NewLvsProxy("sh") vs := "127.0.0.1:6443" ok := lvs.IsVirtualServerAvailable(vs) @@ -100,7 +100,7 @@ func TestLvsProxy_CreateRealServer(t *testing.T) { } func TestLvsProxy_DeleteRealServer(t *testing.T) { - lvs := NewLvsProxy() + lvs := NewLvsProxy("sh") vs := "127.0.0.1:6443" ok := lvs.IsVirtualServerAvailable(vs) @@ -122,7 +122,7 @@ func TestLvsProxy_DeleteRealServer(t *testing.T) { } func TestLvsProxy_ListRealServer(t *testing.T) { - lvs := NewLvsProxy() + lvs := NewLvsProxy("sh") vs := "127.0.0.1:6443" ok := lvs.IsVirtualServerAvailable(vs) diff --git a/bcs-k8s/bcs-apiserver-proxy/pkg/utils/utils.go b/bcs-k8s/bcs-apiserver-proxy/pkg/utils/utils.go index c29577b9e7..7228d63033 100644 --- a/bcs-k8s/bcs-apiserver-proxy/pkg/utils/utils.go +++ b/bcs-k8s/bcs-apiserver-proxy/pkg/utils/utils.go @@ -14,12 +14,21 @@ package utils import ( + "bufio" + "fmt" + "github.com/Tencent/bk-bcs/bcs-common/common/blog" + "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/ipvs" + "log" "net" + "os" + "os/exec" "strconv" "strings" +) - "github.com/Tencent/bk-bcs/bcs-common/common/blog" - "github.com/Tencent/bk-bcs/bcs-k8s/bcs-apiserver-proxy/pkg/ipvs" +const ( + IpvsPersistFileName = "ipvsConfig.yaml" + RcLocalIpvsFlag = "IPVS_START_UP" ) // EndPoint wrap IP&Port @@ -52,13 +61,13 @@ func SplitServer(server string) (string, uint32) { } // BuildVirtualServer build vip to ipvs.VirtualServer -func BuildVirtualServer(vip string) *ipvs.VirtualServer { +func BuildVirtualServer(vip string, scheduler string) *ipvs.VirtualServer { ip, port := SplitServer(vip) virServer := &ipvs.VirtualServer{ Address: net.ParseIP(ip), Protocol: "TCP", Port: port, - Scheduler: "rr", + Scheduler: scheduler, Flags: 0, Timeout: 0, } @@ -75,3 +84,73 @@ func BuildRealServer(real string) *ipvs.RealServer { } return realServer } + +func WriteToFile(filePath string, content string) error { + var file *os.File + var err error + file, err = os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + if err != nil { + log.Printf("open file %s failed; %v", filePath, err) + return err + } + defer file.Close() + _, err = file.Write([]byte(content)) + if err != nil { + log.Printf("write to file %s failed", filePath) + return err + } + log.Printf("write to file %s succeed!", filePath) + return nil +} + +func SetIpvsStartup(ipvsPersistDir string, toolPath string) error { + command := "chmod +x /etc/rc.d/rc.local" + cmd := exec.Command("/bin/sh", "-c", command) + output, err := cmd.Output() + if err != nil { + log.Printf("command [%s] exec failed", command) + } + resp := string(output) + log.Println(resp) + + exist, err := checkFlagExist("/etc/rc.local", RcLocalIpvsFlag) + if err != nil { + return err + } + if exist { + return nil + } else { + command := fmt.Sprintf("%v -cmd reload -persistDir %v", + toolPath, ipvsPersistDir) + command = "# " + RcLocalIpvsFlag + "\n" + command + "\n" + + err = WriteToFile("/etc/rc.local", command) + if err != nil { + log.Printf("write command [%s] to rc.local failed", command) + return err + } + } + return nil +} + +func checkFlagExist(path string, flag string) (bool, error) { + file, err := os.Open(path) + if err != nil { + log.Printf("open file [%v] failed", path) + return false, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.Contains(scanner.Text(), flag) { + log.Printf("ipvs startup flag already exists") + return true, nil + } + } + if err := scanner.Err(); err != nil { + log.Printf("scan file [%s] failed, %v", path, err) + return false, err + } + return false, nil +} diff --git a/docs/features/bcs-apiserver-proxy/bcs-apiserver-proxy.md b/docs/features/bcs-apiserver-proxy/bcs-apiserver-proxy.md index 7b653360f5..a464155884 100644 --- a/docs/features/bcs-apiserver-proxy/bcs-apiserver-proxy.md +++ b/docs/features/bcs-apiserver-proxy/bcs-apiserver-proxy.md @@ -11,62 +11,49 @@ 核心原理:通过本地`ipvs`代理节点解决`master`高可用问题,实现负载均衡。每个node节点上都启动一个负载均衡,上游就是master节点,负载方式有很多 ipvs nginx等,最终使用内核ipvs实现后端rs规则动态刷新,实现自动化。 -bcs-apiserver-proxy架构处理流程如图示: +bcs-apiserver-proxy架构工作流程如图示: -![bcs-apiserver-proxy处理流程图](./img/bcs-apiserver-proxy架构.png) +![bcs-apiserver-proxy工作流程图](./img/bcs-apiserver-proxy-work-flow.png) -* endpoints watch模块,主要负责动态更新`kubernetes`集群的`master`端点IP。集群新增、删除、变更`master`节点、故障时能够动态刷新`endpointsIP` -* ipvs模块 - * 新增virtual server - * 检测virtual server是否可用 - * 删除virtual server - * 新增、删除LVS后端的rs -* manager模块主要负责检测当前节点virtual server是否可用、集群的master端点IP和后端的rs进行diff并进行更新virtual server等 -## 部署指南 +## 使用指南 -通过`daemonSet`进行部署,负责维护本地负载均衡规则,并动态更新后端`endpointIPs` +### 使用步骤 -``` -cd bk-bcs/install/conf/bcs-k8s-master/bcs-apiserver-proxy/ +1. 下载`bk-bcs`代码,进行代码编译,生成 `bcs-apiserver-proxy`和`apiserver-proxy-tools` + ``` + git clone https://github.com/Tencent/bk-bcs.git + make apiserver-proxy + make apiserver-proxy-tools + ``` + +2. 生成镜像并通过`daemonSet`进行部署,负责维护本地负载均衡规则,并动态更新后端 rs ,并将ipvs规则持久化到本地 + +``` +cd bk-bcs/build/bcs.xxxxxxx-21.06.30/bcs-k8s-master/bcs-apiserver-proxy +docker build -t image名称 . +docker push 上传至镜像仓库 kubectl apply -f bcs-apiserver-proxy.yaml ``` -## 使用指南 - -### 使用步骤 -1. 下载`bk-bcs`代码,进行代码编译,生成 `bcs-apiserver-proxy`和`apiserver-proxy-tools` +3. 将工具`apiserver-proxy-tools`分发至各个`node`节点的 `/root`目录下, 通过工具`apiserver-proxy-tools`生成本地负载均衡的代理规则 - ``` - git clone https://github.com/Tencent/bk-bcs.git - make apiserver-proxy - make apiserver-proxy-tools - ``` -2. 生成镜像并将工具`apiserver-proxy-tools`分发至各个`node`节点 - - ``` - cd bk-bcs/build/bcs.xxxxxxx-21.06.30/bcs-k8s-master/bcs-apiserver-proxy - docker build -t image名称 . - docker push 上传至镜像仓库 - 并将生成的 apiserver-proxy-tools分发至各个node节点 - ``` -3. 首先通过工具`apiserver-proxy-tools`生成本地负载均衡的代理规则 - - ``` - apiserver-proxy-tools --help 查看帮助 - 生成vs本地负载均衡规则 - apiserver-proxy-tools -cmd add -vs vip:vport -rs master0:port -rs master1:port -rs master2:port - - 删除vs本地负载均衡 - apiserver-proxy-tools -cmd delete -vs vip:vport - ``` + ``` + apiserver-proxy-tools --help 查看帮助 + 初始化vs本地负载均衡规则 + apiserver-proxy-tools -cmd init -vs vip:vport -rs master0:port -rs master1:port -rs master2:port -scheduler sh + ``` + 4. `kubelet`及`kube-proxy`组件启动时`kube-config`文件配置连接生成的lvs即可并通过部署的`daemonset`动态守护规则 ### 场景 -#### 新增node节点/node重启 +#### 新增node节点 通过`apiserver-proxy-tools`工具生成本地负载均衡的代理规则,并会自动启动`daemonset`的`pod`守护代理规则 +#### 重启node节点 +`apiserver-proxy-tools`第一次初始化同时会创建自动启动任务,重启时从本地持久化文件中恢复负载均衡代理规则。 + #### 新增master节点/master节点IP改变/master节点down/master节点恢复 `node`节点上`pod`自动守护规则,当新增master节点、master节点IP改变、master节点down、master节点恢复,均会自动增加或者剔除后端rs节点,实现内部master节点的高可用访问 @@ -74,6 +61,7 @@ kubectl apply -f bcs-apiserver-proxy.yaml * `kube-proxy`组件启动时必须配置`--ipvs-exclude-cidrs strings`参数,避免清理本地`ipvs`规则 * VIP授权问题,生成证书文件时需要将上述`vip`添加至授权IP列表 * 集群VIP地址不能和集群其他地址段重复 +* bcs-apiserver-proxy组件的参数`lvsScheduler`和`ipvsPersistDir`需要与节点上使用apiserver-proxy-tools初始化时一致,建议默认不修改 ## 参考 [lvscare设计](https://github.com/sealyun/lvscare) diff --git a/docs/features/bcs-apiserver-proxy/img/bcs-apiserver-proxy-work-flow.png b/docs/features/bcs-apiserver-proxy/img/bcs-apiserver-proxy-work-flow.png new file mode 100644 index 0000000000..e64508729a Binary files /dev/null and b/docs/features/bcs-apiserver-proxy/img/bcs-apiserver-proxy-work-flow.png differ diff --git a/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy-static-pod.yaml b/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy-static-pod.yaml deleted file mode 100644 index 3f4434502c..0000000000 --- a/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy-static-pod.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: null - labels: - app: bcs-apiserver-proxy - platform: bk-bcs - name: bcs-apiserver-proxy - namespace: kube-system -spec: - containers: - - args: - - -f - - /data/bcs/bcs-apiserver-proxy/bcs-apiserver-proxy.json - command: - - /data/bcs/bcs-apiserver-proxy/container-start.sh - env: - - name: localIp - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: bcsApiserverProxyPort - value: "8004" - - name: log_dir - value: "/data/bcs/logs/bcs" - - name: pid_dir - value: ./ - - name: alsotostderr - value: "true" - - name: log_level - value: "3" - - name: bcsApiserverProxyDebug - value: "true" - - name: virtualAddress - value: "10.103.97.2:6443" - - name: healthScheme - value: "https" - - name: healthPath - value: "/healthz" - - name: kubeMaster - value: "" - - name: kubeConfig - value: "" - - name: endpointInterval - value: "3" - - name: managerInterval - value: "5" - - name: BCS_CONFIG_TYPE - value: render - image: mirrors.tencent.com/bcs/bcs-apiserver-proxy:v0.1 - imagePullPolicy: IfNotPresent - name: bcs-apiserver-proxy - resources: {} - securityContext: - privileged: true - volumeMounts: - - mountPath: /lib/modules - name: lib-modules - readOnly: true - restartPolicy: Always - hostNetwork: true - nodeName: 10.1.1.1 - serviceAccount: bcs-apiserver-proxy - serviceAccountName: bcs-apiserver-proxy - dnsPolicy: ClusterFirstWithHostNet - priorityClassName: system-cluster-critical - volumes: - - hostPath: - path: /lib/modules - type: "" - name: lib-modules -status: {} diff --git a/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy.json.template b/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy.json.template index 53ceb992ef..1b4df73c79 100644 --- a/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy.json.template +++ b/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy.json.template @@ -7,7 +7,11 @@ "v": ${log_level}, "debugMode": ${bcsApiserverProxyDebug}, "proxyLvs": { - "virtualAddress": "${virtualAddress}" + "virtualAddress": "${virtualAddress}", + "lvsScheduler": "${lvsScheduler}" + }, + "persistConfig": { + "ipvsPersistDir": "${ipvsPersistDir}" }, "healthCheck": { "healthScheme": "${healthScheme}", @@ -18,7 +22,6 @@ "kubeConfig": "${kubeConfig}" }, "systemInterval": { - "endpointInterval": ${endpointInterval}, "managerInterval": ${managerInterval} } } \ No newline at end of file diff --git a/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy.yaml b/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy.yaml index d59305b273..2f5dc92615 100644 --- a/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy.yaml +++ b/install/conf/bcs-k8s-master/bcs-apiserver-proxy/bcs-apiserver-proxy.yaml @@ -72,8 +72,12 @@ spec: value: "3" - name: bcsApiserverProxyDebug value: "true" + - name: ipvsPersistDir + value: "/root/.bcs" - name: virtualAddress value: "10.103.97.2:6443" + - name: lvsScheduler + value: "sh" - name: healthScheme value: "https" - name: healthPath @@ -82,13 +86,11 @@ spec: value: "" - name: kubeConfig value: "" - - name: endpointInterval - value: "3" - name: managerInterval - value: "5" + value: "10" - name: BCS_CONFIG_TYPE value: render - image: mirrors.tencent.com/bcs/bcs-apiserver-proxy:v0.1 + image: mirrors.tencent.com/bcs/bcs-apiserver-proxy:v2.3 imagePullPolicy: IfNotPresent name: bcs-apiserver-proxy resources: {} @@ -98,6 +100,8 @@ spec: - mountPath: /lib/modules name: lib-modules readOnly: true + - mountPath: /root/.bcs + name: bcs-config terminationGracePeriodSeconds: 30 restartPolicy: Always hostNetwork: true @@ -109,4 +113,7 @@ spec: - hostPath: path: /lib/modules type: "" - name: lib-modules \ No newline at end of file + name: lib-modules + - hostPath: + path: /root/.bcs + name: bcs-config \ No newline at end of file