Merge pull request #18139 from deads2k/controller-20

Automatic merge from submit-queue (batch tested with PRs 18163, 18139, 18193, 18194, 16180).

cleanly execute scheduler and kubecontrollermanager via cobra command

I found the upstream bug that was preventing this before and fixed it.  

After this pull, there is no direct rebase impact for kube controllers *except* the ones we directly extend.  HPA, quota, and SA tokens come to mind.

pkg/cmd/server/start/start_kube_controller_manager.go

@@ -1,26 +1,16 @@
 package start
 
 import (
+	"fmt"
 	"strconv"
 
 	"github.com/golang/glog"
-	"github.com/spf13/pflag"
 
-	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	controllerapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
-	controlleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"
-
-	cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
 )
 
-func kubeControllerManagerAddFlags(cmserver *controlleroptions.CMServer) func(flags *pflag.FlagSet) {
-	return func(flags *pflag.FlagSet) {
-		cmserver.AddFlags(flags, controllerapp.KnownControllers(), controllerapp.ControllersDisabledByDefault.List())
-	}
-}
-
-func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, openshiftConfigFile string, dynamicProvisioningEnabled bool) (*controlleroptions.CMServer, error) {
+func computeKubeControllerManagerArgs(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, openshiftConfigFile string, dynamicProvisioningEnabled bool) []string {
 	cmdLineArgs := map[string][]string{}
 	if _, ok := cmdLineArgs["controllers"]; !ok {
 		cmdLineArgs["controllers"] = []string{
@@ -77,22 +67,22 @@ func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, po
 	}
 	cmdLineArgs["openshift-config"] = []string{openshiftConfigFile}
 
-	// resolve arguments
-	controllerManager := controlleroptions.NewCMServer()
-	if err := cmdflags.Resolve(cmdLineArgs, kubeControllerManagerAddFlags(controllerManager)); len(err) > 0 {
-		return nil, kerrors.NewAggregate(err)
+	args := []string{}
+	for key, value := range cmdLineArgs {
+		for _, token := range value {
+			args = append(args, fmt.Sprintf("--%s=%v", key, token))
+		}
 	}
-
-	return controllerManager, nil
+	return args
 }
 
 func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, openshiftConfigFile string, dynamicProvisioningEnabled bool) {
-	// TODO we need a real identity for this.  Right now it's just using the loopback connection like it used to.
-	controllerManager, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, openshiftConfigFile, dynamicProvisioningEnabled)
-	if err != nil {
-		glog.Fatal(err)
-	}
-	if err := controllerapp.Run(controllerManager); err != nil {
+	cmd := controllerapp.NewControllerManagerCommand()
+	args := computeKubeControllerManagerArgs(kubeconfigFile, saPrivateKeyFile, saRootCAFile, podEvictionTimeout, openshiftConfigFile, dynamicProvisioningEnabled)
+	if err := cmd.ParseFlags(args); err != nil {
 		glog.Fatal(err)
 	}
+	glog.Infof("`kube-controller-manager %v`", args)
+	cmd.Run(nil, nil)
+	panic(fmt.Sprintf("`kube-controller-manager %v` exited", args))
 }

pkg/cmd/server/start/start_kube_scheduler.go

@@ -1,17 +1,16 @@
 package start
 
 import (
+	"fmt"
+	"time"
+
 	"github.com/golang/glog"
-	"github.com/spf13/pflag"
 
-	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	schedulerapp "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
 	_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"
-
-	cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
 )
 
-func newScheduler(kubeconfigFile, schedulerConfigFile string, schedulerArgs map[string][]string) (*schedulerapp.Options, error) {
+func computeSchedulerArgs(kubeconfigFile, schedulerConfigFile string, schedulerArgs map[string][]string) []string {
 	cmdLineArgs := map[string][]string{}
 	// deep-copy the input args to avoid mutation conflict.
 	for k, v := range schedulerArgs {
@@ -35,35 +34,23 @@ func newScheduler(kubeconfigFile, schedulerConfigFile string, schedulerArgs map[
 		cmdLineArgs["port"] = []string{"-1"}
 	}
 
-	// resolve arguments
-	schedulerOptions, err := schedulerapp.NewOptions()
-	if err != nil {
-		return nil, err
-	}
-	if err := schedulerOptions.ReallyApplyDefaults(); err != nil {
-		return nil, err
-	}
-	if err := cmdflags.Resolve(cmdLineArgs, func(fs *pflag.FlagSet) {
-		schedulerapp.AddFlags(schedulerOptions, fs)
-	}); len(err) > 0 {
-		return nil, kerrors.NewAggregate(err)
+	args := []string{}
+	for key, value := range cmdLineArgs {
+		for _, token := range value {
+			args = append(args, fmt.Sprintf("--%s=%v", key, token))
+		}
 	}
-	if err := schedulerOptions.Complete(); err != nil {
-		return nil, err
-	}
-
-	return schedulerOptions, nil
+	return args
 }
 
 func runEmbeddedScheduler(kubeconfigFile, schedulerConfigFile string, cmdLineArgs map[string][]string) {
-	// TODO we need a real identity for this.  Right now it's just using the loopback connection like it used to.
-	schedulerOptions, err := newScheduler(kubeconfigFile, schedulerConfigFile, cmdLineArgs)
-	if err != nil {
-		glog.Fatal(err)
-	}
-	// this does a second leader election, but doing the second leader election will allow us to move out process in
-	// 3.8 if we so choose.
-	if err := schedulerOptions.Run(); err != nil {
+	cmd := schedulerapp.NewSchedulerCommand()
+	args := computeSchedulerArgs(kubeconfigFile, schedulerConfigFile, cmdLineArgs)
+	if err := cmd.ParseFlags(args); err != nil {
 		glog.Fatal(err)
 	}
+	glog.Infof("`kube-scheduler %v`", args)
+	cmd.Run(nil, nil)
+	glog.Fatalf("`kube-scheduler %v` exited", args)
+	time.Sleep(10 * time.Second)
 }