From aa4f7b3b21d697696160334cfd8dc04146774af4 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 10 Jul 2020 15:40:21 -0700 Subject: [PATCH 1/6] update control plane properly on multinode restart --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 15 +++++++++------ test/integration/multinode_test.go | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 78c44a99e671..4e95adb149f2 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -722,7 +722,7 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig, n config.Node) er return err } -// UpdateCluster updates the cluster. +// UpdateCluster updates the control plane with cluster-level info. func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion) if err != nil { @@ -745,11 +745,14 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { } } - for _, n := range cfg.Nodes { - err := k.UpdateNode(cfg, n, r) - if err != nil { - return errors.Wrap(err, "updating node") - } + cp, err := config.PrimaryControlPlane(&cfg) + if err != nil { + return errors.Wrap(err, "getting control plane") + } + + err = k.UpdateNode(cfg, cp, r) + if err != nil { + return errors.Wrap(err, "updating control plane") } return nil diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index e43c5a2eb08e..3afd7534f06e 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -231,11 +231,21 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile } if strings.Count(rr.Stdout.String(), "host: Running") != 2 { - t.Errorf("status says both hosts are not running: args %q: %v", rr.Command(), rr.Stdout.String()) + t.Errorf("status says both hosts are not running: args %q: %v", rr.Command(), rr.Output()) } if strings.Count(rr.Stdout.String(), "kubelet: Running") != 2 { - t.Errorf("status says both kubelets are not running: args %q: %v", rr.Command(), rr.Stdout.String()) + t.Errorf("status says both kubelets are not running: args %q: %v", rr.Command(), rr.Output()) + } + + // Make sure kubectl reports that all nodes are ready + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes", "-o", `go-template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}} {{.status}}{{"\n"}}{{end}}{{end}}{{end}}'`)) + if err != nil { + t.Fatalf("failed to kubectl get nodes. args %q : %v", rr.Command(), err) + } + + if strings.Count(rr.Stdout.String(), "True") != 2 { + t.Errorf("expected 2 nodes Ready status to be True, got %v", rr.Output()) } } From 970d954620adf257cfb900b25428cdcfdc12b39c Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 13 Jul 2020 10:32:46 -0700 Subject: [PATCH 2/6] add wait to multinode restart test --- test/integration/multinode_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 3afd7534f06e..2a11ef1e8d8b 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -218,7 +218,7 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile } } // Restart a full cluster with minikube start - startArgs := append([]string{"start", "-p", profile}, StartArgs()...) + startArgs := append([]string{"start", "-p", profile, "--wait=true"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...)) if err != nil { t.Fatalf("failed to start cluster. args %q : %v", rr.Command(), err) From ad9e71dcd13d5fbfced0e4620405d3796176537f Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 13 Jul 2020 14:56:35 -0700 Subject: [PATCH 3/6] fix up multinode test --- test/integration/multinode_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 2a11ef1e8d8b..24de061ce7bb 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -179,7 +179,7 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin } func validateStopMultiNodeCluster(ctx context.Context, t *testing.T, profile string) { - // Run minikube node stop on that node + // Run minikube stop on the cluster rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "stop")) if err != nil { t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err) @@ -198,11 +198,11 @@ func validateStopMultiNodeCluster(ctx context.Context, t *testing.T, profile str t.Fatalf("failed to run minikube status. args %q : %v", rr.Command(), err) } - if strings.Count(rr.Stdout.String(), "host: Stopped") != 2 { + if strings.Count(rr.Stdout.String(), "host: Stopped") != 3 { t.Errorf("incorrect number of stopped hosts: args %q: %v", rr.Command(), rr.Stdout.String()) } - if strings.Count(rr.Stdout.String(), "kubelet: Stopped") != 2 { + if strings.Count(rr.Stdout.String(), "kubelet: Stopped") != 3 { t.Errorf("incorrect number of stopped kubelets: args %q: %v", rr.Command(), rr.Stdout.String()) } } @@ -230,11 +230,11 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile t.Fatalf("failed to run minikube status. args %q : %v", rr.Command(), err) } - if strings.Count(rr.Stdout.String(), "host: Running") != 2 { + if strings.Count(rr.Stdout.String(), "host: Running") != 3 { t.Errorf("status says both hosts are not running: args %q: %v", rr.Command(), rr.Output()) } - if strings.Count(rr.Stdout.String(), "kubelet: Running") != 2 { + if strings.Count(rr.Stdout.String(), "kubelet: Running") != 3 { t.Errorf("status says both kubelets are not running: args %q: %v", rr.Command(), rr.Output()) } @@ -244,8 +244,8 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile t.Fatalf("failed to kubectl get nodes. args %q : %v", rr.Command(), err) } - if strings.Count(rr.Stdout.String(), "True") != 2 { - t.Errorf("expected 2 nodes Ready status to be True, got %v", rr.Output()) + if strings.Count(rr.Stdout.String(), "True") != 3 { + t.Errorf("expected 3 nodes Ready status to be True, got %v", rr.Output()) } } From 9f2f604b93e92cbda6d33d2384cedab05c66beb7 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 13 Jul 2020 18:54:40 -0700 Subject: [PATCH 4/6] several fixes --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 49 ++++++++++---------- pkg/minikube/node/node.go | 36 +++++++++++++- pkg/minikube/node/start.go | 10 ++-- test/integration/multinode_test.go | 12 ++--- 4 files changed, 71 insertions(+), 36 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 3dbfeab02607..4300331a3d96 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -396,16 +396,15 @@ func (k *Bootstrapper) client(ip string, port int) (*kubernetes.Clientset, error func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, timeout time.Duration) error { start := time.Now() - if !n.ControlPlane { - glog.Infof("%s is not a control plane, nothing to wait for", n.Name) - return nil - } - register.Reg.SetStep(register.VerifyingKubernetes) out.T(out.HealthCheck, "Verifying Kubernetes components...") // TODO: #7706: for better performance we could use k.client inside minikube to avoid asking for external IP:PORT - hostname, _, port, err := driver.ControlPlaneEndpoint(&cfg, &n, cfg.Driver) + cp, err := config.PrimaryControlPlane(&cfg) + if err != nil { + return errors.Wrap(err, "get primary control plane") + } + hostname, _, port, err := driver.ControlPlaneEndpoint(&cfg, &cp, cfg.Driver) if err != nil { return errors.Wrap(err, "get control plane endpoint") } @@ -430,31 +429,33 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return errors.Wrapf(err, "create runtme-manager %s", cfg.KubernetesConfig.ContainerRuntime) } - if cfg.VerifyComponents[kverify.APIServerWaitKey] { - if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil { - return errors.Wrap(err, "wait for apiserver proc") - } + if n.ControlPlane { + if cfg.VerifyComponents[kverify.APIServerWaitKey] { + if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil { + return errors.Wrap(err, "wait for apiserver proc") + } - if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, hostname, port, timeout); err != nil { - return errors.Wrap(err, "wait for healthy API server") + if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, hostname, port, timeout); err != nil { + return errors.Wrap(err, "wait for healthy API server") + } } - } - if cfg.VerifyComponents[kverify.SystemPodsWaitKey] { - if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, start, timeout); err != nil { - return errors.Wrap(err, "waiting for system pods") + if cfg.VerifyComponents[kverify.SystemPodsWaitKey] { + if err := kverify.WaitForSystemPods(cr, k, cfg, k.c, client, start, timeout); err != nil { + return errors.Wrap(err, "waiting for system pods") + } } - } - if cfg.VerifyComponents[kverify.DefaultSAWaitKey] { - if err := kverify.WaitForDefaultSA(client, timeout); err != nil { - return errors.Wrap(err, "waiting for default service account") + if cfg.VerifyComponents[kverify.DefaultSAWaitKey] { + if err := kverify.WaitForDefaultSA(client, timeout); err != nil { + return errors.Wrap(err, "waiting for default service account") + } } - } - if cfg.VerifyComponents[kverify.AppsRunningKey] { - if err := kverify.WaitForAppsRunning(client, kverify.AppsRunningList, timeout); err != nil { - return errors.Wrap(err, "waiting for apps_running") + if cfg.VerifyComponents[kverify.AppsRunningKey] { + if err := kverify.WaitForAppsRunning(client, kverify.AppsRunningList, timeout); err != nil { + return errors.Wrap(err, "waiting for apps_running") + } } } diff --git a/pkg/minikube/node/node.go b/pkg/minikube/node/node.go index 8ae5df2735ce..0febcd1a7b43 100644 --- a/pkg/minikube/node/node.go +++ b/pkg/minikube/node/node.go @@ -18,11 +18,13 @@ package node import ( "fmt" + "os/exec" "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/viper" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/machine" @@ -66,12 +68,44 @@ func Delete(cc config.ClusterConfig, name string) (*config.Node, error) { return n, errors.Wrap(err, "retrieve") } + m := driver.MachineName(cc, *n) api, err := machine.NewAPIClient() if err != nil { return n, err } - err = machine.DeleteHost(api, driver.MachineName(cc, *n)) + // grab control plane to use kubeconfig + host, err := machine.LoadHost(api, cc.Name) + if err != nil { + return n, err + } + + runner, err := machine.CommandRunner(host) + if err != nil { + return n, err + } + + // kubectl drain + kubectl := kapi.KubectlBinaryPath(cc.KubernetesConfig.KubernetesVersion) + cmd := exec.Command("sudo", "KUBECONFIG=/var/lib/minikube/kubeconfig", kubectl, "drain", m) + if _, err := runner.RunCmd(cmd); err != nil { + glog.Warningf("unable to scale coredns replicas to 1: %v", err) + } else { + glog.Infof("successfully scaled coredns replicas to 1") + } + + // kubectl delete + client, err := kapi.Client(cc.Name) + if err != nil { + return n, err + } + + err = client.CoreV1().Nodes().Delete(m, nil) + if err != nil { + return n, err + } + + err = machine.DeleteHost(api, m) if err != nil { return n, err } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index e118e0847654..6a1d386db657 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -162,11 +162,6 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { prepareNone() } - glog.Infof("Will wait %s for node ...", waitTimeout) - if err := bs.WaitForNode(*starter.Cfg, *starter.Node, viper.GetDuration(waitTimeout)); err != nil { - return nil, errors.Wrapf(err, "wait %s for node", viper.GetDuration(waitTimeout)) - } - } else { if err := bs.UpdateNode(*starter.Cfg, *starter.Node, cr); err != nil { return nil, errors.Wrap(err, "update node") @@ -197,6 +192,11 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } } + glog.Infof("Will wait %s for node ...", waitTimeout) + if err := bs.WaitForNode(*starter.Cfg, *starter.Node, viper.GetDuration(waitTimeout)); err != nil { + return nil, errors.Wrapf(err, "wait %s for node", viper.GetDuration(waitTimeout)) + } + glog.Infof("waiting for startup goroutines ...") wg.Wait() diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 24de061ce7bb..9a104cdcc29e 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -198,11 +198,11 @@ func validateStopMultiNodeCluster(ctx context.Context, t *testing.T, profile str t.Fatalf("failed to run minikube status. args %q : %v", rr.Command(), err) } - if strings.Count(rr.Stdout.String(), "host: Stopped") != 3 { + if strings.Count(rr.Stdout.String(), "host: Stopped") != 2 { t.Errorf("incorrect number of stopped hosts: args %q: %v", rr.Command(), rr.Stdout.String()) } - if strings.Count(rr.Stdout.String(), "kubelet: Stopped") != 3 { + if strings.Count(rr.Stdout.String(), "kubelet: Stopped") != 2 { t.Errorf("incorrect number of stopped kubelets: args %q: %v", rr.Command(), rr.Stdout.String()) } } @@ -230,11 +230,11 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile t.Fatalf("failed to run minikube status. args %q : %v", rr.Command(), err) } - if strings.Count(rr.Stdout.String(), "host: Running") != 3 { + if strings.Count(rr.Stdout.String(), "host: Running") != 2 { t.Errorf("status says both hosts are not running: args %q: %v", rr.Command(), rr.Output()) } - if strings.Count(rr.Stdout.String(), "kubelet: Running") != 3 { + if strings.Count(rr.Stdout.String(), "kubelet: Running") != 2 { t.Errorf("status says both kubelets are not running: args %q: %v", rr.Command(), rr.Output()) } @@ -244,8 +244,8 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile t.Fatalf("failed to kubectl get nodes. args %q : %v", rr.Command(), err) } - if strings.Count(rr.Stdout.String(), "True") != 3 { - t.Errorf("expected 3 nodes Ready status to be True, got %v", rr.Output()) + if strings.Count(rr.Stdout.String(), "True") != 2 { + t.Errorf("expected 2 nodes Ready status to be True, got %v", rr.Output()) } } From 58c501c322a133085b03f75348ecfd9aa2f7dcf2 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 15 Jul 2020 15:02:54 -0700 Subject: [PATCH 5/6] sleeping for 30 seconds works for some reason? --- test/integration/multinode_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 9a104cdcc29e..e97d32d07dfa 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -24,6 +24,7 @@ import ( "os/exec" "strings" "testing" + "time" ) func TestMultiNode(t *testing.T) { @@ -238,12 +239,15 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile t.Errorf("status says both kubelets are not running: args %q: %v", rr.Command(), rr.Output()) } + time.Sleep(Seconds(30)) + // Make sure kubectl reports that all nodes are ready - rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes", "-o", `go-template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}} {{.status}}{{"\n"}}{{end}}{{end}}{{end}}'`)) - if err != nil { - t.Fatalf("failed to kubectl get nodes. args %q : %v", rr.Command(), err) + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes")) + if strings.Count(rr.Stdout.String(), "NotReady") > 0 { + t.Errorf("expected 2 nodes to be Ready, got %v", rr.Output()) } + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes", "-o", `go-template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}} {{.status}}{{"\n"}}{{end}}{{end}}{{end}}'`)) if strings.Count(rr.Stdout.String(), "True") != 2 { t.Errorf("expected 2 nodes Ready status to be True, got %v", rr.Output()) } From b7445a7483845a12b20cfaff2e8cf673f0312112 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 15 Jul 2020 15:14:12 -0700 Subject: [PATCH 6/6] fix lint --- test/integration/multinode_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index e97d32d07dfa..4b67b09b06ba 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -243,11 +243,17 @@ func validateRestartMultiNodeCluster(ctx context.Context, t *testing.T, profile // Make sure kubectl reports that all nodes are ready rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes")) + if err != nil { + t.Fatalf("failed to run kubectl get nodes. args %q : %v", rr.Command(), err) + } if strings.Count(rr.Stdout.String(), "NotReady") > 0 { t.Errorf("expected 2 nodes to be Ready, got %v", rr.Output()) } rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "get", "nodes", "-o", `go-template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}} {{.status}}{{"\n"}}{{end}}{{end}}{{end}}'`)) + if err != nil { + t.Fatalf("failed to run kubectl get nodes. args %q : %v", rr.Command(), err) + } if strings.Count(rr.Stdout.String(), "True") != 2 { t.Errorf("expected 2 nodes Ready status to be True, got %v", rr.Output()) }