From 1ed86db10911926b384f9471e807dbec6906d4d0 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 5 Mar 2021 13:45:36 -0800 Subject: [PATCH 01/18] auto-detect gce and do not enable gcp auth addon --- pkg/addons/gcpauth/enable.go | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/pkg/addons/gcpauth/enable.go b/pkg/addons/gcpauth/enable.go index 11eecca77383..9a198fb004a3 100644 --- a/pkg/addons/gcpauth/enable.go +++ b/pkg/addons/gcpauth/enable.go @@ -19,10 +19,9 @@ package gcpauth import ( "bytes" "context" - "io/ioutil" + "net/http" "os" "os/exec" - "path" "strconv" "github.com/pkg/errors" @@ -54,6 +53,10 @@ func EnableOrDisable(cfg *config.ClusterConfig, name string, val string) error { } func enableAddon(cfg *config.ClusterConfig) error { + if isGCE() { + exit.Message(reason.InternalCredsNotFound, "It seems like you are on GCE, which means authentication should work without the GCP Auth addon.") + } + // Grab command runner from running cluster cc := mustload.Running(cfg.Name) r := cc.CP.Runner @@ -65,20 +68,9 @@ func enableAddon(cfg *config.ClusterConfig) error { exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") } + // Don't mount in empty credentials file if creds.JSON == nil { - // Cloud Shell sends credential files to an unusual location, let's check that location - // For example, CLOUDSDK_CONFIG=/tmp/tmp.cflmvysoQE - if e := os.Getenv("CLOUDSDK_CONFIG"); e != "" { - credFile := path.Join(e, "application_default_credentials.json") - b, err := ioutil.ReadFile(credFile) - if err != nil { - exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") - } - creds.JSON = b - } else { - // We don't currently support authentication through the metadata server - exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") - } + exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") } f := assets.NewMemoryAssetTarget(creds.JSON, credentialsPath, "0444") @@ -134,3 +126,16 @@ func disableAddon(cfg *config.ClusterConfig) error { return nil } + +func isGCE() bool { + resp, err := http.Get("metadata.google.internal") + if err != nil { + return false + } + + if resp.Header.Get("Metadata-Flavor") == "Google" { + return true + } + + return false +} From 6fc42a549ea3e486daec40dc15e569f1f3e03c4d Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 5 Mar 2021 16:41:27 -0800 Subject: [PATCH 02/18] allow --force --- pkg/addons/gcpauth/enable.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/addons/gcpauth/enable.go b/pkg/addons/gcpauth/enable.go index 9a198fb004a3..dc7fc88e910f 100644 --- a/pkg/addons/gcpauth/enable.go +++ b/pkg/addons/gcpauth/enable.go @@ -25,6 +25,7 @@ import ( "strconv" "github.com/pkg/errors" + "github.com/spf13/viper" "golang.org/x/oauth2/google" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/config" @@ -53,8 +54,8 @@ func EnableOrDisable(cfg *config.ClusterConfig, name string, val string) error { } func enableAddon(cfg *config.ClusterConfig) error { - if isGCE() { - exit.Message(reason.InternalCredsNotFound, "It seems like you are on GCE, which means authentication should work without the GCP Auth addon.") + if isGCE() && !viper.GetBool("force") { + exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to use this addon, use the --force flag.") } // Grab command runner from running cluster From 5a5abedb3e60f7229ddeab09758ec623d977fb16 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 8 Mar 2021 08:47:39 -0800 Subject: [PATCH 03/18] support force in addons enable command --- cmd/minikube/cmd/config/enable.go | 2 ++ test/integration/addons_test.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/config/enable.go b/cmd/minikube/cmd/config/enable.go index a2426ca3fb26..f7a79af631b3 100644 --- a/cmd/minikube/cmd/config/enable.go +++ b/cmd/minikube/cmd/config/enable.go @@ -71,10 +71,12 @@ var addonsEnableCmd = &cobra.Command{ var ( images string registries string + force bool ) func init() { addonsEnableCmd.Flags().StringVar(&images, "images", "", "Images used by this addon. Separated by commas.") addonsEnableCmd.Flags().StringVar(®istries, "registries", "", "Registries used by this addon. Separated by commas.") + addonsEnableCmd.Flags().BoolVar(&force, "force", false, "If true, will force gcp-auth addon to be enabled on GCE.") AddonsCmd.AddCommand(addonsEnableCmd) } diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 7a875a218a76..88e597226f62 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -58,7 +58,7 @@ func TestAddons(t *testing.T) { t.Fatalf("Failed setting GOOGLE_CLOUD_PROJECT env var: %v", err) } - args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=olm", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth"}, StartArgs()...) + args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--force", "--addons=registry", "--addons=metrics-server", "--addons=olm", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth"}, StartArgs()...) if !(runtime.GOOS == "darwin" && KicDriver()) { // macos docker driver does not support ingress args = append(args, "--addons=ingress") } From e6cb33fa6d8d7ba92785fed74c27ca021f15f5f8 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 8 Mar 2021 12:15:05 -0800 Subject: [PATCH 04/18] oh dear global variable --- cmd/minikube/cmd/config/enable.go | 6 +++--- pkg/addons/gcpauth/enable.go | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/minikube/cmd/config/enable.go b/cmd/minikube/cmd/config/enable.go index f7a79af631b3..83cc63bfba4b 100644 --- a/cmd/minikube/cmd/config/enable.go +++ b/cmd/minikube/cmd/config/enable.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "k8s.io/minikube/pkg/addons" + "k8s.io/minikube/pkg/addons/gcpauth" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/exit" @@ -42,7 +43,7 @@ var addonsEnableCmd = &cobra.Command{ addon := args[0] // replace heapster as metrics-server because heapster is deprecated if addon == "heapster" { - out.Styled(style.Waiting, "enable metrics-server addon instead of heapster addon because heapster is deprecated") + out.Styled(style.Waiting, "using metrics-server addon, heapster is deprecated") addon = "metrics-server" } viper.Set(config.AddonImages, images) @@ -71,12 +72,11 @@ var addonsEnableCmd = &cobra.Command{ var ( images string registries string - force bool ) func init() { addonsEnableCmd.Flags().StringVar(&images, "images", "", "Images used by this addon. Separated by commas.") addonsEnableCmd.Flags().StringVar(®istries, "registries", "", "Registries used by this addon. Separated by commas.") - addonsEnableCmd.Flags().BoolVar(&force, "force", false, "If true, will force gcp-auth addon to be enabled on GCE.") + addonsEnableCmd.Flags().BoolVar(&gcpauth.Force, "force", false, "If true, will force gcp-auth addon to be enabled on GCE.") AddonsCmd.AddCommand(addonsEnableCmd) } diff --git a/pkg/addons/gcpauth/enable.go b/pkg/addons/gcpauth/enable.go index dc7fc88e910f..60028db03ba5 100644 --- a/pkg/addons/gcpauth/enable.go +++ b/pkg/addons/gcpauth/enable.go @@ -25,7 +25,6 @@ import ( "strconv" "github.com/pkg/errors" - "github.com/spf13/viper" "golang.org/x/oauth2/google" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/config" @@ -41,6 +40,8 @@ const ( projectPath = "/var/lib/minikube/google_cloud_project" ) +var Force bool + // EnableOrDisable enables or disables the metadata addon depending on the val parameter func EnableOrDisable(cfg *config.ClusterConfig, name string, val string) error { enable, err := strconv.ParseBool(val) @@ -54,7 +55,7 @@ func EnableOrDisable(cfg *config.ClusterConfig, name string, val string) error { } func enableAddon(cfg *config.ClusterConfig) error { - if isGCE() && !viper.GetBool("force") { + if !Force && isGCE() { exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to use this addon, use the --force flag.") } @@ -129,7 +130,7 @@ func disableAddon(cfg *config.ClusterConfig) error { } func isGCE() bool { - resp, err := http.Get("metadata.google.internal") + resp, err := http.Get("http://metadata.google.internal") if err != nil { return false } From 8af2708a8f9cccebb355501a8c5eb1facf81b9e5 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 8 Mar 2021 12:50:27 -0800 Subject: [PATCH 05/18] make it work with start too --- pkg/minikube/node/start.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 017b99540840..e5216a5c20a9 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -37,6 +37,7 @@ import ( kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/addons" + "k8s.io/minikube/pkg/addons/gcpauth" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/bootstrapper" @@ -164,6 +165,9 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { // enable addons, both old and new! if starter.ExistingAddons != nil { + if viper.GetBool("force") { + gcpauth.Force = true + } wg.Add(1) go addons.Start(&wg, starter.Cfg, starter.ExistingAddons, config.AddonList) } From 16a2ac9c47779ac9a3fd828a5f354cfa2dbd6641 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 8 Mar 2021 13:10:43 -0800 Subject: [PATCH 06/18] generate docs --- pkg/addons/gcpauth/enable.go | 1 + site/content/en/docs/commands/addons.md | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/addons/gcpauth/enable.go b/pkg/addons/gcpauth/enable.go index 60028db03ba5..ccc466a03dd8 100644 --- a/pkg/addons/gcpauth/enable.go +++ b/pkg/addons/gcpauth/enable.go @@ -40,6 +40,7 @@ const ( projectPath = "/var/lib/minikube/google_cloud_project" ) +// Force is used to override our GCE check for the addon var Force bool // EnableOrDisable enables or disables the metadata addon depending on the val parameter diff --git a/site/content/en/docs/commands/addons.md b/site/content/en/docs/commands/addons.md index 51023fa24dd9..5fec6cf1d695 100644 --- a/site/content/en/docs/commands/addons.md +++ b/site/content/en/docs/commands/addons.md @@ -128,6 +128,7 @@ minikube addons enable dashboard ### Options ``` + --force If true, will force gcp-auth addon to be enabled on GCE. --images string Images used by this addon. Separated by commas. --registries string Registries used by this addon. Separated by commas. ``` From f0e70b4e1716c08d1a3b0212d109d4a0f9906948 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 8 Mar 2021 14:57:40 -0800 Subject: [PATCH 07/18] initialize bool --- pkg/addons/gcpauth/enable.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/addons/gcpauth/enable.go b/pkg/addons/gcpauth/enable.go index ccc466a03dd8..cd077ef8d5c5 100644 --- a/pkg/addons/gcpauth/enable.go +++ b/pkg/addons/gcpauth/enable.go @@ -41,7 +41,7 @@ const ( ) // Force is used to override our GCE check for the addon -var Force bool +var Force bool = false // EnableOrDisable enables or disables the metadata addon depending on the val parameter func EnableOrDisable(cfg *config.ClusterConfig, name string, val string) error { From afd7a785abab6128dd0d72378cd463976b2d78e9 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 12:53:48 -0800 Subject: [PATCH 08/18] fix message --- pkg/addons/gcpauth/enable.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/addons/gcpauth/enable.go b/pkg/addons/gcpauth/enable.go index cd077ef8d5c5..6e6512d0a5fe 100644 --- a/pkg/addons/gcpauth/enable.go +++ b/pkg/addons/gcpauth/enable.go @@ -57,7 +57,7 @@ func EnableOrDisable(cfg *config.ClusterConfig, name string, val string) error { func enableAddon(cfg *config.ClusterConfig) error { if !Force && isGCE() { - exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to use this addon, use the --force flag.") + exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to authenticate using a credentials file, use the --force flag.") } // Grab command runner from running cluster From 15c44bf7ac41c4c86775f11800c33ca687af2ef0 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 13:32:09 -0800 Subject: [PATCH 09/18] reorganize addons to make force flag more generic --- cmd/minikube/cmd/config/enable.go | 3 +- pkg/addons/addons.go | 116 ++++++++++++++++++- pkg/addons/config.go | 3 +- pkg/addons/gcpauth/enable.go | 144 ------------------------ pkg/minikube/node/start.go | 3 +- site/content/en/docs/commands/addons.md | 2 +- 6 files changed, 118 insertions(+), 153 deletions(-) delete mode 100644 pkg/addons/gcpauth/enable.go diff --git a/cmd/minikube/cmd/config/enable.go b/cmd/minikube/cmd/config/enable.go index 83cc63bfba4b..0df2abb53410 100644 --- a/cmd/minikube/cmd/config/enable.go +++ b/cmd/minikube/cmd/config/enable.go @@ -22,7 +22,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "k8s.io/minikube/pkg/addons" - "k8s.io/minikube/pkg/addons/gcpauth" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/exit" @@ -77,6 +76,6 @@ var ( func init() { addonsEnableCmd.Flags().StringVar(&images, "images", "", "Images used by this addon. Separated by commas.") addonsEnableCmd.Flags().StringVar(®istries, "registries", "", "Registries used by this addon. Separated by commas.") - addonsEnableCmd.Flags().BoolVar(&gcpauth.Force, "force", false, "If true, will force gcp-auth addon to be enabled on GCE.") + addonsEnableCmd.Flags().BoolVar(&addons.Force, "force", false, "If true, will perform potentially dangerous operations. Use with discretion.") AddonsCmd.AddCommand(addonsEnableCmd) } diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 4c44831f22af..9a7008375ba4 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -17,7 +17,12 @@ limitations under the License. package addons import ( + "bytes" + "context" "fmt" + "net/http" + "os" + "os/exec" "path" "runtime" "sort" @@ -28,6 +33,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/viper" + "golang.org/x/oauth2/google" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" @@ -50,8 +56,14 @@ import ( "k8s.io/minikube/pkg/util/retry" ) -// defaultStorageClassProvisioner is the name of the default storage class provisioner -const defaultStorageClassProvisioner = "standard" +const ( + credentialsPath = "/var/lib/minikube/google_application_credentials.json" + projectPath = "/var/lib/minikube/google_cloud_project" + defaultStorageClassProvisioner = "standard" +) + +// Force is used to override checks for addons +var Force bool = false // RunCallbacks runs all actions associated to an addon, but does not set it (thread-safe) func RunCallbacks(cc *config.ClusterConfig, name string, value string) error { @@ -488,3 +500,103 @@ func enableOrDisableAutoPause(cc *config.ClusterConfig, name string, val string) return nil } + +// enableOrDisableGCPAuth enables or disables the gcp-auth addon depending on the val parameter +func enableOrDisableGCPAuth(cfg *config.ClusterConfig, name string, val string) error { + enable, err := strconv.ParseBool(val) + if err != nil { + return errors.Wrapf(err, "parsing bool: %s", name) + } + if enable { + return enableAddonGCPAuth(cfg) + } + return disableAddonGCPAuth(cfg) +} + +func enableAddonGCPAuth(cfg *config.ClusterConfig) error { + if !Force && isGCE() { + exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to authenticate using a credentials file, use the --force flag.") + } + + // Grab command runner from running cluster + cc := mustload.Running(cfg.Name) + r := cc.CP.Runner + + // Grab credentials from where GCP would normally look + ctx := context.Background() + creds, err := google.FindDefaultCredentials(ctx) + if err != nil { + exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") + } + + // Don't mount in empty credentials file + if creds.JSON == nil { + exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") + } + + f := assets.NewMemoryAssetTarget(creds.JSON, credentialsPath, "0444") + + err = r.Copy(f) + if err != nil { + return err + } + + // First check if the project env var is explicitly set + projectEnv := os.Getenv("GOOGLE_CLOUD_PROJECT") + if projectEnv != "" { + f := assets.NewMemoryAssetTarget([]byte(projectEnv), projectPath, "0444") + return r.Copy(f) + } + + // We're currently assuming gcloud is installed and in the user's path + project, err := exec.Command("gcloud", "config", "get-value", "project").Output() + if err == nil && len(project) > 0 { + f := assets.NewMemoryAssetTarget(bytes.TrimSpace(project), projectPath, "0444") + return r.Copy(f) + } + + out.WarningT("Could not determine a Google Cloud project, which might be ok.") + out.Styled(style.Tip, `To set your Google Cloud project, run: + + gcloud config set project + +or set the GOOGLE_CLOUD_PROJECT environment variable.`) + + // Copy an empty file in to avoid errors about missing files + emptyFile := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444") + return r.Copy(emptyFile) +} + +func disableAddonGCPAuth(cfg *config.ClusterConfig) error { + // Grab command runner from running cluster + cc := mustload.Running(cfg.Name) + r := cc.CP.Runner + + // Clean up the files generated when enabling the addon + creds := assets.NewMemoryAssetTarget([]byte{}, credentialsPath, "0444") + err := r.Remove(creds) + if err != nil { + return err + } + + project := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444") + err = r.Remove(project) + if err != nil { + return err + } + + return nil +} + +func isGCE() bool { + resp, err := http.Get("http://metadata.google.internal") + if err != nil { + return false + } + + if resp.Header.Get("Metadata-Flavor") == "Google" { + return true + } + + return false +} diff --git a/pkg/addons/config.go b/pkg/addons/config.go index f33a01f28363..22c242a7a533 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -17,7 +17,6 @@ limitations under the License. package addons import ( - "k8s.io/minikube/pkg/addons/gcpauth" "k8s.io/minikube/pkg/minikube/config" ) @@ -175,7 +174,7 @@ var Addons = []*Addon{ { name: "gcp-auth", set: SetBool, - callbacks: []setFn{gcpauth.EnableOrDisable, EnableOrDisableAddon, verifyGCPAuthAddon}, + callbacks: []setFn{enableOrDisableGCPAuth, EnableOrDisableAddon, verifyGCPAuthAddon}, }, { name: "volumesnapshots", diff --git a/pkg/addons/gcpauth/enable.go b/pkg/addons/gcpauth/enable.go deleted file mode 100644 index 6e6512d0a5fe..000000000000 --- a/pkg/addons/gcpauth/enable.go +++ /dev/null @@ -1,144 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gcpauth - -import ( - "bytes" - "context" - "net/http" - "os" - "os/exec" - "strconv" - - "github.com/pkg/errors" - "golang.org/x/oauth2/google" - "k8s.io/minikube/pkg/minikube/assets" - "k8s.io/minikube/pkg/minikube/config" - "k8s.io/minikube/pkg/minikube/exit" - "k8s.io/minikube/pkg/minikube/mustload" - "k8s.io/minikube/pkg/minikube/out" - "k8s.io/minikube/pkg/minikube/reason" - "k8s.io/minikube/pkg/minikube/style" -) - -const ( - credentialsPath = "/var/lib/minikube/google_application_credentials.json" - projectPath = "/var/lib/minikube/google_cloud_project" -) - -// Force is used to override our GCE check for the addon -var Force bool = false - -// EnableOrDisable enables or disables the metadata addon depending on the val parameter -func EnableOrDisable(cfg *config.ClusterConfig, name string, val string) error { - enable, err := strconv.ParseBool(val) - if err != nil { - return errors.Wrapf(err, "parsing bool: %s", name) - } - if enable { - return enableAddon(cfg) - } - return disableAddon(cfg) -} - -func enableAddon(cfg *config.ClusterConfig) error { - if !Force && isGCE() { - exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to authenticate using a credentials file, use the --force flag.") - } - - // Grab command runner from running cluster - cc := mustload.Running(cfg.Name) - r := cc.CP.Runner - - // Grab credentials from where GCP would normally look - ctx := context.Background() - creds, err := google.FindDefaultCredentials(ctx) - if err != nil { - exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") - } - - // Don't mount in empty credentials file - if creds.JSON == nil { - exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") - } - - f := assets.NewMemoryAssetTarget(creds.JSON, credentialsPath, "0444") - - err = r.Copy(f) - if err != nil { - return err - } - - // First check if the project env var is explicitly set - projectEnv := os.Getenv("GOOGLE_CLOUD_PROJECT") - if projectEnv != "" { - f := assets.NewMemoryAssetTarget([]byte(projectEnv), projectPath, "0444") - return r.Copy(f) - } - - // We're currently assuming gcloud is installed and in the user's path - project, err := exec.Command("gcloud", "config", "get-value", "project").Output() - if err == nil && len(project) > 0 { - f := assets.NewMemoryAssetTarget(bytes.TrimSpace(project), projectPath, "0444") - return r.Copy(f) - } - - out.WarningT("Could not determine a Google Cloud project, which might be ok.") - out.Styled(style.Tip, `To set your Google Cloud project, run: - - gcloud config set project - -or set the GOOGLE_CLOUD_PROJECT environment variable.`) - - // Copy an empty file in to avoid errors about missing files - emptyFile := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444") - return r.Copy(emptyFile) -} - -func disableAddon(cfg *config.ClusterConfig) error { - // Grab command runner from running cluster - cc := mustload.Running(cfg.Name) - r := cc.CP.Runner - - // Clean up the files generated when enabling the addon - creds := assets.NewMemoryAssetTarget([]byte{}, credentialsPath, "0444") - err := r.Remove(creds) - if err != nil { - return err - } - - project := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444") - err = r.Remove(project) - if err != nil { - return err - } - - return nil -} - -func isGCE() bool { - resp, err := http.Get("http://metadata.google.internal") - if err != nil { - return false - } - - if resp.Header.Get("Metadata-Flavor") == "Google" { - return true - } - - return false -} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index e5216a5c20a9..87997f4b6b24 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -37,7 +37,6 @@ import ( kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/addons" - "k8s.io/minikube/pkg/addons/gcpauth" "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/bootstrapper" @@ -166,7 +165,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { // enable addons, both old and new! if starter.ExistingAddons != nil { if viper.GetBool("force") { - gcpauth.Force = true + addons.Force = true } wg.Add(1) go addons.Start(&wg, starter.Cfg, starter.ExistingAddons, config.AddonList) diff --git a/site/content/en/docs/commands/addons.md b/site/content/en/docs/commands/addons.md index 5fec6cf1d695..7a783632a9b9 100644 --- a/site/content/en/docs/commands/addons.md +++ b/site/content/en/docs/commands/addons.md @@ -128,7 +128,7 @@ minikube addons enable dashboard ### Options ``` - --force If true, will force gcp-auth addon to be enabled on GCE. + --force If true, will perform potentially dangerous operations. Use with discretion. --images string Images used by this addon. Separated by commas. --registries string Registries used by this addon. Separated by commas. ``` From eb1ee1f931fbe6aa459f5a13dbcca68c3d2a29e8 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 14:32:02 -0800 Subject: [PATCH 10/18] account for GCE in addons test --- pkg/addons/addons.go | 17 ++--------------- pkg/util/utils.go | 15 +++++++++++++++ test/integration/addons_test.go | 28 +++++++++++++++++++++++++++- test/integration/helpers_test.go | 2 +- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 9a7008375ba4..74a2028a54cd 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "fmt" - "net/http" "os" "os/exec" "path" @@ -53,6 +52,7 @@ import ( "k8s.io/minikube/pkg/minikube/storageclass" "k8s.io/minikube/pkg/minikube/style" "k8s.io/minikube/pkg/minikube/sysinit" + "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" ) @@ -514,7 +514,7 @@ func enableOrDisableGCPAuth(cfg *config.ClusterConfig, name string, val string) } func enableAddonGCPAuth(cfg *config.ClusterConfig) error { - if !Force && isGCE() { + if !Force && util.IsOnGCE() { exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to authenticate using a credentials file, use the --force flag.") } @@ -587,16 +587,3 @@ func disableAddonGCPAuth(cfg *config.ClusterConfig) error { return nil } - -func isGCE() bool { - resp, err := http.Get("http://metadata.google.internal") - if err != nil { - return false - } - - if resp.Header.Get("Metadata-Flavor") == "Google" { - return true - } - - return false -} diff --git a/pkg/util/utils.go b/pkg/util/utils.go index f5fe5a3b226a..de32aa4da884 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -18,6 +18,7 @@ package util import ( "fmt" + "net/http" "os" "os/user" "path/filepath" @@ -109,3 +110,17 @@ func MaybeChownDirRecursiveToMinikubeUser(dir string) error { func ParseKubernetesVersion(version string) (semver.Version, error) { return semver.Make(version[1:]) } + +// IsOnGCE determines whether minikube is currently running on GCE. +func IsOnGCE() bool { + resp, err := http.Get("http://metadata.google.internal") + if err != nil { + return false + } + + if resp.Header.Get("Metadata-Flavor") == "Google" { + return true + } + + return false +} diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 88e597226f62..ccdf50d195b4 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -36,6 +36,7 @@ import ( "github.com/hashicorp/go-retryablehttp" "k8s.io/minikube/pkg/kapi" + "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" ) @@ -58,18 +59,43 @@ func TestAddons(t *testing.T) { t.Fatalf("Failed setting GOOGLE_CLOUD_PROJECT env var: %v", err) } - args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--force", "--addons=registry", "--addons=metrics-server", "--addons=olm", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth"}, StartArgs()...) + args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=olm", "--addons=volumesnapshots", "--addons=csi-hostpath-driver"}, StartArgs()...) if !(runtime.GOOS == "darwin" && KicDriver()) { // macos docker driver does not support ingress args = append(args, "--addons=ingress") } if !arm64Platform() { args = append(args, "--addons=helm-tiller") } + if !util.IsOnGCE() { + args = append(args, "--addons=gcp-auth") + } rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err != nil { t.Fatalf("%s failed: %v", rr.Command(), err) } + // If we're running the integration tests on GCE, which is frequently the case, first check to make sure we exit out properly, + // then use force to actually test using creds. + if util.IsOnGCE() { + args = append([]string{"addons", "enable", "gcp-auth"}) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err == nil { + t.Errorf("Expected error but didn't get one. command %v, output %v", rr.Command(), rr.Output()) + } else { + if !strings.Contains(rr.Output(), "It seems that you are running in GCE") { + t.Errorf("Unexpected error message: %v", rr.Output()) + } else { + // ok, use force here since we are in GCE + // do not use --force unless absolutely necessary + args = append(args, "--force") + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Errorf("%s failed: %v", rr.Command(), err) + } + } + } + } + // Parallelized tests t.Run("parallel", func(t *testing.T) { tests := []struct { diff --git a/test/integration/helpers_test.go b/test/integration/helpers_test.go index c41a20a20985..3e90b282bb73 100644 --- a/test/integration/helpers_test.go +++ b/test/integration/helpers_test.go @@ -86,7 +86,7 @@ func (rr RunResult) Output() string { return sb.String() } -// Run is a test helper to log a command being executed \_(ツ)_/¯ +// Run is a test helper to log a command being executed ¯\_(ツ)_/¯ func Run(t *testing.T, cmd *exec.Cmd) (*RunResult, error) { t.Helper() rr := &RunResult{Args: cmd.Args} From a2b737b8db5835d468264069d60e9fcfa16411aa Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 14:41:19 -0800 Subject: [PATCH 11/18] fix startup --- test/integration/addons_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index ccdf50d195b4..6dd57df85147 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -77,7 +77,7 @@ func TestAddons(t *testing.T) { // If we're running the integration tests on GCE, which is frequently the case, first check to make sure we exit out properly, // then use force to actually test using creds. if util.IsOnGCE() { - args = append([]string{"addons", "enable", "gcp-auth"}) + args = append([]string{"addons", "enable", "gcp-auth"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err == nil { t.Errorf("Expected error but didn't get one. command %v, output %v", rr.Command(), rr.Output()) From d314d73c3a2c136b286e034feba68392f23eddad Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 14:59:54 -0800 Subject: [PATCH 12/18] pull out addon specific code into their own files --- pkg/addons/addons.go | 206 ---------------------------------- pkg/addons/autopause.go | 79 +++++++++++++ pkg/addons/gcpauth.go | 138 +++++++++++++++++++++++ pkg/addons/storage_classes.go | 77 +++++++++++++ 4 files changed, 294 insertions(+), 206 deletions(-) create mode 100644 pkg/addons/autopause.go create mode 100644 pkg/addons/gcpauth.go create mode 100644 pkg/addons/storage_classes.go diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 74a2028a54cd..1ebfef1a869c 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -17,11 +17,7 @@ limitations under the License. package addons import ( - "bytes" - "context" "fmt" - "os" - "os/exec" "path" "runtime" "sort" @@ -32,7 +28,6 @@ import ( "github.com/pkg/errors" "github.com/spf13/viper" - "golang.org/x/oauth2/google" "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" @@ -43,16 +38,12 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" - "k8s.io/minikube/pkg/minikube/kubeconfig" "k8s.io/minikube/pkg/minikube/machine" - "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/out/register" "k8s.io/minikube/pkg/minikube/reason" - "k8s.io/minikube/pkg/minikube/storageclass" "k8s.io/minikube/pkg/minikube/style" "k8s.io/minikube/pkg/minikube/sysinit" - "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" ) @@ -186,7 +177,6 @@ https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Dri } } - // TODO(r2d4): config package should not reference API, pull this out api, err := machine.NewAPIClient() if err != nil { return errors.Wrap(err, "machine client") @@ -291,75 +281,10 @@ func enableOrDisableAddonInternal(cc *config.ClusterConfig, addon *assets.Addon, return retry.Expo(apply, 250*time.Millisecond, 2*time.Minute) } -// enableOrDisableStorageClasses enables or disables storage classes -func enableOrDisableStorageClasses(cc *config.ClusterConfig, name string, val string) error { - klog.Infof("enableOrDisableStorageClasses %s=%v on %q", name, val, cc.Name) - enable, err := strconv.ParseBool(val) - if err != nil { - return errors.Wrap(err, "Error parsing boolean") - } - - class := defaultStorageClassProvisioner - if name == "storage-provisioner-gluster" { - class = "glusterfile" - } - - api, err := machine.NewAPIClient() - if err != nil { - return errors.Wrap(err, "machine client") - } - defer api.Close() - - cp, err := config.PrimaryControlPlane(cc) - if err != nil { - return errors.Wrap(err, "getting control plane") - } - if !machine.IsRunning(api, config.MachineName(*cc, cp)) { - klog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement", config.MachineName(*cc, cp), name, val) - return EnableOrDisableAddon(cc, name, val) - } - - storagev1, err := storageclass.GetStoragev1(cc.Name) - if err != nil { - return errors.Wrapf(err, "Error getting storagev1 interface %v ", err) - } - - if enable { - // Only StorageClass for 'name' should be marked as default - err = storageclass.SetDefaultStorageClass(storagev1, class) - if err != nil { - return errors.Wrapf(err, "Error making %s the default storage class", class) - } - } else { - // Unset the StorageClass as default - err := storageclass.DisableDefaultStorageClass(storagev1, class) - if err != nil { - return errors.Wrapf(err, "Error disabling %s as the default storage class", class) - } - } - - return EnableOrDisableAddon(cc, name, val) -} - func verifyAddonStatus(cc *config.ClusterConfig, name string, val string) error { return verifyAddonStatusInternal(cc, name, val, "kube-system") } -func verifyGCPAuthAddon(cc *config.ClusterConfig, name string, val string) error { - enable, err := strconv.ParseBool(val) - if err != nil { - return errors.Wrapf(err, "parsing bool: %s", name) - } - err = verifyAddonStatusInternal(cc, name, val, "gcp-auth") - - if enable && err == nil { - out.Styled(style.Notice, "Your GCP credentials will now be mounted into every pod created in the {{.name}} cluster.", out.V{"name": cc.Name}) - out.Styled(style.Notice, "If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.") - } - - return err -} - func verifyAddonStatusInternal(cc *config.ClusterConfig, name string, val string, ns string) error { klog.Infof("Verifying addon %s=%s in %q", name, val, cc.Name) enable, err := strconv.ParseBool(val) @@ -456,134 +381,3 @@ func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]boo } } } - -// enableOrDisableAutoPause enables the service after the config was copied by generic enble -func enableOrDisableAutoPause(cc *config.ClusterConfig, name string, val string) error { - enable, err := strconv.ParseBool(val) - if err != nil { - return errors.Wrapf(err, "parsing bool: %s", name) - } - out.Infof("auto-pause addon is an alpha feature and still in early development. Please file issues to help us make it better.") - out.Infof("https://github.com/kubernetes/minikube/labels/co%2Fauto-pause") - - if !driver.IsKIC(cc.Driver) || runtime.GOARCH != "amd64" { - exit.Message(reason.Usage, `auto-pause currently is only supported on docker driver/docker runtime/amd64. Track progress of others here: https://github.com/kubernetes/minikube/issues/10601`) - } - co := mustload.Running(cc.Name) - if enable { - if err := sysinit.New(co.CP.Runner).EnableNow("auto-pause"); err != nil { - klog.ErrorS(err, "failed to enable", "service", "auto-pause") - } - } - - port := co.CP.Port // api server port - if enable { // if enable then need to calculate the forwarded port - port = constants.AutoPauseProxyPort - if driver.NeedsPortForward(cc.Driver) { - port, err = oci.ForwardedPort(cc.Driver, cc.Name, port) - if err != nil { - klog.ErrorS(err, "failed to get forwarded port for", "auto-pause port", port) - } - } - } - - updated, err := kubeconfig.UpdateEndpoint(cc.Name, co.CP.Hostname, port, kubeconfig.PathFromEnv(), kubeconfig.NewExtension()) - if err != nil { - klog.ErrorS(err, "failed to update kubeconfig", "auto-pause proxy endpoint") - return err - } - if updated { - klog.Infof("%s context has been updated to point to auto-pause proxy %s:%s", cc.Name, co.CP.Hostname, co.CP.Port) - } else { - klog.Info("no need to update kube-context for auto-pause proxy") - } - - return nil -} - -// enableOrDisableGCPAuth enables or disables the gcp-auth addon depending on the val parameter -func enableOrDisableGCPAuth(cfg *config.ClusterConfig, name string, val string) error { - enable, err := strconv.ParseBool(val) - if err != nil { - return errors.Wrapf(err, "parsing bool: %s", name) - } - if enable { - return enableAddonGCPAuth(cfg) - } - return disableAddonGCPAuth(cfg) -} - -func enableAddonGCPAuth(cfg *config.ClusterConfig) error { - if !Force && util.IsOnGCE() { - exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to authenticate using a credentials file, use the --force flag.") - } - - // Grab command runner from running cluster - cc := mustload.Running(cfg.Name) - r := cc.CP.Runner - - // Grab credentials from where GCP would normally look - ctx := context.Background() - creds, err := google.FindDefaultCredentials(ctx) - if err != nil { - exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") - } - - // Don't mount in empty credentials file - if creds.JSON == nil { - exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") - } - - f := assets.NewMemoryAssetTarget(creds.JSON, credentialsPath, "0444") - - err = r.Copy(f) - if err != nil { - return err - } - - // First check if the project env var is explicitly set - projectEnv := os.Getenv("GOOGLE_CLOUD_PROJECT") - if projectEnv != "" { - f := assets.NewMemoryAssetTarget([]byte(projectEnv), projectPath, "0444") - return r.Copy(f) - } - - // We're currently assuming gcloud is installed and in the user's path - project, err := exec.Command("gcloud", "config", "get-value", "project").Output() - if err == nil && len(project) > 0 { - f := assets.NewMemoryAssetTarget(bytes.TrimSpace(project), projectPath, "0444") - return r.Copy(f) - } - - out.WarningT("Could not determine a Google Cloud project, which might be ok.") - out.Styled(style.Tip, `To set your Google Cloud project, run: - - gcloud config set project - -or set the GOOGLE_CLOUD_PROJECT environment variable.`) - - // Copy an empty file in to avoid errors about missing files - emptyFile := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444") - return r.Copy(emptyFile) -} - -func disableAddonGCPAuth(cfg *config.ClusterConfig) error { - // Grab command runner from running cluster - cc := mustload.Running(cfg.Name) - r := cc.CP.Runner - - // Clean up the files generated when enabling the addon - creds := assets.NewMemoryAssetTarget([]byte{}, credentialsPath, "0444") - err := r.Remove(creds) - if err != nil { - return err - } - - project := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444") - err = r.Remove(project) - if err != nil { - return err - } - - return nil -} diff --git a/pkg/addons/autopause.go b/pkg/addons/autopause.go new file mode 100644 index 000000000000..cb50a037a6e5 --- /dev/null +++ b/pkg/addons/autopause.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import ( + "runtime" + "strconv" + + "github.com/pkg/errors" + "k8s.io/klog/v2" + "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/kubeconfig" + "k8s.io/minikube/pkg/minikube/mustload" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/reason" + "k8s.io/minikube/pkg/minikube/sysinit" +) + +// enableOrDisableAutoPause enables the service after the config was copied by generic enble +func enableOrDisableAutoPause(cc *config.ClusterConfig, name string, val string) error { + enable, err := strconv.ParseBool(val) + if err != nil { + return errors.Wrapf(err, "parsing bool: %s", name) + } + out.Infof("auto-pause addon is an alpha feature and still in early development. Please file issues to help us make it better.") + out.Infof("https://github.com/kubernetes/minikube/labels/co%2Fauto-pause") + + if !driver.IsKIC(cc.Driver) || runtime.GOARCH != "amd64" { + exit.Message(reason.Usage, `auto-pause currently is only supported on docker driver/docker runtime/amd64. Track progress of others here: https://github.com/kubernetes/minikube/issues/10601`) + } + co := mustload.Running(cc.Name) + if enable { + if err := sysinit.New(co.CP.Runner).EnableNow("auto-pause"); err != nil { + klog.ErrorS(err, "failed to enable", "service", "auto-pause") + } + } + + port := co.CP.Port // api server port + if enable { // if enable then need to calculate the forwarded port + port = constants.AutoPauseProxyPort + if driver.NeedsPortForward(cc.Driver) { + port, err = oci.ForwardedPort(cc.Driver, cc.Name, port) + if err != nil { + klog.ErrorS(err, "failed to get forwarded port for", "auto-pause port", port) + } + } + } + + updated, err := kubeconfig.UpdateEndpoint(cc.Name, co.CP.Hostname, port, kubeconfig.PathFromEnv(), kubeconfig.NewExtension()) + if err != nil { + klog.ErrorS(err, "failed to update kubeconfig", "auto-pause proxy endpoint") + return err + } + if updated { + klog.Infof("%s context has been updated to point to auto-pause proxy %s:%s", cc.Name, co.CP.Hostname, co.CP.Port) + } else { + klog.Info("no need to update kube-context for auto-pause proxy") + } + + return nil +} diff --git a/pkg/addons/gcpauth.go b/pkg/addons/gcpauth.go new file mode 100644 index 000000000000..5dfc0402048f --- /dev/null +++ b/pkg/addons/gcpauth.go @@ -0,0 +1,138 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import ( + "bytes" + "context" + "os" + "os/exec" + "strconv" + + "github.com/pkg/errors" + "golang.org/x/oauth2/google" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/mustload" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/reason" + "k8s.io/minikube/pkg/minikube/style" + "k8s.io/minikube/pkg/util" +) + +// enableOrDisableGCPAuth enables or disables the gcp-auth addon depending on the val parameter +func enableOrDisableGCPAuth(cfg *config.ClusterConfig, name string, val string) error { + enable, err := strconv.ParseBool(val) + if err != nil { + return errors.Wrapf(err, "parsing bool: %s", name) + } + if enable { + return enableAddonGCPAuth(cfg) + } + return disableAddonGCPAuth(cfg) +} + +func enableAddonGCPAuth(cfg *config.ClusterConfig) error { + if !Force && util.IsOnGCE() { + exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to authenticate using a credentials file, use the --force flag.") + } + + // Grab command runner from running cluster + cc := mustload.Running(cfg.Name) + r := cc.CP.Runner + + // Grab credentials from where GCP would normally look + ctx := context.Background() + creds, err := google.FindDefaultCredentials(ctx) + if err != nil { + exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") + } + + // Don't mount in empty credentials file + if creds.JSON == nil { + exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.") + } + + f := assets.NewMemoryAssetTarget(creds.JSON, credentialsPath, "0444") + + err = r.Copy(f) + if err != nil { + return err + } + + // First check if the project env var is explicitly set + projectEnv := os.Getenv("GOOGLE_CLOUD_PROJECT") + if projectEnv != "" { + f := assets.NewMemoryAssetTarget([]byte(projectEnv), projectPath, "0444") + return r.Copy(f) + } + + // We're currently assuming gcloud is installed and in the user's path + project, err := exec.Command("gcloud", "config", "get-value", "project").Output() + if err == nil && len(project) > 0 { + f := assets.NewMemoryAssetTarget(bytes.TrimSpace(project), projectPath, "0444") + return r.Copy(f) + } + + out.WarningT("Could not determine a Google Cloud project, which might be ok.") + out.Styled(style.Tip, `To set your Google Cloud project, run: + + gcloud config set project + +or set the GOOGLE_CLOUD_PROJECT environment variable.`) + + // Copy an empty file in to avoid errors about missing files + emptyFile := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444") + return r.Copy(emptyFile) +} + +func disableAddonGCPAuth(cfg *config.ClusterConfig) error { + // Grab command runner from running cluster + cc := mustload.Running(cfg.Name) + r := cc.CP.Runner + + // Clean up the files generated when enabling the addon + creds := assets.NewMemoryAssetTarget([]byte{}, credentialsPath, "0444") + err := r.Remove(creds) + if err != nil { + return err + } + + project := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444") + err = r.Remove(project) + if err != nil { + return err + } + + return nil +} + +func verifyGCPAuthAddon(cc *config.ClusterConfig, name string, val string) error { + enable, err := strconv.ParseBool(val) + if err != nil { + return errors.Wrapf(err, "parsing bool: %s", name) + } + err = verifyAddonStatusInternal(cc, name, val, "gcp-auth") + + if enable && err == nil { + out.Styled(style.Notice, "Your GCP credentials will now be mounted into every pod created in the {{.name}} cluster.", out.V{"name": cc.Name}) + out.Styled(style.Notice, "If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.") + } + + return err +} diff --git a/pkg/addons/storage_classes.go b/pkg/addons/storage_classes.go new file mode 100644 index 000000000000..6b23c8aff01d --- /dev/null +++ b/pkg/addons/storage_classes.go @@ -0,0 +1,77 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addons + +import ( + "strconv" + + "github.com/pkg/errors" + "k8s.io/klog/v2" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/storageclass" +) + +// enableOrDisableStorageClasses enables or disables storage classes +func enableOrDisableStorageClasses(cc *config.ClusterConfig, name string, val string) error { + klog.Infof("enableOrDisableStorageClasses %s=%v on %q", name, val, cc.Name) + enable, err := strconv.ParseBool(val) + if err != nil { + return errors.Wrap(err, "Error parsing boolean") + } + + class := defaultStorageClassProvisioner + if name == "storage-provisioner-gluster" { + class = "glusterfile" + } + + api, err := machine.NewAPIClient() + if err != nil { + return errors.Wrap(err, "machine client") + } + defer api.Close() + + cp, err := config.PrimaryControlPlane(cc) + if err != nil { + return errors.Wrap(err, "getting control plane") + } + if !machine.IsRunning(api, config.MachineName(*cc, cp)) { + klog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement", config.MachineName(*cc, cp), name, val) + return EnableOrDisableAddon(cc, name, val) + } + + storagev1, err := storageclass.GetStoragev1(cc.Name) + if err != nil { + return errors.Wrapf(err, "Error getting storagev1 interface %v ", err) + } + + if enable { + // Only StorageClass for 'name' should be marked as default + err = storageclass.SetDefaultStorageClass(storagev1, class) + if err != nil { + return errors.Wrapf(err, "Error making %s the default storage class", class) + } + } else { + // Unset the StorageClass as default + err := storageclass.DisableDefaultStorageClass(storagev1, class) + if err != nil { + return errors.Wrapf(err, "Error disabling %s as the default storage class", class) + } + } + + return EnableOrDisableAddon(cc, name, val) +} From a3544ae90acf6a1d6ed7676705e0404323a85be0 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 15:11:54 -0800 Subject: [PATCH 13/18] create detect package --- cmd/minikube/cmd/root.go | 4 +-- pkg/addons/gcpauth.go | 4 +-- pkg/minikube/detect/detect.go | 43 +++++++++++++++++++++++++++++++++ pkg/minikube/driver/driver.go | 10 ++------ pkg/util/utils.go | 15 ------------ test/integration/addons_test.go | 6 ++--- test/integration/main_test.go | 4 +-- 7 files changed, 54 insertions(+), 32 deletions(-) create mode 100644 pkg/minikube/detect/detect.go diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go index 590e523d60ad..d57f6a0e3926 100644 --- a/cmd/minikube/cmd/root.go +++ b/cmd/minikube/cmd/root.go @@ -35,7 +35,7 @@ import ( "k8s.io/minikube/pkg/minikube/audit" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" - "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/detect" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" @@ -80,7 +80,7 @@ func Execute() { defer audit.Log(time.Now()) // Check whether this is a windows binary (.exe) running inisde WSL. - if runtime.GOOS == "windows" && driver.IsMicrosoftWSL() { + if runtime.GOOS == "windows" && detect.IsMicrosoftWSL() { var found = false for _, a := range os.Args { if a == "--force" { diff --git a/pkg/addons/gcpauth.go b/pkg/addons/gcpauth.go index 5dfc0402048f..e5a3a208554b 100644 --- a/pkg/addons/gcpauth.go +++ b/pkg/addons/gcpauth.go @@ -27,12 +27,12 @@ import ( "golang.org/x/oauth2/google" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/detect" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/reason" "k8s.io/minikube/pkg/minikube/style" - "k8s.io/minikube/pkg/util" ) // enableOrDisableGCPAuth enables or disables the gcp-auth addon depending on the val parameter @@ -48,7 +48,7 @@ func enableOrDisableGCPAuth(cfg *config.ClusterConfig, name string, val string) } func enableAddonGCPAuth(cfg *config.ClusterConfig) error { - if !Force && util.IsOnGCE() { + if !Force && detect.IsOnGCE() { exit.Message(reason.InternalCredsNotFound, "It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to authenticate using a credentials file, use the --force flag.") } diff --git a/pkg/minikube/detect/detect.go b/pkg/minikube/detect/detect.go new file mode 100644 index 000000000000..099bf4ad7167 --- /dev/null +++ b/pkg/minikube/detect/detect.go @@ -0,0 +1,43 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package detect + +import ( + "net/http" + "os" +) + +// IsMicrosoftWSL will return true if process is running in WSL in windows +// checking for WSL env var based on this https://github.com/microsoft/WSL/issues/423#issuecomment-608237689 +// also based on https://github.com/microsoft/vscode/blob/90a39ba0d49d75e9a4d7e62a6121ad946ecebc58/resources/win32/bin/code.sh#L24 +func IsMicrosoftWSL() bool { + return os.Getenv("WSL_DISTRO_NAME") != "" || os.Getenv("WSLPATH") != "" +} + +// IsOnGCE determines whether minikube is currently running on GCE. +func IsOnGCE() bool { + resp, err := http.Get("http://metadata.google.internal") + if err != nil { + return false + } + + if resp.Header.Get("Metadata-Flavor") == "Google" { + return true + } + + return false +} diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 2edc1ef742d4..e574493c9c2e 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -26,6 +26,7 @@ import ( "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/detect" "k8s.io/minikube/pkg/minikube/registry" ) @@ -167,14 +168,7 @@ func NeedsPortForward(name string) bool { return true } // Docker for Desktop - return runtime.GOOS == "darwin" || runtime.GOOS == "windows" || IsMicrosoftWSL() -} - -// IsMicrosoftWSL will return true if process is running in WSL in windows -// checking for WSL env var based on this https://github.com/microsoft/WSL/issues/423#issuecomment-608237689 -// also based on https://github.com/microsoft/vscode/blob/90a39ba0d49d75e9a4d7e62a6121ad946ecebc58/resources/win32/bin/code.sh#L24 -func IsMicrosoftWSL() bool { - return os.Getenv("WSL_DISTRO_NAME") != "" || os.Getenv("WSLPATH") != "" + return runtime.GOOS == "darwin" || runtime.GOOS == "windows" || detect.IsMicrosoftWSL() } // HasResourceLimits returns true if driver can set resource limits such as memory size or CPU count. diff --git a/pkg/util/utils.go b/pkg/util/utils.go index de32aa4da884..f5fe5a3b226a 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -18,7 +18,6 @@ package util import ( "fmt" - "net/http" "os" "os/user" "path/filepath" @@ -110,17 +109,3 @@ func MaybeChownDirRecursiveToMinikubeUser(dir string) error { func ParseKubernetesVersion(version string) (semver.Version, error) { return semver.Make(version[1:]) } - -// IsOnGCE determines whether minikube is currently running on GCE. -func IsOnGCE() bool { - resp, err := http.Get("http://metadata.google.internal") - if err != nil { - return false - } - - if resp.Header.Get("Metadata-Flavor") == "Google" { - return true - } - - return false -} diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 6dd57df85147..00aa39957231 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -36,7 +36,7 @@ import ( "github.com/hashicorp/go-retryablehttp" "k8s.io/minikube/pkg/kapi" - "k8s.io/minikube/pkg/util" + "k8s.io/minikube/pkg/minikube/detect" "k8s.io/minikube/pkg/util/retry" ) @@ -66,7 +66,7 @@ func TestAddons(t *testing.T) { if !arm64Platform() { args = append(args, "--addons=helm-tiller") } - if !util.IsOnGCE() { + if !detect.IsOnGCE() { args = append(args, "--addons=gcp-auth") } rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) @@ -76,7 +76,7 @@ func TestAddons(t *testing.T) { // If we're running the integration tests on GCE, which is frequently the case, first check to make sure we exit out properly, // then use force to actually test using creds. - if util.IsOnGCE() { + if detect.IsOnGCE() { args = append([]string{"addons", "enable", "gcp-auth"}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err == nil { diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 1a72688bb748..3e1003df0b08 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -30,7 +30,7 @@ import ( "time" "k8s.io/minikube/pkg/minikube/constants" - "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/detect" ) // General configuration: used to set the VM Driver @@ -169,7 +169,7 @@ func arm64Platform() bool { // NeedsPortForward returns access to endpoints with this driver needs port forwarding // (Docker on non-Linux platforms requires ports to be forwarded to 127.0.0.1) func NeedsPortForward() bool { - return KicDriver() && (runtime.GOOS == "windows" || runtime.GOOS == "darwin") || driver.IsMicrosoftWSL() + return KicDriver() && (runtime.GOOS == "windows" || runtime.GOOS == "darwin") || detect.IsMicrosoftWSL() } // CanCleanup returns if cleanup is allowed From aee9afe2231c1860f60f43f1df65a659eaf59d3e Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 15:20:02 -0800 Subject: [PATCH 14/18] move constants to proper file --- pkg/addons/addons.go | 6 ------ pkg/addons/gcpauth.go | 5 +++++ pkg/addons/storage_classes.go | 2 ++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 1ebfef1a869c..34ae91c0faa1 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -47,12 +47,6 @@ import ( "k8s.io/minikube/pkg/util/retry" ) -const ( - credentialsPath = "/var/lib/minikube/google_application_credentials.json" - projectPath = "/var/lib/minikube/google_cloud_project" - defaultStorageClassProvisioner = "standard" -) - // Force is used to override checks for addons var Force bool = false diff --git a/pkg/addons/gcpauth.go b/pkg/addons/gcpauth.go index e5a3a208554b..b5d54cb55821 100644 --- a/pkg/addons/gcpauth.go +++ b/pkg/addons/gcpauth.go @@ -35,6 +35,11 @@ import ( "k8s.io/minikube/pkg/minikube/style" ) +const ( + credentialsPath = "/var/lib/minikube/google_application_credentials.json" + projectPath = "/var/lib/minikube/google_cloud_project" +) + // enableOrDisableGCPAuth enables or disables the gcp-auth addon depending on the val parameter func enableOrDisableGCPAuth(cfg *config.ClusterConfig, name string, val string) error { enable, err := strconv.ParseBool(val) diff --git a/pkg/addons/storage_classes.go b/pkg/addons/storage_classes.go index 6b23c8aff01d..f25b1187d34d 100644 --- a/pkg/addons/storage_classes.go +++ b/pkg/addons/storage_classes.go @@ -26,6 +26,8 @@ import ( "k8s.io/minikube/pkg/minikube/storageclass" ) +const defaultStorageClassProvisioner = "standard" + // enableOrDisableStorageClasses enables or disables storage classes func enableOrDisableStorageClasses(cc *config.ClusterConfig, name string, val string) error { klog.Infof("enableOrDisableStorageClasses %s=%v on %q", name, val, cc.Name) From fa3173779285ccf6af2c0fb51ab88ce1a2addfcb Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 15:32:49 -0800 Subject: [PATCH 15/18] fix boilerplate date --- pkg/addons/autopause.go | 2 +- pkg/addons/storage_classes.go | 2 +- pkg/minikube/detect/detect.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/addons/autopause.go b/pkg/addons/autopause.go index cb50a037a6e5..936f51ff07e8 100644 --- a/pkg/addons/autopause.go +++ b/pkg/addons/autopause.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Kubernetes Authors All rights reserved. +Copyright 2021 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/addons/storage_classes.go b/pkg/addons/storage_classes.go index f25b1187d34d..cd8b67272900 100644 --- a/pkg/addons/storage_classes.go +++ b/pkg/addons/storage_classes.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Kubernetes Authors All rights reserved. +Copyright 2021 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/minikube/detect/detect.go b/pkg/minikube/detect/detect.go index 099bf4ad7167..b3af332d4fe3 100644 --- a/pkg/minikube/detect/detect.go +++ b/pkg/minikube/detect/detect.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Kubernetes Authors All rights reserved. +Copyright 2021 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 3063e9e720f8ac1d763b520e496d37888b9d0281 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 9 Mar 2021 15:54:30 -0800 Subject: [PATCH 16/18] change filenames to be more descriptive --- pkg/addons/{autopause.go => addons_autopause.go} | 0 pkg/addons/{gcpauth.go => addons_gcpauth.go} | 0 pkg/addons/{storage_classes.go => addons_storage_classes.go} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename pkg/addons/{autopause.go => addons_autopause.go} (100%) rename pkg/addons/{gcpauth.go => addons_gcpauth.go} (100%) rename pkg/addons/{storage_classes.go => addons_storage_classes.go} (100%) diff --git a/pkg/addons/autopause.go b/pkg/addons/addons_autopause.go similarity index 100% rename from pkg/addons/autopause.go rename to pkg/addons/addons_autopause.go diff --git a/pkg/addons/gcpauth.go b/pkg/addons/addons_gcpauth.go similarity index 100% rename from pkg/addons/gcpauth.go rename to pkg/addons/addons_gcpauth.go diff --git a/pkg/addons/storage_classes.go b/pkg/addons/addons_storage_classes.go similarity index 100% rename from pkg/addons/storage_classes.go rename to pkg/addons/addons_storage_classes.go From db79518adc552b98d08e4516ff60b52ed9d9bb89 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 10 Mar 2021 09:09:04 -0800 Subject: [PATCH 17/18] fix gcp auth test --- test/integration/addons_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 00aa39957231..9c3a597d4ba2 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -77,7 +77,7 @@ func TestAddons(t *testing.T) { // If we're running the integration tests on GCE, which is frequently the case, first check to make sure we exit out properly, // then use force to actually test using creds. if detect.IsOnGCE() { - args = append([]string{"addons", "enable", "gcp-auth"}, StartArgs()...) + args = append([]string{"addons", "enable", "gcp-auth", "-p", profile}, StartArgs()...) rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err == nil { t.Errorf("Expected error but didn't get one. command %v, output %v", rr.Command(), rr.Output()) From 86080e50b669a43e218bf0a73349f81cdbd7a4a9 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Wed, 10 Mar 2021 13:57:24 -0800 Subject: [PATCH 18/18] fix gcp-auth test --- test/integration/addons_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 9c3a597d4ba2..7f9428e09eea 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -77,7 +77,7 @@ func TestAddons(t *testing.T) { // If we're running the integration tests on GCE, which is frequently the case, first check to make sure we exit out properly, // then use force to actually test using creds. if detect.IsOnGCE() { - args = append([]string{"addons", "enable", "gcp-auth", "-p", profile}, StartArgs()...) + args = []string{"-p", profile, "addons", "enable", "gcp-auth"} rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) if err == nil { t.Errorf("Expected error but didn't get one. command %v, output %v", rr.Command(), rr.Output())