From 437273a3d84ed928edac85c6c8cbb21de2e5991d Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 22 Nov 2018 00:45:41 -0800 Subject: [PATCH 1/5] config/.../eks: set download paths for kubectl, aws-iam=authenticator Signed-off-by: Gyuho Lee --- config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml b/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml index 5d1da6f346b7..75da7061f780 100644 --- a/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml +++ b/config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml @@ -4,13 +4,19 @@ presets: # URL to download the latest 'aws-k8s-tester' release - name: AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_DOWNLOAD_URL value: https://github.com/aws/aws-k8s-tester/releases/download/0.1.3/aws-k8s-tester-0.1.3-linux-amd64 + - name: AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_DOWNLOAD_PATH + value: /tmp/aws-k8s-tester/aws-k8s-tester # URL to download 'kubectl', required for 'kubectl' calls to EKS # TODO: use upstream 'kubectl' - name: AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_URL value: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/kubectl + - name: AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_PATH + value: /tmp/aws-k8s-tester/kubectl # URL to download 'aws-iam-authenticator', required for 'kubectl' calls to EKS - name: AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_DOWNLOAD_URL value: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator + - name: AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_DOWNLOAD_PATH + value: /tmp/aws-k8s-tester/aws-iam-authenticator # test mode is either "embedded" or "aws-cli" ("embedded" uses native AWS Go client, "aws-cli" will use 'aws') - name: AWS_K8S_TESTER_EKS_TEST_MODE value: "embedded" From 2a768214e4210eed4f0586d34bdd5ce470b6197d Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 22 Nov 2018 01:07:05 -0800 Subject: [PATCH 2/5] kubetest/eks: use "AWSK8sTesterPath" to download test plugins Signed-off-by: Gyuho Lee --- kubetest/eks/eks.go | 60 +++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/kubetest/eks/eks.go b/kubetest/eks/eks.go index 4f1ad782df96..3a4cf8b27d39 100644 --- a/kubetest/eks/eks.go +++ b/kubetest/eks/eks.go @@ -27,7 +27,6 @@ import ( "net/http" "os" "os/exec" - "path/filepath" "syscall" "time" @@ -42,10 +41,9 @@ import ( // Satisfies "k8s.io/test-infra/kubetest/main.go" 'deployer' and 'publisher" interfaces. // Reference https://github.com/kubernetes/test-infra/blob/master/kubetest/main.go. type deployer struct { - stopc chan struct{} - cfg *eksconfig.Config - awsK8sTesterPath string - ctrl *process.Control + stopc chan struct{} + cfg *eksconfig.Config + ctrl *process.Control } // NewDeployer creates a new EKS deployer. @@ -56,15 +54,13 @@ func NewDeployer(timeout time.Duration, verbose bool) (ekstester.Deployer, error return nil, err } var f *os.File - f, err = ioutil.TempFile(os.TempDir(), "aws-k8s-tester") + f, err = ioutil.TempFile(os.TempDir(), "aws-k8s-tester-config") if err != nil { return nil, err } - outputPath := f.Name() - f.Close() - cfg.ConfigPath, err = filepath.Abs(outputPath) - if err != nil { - return nil, err + cfg.ConfigPath = f.Name() + if err = f.Close(); err != nil { + return nil, fmt.Errorf("failed to close aws-k8s-tester-config file %v", err) } if err = cfg.Sync(); err != nil { return nil, err @@ -81,22 +77,22 @@ func NewDeployer(timeout time.Duration, verbose bool) (ekstester.Deployer, error ), } - dp.awsK8sTesterPath, err = exec.LookPath("aws-k8s-tester") + if err = os.RemoveAll(cfg.AWSK8sTesterPath); err != nil { + return nil, err + } + f, err = os.Create(cfg.AWSK8sTesterPath) if err != nil { - var f *os.File - f, err = ioutil.TempFile(os.TempDir(), "aws-k8s-tester") - if err != nil { - return nil, fmt.Errorf("failed to create %q (%v)", dp.awsK8sTesterPath, err) - } - dp.awsK8sTesterPath = f.Name() - dp.awsK8sTesterPath, _ = filepath.Abs(dp.awsK8sTesterPath) - if err = httpRead(cfg.AWSK8sTesterDownloadURL, f); err != nil { - return nil, err - } - f.Close() - if err = util.EnsureExecutable(dp.awsK8sTesterPath); err != nil { - return nil, err - } + return nil, fmt.Errorf("failed to create %q (%v)", cfg.AWSK8sTesterPath, err) + } + cfg.AWSK8sTesterPath = f.Name() + if err = httpRead(cfg.AWSK8sTesterDownloadURL, f); err != nil { + return nil, err + } + if err = f.Close(); err != nil { + return nil, fmt.Errorf("failed to close aws-k8s-tester file %v", err) + } + if err = util.EnsureExecutable(cfg.AWSK8sTesterPath); err != nil { + return nil, err } return dp, nil } @@ -107,7 +103,7 @@ func (dp *deployer) Up() (err error) { // in the configuration file (e.g. VPC ID, ALB DNS names, etc.) // this needs be reloaded for other deployer method calls createCmd := exec.Command( - dp.awsK8sTesterPath, + dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, "create", @@ -135,7 +131,7 @@ func (dp *deployer) Down() (err error) { return err } _, err = dp.ctrl.Output(exec.Command( - dp.awsK8sTesterPath, + dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, "delete", @@ -151,7 +147,7 @@ func (dp *deployer) IsUp() (err error) { return err } _, err = dp.ctrl.Output(exec.Command( - dp.awsK8sTesterPath, + dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, "check", @@ -192,7 +188,7 @@ func (dp *deployer) GetWorkerNodeLogs() (err error) { return err } _, err = dp.ctrl.Output(exec.Command( - dp.awsK8sTesterPath, + dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, "test", "get-worker-node-logs", @@ -209,7 +205,7 @@ func (dp *deployer) DumpClusterLogs(artifactDir, _ string) (err error) { return err } _, err = dp.ctrl.Output(exec.Command( - dp.awsK8sTesterPath, + dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, "test", "get-worker-node-logs", @@ -218,7 +214,7 @@ func (dp *deployer) DumpClusterLogs(artifactDir, _ string) (err error) { return err } _, err = dp.ctrl.Output(exec.Command( - dp.awsK8sTesterPath, + dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, "test", "dump-cluster-logs", From 9af8afc58d48a8ae54f94ba037e5bdda9a975753 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 29 Nov 2018 15:08:59 -0800 Subject: [PATCH 3/5] kubetest/eks: add "KubectlCommand" to EKS provider Signed-off-by: Gyuho Lee --- kubetest/eks/eks.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/kubetest/eks/eks.go b/kubetest/eks/eks.go index 3a4cf8b27d39..92f76091f8e1 100644 --- a/kubetest/eks/eks.go +++ b/kubetest/eks/eks.go @@ -26,13 +26,12 @@ import ( "log" "net/http" "os" - "os/exec" + osexec "os/exec" "syscall" "time" "github.com/aws/aws-k8s-tester/eksconfig" "github.com/aws/aws-k8s-tester/ekstester" - "k8s.io/test-infra/kubetest/process" "k8s.io/test-infra/kubetest/util" ) @@ -102,7 +101,7 @@ func (dp *deployer) Up() (err error) { // "create cluster" command outputs cluster information // in the configuration file (e.g. VPC ID, ALB DNS names, etc.) // this needs be reloaded for other deployer method calls - createCmd := exec.Command( + createCmd := osexec.Command( dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, @@ -130,7 +129,7 @@ func (dp *deployer) Down() (err error) { if _, err = dp.LoadConfig(); err != nil { return err } - _, err = dp.ctrl.Output(exec.Command( + _, err = dp.ctrl.Output(osexec.Command( dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, @@ -146,7 +145,7 @@ func (dp *deployer) IsUp() (err error) { if _, err = dp.LoadConfig(); err != nil { return err } - _, err = dp.ctrl.Output(exec.Command( + _, err = dp.ctrl.Output(osexec.Command( dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, @@ -187,7 +186,7 @@ func (dp *deployer) GetWorkerNodeLogs() (err error) { if _, err = dp.LoadConfig(); err != nil { return err } - _, err = dp.ctrl.Output(exec.Command( + _, err = dp.ctrl.Output(osexec.Command( dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, @@ -204,7 +203,7 @@ func (dp *deployer) DumpClusterLogs(artifactDir, _ string) (err error) { if _, err = dp.LoadConfig(); err != nil { return err } - _, err = dp.ctrl.Output(exec.Command( + _, err = dp.ctrl.Output(osexec.Command( dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, @@ -213,7 +212,7 @@ func (dp *deployer) DumpClusterLogs(artifactDir, _ string) (err error) { if err != nil { return err } - _, err = dp.ctrl.Output(exec.Command( + _, err = dp.ctrl.Output(osexec.Command( dp.cfg.AWSK8sTesterPath, "eks", "--path="+dp.cfg.ConfigPath, @@ -223,6 +222,15 @@ func (dp *deployer) DumpClusterLogs(artifactDir, _ string) (err error) { return err } +// KubectlCommand returns "kubectl" command object for API reachability tests. +func (dp *deployer) KubectlCommand() (*osexec.Cmd, error) { + // reload configuration from disk to read the latest configuration + if _, err := dp.LoadConfig(); err != nil { + return nil, err + } + return osexec.Command(dp.cfg.KubectlPath, "--kubeconfig="+dp.cfg.KubeConfigPath), nil +} + // Stop stops ongoing operations. // This is useful for local development. // For example, one may run "Up" but have to cancel ongoing "Up" From d6e2af93ebe49eb8347360e5335a980d011e26fa Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 29 Nov 2018 15:35:43 -0800 Subject: [PATCH 4/5] kubetest: implement "KubectlCommand" Signed-off-by: Gyuho Lee --- kubetest/anywhere.go | 4 ++++ kubetest/azure.go | 2 ++ kubetest/bash.go | 2 ++ kubetest/conformance/conformance.go | 7 +++++++ kubetest/e2e.go | 19 ++++++++++++++----- kubetest/gke.go | 2 ++ kubetest/kops.go | 2 ++ kubetest/kubeadmdind/kubeadm_dind.go | 2 ++ kubetest/local.go | 2 ++ kubetest/main.go | 1 + kubetest/node.go | 6 ++++++ kubetest/none.go | 6 ++++++ 12 files changed, 50 insertions(+), 5 deletions(-) diff --git a/kubetest/anywhere.go b/kubetest/anywhere.go index 734749040a92..b86c951220db 100644 --- a/kubetest/anywhere.go +++ b/kubetest/anywhere.go @@ -291,6 +291,8 @@ func (k *kubernetesAnywhere) GetClusterCreated(gcpProject string) (time.Time, er return time.Time{}, errors.New("not implemented") } +func (_ *kubernetesAnywhere) KubectlCommand() (*exec.Cmd, error) { return nil, nil } + const defaultConfigFile = ".config" type kubernetesAnywhereMultiCluster struct { @@ -429,3 +431,5 @@ func (k *kubernetesAnywhereMultiCluster) Down() error { } return control.FinishRunningParallel(cmds...) } + +func (_ *kubernetesAnywhereMultiCluster) KubectlCommand() (*exec.Cmd, error) { return nil, nil } diff --git a/kubetest/azure.go b/kubetest/azure.go index fca7aefcebc2..2c3596f0e6c6 100644 --- a/kubetest/azure.go +++ b/kubetest/azure.go @@ -604,3 +604,5 @@ func (c Cluster) TestSetup() error { func (c Cluster) IsUp() error { return isUp(c) } + +func (_ Cluster) KubectlCommand() (*exec.Cmd, error) { return nil, nil } diff --git a/kubetest/bash.go b/kubetest/bash.go index ca263eaea9e8..a91225ed59be 100644 --- a/kubetest/bash.go +++ b/kubetest/bash.go @@ -102,6 +102,8 @@ func (b *bashDeployer) GetClusterCreated(gcpProject string) (time.Time, error) { return created, nil } +func (_ *bashDeployer) KubectlCommand() (*exec.Cmd, error) { return nil, nil } + // Calculates the cluster IP range based on the no. of nodes in the cluster. // Note: This mimics the function get-cluster-ip-range used by kube-up script. func getClusterIPRange(numNodes int) string { diff --git a/kubetest/conformance/conformance.go b/kubetest/conformance/conformance.go index d3f89f2235b2..4688826987b0 100644 --- a/kubetest/conformance/conformance.go +++ b/kubetest/conformance/conformance.go @@ -19,7 +19,9 @@ package conformance import ( "fmt" + "log" "os" + "os/exec" "time" "k8s.io/api/core/v1" @@ -132,3 +134,8 @@ func (d *Deployer) Down() error { func (d *Deployer) GetClusterCreated(gcpProject string) (time.Time, error) { return time.Time{}, fmt.Errorf("cannot get cluster create time for conformance cluster") } + +func (_ *Deployer) KubectlCommand() (*exec.Cmd, error) { + log.Print("Noop - Conformance KubectlCommand()") + return nil, nil +} diff --git a/kubetest/e2e.go b/kubetest/e2e.go index 5f68912ebf9f..fda24ae3d0c9 100644 --- a/kubetest/e2e.go +++ b/kubetest/e2e.go @@ -147,7 +147,7 @@ func run(deploy deployer, o options) error { // If node testing is enabled, check that the api is reachable before // proceeding with further steps. This is accomplished by listing the nodes. if !o.nodeTests { - errs = util.AppendError(errs, control.XMLWrap(&suite, "Check APIReachability", getKubectlVersion)) + errs = util.AppendError(errs, control.XMLWrap(&suite, "Check APIReachability", func() error { return getKubectlVersion(deploy) })) if dump != "" { errs = util.AppendError(errs, control.XMLWrap(&suite, "list nodes", func() error { return listNodes(dump) @@ -201,7 +201,7 @@ func run(deploy deployer, o options) error { })) } else { if o.deployment != "conformance" { - errs = util.AppendError(errs, control.XMLWrap(&suite, "kubectl version", getKubectlVersion)) + errs = util.AppendError(errs, control.XMLWrap(&suite, "kubectl version", func() error { return getKubectlVersion(deploy) })) } if o.skew { @@ -337,10 +337,19 @@ func run(deploy deployer, o options) error { return nil } -func getKubectlVersion() error { +func getKubectlVersion(dp deployer) error { + cmd, err := dp.KubectlCommand() + if err != nil { + return err + } + if cmd == nil { + cmd = exec.Command("./cluster/kubectl.sh") + } + cmd.Args = append(cmd.Args, "--match-server-version=false", "version") + copied := *cmd retries := 5 for { - _, err := control.Output(exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "version")) + _, err := control.Output(&copied) if err == nil { return nil } @@ -348,7 +357,7 @@ func getKubectlVersion() error { if retries == 0 { return err } - log.Print("Failed to reach api. Sleeping for 10 seconds before retrying...") + log.Printf("Failed to reach api. Sleeping for 10 seconds before retrying... (%v)", copied.Args) time.Sleep(10 * time.Second) } } diff --git a/kubetest/gke.go b/kubetest/gke.go index 2e4a5a0ebc15..db6cc72ef962 100644 --- a/kubetest/gke.go +++ b/kubetest/gke.go @@ -652,3 +652,5 @@ func (g *gkeDeployer) GetClusterCreated(gcpProject string) (time.Time, error) { } return created, nil } + +func (_ *gkeDeployer) KubectlCommand() (*exec.Cmd, error) { return nil, nil } diff --git a/kubetest/kops.go b/kubetest/kops.go index f2c2ff823183..0d9e45b26415 100644 --- a/kubetest/kops.go +++ b/kubetest/kops.go @@ -653,6 +653,8 @@ func (k kops) Publish() error { }) } +func (_ kops) KubectlCommand() (*exec.Cmd, error) { return nil, nil } + // getRandomAWSZones looks up all regions, and the availability zones for those regions. A random // region is then chosen and the AZ's for that region is returned. At least masterCount zones will be // returned, all in the same region. diff --git a/kubetest/kubeadmdind/kubeadm_dind.go b/kubetest/kubeadmdind/kubeadm_dind.go index 798f544873ac..a7a766bc2511 100644 --- a/kubetest/kubeadmdind/kubeadm_dind.go +++ b/kubetest/kubeadmdind/kubeadm_dind.go @@ -312,6 +312,8 @@ func (d *Deployer) GetClusterCreated(gcpProject string) (time.Time, error) { return time.Time{}, errors.New("not implemented") } +func (_ *Deployer) KubectlCommand() (*exec.Cmd, error) { return nil, nil } + // findPath looks for the existence of a file or directory based on a // a github organization, github repo, and a relative path. It looks // for the file/directory in this order: diff --git a/kubetest/local.go b/kubetest/local.go index 7d050cf6bf10..90ac7e520353 100644 --- a/kubetest/local.go +++ b/kubetest/local.go @@ -234,3 +234,5 @@ func (n localCluster) Down() error { func (n localCluster) GetClusterCreated(gcpProject string) (time.Time, error) { return time.Time{}, errors.New("GetClusterCreated not implemented in localCluster") } + +func (_ localCluster) KubectlCommand() (*exec.Cmd, error) { return nil, nil } diff --git a/kubetest/main.go b/kubetest/main.go index b9a3a1bc7485..7f0b921d601b 100644 --- a/kubetest/main.go +++ b/kubetest/main.go @@ -226,6 +226,7 @@ type deployer interface { TestSetup() error Down() error GetClusterCreated(gcpProject string) (time.Time, error) + KubectlCommand() (*exec.Cmd, error) } // publisher is implemented by deployers that want to publish status on success diff --git a/kubetest/node.go b/kubetest/node.go index 71ce07f24906..b97a21146e54 100644 --- a/kubetest/node.go +++ b/kubetest/node.go @@ -19,6 +19,7 @@ package main import ( "errors" "log" + "os/exec" "time" ) @@ -55,3 +56,8 @@ func (n nodeDeploy) Down() error { func (n nodeDeploy) GetClusterCreated(gcpProject string) (time.Time, error) { return time.Time{}, errors.New("not implemented") } + +func (_ nodeDeploy) KubectlCommand() (*exec.Cmd, error) { + log.Print("Noop - Node KubectlCommand()") + return nil, nil +} diff --git a/kubetest/none.go b/kubetest/none.go index d4d8d183607e..f82d47f3041c 100644 --- a/kubetest/none.go +++ b/kubetest/none.go @@ -19,6 +19,7 @@ package main import ( "errors" "log" + "os/exec" "time" ) @@ -53,3 +54,8 @@ func (n noneDeploy) Down() error { func (n noneDeploy) GetClusterCreated(gcpProject string) (time.Time, error) { return time.Time{}, errors.New("not implemented") } + +func (_ noneDeploy) KubectlCommand() (*exec.Cmd, error) { + log.Print("Noop KubectlCommand()") + return nil, nil +} From f6b86a0ea17a4751ef2de5270d0bbb0b503410c9 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 29 Nov 2018 15:45:07 -0800 Subject: [PATCH 5/5] vendor: upgrade "aws/aws-k8s-tester" Signed-off-by: Gyuho Lee --- Gopkg.lock | 4 +- .../aws/aws-k8s-tester/ec2config/BUILD.bazel | 2 +- .../aws/aws-k8s-tester/ec2config/config.go | 12 +-- .../ec2config/plugins/plugins.go | 79 ++----------------- .../aws/aws-k8s-tester/eksconfig/BUILD.bazel | 2 +- .../aws/aws-k8s-tester/eksconfig/config.go | 78 +++++++++--------- .../aws/aws-k8s-tester/ekstester/tester.go | 4 + 7 files changed, 65 insertions(+), 116 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 628fa55d9ce2..8eef75cd7cd4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -135,7 +135,7 @@ revision = "1fca145dffbcaa8fe914309b1ec0cfc67500fe61" [[projects]] - digest = "1:1baf5324beded9ad7e1c5b808b89cd3a3c58cb7ae393b168f4f14e516c65f28b" + digest = "1:d47a7946633fc621db8587690253bba78ee390586bc84b94e56831def0a116a5" name = "github.com/aws/aws-k8s-tester" packages = [ "ec2config", @@ -147,7 +147,7 @@ "pkg/awsapi/ec2", ] pruneopts = "NUT" - revision = "7a149356ae339b531b6189f4a801378b62c7b98b" + revision = "4349ca61b2788db7b3197856c7ccf5cd615d70aa" version = "0.1.3" [[projects]] diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/BUILD.bazel b/vendor/github.com/aws/aws-k8s-tester/ec2config/BUILD.bazel index f2e688162739..9e7a6449a004 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/BUILD.bazel +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/BUILD.bazel @@ -9,7 +9,7 @@ go_library( deps = [ "//vendor/github.com/aws/aws-k8s-tester/ec2config/plugins:go_default_library", "//vendor/github.com/aws/aws-k8s-tester/pkg/awsapi/ec2:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go b/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go index 171777d43c6c..0683c381668d 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/config.go @@ -15,8 +15,7 @@ import ( "github.com/aws/aws-k8s-tester/ec2config/plugins" ec2types "github.com/aws/aws-k8s-tester/pkg/awsapi/ec2" - - gyaml "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" ) // Config defines EC2 configuration. @@ -119,7 +118,7 @@ type Config struct { // If empty, it will fetch subnets from a given or created VPC. // And randomly assign them to instances. SubnetIDs []string `json:"subnet-ids,omitempty"` - SubnetIDToAvailibilityZone map[string]string `json:"subnet-id-to-availability-zone,omitempty"` // read-only to user + SubnetIDToAvailabilityZone map[string]string `json:"subnet-id-to-availability-zone,omitempty"` // read-only to user // IngressRulesTCP is a map from TCP port range to CIDR to allow via security groups. IngressRulesTCP map[string]string `json:"ingress-rules-tcp,omitempty"` @@ -136,6 +135,9 @@ type Config struct { // Wait is true to wait until all EC2 instances are ready. Wait bool `json:"wait"` + + // InstanceProfileName is the name of an instance profile with permissions to manage EC2 instances. + InstanceProfileName string `json:"instance-profile-name,omitempty"` } // Instance represents an EC2 instance. @@ -448,7 +450,7 @@ func Load(p string) (cfg *Config, err error) { return nil, err } cfg = new(Config) - if err = gyaml.Unmarshal(d, cfg); err != nil { + if err = yaml.Unmarshal(d, cfg); err != nil { return nil, err } @@ -485,7 +487,7 @@ func (cfg *Config) Sync() (err error) { } cfg.UpdatedAt = time.Now().UTC() var d []byte - d, err = gyaml.Marshal(cfg) + d, err = yaml.Marshal(cfg) if err != nil { return err } diff --git a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go index 7787e14ad759..0342cb4dc288 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go +++ b/vendor/github.com/aws/aws-k8s-tester/ec2config/plugins/plugins.go @@ -5,8 +5,6 @@ package plugins import ( "bytes" "fmt" - "io/ioutil" - "os" "sort" "strconv" "strings" @@ -36,16 +34,14 @@ func (ss scripts) Less(i, j int) bool { return keyPriorities[ss[i].key] < keyPri var keyPriorities = map[string]int{ // in the order of: "update-amazon-linux-2": 1, "update-ubuntu": 2, - "set-env-aws-cred": 3, // TODO: use instance role instead - "mount-aws-cred": 4, // TODO: use instance role instead - "install-go": 5, - "install-csi": 6, - "install-etcd": 7, - "install-aws-k8s-tester": 8, - "install-wrk": 9, - "install-alb": 10, - "install-start-docker-amazon-linux-2": 11, - "install-start-kubeadm-amazon-linux-2": 12, + "install-go": 3, + "install-csi": 4, + "install-etcd": 5, + "install-aws-k8s-tester": 6, + "install-wrk": 7, + "install-alb": 8, + "install-start-docker-amazon-linux-2": 9, + "install-start-kubeadm-amazon-linux-2": 10, } func convertToScript(userName, plugin string) (script, error) { @@ -56,65 +52,6 @@ func convertToScript(userName, plugin string) (script, error) { case plugin == "update-ubuntu": return script{key: "update-ubuntu", data: updateUbuntu}, nil - case strings.HasPrefix(plugin, "set-env-aws-cred-"): - // TODO: use instance role instead - env := strings.Replace(plugin, "set-env-aws-cred-", "", -1) - if os.Getenv(env) == "" { - return script{}, fmt.Errorf("%q is not defined", env) - } - d, derr := ioutil.ReadFile(os.Getenv(env)) - if derr != nil { - return script{}, derr - } - lines := strings.Split(string(d), "\n") - prevDefault := false - accessKey, accessSecret := "", "" - for _, line := range lines { - line = strings.TrimSpace(line) - if line == "" { - continue - } - if line == "[default]" { - prevDefault = true - continue - } - if prevDefault { - if strings.HasPrefix(line, "aws_access_key_id = ") { - accessKey = strings.Replace(line, "aws_access_key_id = ", "", -1) - } - if strings.HasPrefix(line, "aws_secret_access_key = ") { - accessSecret = strings.Replace(line, "aws_secret_access_key = ", "", -1) - break - } - } - } - return script{ - key: "set-env-aws-cred", - data: fmt.Sprintf(`echo "export AWS_ACCESS_KEY_ID=%s" >> /home/%s/.bashrc -echo "export AWS_SECRET_ACCESS_KEY=%s" >> /home/%s/.bashrc -`, accessKey, userName, accessSecret, userName), - }, nil - - case strings.HasPrefix(plugin, "mount-aws-cred-"): - // TODO: use instance role instead - env := strings.Replace(plugin, "mount-aws-cred-", "", -1) - if os.Getenv(env) == "" { - return script{}, fmt.Errorf("%q is not defined", env) - } - d, derr := ioutil.ReadFile(os.Getenv(env)) - if derr != nil { - return script{}, derr - } - return script{ - key: "mount-aws-cred", - data: fmt.Sprintf(` -mkdir -p /home/%s/.aws/ - -cat << EOT > /home/%s/.aws/credentials -%s -EOT`, userName, userName, string(d)), - }, nil - case strings.HasPrefix(plugin, "install-go-"): goVer := strings.Replace(plugin, "install-go-", "", -1) s, err := createInstallGo(goInfo{ diff --git a/vendor/github.com/aws/aws-k8s-tester/eksconfig/BUILD.bazel b/vendor/github.com/aws/aws-k8s-tester/eksconfig/BUILD.bazel index 259260a2dff5..ae28f55ecc31 100644 --- a/vendor/github.com/aws/aws-k8s-tester/eksconfig/BUILD.bazel +++ b/vendor/github.com/aws/aws-k8s-tester/eksconfig/BUILD.bazel @@ -9,8 +9,8 @@ go_library( deps = [ "//vendor/github.com/aws/aws-k8s-tester/ec2config:go_default_library", "//vendor/github.com/aws/aws-k8s-tester/pkg/awsapi/ec2:go_default_library", - "//vendor/github.com/ghodss/yaml:go_default_library", "//vendor/k8s.io/client-go/util/homedir:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/vendor/github.com/aws/aws-k8s-tester/eksconfig/config.go b/vendor/github.com/aws/aws-k8s-tester/eksconfig/config.go index e95a12b84dce..7ec10556b85e 100644 --- a/vendor/github.com/aws/aws-k8s-tester/eksconfig/config.go +++ b/vendor/github.com/aws/aws-k8s-tester/eksconfig/config.go @@ -17,9 +17,8 @@ import ( "github.com/aws/aws-k8s-tester/ec2config" "github.com/aws/aws-k8s-tester/pkg/awsapi/ec2" - - gyaml "github.com/ghodss/yaml" "k8s.io/client-go/util/homedir" + "sigs.k8s.io/yaml" ) // Config defines EKS test configuration. @@ -39,28 +38,18 @@ type Config struct { // Only required when ALB Ingress "TestMode" is "ingress-test-server". AWSK8sTesterImage string `json:"aws-k8s-tester-image,omitempty"` - // AWSK8sTesterDownloadURL is the URL to download the "aws-k8s-tester" from. + // AWSK8sTesterPath is the path to download the "aws-k8s-tester". // This is required for Kubernetes kubetest plugin. + AWSK8sTesterPath string `json:"aws-k8s-tester-path,omitempty"` AWSK8sTesterDownloadURL string `json:"aws-k8s-tester-download-url,omitempty"` - // KubectlDownloadURL is the URL to download the "kubectl" from. + // KubectlPath is the path to download the "kubectl". + KubectlPath string `json:"kubectl-path,omitempty"` KubectlDownloadURL string `json:"kubectl-download-url,omitempty"` - // AWSIAMAuthenticatorDownloadURL is the URL to download the "aws-iam-authenticator" from. + // AWSIAMAuthenticatorPath is the path to download the "aws-iam-authenticator". // This is required for Kubernetes kubetest plugin. + AWSIAMAuthenticatorPath string `json:"aws-iam-authenticator-path,omitempty"` AWSIAMAuthenticatorDownloadURL string `json:"aws-iam-authenticator-download-url,omitempty"` - // WaitBeforeDown is the duration to sleep before cluster tear down. - WaitBeforeDown time.Duration `json:"wait-before-down,omitempty"` - // Down is true to automatically tear down cluster in "test". - // Deployer implementation should not call "Down" inside "Up" method. - // This is meant to be used as a flag for test. - Down bool `json:"down"` - - // EnableWorkerNodeSSH is true to enable SSH access to worker nodes. - EnableWorkerNodeSSH bool `json:"enable-worker-node-ssh"` - // EnableWorkerNodeHA is true to use all 3 subnets to create worker nodes. - // Note that at least 2 subnets are required for EKS cluster. - EnableWorkerNodeHA bool `json:"enable-worker-node-ha"` - // ConfigPath is the configuration file path. // Must be left empty, and let deployer auto-populate this field. // Deployer is expected to update this file with latest status, @@ -77,6 +66,19 @@ type Config struct { KubeConfigPathBucket string `json:"kubeconfig-path-bucket,omitempty"` // read-only to user KubeConfigPathURL string `json:"kubeconfig-path-url,omitempty"` // read-only to user + // WaitBeforeDown is the duration to sleep before cluster tear down. + WaitBeforeDown time.Duration `json:"wait-before-down,omitempty"` + // Down is true to automatically tear down cluster in "test". + // Deployer implementation should not call "Down" inside "Up" method. + // This is meant to be used as a flag for test. + Down bool `json:"down"` + + // EnableWorkerNodeSSH is true to enable SSH access to worker nodes. + EnableWorkerNodeSSH bool `json:"enable-worker-node-ssh"` + // EnableWorkerNodeHA is true to use all 3 subnets to create worker nodes. + // Note that at least 2 subnets are required for EKS cluster. + EnableWorkerNodeHA bool `json:"enable-worker-node-ha"` + // VPCID is the VPC ID. VPCID string `json:"vpc-id"` // SubnetIDs is the subnet IDs. @@ -110,10 +112,10 @@ type Config struct { WorkerNodeAMI string `json:"worker-node-ami,omitempty"` // WorkerNodeInstanceType is the EC2 instance type for worker nodes. WorkerNodeInstanceType string `json:"worker-node-instance-type,omitempty"` - // WorkderNodeASGMin is the minimum number of nodes in worker node ASG. - WorkderNodeASGMin int `json:"worker-node-asg-min,omitempty"` - // WorkderNodeASGMax is the maximum number of nodes in worker node ASG. - WorkderNodeASGMax int `json:"worker-node-asg-max,omitempty"` + // WorkerNodeASGMin is the minimum number of nodes in worker node ASG. + WorkerNodeASGMin int `json:"worker-node-asg-min,omitempty"` + // WorkerNodeASGMax is the maximum number of nodes in worker node ASG. + WorkerNodeASGMax int `json:"worker-node-asg-max,omitempty"` // WorkerNodeVolumeSizeGB is the maximum number of nodes in worker node ASG. // If empty, set default value. WorkerNodeVolumeSizeGB int `json:"worker-node-volume-size-gb,omitempty"` @@ -396,8 +398,11 @@ var defaultConfig = Config{ TestMode: "embedded", AWSK8sTesterDownloadURL: "https://github.com/aws/aws-k8s-tester/releases/download/0.1.2/aws-k8s-tester-0.1.2-linux-amd64", + AWSK8sTesterPath: "/tmp/aws-k8s-tester/aws-k8s-tester", KubectlDownloadURL: "https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/kubectl", + KubectlPath: "/tmp/aws-k8s-tester/kubectl", AWSIAMAuthenticatorDownloadURL: "https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator", + AWSIAMAuthenticatorPath: "/tmp/aws-k8s-tester/aws-iam-authenticator", // enough time for ALB access log WaitBeforeDown: time.Minute, @@ -416,8 +421,8 @@ var defaultConfig = Config{ WorkerNodeAMI: "ami-0f54a2f7d2e9c88b3", WorkerNodeInstanceType: "m5.large", - WorkderNodeASGMin: 1, - WorkderNodeASGMax: 1, + WorkerNodeASGMin: 1, + WorkerNodeASGMax: 1, WorkerNodeVolumeSizeGB: 20, KubernetesVersion: "1.10", @@ -476,7 +481,7 @@ func Load(p string) (cfg *Config, err error) { return nil, err } cfg = new(Config) - if err = gyaml.Unmarshal(d, cfg); err != nil { + if err = yaml.Unmarshal(d, cfg); err != nil { return nil, err } @@ -517,7 +522,7 @@ func (cfg *Config) Sync() (err error) { } cfg.UpdatedAt = time.Now().UTC() var d []byte - d, err = gyaml.Marshal(cfg) + d, err = yaml.Marshal(cfg) if err != nil { return err } @@ -600,25 +605,25 @@ func (cfg *Config) ValidateAndSetDefaults() error { return fmt.Errorf("EKS WorkerNodeInstanceType %q is not valid", cfg.WorkerNodeInstanceType) } if cfg.ALBIngressController != nil && cfg.ALBIngressController.TestServerReplicas > 0 { - if !checkMaxPods(cfg.WorkerNodeInstanceType, cfg.WorkderNodeASGMax, cfg.ALBIngressController.TestServerReplicas) { + if !checkMaxPods(cfg.WorkerNodeInstanceType, cfg.WorkerNodeASGMax, cfg.ALBIngressController.TestServerReplicas) { return fmt.Errorf( "EKS WorkerNodeInstanceType %q only supports %d pods per node (ASG Max %d, allowed up to %d, test server replicas %d)", cfg.WorkerNodeInstanceType, ec2.InstanceTypes[cfg.WorkerNodeInstanceType].MaxPods, - cfg.WorkderNodeASGMax, - ec2.InstanceTypes[cfg.WorkerNodeInstanceType].MaxPods*int64(cfg.WorkderNodeASGMax), + cfg.WorkerNodeASGMax, + ec2.InstanceTypes[cfg.WorkerNodeInstanceType].MaxPods*int64(cfg.WorkerNodeASGMax), cfg.ALBIngressController.TestServerReplicas, ) } } - if cfg.WorkderNodeASGMin == 0 { - return errors.New("EKS WorkderNodeASGMin is not specified") + if cfg.WorkerNodeASGMin == 0 { + return errors.New("EKS WorkerNodeASGMin is not specified") } - if cfg.WorkderNodeASGMax == 0 { - return errors.New("EKS WorkderNodeASGMax is not specified") + if cfg.WorkerNodeASGMax == 0 { + return errors.New("EKS WorkerNodeASGMax is not specified") } - if !checkWorkderNodeASG(cfg.WorkderNodeASGMin, cfg.WorkderNodeASGMax) { - return fmt.Errorf("EKS WorkderNodeASG %d and %d is not valid", cfg.WorkderNodeASGMin, cfg.WorkderNodeASGMax) + if !checkWorkderNodeASG(cfg.WorkerNodeASGMin, cfg.WorkerNodeASGMax) { + return fmt.Errorf("EKS WorkderNodeASG %d and %d is not valid", cfg.WorkerNodeASGMin, cfg.WorkerNodeASGMax) } if cfg.WorkerNodeVolumeSizeGB == 0 { cfg.WorkerNodeVolumeSizeGB = defaultWorkderNodeVolumeSizeGB @@ -667,7 +672,8 @@ func (cfg *Config) ValidateAndSetDefaults() error { } cfg.LogOutputToUploadPathBucket = filepath.Join(cfg.ClusterName, "awsk8stester-eks.log") - cfg.KubeConfigPath = fmt.Sprintf("%s.%s.kubeconfig.generated.yaml", cfg.ConfigPath, cfg.ClusterName) + // cfg.KubeConfigPath = fmt.Sprintf("%s.%s.kubeconfig.generated.yaml", cfg.ConfigPath, cfg.ClusterName) + cfg.KubeConfigPath = "/tmp/aws-k8s-tester/kubeconfig" cfg.KubeConfigPathBucket = filepath.Join(cfg.ClusterName, "kubeconfig") cfg.ALBIngressController.IngressTestServerDeploymentServiceSpecPath = fmt.Sprintf( diff --git a/vendor/github.com/aws/aws-k8s-tester/ekstester/tester.go b/vendor/github.com/aws/aws-k8s-tester/ekstester/tester.go index 3bbe73ec5ce6..b473df577b1a 100644 --- a/vendor/github.com/aws/aws-k8s-tester/ekstester/tester.go +++ b/vendor/github.com/aws/aws-k8s-tester/ekstester/tester.go @@ -2,6 +2,7 @@ package ekstester import ( + osexec "os/exec" "time" "github.com/aws/aws-k8s-tester/eksconfig" @@ -39,6 +40,9 @@ type Deployer interface { // cluster configuration and its states. // It's either reloaded from disk or returned from embedded EKS deployer. LoadConfig() (eksconfig.Config, error) + + // KubectlCommand returns "kubectl" command object for API reachability tests. + KubectlCommand() (*osexec.Cmd, error) } // ALB defines AWS application load balancer tester.