kubetest: add "KubectlCommand" to "deployer" interface

config/jobs/kubernetes/sig-aws/eks/k8s-aws-eks-1.10.yaml

@@ -4,13 +4,19 @@ presets:
   # URL to download the latest 'aws-k8s-tester' release
   - name: AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_DOWNLOAD_URL
     value: https://github.com/aws/aws-k8s-tester/releases/download/0.1.3/aws-k8s-tester-0.1.3-linux-amd64
+  - name: AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_DOWNLOAD_PATH
+    value: /tmp/aws-k8s-tester/aws-k8s-tester
   # URL to download 'kubectl', required for 'kubectl' calls to EKS
   # TODO: use upstream 'kubectl'
   - name: AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_URL
     value: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/kubectl
+  - name: AWS_K8S_TESTER_EKS_KUBECTL_DOWNLOAD_PATH
+    value: /tmp/aws-k8s-tester/kubectl
   # URL to download 'aws-iam-authenticator', required for 'kubectl' calls to EKS
   - name: AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_DOWNLOAD_URL
     value: https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator
+  - name: AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_DOWNLOAD_PATH
+    value: /tmp/aws-k8s-tester/aws-iam-authenticator
   # test mode is either "embedded" or "aws-cli" ("embedded" uses native AWS Go client, "aws-cli" will use 'aws')
   - name: AWS_K8S_TESTER_EKS_TEST_MODE
     value: "embedded"

kubetest/anywhere.go

@@ -291,6 +291,8 @@ func (k *kubernetesAnywhere) GetClusterCreated(gcpProject string) (time.Time, er
 	return time.Time{}, errors.New("not implemented")
 }
 
+func (_ *kubernetesAnywhere) KubectlCommand() (*exec.Cmd, error) { return nil, nil }
+
 const defaultConfigFile = ".config"
 
 type kubernetesAnywhereMultiCluster struct {
@@ -429,3 +431,5 @@ func (k *kubernetesAnywhereMultiCluster) Down() error {
 	}
 	return control.FinishRunningParallel(cmds...)
 }
+
+func (_ *kubernetesAnywhereMultiCluster) KubectlCommand() (*exec.Cmd, error) { return nil, nil }

kubetest/azure.go

@@ -604,3 +604,5 @@ func (c Cluster) TestSetup() error {
 func (c Cluster) IsUp() error {
 	return isUp(c)
 }
+
+func (_ Cluster) KubectlCommand() (*exec.Cmd, error) { return nil, nil }

kubetest/bash.go

@@ -102,6 +102,8 @@ func (b *bashDeployer) GetClusterCreated(gcpProject string) (time.Time, error) {
 	return created, nil
 }
 
+func (_ *bashDeployer) KubectlCommand() (*exec.Cmd, error) { return nil, nil }
+
 // Calculates the cluster IP range based on the no. of nodes in the cluster.
 // Note: This mimics the function get-cluster-ip-range used by kube-up script.
 func getClusterIPRange(numNodes int) string {

kubetest/conformance/conformance.go

@@ -19,7 +19,9 @@ package conformance
 
 import (
 	"fmt"
+	"log"
 	"os"
+	"os/exec"
 	"time"
 
 	"k8s.io/api/core/v1"
@@ -132,3 +134,8 @@ func (d *Deployer) Down() error {
 func (d *Deployer) GetClusterCreated(gcpProject string) (time.Time, error) {
 	return time.Time{}, fmt.Errorf("cannot get cluster create time for conformance cluster")
 }
+
+func (_ *Deployer) KubectlCommand() (*exec.Cmd, error) {
+	log.Print("Noop - Conformance KubectlCommand()")
+	return nil, nil
+}

kubetest/e2e.go

@@ -147,7 +147,7 @@ func run(deploy deployer, o options) error {
 		// If node testing is enabled, check that the api is reachable before
 		// proceeding with further steps. This is accomplished by listing the nodes.
 		if !o.nodeTests {
-			errs = util.AppendError(errs, control.XMLWrap(&suite, "Check APIReachability", getKubectlVersion))
+			errs = util.AppendError(errs, control.XMLWrap(&suite, "Check APIReachability", func() error { return getKubectlVersion(deploy) }))
 			if dump != "" {
 				errs = util.AppendError(errs, control.XMLWrap(&suite, "list nodes", func() error {
 					return listNodes(dump)
@@ -201,7 +201,7 @@ func run(deploy deployer, o options) error {
 			}))
 		} else {
 			if o.deployment != "conformance" {
-				errs = util.AppendError(errs, control.XMLWrap(&suite, "kubectl version", getKubectlVersion))
+				errs = util.AppendError(errs, control.XMLWrap(&suite, "kubectl version", func() error { return getKubectlVersion(deploy) }))
 			}
 
 			if o.skew {
@@ -337,18 +337,27 @@ func run(deploy deployer, o options) error {
 	return nil
 }
 
-func getKubectlVersion() error {
+func getKubectlVersion(dp deployer) error {
+	cmd, err := dp.KubectlCommand()
+	if err != nil {
+		return err
+	}
+	if cmd == nil {
+		cmd = exec.Command("./cluster/kubectl.sh")
+	}
+	cmd.Args = append(cmd.Args, "--match-server-version=false", "version")
+	copied := *cmd
 	retries := 5
 	for {
-		_, err := control.Output(exec.Command("./cluster/kubectl.sh", "--match-server-version=false", "version"))
+		_, err := control.Output(&copied)
 		if err == nil {
 			return nil
 		}
 		retries--
 		if retries == 0 {
 			return err
 		}
-		log.Print("Failed to reach api. Sleeping for 10 seconds before retrying...")
+		log.Printf("Failed to reach api. Sleeping for 10 seconds before retrying... (%v)", copied.Args)
 		time.Sleep(10 * time.Second)
 	}
 }

kubetest/eks/eks.go

@@ -26,14 +26,12 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"os/exec"
-	"path/filepath"
+	osexec "os/exec"
 	"syscall"
 	"time"
 
 	"github.com/aws/aws-k8s-tester/eksconfig"
 	"github.com/aws/aws-k8s-tester/ekstester"
-
 	"k8s.io/test-infra/kubetest/process"
 	"k8s.io/test-infra/kubetest/util"
 )
@@ -42,10 +40,9 @@ import (
 // Satisfies "k8s.io/test-infra/kubetest/main.go" 'deployer' and 'publisher" interfaces.
 // Reference https://github.com/kubernetes/test-infra/blob/master/kubetest/main.go.
 type deployer struct {
-	stopc            chan struct{}
-	cfg              *eksconfig.Config
-	awsK8sTesterPath string
-	ctrl             *process.Control
+	stopc chan struct{}
+	cfg   *eksconfig.Config
+	ctrl  *process.Control
 }
 
 // NewDeployer creates a new EKS deployer.
@@ -56,15 +53,13 @@ func NewDeployer(timeout time.Duration, verbose bool) (ekstester.Deployer, error
 		return nil, err
 	}
 	var f *os.File
-	f, err = ioutil.TempFile(os.TempDir(), "aws-k8s-tester")
+	f, err = ioutil.TempFile(os.TempDir(), "aws-k8s-tester-config")
 	if err != nil {
 		return nil, err
 	}
-	outputPath := f.Name()
-	f.Close()
-	cfg.ConfigPath, err = filepath.Abs(outputPath)
-	if err != nil {
-		return nil, err
+	cfg.ConfigPath = f.Name()
+	if err = f.Close(); err != nil {
+		return nil, fmt.Errorf("failed to close aws-k8s-tester-config file %v", err)
 	}
 	if err = cfg.Sync(); err != nil {
 		return nil, err
@@ -81,22 +76,22 @@ func NewDeployer(timeout time.Duration, verbose bool) (ekstester.Deployer, error
 		),
 	}
 
-	dp.awsK8sTesterPath, err = exec.LookPath("aws-k8s-tester")
+	if err = os.RemoveAll(cfg.AWSK8sTesterPath); err != nil {
+		return nil, err
+	}
+	f, err = os.Create(cfg.AWSK8sTesterPath)
 	if err != nil {
-		var f *os.File
-		f, err = ioutil.TempFile(os.TempDir(), "aws-k8s-tester")
-		if err != nil {
-			return nil, fmt.Errorf("failed to create %q (%v)", dp.awsK8sTesterPath, err)
-		}
-		dp.awsK8sTesterPath = f.Name()
-		dp.awsK8sTesterPath, _ = filepath.Abs(dp.awsK8sTesterPath)
-		if err = httpRead(cfg.AWSK8sTesterDownloadURL, f); err != nil {
-			return nil, err
-		}
-		f.Close()
-		if err = util.EnsureExecutable(dp.awsK8sTesterPath); err != nil {
-			return nil, err
-		}
+		return nil, fmt.Errorf("failed to create %q (%v)", cfg.AWSK8sTesterPath, err)
+	}
+	cfg.AWSK8sTesterPath = f.Name()
+	if err = httpRead(cfg.AWSK8sTesterDownloadURL, f); err != nil {
+		return nil, err
+	}
+	if err = f.Close(); err != nil {
+		return nil, fmt.Errorf("failed to close aws-k8s-tester file %v", err)
+	}
+	if err = util.EnsureExecutable(cfg.AWSK8sTesterPath); err != nil {
+		return nil, err
 	}
 	return dp, nil
 }
@@ -106,8 +101,8 @@ func (dp *deployer) Up() (err error) {
 	// "create cluster" command outputs cluster information
 	// in the configuration file (e.g. VPC ID, ALB DNS names, etc.)
 	// this needs be reloaded for other deployer method calls
-	createCmd := exec.Command(
-		dp.awsK8sTesterPath,
+	createCmd := osexec.Command(
+		dp.cfg.AWSK8sTesterPath,
 		"eks",
 		"--path="+dp.cfg.ConfigPath,
 		"create",
@@ -134,8 +129,8 @@ func (dp *deployer) Down() (err error) {
 	if _, err = dp.LoadConfig(); err != nil {
 		return err
 	}
-	_, err = dp.ctrl.Output(exec.Command(
-		dp.awsK8sTesterPath,
+	_, err = dp.ctrl.Output(osexec.Command(
+		dp.cfg.AWSK8sTesterPath,
 		"eks",
 		"--path="+dp.cfg.ConfigPath,
 		"delete",
@@ -150,8 +145,8 @@ func (dp *deployer) IsUp() (err error) {
 	if _, err = dp.LoadConfig(); err != nil {
 		return err
 	}
-	_, err = dp.ctrl.Output(exec.Command(
-		dp.awsK8sTesterPath,
+	_, err = dp.ctrl.Output(osexec.Command(
+		dp.cfg.AWSK8sTesterPath,
 		"eks",
 		"--path="+dp.cfg.ConfigPath,
 		"check",
@@ -191,8 +186,8 @@ func (dp *deployer) GetWorkerNodeLogs() (err error) {
 	if _, err = dp.LoadConfig(); err != nil {
 		return err
 	}
-	_, err = dp.ctrl.Output(exec.Command(
-		dp.awsK8sTesterPath,
+	_, err = dp.ctrl.Output(osexec.Command(
+		dp.cfg.AWSK8sTesterPath,
 		"eks",
 		"--path="+dp.cfg.ConfigPath,
 		"test", "get-worker-node-logs",
@@ -208,17 +203,17 @@ func (dp *deployer) DumpClusterLogs(artifactDir, _ string) (err error) {
 	if _, err = dp.LoadConfig(); err != nil {
 		return err
 	}
-	_, err = dp.ctrl.Output(exec.Command(
-		dp.awsK8sTesterPath,
+	_, err = dp.ctrl.Output(osexec.Command(
+		dp.cfg.AWSK8sTesterPath,
 		"eks",
 		"--path="+dp.cfg.ConfigPath,
 		"test", "get-worker-node-logs",
 	))
 	if err != nil {
 		return err
 	}
-	_, err = dp.ctrl.Output(exec.Command(
-		dp.awsK8sTesterPath,
+	_, err = dp.ctrl.Output(osexec.Command(
+		dp.cfg.AWSK8sTesterPath,
 		"eks",
 		"--path="+dp.cfg.ConfigPath,
 		"test", "dump-cluster-logs",
@@ -227,6 +222,15 @@ func (dp *deployer) DumpClusterLogs(artifactDir, _ string) (err error) {
 	return err
 }
 
+// KubectlCommand returns "kubectl" command object for API reachability tests.
+func (dp *deployer) KubectlCommand() (*osexec.Cmd, error) {
+	// reload configuration from disk to read the latest configuration
+	if _, err := dp.LoadConfig(); err != nil {
+		return nil, err
+	}
+	return osexec.Command(dp.cfg.KubectlPath, "--kubeconfig="+dp.cfg.KubeConfigPath), nil
+}
+
 // Stop stops ongoing operations.
 // This is useful for local development.
 // For example, one may run "Up" but have to cancel ongoing "Up"

kubetest/gke.go

@@ -652,3 +652,5 @@ func (g *gkeDeployer) GetClusterCreated(gcpProject string) (time.Time, error) {
 	}
 	return created, nil
 }
+
+func (_ *gkeDeployer) KubectlCommand() (*exec.Cmd, error) { return nil, nil }

kubetest/kops.go

@@ -653,6 +653,8 @@ func (k kops) Publish() error {
 	})
 }
 
+func (_ kops) KubectlCommand() (*exec.Cmd, error) { return nil, nil }
+
 // getRandomAWSZones looks up all regions, and the availability zones for those regions.  A random
 // region is then chosen and the AZ's for that region is returned. At least masterCount zones will be
 // returned, all in the same region.

kubetest/kubeadmdind/kubeadm_dind.go

@@ -312,6 +312,8 @@ func (d *Deployer) GetClusterCreated(gcpProject string) (time.Time, error) {
 	return time.Time{}, errors.New("not implemented")
 }
 
+func (_ *Deployer) KubectlCommand() (*exec.Cmd, error) { return nil, nil }
+
 // findPath looks for the existence of a file or directory based on a
 // a github organization, github repo, and a relative path.  It looks
 // for the file/directory in this order:

kubetest/local.go

@@ -234,3 +234,5 @@ func (n localCluster) Down() error {
 func (n localCluster) GetClusterCreated(gcpProject string) (time.Time, error) {
 	return time.Time{}, errors.New("GetClusterCreated not implemented in localCluster")
 }
+
+func (_ localCluster) KubectlCommand() (*exec.Cmd, error) { return nil, nil }

kubetest/main.go

@@ -226,6 +226,7 @@ type deployer interface {
 	TestSetup() error
 	Down() error
 	GetClusterCreated(gcpProject string) (time.Time, error)
+	KubectlCommand() (*exec.Cmd, error)
 }
 
 // publisher is implemented by deployers that want to publish status on success

kubetest/node.go

@@ -19,6 +19,7 @@ package main
 import (
 	"errors"
 	"log"
+	"os/exec"
 	"time"
 )
 
@@ -55,3 +56,8 @@ func (n nodeDeploy) Down() error {
 func (n nodeDeploy) GetClusterCreated(gcpProject string) (time.Time, error) {
 	return time.Time{}, errors.New("not implemented")
 }
+
+func (_ nodeDeploy) KubectlCommand() (*exec.Cmd, error) {
+	log.Print("Noop - Node KubectlCommand()")
+	return nil, nil
+}

kubetest/none.go

@@ -19,6 +19,7 @@ package main
 import (
 	"errors"
 	"log"
+	"os/exec"
 	"time"
 )
 
@@ -53,3 +54,8 @@ func (n noneDeploy) Down() error {
 func (n noneDeploy) GetClusterCreated(gcpProject string) (time.Time, error) {
 	return time.Time{}, errors.New("not implemented")
 }
+
+func (_ noneDeploy) KubectlCommand() (*exec.Cmd, error) {
+	log.Print("Noop KubectlCommand()")
+	return nil, nil
+}