e2e test framework add support for ilb subnet

Adds support for creating and deleting ilb subnets to the e2e test framwork
Also adds "region" CLI flag to e2e binary which is needed for the subnet

cmd/e2e-test/main_test.go

@@ -42,6 +42,7 @@ var (
 		inCluster           bool
 		kubeconfig          string
 		project             string
+		region              string
 		seed                int64
 		destroySandboxes    bool
 		handleSIGINT        bool
@@ -61,6 +62,7 @@ func init() {
 	flag.BoolVar(&flags.run, "run", false, "set to true to run tests (suppresses test suite from 'go test ./...')")
 	flag.BoolVar(&flags.inCluster, "inCluster", false, "set to true if running in the cluster")
 	flag.StringVar(&flags.project, "project", "", "GCP project")
+	flag.StringVar(&flags.region, "region", "", "GCP Region (e.g. us-central1)")
 	flag.Int64Var(&flags.seed, "seed", -1, "random seed")
 	flag.BoolVar(&flags.destroySandboxes, "destroySandboxes", true, "set to false to leave sandboxed resources for debugging")
 	flag.BoolVar(&flags.handleSIGINT, "handleSIGINT", true, "catch SIGINT to perform clean")
@@ -81,6 +83,10 @@ func TestMain(m *testing.M) {
 		fmt.Fprintln(os.Stderr, "-project must be set to the Google Cloud test project")
 		os.Exit(1)
 	}
+	if flags.region == "" {
+		fmt.Println("-region must be set to the region of the cluster")
+		os.Exit(1)
+	}
 
 	fmt.Printf("Version: %q, Commit: %q\n", version.Version, version.GitCommit)
 
@@ -106,6 +112,7 @@ func TestMain(m *testing.M) {
 
 	Framework = e2e.NewFramework(kubeconfig, e2e.Options{
 		Project:             flags.project,
+		Region:              flags.region,
 		Seed:                flags.seed,
 		DestroySandboxes:    flags.destroySandboxes,
 		GceEndpointOverride: flags.gceEndpointOverride,

cmd/e2e-test/run.sh

@@ -34,10 +34,38 @@ if [[ -z "$PROJECT" ]]; then
   exit
 fi
 
+for ATTEMPT in $(seq 60); do
+  ZONE_INFO=$(curl -H'Metadata-Flavor:Google' metadata.google.internal/computeMetadata/v1/instance/zone)
+  if [[ -n "${ZONE_INFO}" ]]; then
+    break
+  fi
+  echo "Error: could not get zone from the metadata server (attempt ${ATTEMPT})"
+  sleep 1
+done
+
+if [[ -z "${ZONE_INFO}" ]]; then
+  echo "Error: could not get zone info from the metadata server"
+  echo "RESULT: 2"
+  echo '--- END ---'
+  exit
+fi
+echo "ZONE_INFO: ${ZONE_INFO}"
+
+# Get Region information from zone info
+ZONE=$(echo ${ZONE_INFO} | sed 's+projects/.*/zones/++')
+REGION=$(echo ${ZONE} | sed 's/-[a-z]$//')
+
+if [[ -z "${REGION}" ]]; then
+  echo "Error: could not parse region from zone info"
+  echo "Result: 2"
+  exit
+fi
+echo "Using Region: ${REGION}"
+
 echo
 echo ==============================================================================
 echo "PROJECT: ${PROJECT}"
-CMD="/e2e-test -test.v -test.parallel=100 -run -project ${PROJECT} -logtostderr -inCluster -v=2"
+CMD="/e2e-test -test.v -test.parallel=100 -run -project ${PROJECT} -region ${REGION} -logtostderr -inCluster -v=2"
 echo "CMD: ${CMD}" $@
 echo
 

pkg/e2e/fixtures.go

@@ -21,9 +21,13 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
+	"k8s.io/ingress-gce/pkg/utils"
+	"net/http"
 	"reflect"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
+	computebeta "google.golang.org/api/compute/v0.beta"
 	"google.golang.org/api/compute/v1"
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
@@ -39,6 +43,8 @@ const (
 	echoheadersImage = "gcr.io/k8s-ingress-image-push/ingress-gce-echo-amd64:master"
 )
 
+var ErrSubnetExists = fmt.Errorf("ILB subnet in region already exists")
+
 // NoopModify does not modify the input deployment
 func NoopModify(*apps.Deployment) {}
 
@@ -254,3 +260,34 @@ func DeleteGCPAddress(s *Sandbox, name string) error {
 
 	return nil
 }
+
+// CreateILBSubnet creates the ILB subnet
+func CreateILBSubnet(s *Sandbox, name string, ipCidrRange string) error {
+	networkID := cloud.ResourceID{ProjectID: s.f.Project, Resource: "networks", Key: meta.GlobalKey("default")}
+	subnet := &computebeta.Subnetwork{
+		Name:        name,
+		IpCidrRange: ipCidrRange,
+		Purpose:     "INTERNAL_HTTPS_LOAD_BALANCER",
+		Network:     networkID.SelfLink(meta.VersionBeta),
+		Role:        "ACTIVE",
+	}
+
+	klog.V(2).Infof("Creating ILB Subnet: %+v", subnet)
+	err := s.f.Cloud.BetaSubnetworks().Insert(context.Background(), meta.RegionalKey(subnet.Name, s.f.Region), subnet)
+	if err != nil {
+		// GCE returns a 409 when there is already an "ACTIVE" subnet set up for ILB
+		if utils.IsHTTPErrorCode(err, http.StatusConflict) {
+			klog.V(3).Info("ILB subnet already exists")
+			return ErrSubnetExists
+		} else {
+			return fmt.Errorf("Error creating ILB subnet: %v", err)
+		}
+	}
+	return nil
+}
+
+// DeleteILBSubnet deletes the ILB subnet
+func DeleteILBSubnet(s *Sandbox, name string) error {
+	klog.V(2).Infof("Deleting ILB Subnet %q", name)
+	return s.f.Cloud.BetaSubnetworks().Delete(context.Background(), meta.RegionalKey(name, s.f.Region))
+}

pkg/e2e/framework.go

@@ -42,6 +42,7 @@ import (
 // Options for the test framework.
 type Options struct {
 	Project             string
+	Region              string
 	Seed                int64
 	DestroySandboxes    bool
 	GceEndpointOverride string
@@ -62,6 +63,7 @@ func NewFramework(config *rest.Config, options Options) *Framework {
 		Clientset:           kubernetes.NewForConfigOrDie(config),
 		BackendConfigClient: backendConfigClient,
 		Project:             options.Project,
+		Region:              options.Region,
 		Cloud:               theCloud,
 		Rand:                rand.New(rand.NewSource(options.Seed)),
 		destroySandboxes:    options.DestroySandboxes,
@@ -76,6 +78,7 @@ type Framework struct {
 	Clientset           *kubernetes.Clientset
 	BackendConfigClient *backendconfigclient.Clientset
 	Project             string
+	Region              string
 	Cloud               cloud.Cloud
 	Rand                *rand.Rand
 	statusManager       *StatusManager