e2e test framework add support for ilb subnet

cmd/e2e-test/main_test.go

@@ -42,6 +42,7 @@ var (
 		inCluster           bool
 		kubeconfig          string
 		project             string
+		region              string
 		seed                int64
 		destroySandboxes    bool
 		handleSIGINT        bool
@@ -61,6 +62,7 @@ func init() {
 	flag.BoolVar(&flags.run, "run", false, "set to true to run tests (suppresses test suite from 'go test ./...')")
 	flag.BoolVar(&flags.inCluster, "inCluster", false, "set to true if running in the cluster")
 	flag.StringVar(&flags.project, "project", "", "GCP project")
+	flag.StringVar(&flags.region, "region", "", "GCP Region (e.g. us-central1)")
 	flag.Int64Var(&flags.seed, "seed", -1, "random seed")
 	flag.BoolVar(&flags.destroySandboxes, "destroySandboxes", true, "set to false to leave sandboxed resources for debugging")
 	flag.BoolVar(&flags.handleSIGINT, "handleSIGINT", true, "catch SIGINT to perform clean")
@@ -81,6 +83,10 @@ func TestMain(m *testing.M) {
 		fmt.Fprintln(os.Stderr, "-project must be set to the Google Cloud test project")
 		os.Exit(1)
 	}
+	if flags.region == "" {
+		fmt.Println("-region must be set to the region of the cluster")
+		os.Exit(1)
+	}
 
 	fmt.Printf("Version: %q, Commit: %q\n", version.Version, version.GitCommit)
 
@@ -106,6 +112,7 @@ func TestMain(m *testing.M) {
 
 	Framework = e2e.NewFramework(kubeconfig, e2e.Options{
 		Project:             flags.project,
+		Region:              flags.region,
 		Seed:                flags.seed,
 		DestroySandboxes:    flags.destroySandboxes,
 		GceEndpointOverride: flags.gceEndpointOverride,

cmd/e2e-test/run.sh

@@ -34,10 +34,38 @@ if [[ -z "$PROJECT" ]]; then
   exit
 fi
 
+for ATTEMPT in $(seq 60); do
+  ZONE_INFO=$(curl -H'Metadata-Flavor:Google' metadata.google.internal/computeMetadata/v1/instance/zone)
+  if [[ -n "${ZONE_INFO}" ]]; then
+    break
+  fi
+  echo "Error: could not get zone from the metadata server (attempt ${ATTEMPT})"
+  sleep 1
+done
+
+if [[ -z "${ZONE_INFO}" ]]; then
+  echo "Error: could not get zone info from the metadata server"
+  echo "RESULT: 2"
+  echo '--- END ---'
+  exit
+fi
+echo "ZONE_INFO: ${ZONE_INFO}"
+
+# Get Region information from zone info
+ZONE=$(echo ${ZONE_INFO} | sed 's+projects/.*/zones/++')
+REGION=$(echo ${ZONE} | sed 's/-[a-z]$//')
+
+if [[ -z "${REGION}" ]]; then
+  echo "Error: could not parse region from zone info"
+  echo "Result: 2"
+  exit
+fi
+echo "Using Region: ${REGION}"
+
 echo
 echo ==============================================================================
 echo "PROJECT: ${PROJECT}"
-CMD="/e2e-test -test.v -test.parallel=100 -run -project ${PROJECT} -logtostderr -inCluster -v=2"
+CMD="/e2e-test -test.v -test.parallel=100 -run -project ${PROJECT} -region ${REGION} -logtostderr -inCluster -v=2"
 echo "CMD: ${CMD}" $@
 echo
 

pkg/e2e/fixtures.go

@@ -21,9 +21,13 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
+	"k8s.io/ingress-gce/pkg/utils"
+	"net/http"
 	"reflect"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
+	computebeta "google.golang.org/api/compute/v0.beta"
 	"google.golang.org/api/compute/v1"
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/api/core/v1"
@@ -39,6 +43,8 @@ const (
 	echoheadersImage = "gcr.io/k8s-ingress-image-push/ingress-gce-echo-amd64:master"
 )
 
+var ErrSubnetExists = fmt.Errorf("ILB subnet in region already exists")
+
 // NoopModify does not modify the input deployment
 func NoopModify(*apps.Deployment) {}
 
@@ -254,3 +260,34 @@ func DeleteGCPAddress(s *Sandbox, name string) error {
 
 	return nil
 }
+
+// CreateILBSubnet creates the ILB subnet
+func CreateILBSubnet(s *Sandbox, name string, ipCidrRange string) error {
+	networkID := cloud.ResourceID{ProjectID: s.f.Project, Resource: "networks", Key: meta.GlobalKey("default")}
+	subnet := &computebeta.Subnetwork{
+		Name:        name,
+		IpCidrRange: ipCidrRange,
+		Purpose:     "INTERNAL_HTTPS_LOAD_BALANCER",
+		Network:     networkID.SelfLink(meta.VersionBeta),
+		Role:        "ACTIVE",
+	}
+
+	klog.V(2).Infof("Creating ILB Subnet: %+v", subnet)
+	err := s.f.Cloud.BetaSubnetworks().Insert(context.Background(), meta.RegionalKey(subnet.Name, s.f.Region), subnet)
+	if err != nil {
+		// GCE returns a 409 when there is already an "ACTIVE" subnet set up for ILB
+		if utils.IsHTTPErrorCode(err, http.StatusConflict) {
+			klog.V(3).Info("ILB subnet already exists")
+			return ErrSubnetExists
+		} else {
+			return fmt.Errorf("Error creating ILB subnet: %v", err)
+		}
+	}
+	return nil
+}
+
+// DeleteILBSubnet deletes the ILB subnet
+func DeleteILBSubnet(s *Sandbox, name string) error {
+	klog.V(2).Infof("Deleting ILB Subnet %q", name)
+	return s.f.Cloud.BetaSubnetworks().Delete(context.Background(), meta.RegionalKey(name, s.f.Region))
+}

pkg/e2e/framework.go

@@ -42,6 +42,7 @@ import (
 // Options for the test framework.
 type Options struct {
 	Project             string
+	Region              string
 	Seed                int64
 	DestroySandboxes    bool
 	GceEndpointOverride string
@@ -62,6 +63,7 @@ func NewFramework(config *rest.Config, options Options) *Framework {
 		Clientset:           kubernetes.NewForConfigOrDie(config),
 		BackendConfigClient: backendConfigClient,
 		Project:             options.Project,
+		Region:              options.Region,
 		Cloud:               theCloud,
 		Rand:                rand.New(rand.NewSource(options.Seed)),
 		destroySandboxes:    options.DestroySandboxes,
@@ -76,6 +78,7 @@ type Framework struct {
 	Clientset           *kubernetes.Clientset
 	BackendConfigClient *backendconfigclient.Clientset
 	Project             string
+	Region              string
 	Cloud               cloud.Cloud
 	Rand                *rand.Rand
 	statusManager       *StatusManager