Added more e2e tests

* Update README with more detail to run E2E
* Cleanup cloud code
* Delete pod E2e test
* Properly delete EBS volumes with Retain reclaimPolicy
* Properly cleanup PV for pre-provisioned tests
* Prefix tester structs with DynamicallyProvisioned
* Automatically create/delete volumes for pre-provisioned tests
* Modify E2E tests Ginkgo structure
* Colocated pods E2E test
* Topology aware E2E test
* ReclaimPolicy E2E test
* Pre-provisioned volume E2E test
* Add more E2E tests
  * should create a volume on demand and mount it as readOnly in a pod
  * should create a volume on demand with provided mountOptions
  * should create a volume with encryption (+2 squashed commits)

docs/README.md

@@ -128,12 +128,27 @@ To execute integration tests, run:
 make test-integration
 ```
 
-To execute e2e tests, run:
+**Note**: EC2 instance is required to run integration test, since it is exercising the actual flow of creating EBS volume, attaching it and read/write on the disk.
+
+To execute e2e tests:
+
+Some tests marked with `[env]` require specific environmental variables to be set, if not set these tests will be skipped.
+
+```
+export AWS_AVAILABILITY_ZONES="us-west-2a,us-west-2b"
 ```
+
+Replacing `us-west-2a,us-west-2b` with the AZ(s) where your Kubernetes worker nodes are located.
+
+These tests also rely on having proper [AWS credentials](https://docs.aws.amazon.com/amazonswf/latest/awsrbflowguide/set-up-creds.html) set either via environmental variables or a credentials file.
+
+Finally run:
+```
+export KUBECONFIG=~/.kube/config
 make test-e2e
 ```
 
-**Note**: EC2 instance is required to run integration test, since it is exercising the actual flow of creating EBS volume, attaching it and read/write on the disk.
+**Note**: By default `make test-e2e` will run 32 tests concurrently, set `GINKGO_NODES` to change the parallelism.
 
 ### Build and Publish Container Image
 

go.mod

@@ -71,9 +71,12 @@ require (
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.9.1 // indirect
 	golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 // indirect
+	golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 // indirect
+	golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect
 	golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a // indirect
 	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect
-	google.golang.org/genproto v0.0.0-20180831171423-11092d34479b // indirect
+	google.golang.org/appengine v1.3.0 // indirect
+	google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 // indirect
 	google.golang.org/grpc v1.17.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect

go.sum

@@ -89,6 +89,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff h1:kOkM9whyQYodu09SJ6W3NCsHG7crFaJILQ22Gozp3lg=
 github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -197,15 +198,23 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58 h1:otZG8yDCO4LVps5+9bxOeNiCvgmOyt96J3roHTYs7oE=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953 h1:LuZIitY8waaxUfNIdtajyE/YzA/zyf0YxXG27VpLrkg=
+golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
+golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -218,9 +227,12 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20180831171423-11092d34479b h1:lohp5blsw53GBXtLyLNaTXPXS9pJ1tiTw61ZHUoE9Qw=
-google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 h1:yvw+zsSmSM02Z5H3ZdEV7B7Ql7eFrjQTnmByJvK+3J8=
+google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
+google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

hack/run-e2e-test

@@ -16,4 +16,6 @@
 
 set -euo pipefail
 
-ginkgo -p -v --focus ebs-csi-e2e tests/e2e
+NODES=${GINKGO_NODES:-32}
+
+ginkgo -p -nodes=$NODES -v --focus ebs-csi-e2e tests/e2e

pkg/cloud/cloud.go

@@ -129,6 +129,7 @@ type Cloud interface {
 	DeleteDisk(ctx context.Context, volumeID string) (success bool, err error)
 	AttachDisk(ctx context.Context, volumeID string, nodeID string) (devicePath string, err error)
 	DetachDisk(ctx context.Context, volumeID string, nodeID string) (err error)
+	WaitForAttachmentState(ctx context.Context, volumeID, state string) error
 	GetDiskByName(ctx context.Context, name string, capacityBytes int64) (disk *Disk, err error)
 	GetDiskByID(ctx context.Context, volumeID string) (disk *Disk, err error)
 	IsExistInstance(ctx context.Context, nodeID string) (success bool)
@@ -143,16 +144,30 @@ type cloud struct {
 var _ Cloud = &cloud{}
 
 // NewCloud returns a new instance of AWS cloud
+// Pass in nil metadata to use an auto created EC2Metadata service
 // It panics if session is invalid
 func NewCloud() (Cloud, error) {
-	sess := session.Must(session.NewSession(&aws.Config{}))
-	svc := ec2metadata.New(sess)
+	svc := newEC2MetadataSvc()
 
+	var err error
 	metadata, err := NewMetadataService(svc)
 	if err != nil {
 		return nil, fmt.Errorf("could not get metadata from AWS: %v", err)
 	}
 
+	return newEC2Cloud(metadata, svc)
+}
+
+func NewCloudWithMetadata(metadata MetadataService) (Cloud, error) {
+	return newEC2Cloud(metadata, newEC2MetadataSvc())
+}
+
+func newEC2MetadataSvc() *ec2metadata.EC2Metadata {
+	sess := session.Must(session.NewSession(&aws.Config{}))
+	return ec2metadata.New(sess)
+}
+
+func newEC2Cloud(metadata MetadataService, svc *ec2metadata.EC2Metadata) (Cloud, error) {
 	provider := []credentials.Provider{
 		&credentials.EnvProvider{},
 		&ec2rolecreds.EC2RoleProvider{Client: svc},
@@ -297,7 +312,7 @@ func (c *cloud) AttachDisk(ctx context.Context, volumeID, nodeID string) (string
 	}
 
 	// This is the only situation where we taint the device
-	if err := c.waitForAttachmentState(ctx, volumeID, "attached"); err != nil {
+	if err := c.WaitForAttachmentState(ctx, volumeID, "attached"); err != nil {
 		device.Taint()
 		return "", err
 	}
@@ -336,13 +351,58 @@ func (c *cloud) DetachDisk(ctx context.Context, volumeID, nodeID string) error {
 		return fmt.Errorf("could not detach volume %q from node %q: %v", volumeID, nodeID, err)
 	}
 
-	if err := c.waitForAttachmentState(ctx, volumeID, "detached"); err != nil {
+	if err := c.WaitForAttachmentState(ctx, volumeID, "detached"); err != nil {
 		return err
 	}
 
 	return nil
 }
 
+// WaitForAttachmentState polls until the attachment status is the expected value.
+func (c *cloud) WaitForAttachmentState(ctx context.Context, volumeID, state string) error {
+	// Most attach/detach operations on AWS finish within 1-4 seconds.
+	// By using 1 second starting interval with a backoff of 1.8,
+	// we get [1, 1.8, 3.24, 5.832000000000001, 10.4976].
+	// In total we wait for 2601 seconds.
+	backoff := wait.Backoff{
+		Duration: 1 * time.Second,
+		Factor:   1.8,
+		Steps:    13,
+	}
+
+	verifyVolumeFunc := func() (bool, error) {
+		request := &ec2.DescribeVolumesInput{
+			VolumeIds: []*string{
+				aws.String(volumeID),
+			},
+		}
+
+		volume, err := c.getVolume(ctx, request)
+		if err != nil {
+			return false, err
+		}
+
+		if len(volume.Attachments) == 0 {
+			if state == "detached" {
+				return true, nil
+			}
+		}
+
+		for _, a := range volume.Attachments {
+			if a.State == nil {
+				klog.Warningf("Ignoring nil attachment state for volume %q: %v", volumeID, a)
+				continue
+			}
+			if *a.State == state {
+				return true, nil
+			}
+		}
+		return false, nil
+	}
+
+	return wait.ExponentialBackoff(backoff, verifyVolumeFunc)
+}
+
 func (c *cloud) GetDiskByName(ctx context.Context, name string, capacityBytes int64) (*Disk, error) {
 	request := &ec2.DescribeVolumesInput{
 		Filters: []*ec2.Filter{
@@ -456,51 +516,6 @@ func (c *cloud) getInstance(ctx context.Context, nodeID string) (*ec2.Instance,
 	return instances[0], nil
 }
 
-// waitForAttachmentStatus polls until the attachment status is the expected value.
-func (c *cloud) waitForAttachmentState(ctx context.Context, volumeID, state string) error {
-	// Most attach/detach operations on AWS finish within 1-4 seconds.
-	// By using 1 second starting interval with a backoff of 1.8,
-	// we get [1, 1.8, 3.24, 5.832000000000001, 10.4976].
-	// In total we wait for 2601 seconds.
-	backoff := wait.Backoff{
-		Duration: 1 * time.Second,
-		Factor:   1.8,
-		Steps:    13,
-	}
-
-	verifyVolumeFunc := func() (bool, error) {
-		request := &ec2.DescribeVolumesInput{
-			VolumeIds: []*string{
-				aws.String(volumeID),
-			},
-		}
-
-		volume, err := c.getVolume(ctx, request)
-		if err != nil {
-			return false, err
-		}
-
-		if len(volume.Attachments) == 0 {
-			if state == "detached" {
-				return true, nil
-			}
-		}
-
-		for _, a := range volume.Attachments {
-			if a.State == nil {
-				klog.Warningf("Ignoring nil attachment state for volume %q: %v", volumeID, a)
-				continue
-			}
-			if *a.State == state {
-				return true, nil
-			}
-		}
-		return false, nil
-	}
-
-	return wait.ExponentialBackoff(backoff, verifyVolumeFunc)
-}
-
 // waitForVolume waits for volume to be in the "available" state.
 // On a random AWS account (shared among several developers) it took 4s on average.
 func (c *cloud) waitForVolume(ctx context.Context, volumeID string) error {

pkg/cloud/fakes.go

@@ -83,6 +83,10 @@ func (c *FakeCloudProvider) DetachDisk(ctx context.Context, volumeID, nodeID str
 	return nil
 }
 
+func (c *FakeCloudProvider) WaitForAttachmentState(ctx context.Context, volumeID, state string) error {
+	return nil
+}
+
 func (c *FakeCloudProvider) GetDiskByName(ctx context.Context, name string, capacityBytes int64) (*Disk, error) {
 	var disks []*fakeDisk
 	for _, d := range c.disks {

pkg/driver/controller.go

@@ -308,13 +308,13 @@ func pickAvailabilityZone(requirement *csi.TopologyRequirement) string {
 		return ""
 	}
 	for _, topology := range requirement.GetPreferred() {
-		zone, exists := topology.GetSegments()[topologyKey]
+		zone, exists := topology.GetSegments()[TopologyKey]
 		if exists {
 			return zone
 		}
 	}
 	for _, topology := range requirement.GetRequisite() {
-		zone, exists := topology.GetSegments()[topologyKey]
+		zone, exists := topology.GetSegments()[TopologyKey]
 		if exists {
 			return zone
 		}
@@ -332,7 +332,7 @@ func newCreateVolumeResponse(disk *cloud.Disk) *csi.CreateVolumeResponse {
 			},
 			AccessibleTopology: []*csi.Topology{
 				{
-					Segments: map[string]string{topologyKey: disk.AvailabilityZone},
+					Segments: map[string]string{TopologyKey: disk.AvailabilityZone},
 				},
 			},
 		},

pkg/driver/controller_test.go

@@ -233,7 +233,7 @@ func TestCreateVolume(t *testing.T) {
 				AccessibilityRequirements: &csi.TopologyRequirement{
 					Requisite: []*csi.Topology{
 						{
-							Segments: map[string]string{topologyKey: expZone},
+							Segments: map[string]string{TopologyKey: expZone},
 						},
 					},
 				},
@@ -248,7 +248,7 @@ func TestCreateVolume(t *testing.T) {
 				AccessibilityRequirements: &csi.TopologyRequirement{
 					Requisite: []*csi.Topology{
 						{
-							Segments: map[string]string{topologyKey: expZone},
+							Segments: map[string]string{TopologyKey: expZone},
 						},
 					},
 				},
@@ -259,7 +259,7 @@ func TestCreateVolume(t *testing.T) {
 				VolumeContext: map[string]string{"fsType": expFsType},
 				AccessibleTopology: []*csi.Topology{
 					{
-						Segments: map[string]string{topologyKey: expZone},
+						Segments: map[string]string{TopologyKey: expZone},
 					},
 				},
 			},
@@ -382,12 +382,12 @@ func TestPickAvailabilityZone(t *testing.T) {
 			requirement: &csi.TopologyRequirement{
 				Requisite: []*csi.Topology{
 					{
-						Segments: map[string]string{topologyKey: expZone},
+						Segments: map[string]string{TopologyKey: expZone},
 					},
 				},
 				Preferred: []*csi.Topology{
 					{
-						Segments: map[string]string{topologyKey: expZone},
+						Segments: map[string]string{TopologyKey: expZone},
 					},
 				},
 			},
@@ -398,7 +398,7 @@ func TestPickAvailabilityZone(t *testing.T) {
 			requirement: &csi.TopologyRequirement{
 				Requisite: []*csi.Topology{
 					{
-						Segments: map[string]string{topologyKey: expZone},
+						Segments: map[string]string{TopologyKey: expZone},
 					},
 				},
 			},

pkg/driver/driver.go

@@ -31,7 +31,7 @@ import (
 
 const (
 	DriverName  = "ebs.csi.aws.com"
-	topologyKey = "topology." + DriverName + "/zone"
+	TopologyKey = "topology." + DriverName + "/zone"
 )
 
 type Driver struct {

pkg/driver/node.go

@@ -265,7 +265,7 @@ func (d *Driver) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (
 	m := d.cloud.GetMetadata()
 
 	topology := &csi.Topology{
-		Segments: map[string]string{topologyKey: m.GetAvailabilityZone()},
+		Segments: map[string]string{TopologyKey: m.GetAvailabilityZone()},
 	}
 
 	return &csi.NodeGetInfoResponse{

pkg/driver/node_test.go

@@ -603,7 +603,7 @@ func TestNodeGetInfo(t *testing.T) {
 	expResp := &csi.NodeGetInfoResponse{
 		NodeId: "instanceID",
 		AccessibleTopology: &csi.Topology{
-			Segments: map[string]string{topologyKey: m.GetAvailabilityZone()},
+			Segments: map[string]string{TopologyKey: m.GetAvailabilityZone()},
 		},
 	}