Skip to content

Commit

Permalink
Merge pull request kubernetes-sigs#1769 from AndrewSirenko/formatting…
Browse files Browse the repository at this point in the history
…OptionE2eTestsSimpler

Add e2e tests for format options
  • Loading branch information
k8s-ci-robot authored Oct 6, 2023
2 parents 280157f + 81acc24 commit 7ca4149
Show file tree
Hide file tree
Showing 7 changed files with 244 additions and 27 deletions.
7 changes: 6 additions & 1 deletion tests/e2e/README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
## E2E Testing
E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc.
E2E test verifies the functionality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc.

### Requirements
1. AWS credential is [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
Expand All @@ -10,6 +10,11 @@ E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernet
1. sed
1. Golang 1.11+

### Running a specific test
1. Make sure you have a cluster up with ebs csi driver deployed
2. Current directory is `/aws-ebs-csi-driver/tests/e2e`
3. Run `'ginkgo run --focus='should create multiple PV objects, bind to PVCs and attach all to a single pod'`

### Notes
Some tests marked with `[env]` require specific environmental variables to be set, if not set these tests will be skipped.

Expand Down
6 changes: 5 additions & 1 deletion tests/e2e/driver/ebs_csi_driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (d *ebsCSIDriver) GetPersistentVolume(volumeID string, fsType string, size
}

// GetParameters returns the parameters specific for this driver
func GetParameters(volumeType string, fsType string, encrypted bool) map[string]string {
func GetParameters(volumeType string, fsType string, encrypted bool, additionalParameters map[string]string) map[string]string {
parameters := map[string]string{
"type": volumeType,
"csi.storage.k8s.io/fstype": fsType,
Expand All @@ -120,6 +120,10 @@ func GetParameters(volumeType string, fsType string, encrypted bool) map[string]
if encrypted {
parameters[ebscsidriver.EncryptedKey] = True
}
for k, v := range additionalParameters {
parameters[k] = v
}

return parameters
}

Expand Down
89 changes: 89 additions & 0 deletions tests/e2e/format_options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"fmt"
ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites"
. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
admissionapi "k8s.io/pod-security-admission/api"
)

var (
testedFsTypes = []string{ebscsidriver.FSTypeExt4}

formatOptionTests = []testsuites.FormatOptionTest{
{
CreateVolumeParameterKey: ebscsidriver.BlockSizeKey,
CreateVolumeParameterValue: "1024",
},
{
CreateVolumeParameterKey: ebscsidriver.INodeSizeKey,
CreateVolumeParameterValue: "512",
},
{
CreateVolumeParameterKey: ebscsidriver.BytesPerINodeKey,
CreateVolumeParameterValue: "8192",
},
{
CreateVolumeParameterKey: ebscsidriver.NumberOfINodesKey,
CreateVolumeParameterValue: "200192",
},
}
)

var _ = Describe("[ebs-csi-e2e] [single-az] [format-options] Formatting a volume", func() {
f := framework.NewDefaultFramework("ebs")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged

var (
cs clientset.Interface
ns *v1.Namespace
ebsDriver driver.PVTestDriver
)

BeforeEach(func() {
cs = f.ClientSet
ns = f.Namespace
ebsDriver = driver.InitEbsCSIDriver()
})

for _, fsType := range testedFsTypes {
Context(fmt.Sprintf("using an %s filesystem", fsType), func() {
for _, formatOptionTestCase := range formatOptionTests {
formatOptionTestCase := formatOptionTestCase
if fsTypeDoesNotSupportFormatOptionParameter(fsType, formatOptionTestCase.CreateVolumeParameterKey) {
continue
}

Context(fmt.Sprintf("with a custom %s parameter", formatOptionTestCase.CreateVolumeParameterKey), func() {
It("successfully mounts and is resizable", func() {
formatOptionTestCase.Run(cs, ns, ebsDriver, fsType)
})
})
}
})
}
})

func fsTypeDoesNotSupportFormatOptionParameter(fsType string, createVolumeParameterKey string) bool {
_, paramNotSupported := ebscsidriver.FileSystemConfigs[fsType].NotSupportedParams[createVolumeParameterKey]
return paramNotSupported
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,25 +45,7 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface,
tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
defer tpvc.Cleanup()

pvcName := tpvc.persistentVolumeClaim.Name
pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), pvcName, metav1.GetOptions{})
By(fmt.Sprintf("Get pvc name: %v", pvc.Name))
originalSize := pvc.Spec.Resources.Requests["storage"]
delta := resource.Quantity{}
delta.Set(util.GiBToBytes(1))
originalSize.Add(delta)
pvc.Spec.Resources.Requests["storage"] = originalSize

By("resizing the pvc")
updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
if err != nil {
framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvcName, err))
}
updatedSize := updatedPvc.Spec.Resources.Requests["storage"]

By("checking the resizing PV result")
error := WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second)
framework.ExpectNoError(error)
ResizeTestPvc(client, namespace, tpvc, 1)

By("Validate volume can be attached")
tpod := NewTestPod(client, namespace, t.Pod.Cmd)
Expand Down Expand Up @@ -92,3 +74,28 @@ func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, d
}
return fmt.Errorf("Gave up after waiting %v for pv %q to complete resizing", timeout, pvName)
}

// ResizeTestPvc increases size of given persistent volume claim by `sizeIncreaseGi` Gigabytes
func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc *TestPersistentVolumeClaim, sizeIncreaseGi int64) (updatedPvc *v1.PersistentVolumeClaim, updatedSize resource.Quantity) {
By(fmt.Sprintf("getting pvc name: %v", testPvc.persistentVolumeClaim.Name))
pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), testPvc.persistentVolumeClaim.Name, metav1.GetOptions{})

originalSize := pvc.Spec.Resources.Requests["storage"]
delta := resource.Quantity{}
delta.Set(util.GiBToBytes(sizeIncreaseGi))
originalSize.Add(delta)
pvc.Spec.Resources.Requests["storage"] = originalSize

By("resizing the pvc")
updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
if err != nil {
framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvc.Name, err))
}
updatedSize = updatedPvc.Spec.Resources.Requests["storage"]

By("checking the resizing PV result")
err = WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second)
framework.ExpectNoError(err)

return
}
26 changes: 26 additions & 0 deletions tests/e2e/testsuites/e2e_utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testsuites

import "fmt"

const (
DefaultVolumeName = "test-volume-1"
DefaultMountPath = "/mnt/default-mount"
)

func PodCmdWriteToVolume(volumeMountPath string) string {
return fmt.Sprintf("echo 'hello world' >> %s/data && grep 'hello world' %s/data && sync", volumeMountPath, volumeMountPath)
}
87 changes: 87 additions & 0 deletions tests/e2e/testsuites/format_options_tester.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testsuites

import (
awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
)

// FormatOptionTest will provision required StorageClass(es), PVC(s) and Pod(s) in order to test that volumes with
// a specified custom format options will mount and are able to be later resized.
type FormatOptionTest struct {
CreateVolumeParameterKey string
CreateVolumeParameterValue string
}

const (
volumeSizeIncreaseAmtGi = 1
)

func (t *FormatOptionTest) Run(client clientset.Interface, namespace *v1.Namespace, ebsDriver driver.PVTestDriver, fsType string) {
By("setting up pvc with custom format option")
volumeDetails := createFormatOptionVolumeDetails(fsType, t)
testPvc, _ := volumeDetails.SetupDynamicPersistentVolumeClaim(client, namespace, ebsDriver)
defer testPvc.Cleanup()

By("deploying pod with custom format option")
formatOptionMountPod := createPodWithVolume(client, namespace, PodCmdWriteToVolume(DefaultMountPath), testPvc, volumeDetails)
defer formatOptionMountPod.Cleanup()
formatOptionMountPod.WaitForSuccess()

By("testing that pvc is able to be resized")
ResizeTestPvc(client, namespace, testPvc, volumeSizeIncreaseAmtGi)

By("validating resized pvc by deploying new pod")
resizeTestPod := createPodWithVolume(client, namespace, PodCmdWriteToVolume(DefaultMountPath), testPvc, volumeDetails)
defer resizeTestPod.Cleanup()

By("confirming new pod can write to resized volume")
resizeTestPod.WaitForSuccess()
}

func createFormatOptionVolumeDetails(fsType string, t *FormatOptionTest) *VolumeDetails {
additionalParameters := map[string]string{
t.CreateVolumeParameterKey: t.CreateVolumeParameterValue,
}

allowVolumeExpansion := true

volume := VolumeDetails{
VolumeType: awscloud.VolumeTypeGP2,
FSType: fsType,
MountOptions: []string{"rw"},
ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2),
VolumeMount: VolumeMountDetails{
NameGenerate: DefaultVolumeName,
MountPathGenerate: DefaultMountPath,
},
AllowVolumeExpansion: &allowVolumeExpansion,
AdditionalParameters: additionalParameters,
}

return &volume
}

func createPodWithVolume(client clientset.Interface, namespace *v1.Namespace, cmd string, testPvc *TestPersistentVolumeClaim, volumeDetails *VolumeDetails) *TestPod {
testPod := NewTestPod(client, namespace, cmd)
testPod.SetupVolume(testPvc.persistentVolumeClaim, volumeDetails.VolumeMount.NameGenerate, volumeDetails.VolumeMount.MountPathGenerate, volumeDetails.VolumeMount.ReadOnly)
testPod.Create()

return testPod
}
11 changes: 5 additions & 6 deletions tests/e2e/testsuites/specs.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,9 @@ type VolumeDetails struct {
VolumeMode VolumeMode
VolumeMount VolumeMountDetails
VolumeDevice VolumeDeviceDetails
// Optional, used with pre-provisioned volumes
VolumeID string
// Optional, used with PVCs created from snapshots
DataSource *DataSource
VolumeID string // Optional, used with pre-provisioned volumes
DataSource *DataSource // Optional, used with PVCs created from snapshots
AdditionalParameters map[string]string // Optional, used when testing formatting options
}

type VolumeMode int
Expand Down Expand Up @@ -121,7 +120,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
volume := pod.Volumes[0]
By("setting up the StorageClass")

storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted, volume.AdditionalParameters), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
tsc := NewTestStorageClass(client, namespace, storageClass)
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
Expand All @@ -141,7 +140,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPersistentVolumeClaim, []func()) {
cleanupFuncs := make([]func(), 0)
By("setting up the StorageClass")
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted, volume.AdditionalParameters), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
tsc := NewTestStorageClass(client, namespace, storageClass)
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
Expand Down

0 comments on commit 7ca4149

Please sign in to comment.