SPBM policy ID support in vsphere cloud provider

examples/volumes/vsphere/README.md

@@ -5,7 +5,9 @@
     - [Volumes](#volumes)
     - [Persistent Volumes](#persistent-volumes)
     - [Storage Class](#storage-class)
-    - [Virtual SAN policy support inside Kubernetes](#virtual-san-policy-support-inside-kubernetes)
+    - [Storage Policy Management inside kubernetes] (#storage-policy-management-inside-kubernetes)
+      - [Using existing vCenter SPBM policy] (#using-existing-vcenter-spbm-policy)
+      - [Virtual SAN policy support](#virtual-san-policy-support)
     - [Stateful Set](#stateful-set)
 
 ## Prerequisites
@@ -374,7 +376,47 @@
       pvpod       1/1     Running   0          48m
       ```
 
-### Virtual SAN policy support inside Kubernetes
+### Storage Policy Management inside kubernetes
+#### Using existing vCenter SPBM policy
+  Admins can use the existing vCenter Storage Policy Based Management (SPBM) policy to configure a persistent volume with the SPBM policy.
+
+  __Note: Here you don't need to create persistent volume it is created for you.__
+   1. Create Storage Class.
+
+      Example 1:
+
+      ```yaml
+      kind: StorageClass
+      apiVersion: storage.k8s.io/v1beta1
+      metadata:
+        name: fast
+      provisioner: kubernetes.io/vsphere-volume
+      parameters:
+          diskformat: zeroedthick
+          storagePolicyName: gold
+      ```
+      [Download example](vsphere-volume-spbm-policy.yaml?raw=true)
+
+      The admin specifies the SPBM policy - "gold" as part of storage class definition for dynamic volume provisioning. When a PVC is created, the persistent volume will be provisioned on a compatible datastore with maximum free space that satisfies the "gold" storage policy requirements.
+
+      Example 2:
+
+      ```yaml
+      kind: StorageClass
+      apiVersion: storage.k8s.io/v1beta1
+      metadata:
+        name: fast
+      provisioner: kubernetes.io/vsphere-volume
+      parameters:
+          diskformat: zeroedthick
+          storagePolicyName: gold
+          datastore: VSANDatastore
+      ```
+      [Download example](vsphere-volume-spbm-policy-with-datastore.yaml?raw=true)
+
+      The admin can also specify a custom datastore where he wants the volume to be provisioned along with the SPBM policy name. When a PVC is created, the vSphere Cloud Provider checks if the user specified datastore satisfies the "gold" storage policy requirements. If yes, it will provision the persistent volume on user specified datastore. If not, it will error out to the user that the user specified datastore is not compatible with "gold" storage policy requirements.
+
+#### Virtual SAN policy support
 
   Vsphere Infrastructure(VI) Admins will have the ability to specify custom Virtual SAN Storage Capabilities during dynamic volume provisioning. You can now define storage requirements, such as performance and availability, in the form of storage capabilities during dynamic volume provisioning. The storage capability requirements are converted into a Virtual SAN policy which are then pushed down to the Virtual SAN layer when a persistent volume (virtual disk) is being created. The virtual disk is distributed across the Virtual SAN datastore to meet the requirements.
 

examples/volumes/vsphere/vsphere-volume-spbm-policy-with-datastore.yaml

@@ -0,0 +1,9 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1beta1
+metadata:
+  name: fast
+provisioner: kubernetes.io/vsphere-volume
+parameters:
+    diskformat: zeroedthick
+    storagePolicyName: gold
+    datastore: VSANDatastore

examples/volumes/vsphere/vsphere-volume-spbm-policy.yaml

@@ -0,0 +1,8 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1beta1
+metadata:
+  name: fast
+provisioner: kubernetes.io/vsphere-volume
+parameters:
+    diskformat: zeroedthick
+    storagePolicyName: gold

pkg/cloudprovider/providers/vsphere/vsphere.go

@@ -45,6 +45,7 @@ import (
 	"github.com/vmware/govmomi/vim25/types"
 	"golang.org/x/net/context"
 
+	pbm "github.com/vmware/govmomi/pbm"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	k8runtime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/api/v1"
@@ -165,7 +166,7 @@ type VSphereConfig struct {
 type Volumes interface {
 	// AttachDisk attaches given disk to given node. Current node
 	// is used when nodeName is empty string.
-	AttachDisk(vmDiskPath string, nodeName k8stypes.NodeName) (diskID string, diskUUID string, err error)
+	AttachDisk(vmDiskPath string, storagePolicyID string, nodeName k8stypes.NodeName) (diskID string, diskUUID string, err error)
 
 	// DetachDisk detaches given disk to given node. Current node
 	// is used when nodeName is empty string.
@@ -189,12 +190,14 @@ type Volumes interface {
 
 // VolumeOptions specifies capacity, tags, name and diskFormat for a volume.
 type VolumeOptions struct {
-	CapacityKB         int
-	Tags               map[string]string
-	Name               string
-	DiskFormat         string
-	Datastore          string
-	StorageProfileData string
+	CapacityKB             int
+	Tags                   map[string]string
+	Name                   string
+	DiskFormat             string
+	Datastore              string
+	VSANStorageProfileData string
+	StoragePolicyName      string
+	StoragePolicyID        string
 }
 
 // Generates Valid Options for Diskformat
@@ -554,14 +557,12 @@ func (vs *VSphere) NodeAddresses(nodeName k8stypes.NodeName) ([]v1.NodeAddress,
 			addressType = v1.NodeInternalIP
 		}
 		for _, ip := range v.IpAddress {
-			if net.ParseIP(ip).To4() != nil {
-				v1helper.AddToNodeAddresses(&addrs,
-					v1.NodeAddress{
-						Type:    addressType,
-						Address: ip,
-					},
-				)
-			}
+			v1helper.AddToNodeAddresses(&addrs,
+				v1.NodeAddress{
+					Type:    addressType,
+					Address: ip,
+				},
+			)
 		}
 	}
 	return addrs, nil
@@ -737,7 +738,7 @@ func cleanUpController(ctx context.Context, newSCSIController types.BaseVirtualD
 }
 
 // Attaches given virtual disk volume to the compute running kubelet.
-func (vs *VSphere) AttachDisk(vmDiskPath string, nodeName k8stypes.NodeName) (diskID string, diskUUID string, err error) {
+func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyID string, nodeName k8stypes.NodeName) (diskID string, diskUUID string, err error) {
 	var newSCSIController types.BaseVirtualDevice
 
 	// Create context
@@ -785,14 +786,8 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, nodeName k8stypes.NodeName) (di
 			return "", "", err
 		}
 
-		// verify scsi controller in virtual machine
-		vmDevices, err := vm.Device(ctx)
-		if err != nil {
-			return "", "", err
-		}
-
 		// Get VM device list
-		_, vmDevices, _, err = getVirtualMachineDevices(ctx, vs.cfg, vs.client, vSphereInstance)
+		_, vmDevices, _, err := getVirtualMachineDevices(ctx, vs.cfg, vs.client, vSphereInstance)
 		if err != nil {
 			glog.Errorf("cannot get vmDevices for VM err=%s", err)
 			return "", "", fmt.Errorf("cannot get vmDevices for VM err=%s", err)
@@ -811,9 +806,9 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, nodeName k8stypes.NodeName) (di
 
 	// Create a new finder
 	f := find.NewFinder(vs.client.Client, true)
-
 	// Set data center
 	f.SetDatacenter(dc)
+
 	datastorePathObj := new(object.DatastorePath)
 	isSuccess := datastorePathObj.FromString(vmDiskPath)
 	if !isSuccess {
@@ -837,28 +832,54 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, nodeName k8stypes.NodeName) (di
 	backing := disk.Backing.(*types.VirtualDiskFlatVer2BackingInfo)
 	backing.DiskMode = string(types.VirtualDiskModeIndependent_persistent)
 
-	// Attach disk to the VM
-	err = vm.AddDevice(ctx, disk)
+	virtualMachineConfigSpec := types.VirtualMachineConfigSpec{}
+	deviceConfigSpec := &types.VirtualDeviceConfigSpec{
+		Device:    disk,
+		Operation: types.VirtualDeviceConfigSpecOperationAdd,
+	}
+	// Configure the disk with the SPBM profile only if ProfileID is not empty.
+	if storagePolicyID != "" {
+		profileSpec := &types.VirtualMachineDefinedProfileSpec{
+			ProfileId: storagePolicyID,
+		}
+		deviceConfigSpec.Profile = append(deviceConfigSpec.Profile, profileSpec)
+	}
+	virtualMachineConfigSpec.DeviceChange = append(virtualMachineConfigSpec.DeviceChange, deviceConfigSpec)
+	task, err := vm.Reconfigure(ctx, virtualMachineConfigSpec)
 	if err != nil {
-		glog.Errorf("cannot attach disk to the vm - %v", err)
+		glog.Errorf("Failed to attach the disk with storagePolicy: %+q with err - %v", storagePolicyID, err)
 		if newSCSICreated {
 			cleanUpController(ctx, newSCSIController, vmDevices, vm)
 		}
 		return "", "", err
 	}
-
-	vmDevices, err = vm.Device(ctx)
+	err = task.Wait(ctx)
 	if err != nil {
+		glog.Errorf("Failed to attach the disk with storagePolicy: %+q with err - %v", storagePolicyID, err)
 		if newSCSICreated {
 			cleanUpController(ctx, newSCSIController, vmDevices, vm)
 		}
 		return "", "", err
 	}
-	devices := vmDevices.SelectByType(disk)
-	if len(devices) < 1 {
+
+	deviceName, diskUUID, err := getVMDiskInfo(ctx, vm, disk)
+	if err != nil {
 		if newSCSICreated {
 			cleanUpController(ctx, newSCSIController, vmDevices, vm)
 		}
+		vs.DetachDisk(deviceName, nodeName)
+		return "", "", err
+	}
+	return deviceName, diskUUID, nil
+}
+
+func getVMDiskInfo(ctx context.Context, vm *object.VirtualMachine, disk *types.VirtualDisk) (string, string, error) {
+	vmDevices, err := vm.Device(ctx)
+	if err != nil {
+		return "", "", err
+	}
+	devices := vmDevices.SelectByType(disk)
+	if len(devices) < 1 {
 		return "", "", ErrNoDevicesFound
 	}
 
@@ -867,18 +888,13 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, nodeName k8stypes.NodeName) (di
 	deviceName := devices.Name(newDevice)
 
 	// get device uuid
-	diskUUID, err = getVirtualDiskUUID(newDevice)
+	diskUUID, err := getVirtualDiskUUID(newDevice)
 	if err != nil {
-		if newSCSICreated {
-			cleanUpController(ctx, newSCSIController, vmDevices, vm)
-		}
-		vs.DetachDisk(deviceName, nodeName)
 		return "", "", err
 	}
 
 	return deviceName, diskUUID, nil
 }
-
 func getNextUnitNumber(devices object.VirtualDeviceList, c types.BaseVirtualController) (int32, error) {
 	// get next available SCSI controller unit number
 	var takenUnitNumbers [SCSIDeviceSlots]bool
@@ -1266,19 +1282,43 @@ func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string
 	dc, err := f.Datacenter(ctx, vs.cfg.Global.Datacenter)
 	f.SetDatacenter(dc)
 
+	if volumeOptions.StoragePolicyName != "" {
+		// Get the pbm client
+		pbmClient, err := pbm.NewClient(ctx, vs.client.Client)
+		if err != nil {
+			return "", err
+		}
+		volumeOptions.StoragePolicyID, err = pbmClient.ProfileIDByName(ctx, volumeOptions.StoragePolicyName)
+		if err != nil {
+			return "", err
+		}
+		// Get the resource pool for current node.
+		resourcePool, err := vs.getCurrentNodeResourcePool(ctx, dc)
+		if err != nil {
+			return "", err
+		}
+
+		dsRefs, err := vs.GetCompatibleDatastores(ctx, pbmClient, resourcePool, volumeOptions.StoragePolicyID)
+		if err != nil {
+			return "", err
+		}
+
+		if volumeOptions.Datastore != "" {
+			if !IsUserSpecifiedDatastoreCompatible(dsRefs, volumeOptions.Datastore) {
+				return "", fmt.Errorf("User specified datastore: %q is not compatible with the StoragePolicy: %q requirements", volumeOptions.Datastore, volumeOptions.StoragePolicyName)
+			}
+		} else {
+			datastore = GetBestFitCompatibleDatastore(dsRefs)
+		}
+	}
+
 	ds, err := f.Datastore(ctx, datastore)
 	if err != nil {
 		glog.Errorf("Failed while searching for datastore %+q. err %s", datastore, err)
 		return "", err
 	}
 
-	// Create a disk with the VSAN storage capabilities specified in the volumeOptions.StorageProfileData.
-	// This is achieved by following steps:
-	// 1. Create dummy VM if not already present.
-	// 2. Add a new disk to the VM by performing VM reconfigure.
-	// 3. Detach the new disk from the dummy VM.
-	// 4. Delete the dummy VM.
-	if volumeOptions.StorageProfileData != "" {
+	if volumeOptions.VSANStorageProfileData != "" {
 		// Check if the datastore is VSAN if any capability requirements are specified.
 		// VSphere cloud provider now only supports VSAN capabilities requirements
 		ok, err := checkIfDatastoreTypeIsVSAN(vs.client, ds)
@@ -1291,7 +1331,14 @@ func (vs *VSphere) CreateVolume(volumeOptions *VolumeOptions) (volumePath string
 				" The policy parameters will work only with VSAN Datastore."+
 				" So, please specify a valid VSAN datastore in Storage class definition.", datastore)
 		}
-
+	}
+	// Create a disk with the VSAN storage capabilities specified in the volumeOptions.VSANStorageProfileData.
+	// This is achieved by following steps:
+	// 1. Create dummy VM if not already present.
+	// 2. Add a new disk to the VM by performing VM reconfigure.
+	// 3. Detach the new disk from the dummy VM.
+	// 4. Delete the dummy VM.
+	if volumeOptions.VSANStorageProfileData != "" || volumeOptions.StoragePolicyName != "" {
 		// Acquire a read lock to ensure multiple PVC requests can be processed simultaneously.
 		cleanUpDummyVMLock.RLock()
 		defer cleanUpDummyVMLock.RUnlock()
@@ -1562,13 +1609,11 @@ func (vs *VSphere) createDummyVM(ctx context.Context, datacenter *object.Datacen
 	if err != nil {
 		return nil, err
 	}
-
 	// Get the folder reference for global working directory where the dummy VM needs to be created.
 	vmFolder, err := getFolder(ctx, vs.client, vs.cfg.Global.Datacenter, vs.cfg.Global.WorkingDir)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to get the folder reference for %q with err: %+v", vs.cfg.Global.WorkingDir, err)
 	}
-
 	task, err := vmFolder.CreateVM(ctx, virtualMachineConfigSpec, resourcePool, nil)
 	if err != nil {
 		return nil, err
@@ -1665,12 +1710,17 @@ func (vs *VSphere) createVirtualDiskWithPolicy(ctx context.Context, datacenter *
 		FileOperation: types.VirtualDeviceConfigSpecFileOperationCreate,
 	}
 
-	storageProfileSpec := &types.VirtualMachineDefinedProfileSpec{
-		ProfileId: "",
-		ProfileData: &types.VirtualMachineProfileRawData{
+	storageProfileSpec := &types.VirtualMachineDefinedProfileSpec{}
+	// Is PBM storage policy ID is present, set the storage spec profile ID,
+	// else, set raw the VSAN policy string.
+	if volumeOptions.StoragePolicyID != "" {
+		storageProfileSpec.ProfileId = volumeOptions.StoragePolicyID
+	} else if volumeOptions.VSANStorageProfileData != "" {
+		storageProfileSpec.ProfileId = ""
+		storageProfileSpec.ProfileData = &types.VirtualMachineProfileRawData{
 			ExtensionKey: "com.vmware.vim.sps",
-			ObjectData:   volumeOptions.StorageProfileData,
-		},
+			ObjectData:   volumeOptions.VSANStorageProfileData,
+		}
 	}
 
 	deviceConfigSpec.Profile = append(deviceConfigSpec.Profile, storageProfileSpec)

pkg/cloudprovider/providers/vsphere/vsphere_test.go

@@ -232,7 +232,7 @@ func TestVolumes(t *testing.T) {
 		t.Fatalf("Cannot create a new VMDK volume: %v", err)
 	}
 
-	_, _, err = vs.AttachDisk(volPath, "")
+	_, _, err = vs.AttachDisk(volPath, "", "")
 	if err != nil {
 		t.Fatalf("Cannot attach volume(%s) to VM(%s): %v", volPath, nodeName, err)
 	}