Skip to content

Commit

Permalink
use vm.Status.Network.PrimaryIP4 for VDS
Browse files Browse the repository at this point in the history
  • Loading branch information
@adikul30
adikul30 committed Mar 23, 2024
1 parent b3403f7 commit 0a3698e
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 121 deletions.
40 changes: 9 additions & 31 deletions pkg/syncer/cnsoperator/util/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
vmoperatortypes "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
apitypes "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
Expand Down Expand Up @@ -104,7 +103,7 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.
return "", err
}

var networkName, interfaceName string
var networkName string

// NSX
// network:
Expand Down Expand Up @@ -132,15 +131,14 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.
// The assumption is that a TKG VM will have only a single network interface.
// This logic needs to be revisited when multiple network interface support
// is added.

interfaceName = networkInterface.Name
networkName = networkInterface.Network.Name
}
log.Debugf("VirtualMachine %s/%s is configured with network %s", vmNamespace, vmName, networkName)

var ip string
var exists bool
if nsxConfiguration {
// todo: don't rely on CRs internal to VM object
virtualNetworkInstance, err := dc.Resource(virtualNetworkGVR).Namespace(vmNamespace).Get(ctx,
networkName, metav1.GetOptions{})
if err != nil {
Expand All @@ -153,34 +151,14 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.
return "", fmt.Errorf("failed to get SNAT IP annotation from VirtualMachine %s/%s", vmNamespace, vmName)
}
} else {
networkInterfaceName := vmName + "-" + networkName + "-" + interfaceName
networkInterfaceInstance, err := dc.Resource(networkInterfaceGVR).Namespace(vmNamespace).Get(ctx,
networkInterfaceName, metav1.GetOptions{})
if err != nil {
return "", err
}
log.Debugf("Got NetworkInterface instance %+v", networkInterfaceInstance)
ipConfigs, exists, err := unstructured.NestedSlice(networkInterfaceInstance.Object, "status", "ipConfigs")
if err != nil {
return "", err
}
if !exists {
return "", fmt.Errorf("status.ipConfigs does not exist in NetworkInterface instance %s/%s",
vmNamespace, networkInterfaceName)
}
if len(ipConfigs) == 0 {
return "", fmt.Errorf("length of status.ipConfigs should be greater than one for NetworkInterface instance %s/%s",
vmNamespace, networkInterfaceName)
networkStatus := virtualMachineInstance.Status.Network
if networkStatus == nil {
return "", fmt.Errorf("vm.Status.Network is not populated for %s/%s", vmNamespace, vmName)
}
// Assuming only a single ipConfig is supported per VM. Revisit this logic
// when multiple ipConfigs are supported.
ip, exists, err = unstructured.NestedString(ipConfigs[0].(map[string]interface{}), "ip")
if err != nil {
return "", err
}
if !exists {
return "", fmt.Errorf("status.ipConfigs.ip does not exist in NetworkInterface instance %s/%s",
vmNamespace, networkInterfaceName)

ip = networkStatus.PrimaryIP4
if ip == "" {
return "", fmt.Errorf("vm.Status.Network.PrimaryIP4 is not populated for %s/%s", vmNamespace, vmName)
}
}
log.Infof("Found external IP Address %s for VirtualMachine %s/%s", ip, vmNamespace, vmName)
Expand Down
86 changes: 44 additions & 42 deletions tests/e2e/vmservice_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import (
"github.com/vmware-tanzu/vm-operator/api/v1alpha2/common"
"golang.org/x/crypto/ssh"

vmopv2 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
vmopv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha2"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -44,6 +44,7 @@ import (
"k8s.io/kubernetes/test/e2e/framework"
fssh "k8s.io/kubernetes/test/e2e/framework/ssh"
ctlrclient "sigs.k8s.io/controller-runtime/pkg/client"

cnsnodevmattachmentv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/cnsnodevmattachment/v1alpha1"
)

Expand Down Expand Up @@ -252,7 +253,7 @@ func invokeVCRestAPIDeleteRequest(vcRestSessionId string, url string) ([]byte, i
func waitNGetVmiForImageName(ctx context.Context, c ctlrclient.Client, namespace string, imageName string) string {
vmi := ""
err := wait.PollImmediate(poll*5, pollTimeout, func() (bool, error) {
vmImagesList := &vmopv2.VirtualMachineImageList{}
vmImagesList := &vmopv1.VirtualMachineImageList{}
err := c.List(ctx, vmImagesList)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
for _, instance := range vmImagesList.Items {
Expand All @@ -270,30 +271,30 @@ func waitNGetVmiForImageName(ctx context.Context, c ctlrclient.Client, namespace

// createVmServiceVmWithPvcs creates VM via VM service with given ns, sc, vmi, pvc(s) and bootstrap data for cloud init
func createVmServiceVmWithPvcs(ctx context.Context, c ctlrclient.Client, namespace string, vmClass string,
pvcs []*v1.PersistentVolumeClaim, vmi string, storageClassName string, secretName string) *vmopv2.VirtualMachine {
pvcs []*v1.PersistentVolumeClaim, vmi string, storageClassName string, secretName string) *vmopv1.VirtualMachine {

r := rand.New(rand.NewSource(time.Now().UnixNano()))
vols := []vmopv2.VirtualMachineVolume{}
vols := []vmopv1.VirtualMachineVolume{}
vmName := fmt.Sprintf("csi-test-vm-%d", r.Intn(10000))
for _, pvc := range pvcs {
vols = append(vols, vmopv2.VirtualMachineVolume{
vols = append(vols, vmopv1.VirtualMachineVolume{
Name: pvc.Name,
VirtualMachineVolumeSource: vmopv2.VirtualMachineVolumeSource{
PersistentVolumeClaim: &vmopv2.PersistentVolumeClaimVolumeSource{
VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{
PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{
PersistentVolumeClaimVolumeSource: v1.PersistentVolumeClaimVolumeSource{ClaimName: pvc.Name},
},
},
})
}
vm := vmopv2.VirtualMachine{
vm := vmopv1.VirtualMachine{
ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: namespace},
Spec: vmopv2.VirtualMachineSpec{
PowerState: vmopv2.VirtualMachinePowerStateOn,
Spec: vmopv1.VirtualMachineSpec{
PowerState: vmopv1.VirtualMachinePowerStateOn,
ImageName: vmi,
ClassName: vmClass,
StorageClass: storageClassName,
Volumes: vols,
Bootstrap: &vmopv2.VirtualMachineBootstrapSpec{CloudInit: &vmopv2.VirtualMachineBootstrapCloudInitSpec{
Bootstrap: &vmopv1.VirtualMachineBootstrapSpec{CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{
RawCloudConfig: &common.SecretKeySelector{
Name: secretName,
Key: "user-data",
Expand All @@ -309,16 +310,16 @@ func createVmServiceVmWithPvcs(ctx context.Context, c ctlrclient.Client, namespa

// getVmsvcVM fetches the vm from the specified ns
func getVmsvcVM(
ctx context.Context, c ctlrclient.Client, namespace string, vmName string) (*vmopv2.VirtualMachine, error) {
ctx context.Context, c ctlrclient.Client, namespace string, vmName string) (*vmopv1.VirtualMachine, error) {
instanceKey := ctlrclient.ObjectKey{Name: vmName, Namespace: namespace}
vm := &vmopv2.VirtualMachine{}
vm := &vmopv1.VirtualMachine{}
err := c.Get(ctx, instanceKey, vm)
return vm, err
}

// waitNgetVmsvcVM wait and fetch the vm CR from the specified ns
func waitNgetVmsvcVM(ctx context.Context, c ctlrclient.Client, namespace string, vmName string) *vmopv2.VirtualMachine {
vm := &vmopv2.VirtualMachine{}
func waitNgetVmsvcVM(ctx context.Context, c ctlrclient.Client, namespace string, vmName string) *vmopv1.VirtualMachine {
vm := &vmopv1.VirtualMachine{}
var err error
err = wait.PollImmediate(poll*5, pollTimeout, func() (bool, error) {
vm, err = getVmsvcVM(ctx, c, namespace, vmName)
Expand Down Expand Up @@ -346,10 +347,11 @@ func waitNgetVmsvcVmIp(ctx context.Context, c ctlrclient.Client, namespace strin
}
return false, nil
}
if vm.Status.Network.PrimaryIP4 == "" {
networkStatus := vm.Status.Network
if networkStatus == nil || networkStatus.PrimaryIP4 == "" {
return false, nil
}
ip = vm.Status.Network.PrimaryIP4
ip = networkStatus.PrimaryIP4
return true, nil
})
framework.Logf("Found IP '%s' for VM '%s'", ip, name)
Expand Down Expand Up @@ -377,12 +379,12 @@ users:

// createService4Vm creates a virtualmachineservice(loadbalancer) for given vm in the specified ns
func createService4Vm(
ctx context.Context, c ctlrclient.Client, namespace string, vmName string) *vmopv2.VirtualMachineService {
ctx context.Context, c ctlrclient.Client, namespace string, vmName string) *vmopv1.VirtualMachineService {
svcName := vmName + "-svc"
vmService := vmopv2.VirtualMachineService{
vmService := vmopv1.VirtualMachineService{
ObjectMeta: metav1.ObjectMeta{Name: svcName, Namespace: namespace},
Spec: vmopv2.VirtualMachineServiceSpec{
Ports: []vmopv2.VirtualMachineServicePort{{Name: "ssh", Port: 22, Protocol: "TCP", TargetPort: 22}},
Spec: vmopv1.VirtualMachineServiceSpec{
Ports: []vmopv1.VirtualMachineServicePort{{Name: "ssh", Port: 22, Protocol: "TCP", TargetPort: 22}},
Type: "LoadBalancer",
Selector: map[string]string{"app": "vmName"},
},
Expand All @@ -394,17 +396,17 @@ func createService4Vm(

// getVmsvcVmLbSvc fetches the virtualmachineservice(loadbalancer) for given vm in the specified ns
func getVmsvcVmLbSvc(ctx context.Context, c ctlrclient.Client, namespace string, name string) (
*vmopv2.VirtualMachineService, error) {
*vmopv1.VirtualMachineService, error) {
instanceKey := ctlrclient.ObjectKey{Name: name, Namespace: namespace}
svc := &vmopv2.VirtualMachineService{}
svc := &vmopv1.VirtualMachineService{}
err := c.Get(ctx, instanceKey, svc)
return svc, err
}

// waitNgetVmLbSvc wait and fetches the virtualmachineservice(loadbalancer) for given vm in the specified ns
func waitNgetVmLbSvc(
ctx context.Context, c ctlrclient.Client, namespace string, name string) *vmopv2.VirtualMachineService {
vmLbSvc := &vmopv2.VirtualMachineService{}
ctx context.Context, c ctlrclient.Client, namespace string, name string) *vmopv1.VirtualMachineService {
vmLbSvc := &vmopv1.VirtualMachineService{}
var err error
err = wait.PollImmediate(poll*5, pollTimeout, func() (bool, error) {
vmLbSvc, err = getVmsvcVmLbSvc(ctx, c, namespace, name)
Expand All @@ -422,7 +424,7 @@ func waitNgetVmLbSvc(

// verifyPvcsAreAttachedToVmsvcVm verify given pvc(s) is(are) attached to given VM via vm and cnsnodevmattachment CRs
func verifyPvcsAreAttachedToVmsvcVm(ctx context.Context, cnsc ctlrclient.Client,
vm *vmopv2.VirtualMachine, pvcs []*v1.PersistentVolumeClaim) bool {
vm *vmopv1.VirtualMachine, pvcs []*v1.PersistentVolumeClaim) bool {

attachmentmap := map[string]int{}
pvcmap := map[string]int{}
Expand Down Expand Up @@ -487,7 +489,7 @@ func getCnsNodeVmAttachmentCR(

// waitNverifyPvcsAreAttachedToVmsvcVm wait for pvc(s) th be attached to VM via vm and cnsnodevmattachment CRs
func waitNverifyPvcsAreAttachedToVmsvcVm(ctx context.Context, vmopC ctlrclient.Client, cnsopC ctlrclient.Client,
vm *vmopv2.VirtualMachine, pvcs []*v1.PersistentVolumeClaim) error {
vm *vmopv1.VirtualMachine, pvcs []*v1.PersistentVolumeClaim) error {

err := wait.PollImmediate(poll*5, pollTimeout, func() (bool, error) {
vm, err := getVmsvcVM(ctx, vmopC, vm.Namespace, vm.Name)
Expand Down Expand Up @@ -658,7 +660,7 @@ func getSshClientForVmThroughGatewayVm(vmIp string) (*ssh.Client, *ssh.Client) {

// wait4PvcAttachmentFailure waits for PVC attachment to given VM to fail
func wait4PvcAttachmentFailure(
ctx context.Context, vmopC ctlrclient.Client, vm *vmopv2.VirtualMachine, pvc *v1.PersistentVolumeClaim) error {
ctx context.Context, vmopC ctlrclient.Client, vm *vmopv1.VirtualMachine, pvc *v1.PersistentVolumeClaim) error {
var returnErr error
waitErr := wait.PollImmediate(poll*5, pollTimeout, func() (bool, error) {
vm, err := getVmsvcVM(ctx, vmopC, vm.Namespace, vm.Name)
Expand Down Expand Up @@ -703,8 +705,8 @@ func mountFormattedVol2Vm(diskUuid string, mountIndex int, vmIp string) string {

// setVmPowerState sets expected power state for the VM
func setVmPowerState(
ctx context.Context, c ctlrclient.Client, vm *vmopv2.VirtualMachine,
powerState vmopv2.VirtualMachinePowerState) *vmopv2.VirtualMachine {
ctx context.Context, c ctlrclient.Client, vm *vmopv1.VirtualMachine,
powerState vmopv1.VirtualMachinePowerState) *vmopv1.VirtualMachine {

vm, err := getVmsvcVM(ctx, c, vm.Namespace, vm.Name) // refresh vm info
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expand All @@ -718,7 +720,7 @@ func setVmPowerState(

// wait4Vm2ReachPowerStateInSpec wait for VM to reach expected power state
func wait4Vm2ReachPowerStateInSpec(
ctx context.Context, c ctlrclient.Client, vm *vmopv2.VirtualMachine) (*vmopv2.VirtualMachine, error) {
ctx context.Context, c ctlrclient.Client, vm *vmopv1.VirtualMachine) (*vmopv1.VirtualMachine, error) {

var err error
waitErr := wait.PollImmediate(poll*5, pollTimeout, func() (bool, error) {
Expand All @@ -739,32 +741,32 @@ func wait4Vm2ReachPowerStateInSpec(
// cloud init on given zone
func createVmServiceVmWithPvcsWithZone(ctx context.Context, c ctlrclient.Client, namespace string, vmClass string,
pvcs []*v1.PersistentVolumeClaim, vmi string, storageClassName string, secretName string,
zone string) *vmopv2.VirtualMachine {
zone string) *vmopv1.VirtualMachine {

r := rand.New(rand.NewSource(time.Now().UnixNano()))
vols := []vmopv2.VirtualMachineVolume{}
vols := []vmopv1.VirtualMachineVolume{}
vmName := fmt.Sprintf("csi-test-vm-%d", r.Intn(10000))
for _, pvc := range pvcs {
vols = append(vols, vmopv2.VirtualMachineVolume{
vols = append(vols, vmopv1.VirtualMachineVolume{
Name: pvc.Name,
VirtualMachineVolumeSource: vmopv2.VirtualMachineVolumeSource{
PersistentVolumeClaim: &vmopv2.PersistentVolumeClaimVolumeSource{
VirtualMachineVolumeSource: vmopv1.VirtualMachineVolumeSource{
PersistentVolumeClaim: &vmopv1.PersistentVolumeClaimVolumeSource{
PersistentVolumeClaimVolumeSource: v1.PersistentVolumeClaimVolumeSource{ClaimName: pvc.Name},
},
},
})
}
labels := make(map[string]string)
labels["topology.kubernetes.io/zone"] = zone
vm := vmopv2.VirtualMachine{
vm := vmopv1.VirtualMachine{
ObjectMeta: metav1.ObjectMeta{Name: vmName, Namespace: namespace, Labels: labels},
Spec: vmopv2.VirtualMachineSpec{
PowerState: vmopv2.VirtualMachinePowerStateOn,
Spec: vmopv1.VirtualMachineSpec{
PowerState: vmopv1.VirtualMachinePowerStateOn,
ImageName: vmi,
ClassName: vmClass,
StorageClass: storageClassName,
Volumes: vols,
Bootstrap: &vmopv2.VirtualMachineBootstrapSpec{CloudInit: &vmopv2.VirtualMachineBootstrapCloudInitSpec{
Bootstrap: &vmopv1.VirtualMachineBootstrapSpec{CloudInit: &vmopv1.VirtualMachineBootstrapCloudInitSpec{
RawCloudConfig: &common.SecretKeySelector{
Name: secretName,
Key: "user-data",
Expand All @@ -779,7 +781,7 @@ func createVmServiceVmWithPvcsWithZone(ctx context.Context, c ctlrclient.Client,
}

// wait4VmSvcVm2BeDeleted waits for the given vmservice vm to get deleted
func wait4VmSvcVm2BeDeleted(ctx context.Context, c ctlrclient.Client, vm *vmopv2.VirtualMachine) {
func wait4VmSvcVm2BeDeleted(ctx context.Context, c ctlrclient.Client, vm *vmopv1.VirtualMachine) {
waitErr := wait.PollImmediate(poll*5, pollTimeout, func() (bool, error) {
_, err := getVmsvcVM(ctx, c, vm.Namespace, vm.Name)
if err != nil {
Expand All @@ -795,7 +797,7 @@ func wait4VmSvcVm2BeDeleted(ctx context.Context, c ctlrclient.Client, vm *vmopv2

// wait4Pvc2Detach waits for PVC to detach from given VM
func wait4Pvc2Detach(
ctx context.Context, vmopC ctlrclient.Client, vm *vmopv2.VirtualMachine, pvc *v1.PersistentVolumeClaim) {
ctx context.Context, vmopC ctlrclient.Client, vm *vmopv1.VirtualMachine, pvc *v1.PersistentVolumeClaim) {
waitErr := wait.PollImmediate(poll*5, pollTimeout, func() (bool, error) {
vm, err := getVmsvcVM(ctx, vmopC, vm.Namespace, vm.Name)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
Expand Down
Loading

0 comments on commit 0a3698e

Please sign in to comment.