Skip to content

Commit

Permalink
Add DHCP IP Retries in PrepareHNSNetwork
Browse files Browse the repository at this point in the history 
To address the potential race condition issue where acquiring a DHCP IP address may fail after CreateHNSNetwork,
we added a retry mechanism to wait for an available IP. If the DHCP IP cannot be acquired within three seconds,
a warning message will be returned.

Signed-off-by: Shuyang Xin <gavinx@vmware.com>
  • Loading branch information
@XinShuYang
XinShuYang committed Dec 25, 2023
1 parent 0c6d471 commit 4b57901
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 2 deletions.
17 changes: 17 additions & 0 deletions ci/jenkins/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,13 @@ function wait_for_antrea_windows_pods_ready {
done
sleep 10
done
WIN_BRINT_DHCP=$(ssh -o StrictHostKeyChecking=no -n administrator@${WINIP} "powershell.exe (Get-NetIPInterface -InterfaceAlias br-int -AddressFamily IPv4).Dhcp")
if [[ $WIN_DHCP == $WIN_BRINT_DHCP ]]; then
echo "New created uplink DHCP status is consistent with the original adapter."
else
echo "New created uplink DHCP status is different with the original adapter."
exit 1
fi
}

function wait_for_antrea_windows_processes_ready {
Expand All @@ -295,6 +302,13 @@ function wait_for_antrea_windows_processes_ready {
done
sleep 10
done
WIN_BRINT_DHCP=$(ssh -o StrictHostKeyChecking=no -n administrator@${WINIP} "powershell.exe (Get-NetIPInterface -InterfaceAlias br-int -AddressFamily IPv4).Dhcp")
if [[ $WIN_DHCP == $WIN_BRINT_DHCP ]]; then
echo "New created uplink DHCP status is consistent with the original adapter."
else
echo "New created uplink DHCP status is different with the original adapter."
exit 1
fi
}

function clean_ns {
Expand Down Expand Up @@ -652,6 +666,9 @@ function deliver_antrea_windows_containerd {
ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "gzip -d antrea-windows.tar.gz && ctr -n k8s.io images import antrea-windows.tar"
fi
done
# Add Windows interface DHCP check using CI script to obtain the original interface status.
WINIP=$(kubectl get nodes -o wide --no-headers=true | awk '$1 ~ /win-0/ {print $6}')
WIN_DHCP=$(ssh -o StrictHostKeyChecking=no -n administrator@${WINIP} "powershell.exe (Get-NetIPInterface -InterfaceAlias "vEthernet0 2" -AddressFamily IPv4).Dhcp")
rm -f antrea-windows.tar
echo "==== Finish building and delivering Windows containerd images ===="
}
Expand Down
38 changes: 36 additions & 2 deletions pkg/agent/util/net_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -460,9 +460,28 @@ func PrepareHNSNetwork(subnetCIDR *net.IPNet, nodeIPNet *net.IPNet, uplinkAdapte
hnsNetworkDelete(hnsNet)
}
}()

adapter, ipFound, err := adapterIPExists(nodeIPNet.IP, uplinkAdapter.HardwareAddr, ContainerVNICPrefix)
var adapter *net.Interface
var ipFound bool
err = wait.PollImmediate(time.Second, 3*time.Second, func() (done bool, err error) {
apt, ipExist, err := adapterIPExists(nodeIPNet.IP, uplinkAdapter.HardwareAddr, ContainerVNICPrefix)
if err != nil {
return false, err
}
adapter = apt
if !ipExist {
return false, nil
}
ipFound = true
return true, nil
})
if err != nil {
if err == wait.ErrWaitTimeout {
dhcpStatus, err := InterfaceIpv4Dhcp(uplinkAdapter.Name)
if err != nil {
klog.ErrorS(err, "Failed to get Ipv4 DHCP status on the network adapter", "adapter", uplinkAdapter.Name)
}
klog.Warningf("Timeout acquiring IP for the adapter, DHCP status: %t", dhcpStatus)
}
return err
}
vNicName, index := adapter.Name, adapter.Index
Expand Down Expand Up @@ -647,6 +666,15 @@ func HostInterfaceExists(ifaceName string) bool {
return true
}

// InterfaceIpv4Dhcp returns the Ipv4 DHCP status on the specified interface.
func InterfaceIpv4Dhcp(ifaceName string) (bool, error) {
adapter, err := getAdapterInAllCompartmentsByName(ifaceName)
if err != nil {
return false, err
}
return adapter.ipv4Dhcp, nil
}

// SetInterfaceMTU configures interface MTU on host for Pods. MTU change cannot be realized with HNSEndpoint because
// there's no MTU field in HNSEndpoint:
// https://github.com/Microsoft/hcsshim/blob/4a468a6f7ae547974bc32911395c51fb1862b7df/internal/hns/hnsendpoint.go#L12
Expand Down Expand Up @@ -1085,6 +1113,7 @@ type updateIPInterfaceFunc func(entry *antreasyscall.MibIPInterfaceRow) *antreas
type adapter struct {
net.Interface
compartmentID uint32
ipv4Dhcp bool
}

func (a *adapter) setMTU(mtu int, family uint16) error {
Expand Down Expand Up @@ -1238,6 +1267,10 @@ func getAdaptersByName(name string) ([]adapter, error) {
case windows.IF_TYPE_ATM:
ifi.Flags |= net.FlagBroadcast | net.FlagPointToPoint | net.FlagMulticast // assume all services available; LANE, point-to-point and point-to-multipoint
}
ipv4Dhcp := false
if aa.Flags&0x00000004 != 0{
ipv4Dhcp = true
}
if aa.Mtu == 0xffffffff {
ifi.MTU = -1
} else {
Expand All @@ -1250,6 +1283,7 @@ func getAdaptersByName(name string) ([]adapter, error) {
adapter := adapter{
Interface: ifi,
compartmentID: aa.CompartmentId,
ipv4Dhcp: ipv4Dhcp,
}
adapters = append(adapters, adapter)
}
Expand Down
14 changes: 14 additions & 0 deletions pkg/agent/util/net_windows_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
setVMCmd := fmt.Sprintf("Set-VMSwitch -ComputerName $(hostname) -Name %s -EnableSoftwareRsc $True", LocalHNSNetwork)
tests := []struct {
name string
testAdapterAddresses *windows.IpAdapterAddresses
nodeIPNet *net.IPNet
dnsServers string
newName string
Expand All @@ -321,6 +322,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
}{
{
name: "Prepare Success",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4PublicIPNet,
dnsServers: testDNSServer,
newName: testNewName,
Expand All @@ -330,6 +332,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
},
{
name: "Create Error",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4PublicIPNet,
dnsServers: testDNSServer,
ipFound: true,
Expand All @@ -338,6 +341,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
},
{
name: "adapter Err",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4PublicIPNet,
dnsServers: testDNSServer,
ipFound: true,
Expand All @@ -346,6 +350,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
},
{
name: "Rename Err",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4PublicIPNet,
dnsServers: testDNSServer,
newName: testNewName,
Expand All @@ -356,6 +361,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
},
{
name: "Enable HNS Err",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4PublicIPNet,
dnsServers: testDNSServer,
ipFound: true,
Expand All @@ -364,6 +370,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
},
{
name: "Enable RSC Err",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4PublicIPNet,
dnsServers: testDNSServer,
ipFound: true,
Expand All @@ -373,13 +380,15 @@ func TestPrepareHNSNetwork(t *testing.T) {
},
{
name: "IP Not Found",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4ZeroIPNet,
dnsServers: testDNSServer,
ipFound: false,
wantCmds: []string{newIPCmd, setServerCmd, getVMCmd, setVMCmd},
},
{
name: "IP Not Found Configure Default Err",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4ZeroIPNet,
dnsServers: testDNSServer,
ipFound: false,
Expand All @@ -389,6 +398,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
},
{
name: "IP Not Found Set adapter Err",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4ZeroIPNet,
dnsServers: testDNSServer,
ipFound: false,
Expand All @@ -398,6 +408,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
},
{
name: "IP Not Found New Net Route Err",
testAdapterAddresses: createTestAdapterAddresses(testAdapterName),
nodeIPNet: &ipv4ZeroIPNet,
ipFound: false,
createRowErr: fmt.Errorf("ip route not found"),
Expand All @@ -415,6 +426,7 @@ func TestPrepareHNSNetwork(t *testing.T) {
defer mockHNSNetworkCreate(tc.hnsNetworkCreateErr)()
defer mockHNSNetworkDelete(nil)()
defer mockAntreaNetIO(&antreasyscalltest.MockNetIO{CreateIPForwardEntryErr: tc.createRowErr})()
defer mockGetAdaptersAddresses(tc.testAdapterAddresses, nil)()
gotErr := PrepareHNSNetwork(testSubnetCIDR, tc.nodeIPNet, testUplinkAdapter, "testGateway", tc.dnsServers, testRoutes, tc.newName)
assert.Equal(t, tc.wantErr, gotErr)
})
Expand Down Expand Up @@ -1139,6 +1151,7 @@ func createTestAdapterAddresses(name string) *windows.IpAdapterAddresses {
PhysicalAddressLength: 6,
PhysicalAddress: testPhysicalAddress,
CompartmentId: 1,
Flags: 0x00000004,
}
}

Expand Down Expand Up @@ -1171,6 +1184,7 @@ func mockGetAdaptersAddresses(testAdaptersAddresses *windows.IpAdapterAddresses,
adapterAddresses.PhysicalAddressLength = testAdaptersAddresses.PhysicalAddressLength
adapterAddresses.PhysicalAddress = testAdaptersAddresses.PhysicalAddress
adapterAddresses.CompartmentId = testAdaptersAddresses.CompartmentId
adapterAddresses.Flags = testAdaptersAddresses.Flags
}
return err
}
Expand Down

0 comments on commit 4b57901

Please sign in to comment.