Add check for endpoint count is zero

Add check to make sure neither the total number of endpoints from EPS or the one from calculated endpoint list is 0. If not, the syncer will enter the error state.
kubernetes · Nov 23, 2022 · 76b6720 · 76b6720
1 parent 978de34
commit 76b6720
Show file tree

Hide file tree

Showing 2 changed files with 191 additions and 0 deletions.
diff --git a/pkg/neg/syncers/transaction.go b/pkg/neg/syncers/transaction.go
@@ -363,9 +363,14 @@ func (s *transactionSyncer) ensureNetworkEndpointGroups() error {
 //     endpiontPodMap removes the duplicated endpoints, and dupCount stores the number of duplicated it removed
 //     and we compare the endpoint counts with duplicates
 //  2. There is at least one endpoint in endpointData with missing nodeName
+//  3. The endpoint count from endpointData or the one from endpointPodMap is 0
 func (s *transactionSyncer) invalidEndpointInfo(eds []negtypes.EndpointsData, endpointPodMap negtypes.EndpointPodMap, dupCount int) bool {
 	// Endpoint count from EndpointPodMap
 	countFromPodMap := len(endpointPodMap) + dupCount
+	if countFromPodMap == 0 {
+		s.logger.Info("Detected endpoint count from endpointPodMap going to zero, marking syncer in error state", "endpointPodMap", endpointPodMap)
+		return true
+	}
 
 	// Endpoint count from EndpointData
 	countFromEndpointData := 0
@@ -379,6 +384,10 @@ func (s *transactionSyncer) invalidEndpointInfo(eds []negtypes.EndpointsData, en
 			}
 		}
 	}
+	if countFromEndpointData == 0 {
+		s.logger.Info("Detected endpoint count from endpointData going to zero, marking syncer in error state", "endpointData", eds)
+		return true
+	}
 
 	if countFromEndpointData != countFromPodMap {
 		s.logger.Info("Detected error when comparing endpoint counts, marking syncer in error state", "endpointData", eds, "endpointPodMap", endpointPodMap, "dupCount", dupCount)

diff --git a/pkg/neg/syncers/transaction_test.go b/pkg/neg/syncers/transaction_test.go
@@ -2050,6 +2050,188 @@ func TestIsZoneMissing(t *testing.T) {
 	}
 }
 
+func TestInvalidEndpointInfoEndpointCountZero(t *testing.T) {
+	t.Parallel()
+	_, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(gce.DefaultTestClusterValues())), negtypes.VmIpPortEndpointType, false, true)
+
+	instance1 := testInstance1
+	testPortName := "port1"
+	testServiceNamespace := "namespace"
+	port80 := int32(80)
+	testIP1 := "10.100.1.1"
+	testIP2 := "10.100.1.2"
+	testPodName1 := "pod1"
+	testPodName2 := "pod2"
+
+	testCases := []struct {
+		desc           string
+		endpointsData  []negtypes.EndpointsData
+		endpointPodMap map[negtypes.NetworkEndpoint]types.NamespacedName
+		expect         bool
+	}{
+		{
+			desc: "endpointData has zero endpoint",
+			endpointsData: []negtypes.EndpointsData{
+				{
+					Meta: &metav1.ObjectMeta{
+						Name:      testServiceName + "-1",
+						Namespace: testServiceNamespace,
+					},
+					Ports: []negtypes.PortData{
+						{
+							Name: testPortName,
+							Port: port80,
+						},
+					},
+					Addresses: []negtypes.AddressData{},
+				},
+			},
+			endpointPodMap: map[negtypes.NetworkEndpoint]types.NamespacedName{
+				{
+					IP:   testIP1,
+					Port: "80",
+					Node: instance1,
+				}: {
+					Namespace: testServiceNamespace,
+					Name:      testPodName1,
+				},
+				{
+					IP:   testIP2,
+					Port: "80",
+					Node: instance1,
+				}: {
+					Namespace: testServiceNamespace,
+					Name:      testPodName2,
+				},
+			},
+			expect: true,
+		},
+		{
+			desc: "endpointPodMap has zero endpoint",
+			endpointsData: []negtypes.EndpointsData{
+				{
+					Meta: &metav1.ObjectMeta{
+						Name:      testServiceName + "-1",
+						Namespace: testServiceNamespace,
+					},
+					Ports: []negtypes.PortData{
+						{
+							Name: testPortName,
+							Port: port80,
+						},
+					},
+					Addresses: []negtypes.AddressData{
+						{
+							TargetRef: &corev1.ObjectReference{
+								Namespace: testServiceNamespace,
+								Name:      testPodName1,
+							},
+							NodeName:  &instance1,
+							Addresses: []string{testIP1},
+							Ready:     true,
+						},
+						{
+							TargetRef: &corev1.ObjectReference{
+								Namespace: testServiceNamespace,
+								Name:      testPodName2,
+							},
+							NodeName:  &instance1,
+							Addresses: []string{testIP2},
+							Ready:     true,
+						},
+					},
+				},
+			},
+			endpointPodMap: map[negtypes.NetworkEndpoint]types.NamespacedName{},
+			expect:         true,
+		},
+		{
+			desc: "endpointData and endpointPodMap both have zero endpoint",
+			endpointsData: []negtypes.EndpointsData{
+				{
+					Meta: &metav1.ObjectMeta{
+						Name:      testServiceName + "-1",
+						Namespace: testServiceNamespace,
+					},
+					Ports: []negtypes.PortData{
+						{
+							Name: testPortName,
+							Port: port80,
+						},
+					},
+					Addresses: []negtypes.AddressData{},
+				},
+			},
+			endpointPodMap: map[negtypes.NetworkEndpoint]types.NamespacedName{},
+			expect:         true,
+		},
+		{
+			desc: "endpointData and endpointPodMap both have non-zero endpoints",
+			endpointsData: []negtypes.EndpointsData{
+				{
+					Meta: &metav1.ObjectMeta{
+						Name:      testServiceName + "-1",
+						Namespace: testServiceNamespace,
+					},
+					Ports: []negtypes.PortData{
+						{
+							Name: testPortName,
+							Port: port80,
+						},
+					},
+					Addresses: []negtypes.AddressData{
+						{
+							TargetRef: &corev1.ObjectReference{
+								Namespace: testServiceNamespace,
+								Name:      testPodName1,
+							},
+							NodeName:  &instance1,
+							Addresses: []string{testIP1},
+							Ready:     true,
+						},
+						{
+							TargetRef: &corev1.ObjectReference{
+								Namespace: testServiceNamespace,
+								Name:      testPodName2,
+							},
+							NodeName:  &instance1,
+							Addresses: []string{testIP2},
+							Ready:     true,
+						},
+					},
+				},
+			},
+			endpointPodMap: map[negtypes.NetworkEndpoint]types.NamespacedName{
+				{
+					IP:   testIP1,
+					Port: "80",
+					Node: instance1,
+				}: {
+					Namespace: testServiceNamespace,
+					Name:      testPodName1,
+				},
+				{
+					IP:   testIP2,
+					Port: "80",
+					Node: instance1,
+				}: {
+					Namespace: testServiceNamespace,
+					Name:      testPodName2,
+				},
+			},
+			expect: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			if got := transactionSyncer.invalidEndpointInfo(tc.endpointsData, tc.endpointPodMap, 0); got != tc.expect {
+				t.Errorf("invalidEndpointInfo() = %t, expected %t", got, tc.expect)
+			}
+		})
+	}
+}
+
 func newL4ILBTestTransactionSyncer(fakeGCE negtypes.NetworkEndpointGroupCloud, mode negtypes.EndpointsCalculatorMode, enableEndpointSlices bool) (negtypes.NegSyncer, *transactionSyncer) {
 	negsyncer, ts := newTestTransactionSyncer(fakeGCE, negtypes.VmIpEndpointType, false, enableEndpointSlices)
 	ts.endpointsCalculator = GetEndpointsCalculator(ts.nodeLister, ts.podLister, ts.zoneGetter, ts.NegSyncerKey, mode, klog.TODO())