pod: block the vsw if ip not enough

pkg/controller/pod/pod_controller.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	aliyunClient "github.com/AliyunContainerService/terway/pkg/aliyun/client"
+	apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/client/errors"
 	"github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
 	"github.com/AliyunContainerService/terway/pkg/backoff"
 	register "github.com/AliyunContainerService/terway/pkg/controller"
@@ -656,6 +657,11 @@ func (m *ReconcilePod) createENI(ctx context.Context, allocs *[]*v1beta1.Allocat
 			}
 			eni, err := m.aliyun.CreateNetworkInterface(ctx, option)
 			if err != nil {
+
+				if apiErr.ErrorCodeIs(err, apiErr.InvalidVSwitchIDIPNotEnough) {
+					m.swPool.Block(alloc.ENI.VSwitchID)
+				}
+
 				return fmt.Errorf("create eni with openAPI err, %w", err)
 			}
 

pkg/vswitch/vswitch.go

@@ -26,12 +26,14 @@
 
 	"golang.org/x/sync/singleflight"
 	"k8s.io/apimachinery/pkg/util/cache"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/AliyunContainerService/terway/pkg/aliyun/client"
 )
 
 var ErrNoAvailableVSwitch = errors.New("no available vSwitch")
+var ErrIPNotEnough = errors.New("no ip left")
 
 // Switch hole all switch info from both terway config and podNetworking
 type Switch struct {
@@ -89,12 +91,6 @@
 				continue
 			}
 
-			if vsw.Zone != zone {
-				continue
-			}
-			if vsw.AvailableIPCount == 0 {
-				continue
-			}
 			byAvailableIP = append(byAvailableIP, *vsw)
 		}
 
@@ -107,6 +103,8 @@
 		ids = newOrder
 	}
 
+	var errs []error
+
 	// lookup all vsw in cache and get one matched
 	for _, id := range ids {
 		vsw, err := s.GetByID(ctx, client, id)
@@ -122,19 +120,22 @@
 			continue
 		}
 		if vsw.AvailableIPCount == 0 {
+			errs = append(errs, fmt.Errorf("%s %w", vsw.ID, ErrIPNotEnough))
 			continue
 		}
 		return vsw, nil
 	}
 
 	for _, vsw := range fallBackSwitches {
 		if vsw.AvailableIPCount == 0 {
+			errs = append(errs, fmt.Errorf("%s %w", vsw.ID, ErrIPNotEnough))
 			continue
 		}
 		return vsw, nil
 	}
+	errs = append(errs, fmt.Errorf("%w for zone %s, vswList %v", ErrNoAvailableVSwitch, zone, ids))
 
-	return nil, fmt.Errorf("no available vSwitch for zone %s, vswList %v, %w", zone, ids, ErrNoAvailableVSwitch)
+	return nil, utilerrors.NewAggregate(errs)
 }
 
 // GetByID will get vSwitch info from local store or openAPI

pkg/vswitch/vswitch_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/aliyun/alibaba-cloud-sdk-go/services/vpc"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 
@@ -30,6 +31,11 @@ import (
 
 func TestSwitchPool_GetOne(t *testing.T) {
 	openAPI := mocks.NewVPC(t)
+	openAPI.On("DescribeVSwitchByID", mock.Anything, "vsw-0").Return(&vpc.VSwitch{
+		VSwitchId:               "vsw-0",
+		ZoneId:                  "zone-0",
+		AvailableIpAddressCount: 0,
+	}, nil).Maybe()
 	openAPI.On("DescribeVSwitchByID", mock.Anything, "vsw-1").Return(&vpc.VSwitch{
 		VSwitchId:               "vsw-1",
 		ZoneId:                  "zone-1",
@@ -69,6 +75,22 @@ func TestSwitchPool_GetOne(t *testing.T) {
 	}
 
 	assert.Equal(t, 2, len(ids))
+
+	_, err = switchPool.GetOne(context.Background(), openAPI, "zone-x", []string{"vsw-2", "vsw-3"}, &SelectOptions{
+		IgnoreZone:          false,
+		VSwitchSelectPolicy: VSwitchSelectionPolicyRandom,
+	})
+
+	assert.True(t, errors.Is(err, ErrNoAvailableVSwitch))
+	assert.False(t, errors.Is(err, ErrIPNotEnough))
+
+	_, err = switchPool.GetOne(context.Background(), openAPI, "zone-0", []string{"vsw-0"}, &SelectOptions{
+		IgnoreZone:          false,
+		VSwitchSelectPolicy: VSwitchSelectionPolicyRandom,
+	})
+
+	assert.True(t, errors.Is(err, ErrNoAvailableVSwitch))
+	assert.True(t, errors.Is(err, ErrIPNotEnough))
 }
 
 func TestGetByID(t *testing.T) {