Merge pull request #127 from liwenwu-amazon/add-fail-hdlr

Release IP back to datastore if CNI failed to set Pod's namespace

ipamd/ipamd.go

@@ -83,6 +83,19 @@ var (
 		},
 		[]string{"fn"},
 	)
+	addIPCnt = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "add_ip_req_count",
+			Help: "The number of add IP address request",
+		},
+	)
+	delIPCnt = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "del_ip_req_count",
+			Help: "The number of delete IP address request",
+		},
+		[]string{"reason"},
+	)
 	prometheusRegistered = false
 )
 
@@ -111,6 +124,8 @@ func prometheusRegister() {
 		prometheus.MustRegister(enisMax)
 		prometheus.MustRegister(ipMax)
 		prometheus.MustRegister(reconcileCnt)
+		prometheus.MustRegister(addIPCnt)
+		prometheus.MustRegister(delIPCnt)
 		prometheusRegistered = true
 	}
 }

ipamd/rpc_handler.go

@@ -19,6 +19,7 @@ import (
 	"github.com/pkg/errors"
 
 	pb "github.com/aws/amazon-vpc-cni-k8s/rpc"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
@@ -47,12 +48,14 @@ func (s *server) AddNetwork(ctx context.Context, in *pb.AddNetworkRequest) (*pb.
 		Namespace: in.K8S_POD_NAMESPACE,
 		Container: in.K8S_POD_INFRA_CONTAINER_ID})
 	log.Infof("Send AddNetworkReply: IPv4Addr %s, DeviceNumber: %d, err: %v", addr, deviceNumber, err)
+	addIPCnt.Inc()
 	return &pb.AddNetworkReply{Success: err == nil, IPv4Addr: addr, IPv4Subnet: "", DeviceNumber: int32(deviceNumber)}, nil
 }
 
 func (s *server) DelNetwork(ctx context.Context, in *pb.DelNetworkRequest) (*pb.DelNetworkReply, error) {
 	log.Infof("Received DelNetwork for IP %s, Pod %s, Namespace %s, Container %s",
 		in.IPv4Addr, in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, in.K8S_POD_INFRA_CONTAINER_ID)
+	delIPCnt.With(prometheus.Labels{"reason": in.Reason}).Inc()
 
 	var err error
 

plugins/routed-eni/cni.go

@@ -179,6 +179,25 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 	if err != nil {
 		log.Errorf("Failed SetupPodNetwork for pod %s namespace %s container %s: %v",
 			string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), err)
+
+		// return allocated IP back to IP pool
+		r, delErr := c.DelNetwork(context.Background(),
+			&pb.DelNetworkRequest{
+				K8S_POD_NAME:               string(k8sArgs.K8S_POD_NAME),
+				K8S_POD_NAMESPACE:          string(k8sArgs.K8S_POD_NAMESPACE),
+				K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID),
+				IPv4Addr:                   r.IPv4Addr,
+				Reason:                     "SetupNSFailed"})
+
+		if delErr != nil {
+			log.Errorf("Error received from DelNetwork grpc call for pod %s namespace %s container %s: %v",
+				string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), delErr)
+		}
+
+		if !r.Success {
+			log.Errorf("Failed to release IP of pod %s namespace %s container %s: %v",
+				string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), delErr)
+		}
 		return errors.Wrap(err, "add command: failed to setup network")
 	}
 
@@ -246,7 +265,8 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap
 			K8S_POD_NAME:               string(k8sArgs.K8S_POD_NAME),
 			K8S_POD_NAMESPACE:          string(k8sArgs.K8S_POD_NAMESPACE),
 			K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID),
-			IPv4Addr:                   k8sArgs.IP.String()})
+			IPv4Addr:                   k8sArgs.IP.String(),
+			Reason:                     "PodDeleted"})
 
 	if err != nil {
 		log.Errorf("Error received from DelNetwork grpc call for pod %s namespace %s container %s: %v",

plugins/routed-eni/cni_test.go

@@ -181,6 +181,10 @@ func TestCmdAddErrSetupPodNetwork(t *testing.T) {
 	mocksNetwork.EXPECT().SetupNS(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns,
 		addr, int(addNetworkReply.DeviceNumber)).Return(errors.New("Error on SetupPodNetwork"))
 
+	// when SetupPodNetwork fails, expect to return IP back to datastore
+	delNetworkReply := &rpc.DelNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum}
+	mockC.EXPECT().DelNetwork(gomock.Any(), gomock.Any()).Return(delNetworkReply, nil)
+
 	err := add(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork)
 
 	assert.Error(t, err)

rpc/rpc.proto

@@ -32,6 +32,7 @@ message DelNetworkRequest {
   string K8S_POD_NAMESPACE = 2;
   string K8S_POD_INFRA_CONTAINER_ID = 3;
   string IPv4Addr = 4;
+  string Reason = 5;
 }
 
 message DelNetworkReply {