From 3d58d4b09a459af701f188da244951c62d323af5 Mon Sep 17 00:00:00 2001 From: liwen wu Date: Sun, 8 Jul 2018 18:29:11 +0000 Subject: [PATCH] Release IP back to datastore if CNI failed to set Pod's namespace --- ipamd/ipamd.go | 15 +++++++++ ipamd/rpc_handler.go | 3 ++ plugins/routed-eni/cni.go | 22 ++++++++++++- plugins/routed-eni/cni_test.go | 4 +++ rpc/mocks/rpc_mocks.go | 4 +-- rpc/rpc.pb.go | 59 ++++++++++++++++++++-------------- rpc/rpc.proto | 1 + 7 files changed, 80 insertions(+), 28 deletions(-) diff --git a/ipamd/ipamd.go b/ipamd/ipamd.go index 3e568832de..9b6ad9a90d 100644 --- a/ipamd/ipamd.go +++ b/ipamd/ipamd.go @@ -83,6 +83,19 @@ var ( }, []string{"fn"}, ) + addIPCnt = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "add_ip_req_count", + Help: "The number of add IP address request", + }, + ) + delIPCnt = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "del_ip_req_count", + Help: "The number of delete IP address request", + }, + []string{"reason"}, + ) prometheusRegistered = false ) @@ -111,6 +124,8 @@ func prometheusRegister() { prometheus.MustRegister(enisMax) prometheus.MustRegister(ipMax) prometheus.MustRegister(reconcileCnt) + prometheus.MustRegister(addIPCnt) + prometheus.MustRegister(delIPCnt) prometheusRegistered = true } } diff --git a/ipamd/rpc_handler.go b/ipamd/rpc_handler.go index 718e6d4d81..45d5cbd457 100644 --- a/ipamd/rpc_handler.go +++ b/ipamd/rpc_handler.go @@ -19,6 +19,7 @@ import ( "github.com/pkg/errors" pb "github.com/aws/amazon-vpc-cni-k8s/rpc" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/reflection" @@ -47,12 +48,14 @@ func (s *server) AddNetwork(ctx context.Context, in *pb.AddNetworkRequest) (*pb. Namespace: in.K8S_POD_NAMESPACE, Container: in.K8S_POD_INFRA_CONTAINER_ID}) log.Infof("Send AddNetworkReply: IPv4Addr %s, DeviceNumber: %d, err: %v", addr, deviceNumber, err) + addIPCnt.Inc() return &pb.AddNetworkReply{Success: err == nil, IPv4Addr: addr, IPv4Subnet: "", DeviceNumber: int32(deviceNumber)}, nil } func (s *server) DelNetwork(ctx context.Context, in *pb.DelNetworkRequest) (*pb.DelNetworkReply, error) { log.Infof("Received DelNetwork for IP %s, Pod %s, Namespace %s, Container %s", in.IPv4Addr, in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, in.K8S_POD_INFRA_CONTAINER_ID) + delIPCnt.With(prometheus.Labels{"reason": in.Reason}).Inc() var err error diff --git a/plugins/routed-eni/cni.go b/plugins/routed-eni/cni.go index dee23d5005..ea07d4361b 100644 --- a/plugins/routed-eni/cni.go +++ b/plugins/routed-eni/cni.go @@ -179,6 +179,25 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap if err != nil { log.Errorf("Failed SetupPodNetwork for pod %s namespace %s container %s: %v", string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), err) + + // return allocated IP back to IP pool + r, delErr := c.DelNetwork(context.Background(), + &pb.DelNetworkRequest{ + K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME), + K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE), + K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), + IPv4Addr: r.IPv4Addr, + Reason: "SetupNSFailed"}) + + if delErr != nil { + log.Errorf("Error received from DelNetwork grpc call for pod %s namespace %s container %s: %v", + string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), delErr) + } + + if !r.Success { + log.Errorf("Failed to release IP of pod %s namespace %s container %s: %v", + string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), delErr) + } return errors.Wrap(err, "add command: failed to setup network") } @@ -246,7 +265,8 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME), K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE), K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), - IPv4Addr: k8sArgs.IP.String()}) + IPv4Addr: k8sArgs.IP.String(), + Reason: "PodDeleted"}) if err != nil { log.Errorf("Error received from DelNetwork grpc call for pod %s namespace %s container %s: %v", diff --git a/plugins/routed-eni/cni_test.go b/plugins/routed-eni/cni_test.go index 0a1d632169..75a7237cb3 100644 --- a/plugins/routed-eni/cni_test.go +++ b/plugins/routed-eni/cni_test.go @@ -181,6 +181,10 @@ func TestCmdAddErrSetupPodNetwork(t *testing.T) { mocksNetwork.EXPECT().SetupNS(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns, addr, int(addNetworkReply.DeviceNumber)).Return(errors.New("Error on SetupPodNetwork")) + // when SetupPodNetwork fails, expect to return IP back to datastore + delNetworkReply := &rpc.DelNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum} + mockC.EXPECT().DelNetwork(gomock.Any(), gomock.Any()).Return(delNetworkReply, nil) + err := add(cmdArgs, mocksTypes, mocksGRPC, mocksRPC, mocksNetwork) assert.Error(t, err) diff --git a/rpc/mocks/rpc_mocks.go b/rpc/mocks/rpc_mocks.go index 402fb7b982..51d7eaae9c 100644 --- a/rpc/mocks/rpc_mocks.go +++ b/rpc/mocks/rpc_mocks.go @@ -1,4 +1,4 @@ -// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"). You may // not use this file except in compliance with the License. A copy of the @@ -18,11 +18,11 @@ package mock_rpc import ( - context "golang.org/x/net/context" reflect "reflect" rpc "github.com/aws/amazon-vpc-cni-k8s/rpc" gomock "github.com/golang/mock/gomock" + context "golang.org/x/net/context" grpc "google.golang.org/grpc" ) diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index 9f7dd38e6d..d4aeb0561a 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -128,6 +128,7 @@ type DelNetworkRequest struct { K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE" json:"K8S_POD_NAMESPACE,omitempty"` K8S_POD_INFRA_CONTAINER_ID string `protobuf:"bytes,3,opt,name=K8S_POD_INFRA_CONTAINER_ID,json=K8SPODINFRACONTAINERID" json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` IPv4Addr string `protobuf:"bytes,4,opt,name=IPv4Addr" json:"IPv4Addr,omitempty"` + Reason string `protobuf:"bytes,5,opt,name=Reason" json:"Reason,omitempty"` } func (m *DelNetworkRequest) Reset() { *m = DelNetworkRequest{} } @@ -163,6 +164,13 @@ func (m *DelNetworkRequest) GetIPv4Addr() string { return "" } +func (m *DelNetworkRequest) GetReason() string { + if m != nil { + return m.Reason + } + return "" +} + type DelNetworkReply struct { Success bool `protobuf:"varint,1,opt,name=Success" json:"Success,omitempty"` IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr" json:"IPv4Addr,omitempty"` @@ -310,29 +318,30 @@ var _CNIBackend_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("rpc.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 384 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x53, 0xcd, 0x6a, 0xf2, 0x50, - 0x10, 0x35, 0x9f, 0x3f, 0x9f, 0x0e, 0x42, 0xf0, 0x22, 0x12, 0x2c, 0x14, 0xc9, 0xaa, 0x74, 0x11, - 0x4a, 0xdb, 0x85, 0x94, 0x6e, 0xa2, 0xb1, 0x34, 0x48, 0xaf, 0x21, 0x29, 0x74, 0x19, 0x34, 0x99, - 0xb6, 0x62, 0x34, 0x69, 0x7e, 0xb4, 0xbe, 0x41, 0xa1, 0x4f, 0xd4, 0x67, 0xe8, 0x4b, 0x95, 0x7b, - 0x8d, 0x1a, 0xcd, 0xae, 0xab, 0xee, 0x66, 0xce, 0x9c, 0x03, 0xe7, 0xcc, 0xbd, 0x03, 0xb5, 0x30, - 0x70, 0x94, 0x20, 0xf4, 0x63, 0x9f, 0x14, 0xc3, 0xc0, 0x91, 0xbf, 0x05, 0x68, 0xa8, 0xae, 0x4b, - 0x31, 0x5e, 0xf9, 0xe1, 0xcc, 0xc4, 0xb7, 0x04, 0xa3, 0x98, 0x74, 0xa0, 0x3e, 0xec, 0x5a, 0xb6, - 0x31, 0xd2, 0x6c, 0xaa, 0x3e, 0x0c, 0x24, 0xa1, 0x23, 0x9c, 0xd5, 0x4c, 0x18, 0x76, 0x2d, 0x63, - 0xa4, 0x31, 0x84, 0x9c, 0x43, 0x23, 0xcb, 0xb0, 0x0c, 0xb5, 0x3f, 0x90, 0xfe, 0x71, 0x9a, 0xb8, - 0xa7, 0x71, 0x98, 0xdc, 0x40, 0x7b, 0xcb, 0xd5, 0xe9, 0x9d, 0xa9, 0xda, 0xfd, 0x11, 0x7d, 0x54, - 0x75, 0x3a, 0x30, 0x6d, 0x5d, 0x93, 0x8a, 0x5c, 0xd4, 0xda, 0x88, 0xf8, 0x7c, 0x37, 0xd6, 0x35, - 0xd2, 0x84, 0x32, 0xc5, 0x78, 0x11, 0x49, 0x25, 0x4e, 0xdb, 0x34, 0xa4, 0x05, 0x15, 0xfd, 0x99, - 0x8e, 0xe7, 0x28, 0x95, 0x39, 0x9c, 0x76, 0xf2, 0xa7, 0x00, 0x62, 0x36, 0x4d, 0xe0, 0xad, 0x89, - 0x04, 0xff, 0xad, 0xc4, 0x71, 0x30, 0x8a, 0x78, 0x8c, 0xaa, 0xb9, 0x6d, 0x49, 0x1b, 0xaa, 0xba, - 0xb1, 0xbc, 0x56, 0x5d, 0x37, 0x4c, 0xad, 0xef, 0x7a, 0x72, 0x0a, 0xc0, 0x6a, 0x2b, 0x99, 0x2c, - 0x30, 0x4e, 0x3d, 0x66, 0x10, 0x22, 0x43, 0x5d, 0xc3, 0xe5, 0xd4, 0x41, 0x9a, 0xcc, 0x27, 0x18, - 0x72, 0x7b, 0x65, 0xf3, 0x00, 0x93, 0xbf, 0x04, 0x68, 0x68, 0xe8, 0xfd, 0xd9, 0xdd, 0x66, 0xf3, - 0x97, 0x0e, 0xf3, 0xcb, 0x33, 0x10, 0xb3, 0xd6, 0x7f, 0xbf, 0xc8, 0xe3, 0x45, 0x15, 0xf3, 0x8b, - 0xba, 0xfc, 0x10, 0x00, 0xfa, 0x54, 0xef, 0x8d, 0x9d, 0x19, 0x2e, 0x5c, 0x72, 0x0b, 0xb0, 0x7f, - 0x44, 0xd2, 0x52, 0xd8, 0x97, 0xcd, 0xfd, 0xd1, 0x76, 0x33, 0x87, 0x07, 0xde, 0x5a, 0x2e, 0x30, - 0xf5, 0xde, 0x79, 0xaa, 0xce, 0xbd, 0x42, 0xaa, 0x3e, 0x8a, 0x28, 0x17, 0x7a, 0x17, 0x70, 0x32, - 0xf5, 0x95, 0x17, 0x36, 0xc4, 0xf7, 0xf1, 0x3c, 0xf0, 0x30, 0x52, 0x5e, 0xd1, 0xf3, 0xfc, 0x95, - 0x1f, 0x7a, 0x6e, 0x4f, 0xbc, 0x67, 0xf5, 0x13, 0xab, 0x0d, 0x76, 0x44, 0x86, 0x30, 0xa9, 0xf0, - 0x6b, 0xba, 0xfa, 0x09, 0x00, 0x00, 0xff, 0xff, 0x23, 0x65, 0x88, 0xdb, 0x5a, 0x03, 0x00, 0x00, + // 396 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x53, 0x4d, 0x6b, 0xe2, 0x50, + 0x14, 0x35, 0xe3, 0xc7, 0xe8, 0x45, 0x08, 0x3e, 0x24, 0x04, 0x07, 0x06, 0xc9, 0x6a, 0x98, 0x45, + 0x18, 0x66, 0x66, 0x21, 0xc3, 0x6c, 0xa2, 0xb1, 0x34, 0x48, 0x9f, 0x21, 0x29, 0x74, 0x19, 0x62, + 0x72, 0xdb, 0x8a, 0x31, 0x49, 0xf3, 0xa1, 0xf5, 0x1f, 0x14, 0xfa, 0xd3, 0xba, 0xea, 0x3f, 0x2a, + 0xef, 0x19, 0x35, 0x35, 0xbb, 0xae, 0xba, 0x7b, 0xe7, 0xdc, 0x73, 0xe0, 0x9e, 0xf3, 0xde, 0x83, + 0x4e, 0x12, 0x7b, 0x6a, 0x9c, 0x44, 0x59, 0x44, 0xea, 0x49, 0xec, 0x29, 0x2f, 0x02, 0xf4, 0x34, + 0xdf, 0xa7, 0x98, 0x6d, 0xa3, 0x64, 0x65, 0xe1, 0x43, 0x8e, 0x69, 0x46, 0x86, 0xd0, 0x9d, 0x8d, + 0x6c, 0xc7, 0x9c, 0xeb, 0x0e, 0xd5, 0xae, 0xa6, 0xb2, 0x30, 0x14, 0x7e, 0x74, 0x2c, 0x98, 0x8d, + 0x6c, 0x73, 0xae, 0x33, 0x86, 0xfc, 0x84, 0x5e, 0x59, 0x61, 0x9b, 0xda, 0x64, 0x2a, 0x7f, 0xe1, + 0x32, 0xf1, 0x24, 0xe3, 0x34, 0xf9, 0x07, 0x83, 0x83, 0xd6, 0xa0, 0x17, 0x96, 0xe6, 0x4c, 0xe6, + 0xf4, 0x5a, 0x33, 0xe8, 0xd4, 0x72, 0x0c, 0x5d, 0xae, 0x73, 0x93, 0xb4, 0x37, 0xf1, 0xf9, 0x71, + 0x6c, 0xe8, 0xa4, 0x0f, 0x4d, 0x8a, 0x59, 0x98, 0xca, 0x0d, 0x2e, 0xdb, 0x03, 0x22, 0x41, 0xcb, + 0xb8, 0xa5, 0xee, 0x1a, 0xe5, 0x26, 0xa7, 0x0b, 0xa4, 0x3c, 0x0b, 0x20, 0x96, 0xd3, 0xc4, 0xc1, + 0x8e, 0xc8, 0xf0, 0xd5, 0xce, 0x3d, 0x0f, 0xd3, 0x94, 0xc7, 0x68, 0x5b, 0x07, 0x48, 0x06, 0xd0, + 0x36, 0xcc, 0xcd, 0x5f, 0xcd, 0xf7, 0x93, 0x62, 0xf5, 0x23, 0x26, 0xdf, 0x01, 0xd8, 0xd9, 0xce, + 0x17, 0x21, 0x66, 0xc5, 0x8e, 0x25, 0x86, 0x28, 0xd0, 0xd5, 0x71, 0xb3, 0xf4, 0x90, 0xe6, 0xeb, + 0x05, 0x26, 0x7c, 0xbd, 0xa6, 0xf5, 0x8e, 0x53, 0x5e, 0x05, 0xe8, 0xe9, 0x18, 0x7c, 0xda, 0x6e, + 0xcb, 0xf9, 0x1b, 0x67, 0xf9, 0x25, 0x68, 0x59, 0xe8, 0xa6, 0x51, 0x78, 0x68, 0x78, 0x8f, 0x94, + 0x15, 0x88, 0xe5, 0x48, 0x1f, 0x2f, 0xf8, 0xbc, 0xc0, 0x7a, 0xb5, 0xc0, 0xdf, 0x4f, 0x02, 0xc0, + 0x84, 0x1a, 0x63, 0xd7, 0x5b, 0x61, 0xe8, 0x93, 0xff, 0x00, 0xa7, 0xcb, 0x25, 0x92, 0xca, 0x9e, + 0x72, 0xe5, 0xed, 0x0e, 0xfa, 0x15, 0x3e, 0x0e, 0x76, 0x4a, 0x8d, 0xb9, 0x4f, 0x9b, 0x17, 0xee, + 0xca, 0xed, 0x14, 0xee, 0xb3, 0x88, 0x4a, 0x6d, 0xfc, 0x0b, 0xbe, 0x2d, 0x23, 0xf5, 0x8e, 0x0d, + 0xf1, 0xd1, 0x5d, 0xc7, 0x01, 0xa6, 0xea, 0x3d, 0x06, 0x41, 0xb4, 0x8d, 0x92, 0xc0, 0x1f, 0x8b, + 0x97, 0xec, 0x7c, 0xc3, 0xce, 0x26, 0xfb, 0x5c, 0xa6, 0xb0, 0x68, 0xf1, 0x5f, 0xf6, 0xe7, 0x2d, + 0x00, 0x00, 0xff, 0xff, 0xb0, 0x25, 0x98, 0x3e, 0x72, 0x03, 0x00, 0x00, } diff --git a/rpc/rpc.proto b/rpc/rpc.proto index e91e02dd0e..db5fcea50b 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -32,6 +32,7 @@ message DelNetworkRequest { string K8S_POD_NAMESPACE = 2; string K8S_POD_INFRA_CONTAINER_ID = 3; string IPv4Addr = 4; + string Reason = 5; } message DelNetworkReply {