From 11148995985c3553f6447dc14d74e632f2dd5c56 Mon Sep 17 00:00:00 2001 From: Peng Gao Date: Fri, 1 Jun 2018 00:29:50 +0800 Subject: [PATCH] subnet: handle unknown state in delete event To fix panic (see log below), add a handler for cache.DeletedFinalStateUnknown. ``` /usr/local/go/src/runtime/asm_amd64.s:2197 panic: interface conversion: interface {} is cache.DeletedFinalStateUnknown, not *v1.Node [recovered] panic: interface conversion: interface {} is cache.DeletedFinalStateUnknown, not *v1.Node [recovered] panic: interface conversion: interface {} is cache.DeletedFinalStateUnknown, not *v1.Node goroutine 19 [running]: github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) /go/src/github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:58 +0x126 panic(0x1782360, 0xc420945e80) /usr/local/go/src/runtime/panic.go:489 +0x2cf github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) /go/src/github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:58 +0x126 panic(0x1782360, 0xc420945e80) /usr/local/go/src/runtime/panic.go:489 +0x2cf github.com/coreos/flannel/subnet/kube.(*kubeSubnetManager).handleAddLeaseEvent(0xc420014f00, 0x1, 0x17e38a0, 0xc4204f61c0) /go/src/github.com/coreos/flannel/subnet/kube/kube.go:176 +0x2d3 github.com/coreos/flannel/subnet/kube.newKubeSubnetManager.func4(0x17e38a0, 0xc4204f61c0) /go/src/github.com/coreos/flannel/subnet/kube/kube.go:165 +0x47 github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnDelete(0xc4202ba6e0, 0xc4202ba6f0, 0xc4202ba700, 0x17e38a0, 0xc4204f61c0) /go/src/github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache/controller.go:206 +0x49 github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnDelete(0xc42018c780, 0x17e38a0, 0xc4204f61c0) :56 +0x73 github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache.NewIndexerInformer.func1(0x1792520, 0xc4204f6680, 0x1792520, 0xc4204f6680) /go/src/github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache/controller.go:342 +0x519 github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc4203120c0, 0xc4203aede0, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache/delta_fifo.go:451 +0x27e github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache.(*controller).processLoop(0xc420067180) /go/src/github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache/controller.go:147 +0x40 github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache.(*controller).(github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache.processLoop)-fm() /go/src/github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x2a github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc4205fcf70) /go/src/github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:96 +0x5e github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc420865f70, 0x3b9aca00, 0x0, 0x166ac01, 0x0) /go/src/github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:97 +0xad github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/wait.Until(0xc4205fcf70, 0x3b9aca00, 0x0) /go/src/github.com/coreos/flannel/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:52 +0x4d github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache.(*controller).Run(0xc420067180, 0x0) /go/src/github.com/coreos/flannel/vendor/k8s.io/client-go/tools/cache/controller.go:121 +0x237 github.com/coreos/flannel/subnet/kube.(*kubeSubnetManager).Run(0xc420014f00, 0x7f340ee4d3b0, 0xc4200161a8) /go/src/github.com/coreos/flannel/subnet/kube/kube.go:284 +0x8a created by github.com/coreos/flannel/subnet/kube.NewSubnetManager /go/src/github.com/coreos/flannel/subnet/kube/kube.go:128 +0x592 ``` Signed-off-by: Peng Gao --- subnet/kube/kube.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/subnet/kube/kube.go b/subnet/kube/kube.go index 38375d8d03..dff4e1eaf1 100644 --- a/subnet/kube/kube.go +++ b/subnet/kube/kube.go @@ -163,6 +163,21 @@ func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName str }, UpdateFunc: ksm.handleUpdateLeaseEvent, DeleteFunc: func(obj interface{}) { + node, isNode := obj.(*v1.Node) + // We can get DeletedFinalStateUnknown instead of *api.Node here and we need to handle that correctly. + if !isNode { + deletedState, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + glog.Infof("Error received unexpected object: %v", obj) + return + } + node, ok = deletedState.Obj.(*v1.Node) + if !ok { + glog.Infof("Error deletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj) + return + } + obj = node + } ksm.handleAddLeaseEvent(subnet.EventRemoved, obj) }, },