Skip to content

Commit

Permalink
fix: fix 熔断器 panic (#193)
Browse files Browse the repository at this point in the history
  • Loading branch information
@WTIFS
WTIFS authored Dec 19, 2023
1 parent b7d2762 commit 2022aff
Show file tree
Hide file tree
Showing 8 changed files with 49 additions and 33 deletions.
5 changes: 4 additions & 1 deletion examples/circuitbreaker/instance/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,4 +104,7 @@ Hello, I'm CircuitBreakerEchoServer Provider, My host : 127.0.0.1:9999
Hello, I'm CircuitBreakerEchoServer Provider, My host : 27.0.0.1:9999
...
Hello, I'm CircuitBreakerEchoServer Provider, My host : 127.0.0.1:9999
```
```

可启动 provider 两次，使北极星上有两个 CircuitBreakerEchoServer 服务（其中一个正常，一个不正常）
然后启动 consumer，并通过 webUI 访问 http://localhost:18080/echo 接口，观察控制台输出，判断路由结果是否正确
6 changes: 0 additions & 6 deletions examples/circuitbreaker/instance/consumer/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,6 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/agiledragon/gomonkey v2.0.2+incompatible h1:eXKi9/piiC3cjJD1658mEE2o3NjkJ5vDLgYjCQu0Xlw=
github.com/agiledragon/gomonkey v2.0.2+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
Expand All @@ -186,7 +185,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
Expand Down Expand Up @@ -321,7 +319,6 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99
github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
Expand All @@ -340,7 +337,6 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
Expand Down Expand Up @@ -404,9 +400,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
Expand Down
17 changes: 17 additions & 0 deletions examples/circuitbreaker/instance/consumer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,23 @@ func (svr *PolarisConsumer) Run() {
}

func (svr *PolarisConsumer) discoverInstance() (string, model.Resource, error) {
req := &polaris.GetInstancesRequest{}
req.Namespace = namespace
req.Service = service

instancesResp, err := svr.consumer.GetInstances(req)
if err != nil {
log.Printf("[error] fail to getInstances, err is %v", err)
return "", nil, err
}
for _, ins := range instancesResp.GetInstances() {
cbStatus := "close"
if ins.GetCircuitBreakerStatus() != nil {
cbStatus = ins.GetCircuitBreakerStatus().GetStatus().String()
}
log.Printf("%s:%d. cb status: %s", ins.GetHost(), ins.GetPort(), cbStatus)
}

getOneRequest := &polaris.GetOneInstanceRequest{}
getOneRequest.Namespace = namespace
getOneRequest.Service = service
Expand Down
6 changes: 0 additions & 6 deletions examples/circuitbreaker/instance/provider/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,6 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/agiledragon/gomonkey v2.0.2+incompatible h1:eXKi9/piiC3cjJD1658mEE2o3NjkJ5vDLgYjCQu0Xlw=
github.com/agiledragon/gomonkey v2.0.2+incompatible/go.mod h1:2NGfXu1a80LLr2cmWXGBDaHEjb1idR6+FVlX5T3D9hw=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
Expand All @@ -186,7 +185,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
Expand Down Expand Up @@ -321,7 +319,6 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99
github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
Expand All @@ -340,7 +337,6 @@ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
Expand Down Expand Up @@ -404,9 +400,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
Expand Down
5 changes: 5 additions & 0 deletions pkg/model/circuitbreaker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,11 @@ type CircuitBreakerStatus interface {
SetFallbackInfo(*FallbackInfo)
}

// CircuitBreakerStatusWrapper 上方熔断管理器的包装，用于存入 atomic.Value
type CircuitBreakerStatusWrapper struct {
Val CircuitBreakerStatus
}

// Status 断路器状态
type Status int

Expand Down
7 changes: 5 additions & 2 deletions pkg/model/local/value.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func (lv *DefaultInstanceLocalValue) SetSliceWindows(pluginIndex int32, windows

// SetCircuitBreakerStatus 设置熔断信息
func (lv *DefaultInstanceLocalValue) SetCircuitBreakerStatus(st model.CircuitBreakerStatus) {
lv.cbStatus.Store(st)
lv.cbStatus.Store(model.CircuitBreakerStatusWrapper{Val: st})
}

// SetActiveDetectStatus 设置健康检测信息
Expand All @@ -96,7 +96,10 @@ func (lv *DefaultInstanceLocalValue) GetCircuitBreakerStatus() model.CircuitBrea
if nil == res {
return nil
}
return res.(model.CircuitBreakerStatus)
if wrapper, ok := res.(model.CircuitBreakerStatusWrapper); ok {
return wrapper.Val.(model.CircuitBreakerStatus)

Check failure on line 100 in pkg/model/local/value.go

View workflow job for this annotation

GitHub Actions / golangci-lint (1.15.x) 

sloppyTypeAssert: type assertion from/to types are identical (gocritic)
}
return nil
}

// GetActiveDetectStatus 返回健康检测信息
Expand Down
14 changes: 6 additions & 8 deletions plugin/circuitbreaker/composite/counter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,8 @@ func (rc *ResourceCounters) CurrentActiveRule() *fault_tolerance.CircuitBreakerR
}

func (rc *ResourceCounters) updateCircuitBreakerStatus(status model.CircuitBreakerStatus) {
rc.statusRef.Store(&circuitBreakerStatusWrapper{
val: status,
rc.statusRef.Store(&model.CircuitBreakerStatusWrapper{
Val: status,
})
}

Expand All @@ -132,8 +132,10 @@ func (rc *ResourceCounters) CurrentCircuitBreakerStatus() model.CircuitBreakerSt
if val == nil {
return nil
}
wrapper := val.(circuitBreakerStatusWrapper)
return wrapper.val
if wrapper, ok := val.(*model.CircuitBreakerStatusWrapper); ok {
return wrapper.Val
}
return nil
}

func (rc *ResourceCounters) CloseToOpen(breaker string) {
Expand Down Expand Up @@ -304,7 +306,3 @@ func buildFallbackInfo(rule *fault_tolerance.CircuitBreakerRule) *model.Fallback
}
return ret
}

type circuitBreakerStatusWrapper struct {
val model.CircuitBreakerStatus
}
22 changes: 12 additions & 10 deletions plugin/circuitbreaker/composite/task_pool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@ package composite

import (
"context"
"github.com/polarismesh/polaris-go/pkg/log"

Check failure on line 22 in plugin/circuitbreaker/composite/task_pool.go

View workflow job for this annotation

GitHub Actions / golangci-lint (1.15.x) 

File is not `gci`-ed with --skip-generated -s standard -s default -s prefix(github.com/polarismesh/polaris-go) --custom-order (gci)
"hash/fnv"
"math/rand"
"runtime"
"sync"
"sync/atomic"
"time"

Check failure on line 28 in plugin/circuitbreaker/composite/task_pool.go

View workflow job for this annotation

GitHub Actions / golangci-lint (1.15.x) 

File is not `gci`-ed with --skip-generated -s standard -s default -s prefix(github.com/polarismesh/polaris-go) --custom-order (gci)
Expand Down Expand Up @@ -102,6 +104,14 @@ type worker struct {
delayQueue sync.Map
}

func recovery() {
if r := recover(); r != nil {
buf := make([]byte, 1<<18)
n := runtime.Stack(buf, false)
log.GetBaseLogger().Errorf("[CircuitBreaker] panic recovered: %v\nruntime stack: %s", r, buf[0:n])
}
}

func (w *worker) add(f func()) {
w.lock.RLock()
defer w.lock.RUnlock()
Expand All @@ -111,11 +121,7 @@ func (w *worker) add(f func()) {
}

w.queue <- func() {
defer func() {
if err := recover(); err != nil {
panic(err)
}
}()
defer recovery()
f()
}
}
Expand All @@ -129,11 +135,7 @@ func (w *worker) addDelay(delay time.Duration, f func(), isInterval bool) {
}

wf := func() {
defer func() {
if err := recover(); err != nil {
// do nothing
}
}()
defer recovery()
f()
}

Expand Down

0 comments on commit 2022aff

Please sign in to comment.