diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java index b212c782f6..3385e5efe1 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/Node.java @@ -16,6 +16,9 @@ package com.alibaba.csp.sentinel.node; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import com.alibaba.csp.sentinel.Entry; import com.alibaba.csp.sentinel.node.metric.MetricNode; @@ -126,10 +129,21 @@ public interface Node { */ void addPassRequest(int count); + /** + * minus exception + */ + void minusMinuteException(); + + void minusSecondException(); + /** + * minus Rt + */ + void minusRt(int count); + /** * Add rt and success count. * - * @param rt response time + * @param rt response time * @param success success count to add */ void addRtAndSuccess(long rt, int success); @@ -164,4 +178,25 @@ public interface Node { * Debug only. */ void debug(); + + /** + * reset lastRT + */ + void resetLastRt(); + + /** + * get lastRT + */ + AtomicLong getLastRt(); + + /** + * get last Result (true:success;false:exception) + */ + AtomicBoolean getLastResult(); + + /** + * get last rt(same TimeWindow) sum + */ + AtomicInteger getLastRtSum(); + } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java index a27ef0fe52..b8a6417f0b 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/node/StatisticNode.java @@ -18,7 +18,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import com.alibaba.csp.sentinel.util.TimeUtil; import com.alibaba.csp.sentinel.node.metric.MetricNode; @@ -93,7 +95,7 @@ public class StatisticNode implements Node { * by given {@code sampleCount}. */ private transient volatile Metric rollingCounterInSecond = new ArrayMetric(SampleCountProperty.SAMPLE_COUNT, - IntervalProperty.INTERVAL); + IntervalProperty.INTERVAL); /** * Holds statistics of the recent 60 seconds. The windowLengthInMs is deliberately set to 1000 milliseconds, @@ -110,6 +112,21 @@ public class StatisticNode implements Node { * The last timestamp when metrics were fetched. */ private long lastFetchTime = -1; + /** + * record last rt(Prepare degrade rt sum and same TimeWindow) + */ + private AtomicLong lastRt = new AtomicLong(0); + + /** + * record last rt(same TimeWindow) sum + */ + private AtomicInteger lastRtSum = new AtomicInteger(0); + /** + * lastResult + * + * @return + */ + private AtomicBoolean lastResult = new AtomicBoolean(true); @Override public Map metrics() { @@ -137,7 +154,7 @@ private boolean isNodeInTime(MetricNode node, long currentTime) { private boolean isValidMetricNode(MetricNode node) { return node.getPassQps() > 0 || node.getBlockQps() > 0 || node.getSuccessQps() > 0 - || node.getExceptionQps() > 0 || node.getRt() > 0; + || node.getExceptionQps() > 0 || node.getRt() > 0; } @Override @@ -232,11 +249,36 @@ public void addPassRequest(int count) { rollingCounterInMinute.addPass(count); } + @Override + public void minusSecondException() { + rollingCounterInSecond.minusException(); + } + @Override + public void minusMinuteException() { + rollingCounterInMinute.minusException(); + } + + @Override + public void minusRt(int count) { + rollingCounterInSecond.minusRt(lastRt.get()); + rollingCounterInSecond.minusSuccess(count); + rollingCounterInMinute.minusRt(lastRt.get()); + rollingCounterInMinute.minusSuccess(count); + lastRt.set(0); + } + @Override public void addRtAndSuccess(long rt, int successCount) { rollingCounterInSecond.addSuccess(successCount); rollingCounterInSecond.addRT(rt); - + if (rollingCounterInSecond.newTimeWindow()) { + lastRt.set(rt); + lastRtSum.set(1); + } else { + lastRt.set(rt + lastRt.get()); + lastRtSum.getAndAdd(1); + } + lastResult.set(true); rollingCounterInMinute.addSuccess(successCount); rollingCounterInMinute.addRT(rt); } @@ -251,7 +293,13 @@ public void increaseBlockQps(int count) { public void increaseExceptionQps(int count) { rollingCounterInSecond.addException(count); rollingCounterInMinute.addException(count); + lastResult.set(false); + + } + @Override + public void resetLastRt() { + lastRt.set(0); } @Override @@ -268,4 +316,20 @@ public void decreaseThreadNum() { public void debug() { rollingCounterInSecond.debugQps(); } + + @Override + public AtomicLong getLastRt() { + return lastRt; + } + + @Override + public AtomicBoolean getLastResult() { + return lastResult; + } + + @Override + public AtomicInteger getLastRtSum() { + return lastRtSum; + } + } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java index 978b169fbd..bff26806b2 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/RuleConstant.java @@ -50,9 +50,14 @@ public final class RuleConstant { public static final String LIMIT_APP_DEFAULT = "default"; public static final String LIMIT_APP_OTHER = "other"; + /** + * Three states of the degraded switch + */ + public static final int DEGRADE_CUT_CLOSE = 0; + public static final int DEGRADE_CUT_OPEN = 1; + public static final int DEGRADE_CUT_HALF_OPEN = 2; public static final int DEFAULT_SAMPLE_COUNT = 2; public static final int DEFAULT_WINDOW_INTERVAL_MS = 1000; - - private RuleConstant() {} + private RuleConstant(){} } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java index 8abb1ddb83..78c65b85d8 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeRule.java @@ -58,9 +58,10 @@ public class DegradeRule extends AbstractRule { private static final int RT_MAX_EXCEED_N = 5; private static ScheduledExecutorService pool = Executors.newScheduledThreadPool( - Runtime.getRuntime().availableProcessors(), new NamedThreadFactory("sentinel-degrade-reset-task", true)); + Runtime.getRuntime().availableProcessors(), new NamedThreadFactory("sentinel-degrade-reset-task", true)); - public DegradeRule() {} + public DegradeRule() { + } public DegradeRule(String resourceName) { setResource(resourceName); @@ -81,7 +82,16 @@ public DegradeRule(String resourceName) { */ private int grade = RuleConstant.DEGRADE_GRADE_RT; - private final AtomicBoolean cut = new AtomicBoolean(false); + /** + * First request after degrade + */ + private AtomicBoolean firstRequest = new AtomicBoolean(true); + + private volatile int cut = RuleConstant.DEGRADE_CUT_CLOSE; + /** + * Degrade half-open-switch(default=false) + */ + private boolean enableHalfOpen = false; public int getGrade() { return grade; @@ -94,6 +104,8 @@ public DegradeRule setGrade(int grade) { private AtomicLong passCount = new AtomicLong(0); + private final Object lock = new Object(); + public double getCount() { return count; } @@ -103,14 +115,23 @@ public DegradeRule setCount(double count) { return this; } - private boolean isCut() { - return cut.get(); + public int getCut() { + return cut; + } + + public void setCut(int cut) { + this.cut = cut; + } + + public boolean isEnableHalfOpen() { + return enableHalfOpen; } - private void setCut(boolean cut) { - this.cut.set(cut); + public void setEnableHalfOpen(boolean enableHalfOpen) { + this.enableHalfOpen = enableHalfOpen; } + public AtomicLong getPassCount() { return passCount; } @@ -136,7 +157,7 @@ public boolean equals(Object o) { return false; } - DegradeRule that = (DegradeRule)o; + DegradeRule that = (DegradeRule) o; if (count != that.count) { return false; @@ -161,27 +182,65 @@ public int hashCode() { @Override public boolean passCheck(Context context, DefaultNode node, int acquireCount, Object... args) { - if (cut.get()) { + if (cut == RuleConstant.DEGRADE_CUT_OPEN) { return false; } - ClusterNode clusterNode = ClusterBuilderSlot.getClusterNode(this.getResource()); if (clusterNode == null) { return true; } - + // degrade_cut_half_open + if (cut == RuleConstant.DEGRADE_CUT_HALF_OPEN) { + //First request after downgrade + if (firstRequest.get()) { + firstRequest.set(false); + return true; + } + //In the half-open state, only five requests are all normal and will be fully opened. + if (grade == RuleConstant.DEGRADE_GRADE_RT) { + long rt = clusterNode.getLastRt().get(); + if (rt == 0) { + return degradeCutOpen(); + } + if (rt < this.count) { + cut = RuleConstant.DEGRADE_CUT_CLOSE; + return true; + } + clusterNode.minusRt(1); + return degradeCutOpen(); + } + if (grade != RuleConstant.DEGRADE_GRADE_RT) { + //When the sliding window slides over the next window, it needs to be cleared. + if (clusterNode.getLastResult().get()) { + cut = RuleConstant.DEGRADE_CUT_CLOSE; + return true; + } + if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { + clusterNode.minusSecondException(); + } else { + clusterNode.minusMinuteException(); + } + return degradeCutOpen(); + } + } + // degrade_cut_close if (grade == RuleConstant.DEGRADE_GRADE_RT) { double rt = clusterNode.avgRt(); if (rt < this.count) { passCount.set(0); + clusterNode.resetLastRt(); return true; } - // Sentinel will degrade the service only if count exceeds. if (passCount.incrementAndGet() < RT_MAX_EXCEED_N) { return true; } - } else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { + clusterNode.minusRt(clusterNode.getLastRtSum().get()); + clusterNode.resetLastRt(); + passCount.set(0); + return degradeCutOpen(); + } + if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_RATIO) { double exception = clusterNode.exceptionQps(); double success = clusterNode.successQps(); long total = clusterNode.totalQps(); @@ -198,30 +257,29 @@ public boolean passCheck(Context context, DefaultNode node, int acquireCount, Ob if (exception / success < count) { return true; } - } else if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) { + clusterNode.minusSecondException(); + return degradeCutOpen(); + } + if (grade == RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT) { double exception = clusterNode.totalException(); if (exception < count) { return true; } + // after degrade-open minus exceptionCount + clusterNode.minusMinuteException(); } - - if (cut.compareAndSet(false, true)) { - ResetTask resetTask = new ResetTask(this); - pool.schedule(resetTask, timeWindow, TimeUnit.SECONDS); - } - - return false; + return degradeCutOpen(); } @Override public String toString() { return "DegradeRule{" + - "resource=" + getResource() + - ", grade=" + grade + - ", count=" + count + - ", limitApp=" + getLimitApp() + - ", timeWindow=" + timeWindow + - "}"; + "resource=" + getResource() + + ", grade=" + grade + + ", count=" + count + + ", limitApp=" + getLimitApp() + + ", timeWindow=" + timeWindow + + "}"; } private static final class ResetTask implements Runnable { @@ -234,9 +292,26 @@ private static final class ResetTask implements Runnable { @Override public void run() { - rule.getPassCount().set(0); - rule.cut.set(false); + //enableHalfOpen + if (rule.enableHalfOpen) { + rule.setCut(RuleConstant.DEGRADE_CUT_HALF_OPEN); + } else { + rule.setCut(RuleConstant.DEGRADE_CUT_CLOSE); + } + } + } + + private boolean degradeCutOpen() { + synchronized (lock) { + if (cut != RuleConstant.DEGRADE_CUT_OPEN) { + // Automatically degrade. + cut = RuleConstant.DEGRADE_CUT_OPEN; + ResetTask resetTask = new ResetTask(this); + pool.schedule(resetTask, timeWindow, TimeUnit.SECONDS); + } + return false; } } + } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/controller/RateLimiterController.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/controller/RateLimiterController.java index ca07b028cf..354988f763 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/controller/RateLimiterController.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/controller/RateLimiterController.java @@ -65,27 +65,25 @@ public boolean canPass(Node node, int acquireCount, boolean prioritized) { // Contention may exist here, but it's okay. latestPassedTime.set(currentTime); return true; - } else { - // Calculate the time to wait. - long waitTime = costTime + latestPassedTime.get() - TimeUtil.currentTimeMillis(); + } + // Calculate the time to wait. + long waitTime = costTime + latestPassedTime.get() - TimeUtil.currentTimeMillis(); + if (waitTime > maxQueueingTimeMs) { + return false; + } + long oldTime = latestPassedTime.addAndGet(costTime); + try { + waitTime = oldTime - TimeUtil.currentTimeMillis(); if (waitTime > maxQueueingTimeMs) { + latestPassedTime.addAndGet(-costTime); return false; - } else { - long oldTime = latestPassedTime.addAndGet(costTime); - try { - waitTime = oldTime - TimeUtil.currentTimeMillis(); - if (waitTime > maxQueueingTimeMs) { - latestPassedTime.addAndGet(-costTime); - return false; - } - // in race condition waitTime may <= 0 - if (waitTime > 0) { - Thread.sleep(waitTime); - } - return true; - } catch (InterruptedException e) { - } } + // in race condition waitTime may <= 0 + if (waitTime > 0) { + Thread.sleep(waitTime); + } + return true; + } catch (InterruptedException e) { } return false; } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/controller/WarmUpRateLimiterController.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/controller/WarmUpRateLimiterController.java index ca8d953b2a..0fce5888b2 100644 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/controller/WarmUpRateLimiterController.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/block/flow/controller/WarmUpRateLimiterController.java @@ -66,25 +66,23 @@ public boolean canPass(Node node, int acquireCount, boolean prioritized) { if (expectedTime <= currentTime) { latestPassedTime.set(currentTime); return true; - } else { - long waitTime = costTime + latestPassedTime.get() - currentTime; + } + long waitTime = costTime + latestPassedTime.get() - currentTime; + if (waitTime > timeOutInMs) { + return false; + } + long oldTime = latestPassedTime.addAndGet(costTime); + try { + waitTime = oldTime - TimeUtil.currentTimeMillis(); if (waitTime > timeOutInMs) { + latestPassedTime.addAndGet(-costTime); return false; - } else { - long oldTime = latestPassedTime.addAndGet(costTime); - try { - waitTime = oldTime - TimeUtil.currentTimeMillis(); - if (waitTime > timeOutInMs) { - latestPassedTime.addAndGet(-costTime); - return false; - } - if (waitTime > 0) { - Thread.sleep(waitTime); - } - return true; - } catch (InterruptedException e) { - } } + if (waitTime > 0) { + Thread.sleep(waitTime); + } + return true; + } catch (InterruptedException e) { } return false; } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/base/LongAdder.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/base/LongAdder.java index beeaedcf67..671f1ad655 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/base/LongAdder.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/base/LongAdder.java @@ -82,10 +82,10 @@ public void increment() { } /** - * Equivalent to {@code add(-1)}. + * Equivalent to {@code add(-x)}. */ - public void decrement() { - add(-1L); + public void decrement(long x) { + add(-x); } /** diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/data/MetricBucket.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/data/MetricBucket.java index ab1dc79634..c8a9c6f660 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/data/MetricBucket.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/data/MetricBucket.java @@ -57,6 +57,11 @@ public MetricBucket reset() { return this; } + public MetricBucket reset(MetricEvent event) { + counters[event.ordinal()].reset(); + return this; + } + public long get(MetricEvent event) { return counters[event.ordinal()].sum(); } @@ -66,6 +71,11 @@ public MetricBucket add(MetricEvent event, long n) { return this; } + public MetricBucket minus(MetricEvent event, long n) { + counters[event.ordinal()].decrement(n); + return this; + } + public long pass() { return get(MetricEvent.PASS); } @@ -98,6 +108,17 @@ public void addException(int n) { add(MetricEvent.EXCEPTION, n); } + public void minusException(int n) { + minus(MetricEvent.EXCEPTION, n); + } + + public void minusRt(long rt) { + minus(MetricEvent.RT, rt); + + } + public void minusSuccess(int n){ + minus(MetricEvent.SUCCESS,n); + } public void addBlock(int n) { add(MetricEvent.BLOCK, n); } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java index cadec6ea58..2327e340fe 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/ArrayMetric.java @@ -165,6 +165,7 @@ public MetricBucket[] windows() { public void addException(int count) { WindowWrap wrap = data.currentWindow(); wrap.value().addException(count); + } @Override @@ -191,6 +192,33 @@ public void addRT(long rt) { wrap.value().addRT(rt); } + @Override + public void minusSuccess(int n) { + if (data.values().isEmpty()) { + return; + } + WindowWrap wrap = data.currentWindow(); + wrap.value().minusSuccess(n); + } + + @Override + public void minusRt(long rt) { + if (data.values().isEmpty()) { + return; + } + WindowWrap wrap = data.currentWindow(); + wrap.value().minusRt(rt); + } + + @Override + public void minusException() { + if (data.values().isEmpty()) { + return; + } + WindowWrap wrap = data.currentWindow(); + wrap.value().minusException(1); + } + @Override public void debugQps() { data.currentWindow(); @@ -199,7 +227,7 @@ public void debugQps() { for (WindowWrap windowWrap : data.list()) { sb.append(windowWrap.windowStart()).append(":").append(windowWrap.value().pass()).append(":") - .append(windowWrap.value().block()); + .append(windowWrap.value().block()); sb.append(","); } @@ -235,4 +263,12 @@ public double getWindowIntervalInSec() { public int getSampleCount() { return data.getSampleCount(); } + + @Override + public boolean newTimeWindow() { + if (data.values().isEmpty()) { + return true; + } + return false; + } } diff --git a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java index a3fb933bf6..9c6a527cd8 100755 --- a/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java +++ b/sentinel-core/src/main/java/com/alibaba/csp/sentinel/slots/statistic/metric/Metric.java @@ -118,6 +118,28 @@ public interface Metric { */ void addRT(long rt); + /** + * is New TimeWindow + */ + boolean newTimeWindow(); + + /** + * minus success count + */ + void minusSuccess(int n); + + /** + * minus rt + */ + void minusRt(long rt); + + /** + * minus exception + * + * @return + */ + void minusException(); + double getWindowIntervalInSec(); int getSampleCount(); diff --git a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java index faf78931d6..647a10f542 100755 --- a/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java +++ b/sentinel-core/src/test/java/com/alibaba/csp/sentinel/slots/block/degrade/DegradeTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.when; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; @@ -41,12 +42,11 @@ public void testAverageRtDegrade() throws InterruptedException { String key = "test_degrade_average_rt"; ClusterNode cn = mock(ClusterNode.class); ClusterBuilderSlot.getClusterNodeMap().put(new StringResourceWrapper(key, EntryType.IN), cn); - Context context = mock(Context.class); DefaultNode node = mock(DefaultNode.class); when(node.getClusterNode()).thenReturn(cn); when(cn.avgRt()).thenReturn(2L); - + when(cn.getLastRtSum()).thenReturn(new AtomicInteger(0)); DegradeRule rule = new DegradeRule(); rule.setCount(1); rule.setResource(key); @@ -59,10 +59,14 @@ public void testAverageRtDegrade() throws InterruptedException { // The third time will fail. assertFalse(rule.passCheck(context, node, 1)); assertFalse(rule.passCheck(context, node, 1)); - // Restore. - TimeUnit.MILLISECONDS.sleep(2200); - assertTrue(rule.passCheck(context, node, 1)); + + TimeUnit.SECONDS.sleep(6); + rule.setCount(3); + //When the fifth request ends, degrade will blow close. + for (int i = 0; i < 10; i++) { + assertTrue(rule.passCheck(context, node, 1)); + } } @Test @@ -92,9 +96,21 @@ public void testExceptionRatioModeDegrade() throws Throwable { // Restore from the degrade timeout. TimeUnit.MILLISECONDS.sleep(2200); - when(cn.successQps()).thenReturn(20L); + // when(cn.successQps()).thenReturn(20L); // Will pass. - assertTrue(rule.passCheck(context, node, 1)); + when(cn.exceptionQps()).thenReturn(0L); + for (int i = 0; i < 5; i++) { + assertTrue(rule.passCheck(context, node, 1)); + } + when(cn.exceptionQps()).thenReturn(20L); + assertFalse(rule.passCheck(context, node, 1)); + TimeUnit.SECONDS.sleep(6); + // Will pass. + //When the fifth request ends, degrade will blow close. + when(cn.exceptionQps()).thenReturn(0L); + for (int i = 0; i < 5; i++) { + assertTrue(rule.passCheck(context, node, 1)); + } } @Test @@ -115,16 +131,16 @@ public void testExceptionCountModeDegrade() throws Throwable { rule.setGrade(RuleConstant.DEGRADE_GRADE_EXCEPTION_COUNT); when(cn.totalException()).thenReturn(4L); - // Will fail. assertFalse(rule.passCheck(context, node, 1)); - // Restore from the degrade timeout. - TimeUnit.MILLISECONDS.sleep(2200); - when(cn.totalException()).thenReturn(0L); - // Will pass. - assertTrue(rule.passCheck(context, node, 1)); + TimeUnit.SECONDS.sleep(3); + when(cn.totalException()).thenReturn(3L); + //When the fifth request ends, degrade will blow close. + for (int i = 0; i < 10; i++) { + assertTrue(rule.passCheck(context, node, 1)); + } } }