fix all the pipeline failues for release 3.4

Items resolved: 1. fix the vet error: possible misuse of reflect.SliceHeader; 2. fix the vet error: call to (*T).Fatal from a non-test goroutine; 3. bump package golang.org/x/crypto, net and sys; 4. bump boltdb from 1.3.3 to 1.3.6; 5. remove the vendor directory; 6. remove go 1.12.17 and 1.15.15, add go 1.16.15 into pipeline; 7. bump go version to 1.16 in go.mod; 8. fix the issue: compile: version go1.16.15 does not match go tool version go1.17.11; 9. fix data race on compactMainRev and watcherGauge. Signed-off-by: Benjamin Wang <wachao@vmware.com>
etcd-io · Jun 20, 2022 · b0e451b · b0e451b
1 parent c2c9e7d
commit b0e451b
Show file tree

Hide file tree

Showing 1,107 changed files with 79 additions and 405,173 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -6,7 +6,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.12.17, 1.15.15]
         target:
         - linux-amd64-fmt
         - linux-amd64-integration-1-cpu
@@ -17,53 +16,38 @@ jobs:
         - all-build
         - linux-amd64-grpcproxy
         - linux-386-unit
-        exclude:
-        - go: 1.12.17
-          target: linux-amd64-grpcproxy
-        - go: 1.12.17
-          target: linux-386-unit
-        - go: 1.15.15
-          target: linux-amd64-integration-1-cpu
-        - go: 1.15.15
-          target: linux-amd64-integration-2-cpu
-        - go: 1.12.17
-          target: linux-amd64-unit
-        - go: 1.15.15
-          target: linux-amd64-coverage
-        - go: 1.12.17
-          target: linux-amd64-fmt
-        - go: 1.15.15
-          target: linux-386-unit
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go }}
+        go-version: "1.16.15"
     - run: date
     - env:
         TARGET: ${{ matrix.target }}
-        GO_VERSION: ${{ matrix.go }}
       run: |
-        RACE='true'; if [[ ${GO_VERSION} == 1.15.15 ]]; then echo 'setting race off'; RACE='false'; fi
+        go version
+        echo ${GOROOT}
+        sudo unlink /usr/bin/go; sudo ln -s ${GOROOT}/bin/go /usr/bin/go; ls -lrt /usr/bin/go
+
         echo "${TARGET}"
         case "${TARGET}" in
           linux-amd64-fmt)
             GOARCH=amd64 PASSES='fmt bom dep' ./test
             ;;
           linux-amd64-integration-1-cpu)
-            GOARCH=amd64 CPU=1 PASSES='integration' RACE="${RACE}" ./test
+            GOARCH=amd64 CPU=1 PASSES='integration' RACE='false' ./test
             ;;
           linux-amd64-integration-2-cpu)
-            GOARCH=amd64 CPU=2 PASSES='integration' RACE="${RACE}" ./test
+            GOARCH=amd64 CPU=2 PASSES='integration' RACE='false' ./test
             ;;
           linux-amd64-integration-4-cpu)
-            GOARCH=amd64 CPU=4 PASSES='integration' RACE="${RACE}" ./test
+            GOARCH=amd64 CPU=4 PASSES='integration' RACE='false' ./test
             ;;
           linux-amd64-functional)
-            ./build && GOARCH=amd64 PASSES='functional' RACE="${RACE}" ./test
+            ./build && GOARCH=amd64 PASSES='functional' ./test
             ;;
           linux-amd64-unit)
-            GOARCH=amd64 PASSES='unit' RACE="${RACE}" ./test
+            GOARCH=amd64 PASSES='unit' RACE='false' ./test
             ;;
           all-build)
             GOARCH=amd64 PASSES='build' ./test

diff --git a/clientv3/concurrency/election_test.go b/clientv3/concurrency/election_test.go
@@ -16,6 +16,7 @@ package concurrency_test
 
 import (
 	"context"
+	"errors"
 	"log"
 	"strings"
 	"testing"
@@ -64,14 +65,15 @@ func TestResumeElection(t *testing.T) {
 		string(leader.Kvs[0].Key), leader.Kvs[0].CreateRevision)
 
 	respChan := make(chan *clientv3.GetResponse)
+	errCHan := make(chan error)
 	go func() {
 		o := e.Observe(ctx)
 		respChan <- nil
 		for {
 			select {
 			case resp, ok := <-o:
 				if !ok {
-					t.Fatal("Observe() channel closed prematurely")
+					errCHan <- errors.New("observe() channel closed prematurely")
 				}
 				// Ignore any observations that candidate1 was elected
 				if string(resp.Kvs[0].Value) == "candidate1" {
@@ -104,7 +106,12 @@ func TestResumeElection(t *testing.T) {
 	}
 
 	// wait for observed leader change
-	resp := <-respChan
+	var resp *clientv3.GetResponse
+	select {
+	case resp = <-respChan:
+	case err := <-errCHan:
+		t.Fatal(err)
+	}
 
 	kv := resp.Kvs[0]
 	if !strings.HasPrefix(string(kv.Key), prefix) {

diff --git a/etcdserver/api/v2store/node_extern_test.go b/etcdserver/api/v2store/node_extern_test.go
@@ -104,5 +104,5 @@ func TestNodeExternClone(t *testing.T) {
 func sameSlice(a, b []*NodeExtern) bool {
 	ah := (*reflect.SliceHeader)(unsafe.Pointer(&a))
 	bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	return *ah == *bh
+	return ah.Data == bh.Data && ah.Len == bh.Len && ah.Cap == bh.Cap
 }
diff --git a/etcdserver/server_test.go b/etcdserver/server_test.go
@@ -989,12 +989,14 @@ func TestSnapshot(t *testing.T) {
 
 	ch := make(chan struct{}, 2)
 
+	errChan := make(chan error)
+
 	go func() {
 		gaction, _ := p.Wait(2)
 		defer func() { ch <- struct{}{} }()
 
 		if len(gaction) != 2 {
-			t.Fatalf("len(action) = %d, want 2", len(gaction))
+			errChan <- fmt.Errorf("len(action) = %d, want 2", len(gaction))
 		}
 		if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "SaveSnap"}) {
 			t.Errorf("action = %s, want SaveSnap", gaction[0])
@@ -1023,6 +1025,12 @@ func TestSnapshot(t *testing.T) {
 	srv.snapshot(1, raftpb.ConfState{Voters: []uint64{1}})
 	<-ch
 	<-ch
+
+	select {
+	case err := <-errChan:
+		t.Fatal(err)
+	default:
+	}
 }
 
 // TestSnapshotOrdering ensures raft persists snapshot onto disk before
@@ -1148,6 +1156,7 @@ func TestTriggerSnap(t *testing.T) {
 	srv.start()
 
 	donec := make(chan struct{})
+	errChan := make(chan error)
 	go func() {
 		wcnt := 3 + snapc
 		gaction, _ := p.Wait(wcnt)
@@ -1156,7 +1165,7 @@ func TestTriggerSnap(t *testing.T) {
 		// (SnapshotCount+1) * Puts + SaveSnap = (SnapshotCount+1) * Save + SaveSnap + Release
 		if len(gaction) != wcnt {
 			t.Logf("gaction: %v", gaction)
-			t.Fatalf("len(action) = %d, want %d", len(gaction), wcnt)
+			errChan <- fmt.Errorf("len(action) = %d, want %d", len(gaction), wcnt)
 		}
 
 		if !reflect.DeepEqual(gaction[wcnt-2], testutil.Action{Name: "SaveSnap"}) {
@@ -1175,6 +1184,12 @@ func TestTriggerSnap(t *testing.T) {
 
 	<-donec
 	srv.Stop()
+
+	select {
+	case err := <-errChan:
+		t.Fatal(err)
+	default:
+	}
 }
 
 // TestConcurrentApplyAndSnapshotV3 will send out snapshots concurrently with

diff --git a/go.mod b/go.mod
@@ -1,6 +1,6 @@
 module go.etcd.io/etcd
 
-go 1.12
+go 1.16
 
 require (
 	github.com/bgentry/speakeasy v0.1.0
@@ -38,13 +38,14 @@ require (
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966
 	github.com/urfave/cli v1.20.0
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
-	go.etcd.io/bbolt v1.3.3
+	go.etcd.io/bbolt v1.3.6
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-	golang.org/x/net v0.0.0-20201021035429-f5854403a974
-	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
+	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2
 	google.golang.org/grpc v1.26.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.25

diff --git a/go.sum b/go.sum
@@ -133,19 +133,18 @@ github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
+go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -156,9 +155,9 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -169,22 +168,26 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

diff --git a/integration/cluster.go b/integration/cluster.go
@@ -1153,7 +1153,12 @@ func (m *member) Metric(metricName string) (string, error) {
 	}
 	lines := strings.Split(string(b), "\n")
 	for _, l := range lines {
+		if strings.Contains(l, metricName) {
+			fmt.Printf("######Contains:  metricName: %s\n", l)
+		}
+
 		if strings.HasPrefix(l, metricName) {
+			fmt.Printf("######HasPrefix:  metricName: %s\n", l)
 			return strings.Split(l, " ")[1], nil
 		}
 	}

diff --git a/integration/v3_watch_test.go b/integration/v3_watch_test.go
@@ -1249,6 +1249,7 @@ func TestV3WatchCancellation(t *testing.T) {
 		expected = "1"
 	}
 
+	fmt.Printf("###### expected: %s, minWatches: %s\n", expected, minWatches)
 	if minWatches != expected {
 		t.Fatalf("expected %s watch, got %s", expected, minWatches)
 	}

diff --git a/mvcc/kvstore.go b/mvcc/kvstore.go
@@ -351,8 +351,15 @@ func (s *store) Restore(b backend.Backend) error {
 	atomic.StoreUint64(&s.consistentIndex, 0)
 	s.b = b
 	s.kvindex = newTreeIndex(s.lg)
-	s.currentRev = 1
-	s.compactMainRev = -1
+
+	{
+		// During restore the metrics might report 'special' values.
+		s.revMu.Lock()
+		s.currentRev = 1
+		s.compactMainRev = -1
+		s.revMu.Unlock()
+	}
+
 	s.fifoSched = schedule.NewFIFOScheduler()
 	s.stopc = make(chan struct{})
 

diff --git a/mvcc/watchable_store.go b/mvcc/watchable_store.go
@@ -152,10 +152,13 @@ func (s *watchableStore) cancelWatcher(wa *watcher) {
 		s.mu.Lock()
 		if s.unsynced.delete(wa) {
 			slowWatcherGauge.Dec()
+			watcherGauge.Dec()
 			break
 		} else if s.synced.delete(wa) {
+			watcherGauge.Dec()
 			break
 		} else if wa.compacted {
+			watcherGauge.Dec()
 			break
 		} else if wa.ch == nil {
 			// already canceled (e.g., cancel/close race)
@@ -175,6 +178,7 @@ func (s *watchableStore) cancelWatcher(wa *watcher) {
 		}
 		if victimBatch != nil {
 			slowWatcherGauge.Dec()
+			watcherGauge.Dec()
 			delete(victimBatch, wa)
 			break
 		}
@@ -184,7 +188,6 @@ func (s *watchableStore) cancelWatcher(wa *watcher) {
 		time.Sleep(time.Millisecond)
 	}
 
-	watcherGauge.Dec()
 	wa.ch = nil
 	s.mu.Unlock()
 }

diff --git a/vendor/github.com/beorn7/perks/LICENSE b/vendor/github.com/beorn7/perks/LICENSE