server/tso: fix the panic issue when getTS in the follower (tikv#1930)

Signed-off-by: nolouch <nolouch@gmail.com>
nolouch · Nov 24, 2019 · 8931c60 · 8931c60
1 parent c6fb77f
commit 8931c60
Show file tree

Hide file tree

Showing 2 changed files with 59 additions and 2 deletions.
diff --git a/server/tso/tso.go b/server/tso/tso.go
@@ -255,7 +255,7 @@ func (t *TimestampOracle) ResetTimestamp() {
 	atomic.StorePointer(&t.ts, unsafe.Pointer(zero))
 }
 
-const maxRetryCount = 100
+var maxRetryCount = 100
 
 // GetRespTS is used to get a timestamp.
 func (t *TimestampOracle) GetRespTS(count uint32) (pdpb.Timestamp, error) {
@@ -265,9 +265,13 @@ func (t *TimestampOracle) GetRespTS(count uint32) (pdpb.Timestamp, error) {
 		return resp, errors.New("tso count should be positive")
 	}
 
+	failpoint.Inject("skipRetryGetTS", func() {
+		maxRetryCount = 1
+	})
+
 	for i := 0; i < maxRetryCount; i++ {
 		current := (*atomicObject)(atomic.LoadPointer(&t.ts))
-		if current.physical == typeutil.ZeroTime {
+		if current == nil || current.physical == typeutil.ZeroTime {
 			log.Error("we haven't synced timestamp ok, wait and retry", zap.Int("retry-count", i))
 			time.Sleep(200 * time.Millisecond)
 			continue

diff --git a/tests/server/tso/tso_test.go b/tests/server/tso/tso_test.go
@@ -15,6 +15,7 @@ package tso_test
 
 import (
 	"context"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -51,6 +52,7 @@ func (s *testTsoSuite) SetUpSuite(c *C) {
 func (s *testTsoSuite) TearDownSuite(c *C) {
 	s.cancel()
 }
+
 func (s *testTsoSuite) testGetTimestamp(c *C, n int) *pdpb.Timestamp {
 	var err error
 	cluster, err := tests.NewTestCluster(1)
@@ -229,3 +231,54 @@ func (s *testTimeFallBackSuite) TestTimeFallBack(c *C) {
 
 	wg.Wait()
 }
+
+var _ = Suite(&testFollowerTsoSuite{})
+
+type testFollowerTsoSuite struct {
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+func (s *testFollowerTsoSuite) SetUpSuite(c *C) {
+	s.ctx, s.cancel = context.WithCancel(context.Background())
+	server.EnableZap = true
+}
+
+func (s *testFollowerTsoSuite) TearDownSuite(c *C) {
+	s.cancel()
+}
+func (s *testFollowerTsoSuite) TestRequest(c *C) {
+	c.Assert(failpoint.Enable("github.com/pingcap/pd/server/tso/skipRetryGetTS", `return(true)`), IsNil)
+	var err error
+	cluster, err := tests.NewTestCluster(2)
+	defer cluster.Destroy()
+	c.Assert(err, IsNil)
+
+	err = cluster.RunInitialServers(s.ctx)
+	c.Assert(err, IsNil)
+	cluster.WaitLeader()
+
+	servers := cluster.GetServers()
+	var followerServer *tests.TestServer
+	for _, s := range servers {
+		if !s.IsLeader() {
+			followerServer = s
+		}
+	}
+	c.Assert(followerServer, NotNil)
+	grpcPDClient := testutil.MustNewGrpcClient(c, followerServer.GetAddr())
+	clusterID := followerServer.GetClusterID()
+
+	req := &pdpb.TsoRequest{Header: testutil.NewRequestHeader(clusterID), Count: 1}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	tsoClient, err := grpcPDClient.Tso(ctx)
+	c.Assert(err, IsNil)
+	defer tsoClient.CloseSend()
+	err = tsoClient.Send(req)
+	c.Assert(err, IsNil)
+	_, err = tsoClient.Recv()
+	c.Assert(err, NotNil)
+	c.Assert(strings.Contains(err.Error(), "can not get timestamp"), IsTrue)
+	failpoint.Disable("github.com/pingcap/pd/server/tso/skipRetryGetTS")
+}