Skip to content

Commit

Permalink
pd-ctl: retry when keyspace group manager not initialized (#7442)
Browse files Browse the repository at this point in the history 
close #7441
  • Loading branch information
@AmoebaProtozoa
AmoebaProtozoa authored Nov 29, 2023
1 parent 8c8b4d4 commit 4356aeb
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 19 deletions.
2 changes: 1 addition & 1 deletion server/apiv2/handlers/keyspace.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func LoadKeyspace(c *gin.Context) {
if value, ok := c.GetQuery("force_refresh_group_id"); ok && value == "true" {
groupManager := svr.GetKeyspaceGroupManager()
if groupManager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, managerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
// keyspace has been checked in LoadKeyspace, so no need to check again.
Expand Down
25 changes: 13 additions & 12 deletions server/apiv2/handlers/tso_keyspace_group.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ import (
"github.com/tikv/pd/server/apiv2/middlewares"
)

const groupManagerUninitializedErr = "keyspace group manager is not initialized"
// GroupManagerUninitializedErr is the error message for uninitialized keyspace group manager.
const GroupManagerUninitializedErr = "keyspace group manager is not initialized"

// RegisterTSOKeyspaceGroup registers keyspace group handlers to the server.
func RegisterTSOKeyspaceGroup(r *gin.RouterGroup) {
Expand Down Expand Up @@ -78,7 +79,7 @@ func CreateKeyspaceGroups(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
err = manager.CreateKeyspaceGroups(createParams.KeyspaceGroups)
Expand All @@ -101,7 +102,7 @@ func GetKeyspaceGroups(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
keyspaceGroups, err := manager.GetKeyspaceGroups(scanStart, scanLimit)
Expand Down Expand Up @@ -152,7 +153,7 @@ func GetKeyspaceGroupByID(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}

Expand Down Expand Up @@ -189,7 +190,7 @@ func DeleteKeyspaceGroupByID(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
kg, err := manager.DeleteKeyspaceGroupByID(id)
Expand Down Expand Up @@ -250,7 +251,7 @@ func SplitKeyspaceGroupByID(c *gin.Context) {
}
groupManager := svr.GetKeyspaceGroupManager()
if groupManager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}

Expand Down Expand Up @@ -289,7 +290,7 @@ func FinishSplitKeyspaceByID(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
err = manager.FinishSplitKeyspaceByID(id)
Expand Down Expand Up @@ -337,7 +338,7 @@ func MergeKeyspaceGroups(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
groupManager := svr.GetKeyspaceGroupManager()
if groupManager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
// Merge keyspace group.
Expand All @@ -364,7 +365,7 @@ func FinishMergeKeyspaceByID(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
err = manager.FinishMergeKeyspaceByID(id)
Expand All @@ -390,7 +391,7 @@ func AllocNodesForKeyspaceGroup(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
allocParams := &AllocNodesForKeyspaceGroupParams{}
Expand Down Expand Up @@ -437,7 +438,7 @@ func SetNodesForKeyspaceGroup(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
setParams := &SetNodesForKeyspaceGroupParams{}
Expand Down Expand Up @@ -493,7 +494,7 @@ func SetPriorityForKeyspaceGroup(c *gin.Context) {
svr := c.MustGet(middlewares.ServerContextKey).(*server.Server)
manager := svr.GetKeyspaceGroupManager()
if manager == nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, groupManagerUninitializedErr)
c.AbortWithStatusJSON(http.StatusInternalServerError, GroupManagerUninitializedErr)
return
}
setParams := &SetPriorityForKeyspaceGroupParams{}
Expand Down
29 changes: 29 additions & 0 deletions tests/pdctl/keyspace/keyspace_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,35 @@ func TestKeyspace(t *testing.T) {
re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop"))
}

// Show command should auto retry without refresh_group_id if keyspace group manager not initialized.
// See issue: #7441
func TestKeyspaceGroupUninitialized(t *testing.T) {
re := require.New(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`))
re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)"))
tc, err := tests.NewTestCluster(ctx, 1)
re.NoError(err)
re.NoError(tc.RunInitialServers())
tc.WaitLeader()
re.NoError(tc.GetLeaderServer().BootstrapCluster())
pdAddr := tc.GetConfig().GetClientURL()

keyspaceName := "DEFAULT"
keyspaceID := uint32(0)
args := []string{"-u", pdAddr, "keyspace", "show", "name", keyspaceName}
output, err := pdctl.ExecuteCommand(pdctlCmd.GetRootCmd(), args...)
re.NoError(err)
var meta api.KeyspaceMeta
re.NoError(json.Unmarshal(output, &meta))
re.Equal(keyspaceName, meta.GetName())
re.Equal(keyspaceID, meta.GetId())

re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop"))
re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion"))
}

type keyspaceTestSuite struct {
suite.Suite
ctx context.Context
Expand Down
28 changes: 22 additions & 6 deletions tools/pd-ctl/pdctl/command/keyspace_command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,12 @@ import (
const (
keyspacePrefix = "pd/api/v2/keyspaces"
// flags
nmConfig = "config"
nmLimit = "limit"
nmPageToken = "page_token"
nmRemove = "remove"
nmUpdate = "update"
nmConfig = "config"
nmLimit = "limit"
nmPageToken = "page_token"
nmRemove = "remove"
nmUpdate = "update"
nmForceRefreshGroupID = "force_refresh_group_id"
)

// NewKeyspaceCommand returns a keyspace subcommand of rootCmd.
Expand Down Expand Up @@ -64,6 +65,7 @@ func newShowKeyspaceCommand() *cobra.Command {
Short: "show keyspace metadata specified by keyspace name",
Run: showKeyspaceNameCommandFunc,
}
showByName.Flags().Bool(nmForceRefreshGroupID, true, "force refresh keyspace group id")
r.AddCommand(showByID)
r.AddCommand(showByName)
return r
Expand All @@ -87,7 +89,21 @@ func showKeyspaceNameCommandFunc(cmd *cobra.Command, args []string) {
cmd.Usage()
return
}
resp, err := doRequest(cmd, fmt.Sprintf("%s/%s?force_refresh_group_id=true", keyspacePrefix, args[0]), http.MethodGet, http.Header{})
refreshGroupID, err := cmd.Flags().GetBool(nmForceRefreshGroupID)
if err != nil {
cmd.PrintErrln("Failed to parse flag: ", err)
return
}
url := fmt.Sprintf("%s/%s", keyspacePrefix, args[0])
if refreshGroupID {
url += "?force_refresh_group_id=true"
}
resp, err := doRequest(cmd, url, http.MethodGet, http.Header{})
// Retry without the force_refresh_group_id if the keyspace group manager is not initialized.
// This can happen when PD is not running in API mode.
if err != nil && refreshGroupID && strings.Contains(err.Error(), handlers.GroupManagerUninitializedErr) {
resp, err = doRequest(cmd, fmt.Sprintf("%s/%s", keyspacePrefix, args[0]), http.MethodGet, http.Header{})
}
if err != nil {
cmd.PrintErrln("Failed to get the keyspace information: ", err)
return
Expand Down

0 comments on commit 4356aeb

Please sign in to comment.