diff --git a/node.go b/node.go index 9c53aed2..fe8f8dee 100644 --- a/node.go +++ b/node.go @@ -189,6 +189,32 @@ type Node interface { // TransferLeadership attempts to transfer leadership to the given transferee. TransferLeadership(ctx context.Context, lead, transferee uint64) + // ForgetLeader forgets a follower's current leader, changing it to None. It + // remains a leaderless follower in the current term, without campaigning. + // + // This is useful with PreVote+CheckQuorum, where followers will normally not + // grant pre-votes if they've heard from the leader in the past election + // timeout interval. Leaderless followers can grant pre-votes immediately, so + // if a quorum of followers have strong reason to believe the leader is dead + // (for example via a side-channel or external failure detector) and forget it + // then they can elect a new leader immediately, without waiting out the + // election timeout. They will also revert to normal followers if they hear + // from the leader again, or transition to candidates on an election timeout. + // + // For example, consider a three-node cluster where 1 is the leader and 2+3 + // have just received a heartbeat from it. If 2 and 3 believe the leader has + // now died (maybe they know that an orchestration system shut down 1's VM), + // we can instruct 2 to forget the leader and 3 to campaign. 2 will then be + // able to grant 3's pre-vote and elect 3 as leader immediately (normally 2 + // would reject the vote until an election timeout passes because it has heard + // from the leader recently). However, 3 can not campaign unilaterally, a + // quorum have to agree that the leader is dead, which avoids disrupting the + // leader if individual nodes are wrong about it being dead. + // + // This does nothing with ReadOnlyLeaseBased, since it would allow a new + // leader to be elected without the old leader knowing. + ForgetLeader(ctx context.Context) error + // ReadIndex request a read state. The read state will be set in the ready. // Read state has a read index. Once the application advances further than the read // index, any linearizable read requests issued before the read request can be @@ -575,6 +601,10 @@ func (n *node) TransferLeadership(ctx context.Context, lead, transferee uint64) } } +func (n *node) ForgetLeader(ctx context.Context) error { + return n.step(ctx, pb.Message{Type: pb.MsgForgetLeader}) +} + func (n *node) ReadIndex(ctx context.Context, rctx []byte) error { return n.step(ctx, pb.Message{Type: pb.MsgReadIndex, Entries: []pb.Entry{{Data: rctx}}}) } diff --git a/raft.go b/raft.go index 535889ea..63b4e08c 100644 --- a/raft.go +++ b/raft.go @@ -1284,6 +1284,8 @@ func stepLeader(r *raft, m pb.Message) error { sendMsgReadIndexResponse(r, m) return nil + case pb.MsgForgetLeader: + return nil // noop on leader } // All other message types require a progress for m.From (pr). @@ -1661,6 +1663,15 @@ func stepFollower(r *raft, m pb.Message) error { } m.To = r.lead r.send(m) + case pb.MsgForgetLeader: + if r.readOnly.option == ReadOnlyLeaseBased { + r.logger.Error("ignoring MsgForgetLeader due to ReadOnlyLeaseBased") + return nil + } + if r.lead != None { + r.logger.Infof("%x forgetting leader %x at term %d", r.id, r.lead, r.Term) + r.lead = None + } case pb.MsgTimeoutNow: r.logger.Infof("%x [term %d] received MsgTimeoutNow from %x and starts an election to get leadership.", r.id, r.Term, m.From) // Leadership transfers never use pre-vote even if r.preVote is true; we diff --git a/raftpb/raft.pb.go b/raftpb/raft.pb.go index 14802639..7dcdef0c 100644 --- a/raftpb/raft.pb.go +++ b/raftpb/raft.pb.go @@ -95,6 +95,7 @@ const ( MsgStorageAppendResp MessageType = 20 MsgStorageApply MessageType = 21 MsgStorageApplyResp MessageType = 22 + MsgForgetLeader MessageType = 23 ) var MessageType_name = map[int32]string{ @@ -121,6 +122,7 @@ var MessageType_name = map[int32]string{ 20: "MsgStorageAppendResp", 21: "MsgStorageApply", 22: "MsgStorageApplyResp", + 23: "MsgForgetLeader", } var MessageType_value = map[string]int32{ @@ -147,6 +149,7 @@ var MessageType_value = map[string]int32{ "MsgStorageAppendResp": 20, "MsgStorageApply": 21, "MsgStorageApplyResp": 22, + "MsgForgetLeader": 23, } func (x MessageType) Enum() *MessageType { @@ -726,76 +729,76 @@ func init() { func init() { proto.RegisterFile("raft.proto", fileDescriptor_b042552c306ae59b) } var fileDescriptor_b042552c306ae59b = []byte{ - // 1091 bytes of a gzipped FileDescriptorProto + // 1102 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xcb, 0x6e, 0x23, 0x45, - 0x14, 0xed, 0x6e, 0x77, 0xfc, 0xb8, 0x76, 0x9c, 0x4a, 0xc5, 0x13, 0x5a, 0x51, 0xe4, 0x31, 0x9e, - 0x41, 0x63, 0x05, 0x4d, 0x40, 0x46, 0x42, 0x88, 0x5d, 0x1e, 0x23, 0x25, 0x28, 0x0e, 0x83, 0x93, - 0xc9, 0x02, 0x09, 0x45, 0x15, 0x77, 0xa5, 0xd3, 0x60, 0x57, 0xb5, 0xaa, 0xcb, 0x21, 0xd9, 0x20, - 0xc4, 0x17, 0xb0, 0x64, 0xc3, 0x96, 0x3d, 0x7c, 0x45, 0x96, 0x59, 0xb2, 0x1a, 0x31, 0xc9, 0x1f, - 0xf0, 0x05, 0xa8, 0xaa, 0xab, 0x1f, 0x76, 0xa2, 0x59, 0xb0, 0xab, 0x3a, 0xf7, 0xd4, 0xbd, 0xe7, - 0x9e, 0xdb, 0x55, 0x0d, 0x20, 0xc8, 0xb9, 0xdc, 0x8c, 0x04, 0x97, 0x1c, 0x97, 0xd5, 0x3a, 0x3a, - 0x5b, 0x6b, 0x05, 0x3c, 0xe0, 0x1a, 0xfa, 0x44, 0xad, 0x92, 0x68, 0xf7, 0x27, 0x58, 0x78, 0xc5, - 0xa4, 0xb8, 0xc6, 0x1e, 0xb8, 0xc7, 0x54, 0x4c, 0x3c, 0xa7, 0x63, 0xf7, 0xdc, 0x6d, 0xf7, 0xe6, - 0xed, 0x53, 0x6b, 0xa8, 0x11, 0xbc, 0x06, 0x0b, 0xfb, 0xcc, 0xa7, 0x57, 0x5e, 0xa9, 0x10, 0x4a, - 0x20, 0xfc, 0x31, 0xb8, 0xc7, 0xd7, 0x11, 0xf5, 0xec, 0x8e, 0xdd, 0x6b, 0xf6, 0x97, 0x37, 0x93, - 0x5a, 0x9b, 0x3a, 0xa5, 0x0a, 0x64, 0x89, 0xae, 0x23, 0x8a, 0x31, 0xb8, 0xbb, 0x44, 0x12, 0xcf, - 0xed, 0xd8, 0xbd, 0xc6, 0x50, 0xaf, 0xbb, 0x3f, 0xdb, 0x80, 0x8e, 0x18, 0x89, 0xe2, 0x0b, 0x2e, - 0x07, 0x54, 0x12, 0x9f, 0x48, 0x82, 0x3f, 0x07, 0x18, 0x71, 0x76, 0x7e, 0x1a, 0x4b, 0x22, 0x93, - 0xdc, 0xf5, 0x3c, 0xf7, 0x0e, 0x67, 0xe7, 0x47, 0x2a, 0x60, 0x72, 0xd7, 0x46, 0x29, 0xa0, 0x94, - 0x86, 0x5a, 0x69, 0xb1, 0x89, 0x04, 0x52, 0xfd, 0x49, 0xd5, 0x5f, 0xb1, 0x09, 0x8d, 0x74, 0xbf, - 0x85, 0x6a, 0xaa, 0x40, 0x49, 0x54, 0x0a, 0x74, 0xcd, 0xc6, 0x50, 0xaf, 0xf1, 0x97, 0x50, 0x9d, - 0x18, 0x65, 0x3a, 0x71, 0xbd, 0xef, 0xa5, 0x5a, 0xe6, 0x95, 0x9b, 0xbc, 0x19, 0xbf, 0xfb, 0x6f, - 0x09, 0x2a, 0x03, 0x1a, 0xc7, 0x24, 0xa0, 0xf8, 0x25, 0xb8, 0x32, 0xf7, 0x6a, 0x25, 0xcd, 0x61, - 0xc2, 0x45, 0xb7, 0x14, 0x0d, 0xb7, 0xc0, 0x91, 0x7c, 0xa6, 0x13, 0x47, 0x72, 0xd5, 0xc6, 0xb9, - 0xe0, 0x73, 0x6d, 0x28, 0x24, 0x6b, 0xd0, 0x9d, 0x6f, 0x10, 0xb7, 0xa1, 0x32, 0xe6, 0x81, 0x9e, - 0xee, 0x42, 0x21, 0x98, 0x82, 0xb9, 0x6d, 0xe5, 0x87, 0xb6, 0xbd, 0x84, 0x0a, 0x65, 0x52, 0x84, - 0x34, 0xf6, 0x2a, 0x9d, 0x52, 0xaf, 0xde, 0x5f, 0x9c, 0x99, 0x71, 0x9a, 0xca, 0x70, 0xf0, 0x3a, - 0x94, 0x47, 0x7c, 0x32, 0x09, 0xa5, 0x57, 0x2d, 0xe4, 0x32, 0x98, 0x92, 0x78, 0xc9, 0x25, 0xf5, - 0x16, 0x8b, 0x12, 0x15, 0x82, 0xfb, 0x50, 0x8d, 0x8d, 0x97, 0x5e, 0x4d, 0x7b, 0x8c, 0xe6, 0x3d, - 0xd6, 0x7c, 0x7b, 0x98, 0xf1, 0x54, 0x2d, 0x41, 0xbf, 0xa7, 0x23, 0xe9, 0x41, 0xc7, 0xee, 0x55, - 0xd3, 0x5a, 0x09, 0x86, 0x9f, 0x03, 0x24, 0xab, 0xbd, 0x90, 0x49, 0xaf, 0x5e, 0xa8, 0x58, 0xc0, - 0x95, 0x35, 0x23, 0xce, 0x24, 0xbd, 0x92, 0x5e, 0x43, 0x8d, 0xdc, 0x14, 0x49, 0x41, 0xfc, 0x19, - 0xd4, 0x04, 0x8d, 0x23, 0xce, 0x62, 0x1a, 0x7b, 0x4d, 0x6d, 0xc0, 0xd2, 0xdc, 0xe0, 0xd2, 0xcf, - 0x30, 0xe3, 0x75, 0xbf, 0x83, 0xda, 0x1e, 0x11, 0x7e, 0xf2, 0x4d, 0xa6, 0x63, 0xb1, 0x1f, 0x8c, - 0x25, 0x75, 0xc3, 0x79, 0xe0, 0x46, 0xee, 0x62, 0xe9, 0xa1, 0x8b, 0xdd, 0xbf, 0x6c, 0xa8, 0x65, - 0x97, 0x00, 0xaf, 0x42, 0x59, 0x9d, 0x11, 0xb1, 0x67, 0x77, 0x4a, 0x3d, 0x77, 0x68, 0x76, 0x78, - 0x0d, 0xaa, 0x63, 0x4a, 0x04, 0x53, 0x11, 0x47, 0x47, 0xb2, 0x3d, 0x7e, 0x01, 0x4b, 0x09, 0xeb, - 0x94, 0x4f, 0x65, 0xc0, 0x43, 0x16, 0x78, 0x25, 0x4d, 0x69, 0x26, 0xf0, 0xd7, 0x06, 0xc5, 0xcf, - 0x60, 0x31, 0x3d, 0x74, 0xca, 0x94, 0x49, 0xae, 0xa6, 0x35, 0x52, 0xf0, 0x50, 0x79, 0xf4, 0x0c, - 0x80, 0x4c, 0x25, 0x3f, 0x1d, 0x53, 0x72, 0x49, 0xf5, 0x17, 0x96, 0xce, 0xa2, 0xa6, 0xf0, 0x03, - 0x05, 0x77, 0x7f, 0xb7, 0x01, 0x94, 0xe8, 0x9d, 0x0b, 0xc2, 0x02, 0x8a, 0x3f, 0x35, 0x77, 0xc1, - 0xd1, 0x77, 0x61, 0xb5, 0x78, 0xb7, 0x13, 0xc6, 0x83, 0xeb, 0xf0, 0x02, 0x2a, 0x8c, 0xfb, 0xf4, - 0x34, 0xf4, 0x8d, 0x29, 0x4d, 0x15, 0xbc, 0x7b, 0xfb, 0xb4, 0x7c, 0xc8, 0x7d, 0xba, 0xbf, 0x3b, - 0x2c, 0xab, 0xf0, 0xbe, 0x8f, 0xbd, 0x7c, 0xa4, 0xc9, 0x43, 0x93, 0x0d, 0x73, 0x0d, 0x9c, 0xd0, - 0x37, 0x83, 0x00, 0x73, 0xda, 0xd9, 0xdf, 0x1d, 0x3a, 0xa1, 0xdf, 0x9d, 0x00, 0xca, 0x8b, 0x1f, - 0x85, 0x2c, 0x18, 0xe7, 0x22, 0xed, 0xff, 0x23, 0xd2, 0x79, 0x9f, 0xc8, 0xee, 0x1f, 0x36, 0x34, - 0xf2, 0x3c, 0x27, 0x7d, 0xbc, 0x0d, 0x20, 0x05, 0x61, 0x71, 0x28, 0x43, 0xce, 0x4c, 0xc5, 0xf5, - 0x47, 0x2a, 0x66, 0x9c, 0xf4, 0x63, 0xce, 0x4f, 0xe1, 0x2f, 0xa0, 0x32, 0xd2, 0xac, 0x64, 0xe2, - 0x85, 0x77, 0x6a, 0xbe, 0xb5, 0xf4, 0xda, 0x1a, 0x7a, 0xd1, 0xb3, 0xd2, 0x8c, 0x67, 0x1b, 0x7b, - 0x50, 0xcb, 0x1e, 0x73, 0xbc, 0x04, 0x75, 0xbd, 0x39, 0xe4, 0x62, 0x42, 0xc6, 0xc8, 0xc2, 0x2b, - 0xb0, 0xa4, 0x81, 0x3c, 0x3f, 0xb2, 0xf1, 0x13, 0x58, 0x9e, 0x03, 0x4f, 0xfa, 0xc8, 0xd9, 0xf8, - 0xb3, 0x04, 0xf5, 0xc2, 0x5b, 0x87, 0x01, 0xca, 0x83, 0x38, 0xd8, 0x9b, 0x46, 0xc8, 0xc2, 0x75, - 0xa8, 0x0c, 0xe2, 0x60, 0x9b, 0x12, 0x89, 0x6c, 0xb3, 0x79, 0x2d, 0x78, 0x84, 0x1c, 0xc3, 0xda, - 0x8a, 0x22, 0x54, 0xc2, 0x4d, 0x80, 0x64, 0x3d, 0xa4, 0x71, 0x84, 0x5c, 0x43, 0x3c, 0xe1, 0x92, - 0xa2, 0x05, 0xa5, 0xcd, 0x6c, 0x74, 0xb4, 0x6c, 0xa2, 0xea, 0xf5, 0x40, 0x15, 0x8c, 0xa0, 0xa1, - 0x8a, 0x51, 0x22, 0xe4, 0x99, 0xaa, 0x52, 0xc5, 0x2d, 0x40, 0x45, 0x44, 0x1f, 0xaa, 0x61, 0x0c, - 0xcd, 0x41, 0x1c, 0xbc, 0x61, 0x82, 0x92, 0xd1, 0x05, 0x39, 0x1b, 0x53, 0x04, 0x78, 0x19, 0x16, - 0x4d, 0x22, 0x75, 0xe3, 0xa6, 0x31, 0xaa, 0x1b, 0xda, 0xce, 0x05, 0x1d, 0xfd, 0xf0, 0xcd, 0x94, - 0x8b, 0xe9, 0x04, 0x35, 0x54, 0xdb, 0x83, 0x38, 0xd0, 0x03, 0x3a, 0xa7, 0xe2, 0x80, 0x12, 0x9f, - 0x0a, 0xb4, 0x68, 0x4e, 0x1f, 0x87, 0x13, 0xca, 0xa7, 0xf2, 0x90, 0xff, 0x88, 0x9a, 0x46, 0xcc, - 0x90, 0x12, 0x5f, 0xff, 0x44, 0xd1, 0x92, 0x11, 0x93, 0x21, 0x5a, 0x0c, 0x32, 0xfd, 0xbe, 0x16, - 0x54, 0xb7, 0xb8, 0x6c, 0xaa, 0x9a, 0xbd, 0xe6, 0x60, 0x73, 0xf2, 0x48, 0x72, 0x41, 0x02, 0xba, - 0x15, 0x45, 0x94, 0xf9, 0x68, 0x05, 0x7b, 0xd0, 0x9a, 0x47, 0x35, 0xbf, 0xa5, 0x26, 0x36, 0x13, - 0x19, 0x5f, 0xa3, 0x27, 0xf8, 0x03, 0x58, 0x99, 0x03, 0x35, 0x7b, 0x75, 0xe3, 0x17, 0x1b, 0x5a, - 0x8f, 0x7d, 0x7c, 0x78, 0x1d, 0xbc, 0xc7, 0xf0, 0xad, 0xa9, 0xe4, 0xc8, 0xc2, 0x1f, 0xc1, 0x87, - 0x8f, 0x45, 0xbf, 0xe2, 0x21, 0x93, 0xfb, 0x93, 0x68, 0x1c, 0x8e, 0x42, 0x35, 0xe8, 0xf7, 0xd1, - 0x5e, 0x5d, 0x19, 0x9a, 0xb3, 0x71, 0x0d, 0xcd, 0xd9, 0x2b, 0xa7, 0xac, 0xce, 0x91, 0x2d, 0xdf, - 0x57, 0x97, 0x0b, 0x59, 0xaa, 0xeb, 0x1c, 0x1e, 0xd2, 0x09, 0xbf, 0xa4, 0x3a, 0x62, 0xcf, 0x46, - 0xde, 0x44, 0x3e, 0x91, 0x49, 0xc4, 0x99, 0x6d, 0x64, 0xcb, 0xf7, 0x0f, 0x92, 0x97, 0x4d, 0x47, - 0x4b, 0xdb, 0xcf, 0x6f, 0xde, 0xb5, 0xad, 0xdb, 0x77, 0x6d, 0xeb, 0xe6, 0xae, 0x6d, 0xdf, 0xde, - 0xb5, 0xed, 0x7f, 0xee, 0xda, 0xf6, 0xaf, 0xf7, 0x6d, 0xeb, 0xb7, 0xfb, 0xb6, 0x75, 0x7b, 0xdf, - 0xb6, 0xfe, 0xbe, 0x6f, 0x5b, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xe4, 0x8e, 0x4c, 0xc0, 0x6e, - 0x09, 0x00, 0x00, + 0x14, 0xed, 0x6e, 0x77, 0xfc, 0xb8, 0x76, 0x9c, 0x4a, 0xc5, 0x33, 0xd3, 0x8a, 0x22, 0x8f, 0xf1, + 0x0c, 0x1a, 0x2b, 0x68, 0x02, 0x32, 0x12, 0x42, 0xec, 0xf2, 0x18, 0x94, 0xa0, 0x38, 0x0c, 0x4e, + 0x26, 0x0b, 0x24, 0x14, 0x55, 0xdc, 0x95, 0x4e, 0x83, 0x5d, 0xd5, 0xaa, 0x2e, 0x87, 0x64, 0x83, + 0x10, 0x5f, 0xc0, 0x92, 0x0d, 0x5b, 0x3e, 0x80, 0x8f, 0x40, 0x59, 0x66, 0xc9, 0x6a, 0xc4, 0x24, + 0x7f, 0xc0, 0x17, 0xa0, 0xaa, 0xae, 0x7e, 0xd8, 0x89, 0x66, 0xc1, 0xae, 0xea, 0xdc, 0x53, 0xf7, + 0x9e, 0x7b, 0x6e, 0x57, 0x35, 0x80, 0x20, 0x67, 0x72, 0x23, 0x12, 0x5c, 0x72, 0x5c, 0x56, 0xeb, + 0xe8, 0x74, 0xb5, 0x15, 0xf0, 0x80, 0x6b, 0xe8, 0x63, 0xb5, 0x4a, 0xa2, 0xdd, 0x9f, 0x60, 0xe1, + 0x15, 0x93, 0xe2, 0x0a, 0x7b, 0xe0, 0x1e, 0x51, 0x31, 0xf1, 0x9c, 0x8e, 0xdd, 0x73, 0xb7, 0xdc, + 0xeb, 0xb7, 0x4f, 0xad, 0xa1, 0x46, 0xf0, 0x2a, 0x2c, 0xec, 0x31, 0x9f, 0x5e, 0x7a, 0xa5, 0x42, + 0x28, 0x81, 0xf0, 0x47, 0xe0, 0x1e, 0x5d, 0x45, 0xd4, 0xb3, 0x3b, 0x76, 0xaf, 0xd9, 0x5f, 0xde, + 0x48, 0x6a, 0x6d, 0xe8, 0x94, 0x2a, 0x90, 0x25, 0xba, 0x8a, 0x28, 0xc6, 0xe0, 0xee, 0x10, 0x49, + 0x3c, 0xb7, 0x63, 0xf7, 0x1a, 0x43, 0xbd, 0xee, 0xfe, 0x6c, 0x03, 0x3a, 0x64, 0x24, 0x8a, 0xcf, + 0xb9, 0x1c, 0x50, 0x49, 0x7c, 0x22, 0x09, 0xfe, 0x0c, 0x60, 0xc4, 0xd9, 0xd9, 0x49, 0x2c, 0x89, + 0x4c, 0x72, 0xd7, 0xf3, 0xdc, 0xdb, 0x9c, 0x9d, 0x1d, 0xaa, 0x80, 0xc9, 0x5d, 0x1b, 0xa5, 0x80, + 0x52, 0x1a, 0x6a, 0xa5, 0xc5, 0x26, 0x12, 0x48, 0xf5, 0x27, 0x55, 0x7f, 0xc5, 0x26, 0x34, 0xd2, + 0xfd, 0x16, 0xaa, 0xa9, 0x02, 0x25, 0x51, 0x29, 0xd0, 0x35, 0x1b, 0x43, 0xbd, 0xc6, 0x5f, 0x40, + 0x75, 0x62, 0x94, 0xe9, 0xc4, 0xf5, 0xbe, 0x97, 0x6a, 0x99, 0x57, 0x6e, 0xf2, 0x66, 0xfc, 0xee, + 0xbf, 0x25, 0xa8, 0x0c, 0x68, 0x1c, 0x93, 0x80, 0xe2, 0x97, 0xe0, 0xca, 0xdc, 0xab, 0x95, 0x34, + 0x87, 0x09, 0x17, 0xdd, 0x52, 0x34, 0xdc, 0x02, 0x47, 0xf2, 0x99, 0x4e, 0x1c, 0xc9, 0x55, 0x1b, + 0x67, 0x82, 0xcf, 0xb5, 0xa1, 0x90, 0xac, 0x41, 0x77, 0xbe, 0x41, 0xdc, 0x86, 0xca, 0x98, 0x07, + 0x7a, 0xba, 0x0b, 0x85, 0x60, 0x0a, 0xe6, 0xb6, 0x95, 0xef, 0xdb, 0xf6, 0x12, 0x2a, 0x94, 0x49, + 0x11, 0xd2, 0xd8, 0xab, 0x74, 0x4a, 0xbd, 0x7a, 0x7f, 0x71, 0x66, 0xc6, 0x69, 0x2a, 0xc3, 0xc1, + 0x6b, 0x50, 0x1e, 0xf1, 0xc9, 0x24, 0x94, 0x5e, 0xb5, 0x90, 0xcb, 0x60, 0x4a, 0xe2, 0x05, 0x97, + 0xd4, 0x5b, 0x2c, 0x4a, 0x54, 0x08, 0xee, 0x43, 0x35, 0x36, 0x5e, 0x7a, 0x35, 0xed, 0x31, 0x9a, + 0xf7, 0x58, 0xf3, 0xed, 0x61, 0xc6, 0x53, 0xb5, 0x04, 0xfd, 0x9e, 0x8e, 0xa4, 0x07, 0x1d, 0xbb, + 0x57, 0x4d, 0x6b, 0x25, 0x18, 0x7e, 0x0e, 0x90, 0xac, 0x76, 0x43, 0x26, 0xbd, 0x7a, 0xa1, 0x62, + 0x01, 0x57, 0xd6, 0x8c, 0x38, 0x93, 0xf4, 0x52, 0x7a, 0x0d, 0x35, 0x72, 0x53, 0x24, 0x05, 0xf1, + 0xa7, 0x50, 0x13, 0x34, 0x8e, 0x38, 0x8b, 0x69, 0xec, 0x35, 0xb5, 0x01, 0x4b, 0x73, 0x83, 0x4b, + 0x3f, 0xc3, 0x8c, 0xd7, 0xfd, 0x0e, 0x6a, 0xbb, 0x44, 0xf8, 0xc9, 0x37, 0x99, 0x8e, 0xc5, 0xbe, + 0x37, 0x96, 0xd4, 0x0d, 0xe7, 0x9e, 0x1b, 0xb9, 0x8b, 0xa5, 0xfb, 0x2e, 0x76, 0xff, 0xb4, 0xa1, + 0x96, 0x5d, 0x02, 0xfc, 0x18, 0xca, 0xea, 0x8c, 0x88, 0x3d, 0xbb, 0x53, 0xea, 0xb9, 0x43, 0xb3, + 0xc3, 0xab, 0x50, 0x1d, 0x53, 0x22, 0x98, 0x8a, 0x38, 0x3a, 0x92, 0xed, 0xf1, 0x0b, 0x58, 0x4a, + 0x58, 0x27, 0x7c, 0x2a, 0x03, 0x1e, 0xb2, 0xc0, 0x2b, 0x69, 0x4a, 0x33, 0x81, 0xbf, 0x36, 0x28, + 0x7e, 0x06, 0x8b, 0xe9, 0xa1, 0x13, 0xa6, 0x4c, 0x72, 0x35, 0xad, 0x91, 0x82, 0x07, 0xca, 0xa3, + 0x67, 0x00, 0x64, 0x2a, 0xf9, 0xc9, 0x98, 0x92, 0x0b, 0xaa, 0xbf, 0xb0, 0x74, 0x16, 0x35, 0x85, + 0xef, 0x2b, 0xb8, 0xfb, 0xbb, 0x0d, 0xa0, 0x44, 0x6f, 0x9f, 0x13, 0x16, 0x50, 0xfc, 0x89, 0xb9, + 0x0b, 0x8e, 0xbe, 0x0b, 0x8f, 0x8b, 0x77, 0x3b, 0x61, 0xdc, 0xbb, 0x0e, 0x2f, 0xa0, 0xc2, 0xb8, + 0x4f, 0x4f, 0x42, 0xdf, 0x98, 0xd2, 0x54, 0xc1, 0xdb, 0xb7, 0x4f, 0xcb, 0x07, 0xdc, 0xa7, 0x7b, + 0x3b, 0xc3, 0xb2, 0x0a, 0xef, 0xf9, 0xd8, 0xcb, 0x47, 0x9a, 0x3c, 0x34, 0xd9, 0x30, 0x57, 0xc1, + 0x09, 0x7d, 0x33, 0x08, 0x30, 0xa7, 0x9d, 0xbd, 0x9d, 0xa1, 0x13, 0xfa, 0xdd, 0x09, 0xa0, 0xbc, + 0xf8, 0x61, 0xc8, 0x82, 0x71, 0x2e, 0xd2, 0xfe, 0x3f, 0x22, 0x9d, 0xf7, 0x89, 0xec, 0xfe, 0x61, + 0x43, 0x23, 0xcf, 0x73, 0xdc, 0xc7, 0x5b, 0x00, 0x52, 0x10, 0x16, 0x87, 0x32, 0xe4, 0xcc, 0x54, + 0x5c, 0x7b, 0xa0, 0x62, 0xc6, 0x49, 0x3f, 0xe6, 0xfc, 0x14, 0xfe, 0x1c, 0x2a, 0x23, 0xcd, 0x4a, + 0x26, 0x5e, 0x78, 0xa7, 0xe6, 0x5b, 0x4b, 0xaf, 0xad, 0xa1, 0x17, 0x3d, 0x2b, 0xcd, 0x78, 0xb6, + 0xbe, 0x0b, 0xb5, 0xec, 0x31, 0xc7, 0x4b, 0x50, 0xd7, 0x9b, 0x03, 0x2e, 0x26, 0x64, 0x8c, 0x2c, + 0xbc, 0x02, 0x4b, 0x1a, 0xc8, 0xf3, 0x23, 0x1b, 0x3f, 0x82, 0xe5, 0x39, 0xf0, 0xb8, 0x8f, 0x9c, + 0xf5, 0xbf, 0x4a, 0x50, 0x2f, 0xbc, 0x75, 0x18, 0xa0, 0x3c, 0x88, 0x83, 0xdd, 0x69, 0x84, 0x2c, + 0x5c, 0x87, 0xca, 0x20, 0x0e, 0xb6, 0x28, 0x91, 0xc8, 0x36, 0x9b, 0xd7, 0x82, 0x47, 0xc8, 0x31, + 0xac, 0xcd, 0x28, 0x42, 0x25, 0xdc, 0x04, 0x48, 0xd6, 0x43, 0x1a, 0x47, 0xc8, 0x35, 0xc4, 0x63, + 0x2e, 0x29, 0x5a, 0x50, 0xda, 0xcc, 0x46, 0x47, 0xcb, 0x26, 0xaa, 0x5e, 0x0f, 0x54, 0xc1, 0x08, + 0x1a, 0xaa, 0x18, 0x25, 0x42, 0x9e, 0xaa, 0x2a, 0x55, 0xdc, 0x02, 0x54, 0x44, 0xf4, 0xa1, 0x1a, + 0xc6, 0xd0, 0x1c, 0xc4, 0xc1, 0x1b, 0x26, 0x28, 0x19, 0x9d, 0x93, 0xd3, 0x31, 0x45, 0x80, 0x97, + 0x61, 0xd1, 0x24, 0x52, 0x37, 0x6e, 0x1a, 0xa3, 0xba, 0xa1, 0x6d, 0x9f, 0xd3, 0xd1, 0x0f, 0xdf, + 0x4c, 0xb9, 0x98, 0x4e, 0x50, 0x43, 0xb5, 0x3d, 0x88, 0x03, 0x3d, 0xa0, 0x33, 0x2a, 0xf6, 0x29, + 0xf1, 0xa9, 0x40, 0x8b, 0xe6, 0xf4, 0x51, 0x38, 0xa1, 0x7c, 0x2a, 0x0f, 0xf8, 0x8f, 0xa8, 0x69, + 0xc4, 0x0c, 0x29, 0xf1, 0xf5, 0x4f, 0x14, 0x2d, 0x19, 0x31, 0x19, 0xa2, 0xc5, 0x20, 0xd3, 0xef, + 0x6b, 0x41, 0x75, 0x8b, 0xcb, 0xa6, 0xaa, 0xd9, 0x6b, 0x0e, 0x36, 0x27, 0x0f, 0x25, 0x17, 0x24, + 0xa0, 0x9b, 0x51, 0x44, 0x99, 0x8f, 0x56, 0xb0, 0x07, 0xad, 0x79, 0x54, 0xf3, 0x5b, 0x6a, 0x62, + 0x33, 0x91, 0xf1, 0x15, 0x7a, 0x84, 0x9f, 0xc0, 0xca, 0x1c, 0xa8, 0xd9, 0x8f, 0x0d, 0xfb, 0x4b, + 0x2e, 0x02, 0x2a, 0x4d, 0x47, 0x4f, 0xd6, 0x7f, 0xb1, 0xa1, 0xf5, 0xd0, 0x17, 0x89, 0xd7, 0xc0, + 0x7b, 0x08, 0xdf, 0x9c, 0x4a, 0x8e, 0x2c, 0xfc, 0x21, 0x7c, 0xf0, 0x50, 0xf4, 0x2b, 0x1e, 0x32, + 0xb9, 0x37, 0x89, 0xc6, 0xe1, 0x28, 0x54, 0xd3, 0x7f, 0x1f, 0xed, 0xd5, 0xa5, 0xa1, 0x39, 0xeb, + 0x57, 0xd0, 0x9c, 0xbd, 0x87, 0xca, 0xff, 0x1c, 0xd9, 0xf4, 0x7d, 0x75, 0xe3, 0x90, 0xa5, 0xac, + 0xc8, 0xe1, 0x21, 0x9d, 0xf0, 0x0b, 0xaa, 0x23, 0xf6, 0x6c, 0xe4, 0x4d, 0xe4, 0x13, 0x99, 0x44, + 0x9c, 0xd9, 0x46, 0x36, 0x7d, 0x7f, 0x3f, 0x79, 0xee, 0x74, 0xb4, 0xb4, 0xf5, 0xfc, 0xfa, 0x5d, + 0xdb, 0xba, 0x79, 0xd7, 0xb6, 0xae, 0x6f, 0xdb, 0xf6, 0xcd, 0x6d, 0xdb, 0xfe, 0xe7, 0xb6, 0x6d, + 0xff, 0x7a, 0xd7, 0xb6, 0x7e, 0xbb, 0x6b, 0x5b, 0x37, 0x77, 0x6d, 0xeb, 0xef, 0xbb, 0xb6, 0xf5, + 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x67, 0x2b, 0x47, 0x0c, 0x83, 0x09, 0x00, 0x00, } func (m *Entry) Marshal() (dAtA []byte, err error) { diff --git a/raftpb/raft.proto b/raftpb/raft.proto index a70f736a..a8598ee5 100644 --- a/raftpb/raft.proto +++ b/raftpb/raft.proto @@ -62,6 +62,7 @@ enum MessageType { MsgStorageAppendResp = 20; MsgStorageApply = 21; MsgStorageApplyResp = 22; + MsgForgetLeader = 23; // NOTE: when adding new message types, remember to update the isLocalMsg and // isResponseMsg arrays in raft/util.go and update the corresponding tests in // raft/util_test.go. diff --git a/rafttest/interaction_env_handler.go b/rafttest/interaction_env_handler.go index d78adbb7..c4c84fe6 100644 --- a/rafttest/interaction_env_handler.go +++ b/rafttest/interaction_env_handler.go @@ -142,6 +142,13 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { // // transfer-leadership from=1 to=4 err = env.handleTransferLeadership(t, d) + case "forget-leader": + // Forgets the current leader of the given node. + // + // Example: + // + // forget-leader 1 + err = env.handleForgetLeader(t, d) case "propose": // Propose an entry. // @@ -182,20 +189,17 @@ func (env *InteractionEnv) Handle(t *testing.T, d datadriven.TestData) string { default: err = fmt.Errorf("unknown command") } - if err != nil { - env.Output.WriteString(err.Error()) - } // NB: the highest log level suppresses all output, including that of the // handlers. This comes in useful during setup which can be chatty. // However, errors are always logged. - if env.Output.Len() == 0 { - return "ok" - } - if env.Output.Lvl == len(lvlNames)-1 { - if err != nil { + if err != nil { + if env.Output.Quiet() { return err.Error() } - return "ok (quiet)" + env.Output.WriteString(err.Error()) + } + if env.Output.Len() == 0 { + return "ok" } return env.Output.String() } diff --git a/rafttest/interaction_env_handler_add_nodes.go b/rafttest/interaction_env_handler_add_nodes.go index 3f2dd629..1c3ed773 100644 --- a/rafttest/interaction_env_handler_add_nodes.go +++ b/rafttest/interaction_env_handler_add_nodes.go @@ -54,6 +54,15 @@ func (env *InteractionEnv) handleAddNodes(t *testing.T, d datadriven.TestData) e arg.Scan(t, i, &cfg.PreVote) case "checkquorum": arg.Scan(t, i, &cfg.CheckQuorum) + case "read-only": + switch arg.Vals[i] { + case "safe": + cfg.ReadOnlyOption = raft.ReadOnlySafe + case "lease-based": + cfg.ReadOnlyOption = raft.ReadOnlyLeaseBased + default: + return fmt.Errorf("invalid read-only option %q", arg.Vals[i]) + } } } } diff --git a/rafttest/interaction_env_handler_forget_leader.go b/rafttest/interaction_env_handler_forget_leader.go new file mode 100644 index 00000000..8884e0d6 --- /dev/null +++ b/rafttest/interaction_env_handler_forget_leader.go @@ -0,0 +1,32 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rafttest + +import ( + "testing" + + "github.com/cockroachdb/datadriven" +) + +func (env *InteractionEnv) handleForgetLeader(t *testing.T, d datadriven.TestData) error { + idx := firstAsNodeIdx(t, d) + env.ForgetLeader(t, idx) + return nil +} + +// ForgetLeader makes the follower at the given index forget its leader. +func (env *InteractionEnv) ForgetLeader(t *testing.T, idx int) { + env.Nodes[idx].ForgetLeader() +} diff --git a/rafttest/interaction_env_handler_raftstate.go b/rafttest/interaction_env_handler_raftstate.go index 3117b2f7..e40a192b 100644 --- a/rafttest/interaction_env_handler_raftstate.go +++ b/rafttest/interaction_env_handler_raftstate.go @@ -42,7 +42,8 @@ func (env *InteractionEnv) handleRaftState() error { } else { voterStatus = "(Non-Voter)" } - fmt.Fprintf(env.Output, "%d: %s %s\n", st.ID, st.RaftState, voterStatus) + fmt.Fprintf(env.Output, "%d: %s %s Term:%d Lead:%d\n", + st.ID, st.RaftState, voterStatus, st.Term, st.Lead) } return nil } diff --git a/rafttest/interaction_env_handler_stabilize.go b/rafttest/interaction_env_handler_stabilize.go index d79a9aa4..084022ab 100644 --- a/rafttest/interaction_env_handler_stabilize.go +++ b/rafttest/interaction_env_handler_stabilize.go @@ -25,7 +25,22 @@ import ( ) func (env *InteractionEnv) handleStabilize(t *testing.T, d datadriven.TestData) error { - idxs := nodeIdxs(t, d) + idxs := nodeIdxs(t, d) // skips key=value args + for _, arg := range d.CmdArgs { + for i := range arg.Vals { + switch arg.Key { + case "log-level": + defer func(old int) { + env.Output.Lvl = old + }(env.Output.Lvl) + var level string + arg.Scan(t, i, &level) + if err := env.LogLevel(level); err != nil { + return err + } + } + } + } return env.Stabilize(idxs...) } diff --git a/rafttest/interaction_env_logger.go b/rafttest/interaction_env_logger.go index 78298f54..0fcf94d6 100644 --- a/rafttest/interaction_env_logger.go +++ b/rafttest/interaction_env_logger.go @@ -102,3 +102,37 @@ func (l *RedirectLogger) Panicf(format string, v ...interface{}) { // and testing the cases when panic is expected. panic(fmt.Sprintf(format, v...)) } + +// Override StringBuilder write methods to silence them under NONE. + +func (l *RedirectLogger) Quiet() bool { + return l.Lvl == len(lvlNames)-1 +} + +func (l *RedirectLogger) Write(p []byte) (int, error) { + if l.Quiet() { + return 0, nil + } + return l.Builder.Write(p) +} + +func (l *RedirectLogger) WriteByte(c byte) error { + if l.Quiet() { + return nil + } + return l.Builder.WriteByte(c) +} + +func (l *RedirectLogger) WriteRune(r rune) (int, error) { + if l.Quiet() { + return 0, nil + } + return l.Builder.WriteRune(r) +} + +func (l *RedirectLogger) WriteString(s string) (int, error) { + if l.Quiet() { + return 0, nil + } + return l.Builder.WriteString(s) +} diff --git a/rawnode.go b/rawnode.go index bad7ae98..a474f97c 100644 --- a/rawnode.go +++ b/rawnode.go @@ -544,6 +544,12 @@ func (rn *RawNode) TransferLeader(transferee uint64) { _ = rn.raft.Step(pb.Message{Type: pb.MsgTransferLeader, From: transferee}) } +// ForgetLeader forgets a follower's current leader, changing it to None. +// See (Node).ForgetLeader for details. +func (rn *RawNode) ForgetLeader() error { + return rn.raft.Step(pb.Message{Type: pb.MsgForgetLeader}) +} + // ReadIndex requests a read state. The read state will be set in ready. // Read State has a read index. Once the application advances further than the read // index, any linearizable read requests issued before the read request can be diff --git a/rawnode_test.go b/rawnode_test.go index 553d74fd..e570be3b 100644 --- a/rawnode_test.go +++ b/rawnode_test.go @@ -41,6 +41,9 @@ func (a *rawNodeAdapter) TransferLeadership(ctx context.Context, lead, transfere a.RawNode.TransferLeader(transferee) } +// ForgetLeader takes a context, RawNode doesn't need it. +func (a *rawNodeAdapter) ForgetLeader(context.Context) error { return a.RawNode.ForgetLeader() } + // Stop when node has a goroutine, RawNode doesn't need this. func (a *rawNodeAdapter) Stop() {} diff --git a/testdata/async_storage_writes_append_aba_race.txt b/testdata/async_storage_writes_append_aba_race.txt index b1e82f12..4b68330f 100644 --- a/testdata/async_storage_writes_append_aba_race.txt +++ b/testdata/async_storage_writes_append_aba_race.txt @@ -18,7 +18,7 @@ ok stabilize ---- -ok (quiet) +ok log-level info ---- diff --git a/testdata/campaign_learner_must_vote.txt b/testdata/campaign_learner_must_vote.txt index 14530b55..85617df3 100644 --- a/testdata/campaign_learner_must_vote.txt +++ b/testdata/campaign_learner_must_vote.txt @@ -25,7 +25,7 @@ ok stabilize ---- -ok (quiet) +ok # Propose a conf change on n1 that promotes n3 to voter. propose-conf-change 1 @@ -36,13 +36,13 @@ ok # Commit and fully apply said conf change. n1 and n2 now consider n3 a voter. stabilize 1 2 ---- -ok (quiet) +ok # Drop all inflight messages to 3. We don't want it to be caught up when it is # asked to vote. deliver-msgs drop=(3) ---- -ok (quiet) +ok # We now pretend that n1 is dead, and n2 is trying to become leader. diff --git a/testdata/checkquorum.txt b/testdata/checkquorum.txt index ed69a529..b25c1e63 100644 --- a/testdata/checkquorum.txt +++ b/testdata/checkquorum.txt @@ -18,7 +18,7 @@ ok stabilize ---- -ok (quiet) +ok log-level debug ---- diff --git a/testdata/confchange_v1_remove_leader.txt b/testdata/confchange_v1_remove_leader.txt index 68aab846..1de8d98f 100644 --- a/testdata/confchange_v1_remove_leader.txt +++ b/testdata/confchange_v1_remove_leader.txt @@ -15,7 +15,7 @@ ok stabilize ---- -ok (quiet) +ok log-level debug ---- @@ -23,9 +23,9 @@ ok raft-state ---- -1: StateLeader (Voter) -2: StateFollower (Voter) -3: StateFollower (Voter) +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 # Start removing n1. propose-conf-change 1 v1=true @@ -35,9 +35,9 @@ ok raft-state ---- -1: StateLeader (Voter) -2: StateFollower (Voter) -3: StateFollower (Voter) +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 # Propose an extra entry which will be sent out together with the conf change. propose 1 foo @@ -109,9 +109,9 @@ stabilize 1 raft-state ---- -1: StateLeader (Non-Voter) -2: StateFollower (Voter) -3: StateFollower (Voter) +1: StateLeader (Non-Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 # n2 responds, n3 doesn't yet. Quorum for 'bar' should not be reached... stabilize 2 @@ -244,6 +244,6 @@ stabilize # Just confirming the issue above - leader does not automatically step down. raft-state ---- -1: StateLeader (Non-Voter) -2: StateFollower (Voter) -3: StateFollower (Voter) +1: StateLeader (Non-Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 diff --git a/testdata/confchange_v2_replace_leader.txt b/testdata/confchange_v2_replace_leader.txt index 8074188d..27a3e1ab 100644 --- a/testdata/confchange_v2_replace_leader.txt +++ b/testdata/confchange_v2_replace_leader.txt @@ -21,7 +21,7 @@ ok stabilize ---- -ok (quiet) +ok log-level info ---- @@ -29,9 +29,9 @@ ok raft-state ---- -1: StateLeader (Voter) -2: StateFollower (Voter) -3: StateFollower (Voter) +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 log-level info ---- @@ -168,10 +168,10 @@ INFO 1 sends MsgTimeoutNow to 4 immediately as 4 already has up-to-date log # Leadership transfer wasn't processed yet. raft-state ---- -1: StateLeader (Voter) -2: StateFollower (Voter) -3: StateFollower (Voter) -4: StateFollower (Voter) +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Voter) Term:1 Lead:1 # Leadership transfer is happening here. stabilize @@ -328,10 +328,10 @@ stabilize # Leadership transfer succeeded. raft-state ---- -1: StateFollower (Voter) -2: StateFollower (Voter) -3: StateFollower (Voter) -4: StateLeader (Voter) +1: StateFollower (Voter) Term:2 Lead:4 +2: StateFollower (Voter) Term:2 Lead:4 +3: StateFollower (Voter) Term:2 Lead:4 +4: StateLeader (Voter) Term:2 Lead:4 # n4 will propose a transition out of the joint config. propose-conf-change 4 @@ -426,10 +426,10 @@ stabilize # n1 is out of the configuration. raft-state ---- -1: StateFollower (Non-Voter) -2: StateFollower (Voter) -3: StateFollower (Voter) -4: StateLeader (Voter) +1: StateFollower (Non-Voter) Term:2 Lead:4 +2: StateFollower (Voter) Term:2 Lead:4 +3: StateFollower (Voter) Term:2 Lead:4 +4: StateLeader (Voter) Term:2 Lead:4 # Make sure n1 cannot campaign to become leader. campaign 1 diff --git a/testdata/forget_leader.txt b/testdata/forget_leader.txt new file mode 100644 index 00000000..a674b346 --- /dev/null +++ b/testdata/forget_leader.txt @@ -0,0 +1,192 @@ +log-level none +---- +ok + +add-nodes 4 voters=(1,2,3) learners=(4) index=10 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# ForgetLeader is a noop on the leader. +forget-leader 1 +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +# ForgetLeader causes a follower to forget its leader, but remain in the current +# term. It's a noop if it's called again. +forget-leader 2 +---- +INFO 2 forgetting leader 1 at term 1 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:0 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +forget-leader 2 +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:0 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +# ForgetLeader also works on learners. +forget-leader 4 +---- +INFO 4 forgetting leader 1 at term 1 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:0 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:0 + +# When receiving a heartbeat from the leader, they revert to followers. +tick-heartbeat 1 +---- +ok + +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 4 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 4 receiving messages + 1->4 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 2->1 MsgHeartbeatResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 4 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 4->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + 4->1 MsgHeartbeatResp Term:1 Log:0/0 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +# ForgetLeader is a noop on candidates. +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became candidate at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 1 at term 2 +INFO 3 [logterm: 1, index: 11] sent MsgVote request to 2 at term 2 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateCandidate (Voter) Term:2 Lead:0 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +forget-leader 3 +---- +ok + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateCandidate (Voter) Term:2 Lead:0 +4: StateFollower (Non-Voter) Term:1 Lead:1 + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Voter) Term:2 Lead:3 +2: StateFollower (Voter) Term:2 Lead:3 +3: StateLeader (Voter) Term:2 Lead:3 +4: StateFollower (Non-Voter) Term:2 Lead:3 + +# ForgetLeader shouldn't affect the election timeout: if a follower +# forgets the leader 1 tick before the election timeout fires, it +# will still campaign on the next tick. +set-randomized-election-timeout 2 timeout=3 +---- +ok + +tick-heartbeat 2 +---- +ok + +tick-heartbeat 2 +---- +ok + +forget-leader 2 +---- +INFO 2 forgetting leader 3 at term 2 + +tick-heartbeat 2 +---- +INFO 2 is starting a new election at term 2 +INFO 2 became candidate at term 3 +INFO 2 [logterm: 2, index: 12] sent MsgVote request to 1 at term 3 +INFO 2 [logterm: 2, index: 12] sent MsgVote request to 3 at term 3 + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Voter) Term:3 Lead:2 +2: StateLeader (Voter) Term:3 Lead:2 +3: StateFollower (Voter) Term:3 Lead:2 +4: StateFollower (Non-Voter) Term:3 Lead:2 diff --git a/testdata/forget_leader_prevote_checkquorum.txt b/testdata/forget_leader_prevote_checkquorum.txt new file mode 100644 index 00000000..9b3b80ff --- /dev/null +++ b/testdata/forget_leader_prevote_checkquorum.txt @@ -0,0 +1,235 @@ +# Tests that a follower with PreVote+CheckQuorum can forget the leader, allowing +# it to grant prevotes despite having heard from the leader recently. +# +# Also tests that forgetting the leader still won't grant prevotes to a +# replica that isn't up-to-date. +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 prevote=true checkquorum=true +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# If 3 attempts to campaign, 2 rejects it because it has a leader. +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became pre-candidate at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 1 at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 + +stabilize 3 +---- +> 3 handling Ready + Ready MustSync=false: + Lead:0 State:StatePreCandidate + Messages: + 3->1 MsgPreVote Term:2 Log:1/11 + 3->2 MsgPreVote Term:2 Log:1/11 + INFO 3 received MsgPreVoteResp from 3 at term 1 + INFO 3 has received 1 MsgPreVoteResp votes and 0 vote rejections + +deliver-msgs 1 2 +---- +3->1 MsgPreVote Term:2 Log:1/11 +INFO 1 [logterm: 1, index: 11, vote: 1] ignored MsgPreVote from 3 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) +3->2 MsgPreVote Term:2 Log:1/11 +INFO 2 [logterm: 1, index: 11, vote: 1] ignored MsgPreVote from 3 [logterm: 1, index: 11] at term 1: lease is not expired (remaining ticks: 3) + +# Make 1 assert leadership over 3 again. +tick-heartbeat 1 +---- +ok + +stabilize +---- +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 2 receiving messages + 1->2 MsgHeartbeat Term:1 Log:0/0 Commit:11 +> 3 receiving messages + 1->3 MsgHeartbeat Term:1 Log:0/0 Commit:11 + INFO 3 became follower at term 1 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgHeartbeatResp Term:1 Log:0/0 +> 3 handling Ready + Ready MustSync=false: + Lead:1 State:StateFollower + Messages: + 3->1 MsgHeartbeatResp Term:1 Log:0/0 +> 1 receiving messages + 2->1 MsgHeartbeatResp Term:1 Log:0/0 + 3->1 MsgHeartbeatResp Term:1 Log:0/0 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 + +# If 2 forgets the leader, then 3 can obtain prevotes and hold an election +# despite 2 having heard from the leader recently. +forget-leader 2 +---- +INFO 2 forgetting leader 1 at term 1 + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:0 +3: StateFollower (Voter) Term:1 Lead:1 + +campaign 3 +---- +INFO 3 is starting a new election at term 1 +INFO 3 became pre-candidate at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 1 at term 1 +INFO 3 [logterm: 1, index: 11] sent MsgPreVote request to 2 at term 1 + +stabilize 3 +---- +> 3 handling Ready + Ready MustSync=false: + Lead:0 State:StatePreCandidate + Messages: + 3->1 MsgPreVote Term:2 Log:1/11 + 3->2 MsgPreVote Term:2 Log:1/11 + INFO 3 received MsgPreVoteResp from 3 at term 1 + INFO 3 has received 1 MsgPreVoteResp votes and 0 vote rejections + +stabilize 2 +---- +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 2 receiving messages + 3->2 MsgPreVote Term:2 Log:1/11 + INFO 2 [logterm: 1, index: 11, vote: 1] cast MsgPreVote for 3 [logterm: 1, index: 11] at term 1 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->3 MsgPreVoteResp Term:2 Log:0/0 + +stabilize 3 +---- +> 3 receiving messages + 2->3 MsgPreVoteResp Term:2 Log:0/0 + INFO 3 received MsgPreVoteResp from 2 at term 1 + INFO 3 has received 2 MsgPreVoteResp votes and 0 vote rejections + INFO 3 became candidate at term 2 + INFO 3 [logterm: 1, index: 11] sent MsgVote request to 1 at term 2 + INFO 3 [logterm: 1, index: 11] sent MsgVote request to 2 at term 2 +> 3 handling Ready + Ready MustSync=true: + Lead:0 State:StateCandidate + HardState Term:2 Vote:3 Commit:11 + Messages: + 3->1 MsgVote Term:2 Log:1/11 + 3->2 MsgVote Term:2 Log:1/11 + INFO 3 received MsgVoteResp from 3 at term 2 + INFO 3 has received 1 MsgVoteResp votes and 0 vote rejections + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Voter) Term:2 Lead:3 +2: StateFollower (Voter) Term:2 Lead:3 +3: StateLeader (Voter) Term:2 Lead:3 + +# Test that forgetting the leader still won't grant prevotes if the candidate +# isn't up-to-date. We first replicate a proposal on 3 and 2. +propose 3 prop_1 +---- +ok + +stabilize 3 +---- +> 3 handling Ready + Ready MustSync=true: + Entries: + 2/13 EntryNormal "prop_1" + Messages: + 3->1 MsgApp Term:2 Log:2/12 Commit:12 Entries:[2/13 EntryNormal "prop_1"] + 3->2 MsgApp Term:2 Log:2/12 Commit:12 Entries:[2/13 EntryNormal "prop_1"] + +stabilize 2 +---- +> 2 receiving messages + 3->2 MsgApp Term:2 Log:2/12 Commit:12 Entries:[2/13 EntryNormal "prop_1"] +> 2 handling Ready + Ready MustSync=true: + Entries: + 2/13 EntryNormal "prop_1" + Messages: + 2->3 MsgAppResp Term:2 Log:0/13 + +forget-leader 2 +---- +INFO 2 forgetting leader 3 at term 2 + +# 1 is now behind on its log. It tries to campaign, but fails. +raft-log 1 +---- +1/11 EntryNormal "" +2/12 EntryNormal "" + +campaign 1 +---- +INFO 1 is starting a new election at term 2 +INFO 1 became pre-candidate at term 2 +INFO 1 [logterm: 2, index: 12] sent MsgPreVote request to 2 at term 2 +INFO 1 [logterm: 2, index: 12] sent MsgPreVote request to 3 at term 2 + +process-ready 1 +---- +Ready MustSync=false: +Lead:0 State:StatePreCandidate +Messages: +1->2 MsgPreVote Term:3 Log:2/12 +1->3 MsgPreVote Term:3 Log:2/12 +INFO 1 received MsgPreVoteResp from 1 at term 2 +INFO 1 has received 1 MsgPreVoteResp votes and 0 vote rejections + +stabilize 2 +---- +> 2 handling Ready + Ready MustSync=false: + Lead:0 State:StateFollower +> 2 receiving messages + 1->2 MsgPreVote Term:3 Log:2/12 + INFO 2 [logterm: 2, index: 13, vote: 3] rejected MsgPreVote from 1 [logterm: 2, index: 12] at term 2 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgPreVoteResp Term:2 Log:0/0 Rejected (Hint: 0) + +stabilize log-level=none +---- +ok + +raft-state +---- +1: StateFollower (Voter) Term:2 Lead:3 +2: StateFollower (Voter) Term:2 Lead:3 +3: StateLeader (Voter) Term:2 Lead:3 diff --git a/testdata/forget_leader_read_only_lease_based.txt b/testdata/forget_leader_read_only_lease_based.txt new file mode 100644 index 00000000..4d4bd188 --- /dev/null +++ b/testdata/forget_leader_read_only_lease_based.txt @@ -0,0 +1,30 @@ +log-level none +---- +ok + +add-nodes 3 voters=(1,2,3) index=10 checkquorum=true read-only=lease-based +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok + +log-level debug +---- +ok + +# ForgetLeader fails with lease-based reads, as it's not safe. +forget-leader 2 +---- +ERROR ignoring MsgForgetLeader due to ReadOnlyLeaseBased + +raft-state +---- +1: StateLeader (Voter) Term:1 Lead:1 +2: StateFollower (Voter) Term:1 Lead:1 +3: StateFollower (Voter) Term:1 Lead:1 diff --git a/testdata/heartbeat_resp_recovers_from_probing.txt b/testdata/heartbeat_resp_recovers_from_probing.txt index 690f563a..e606a155 100644 --- a/testdata/heartbeat_resp_recovers_from_probing.txt +++ b/testdata/heartbeat_resp_recovers_from_probing.txt @@ -20,7 +20,7 @@ ok stabilize ---- -ok (quiet) +ok log-level debug ---- diff --git a/testdata/prevote.txt b/testdata/prevote.txt index d65103ed..db763d35 100644 --- a/testdata/prevote.txt +++ b/testdata/prevote.txt @@ -19,7 +19,7 @@ ok stabilize ---- -ok (quiet) +ok log-level debug ---- diff --git a/testdata/prevote_checkquorum.txt b/testdata/prevote_checkquorum.txt index 939d6f36..6db6662b 100644 --- a/testdata/prevote_checkquorum.txt +++ b/testdata/prevote_checkquorum.txt @@ -17,7 +17,7 @@ ok stabilize ---- -ok (quiet) +ok log-level debug ---- diff --git a/testdata/probe_and_replicate.txt b/testdata/probe_and_replicate.txt index 8e61b618..d5829704 100644 --- a/testdata/probe_and_replicate.txt +++ b/testdata/probe_and_replicate.txt @@ -44,7 +44,7 @@ ok stabilize ---- -ok (quiet) +ok propose 1 prop_1_12 ---- @@ -56,7 +56,7 @@ ok stabilize ---- -ok (quiet) +ok ## Create term 2 entries. campaign 2 @@ -65,15 +65,15 @@ ok stabilize 2 ---- -ok (quiet) +ok stabilize 6 ---- -ok (quiet) +ok stabilize 2 5 7 ---- -ok (quiet) +ok propose 2 prop_2_15 ---- @@ -85,11 +85,11 @@ ok stabilize 2 7 ---- -ok (quiet) +ok deliver-msgs drop=(1,2,3,4,5,6,7) ---- -ok (quiet) +ok ## Create term 3 entries. campaign 7 @@ -98,15 +98,15 @@ ok stabilize 7 ---- -ok (quiet) +ok stabilize 1 2 3 4 5 6 ---- -ok (quiet) +ok stabilize 7 ---- -ok (quiet) +ok propose 7 prop_3_18 ---- @@ -126,11 +126,11 @@ ok stabilize 7 ---- -ok (quiet) +ok deliver-msgs drop=(1,2,3,4,5,6,7) ---- -ok (quiet) +ok ## Create term 4 entries. campaign 6 @@ -139,7 +139,7 @@ ok stabilize 1 2 3 4 5 6 ---- -ok (quiet) +ok propose 6 prop_4_15 ---- @@ -147,7 +147,7 @@ ok stabilize 1 2 4 5 6 ---- -ok (quiet) +ok propose 6 prop_4_16 ---- @@ -159,11 +159,11 @@ ok stabilize 6 ---- -ok (quiet) +ok deliver-msgs drop=(1,2,3,4,5,6,7) ---- -ok (quiet) +ok ## Create term 5 entries. campaign 5 @@ -172,7 +172,7 @@ ok stabilize 1 2 4 5 ---- -ok (quiet) +ok propose 5 prop_5_17 ---- @@ -180,11 +180,11 @@ ok stabilize 1 2 4 5 ---- -ok (quiet) +ok deliver-msgs drop=(1,2,3,4,5,6,7) ---- -ok (quiet) +ok ## Create term 6 entries. campaign 4 @@ -193,7 +193,7 @@ ok stabilize 1 2 4 5 ---- -ok (quiet) +ok propose 4 prop_6_19 ---- @@ -201,7 +201,7 @@ ok stabilize 1 2 4 ---- -ok (quiet) +ok propose 4 prop_6_20 ---- @@ -209,7 +209,7 @@ ok stabilize 1 4 ---- -ok (quiet) +ok propose 4 prop_6_21 ---- @@ -217,11 +217,11 @@ ok stabilize 4 ---- -ok (quiet) +ok deliver-msgs drop=(1,2,3,4,5,6,7) ---- -ok (quiet) +ok ## Create term 7 entries. campaign 5 @@ -230,15 +230,15 @@ ok stabilize 5 ---- -ok (quiet) +ok stabilize 1 3 6 7 ---- -ok (quiet) +ok stabilize 5 ---- -ok (quiet) +ok propose 5 prop_7_20 ---- @@ -254,11 +254,11 @@ ok stabilize 5 ---- -ok (quiet) +ok deliver-msgs drop=(1,2,3,4,5,6,7) ---- -ok (quiet) +ok # Show the Raft log from each node. diff --git a/testdata/replicate_pause.txt b/testdata/replicate_pause.txt index 67e64a95..d9cee59f 100644 --- a/testdata/replicate_pause.txt +++ b/testdata/replicate_pause.txt @@ -18,7 +18,7 @@ ok stabilize ---- -ok (quiet) +ok # Propose 3 entries. propose 1 prop_1_12 @@ -36,7 +36,7 @@ ok # Store entries and send proposals. process-ready 1 ---- -ok (quiet) +ok # Re-enable log messages. log-level debug @@ -57,7 +57,7 @@ ok # Commit entries between nodes 1 and 2. stabilize 1 2 ---- -ok (quiet) +ok log-level debug ---- @@ -105,7 +105,7 @@ ok # Commit entries between nodes 1 and 2 again. stabilize 1 2 ---- -ok (quiet) +ok log-level debug ---- @@ -176,7 +176,7 @@ ok stabilize ---- -ok (quiet) +ok log-level debug ---- diff --git a/testdata/slow_follower_after_compaction.txt b/testdata/slow_follower_after_compaction.txt index 0b169b78..0d3d48c8 100644 --- a/testdata/slow_follower_after_compaction.txt +++ b/testdata/slow_follower_after_compaction.txt @@ -16,7 +16,7 @@ ok stabilize ---- -ok (quiet) +ok # Propose 3 entries. propose 1 prop_1_12 @@ -33,7 +33,7 @@ ok stabilize ---- -ok (quiet) +ok # Re-enable log messages. log-level debug @@ -70,7 +70,7 @@ ok # Commit entries on nodes 1 and 2. stabilize 1 2 ---- -ok (quiet) +ok log-level debug ---- @@ -107,7 +107,7 @@ ok stabilize ---- -ok (quiet) +ok log-level debug ---- diff --git a/testdata/snapshot_succeed_via_app_resp.txt b/testdata/snapshot_succeed_via_app_resp.txt index a52a683e..80ed3646 100644 --- a/testdata/snapshot_succeed_via_app_resp.txt +++ b/testdata/snapshot_succeed_via_app_resp.txt @@ -23,16 +23,16 @@ ok # Fully replicate everything, including the leader's empty index. stabilize ---- -ok (quiet) +ok compact 1 11 ---- -ok (quiet) +ok # Drop inflight messages to n3. deliver-msgs drop=(3) ---- -ok (quiet) +ok # Show the Raft log messages from now on. log-level debug