From 5c83a222e8fba773f4b639ca48e58278d173f8c6 Mon Sep 17 00:00:00 2001 From: vsmk98 Date: Fri, 13 Dec 2019 11:18:05 +0800 Subject: [PATCH 1/6] updating documentation for DNS feature --- docs/Features/dns.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/Features/dns.md b/docs/Features/dns.md index c4a008241c..4a19af0ced 100644 --- a/docs/Features/dns.md +++ b/docs/Features/dns.md @@ -17,7 +17,10 @@ DNS is not supported for the discovery protocol. Use a bootnode instead, which c resolved. ## Compatibility -For Raft, the whole network must be on version 2.3.1 of Quorum for DNS to function properly; because of this, DNS must -be explicitly enabled using the `--raftdnsenable` flag. -The network will support older nodes mixed with newer nodes if DNS is not enabled via this flag, and it is safe to -enable DNS only on some nodes if all nodes are on at least version 2.3.1. This allows for a clear upgrade path. \ No newline at end of file +For Raft, the whole network must be on version 2.4.0 of Quorum for DNS to function properly. DNS must +be explicitly enabled using the `--raftdnsenable` flag for each node once the node has migrated to version 2.4.0 of Quorum +The network can still run in fine when some nodes are in 2.4.0 version and some in older version as long as this feature is not enabled. For safe migration the recommended approach is as below: +* migrate the nodes to `geth` 2.4.0 version without using `--raftdnsenable` flag +* once the network is fully migrated, restart the nodes with `--raftdnsenable` to enable the feature + +Please note that in a partially migrated network (where some nodes are on version 2.4.0 and others on lower version) **with DNS feature enabled** for migrated nodes, `raft.addPeer` should not be invoked with Hostname till entire network migrates to 2.4.0 version. `raft.addPeer` can still be invoked with IP address and network will work fine. \ No newline at end of file From 7858c7bb8beffff8cebdfb722d37acab837f1247 Mon Sep 17 00:00:00 2001 From: vsmk98 Date: Fri, 13 Dec 2019 15:50:38 +0800 Subject: [PATCH 2/6] update dns documentation with known issues --- docs/Features/dns.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/Features/dns.md b/docs/Features/dns.md index 4a19af0ced..4f0b1d4676 100644 --- a/docs/Features/dns.md +++ b/docs/Features/dns.md @@ -23,4 +23,7 @@ The network can still run in fine when some nodes are in 2.4.0 version and some * migrate the nodes to `geth` 2.4.0 version without using `--raftdnsenable` flag * once the network is fully migrated, restart the nodes with `--raftdnsenable` to enable the feature -Please note that in a partially migrated network (where some nodes are on version 2.4.0 and others on lower version) **with DNS feature enabled** for migrated nodes, `raft.addPeer` should not be invoked with Hostname till entire network migrates to 2.4.0 version. `raft.addPeer` can still be invoked with IP address and network will work fine. \ No newline at end of file +Please note that in a partially migrated network (where some nodes are on version 2.4.0 and others on lower version) **with DNS feature enabled** for migrated nodes, `raft.addPeer` should not be invoked with Hostname till entire network migrates to 2.4.0 version. If invoked, this call will crash all nodes running in older version and these nodes will have to restarted with `geth` of version 2.4.0 of Quorum. `raft.addPeer` can still be invoked with IP address and network will work fine. + +### Known Issue +In a network where all nodes are running on Quorum version 2.4.0, with few nodes enabled for DNS, we recommend the `--verbosity` to be 3 or below. We have observed that nodes which are not enabled for DNS fail to restart if `raft.addPeer` is invoked with host name if `--verbosity` is set above 3. We are working on this issue. \ No newline at end of file From 39191ffe0315215cb60c862b4cb7460199ab1af9 Mon Sep 17 00:00:00 2001 From: vsmk98 Date: Fri, 13 Dec 2019 17:36:52 +0800 Subject: [PATCH 3/6] Correcting typo. --- docs/Features/dns.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Features/dns.md b/docs/Features/dns.md index 4f0b1d4676..5b3c251647 100644 --- a/docs/Features/dns.md +++ b/docs/Features/dns.md @@ -19,11 +19,11 @@ resolved. ## Compatibility For Raft, the whole network must be on version 2.4.0 of Quorum for DNS to function properly. DNS must be explicitly enabled using the `--raftdnsenable` flag for each node once the node has migrated to version 2.4.0 of Quorum -The network can still run in fine when some nodes are in 2.4.0 version and some in older version as long as this feature is not enabled. For safe migration the recommended approach is as below: +The network runs fine when some nodes are in 2.4.0 version and some in older version as long as this feature is not enabled. For safe migration the recommended approach is as below: * migrate the nodes to `geth` 2.4.0 version without using `--raftdnsenable` flag * once the network is fully migrated, restart the nodes with `--raftdnsenable` to enable the feature Please note that in a partially migrated network (where some nodes are on version 2.4.0 and others on lower version) **with DNS feature enabled** for migrated nodes, `raft.addPeer` should not be invoked with Hostname till entire network migrates to 2.4.0 version. If invoked, this call will crash all nodes running in older version and these nodes will have to restarted with `geth` of version 2.4.0 of Quorum. `raft.addPeer` can still be invoked with IP address and network will work fine. -### Known Issue -In a network where all nodes are running on Quorum version 2.4.0, with few nodes enabled for DNS, we recommend the `--verbosity` to be 3 or below. We have observed that nodes which are not enabled for DNS fail to restart if `raft.addPeer` is invoked with host name if `--verbosity` is set above 3. We are working on this issue. \ No newline at end of file +### Note +In a network where all nodes are running on Quorum version 2.4.0, with few nodes enabled for DNS, we recommend the `--verbosity` to be 3 or below. We have observed that nodes which are not enabled for DNS fail to restart if `raft.addPeer` is invoked with host name if `--verbosity` is set above 3. We are currently fixing the same. \ No newline at end of file From 48ffc863360ea77422456bd117bac8bcd6f23e8a Mon Sep 17 00:00:00 2001 From: vsmk98 Date: Wed, 29 Jan 2020 17:41:37 +0800 Subject: [PATCH 4/6] raft.cluster was throwing an error when there was no leader in the network. Fixed the same. added nodeActive attribute to the api response which indicates if the node is active in raft cluster. added error handling for other apis --- raft/api.go | 74 +++++++++++++++++++++++++++++++++++++++++++++------- raft/peer.go | 3 ++- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/raft/api.go b/raft/api.go index c1c38c532f..72b247f20c 100755 --- a/raft/api.go +++ b/raft/api.go @@ -1,5 +1,10 @@ package raft +import ( + "errors" + "github.com/coreos/etcd/pkg/types" +) + type RaftNodeInfo struct { ClusterSize int `json:"clusterSize"` Role string `json:"role"` @@ -19,22 +24,49 @@ func NewPublicRaftAPI(raftService *RaftService) *PublicRaftAPI { } func (s *PublicRaftAPI) Role() string { + if err := s.checkIfNodeInCluster(); err != nil { + return "" + } + _, err := s.raftService.raftProtocolManager.LeaderAddress() + if err != nil { + return "" + } return s.raftService.raftProtocolManager.NodeInfo().Role } +// helper function to check if self node is part of cluster +func (s *PublicRaftAPI) checkIfNodeInCluster() error { + if s.raftService.raftProtocolManager.IsIDRemoved(uint64(s.raftService.raftProtocolManager.raftId)) { + return errors.New("node not part of raft cluster. operations not allowed") + } + return nil +} + func (s *PublicRaftAPI) AddPeer(enodeId string) (uint16, error) { + if err := s.checkIfNodeInCluster(); err != nil { + return 0, err + } return s.raftService.raftProtocolManager.ProposeNewPeer(enodeId, false) } func (s *PublicRaftAPI) AddLearner(enodeId string) (uint16, error) { + if err := s.checkIfNodeInCluster(); err != nil { + return 0, err + } return s.raftService.raftProtocolManager.ProposeNewPeer(enodeId, true) } func (s *PublicRaftAPI) PromoteToPeer(raftId uint16) (bool, error) { + if err := s.checkIfNodeInCluster(); err != nil { + return false, err + } return s.raftService.raftProtocolManager.PromoteToPeer(raftId) } func (s *PublicRaftAPI) RemovePeer(raftId uint16) error { + if err := s.checkIfNodeInCluster(); err != nil { + return err + } return s.raftService.raftProtocolManager.ProposePeerRemoval(raftId) } @@ -48,33 +80,57 @@ func (s *PublicRaftAPI) Leader() (string, error) { } func (s *PublicRaftAPI) Cluster() ([]ClusterInfo, error) { + // check if the node has already been removed from cluster + // if yes return nil + if err := s.checkIfNodeInCluster(); err != nil { + return []ClusterInfo{}, nil + } + nodeInfo := s.raftService.raftProtocolManager.NodeInfo() if nodeInfo.Role == "" { return []ClusterInfo{}, nil } + + noLeader := false leaderAddr, err := s.raftService.raftProtocolManager.LeaderAddress() if err != nil { - if err == errNoLeaderElected && s.Role() == "" { + noLeader = true + if s.raftService.raftProtocolManager.NodeInfo().Role == "" { return []ClusterInfo{}, nil } - return []ClusterInfo{}, err } + peerAddresses := append(nodeInfo.PeerAddresses, nodeInfo.Address) clustInfo := make([]ClusterInfo, len(peerAddresses)) for i, a := range peerAddresses { role := "" - if a.RaftId == leaderAddr.RaftId { - role = "minter" - } else if s.raftService.raftProtocolManager.isLearner(a.RaftId) { - role = "learner" - } else if s.raftService.raftProtocolManager.isVerifier(a.RaftId) { - role = "verifier" + if !noLeader { + if a.RaftId == leaderAddr.RaftId { + role = "minter" + } else if s.raftService.raftProtocolManager.isLearner(a.RaftId) { + role = "learner" + } else if s.raftService.raftProtocolManager.isVerifier(a.RaftId) { + role = "verifier" + } } - clustInfo[i] = ClusterInfo{*a, role} + clustInfo[i] = ClusterInfo{*a, role, s.checkIfNodeIsActive(a.RaftId)} } return clustInfo, nil } +// helper function to check of the raft node is active +// of the raftnode is active ActiveSince returns non-zero time +func (s *PublicRaftAPI) checkIfNodeIsActive(raftId uint16) bool { + if raftId == s.raftService.raftProtocolManager.raftId { + return true + } + activeSince := s.raftService.raftProtocolManager.transport.ActiveSince(types.ID(raftId)) + if activeSince.IsZero() { + return false + } + return true +} + func (s *PublicRaftAPI) GetRaftId(enodeId string) (uint16, error) { return s.raftService.raftProtocolManager.FetchRaftId(enodeId) } diff --git a/raft/peer.go b/raft/peer.go index 12b5a6414f..e5c5c84764 100644 --- a/raft/peer.go +++ b/raft/peer.go @@ -29,7 +29,8 @@ type Address struct { type ClusterInfo struct { Address - Role string `json:"role"` + Role string `json:"role"` + NodeActive bool `json:"nodeActive"` } func newAddress(raftId uint16, raftPort int, node *enode.Node, withHostname bool) *Address { From 064e048d6fd49164cc7c4f427de49407803abeca Mon Sep 17 00:00:00 2001 From: vsmk98 Date: Fri, 31 Jan 2020 12:26:25 +0800 Subject: [PATCH 5/6] added api documentation for raft apis. --- docs/Consensus/raft/raft-rpc-api.md | 192 ++++++++++++++++++++++++++++ docs/Consensus/{ => raft}/raft.md | 4 +- mkdocs.yml | 4 +- raft/api.go | 2 +- 4 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 docs/Consensus/raft/raft-rpc-api.md rename docs/Consensus/{ => raft}/raft.md (99%) diff --git a/docs/Consensus/raft/raft-rpc-api.md b/docs/Consensus/raft/raft-rpc-api.md new file mode 100644 index 0000000000..1ad188de4c --- /dev/null +++ b/docs/Consensus/raft/raft-rpc-api.md @@ -0,0 +1,192 @@ +# Raft RPC API +# APIs +### raft_cluster +Returns the details of all nodes part of the raft cluster +#### Parameters +None +#### Returns +* `hostName`: DNS name or the host IP address +* `nodeActive`: true if the node is active in raft cluster else false +* `nodeId`: enode id of the node +* `p2pPort`: p2p port +* `raftId`: raft id of the node +* `raftPort`: raft port +* `role`: role of the node in raft quorum. Can be minter/ verifier/ learner. In case there is no leader at network level it will be returned as `""` +#### Examples +```jshelllanguage tab="JSON RPC" +// Request +curl -X POST http://127.0.0.1:22001 --data '{"jsonrpc":"2.0","method":"raft_cluster", "id":10}' --header "Content-Type: application/json" + +// Response +{"jsonrpc":"2.0","id":10,"result":[{"raftId":1,"nodeId":"ac6b1096ca56b9f6d004b779ae3728bf83f8e22453404cc3cef16a3d9b96608bc67c4b30db88e0a5a6c6390213f7acbe1153ff6d23ce57380104288ae19373ef","p2pPort":21000,"raftPort":50401,"hostname":"127.0.0.1","role":"minter","nodeActive":true},{"raftId":3,"nodeId":"579f786d4e2830bbcc02815a27e8a9bacccc9605df4dc6f20bcc1a6eb391e7225fff7cb83e5b4ecd1f3a94d8b733803f2f66b7e871961e7b029e22c155c3a778","p2pPort":21002,"raftPort":50403,"hostname":"127.0.0.1","role":"verifier","nodeActive":true},{"raftId":2,"nodeId":"0ba6b9f606a43a95edc6247cdb1c1e105145817be7bcafd6b2c0ba15d58145f0dc1a194f70ba73cd6f4cdd6864edc7687f311254c7555cc32e4d45aeb1b80416","p2pPort":21001,"raftPort":50402,"hostname":"127.0.0.1","role":"verifier","nodeActive":true}]} +``` + +```javascript tab="geth console" +> raft.cluster +[{ + hostname: "127.0.0.1", + nodeActive: true, + nodeId: "0ba6b9f606a43a95edc6247cdb1c1e105145817be7bcafd6b2c0ba15d58145f0dc1a194f70ba73cd6f4cdd6864edc7687f311254c7555cc32e4d45aeb1b80416", + p2pPort: 21001, + raftId: 2, + raftPort: 50402, + role: "verifier" +}, { + hostname: "127.0.0.1", + nodeActive: true, + nodeId: "579f786d4e2830bbcc02815a27e8a9bacccc9605df4dc6f20bcc1a6eb391e7225fff7cb83e5b4ecd1f3a94d8b733803f2f66b7e871961e7b029e22c155c3a778", + p2pPort: 21002, + raftId: 3, + raftPort: 50403, + role: "verifier" +}, { + hostname: "127.0.0.1", + nodeActive: true, + nodeId: "ac6b1096ca56b9f6d004b779ae3728bf83f8e22453404cc3cef16a3d9b96608bc67c4b30db88e0a5a6c6390213f7acbe1153ff6d23ce57380104288ae19373ef", + p2pPort: 21000, + raftId: 1, + raftPort: 50401, + role: "minter" +}] +``` +### raft_role +Returns the role of the current node in raft cluster +#### Parameters +None +#### Returns +* `result`: role of the node in raft cluster. Can be minter/ verifier/ learner. In case there is no leader at network level it will be returned as `""` +#### Examples +```jshelllanguage tab="JSON RPC" +// Request +curl -X POST http://127.0.0.1:22001 --data '{"jsonrpc":"2.0","method":"raft_role", "id":10}' --header "Content-Type: application/json" + +// Response +{"jsonrpc":"2.0","id":10,"result":"verifier"} +``` + +```javascript tab="geth console" +> raft.role +"minter" +``` +### raft_leader +Returns enode id of the leader node +#### Parameters +None +#### Returns +* `result`: enode id of the leader +#### Examples +```jshelllanguage tab="JSON RPC" +// Request +curl -X POST http://127.0.0.1:22001 --data '{"jsonrpc":"2.0","method":"raft_leader", "id":10}' --header "Content-Type: application/json" + +// Response +{"jsonrpc":"2.0","id":10,"result":"ac6b1096ca56b9f6d004b779ae3728bf83f8e22453404cc3cef16a3d9b96608bc67c4b30db88e0a5a6c6390213f7acbe1153ff6d23ce57380104288ae19373ef"} +``` + + +```javascript tab="geth console" +> raft.leader +"ac6b1096ca56b9f6d004b779ae3728bf83f8e22453404cc3cef16a3d9b96608bc67c4b30db88e0a5a6c6390213f7acbe1153ff6d23ce57380104288ae19373ef" +``` + +If there is no leader at the network level, the call to the api will result in the following error: +```javascript +> raft.leader +Error: no leader is currently elected + at web3.js:3143:20 + at web3.js:6347:15 + at get (web3.js:6247:38) + at +``` + +### raft_addPeer +API for adding a new peer to the network. +#### Parameters +* `enodeId`: enode id of the node to be added to the network +#### Returns +* `result`: raft id for the node being added +#### Examples +```jshelllanguage tab="JSON RPC" +// Request +curl -X POST http://127.0.0.1:22001 --data '{"jsonrpc":"2.0","method":"raft_addPeer","params": ["enode://3701f007bfa4cb26512d7df18e6bbd202e8484a6e11d387af6e482b525fa25542d46ff9c99db87bd419b980c24a086117a397f6d8f88e74351b41693880ea0cb@127.0.0.1:21004?discport=0&raftport=50405"], "id":10}' --header "Content-Type: application/json" + +// Response +{"jsonrpc":"2.0","id":10,"result":5} +``` + +```javascript tab="geth console" +> raft.addPeer("enode://3701f007bfa4cb26512d7df18e6bbd202e8484a6e11d387af6e482b525fa25542d46ff9c99db87bd419b980c24a086117a397f6d8f88e74351b41693880ea0cb@127.0.0.1:21004?discport=0&raftport=50405") +5 +``` + +The new node can join the network with `geth` option of `--raftjoinexisting <>` + +If the node being added is already part of the network the of the network, the following error is thrown: +```javascript +> raft.addPeer("enode://3701f007bfa4cb26512d7df18e6bbd202e8484a6e11d387af6e482b525fa25542d46ff9c99db87bd419b980c24a086117a397f6d8f88e74351b41693880ea0cb@127.0.0.1:21004?discport=0&raftport=50405") +Error: node with this enode has already been added to the cluster: f06c06f1e958cb2edf90d8bfb912de287f9b047b4228436e94b5b78e3ee16171 + at web3.js:3143:20 + at web3.js:6347:15 + at web3.js:5081:36 + at :1:1 +``` +### raft_removePeer +API to remove a node from raft cluster +#### Parameters +* `raftId` : raft id of the node to be removed from the cluster +#### Returns +* `result`: null +#### Examples +```jshelllanguage tab="JSON RPC" +// Request +curl -X POST http://127.0.0.1:22001 --data '{"jsonrpc":"2.0","method":"raft_removePeer","params": [4], "id":10}' --header "Content-Type: application/json" + +// Response +{"jsonrpc":"2.0","id":10,"result":null} +``` + +```javascript tab="geth console" +> raft.removePeer(4) +null +``` + +### raft_addLearner +API to add a new node to the network as a learner node. The learner node syncs with network and can transact but will not be part of raft quorum and hence will not provide block confirmation to minter node. +#### Parameters +* `enodeId` +#### Returns +* `result`: raft id for the node being added +#### Examples +```jshelllanguage tab="JSON RPC" +// Request +curl -X POST http://127.0.0.1:22001 --data '{"jsonrpc":"2.0","method":"raft_addLearner","params": ["enode://3701f007bfa4cb26512d7df18e6bbd202e8484a6e11d387af6e482b525fa25542d46ff9c99db87bd419b980c24a086117a397f6d8f88e74351b41693880ea0cb@127.0.0.1:21004?discport=0&raftport=50405"], "id":10}' --header "Content-Type: application/json" + +// Response +{"jsonrpc":"2.0","id":10,"result":5} + +``` + +```javascript tab="geth console" +> raft.addLearner("enode://3701f007bfa4cb26512d7df18e6bbd202e8484a6e11d387af6e482b525fa25542d46ff9c99db87bd419b980c24a086117a397f6d8f88e74351b41693880ea0cb@127.0.0.1:21004?discport=0&raftport=50405") +5 +``` +### raft_promoteToPeer +API for promoting a learner node to peer and thus be part of the raft quorum. +#### Parameters +* `raftId`: raft id of the node to be promoted +#### Returns +* `result`: true or false +#### Examples +```jshelllanguage tab="JSON RPC" +// Request +curl -X POST http://127.0.0.1:22001 --data '{"jsonrpc":"2.0","method":"raft_promoteToPeer","params": [4], "id":10}' --header "Content-Type: application/json" + +// Response +{// Response + {"jsonrpc":"2.0","id":10,"result":true} +``` + +```javascript tab="geth console" +> raft.promoteToPeer(4) +true +``` diff --git a/docs/Consensus/raft.md b/docs/Consensus/raft/raft.md similarity index 99% rename from docs/Consensus/raft.md rename to docs/Consensus/raft/raft.md index 2c4960a0ff..2fc9a57c73 100644 --- a/docs/Consensus/raft.md +++ b/docs/Consensus/raft/raft.md @@ -1,4 +1,4 @@ -# Raft-based consensus for Ethereum/Quorum +# Raft Consensus Overview ## Introduction @@ -190,4 +190,4 @@ Note that like the enode IDs listed in the static peers JSON file, this enode ID ## FAQ -Answers to frequently asked questions can be found on the main [Quorum FAQ page](../FAQ.md). +Answers to frequently asked questions can be found on the main [Quorum FAQ page](../../FAQ.md). diff --git a/mkdocs.yml b/mkdocs.yml index cf02102c06..79d125579e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -20,7 +20,9 @@ nav: - Quorum API: Getting Started/api.md - Consensus: - Consensus: Consensus/Consensus.md - - Raft: Consensus/raft.md + - Raft BFT: + - Consensus/raft/raft.md + - Consensus/raft/raft-rpc-api.md - Istanbul BFT: - Consensus/ibft/istanbul-rpc-api.md - Consensus/ibft/ibft-parameters.md diff --git a/raft/api.go b/raft/api.go index 72b247f20c..518b413f33 100755 --- a/raft/api.go +++ b/raft/api.go @@ -124,7 +124,7 @@ func (s *PublicRaftAPI) checkIfNodeIsActive(raftId uint16) bool { if raftId == s.raftService.raftProtocolManager.raftId { return true } - activeSince := s.raftService.raftProtocolManager.transport.ActiveSince(types.ID(raftId)) + activeSince := s.raftService.raftProtocolManager.transport.ActiveSince(types.ID(raftId)) if activeSince.IsZero() { return false } From 2608b97ae50c9b07e636b0f28b326d2c6569cd7a Mon Sep 17 00:00:00 2001 From: Jitendra Bhurat Date: Tue, 4 Feb 2020 12:26:51 -0500 Subject: [PATCH 6/6] Updated the comment for checkIfNodeIsActive() to fix typo --- raft/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/raft/api.go b/raft/api.go index 518b413f33..c035202489 100755 --- a/raft/api.go +++ b/raft/api.go @@ -118,8 +118,8 @@ func (s *PublicRaftAPI) Cluster() ([]ClusterInfo, error) { return clustInfo, nil } -// helper function to check of the raft node is active -// of the raftnode is active ActiveSince returns non-zero time +// checkIfNodeIsActive checks if the raft node is active +// if the raft node is active ActiveSince returns non-zero time func (s *PublicRaftAPI) checkIfNodeIsActive(raftId uint16) bool { if raftId == s.raftService.raftProtocolManager.raftId { return true