From 45ee8d6e7450d8b70c1abf0763282eda01278f44 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Sat, 20 Apr 2024 15:53:34 +0900 Subject: [PATCH 01/19] WIP --- raftify/Cargo.toml | 5 + raftify/src/raft_node/mod.rs | 15 ++- raftify/src/storage/heed_storage/mod.rs | 1 + raftify/src/storage/heed_storage/utils.rs | 7 ++ raftify/src/storage/inmemory_storage/mod.rs | 121 ++++++++++++++++++++ raftify/src/storage/mod.rs | 1 + raftify/src/storage/utils.rs | 8 -- 7 files changed, 148 insertions(+), 10 deletions(-) create mode 100644 raftify/src/storage/heed_storage/utils.rs create mode 100644 raftify/src/storage/inmemory_storage/mod.rs diff --git a/raftify/Cargo.toml b/raftify/Cargo.toml index 30261639..ab0569e3 100644 --- a/raftify/Cargo.toml +++ b/raftify/Cargo.toml @@ -30,6 +30,11 @@ built = "0.5" clap = { version = "4.5.18", features = ["derive"] } chrono = "0.4.38" +[features] +default = ["inmemory_storage"] +inmemory_storage = [] +heed_storage =[] + [dev-dependencies] tempfile = "3" diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index b318943d..5cd5270c 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -51,8 +51,9 @@ use crate::{ ResponseMessage, }, storage::{ - heed_storage::HeedStorage, - utils::{clear_storage_path, ensure_directory_exist, get_data_mdb_path, get_storage_path}, + heed_storage::{utils::{get_data_mdb_path, get_storage_path}, HeedStorage}, + inmemory_storage::MemStorage, + utils::{clear_storage_path, ensure_directory_exist}, }, utils::{membership::to_confchange_v2, oneshot_mutex::OneShotMutex}, AbstractLogEntry, AbstractStateMachine, ClusterJoinTicket, Config, Error, InitialRole, Peers, @@ -478,7 +479,10 @@ pub struct RaftNodeCore< LogEntry: AbstractLogEntry + Send + 'static, FSM: AbstractStateMachine + Clone + 'static, > { + #[cfg(feature = "heed_storage")] pub raw_node: RawNode, + #[cfg(feature = "inmemory_storage")] + pub raw_node: RawNode, pub fsm: FSM, pub peers: Arc>, response_seq: AtomicU64, @@ -531,8 +535,14 @@ impl< ensure_directory_exist(storage_pth.as_str())?; }; + #[cfg(feature = "heed_storage")] let mut storage = HeedStorage::create(storage_pth.as_str(), &config, logger.clone())?; + + #[cfg(feature = "inmemory_storage")] + let mut storage = MemStorage::create(); + let mut snapshot = storage.snapshot(0, storage.last_index()?)?; + let conf_state = snapshot.mut_metadata().mut_conf_state(); let peers = config @@ -1120,6 +1130,7 @@ impl< .unwrap(); } LocalRequestMsg::GetStorage { tx_msg } => { + #[cfg(feature = "heed_storage")] tx_msg .send(LocalResponseMsg::GetStorage { storage: self.raw_node.store().clone(), diff --git a/raftify/src/storage/heed_storage/mod.rs b/raftify/src/storage/heed_storage/mod.rs index db2716b3..e7e30bad 100644 --- a/raftify/src/storage/heed_storage/mod.rs +++ b/raftify/src/storage/heed_storage/mod.rs @@ -1,5 +1,6 @@ mod codec; mod constant; +pub mod utils; use bincode::{deserialize, serialize}; use constant::{CONF_STATE_KEY, HARD_STATE_KEY, LAST_INDEX_KEY, SNAPSHOT_KEY}; diff --git a/raftify/src/storage/heed_storage/utils.rs b/raftify/src/storage/heed_storage/utils.rs new file mode 100644 index 00000000..a2c26255 --- /dev/null +++ b/raftify/src/storage/heed_storage/utils.rs @@ -0,0 +1,7 @@ +pub fn get_storage_path(log_dir: &str, node_id: u64) -> String { + format!("{}/node-{}", log_dir, node_id) +} + +pub fn get_data_mdb_path(log_dir: &str, node_id: u64) -> String { + format!("{}/data.mdb", get_storage_path(log_dir, node_id)) +} diff --git a/raftify/src/storage/inmemory_storage/mod.rs b/raftify/src/storage/inmemory_storage/mod.rs new file mode 100644 index 00000000..65c47470 --- /dev/null +++ b/raftify/src/storage/inmemory_storage/mod.rs @@ -0,0 +1,121 @@ +use crate::{ + error::Result, + raft::{ + self, + eraftpb::{ConfState, Entry, HardState, Snapshot}, + storage::{MemStorage as MemStorageCore, Storage}, + }, + StableStorage, +}; + +#[derive(Clone)] +pub struct MemStorage { + core: MemStorageCore, + snapshot: Snapshot, +} + +impl MemStorage { + pub fn create() -> Self { + let core = MemStorageCore::default(); + let snapshot: Snapshot = Default::default(); + Self { core, snapshot } + } +} + +impl StableStorage for MemStorage { + fn append(&mut self, entries: &[Entry]) -> Result<()> { + let mut store = self.core.wl(); + store.append(entries)?; + Ok(()) + } + + fn hard_state(&self) -> Result { + let store = self.core.rl(); + Ok(store.hard_state().clone()) + } + + fn set_hard_state(&mut self, hard_state: &HardState) -> Result<()> { + let mut store = self.core.wl(); + store.set_hardstate(hard_state.clone()); + Ok(()) + } + + fn set_hard_state_commit(&mut self, commit: u64) -> Result<()> { + let mut store = self.core.wl(); + let mut hard_state = store.hard_state().clone(); + hard_state.set_commit(commit); + store.set_hardstate(hard_state); + Ok(()) + } + + fn conf_state(&self) -> Result { + todo!() + } + + fn set_conf_state(&mut self, conf_state: &ConfState) -> Result<()> { + let mut store = self.core.wl(); + store.set_conf_state(conf_state.clone()); + Ok(()) + } + + fn create_snapshot(&mut self, data: Vec, index: u64, term: u64) -> Result<()> { + let mut snapshot = self.core.snapshot(0, 0)?; + snapshot.set_data(data); + let mut metadata = snapshot.take_metadata(); + metadata.set_index(index); + metadata.set_term(term); + snapshot.set_metadata(metadata); + self.snapshot = snapshot; + Ok(()) + } + + fn apply_snapshot(&mut self, snapshot: Snapshot) -> Result<()> { + let mut store = self.core.wl(); + store.apply_snapshot(snapshot)?; + Ok(()) + } + + fn compact(&mut self, index: u64) -> Result<()> { + let mut store = self.core.wl(); + store.compact(index)?; + Ok(()) + } + + fn all_entries(&self) -> raft::Result> { + todo!() + } +} + +impl Storage for MemStorage { + fn initial_state(&self) -> raft::Result { + let raft_state = self.core.initial_state()?; + Ok(raft_state) + } + + fn entries( + &self, + low: u64, + high: u64, + max_size: impl Into>, + context: raft::GetEntriesContext, + ) -> raft::Result> { + let entries = self.core.entries(low, high, max_size, context)?; + Ok(entries) + } + + fn term(&self, idx: u64) -> raft::Result { + self.core.term(idx) + } + + fn first_index(&self) -> raft::Result { + self.core.first_index() + } + + fn last_index(&self) -> raft::Result { + self.core.last_index() + } + + fn snapshot(&self, _request_index: u64, _to: u64) -> raft::Result { + Ok(self.snapshot.clone()) + } +} diff --git a/raftify/src/storage/mod.rs b/raftify/src/storage/mod.rs index d5e68da9..1caba081 100644 --- a/raftify/src/storage/mod.rs +++ b/raftify/src/storage/mod.rs @@ -1,4 +1,5 @@ pub mod heed_storage; +pub mod inmemory_storage; pub mod utils; use crate::{ diff --git a/raftify/src/storage/utils.rs b/raftify/src/storage/utils.rs index ae6b1f8a..a266cf16 100644 --- a/raftify/src/storage/utils.rs +++ b/raftify/src/storage/utils.rs @@ -11,14 +11,6 @@ use crate::{ Result, }; -pub fn get_storage_path(log_dir: &str, node_id: u64) -> String { - format!("{}/node-{}", log_dir, node_id) -} - -pub fn get_data_mdb_path(log_dir: &str, node_id: u64) -> String { - format!("{}/data.mdb", get_storage_path(log_dir, node_id)) -} - pub fn clear_storage_path(log_dir_path: &str) -> Result<()> { let log_dir_path = Path::new(&log_dir_path); From 6f640e6e39a4d9cc91c6df153e6821445e3ac662 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Sat, 20 Apr 2024 18:14:41 +0900 Subject: [PATCH 02/19] Implement correct `SnapshotOutOfDate` check logic in HeedStorage --- raftify/src/storage/heed_storage/mod.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/raftify/src/storage/heed_storage/mod.rs b/raftify/src/storage/heed_storage/mod.rs index e7e30bad..91c3569e 100644 --- a/raftify/src/storage/heed_storage/mod.rs +++ b/raftify/src/storage/heed_storage/mod.rs @@ -10,7 +10,7 @@ use heed::{ }; use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use prost::Message as PMessage; -use raft::{logger::Logger, util::limit_size}; +use raft::{logger::Logger, util::limit_size, INVALID_INDEX}; use std::{ cmp::max, fs, @@ -24,6 +24,7 @@ use crate::{ config::Config, error::Result, raft::{self, prelude::*, GetEntriesContext}, + Error, }; #[derive(Clone)] @@ -128,13 +129,12 @@ impl StableStorage for HeedStorage { let metadata = snapshot.get_metadata(); let conf_state = metadata.get_conf_state(); - // TODO: Investigate if this is necessary. It broke the static bootstrap. - // let first_index = store.first_index(&writer)?; - // if first_index > metadata.index { - // return Err(Error::RaftStorageError( - // raft::StorageError::SnapshotOutOfDate, - // )); - // } + let first_index = store.first_index(&writer)?; + if metadata.index != INVALID_INDEX && first_index > metadata.index { + return Err(Error::RaftStorageError( + raft::StorageError::SnapshotOutOfDate, + )); + } let mut hard_state = store.hard_state(&writer)?; hard_state.set_term(max(hard_state.term, metadata.term)); @@ -938,9 +938,8 @@ mod test { storage.apply_snapshot(snap).unwrap(); // Apply snapshot fails due to StorageError::SnapshotOutOfDate - // TODO: Support the below test case - // let snap = new_snapshot(3, 3, nodes); - // storage.apply_snapshot(snap).unwrap_err(); + let snap = new_snapshot(3, 3, nodes); + storage.apply_snapshot(snap).unwrap_err(); teardown(tempdir); } From 3f8746a04fb1e62953fcfcdfacba550e6db71de5 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Sat, 20 Apr 2024 18:14:56 +0900 Subject: [PATCH 03/19] Update `apply_snapshot` in MemStorage --- raftify/src/raft_node/mod.rs | 5 ++++- raftify/src/storage/inmemory_storage/mod.rs | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index 5cd5270c..bc01478e 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -51,7 +51,10 @@ use crate::{ ResponseMessage, }, storage::{ - heed_storage::{utils::{get_data_mdb_path, get_storage_path}, HeedStorage}, + heed_storage::{ + utils::{get_data_mdb_path, get_storage_path}, + HeedStorage, + }, inmemory_storage::MemStorage, utils::{clear_storage_path, ensure_directory_exist}, }, diff --git a/raftify/src/storage/inmemory_storage/mod.rs b/raftify/src/storage/inmemory_storage/mod.rs index 65c47470..c4adef3a 100644 --- a/raftify/src/storage/inmemory_storage/mod.rs +++ b/raftify/src/storage/inmemory_storage/mod.rs @@ -4,6 +4,7 @@ use crate::{ self, eraftpb::{ConfState, Entry, HardState, Snapshot}, storage::{MemStorage as MemStorageCore, Storage}, + INVALID_INDEX, }, StableStorage, }; @@ -71,6 +72,14 @@ impl StableStorage for MemStorage { fn apply_snapshot(&mut self, snapshot: Snapshot) -> Result<()> { let mut store = self.core.wl(); + + // Pass apply snapshot if the snapshot is empty + if snapshot.get_metadata().get_index() == INVALID_INDEX { + // Update conf states. + store.set_conf_state(snapshot.get_metadata().clone().take_conf_state()); + return Ok(()); + } + store.apply_snapshot(snapshot)?; Ok(()) } From 48bc8f85f7e2ec5baec18335b1ed17ff12b1b852 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 23 Sep 2024 17:10:14 +0900 Subject: [PATCH 04/19] chore: Set default storage to heed --- raftify/Cargo.toml | 2 +- raftify/src/storage/inmemory_storage/mod.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/raftify/Cargo.toml b/raftify/Cargo.toml index ab0569e3..bef6972e 100644 --- a/raftify/Cargo.toml +++ b/raftify/Cargo.toml @@ -31,7 +31,7 @@ clap = { version = "4.5.18", features = ["derive"] } chrono = "0.4.38" [features] -default = ["inmemory_storage"] +default = ["heed_storage"] inmemory_storage = [] heed_storage =[] diff --git a/raftify/src/storage/inmemory_storage/mod.rs b/raftify/src/storage/inmemory_storage/mod.rs index c4adef3a..bae1c5ec 100644 --- a/raftify/src/storage/inmemory_storage/mod.rs +++ b/raftify/src/storage/inmemory_storage/mod.rs @@ -75,7 +75,6 @@ impl StableStorage for MemStorage { // Pass apply snapshot if the snapshot is empty if snapshot.get_metadata().get_index() == INVALID_INDEX { - // Update conf states. store.set_conf_state(snapshot.get_metadata().clone().take_conf_state()); return Ok(()); } From 1b62ccf774adb2c83c4af951572cae2dccacc178 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 23 Sep 2024 17:13:12 +0900 Subject: [PATCH 05/19] chore: Mark todo --- raftify/src/raft_node/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index bc01478e..459b1c93 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -1139,6 +1139,9 @@ impl< storage: self.raw_node.store().clone(), }) .unwrap(); + + #[cfg(feature = "inmemory_storage")] + todo!("Implement this for inmemory storage"); } LocalRequestMsg::DebugNode { tx_msg } => { tx_msg From 0d1e75624b8e4949b0884408c54b1f29a251b29b Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 23 Sep 2024 17:21:43 +0900 Subject: [PATCH 06/19] fix: Use only selected storage source code --- raftify/src/raft_node/mod.rs | 18 ++++++++++-------- raftify/src/storage/inmemory_storage/mod.rs | 5 +++++ raftify/src/storage/mod.rs | 4 ++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index 459b1c93..30eff696 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -50,19 +50,21 @@ use crate::{ }, ResponseMessage, }, - storage::{ - heed_storage::{ - utils::{get_data_mdb_path, get_storage_path}, - HeedStorage, - }, - inmemory_storage::MemStorage, - utils::{clear_storage_path, ensure_directory_exist}, - }, + storage::utils::{clear_storage_path, ensure_directory_exist}, utils::{membership::to_confchange_v2, oneshot_mutex::OneShotMutex}, AbstractLogEntry, AbstractStateMachine, ClusterJoinTicket, Config, Error, InitialRole, Peers, RaftServiceClient, StableStorage, }; +#[cfg(feature = "heed_storage")] +use crate::storage::heed_storage::{ + utils::{get_data_mdb_path, get_storage_path}, + HeedStorage, +}; + +#[cfg(feature = "inmemory_storage")] +use crate::storage::inmemory_storage::MemStorage; + #[derive(Clone)] pub struct RaftNode< LogEntry: AbstractLogEntry + Send + 'static, diff --git a/raftify/src/storage/inmemory_storage/mod.rs b/raftify/src/storage/inmemory_storage/mod.rs index bae1c5ec..368ff874 100644 --- a/raftify/src/storage/inmemory_storage/mod.rs +++ b/raftify/src/storage/inmemory_storage/mod.rs @@ -127,3 +127,8 @@ impl Storage for MemStorage { Ok(self.snapshot.clone()) } } + +// Ref: https://github.com/tikv/raft-rs/blob/master/src/storage.rs +// TODO: Implement these tests for MemStorage +#[cfg(test)] +mod test {} diff --git a/raftify/src/storage/mod.rs b/raftify/src/storage/mod.rs index 1caba081..725574fc 100644 --- a/raftify/src/storage/mod.rs +++ b/raftify/src/storage/mod.rs @@ -1,5 +1,9 @@ +#[cfg(feature = "heed_storage")] pub mod heed_storage; + +#[cfg(feature = "inmemory_storage")] pub mod inmemory_storage; + pub mod utils; use crate::{ From a12d1d14994db635d364128b51c5e6274976bdec Mon Sep 17 00:00:00 2001 From: Gyubong Date: Fri, 27 Sep 2024 15:45:27 +0900 Subject: [PATCH 07/19] refactor: Add `LogStorage` generic type and extract log storage variable --- Cargo.lock | 97 ------------- Cargo.toml | 4 +- examples/memstore/Cargo.toml | 5 + examples/memstore/dynamic-members/Cargo.toml | 5 + examples/memstore/dynamic-members/src/main.rs | 46 +++++- examples/memstore/src/cli/main.rs | 9 +- examples/memstore/src/state_machine.rs | 10 +- examples/memstore/src/web_server_api.rs | 49 ++++--- .../memstore/{ => static-members}/.gitignore | 0 examples/memstore/static-members/Cargo.toml | 5 + examples/memstore/static-members/src/main.rs | 32 +++-- examples/src/lib.rs | 1 + examples/src/utils.rs | 19 +++ harness/src/raft.rs | 3 +- raftify/src/cli/commands/debug.rs | 42 ++++-- raftify/src/cli/commands/dump.rs | 119 --------------- raftify/src/cli/commands/mod.rs | 2 - raftify/src/cli/commands/utils.rs | 22 --- raftify/src/cli/mod.rs | 43 +++--- raftify/src/config.rs | 25 +--- raftify/src/lib.rs | 7 +- raftify/src/raft_bootstrapper.rs | 32 +++-- raftify/src/raft_node/mod.rs | 135 +++++++----------- raftify/src/raft_node/response_sender.rs | 16 ++- raftify/src/raft_server.rs | 33 +++-- raftify/src/request/local_request_message.rs | 50 ++++--- raftify/src/request/server_request_message.rs | 9 +- .../src/response/local_response_message.rs | 25 ++-- raftify/src/response/mod.rs | 10 +- .../src/response/server_response_message.rs | 6 +- raftify/src/storage/heed_storage/mod.rs | 1 - raftify/src/storage/heed_storage/utils.rs | 7 - raftify/src/storage/utils.rs | 26 +--- 33 files changed, 378 insertions(+), 517 deletions(-) rename examples/memstore/{ => static-members}/.gitignore (100%) create mode 100644 examples/src/utils.rs delete mode 100644 raftify/src/cli/commands/dump.rs delete mode 100644 raftify/src/cli/commands/utils.rs delete mode 100644 raftify/src/storage/heed_storage/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 6d9b7855..d56e04cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -636,26 +636,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" -[[package]] -name = "const_format" -version = "0.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c655d81ff1114fb0dcdea9225ea9f0cc712a6f8d189378e82bdf62a473a64b" -dependencies = [ - "const_format_proc_macros", -] - -[[package]] -name = "const_format_proc_macros" -version = "0.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff1a44b93f47b1bac19a27932f5c591e43d1ba357ee4f61526c8a25603f0eb1" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - [[package]] name = "convert_case" version = "0.4.0" @@ -868,21 +848,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "futures" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - [[package]] name = "futures-channel" version = "0.3.30" @@ -890,7 +855,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", - "futures-sink", ] [[package]] @@ -899,34 +863,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" -[[package]] -name = "futures-executor" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" - -[[package]] -name = "futures-macro" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.77", -] - [[package]] name = "futures-sink" version = "0.3.30" @@ -945,16 +881,10 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ - "futures-channel", "futures-core", - "futures-io", - "futures-macro", - "futures-sink", "futures-task", - "memchr", "pin-project-lite", "pin-utils", - "slab", ] [[package]] @@ -1024,27 +954,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "harness" -version = "0.1.69" -dependencies = [ - "actix-rt", - "actix-web", - "bincode", - "const_format", - "futures", - "log", - "raftify", - "serde", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "structopt", - "tokio", - "toml 0.8.19", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -2676,12 +2585,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" -[[package]] -name = "unicode-xid" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" - [[package]] name = "url" version = "2.5.2" diff --git a/Cargo.toml b/Cargo.toml index 7d46525f..84cab6b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,6 @@ resolver = "2" members = [ "raftify", - "harness", "examples", "examples/memstore", "examples/memstore/dynamic-members", @@ -10,7 +9,6 @@ members = [ ] default-members = [ "raftify", - "harness", "examples", "examples/memstore", "examples/memstore/dynamic-members", @@ -35,4 +33,4 @@ example-harness = { path = "examples" } memstore-example-harness = { path = "examples/memstore" } [patch.crates-io] -jopemachine-raft = { path = "./raft-rs" } \ No newline at end of file +jopemachine-raft = { path = "./raft-rs" } diff --git a/examples/memstore/Cargo.toml b/examples/memstore/Cargo.toml index dd4b7c22..40af1cc3 100644 --- a/examples/memstore/Cargo.toml +++ b/examples/memstore/Cargo.toml @@ -28,3 +28,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1.40", features = ["full"] } color-backtrace = "0.6.1" + +[features] +default = ["heed_storage"] +inmemory_storage = [] +heed_storage =[] diff --git a/examples/memstore/dynamic-members/Cargo.toml b/examples/memstore/dynamic-members/Cargo.toml index 0428ac7c..41be2bde 100644 --- a/examples/memstore/dynamic-members/Cargo.toml +++ b/examples/memstore/dynamic-members/Cargo.toml @@ -28,3 +28,8 @@ structopt = "0.3" tokio = { version = "1.40", features = ["full"] } toml = "0.8.19" color-backtrace = "0.6.1" + +[features] +default = ["heed_storage"] +inmemory_storage = [] +heed_storage =[] diff --git a/examples/memstore/dynamic-members/src/main.rs b/examples/memstore/dynamic-members/src/main.rs index c4eb7468..f24a8174 100644 --- a/examples/memstore/dynamic-members/src/main.rs +++ b/examples/memstore/dynamic-members/src/main.rs @@ -6,23 +6,30 @@ extern crate slog_term; use actix_web::{web, App, HttpServer}; use raftify::{ raft::{formatter::set_custom_formatter, logger::Slogger}, - CustomFormatter, Raft as Raft_, + CustomFormatter, }; use slog::Drain; use slog_envlogger::LogBuilder; use std::sync::Arc; use structopt::StructOpt; -use example_harness::config::build_config; +use example_harness::{ + config::build_config, + utils::{ensure_directory_exist, get_storage_path}, +}; use memstore_example_harness::{ - state_machine::{HashStore, LogEntry}, + state_machine::{HashStore, LogEntry, Raft}, web_server_api::{ campaign, debug, demote, get, leader_id, leave, leave_joint, peers, put, snapshot, transfer_leader, }, }; -type Raft = Raft_; +#[cfg(feature = "inmemory_storage")] +use raftify::MemStorage; + +#[cfg(feature = "heed_storage")] +use raftify::HeedStorage; #[derive(Debug, StructOpt)] struct Options { @@ -71,9 +78,20 @@ async fn main() -> std::result::Result<(), Box> { let node_id = ticket.reserved_id; cfg.initial_peers = Some(ticket.peers.clone().into()); + let storage_pth = get_storage_path(cfg.log_dir.as_str(), node_id); + ensure_directory_exist(storage_pth.as_str())?; + + #[cfg(feature = "inmemory_storage")] + let log_storage = MemStorage::create(); + + #[cfg(feature = "heed_storage")] + let log_storage = HeedStorage::create(&storage_pth, &cfg.clone(), logger.clone()) + .expect("Failed to create heed storage"); + let raft = Raft::bootstrap( node_id, options.raft_addr, + log_storage, store.clone(), cfg.clone(), logger.clone(), @@ -87,7 +105,25 @@ async fn main() -> std::result::Result<(), Box> { } None => { log::info!("Bootstrap a Raft Cluster"); - let raft = Raft::bootstrap(1, options.raft_addr, store.clone(), cfg, logger.clone())?; + + let storage_pth = get_storage_path(cfg.log_dir.as_str(), 1); + ensure_directory_exist(storage_pth.as_str())?; + + #[cfg(feature = "inmemory_storage")] + let log_storage = MemStorage::create(); + + #[cfg(feature = "heed_storage")] + let log_storage = HeedStorage::create(&storage_pth, &cfg.clone(), logger.clone()) + .expect("Failed to create heed storage"); + + let raft = Raft::bootstrap( + 1, + options.raft_addr, + log_storage, + store.clone(), + cfg, + logger.clone(), + )?; let handle = tokio::spawn(raft.clone().run()); (raft, handle) } diff --git a/examples/memstore/src/cli/main.rs b/examples/memstore/src/cli/main.rs index 67920372..b3491fd9 100644 --- a/examples/memstore/src/cli/main.rs +++ b/examples/memstore/src/cli/main.rs @@ -2,8 +2,15 @@ use raftify::{cli::cli_handler, Result}; use memstore_example_harness::state_machine::{HashStore, LogEntry}; +#[cfg(feature = "inmemory_storage")] +use raftify::MemStorage as StorageType; + +#[cfg(feature = "heed_storage")] +use raftify::HeedStorage as StorageType; + #[tokio::main] async fn main() -> Result<()> { - cli_handler::(None).await?; + cli_handler::(None).await?; + Ok(()) } diff --git a/examples/memstore/src/state_machine.rs b/examples/memstore/src/state_machine.rs index ff556688..c2e57ec0 100644 --- a/examples/memstore/src/state_machine.rs +++ b/examples/memstore/src/state_machine.rs @@ -1,12 +1,20 @@ use async_trait::async_trait; use bincode::{deserialize, serialize}; -use raftify::{AbstractLogEntry, AbstractStateMachine, Result}; +use raftify::{AbstractLogEntry, AbstractStateMachine, Raft as Raft_, Result}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, sync::{Arc, RwLock}, }; +#[cfg(feature = "inmemory_storage")] +use raftify::MemStorage as StorageType; + +#[cfg(feature = "heed_storage")] +use raftify::HeedStorage as StorageType; + +pub type Raft = Raft_; + #[derive(Clone, Debug, Serialize, Deserialize)] pub enum LogEntry { Insert { key: u64, value: String }, diff --git a/examples/memstore/src/web_server_api.rs b/examples/memstore/src/web_server_api.rs index 36e16675..d06424cf 100644 --- a/examples/memstore/src/web_server_api.rs +++ b/examples/memstore/src/web_server_api.rs @@ -1,12 +1,11 @@ use std::collections::HashMap; use actix_web::{get, put, web, Responder}; -use raftify::{raft::Storage, AbstractLogEntry, Raft as Raft_, StableStorage}; +use raftify::raft::Storage; +use raftify::{AbstractLogEntry, StableStorage}; use serde_json::Value; -use super::state_machine::{HashStore, LogEntry}; - -type Raft = Raft_; +use crate::state_machine::{HashStore, LogEntry, Raft}; #[put("/store/{id}/{value}")] async fn put(data: web::Data<(HashStore, Raft)>, path: web::Path<(u64, String)>) -> impl Responder { @@ -58,23 +57,31 @@ async fn peers(data: web::Data<(HashStore, Raft)>) -> impl Responder { #[get("/snapshot")] async fn snapshot(data: web::Data<(HashStore, Raft)>) -> impl Responder { - let raft = data.clone(); - let last_index = raft - .1 - .storage() - .await - .last_index() - .expect("Failed to get last index"); - - let hard_state = raft - .1 - .storage() - .await - .hard_state() - .expect("Failed to get hard state"); - - raft.1.make_snapshot(last_index, hard_state.term).await; - "OK".to_string() + #[cfg(feature = "inmemory_storage")] + { + return "Snapshot is not supported with inmemory storage".to_string(); + } + + #[cfg(not(feature = "inmemory_storage"))] + { + let raft = data.clone(); + let last_index = raft + .1 + .storage() + .await + .last_index() + .expect("Failed to get last index"); + + let hard_state = raft + .1 + .storage() + .await + .hard_state() + .expect("Failed to get hard state"); + + raft.1.make_snapshot(last_index, hard_state.term).await; + "OK".to_string() + } } #[get("/leave_joint")] diff --git a/examples/memstore/.gitignore b/examples/memstore/static-members/.gitignore similarity index 100% rename from examples/memstore/.gitignore rename to examples/memstore/static-members/.gitignore diff --git a/examples/memstore/static-members/Cargo.toml b/examples/memstore/static-members/Cargo.toml index 6077d243..52b3a09f 100644 --- a/examples/memstore/static-members/Cargo.toml +++ b/examples/memstore/static-members/Cargo.toml @@ -27,3 +27,8 @@ structopt = "0.3" tokio = { version = "1.40", features = ["full"] } toml = "0.8.19" color-backtrace = "0.6.1" + +[features] +default = ["heed_storage"] +inmemory_storage = [] +heed_storage =[] diff --git a/examples/memstore/static-members/src/main.rs b/examples/memstore/static-members/src/main.rs index b732b2ff..6676246d 100644 --- a/examples/memstore/static-members/src/main.rs +++ b/examples/memstore/static-members/src/main.rs @@ -6,16 +6,19 @@ extern crate slog_term; use actix_web::{web, App, HttpServer}; use raftify::{ raft::{formatter::set_custom_formatter, logger::Slogger}, - CustomFormatter, Raft as Raft_, + CustomFormatter, }; use slog::Drain; use slog_envlogger::LogBuilder; use std::sync::Arc; use structopt::StructOpt; -use example_harness::config::build_config; +use example_harness::{ + config::build_config, + utils::{ensure_directory_exist, get_storage_path}, +}; use memstore_example_harness::{ - state_machine::{HashStore, LogEntry}, + state_machine::{HashStore, LogEntry, Raft}, web_server_api::{ campaign, debug, demote, get, leader_id, leave, leave_joint, peers, put, snapshot, transfer_leader, @@ -23,7 +26,11 @@ use memstore_example_harness::{ }; use memstore_static_members::utils::load_peers; -type Raft = Raft_; +#[cfg(feature = "inmemory_storage")] +use raftify::MemStorage; + +#[cfg(feature = "heed_storage")] +use raftify::HeedStorage; #[derive(Debug, StructOpt)] struct Options { @@ -31,10 +38,6 @@ struct Options { raft_addr: String, #[structopt(long)] web_server: Option, - #[structopt(long)] - restore_wal_from: Option, - #[structopt(long)] - restore_wal_snapshot_from: Option, } #[actix_rt::main] @@ -65,16 +68,25 @@ async fn main() -> std::result::Result<(), Box> { let mut cfg = build_config(); cfg.initial_peers = Some(initial_peers.clone()); - cfg.restore_wal_from = options.restore_wal_from; - cfg.restore_wal_snapshot_from = options.restore_wal_snapshot_from; let node_id = initial_peers .get_node_id_by_addr(options.raft_addr.clone()) .unwrap(); + let storage_pth = get_storage_path(cfg.log_dir.as_str(), node_id); + ensure_directory_exist(storage_pth.as_str())?; + + #[cfg(feature = "inmemory_storage")] + let log_storage = MemStorage::create(); + + #[cfg(feature = "heed_storage")] + let log_storage = HeedStorage::create(&storage_pth, &cfg.clone(), logger.clone()) + .expect("Failed to create heed storage"); + let raft = Raft::bootstrap( node_id, options.raft_addr.clone(), + log_storage, store.clone(), cfg.clone(), logger.clone(), diff --git a/examples/src/lib.rs b/examples/src/lib.rs index ef68c369..472f7fce 100644 --- a/examples/src/lib.rs +++ b/examples/src/lib.rs @@ -1 +1,2 @@ pub mod config; +pub mod utils; diff --git a/examples/src/utils.rs b/examples/src/utils.rs new file mode 100644 index 00000000..22f2d0fc --- /dev/null +++ b/examples/src/utils.rs @@ -0,0 +1,19 @@ +use raftify::Result; +use std::{fs, path::Path}; + +pub fn get_storage_path(log_dir: &str, node_id: u64) -> String { + format!("{}/node-{}", log_dir, node_id) +} + +pub fn get_data_mdb_path(log_dir: &str, node_id: u64) -> String { + format!("{}/data.mdb", get_storage_path(log_dir, node_id)) +} + +pub fn ensure_directory_exist(dir_pth: &str) -> Result<()> { + let dir_pth: &Path = Path::new(&dir_pth); + + if fs::metadata(dir_pth).is_err() { + fs::create_dir_all(dir_pth)?; + } + Ok(()) +} diff --git a/harness/src/raft.rs b/harness/src/raft.rs index ff178c72..8f29e65d 100644 --- a/harness/src/raft.rs +++ b/harness/src/raft.rs @@ -2,6 +2,7 @@ use futures::future; use raftify::{ raft::{formatter::set_custom_formatter, logger::Slogger}, CustomFormatter, Peers, Raft as Raft_, Result, + HeedStorage, }; use std::{ collections::HashMap, @@ -16,7 +17,7 @@ use crate::{ utils::build_logger, }; -pub type Raft = Raft_; +pub type Raft = Raft_; pub async fn wait_until_rafts_ready( rafts: Option>, diff --git a/raftify/src/cli/commands/debug.rs b/raftify/src/cli/commands/debug.rs index c704350e..2f2a8e2f 100644 --- a/raftify/src/cli/commands/debug.rs +++ b/raftify/src/cli/commands/debug.rs @@ -7,26 +7,18 @@ use crate::{ raft::{ formatter::{format_entry, format_snapshot}, logger::Slogger, - Storage, }, raft_node::utils::format_debugging_info, - raft_service, Config, HeedStorage, Result, StableStorage, + raft_service, Config, Result, StableStorage, }; -pub fn debug_persisted(path: &str, logger: slog::Logger) -> Result<()> { - let config = Config { - log_dir: path.to_string(), - ..Default::default() - }; +#[cfg(feature = "inmemory_storage")] +use raftify::MemStorage; - let storage = HeedStorage::create( - config.log_dir.as_str(), - &config, - Arc::new(Slogger { - slog: logger.clone(), - }), - )?; +#[cfg(feature = "heed_storage")] +use crate::HeedStorage; +pub fn debug_persisted(storage: impl StableStorage) -> Result<()> { let entries = storage.all_entries()?; if !entries.is_empty() { @@ -82,7 +74,27 @@ pub fn debug_persitsted_all(path_str: &str, logger: slog::Logger) -> Result<()> for name in dir_entries { println!("*----- {name} -----*"); - debug_persisted(&format!("{}/{}", path_str, name), logger.clone())?; + #[cfg(feature = "heed_storage")] + { + let config = Config { + log_dir: path_str.to_string(), + ..Default::default() + }; + let storage = HeedStorage::create( + format!("{}/{}", path_str, name).as_str(), + &config, + Arc::new(Slogger { + slog: logger.clone(), + }), + )?; + debug_persisted(storage)?; + } + + #[cfg(feature = "inmemory_storage")] + { + eprintln!("Inmemory storage does not support this feature"); + } + println!(); } } else { diff --git a/raftify/src/cli/commands/dump.rs b/raftify/src/cli/commands/dump.rs deleted file mode 100644 index 25fd718f..00000000 --- a/raftify/src/cli/commands/dump.rs +++ /dev/null @@ -1,119 +0,0 @@ -use crate::raft::{ - eraftpb::{ConfChange, ConfChangeSingle, ConfChangeType, ConfChangeV2, Entry, EntryType}, - logger::Slogger, - Storage, -}; -use bincode::{deserialize, serialize}; -use prost::Message as _; -use std::{collections::HashMap, net::SocketAddr, sync::Arc}; - -use crate::{utils::membership::to_confchange_v2, Config, HeedStorage, Result, StableStorage}; - -/// Read all_entries and make the appropriate ConfChanges to make it peers compared to the peers given in json format. -pub fn dump_peers(path: &str, peers: HashMap, logger: slog::Logger) -> Result<()> { - let config = Config { - log_dir: path.to_string(), - ..Default::default() - }; - - let mut storage = HeedStorage::create( - config.log_dir.as_str(), - &config, - Arc::new(Slogger { - slog: logger.clone(), - }), - )?; - - let entries = storage.all_entries()?; - - let mut persisted_peers_configs: HashMap = HashMap::new(); - - for entry in entries.iter() { - let conf_change_v2 = match entry.get_entry_type() { - EntryType::EntryConfChange => to_confchange_v2(ConfChange::decode(entry.get_data())?), - EntryType::EntryConfChangeV2 => ConfChangeV2::decode(entry.get_data())?, - _ => continue, - }; - - let conf_changes = conf_change_v2.get_changes(); - let addrs: Vec = deserialize(conf_change_v2.get_context())?; - - for (cc_idx, conf_change) in conf_changes.iter().enumerate() { - let addr = addrs[cc_idx]; - - let node_id = conf_change.get_node_id(); - let change_type = conf_change.get_change_type(); - - match change_type { - ConfChangeType::AddNode | ConfChangeType::AddLearnerNode => { - persisted_peers_configs.insert(node_id, addr); - } - ConfChangeType::RemoveNode => { - persisted_peers_configs.remove(&node_id); - } - } - } - } - - // old - new = to be removed - let diffs_to_remove: HashMap<_, _> = persisted_peers_configs - .iter() - .filter(|(k, v)| peers.get(k) != Some(v)) - .collect(); - - // new - old = to be added - let diffs_to_add: HashMap<_, _> = peers - .iter() - .filter(|(k, v)| persisted_peers_configs.get(k) != Some(v)) - .collect(); - - let mut new_cc_v2 = ConfChangeV2::default(); - let mut conf_changes: Vec = vec![]; - let mut cc_addrs: Vec = vec![]; - - for (k, v) in diffs_to_remove.into_iter() { - let mut cs = ConfChangeSingle::default(); - cs.set_node_id(*k); - cs.set_change_type(ConfChangeType::RemoveNode); - conf_changes.push(cs); - cc_addrs.push(*v); - } - - // TODO: Support AddLearnerNode - for (k, v) in diffs_to_add.into_iter() { - let mut cs = ConfChangeSingle::default(); - cs.set_node_id(*k); - cs.set_change_type(ConfChangeType::AddNode); - conf_changes.push(cs); - cc_addrs.push(*v); - } - - if !conf_changes.is_empty() { - new_cc_v2.set_context(serialize(&cc_addrs)?); - new_cc_v2.set_changes(conf_changes); - - let last_idx = storage.last_index()?; - let last_term = storage.term(last_idx)?; - - let mut new_entry = Entry::default(); - new_entry.set_index(last_idx + 1); - new_entry.set_term(last_term); - new_entry.set_entry_type(EntryType::EntryConfChangeV2); - new_entry.set_data(new_cc_v2.encode_to_vec()); - new_entry.set_context(vec![]); - - storage.append(vec![new_entry].as_slice())?; - - let mut snapshot = storage.snapshot(0, last_idx)?; - let mut meta = snapshot.get_metadata().clone(); - meta.set_index(last_idx + 1); - snapshot.set_metadata(meta); - storage.apply_snapshot(snapshot)?; - - println!("Changes applied successfully"); - } else { - println!("No changes to be made"); - } - - Ok(()) -} diff --git a/raftify/src/cli/commands/mod.rs b/raftify/src/cli/commands/mod.rs index e2798c6d..2f365233 100644 --- a/raftify/src/cli/commands/mod.rs +++ b/raftify/src/cli/commands/mod.rs @@ -1,3 +1 @@ pub mod debug; -pub mod dump; -pub mod utils; diff --git a/raftify/src/cli/commands/utils.rs b/raftify/src/cli/commands/utils.rs deleted file mode 100644 index dac74b20..00000000 --- a/raftify/src/cli/commands/utils.rs +++ /dev/null @@ -1,22 +0,0 @@ -use serde_json::Value; -use std::collections::HashMap; -use std::net::SocketAddr; -use std::str::FromStr; - -pub fn parse_peers_json( - peers: &str, -) -> Result, Box> { - let peers: Value = serde_json::from_str(peers)?; - - let mut result = HashMap::new(); - - if let Value::Object(peer) = peers { - for (node_id, addr) in peer { - let key = node_id.parse::()?; - let value = SocketAddr::from_str(addr.as_str().unwrap())?; - result.insert(key, value); - } - } - - Ok(result) -} diff --git a/raftify/src/cli/mod.rs b/raftify/src/cli/mod.rs index 67ba6158..4c702c2d 100644 --- a/raftify/src/cli/mod.rs +++ b/raftify/src/cli/mod.rs @@ -3,16 +3,14 @@ include!(concat!(env!("OUT_DIR"), "/built.rs")); mod commands; use clap::{Args, Parser, Subcommand}; -use commands::{ - debug::{debug_entries, debug_node, debug_persisted, debug_persitsted_all}, - dump::dump_peers, - utils::parse_peers_json, -}; -use std::fmt::Debug; +use commands::debug::{debug_entries, debug_node, debug_persisted, debug_persitsted_all}; +use raft::logger::Slogger; +use std::{fmt::Debug, sync::Arc}; use crate::{ raft::{default_logger, formatter::set_custom_formatter}, - AbstractLogEntry, AbstractStateMachine, CustomFormatter, Result, + AbstractLogEntry, AbstractStateMachine, Config, CustomFormatter, HeedStorage, Result, + StableStorage, }; #[derive(Parser)] @@ -30,8 +28,6 @@ enum Commands { /// Debug tools #[command(subcommand)] Debug(DebugSubcommands), - /// Dump tools - Dump(Dump), } #[derive(Subcommand)] @@ -68,6 +64,7 @@ struct Dump { pub async fn cli_handler< LogEntry: AbstractLogEntry + Debug + Send + 'static, + LogStorage: StableStorage + Send + Sync + Clone + 'static, FSM: AbstractStateMachine + Debug + Clone + Send + Sync + 'static, >( args: Option>, @@ -82,7 +79,26 @@ pub async fn cli_handler< match app.command { Commands::Debug(x) => match x { DebugSubcommands::Persisted { path } => { - debug_persisted(path.as_str(), logger.clone())?; + #[cfg(feature = "heed_storage")] + { + let config = Config { + log_dir: path.to_string(), + ..Default::default() + }; + let storage = HeedStorage::create( + config.log_dir.as_str(), + &config, + Arc::new(Slogger { + slog: logger.clone(), + }), + )?; + debug_persisted(storage)?; + } + + #[cfg(feature = "inmemory_storage")] + { + eprintln!("Inmemory storage does not support this feature"); + } } DebugSubcommands::PersistedAll { path } => { debug_persitsted_all(path.as_str(), logger.clone())?; @@ -94,13 +110,6 @@ pub async fn cli_handler< debug_node(address.as_str()).await?; } }, - Commands::Dump(x) => { - dump_peers( - x.path.as_str(), - parse_peers_json(x.peers.as_str()).unwrap(), - logger.clone(), - )?; - } } Ok(()) diff --git a/raftify/src/config.rs b/raftify/src/config.rs index 92c9b6a0..e5606f71 100644 --- a/raftify/src/config.rs +++ b/raftify/src/config.rs @@ -7,6 +7,7 @@ pub struct Config { pub raft_config: RaftConfig, pub log_dir: String, + pub bootstrap_from_snapshot: bool, pub save_compacted_logs: bool, pub compacted_log_dir: String, pub compacted_log_size_threshold: u64, @@ -18,8 +19,6 @@ pub struct Config { pub initial_peers: Option, pub snapshot_interval: Option, - pub restore_wal_from: Option, - pub restore_wal_snapshot_from: Option, } impl Config { @@ -33,11 +32,10 @@ impl Config { tick_interval: f32, lmdb_map_size: u64, cluster_id: String, + bootstrap_from_snapshot: bool, conf_change_request_timeout: f32, initial_peers: Option, snapshot_interval: Option, - restore_wal_from: Option, - restore_wal_snapshot_from: Option, ) -> Self { Self { raft_config, @@ -51,8 +49,7 @@ impl Config { initial_peers, cluster_id, conf_change_request_timeout, - restore_wal_from, - restore_wal_snapshot_from, + bootstrap_from_snapshot, } } } @@ -78,13 +75,6 @@ impl Config { } self.raft_config.validate()?; - if self.restore_wal_from.is_some() && self.restore_wal_snapshot_from.is_some() { - return Err(Error::ConfigInvalid( - "restore_wal_from and restore_wal_snapshot_from cannot be set at the same time" - .to_owned(), - )); - } - Ok(()) } } @@ -103,8 +93,7 @@ impl Default for Config { conf_change_request_timeout: 2.0, initial_peers: None, snapshot_interval: None, - restore_wal_from: None, - restore_wal_snapshot_from: None, + bootstrap_from_snapshot: false, } } } @@ -133,6 +122,7 @@ impl fmt::Debug for Config { max_committed_size_per_ready: {max_committed_size_per_ready}, \ }}, \ log_dir: {log_dir}, \ + bootstrap_from_snapshot: {bootstrap_from_snapshot}, \ save_compacted_logs: {save_compacted_logs}, \ compacted_log_dir: {compacted_log_dir}, \ compacted_log_size_threshold: {compacted_log_size_threshold}, \ @@ -142,8 +132,6 @@ impl fmt::Debug for Config { lmdb_map_size: {lmdb_map_size}, \ cluster_id: {cluster_id}, \ conf_change_request_timeout: {conf_change_request_timeout}, \ - restore_wal_from: {restore_wal_from:?}, \ - restore_wal_snapshot_from: {restore_wal_snapshot_from:?}, \ }}", id = self.raft_config.id, election_tick = self.raft_config.election_tick, @@ -171,8 +159,7 @@ impl fmt::Debug for Config { initial_peers = self.initial_peers, cluster_id = self.cluster_id, conf_change_request_timeout = self.conf_change_request_timeout, - restore_wal_from = self.restore_wal_from, - restore_wal_snapshot_from = self.restore_wal_snapshot_from, + bootstrap_from_snapshot = self.bootstrap_from_snapshot, ) } } diff --git a/raftify/src/lib.rs b/raftify/src/lib.rs index e1614d77..ebd00958 100644 --- a/raftify/src/lib.rs +++ b/raftify/src/lib.rs @@ -37,8 +37,13 @@ pub use crate::{ raft_service::raft_service_client::RaftServiceClient, request::common::confchange_request::ConfChangeRequest, state_machine::AbstractStateMachine, - storage::heed_storage::HeedStorage, storage::StableStorage, }; +#[cfg(feature = "heed_storage")] +pub use storage::heed_storage::HeedStorage; + +#[cfg(feature = "inmemory_storage")] +pub use storage::inmemory_storage::MemStorage; + pub(crate) use crate::utils::macros::macro_utils; diff --git a/raftify/src/raft_bootstrapper.rs b/raftify/src/raft_bootstrapper.rs index 59e37134..8815b435 100644 --- a/raftify/src/raft_bootstrapper.rs +++ b/raftify/src/raft_bootstrapper.rs @@ -1,6 +1,6 @@ use crate::{ raft::logger::Logger, request::server_request_message::ServerRequestMsg, ClusterJoinTicket, - InitialRole, Peers, + InitialRole, Peers, StableStorage, }; use bincode::deserialize; use std::{net::ToSocketAddrs, ops::Deref, sync::Arc}; @@ -22,25 +22,35 @@ use super::{ /// The bootstrap function returns an instance of the Raft type that deref to RaftNode type, /// allowing the use of functions necessary for interaction with the cluster. #[derive(Clone)] -pub struct Raft { - pub raft_node: RaftNode, - pub raft_server: RaftServer, - pub tx_server: mpsc::Sender>, +pub struct Raft< + LogEntry: AbstractLogEntry + 'static, + LogStorage: StableStorage + Send + Clone + 'static, + FSM: AbstractStateMachine + Clone + 'static, +> { + pub raft_node: RaftNode, + pub raft_server: RaftServer, + pub tx_server: mpsc::Sender>, pub logger: Arc, } -impl Deref - for Raft +impl< + LogEntry: AbstractLogEntry + 'static, + LogStorage: StableStorage + Send + Clone + 'static, + FSM: AbstractStateMachine + Clone + 'static, + > Deref for Raft { - type Target = RaftNode; + type Target = RaftNode; fn deref(&self) -> &Self::Target { &self.raft_node } } -impl - Raft +impl< + LogEntry: AbstractLogEntry, + LogStorage: StableStorage + Send + Sync + Clone + 'static, + FSM: AbstractStateMachine + Send + Sync + Clone + 'static, + > Raft { /// Creates a new Raft instance. /// Cloning a Raft instance does not bootstrap a new Raft instance. @@ -48,6 +58,7 @@ impl( node_id: u64, raft_addr: A, + log_storage: LogStorage, fsm: FSM, config: Config, logger: Arc, @@ -76,6 +87,7 @@ impl { // The lock of RaftNodeCore is locked when RaftNode.run is called and is never released until program terminates. // However, to implement the Clone trait in RaftNode, Arc and Mutex are necessary. That's why we use OneShotMutex here. - inner: Arc>>, + inner: Arc>>, // RaftNode.(method_call) >>> RaftNodeCore.run - tx_local: mpsc::Sender>, + tx_local: mpsc::Sender>, } impl< LogEntry: AbstractLogEntry + Send + 'static, + LogStorage: StableStorage + Send + Clone + 'static, FSM: AbstractStateMachine + Clone + Send + 'static, - > RaftNode + > RaftNode { #[allow(clippy::too_many_arguments)] pub fn bootstrap( node_id: u64, should_be_leader: bool, + log_storage: LogStorage, fsm: FSM, config: Config, raft_addr: SocketAddr, logger: Arc, - tx_server: mpsc::Sender>, - rx_server: mpsc::Receiver>, + tx_server: mpsc::Sender>, + rx_server: mpsc::Receiver>, ) -> Result { let (tx_local, rx_local) = mpsc::channel(100); - RaftNodeCore::::bootstrap( + RaftNodeCore::::bootstrap( node_id, should_be_leader, + log_storage, fsm, config, raft_addr, @@ -231,7 +224,7 @@ impl< } } - pub async fn storage(&self) -> HeedStorage { + pub async fn storage(&self) -> LogStorage { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetStorage { tx_msg: tx }) @@ -456,7 +449,7 @@ impl< /// # Safety /// TODO: Write this. - pub async unsafe fn get_raw_node(&self) -> Arc>> { + pub async unsafe fn get_raw_node(&self) -> Arc>> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetRawNode { tx_msg: tx }) @@ -482,12 +475,12 @@ impl< pub struct RaftNodeCore< LogEntry: AbstractLogEntry + Send + 'static, + LogStorage: StableStorage + Send + Clone + 'static, FSM: AbstractStateMachine + Clone + 'static, > { - #[cfg(feature = "heed_storage")] - pub raw_node: RawNode, - #[cfg(feature = "inmemory_storage")] - pub raw_node: RawNode, + pub raw_node: RawNode, + pub log_storage: LogStorage, + pub fsm: FSM, pub peers: Arc>, response_seq: AtomicU64, @@ -496,15 +489,15 @@ pub struct RaftNodeCore< should_exit: bool, last_snapshot_created: Instant, logger: Arc, - response_senders: HashMap>, + response_senders: HashMap>, #[allow(dead_code)] - tx_server: mpsc::Sender>, - rx_server: mpsc::Receiver>, + tx_server: mpsc::Sender>, + rx_server: mpsc::Receiver>, #[allow(dead_code)] - tx_local: mpsc::Sender>, - rx_local: mpsc::Receiver>, + tx_local: mpsc::Sender>, + rx_local: mpsc::Receiver>, tx_self: mpsc::Sender, rx_self: mpsc::Receiver, @@ -514,39 +507,30 @@ pub struct RaftNodeCore< impl< LogEntry: AbstractLogEntry + Send + 'static, + LogStorage: StableStorage + Send + Clone + 'static, FSM: AbstractStateMachine + Clone + Send + 'static, - > RaftNodeCore + > RaftNodeCore { #[allow(clippy::too_many_arguments)] pub fn bootstrap( node_id: u64, should_be_leader: bool, + mut log_storage: LogStorage, fsm: FSM, mut config: Config, raft_addr: SocketAddr, logger: Arc, - tx_server: mpsc::Sender>, - rx_server: mpsc::Receiver>, - tx_local: mpsc::Sender>, - rx_local: mpsc::Receiver>, + tx_server: mpsc::Sender>, + rx_server: mpsc::Receiver>, + tx_local: mpsc::Sender>, + rx_local: mpsc::Receiver>, ) -> Result { config.raft_config.id = node_id; config.validate()?; - let storage_pth = get_storage_path(config.log_dir.as_str(), node_id); + let mut snapshot = log_storage.snapshot(0, log_storage.last_index()?)?; - if let (None, None) = (config.restore_wal_from, config.restore_wal_snapshot_from) { - clear_storage_path(storage_pth.as_str())?; - ensure_directory_exist(storage_pth.as_str())?; - }; - - #[cfg(feature = "heed_storage")] - let mut storage = HeedStorage::create(storage_pth.as_str(), &config, logger.clone())?; - - #[cfg(feature = "inmemory_storage")] - let mut storage = MemStorage::create(); - - let mut snapshot = storage.snapshot(0, storage.last_index()?)?; + let last_idx = log_storage.last_index()?; let conf_state = snapshot.mut_metadata().mut_conf_state(); @@ -576,33 +560,11 @@ impl< conf_state.set_learners(learners); } - match (config.restore_wal_from, config.restore_wal_snapshot_from) { - (Some(restore_wal_from), None) => { - if restore_wal_from != node_id { - std::fs::copy( - get_data_mdb_path(config.log_dir.as_str(), restore_wal_from), - get_data_mdb_path(config.log_dir.as_str(), node_id), - )?; - } - } - (None, Some(restore_wal_snapshot_from)) => { - if restore_wal_snapshot_from != node_id { - std::fs::copy( - get_data_mdb_path(config.log_dir.as_str(), restore_wal_snapshot_from), - get_data_mdb_path(config.log_dir.as_str(), node_id), - )?; - } - storage.apply_snapshot(snapshot)?; - } - (Some(_), Some(_)) => { - unreachable!() - } - _ => { - storage.apply_snapshot(snapshot)?; - } + if last_idx == 0 || config.bootstrap_from_snapshot { + log_storage.apply_snapshot(snapshot)?; } - let mut raw_node = RawNode::new(&config.raft_config, storage, logger.clone())?; + let mut raw_node = RawNode::new(&config.raft_config, log_storage.clone(), logger.clone())?; let response_seq = AtomicU64::new(0); let last_snapshot_created = Instant::now(); @@ -615,6 +577,7 @@ impl< Ok(RaftNodeCore { raw_node, + log_storage, fsm, response_seq, config, @@ -727,7 +690,7 @@ impl< async fn send_messages(&mut self, messages: Vec) { for message in messages { - tokio::spawn(RaftNodeCore::::send_message( + tokio::spawn(RaftNodeCore::::send_message( message, self.peers.clone(), self.tx_self.clone(), @@ -961,7 +924,7 @@ impl< async fn handle_propose_request( &mut self, proposal: Vec, - response_sender: ResponseSender, + response_sender: ResponseSender, ) -> Result<()> { if !self.is_leader() { let leader_id = self.get_leader_id(); @@ -981,7 +944,7 @@ impl< .addr .to_string(); - let raft_response: ResponseMessage = match response_sender { + let raft_response: ResponseMessage = match response_sender { ResponseSender::Local(_) => LocalResponseMsg::Propose { result: ResponseResult::WrongLeader { leader_id, @@ -1021,7 +984,7 @@ impl< async fn handle_confchange_request( &mut self, conf_change: ConfChangeV2, - response_sender: ResponseSender, + response_sender: ResponseSender, ) -> Result<()> { if self.raw_node.raft.has_pending_conf() { self.logger.warn(&format!("Reject the conf change because pending conf change exist! (pending_conf_index={}), try later...", self.raw_node.raft.pending_conf_index)); @@ -1085,7 +1048,7 @@ impl< async fn handle_local_request_msg( &mut self, - message: LocalRequestMsg, + message: LocalRequestMsg, ) -> Result<()> { match message { LocalRequestMsg::IsLeader { tx_msg } => { @@ -1135,7 +1098,6 @@ impl< .unwrap(); } LocalRequestMsg::GetStorage { tx_msg } => { - #[cfg(feature = "heed_storage")] tx_msg .send(LocalResponseMsg::GetStorage { storage: self.raw_node.store().clone(), @@ -1254,7 +1216,7 @@ impl< async fn handle_server_request_msg( &mut self, - message: ServerRequestMsg, + message: ServerRequestMsg, ) -> Result<()> { match message { ServerRequestMsg::ChangeConfig { @@ -1363,6 +1325,7 @@ impl< } ServerRequestMsg::_Phantom(_) => unreachable!(), ServerRequestMsg::_Phantom2(_) => unreachable!(), + ServerRequestMsg::_Phantom3(_) => unreachable!(), } Ok(()) @@ -1414,13 +1377,14 @@ impl< async fn on_ready(&mut self) -> Result<()> { // TODO: Improve this logic. // I'd like to move this logic into RaftNodeCore::bootstrap, but it's currently not possible because it is not async. - if self.config.restore_wal_snapshot_from.is_some() { - self.logger.info("Restoring state machine from snapshot..."); - let store = self.raw_node.store(); - let snap = store.snapshot(0, store.last_index()?)?; - self.fsm.restore(snap.get_data().to_vec()).await?; - self.config.restore_wal_snapshot_from = None; - } + + // if self.config.restore_wal_snapshot_from.is_some() { + // self.logger.info("Restoring state machine from snapshot..."); + // let store = self.raw_node.store(); + // let snap = store.snapshot(0, store.last_index()?)?; + // self.fsm.restore(snap.get_data().to_vec()).await?; + // self.config.restore_wal_snapshot_from = None; + // } if !self.raw_node.has_ready() { return Ok(()); @@ -1448,6 +1412,7 @@ impl< if !ready.entries().is_empty() { let entries = &ready.entries()[..]; let store = self.raw_node.mut_store(); + println!("Applying entries: {:?}", entries); store.append(entries)?; } diff --git a/raftify/src/raft_node/response_sender.rs b/raftify/src/raft_node/response_sender.rs index e267a913..efc74841 100644 --- a/raftify/src/raft_node/response_sender.rs +++ b/raftify/src/raft_node/response_sender.rs @@ -5,16 +5,22 @@ use crate::{ local_response_message::LocalResponseMsg, server_response_message::ServerResponseMsg, ResponseMessage, }, - AbstractLogEntry, AbstractStateMachine, + AbstractLogEntry, AbstractStateMachine, StableStorage, }; -pub(crate) enum ResponseSender { - Local(oneshot::Sender>), +pub(crate) enum ResponseSender< + LogEntry: AbstractLogEntry, + LogStorage: StableStorage + 'static, + FSM: AbstractStateMachine, +> { + Local(oneshot::Sender>), Server(oneshot::Sender), } -impl ResponseSender { - pub fn send(self, response: ResponseMessage) { +impl + ResponseSender +{ + pub fn send(self, response: ResponseMessage) { match self { ResponseSender::Local(tx_local) => { if let ResponseMessage::Local(response) = response { diff --git a/raftify/src/raft_server.rs b/raftify/src/raft_server.rs index 8ba26b74..f3718b2c 100644 --- a/raftify/src/raft_server.rs +++ b/raftify/src/raft_server.rs @@ -31,22 +31,29 @@ use crate::{ response::server_response_message::{ ConfChangeResponseResult, RequestIdResponseResult, ResponseResult, ServerResponseMsg, }, - AbstractLogEntry, AbstractStateMachine, + AbstractLogEntry, AbstractStateMachine, StableStorage, }; #[derive(Clone)] -pub struct RaftServer { - tx: mpsc::Sender>, +pub struct RaftServer< + LogEntry: AbstractLogEntry, + LogStorage: StableStorage + 'static, + FSM: AbstractStateMachine, +> { + tx: mpsc::Sender>, raft_addr: SocketAddr, config: Config, logger: Arc, } -impl - RaftServer +impl< + LogEntry: AbstractLogEntry + 'static, + LogStorage: StableStorage + Send + Sync + 'static, + FSM: AbstractStateMachine + 'static, + > RaftServer { pub fn new( - tx: mpsc::Sender>, + tx: mpsc::Sender>, raft_addr: A, config: Config, logger: Arc, @@ -81,8 +88,11 @@ impl } } -impl - RaftServer +impl< + LogEntry: AbstractLogEntry + 'static, + LogStorage: StableStorage + 'static, + FSM: AbstractStateMachine + 'static, + > RaftServer { fn print_send_error(&self, function_name: &str) { self.logger.error(&format!( @@ -93,8 +103,11 @@ impl } #[tonic::async_trait] -impl RaftService - for RaftServer +impl< + LogEntry: AbstractLogEntry + 'static, + LogStorage: StableStorage + Sync + Send + 'static, + FSM: AbstractStateMachine + 'static, + > RaftService for RaftServer { async fn request_id( &self, diff --git a/raftify/src/request/local_request_message.rs b/raftify/src/request/local_request_message.rs index 84427cec..3eb9cf1e 100644 --- a/raftify/src/request/local_request_message.rs +++ b/raftify/src/request/local_request_message.rs @@ -1,6 +1,6 @@ use std::{collections::HashMap, net::SocketAddr}; -use crate::raft::eraftpb::Message as RaftMessage; +use crate::{raft::eraftpb::Message as RaftMessage, StableStorage}; use tokio::sync::oneshot::Sender; use crate::{ @@ -12,82 +12,86 @@ use super::common::confchange_request::ConfChangeRequest; /// Request type used for communication (method calls) between user side and RaftNode #[derive(Debug)] -pub enum LocalRequestMsg { +pub enum LocalRequestMsg< + LogEntry: AbstractLogEntry, + LogStorage: StableStorage + 'static, + FSM: AbstractStateMachine, +> { IsLeader { - tx_msg: Sender>, + tx_msg: Sender>, }, GetId { - tx_msg: Sender>, + tx_msg: Sender>, }, GetLeaderId { - tx_msg: Sender>, + tx_msg: Sender>, }, GetPeers { - tx_msg: Sender>, + tx_msg: Sender>, }, AddPeer { id: u64, addr: String, - tx_msg: Sender>, + tx_msg: Sender>, role: Option, }, AddPeers { peers: HashMap, - tx_msg: Sender>, + tx_msg: Sender>, }, DebugNode { - tx_msg: Sender>, + tx_msg: Sender>, }, GetStateMachine { - tx_msg: Sender>, + tx_msg: Sender>, }, GetStorage { - tx_msg: Sender>, + tx_msg: Sender>, }, GetClusterSize { - tx_msg: Sender>, + tx_msg: Sender>, }, Quit { - tx_msg: Sender>, + tx_msg: Sender>, }, Campaign { - tx_msg: Sender>, + tx_msg: Sender>, }, Demote { term: u64, leader_id: u64, - tx_msg: Sender>, + tx_msg: Sender>, }, TransferLeader { node_id: u64, - tx_msg: Sender>, + tx_msg: Sender>, }, Leave { - tx_msg: Sender>, + tx_msg: Sender>, }, MakeSnapshot { index: u64, term: u64, - tx_msg: Sender>, + tx_msg: Sender>, }, Propose { proposal: Vec, - tx_msg: Sender>, + tx_msg: Sender>, }, ChangeConfig { conf_change: ConfChangeRequest, - tx_msg: Sender>, + tx_msg: Sender>, }, SendMessage { message: Box, - tx_msg: Sender>, + tx_msg: Sender>, }, JoinCluster { tickets: Vec, - tx_msg: Sender>, + tx_msg: Sender>, }, LeaveJoint {}, GetRawNode { - tx_msg: Sender>, + tx_msg: Sender>, }, } diff --git a/raftify/src/request/server_request_message.rs b/raftify/src/request/server_request_message.rs index 3805997b..e01ff62f 100644 --- a/raftify/src/request/server_request_message.rs +++ b/raftify/src/request/server_request_message.rs @@ -4,14 +4,18 @@ use tokio::sync::oneshot::Sender; use crate::{ raft::eraftpb::Message as RaftMessage, response::server_response_message::ServerResponseMsg, - AbstractLogEntry, AbstractStateMachine, Peers, + AbstractLogEntry, AbstractStateMachine, Peers, StableStorage, }; use super::common::confchange_request::ConfChangeRequest; /// Request type processed through network calls (gRPC) #[derive(Debug)] -pub enum ServerRequestMsg { +pub enum ServerRequestMsg< + LogEntry: AbstractLogEntry, + LogStorage: StableStorage, + FSM: AbstractStateMachine, +> { RequestId { raft_addr: String, tx_msg: Sender, @@ -45,4 +49,5 @@ pub enum ServerRequestMsg }, _Phantom(PhantomData), _Phantom2(PhantomData), + _Phantom3(PhantomData), } diff --git a/raftify/src/response/local_response_message.rs b/raftify/src/response/local_response_message.rs index 17c0666c..2b816333 100644 --- a/raftify/src/response/local_response_message.rs +++ b/raftify/src/response/local_response_message.rs @@ -1,16 +1,20 @@ use std::{fmt, marker::PhantomData, sync::Arc}; -use crate::raft::RawNode; +use crate::{raft::RawNode, StableStorage}; use tokio::sync::Mutex; -use crate::{AbstractLogEntry, AbstractStateMachine, HeedStorage, Peers}; +use crate::{AbstractLogEntry, AbstractStateMachine, Peers}; use super::{ server_response_message::{ConfChangeResponseResult, ResponseResult}, ResponseMessage, }; -pub enum LocalResponseMsg { +pub enum LocalResponseMsg< + LogEntry: AbstractLogEntry, + LogStorage: StableStorage + 'static, + FSM: AbstractStateMachine, +> { IsLeader { is_leader: bool, }, @@ -29,13 +33,13 @@ pub enum LocalResponseMsg store: FSM, }, GetStorage { - storage: HeedStorage, + storage: LogStorage, }, GetClusterSize { size: usize, }, GetRawNode { - raw_node: Arc>>, + raw_node: Arc>>, }, Quit {}, Campaign {}, @@ -59,8 +63,8 @@ pub enum LocalResponseMsg }, } -impl fmt::Debug - for LocalResponseMsg +impl fmt::Debug + for LocalResponseMsg { #[allow(clippy::recursive_format_impl)] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -75,10 +79,11 @@ impl fmt::Debug } } -impl From> - for ResponseMessage +impl + From> + for ResponseMessage { - fn from(msg: LocalResponseMsg) -> Self { + fn from(msg: LocalResponseMsg) -> Self { ResponseMessage::Local(msg) } } diff --git a/raftify/src/response/mod.rs b/raftify/src/response/mod.rs index ae9bb3d4..81e9967d 100644 --- a/raftify/src/response/mod.rs +++ b/raftify/src/response/mod.rs @@ -1,14 +1,18 @@ use std::marker::PhantomData; -use crate::{AbstractLogEntry, AbstractStateMachine}; +use crate::{AbstractLogEntry, AbstractStateMachine, StableStorage}; use self::{local_response_message::LocalResponseMsg, server_response_message::ServerResponseMsg}; pub mod local_response_message; pub mod server_response_message; -pub enum ResponseMessage { +pub enum ResponseMessage< + LogEntry: AbstractLogEntry, + LogStorage: StableStorage + 'static, + FSM: AbstractStateMachine, +> { Server(ServerResponseMsg), - Local(LocalResponseMsg), + Local(LocalResponseMsg), _Phantom(PhantomData), } diff --git a/raftify/src/response/server_response_message.rs b/raftify/src/response/server_response_message.rs index 61795a12..c9d0f5ce 100644 --- a/raftify/src/response/server_response_message.rs +++ b/raftify/src/response/server_response_message.rs @@ -1,4 +1,4 @@ -use crate::{AbstractLogEntry, AbstractStateMachine, Error, Peers}; +use crate::{AbstractLogEntry, AbstractStateMachine, Error, Peers, StableStorage}; use super::ResponseMessage; @@ -54,8 +54,8 @@ pub enum ServerResponseMsg { RequestId { result: RequestIdResponseResult }, } -impl From - for ResponseMessage +impl + From for ResponseMessage { fn from(msg: ServerResponseMsg) -> Self { ResponseMessage::Server(msg) diff --git a/raftify/src/storage/heed_storage/mod.rs b/raftify/src/storage/heed_storage/mod.rs index 91c3569e..04f513fd 100644 --- a/raftify/src/storage/heed_storage/mod.rs +++ b/raftify/src/storage/heed_storage/mod.rs @@ -1,6 +1,5 @@ mod codec; mod constant; -pub mod utils; use bincode::{deserialize, serialize}; use constant::{CONF_STATE_KEY, HARD_STATE_KEY, LAST_INDEX_KEY, SNAPSHOT_KEY}; diff --git a/raftify/src/storage/heed_storage/utils.rs b/raftify/src/storage/heed_storage/utils.rs deleted file mode 100644 index a2c26255..00000000 --- a/raftify/src/storage/heed_storage/utils.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub fn get_storage_path(log_dir: &str, node_id: u64) -> String { - format!("{}/node-{}", log_dir, node_id) -} - -pub fn get_data_mdb_path(log_dir: &str, node_id: u64) -> String { - format!("{}/data.mdb", get_storage_path(log_dir, node_id)) -} diff --git a/raftify/src/storage/utils.rs b/raftify/src/storage/utils.rs index a266cf16..727eeb9c 100644 --- a/raftify/src/storage/utils.rs +++ b/raftify/src/storage/utils.rs @@ -1,34 +1,12 @@ use chrono::Utc; use serde_json::{json, Value}; use std::{ - fs::{self, File, OpenOptions}, + fs::{File, OpenOptions}, io::{self, Read, Seek, Write as StdIoWrite}, path::Path, }; -use crate::{ - raft::{eraftpb::Entry, formatter::Bytes, formatter::CUSTOM_FORMATTER}, - Result, -}; - -pub fn clear_storage_path(log_dir_path: &str) -> Result<()> { - let log_dir_path = Path::new(&log_dir_path); - - if fs::metadata(log_dir_path).is_ok() { - fs::remove_dir_all(log_dir_path)?; - } - - Ok(()) -} - -pub fn ensure_directory_exist(dir_pth: &str) -> Result<()> { - let dir_pth: &Path = Path::new(&dir_pth); - - if fs::metadata(dir_pth).is_err() { - fs::create_dir_all(dir_pth)?; - } - Ok(()) -} +use crate::raft::{eraftpb::Entry, formatter::Bytes, formatter::CUSTOM_FORMATTER}; fn entry_type_to_str(entry_type: i32) -> &'static str { match entry_type { From 1714cd0f4e170bba234ef98a9e5abcf860019f58 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Fri, 27 Sep 2024 15:50:12 +0900 Subject: [PATCH 08/19] wip --- examples/memstore/src/cli/main.rs | 8 +------- examples/memstore/src/state_machine.rs | 4 ++-- raftify/src/raft_node/mod.rs | 11 ----------- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/examples/memstore/src/cli/main.rs b/examples/memstore/src/cli/main.rs index b3491fd9..c2a4ef59 100644 --- a/examples/memstore/src/cli/main.rs +++ b/examples/memstore/src/cli/main.rs @@ -1,12 +1,6 @@ use raftify::{cli::cli_handler, Result}; -use memstore_example_harness::state_machine::{HashStore, LogEntry}; - -#[cfg(feature = "inmemory_storage")] -use raftify::MemStorage as StorageType; - -#[cfg(feature = "heed_storage")] -use raftify::HeedStorage as StorageType; +use memstore_example_harness::state_machine::{HashStore, LogEntry, StorageType}; #[tokio::main] async fn main() -> Result<()> { diff --git a/examples/memstore/src/state_machine.rs b/examples/memstore/src/state_machine.rs index c2e57ec0..05928b10 100644 --- a/examples/memstore/src/state_machine.rs +++ b/examples/memstore/src/state_machine.rs @@ -8,10 +8,10 @@ use std::{ }; #[cfg(feature = "inmemory_storage")] -use raftify::MemStorage as StorageType; +pub use raftify::MemStorage as StorageType; #[cfg(feature = "heed_storage")] -use raftify::HeedStorage as StorageType; +pub use raftify::HeedStorage as StorageType; pub type Raft = Raft_; diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index 6ff3b613..255bc8fb 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -1375,17 +1375,6 @@ impl< } async fn on_ready(&mut self) -> Result<()> { - // TODO: Improve this logic. - // I'd like to move this logic into RaftNodeCore::bootstrap, but it's currently not possible because it is not async. - - // if self.config.restore_wal_snapshot_from.is_some() { - // self.logger.info("Restoring state machine from snapshot..."); - // let store = self.raw_node.store(); - // let snap = store.snapshot(0, store.last_index()?)?; - // self.fsm.restore(snap.get_data().to_vec()).await?; - // self.config.restore_wal_snapshot_from = None; - // } - if !self.raw_node.has_ready() { return Ok(()); } From 962409666c7c19305d3d7f88fbd694cc8456818b Mon Sep 17 00:00:00 2001 From: Gyubong Date: Fri, 27 Sep 2024 15:58:12 +0900 Subject: [PATCH 09/19] wip --- Cargo.lock | 97 +++++++++++++++++++++++++ Cargo.toml | 2 + harness/src/raft.rs | 30 ++++++-- harness/src/utils.rs | 19 ++++- raftify/src/storage/heed_storage/mod.rs | 21 +++--- 5 files changed, 153 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d56e04cc..3be7bada 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -636,6 +636,26 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +[[package]] +name = "const_format" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c655d81ff1114fb0dcdea9225ea9f0cc712a6f8d189378e82bdf62a473a64b" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff1a44b93f47b1bac19a27932f5c591e43d1ba357ee4f61526c8a25603f0eb1" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -848,6 +868,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.30" @@ -855,6 +890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -863,6 +899,34 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "futures-sink" version = "0.3.30" @@ -881,10 +945,16 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -954,6 +1024,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "harness" +version = "0.1.69" +dependencies = [ + "actix-rt", + "actix-web", + "bincode", + "const_format", + "futures", + "log", + "raftify", + "serde", + "slog", + "slog-async", + "slog-envlogger", + "slog-term", + "structopt", + "tokio", + "toml 0.8.19", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2585,6 +2676,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "url" version = "2.5.2" diff --git a/Cargo.toml b/Cargo.toml index 84cab6b9..cff15e7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "raftify", + "harness", "examples", "examples/memstore", "examples/memstore/dynamic-members", @@ -9,6 +10,7 @@ members = [ ] default-members = [ "raftify", + "harness", "examples", "examples/memstore", "examples/memstore/dynamic-members", diff --git a/harness/src/raft.rs b/harness/src/raft.rs index 8f29e65d..f21662eb 100644 --- a/harness/src/raft.rs +++ b/harness/src/raft.rs @@ -1,8 +1,7 @@ use futures::future; use raftify::{ raft::{formatter::set_custom_formatter, logger::Slogger}, - CustomFormatter, Peers, Raft as Raft_, Result, - HeedStorage, + CustomFormatter, HeedStorage, Peers, Raft as Raft_, Result, }; use std::{ collections::HashMap, @@ -14,7 +13,7 @@ use crate::{ config::build_config, logger::get_logger, state_machine::{HashStore, LogEntry}, - utils::build_logger, + utils::{build_logger, ensure_directory_exist, get_storage_path}, }; pub type Raft = Raft_; @@ -61,10 +60,21 @@ fn run_raft( let store = HashStore::new(); let logger = build_logger(); + let storage_pth = get_storage_path(cfg.log_dir.as_str(), 1); + ensure_directory_exist(storage_pth.as_str())?; + + let storage = HeedStorage::create( + &storage_pth, + &cfg, + Arc::new(Slogger { + slog: logger.clone(), + }), + )?; let raft = Raft::bootstrap( *node_id, peer.addr, + storage, store, cfg, Arc::new(Slogger { @@ -133,7 +143,12 @@ pub async fn spawn_extra_node( let cfg = build_config(); let store = HashStore::new(); - let raft = Raft::bootstrap(node_id, raft_addr, store, cfg, logger).expect("Raft build failed!"); + let storage_pth = get_storage_path(cfg.log_dir.as_str(), 1); + ensure_directory_exist(storage_pth.as_str())?; + + let storage = HeedStorage::create(&storage_pth, &cfg, logger.clone())?; + let raft = Raft::bootstrap(node_id, raft_addr, storage, store, cfg, logger) + .expect("Raft build failed!"); tx_initialized_raft .send((node_id, raft.clone())) @@ -161,7 +176,12 @@ pub async fn spawn_and_join_extra_node( cfg.initial_peers = Some(join_ticket.peers.clone().into()); let store = HashStore::new(); - let raft = Raft::bootstrap(node_id, raft_addr, store, cfg, logger).expect("Raft build failed!"); + let storage_pth = get_storage_path(cfg.log_dir.as_str(), 1); + ensure_directory_exist(storage_pth.as_str())?; + + let storage = HeedStorage::create(&storage_pth, &cfg, logger.clone())?; + let raft = Raft::bootstrap(node_id, raft_addr, storage, store, cfg, logger) + .expect("Raft build failed!"); tx_initialized_raft .send((node_id, raft.clone())) diff --git a/harness/src/utils.rs b/harness/src/utils.rs index a42570be..679c1f9e 100644 --- a/harness/src/utils.rs +++ b/harness/src/utils.rs @@ -1,4 +1,4 @@ -use raftify::{InitialRole, Peers}; +use raftify::{Error, InitialRole, Peers}; use serde::Deserialize; use slog::{o, Drain}; use slog_envlogger::LogBuilder; @@ -180,3 +180,20 @@ pub fn kill_previous_raft_processes() { kill_process_using_port(*port); }); } + +pub fn get_storage_path(log_dir: &str, node_id: u64) -> String { + format!("{}/node-{}", log_dir, node_id) +} + +pub fn get_data_mdb_path(log_dir: &str, node_id: u64) -> String { + format!("{}/data.mdb", get_storage_path(log_dir, node_id)) +} + +pub fn ensure_directory_exist(dir_pth: &str) -> Result<(), Error> { + let dir_pth: &Path = Path::new(&dir_pth); + + if fs::metadata(dir_pth).is_err() { + fs::create_dir_all(dir_pth)?; + } + Ok(()) +} diff --git a/raftify/src/storage/heed_storage/mod.rs b/raftify/src/storage/heed_storage/mod.rs index 04f513fd..db2716b3 100644 --- a/raftify/src/storage/heed_storage/mod.rs +++ b/raftify/src/storage/heed_storage/mod.rs @@ -9,7 +9,7 @@ use heed::{ }; use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use prost::Message as PMessage; -use raft::{logger::Logger, util::limit_size, INVALID_INDEX}; +use raft::{logger::Logger, util::limit_size}; use std::{ cmp::max, fs, @@ -23,7 +23,6 @@ use crate::{ config::Config, error::Result, raft::{self, prelude::*, GetEntriesContext}, - Error, }; #[derive(Clone)] @@ -128,12 +127,13 @@ impl StableStorage for HeedStorage { let metadata = snapshot.get_metadata(); let conf_state = metadata.get_conf_state(); - let first_index = store.first_index(&writer)?; - if metadata.index != INVALID_INDEX && first_index > metadata.index { - return Err(Error::RaftStorageError( - raft::StorageError::SnapshotOutOfDate, - )); - } + // TODO: Investigate if this is necessary. It broke the static bootstrap. + // let first_index = store.first_index(&writer)?; + // if first_index > metadata.index { + // return Err(Error::RaftStorageError( + // raft::StorageError::SnapshotOutOfDate, + // )); + // } let mut hard_state = store.hard_state(&writer)?; hard_state.set_term(max(hard_state.term, metadata.term)); @@ -937,8 +937,9 @@ mod test { storage.apply_snapshot(snap).unwrap(); // Apply snapshot fails due to StorageError::SnapshotOutOfDate - let snap = new_snapshot(3, 3, nodes); - storage.apply_snapshot(snap).unwrap_err(); + // TODO: Support the below test case + // let snap = new_snapshot(3, 3, nodes); + // storage.apply_snapshot(snap).unwrap_err(); teardown(tempdir); } From bc463fc3edcd52908ca53705efe8f83a3eeb66c3 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Fri, 27 Sep 2024 16:04:17 +0900 Subject: [PATCH 10/19] wip --- examples/memstore/src/web_server_api.rs | 3 +-- raftify/src/cli/commands/debug.rs | 2 +- raftify/src/cli/mod.rs | 9 +++++++-- raftify/src/raft_node/mod.rs | 5 ++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/memstore/src/web_server_api.rs b/examples/memstore/src/web_server_api.rs index d06424cf..143004e1 100644 --- a/examples/memstore/src/web_server_api.rs +++ b/examples/memstore/src/web_server_api.rs @@ -1,8 +1,7 @@ use std::collections::HashMap; use actix_web::{get, put, web, Responder}; -use raftify::raft::Storage; -use raftify::{AbstractLogEntry, StableStorage}; +use raftify::{raft::Storage, AbstractLogEntry, StableStorage}; use serde_json::Value; use crate::state_machine::{HashStore, LogEntry, Raft}; diff --git a/raftify/src/cli/commands/debug.rs b/raftify/src/cli/commands/debug.rs index 2f2a8e2f..fdbe76c3 100644 --- a/raftify/src/cli/commands/debug.rs +++ b/raftify/src/cli/commands/debug.rs @@ -13,7 +13,7 @@ use crate::{ }; #[cfg(feature = "inmemory_storage")] -use raftify::MemStorage; +use crate::MemStorage; #[cfg(feature = "heed_storage")] use crate::HeedStorage; diff --git a/raftify/src/cli/mod.rs b/raftify/src/cli/mod.rs index 4c702c2d..5d607fc4 100644 --- a/raftify/src/cli/mod.rs +++ b/raftify/src/cli/mod.rs @@ -9,10 +9,15 @@ use std::{fmt::Debug, sync::Arc}; use crate::{ raft::{default_logger, formatter::set_custom_formatter}, - AbstractLogEntry, AbstractStateMachine, Config, CustomFormatter, HeedStorage, Result, - StableStorage, + AbstractLogEntry, AbstractStateMachine, Config, CustomFormatter, Result, StableStorage, }; +#[cfg(feature = "inmemory_storage")] +use crate::MemStorage; + +#[cfg(feature = "heed_storage")] +use crate::HeedStorage; + #[derive(Parser)] #[command(name = "raftify")] #[command(version = PKG_VERSION)] diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index 255bc8fb..f672e4e2 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -479,8 +479,7 @@ pub struct RaftNodeCore< FSM: AbstractStateMachine + Clone + 'static, > { pub raw_node: RawNode, - pub log_storage: LogStorage, - + // pub log_storage: LogStorage, pub fsm: FSM, pub peers: Arc>, response_seq: AtomicU64, @@ -577,7 +576,6 @@ impl< Ok(RaftNodeCore { raw_node, - log_storage, fsm, response_seq, config, @@ -594,6 +592,7 @@ impl< tx_self, rx_self, _phantom_log_entry_typ: PhantomData, + // log_storage, }) } From fed08e0fd3ecdb4134c540281d5ec00b33c55cc3 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Fri, 27 Sep 2024 17:11:39 +0900 Subject: [PATCH 11/19] wip --- Cargo.lock | 12 +- Cargo.toml | 4 +- examples/memstore/src/cli/main.rs | 5 +- raftify-cli/Cargo.lock | 2119 +++++++++++++++++ raftify-cli/Cargo.toml | 34 + raftify-cli/build.rs | 4 + .../cli => raftify-cli/src}/commands/debug.rs | 55 +- .../cli => raftify-cli/src}/commands/mod.rs | 0 {raftify/src/cli => raftify-cli/src}/mod.rs | 38 +- raftify/Cargo.toml | 2 +- raftify/src/lib.rs | 7 +- raftify/src/storage/heed_storage/mod.rs | 6 +- raftify/src/storage/inmemory_storage/mod.rs | 8 +- raftify/src/storage/mod.rs | 8 + 14 files changed, 2222 insertions(+), 80 deletions(-) create mode 100644 raftify-cli/Cargo.lock create mode 100644 raftify-cli/Cargo.toml create mode 100644 raftify-cli/build.rs rename {raftify/src/cli => raftify-cli/src}/commands/debug.rs (70%) rename {raftify/src/cli => raftify-cli/src}/commands/mod.rs (100%) rename {raftify/src/cli => raftify-cli/src}/mod.rs (66%) diff --git a/Cargo.lock b/Cargo.lock index 3be7bada..38f831ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -826,7 +826,7 @@ dependencies = [ [[package]] name = "example-harness" -version = "0.1.69" +version = "0.1.78" dependencies = [ "raftify", ] @@ -1026,7 +1026,7 @@ dependencies = [ [[package]] name = "harness" -version = "0.1.69" +version = "0.1.78" dependencies = [ "actix-rt", "actix-web", @@ -1435,7 +1435,7 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memstore-dynamic-members" -version = "0.1.69" +version = "0.1.78" dependencies = [ "actix-rt", "actix-web", @@ -1459,7 +1459,7 @@ dependencies = [ [[package]] name = "memstore-example-harness" -version = "0.1.69" +version = "0.1.78" dependencies = [ "actix-rt", "actix-web", @@ -1475,7 +1475,7 @@ dependencies = [ [[package]] name = "memstore-static-members" -version = "0.1.69" +version = "0.1.78" dependencies = [ "actix-rt", "actix-web", @@ -1880,7 +1880,7 @@ dependencies = [ [[package]] name = "raftify" -version = "0.1.69" +version = "0.1.78" dependencies = [ "async-trait", "bincode", diff --git a/Cargo.toml b/Cargo.toml index cff15e7e..a415dc18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,10 +16,10 @@ default-members = [ "examples/memstore/dynamic-members", "examples/memstore/static-members", ] -exclude = ["raft-rs"] +exclude = ["raft-rs", "raftify-cli"] [workspace.package] -version = "0.1.69" +version = "0.1.78" authors = ["Lablup Inc."] edition = "2021" description = "Experimental High level Raft framework" diff --git a/examples/memstore/src/cli/main.rs b/examples/memstore/src/cli/main.rs index c2a4ef59..50bf79bb 100644 --- a/examples/memstore/src/cli/main.rs +++ b/examples/memstore/src/cli/main.rs @@ -1,10 +1,11 @@ -use raftify::{cli::cli_handler, Result}; +// use raftify::{cli::cli_handler, Result}; +use raftify::Result; use memstore_example_harness::state_machine::{HashStore, LogEntry, StorageType}; #[tokio::main] async fn main() -> Result<()> { - cli_handler::(None).await?; + // cli_handler::(None).await?; Ok(()) } diff --git a/raftify-cli/Cargo.lock b/raftify-cli/Cargo.lock new file mode 100644 index 00000000..3575ba14 --- /dev/null +++ b/raftify-cli/Cargo.lock @@ -0,0 +1,2119 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "anyhow" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] + +[[package]] +name = "built" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9c056b9ed43aee5e064b683aa1ec783e19c6acec7559e3ae931b7490472fbe" +dependencies = [ + "cargo-lock", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" + +[[package]] +name = "cargo-lock" +version = "8.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031718ddb8f78aa5def78a09e90defe30151d1f6c672f937af4dd916429ed996" +dependencies = [ + "semver", + "serde", + "toml", + "url", +] + +[[package]] +name = "cc" +version = "1.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "clap" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getset" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f636605b743120a8d32ed92fc27b6cde1a769f8f936c065151eb66f88ded513c" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "gimli" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 2.5.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "heed" +version = "0.20.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb" +dependencies = [ + "bitflags 2.6.0", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-master-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" + +[[package]] +name = "heed-types" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" +dependencies = [ + "bincode", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jopemachine-raft" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57206b3af8ccd4a755cac94abb0f0692544dfb10869f64dbc19b69ab73c6e251" +dependencies = [ + "bytes", + "fxhash", + "getset", + "lazy_static", + "protobuf", + "raft-proto", + "rand", + "slog", + "slog-envlogger", + "slog-stdlog", + "slog-term", + "thiserror", +] + +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.159" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lmdb-master-sys" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "472c3760e2a8d0f61f322fb36788021bb36d573c502b50fa3e2bcaac3ec326c9" +dependencies = [ + "cc", + "doxygen-rs", + "libc", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.5.0", +] + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 1.0.109", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + +[[package]] +name = "protobuf-build" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2df9942df2981178a930a72d442de47e2f0df18ad68e50a30f816f1848215ad0" +dependencies = [ + "bitflags 1.3.2", + "proc-macro2", + "prost-build", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "raft-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb6884896294f553e8d5cfbdb55080b9f5f2f43394afff59c9f077e0f4b46d6b" +dependencies = [ + "lazy_static", + "prost", + "protobuf", + "protobuf-build", +] + +[[package]] +name = "raftify" +version = "0.1.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f42c008e7e6a5b6431062fac27c4b61e7a78a7e87feeffba5d44d80cfdd657d8" +dependencies = [ + "async-trait", + "bincode", + "built", + "bytes", + "chrono", + "clap", + "heed", + "heed-traits", + "jopemachine-raft", + "log", + "parking_lot", + "prost", + "serde", + "serde_json", + "slog", + "slog-stdlog", + "thiserror", + "tokio", + "tonic", + "tonic-build", +] + +[[package]] +name = "raftify_cli" +version = "0.1.0" +dependencies = [ + "async-trait", + "bincode", + "built", + "bytes", + "chrono", + "clap", + "heed", + "heed-traits", + "jopemachine-raft", + "log", + "parking_lot", + "prost", + "raftify", + "serde", + "serde_json", + "slog", + "slog-stdlog", + "thiserror", + "tokio", + "tonic", + "tonic-build", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "355ae415ccd3a04315d3f8246e86d67689ea74d88d915576e1589a351062a13b" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustix" +version = "0.38.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "serde_json" +version = "1.0.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-async" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-envlogger" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" +dependencies = [ + "log", + "regex", + "slog", + "slog-async", + "slog-scope", + "slog-stdlog", + "slog-term", +] + +[[package]] +name = "slog-scope" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" +dependencies = [ + "arc-swap", + "lazy_static", + "slog", +] + +[[package]] +name = "slog-stdlog" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" +dependencies = [ + "log", + "slog", + "slog-scope", +] + +[[package]] +name = "slog-term" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" +dependencies = [ + "is-terminal", + "slog", + "term", + "thread_local", + "time", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tempfile" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + +[[package]] +name = "thiserror" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "tokio-stream" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unicode-normalization" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.77", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] diff --git a/raftify-cli/Cargo.toml b/raftify-cli/Cargo.toml new file mode 100644 index 00000000..c75a6e9b --- /dev/null +++ b/raftify-cli/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "raftify_cli" +version = "0.1.0" +edition = "2021" + +[dependencies] +async-trait = "0.1.82" +bincode = "1.3" +bytes = "1.7.2" +heed = "0.20.5" +heed-traits = "0.20" +log = { version = "0.4", features = ["std"] } +parking_lot = "0.12.3" +prost = "0.11" +raft = { version = "0.7.9", features = ["prost-codec", "default-logger"], default-features = false, package = "jopemachine-raft" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +slog = "2" +slog-stdlog = "4" +thiserror = "1.0" +tokio = { version = "1.40", features = ["full"] } +tonic = "0.9.2" +built = "0.5" +clap = { version = "4.5.18", features = ["derive"] } +chrono = "0.4.38" +raftify = { version = "0.1.78", features = ["heed_storage", "inmemory_storage"] } + +[lib] +name = "raftify_cli" +path = "src/mod.rs" + +[build-dependencies] +tonic-build = "0.9.2" +built = "0.5" diff --git a/raftify-cli/build.rs b/raftify-cli/build.rs new file mode 100644 index 00000000..a5c32511 --- /dev/null +++ b/raftify-cli/build.rs @@ -0,0 +1,4 @@ +fn main() -> Result<(), Box> { + built::write_built_file().expect("Failed to acquire build-time information"); + Ok(()) +} diff --git a/raftify/src/cli/commands/debug.rs b/raftify-cli/src/commands/debug.rs similarity index 70% rename from raftify/src/cli/commands/debug.rs rename to raftify-cli/src/commands/debug.rs index fdbe76c3..1a9f6316 100644 --- a/raftify/src/cli/commands/debug.rs +++ b/raftify-cli/src/commands/debug.rs @@ -2,23 +2,41 @@ use core::panic; use serde_json::Value; use std::{collections::HashMap, fs, path::Path, sync::Arc}; -use crate::{ +use raftify::{ create_client, raft::{ formatter::{format_entry, format_snapshot}, logger::Slogger, + Storage, }, raft_node::utils::format_debugging_info, raft_service, Config, Result, StableStorage, }; -#[cfg(feature = "inmemory_storage")] -use crate::MemStorage; +use raftify::{HeedStorage, StorageType}; -#[cfg(feature = "heed_storage")] -use crate::HeedStorage; +pub fn debug_persisted(path: &str, logger: slog::Logger) -> Result<()> { + let config = Config { + log_dir: path.to_string(), + ..Default::default() + }; + + let storage = match LogStorage::STORAGE_TYPE { + StorageType::Heed => HeedStorage::create( + config.log_dir.as_str(), + &config, + Arc::new(Slogger { + slog: logger.clone(), + }), + )?, + StorageType::InMemory => { + panic!("Inmemory storage does not support this feature"); + } + _ => { + panic!("Unsupported storage type"); + } + }; -pub fn debug_persisted(storage: impl StableStorage) -> Result<()> { let entries = storage.all_entries()?; if !entries.is_empty() { @@ -43,7 +61,7 @@ pub fn debug_persisted(storage: impl StableStorage) -> Result<()> { Ok(()) } -pub fn debug_persitsted_all(path_str: &str, logger: slog::Logger) -> Result<()> { +pub fn debug_persitsted_all(path_str: &str, logger: slog::Logger) -> Result<()> { let path = match fs::canonicalize(Path::new(&path_str)) { Ok(absolute_path) => absolute_path, Err(e) => { @@ -73,28 +91,7 @@ pub fn debug_persitsted_all(path_str: &str, logger: slog::Logger) -> Result<()> dir_entries.sort(); for name in dir_entries { - println!("*----- {name} -----*"); - #[cfg(feature = "heed_storage")] - { - let config = Config { - log_dir: path_str.to_string(), - ..Default::default() - }; - let storage = HeedStorage::create( - format!("{}/{}", path_str, name).as_str(), - &config, - Arc::new(Slogger { - slog: logger.clone(), - }), - )?; - debug_persisted(storage)?; - } - - #[cfg(feature = "inmemory_storage")] - { - eprintln!("Inmemory storage does not support this feature"); - } - + debug_persisted::(&format!("{}/{}", path_str, name), logger.clone())?; println!(); } } else { diff --git a/raftify/src/cli/commands/mod.rs b/raftify-cli/src/commands/mod.rs similarity index 100% rename from raftify/src/cli/commands/mod.rs rename to raftify-cli/src/commands/mod.rs diff --git a/raftify/src/cli/mod.rs b/raftify-cli/src/mod.rs similarity index 66% rename from raftify/src/cli/mod.rs rename to raftify-cli/src/mod.rs index 5d607fc4..e325fc4b 100644 --- a/raftify/src/cli/mod.rs +++ b/raftify-cli/src/mod.rs @@ -3,21 +3,14 @@ include!(concat!(env!("OUT_DIR"), "/built.rs")); mod commands; use clap::{Args, Parser, Subcommand}; -use commands::debug::{debug_entries, debug_node, debug_persisted, debug_persitsted_all}; -use raft::logger::Slogger; -use std::{fmt::Debug, sync::Arc}; +use commands::debug::{debug_entries, debug_node, debug_persisted, debug_persisted_all}; +use std::fmt::Debug; -use crate::{ +use raftify::{ raft::{default_logger, formatter::set_custom_formatter}, - AbstractLogEntry, AbstractStateMachine, Config, CustomFormatter, Result, StableStorage, + AbstractLogEntry, AbstractStateMachine, CustomFormatter, Result, StableStorage, }; -#[cfg(feature = "inmemory_storage")] -use crate::MemStorage; - -#[cfg(feature = "heed_storage")] -use crate::HeedStorage; - #[derive(Parser)] #[command(name = "raftify")] #[command(version = PKG_VERSION)] @@ -84,29 +77,10 @@ pub async fn cli_handler< match app.command { Commands::Debug(x) => match x { DebugSubcommands::Persisted { path } => { - #[cfg(feature = "heed_storage")] - { - let config = Config { - log_dir: path.to_string(), - ..Default::default() - }; - let storage = HeedStorage::create( - config.log_dir.as_str(), - &config, - Arc::new(Slogger { - slog: logger.clone(), - }), - )?; - debug_persisted(storage)?; - } - - #[cfg(feature = "inmemory_storage")] - { - eprintln!("Inmemory storage does not support this feature"); - } + debug_persisted::(path.as_str(), logger.clone())?; } DebugSubcommands::PersistedAll { path } => { - debug_persitsted_all(path.as_str(), logger.clone())?; + debug_persisted_all::(path.as_str(), logger.clone())?; } DebugSubcommands::Entries { address } => { debug_entries(address.as_str()).await?; diff --git a/raftify/Cargo.toml b/raftify/Cargo.toml index bef6972e..a54c0c89 100644 --- a/raftify/Cargo.toml +++ b/raftify/Cargo.toml @@ -18,7 +18,7 @@ heed-traits = "0.20" log = { version = "0.4", features = ["std"] } parking_lot = "0.12.3" prost = "0.11" -raft = { version = "0.7.9", features = ["prost-codec", "default-logger"], default-features = false, package = "jopemachine-raft" } +raft = { version = "0.7.10", features = ["prost-codec", "default-logger"], default-features = false, package = "jopemachine-raft" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" slog = "2" diff --git a/raftify/src/lib.rs b/raftify/src/lib.rs index ebd00958..94efa426 100644 --- a/raftify/src/lib.rs +++ b/raftify/src/lib.rs @@ -6,7 +6,6 @@ mod peer; mod peers; mod raft_bootstrapper; mod raft_client; -mod raft_node; mod raft_server; mod state_machine; mod storage; @@ -15,8 +14,8 @@ mod utils; mod request; mod response; -pub mod cli; pub mod cluster_join_ticket; +pub mod raft_node; pub mod raft_service; pub use { @@ -33,11 +32,11 @@ pub use crate::{ peers::Peers, raft_bootstrapper::Raft, raft_client::create_client, - raft_node::{role::InitialRole, RaftNode}, + raft_node::{role::InitialRole, utils::format_debugging_info, RaftNode}, raft_service::raft_service_client::RaftServiceClient, request::common::confchange_request::ConfChangeRequest, state_machine::AbstractStateMachine, - storage::StableStorage, + storage::{StableStorage, StorageType}, }; #[cfg(feature = "heed_storage")] diff --git a/raftify/src/storage/heed_storage/mod.rs b/raftify/src/storage/heed_storage/mod.rs index db2716b3..d2238b47 100644 --- a/raftify/src/storage/heed_storage/mod.rs +++ b/raftify/src/storage/heed_storage/mod.rs @@ -18,7 +18,7 @@ use std::{ }; use self::codec::{format_entry_key_string, HeedEntry, HeedEntryKeyString}; -use super::{utils::append_compacted_logs, StableStorage}; +use super::{utils::append_compacted_logs, StableStorage, StorageType}; use crate::{ config::Config, error::Result, @@ -47,6 +47,8 @@ impl HeedStorage { } impl StableStorage for HeedStorage { + const STORAGE_TYPE: StorageType = StorageType::Heed; + fn compact(&mut self, index: u64) -> Result<()> { let store = self.wl(); let mut writer = store.env.write_txn()?; @@ -433,7 +435,7 @@ impl HeedStorageCore { reader: &heed::RoTxn, low: u64, high: u64, - max_size: impl Into>, + max_size: Option, _ctx: GetEntriesContext, ) -> Result> { self.logger diff --git a/raftify/src/storage/inmemory_storage/mod.rs b/raftify/src/storage/inmemory_storage/mod.rs index 368ff874..c64d71f8 100644 --- a/raftify/src/storage/inmemory_storage/mod.rs +++ b/raftify/src/storage/inmemory_storage/mod.rs @@ -9,6 +9,8 @@ use crate::{ StableStorage, }; +use super::StorageType; + #[derive(Clone)] pub struct MemStorage { core: MemStorageCore, @@ -24,6 +26,8 @@ impl MemStorage { } impl StableStorage for MemStorage { + const STORAGE_TYPE: StorageType = StorageType::InMemory; + fn append(&mut self, entries: &[Entry]) -> Result<()> { let mut store = self.core.wl(); store.append(entries)?; @@ -104,10 +108,10 @@ impl Storage for MemStorage { &self, low: u64, high: u64, - max_size: impl Into>, + max_size: Option, context: raft::GetEntriesContext, ) -> raft::Result> { - let entries = self.core.entries(low, high, max_size, context)?; + let entries = self.core.entries(low, high, max_size.into(), context)?; Ok(entries) } diff --git a/raftify/src/storage/mod.rs b/raftify/src/storage/mod.rs index 725574fc..9ec4fa26 100644 --- a/raftify/src/storage/mod.rs +++ b/raftify/src/storage/mod.rs @@ -11,7 +11,15 @@ use crate::{ raft::{self, prelude::*}, }; +pub enum StorageType { + InMemory, + Heed, + Custom, +} + pub trait StableStorage: Storage { + const STORAGE_TYPE: StorageType; + fn append(&mut self, entries: &[Entry]) -> Result<()>; fn hard_state(&self) -> Result; fn set_hard_state(&mut self, hard_state: &HardState) -> Result<()>; From f362d9333fa47d1b25cfa1c951f97231bfbe3fd7 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Fri, 27 Sep 2024 17:18:04 +0900 Subject: [PATCH 12/19] chore: Correct dependencies --- Cargo.lock | 124 +---------------------------------------- raftify-cli/Cargo.lock | 13 ----- raftify-cli/Cargo.toml | 15 +---- raftify/Cargo.toml | 1 - 4 files changed, 4 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38f831ee..481147ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -262,55 +262,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "anstream" -version = "0.6.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" - -[[package]] -name = "anstyle-parse" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" -dependencies = [ - "anstyle", - "windows-sys 0.52.0", -] - [[package]] name = "anyhow" version = "1.0.87" @@ -574,52 +525,12 @@ dependencies = [ "ansi_term", "atty", "bitflags 1.3.2", - "strsim 0.8.0", + "strsim", "textwrap", "unicode-width", "vec_map", ] -[[package]] -name = "clap" -version = "4.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim 0.11.1", -] - -[[package]] -name = "clap_derive" -version = "4.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.77", -] - -[[package]] -name = "clap_lex" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" - [[package]] name = "color-backtrace" version = "0.6.1" @@ -630,12 +541,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "colorchoice" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" - [[package]] name = "const_format" version = "0.2.33" @@ -1072,12 +977,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - [[package]] name = "heed" version = "0.20.5" @@ -1286,12 +1185,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - [[package]] name = "itertools" version = "0.10.5" @@ -1887,7 +1780,6 @@ dependencies = [ "built", "bytes", "chrono", - "clap 4.5.18", "heed", "heed-traits", "jopemachine-raft", @@ -2229,19 +2121,13 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "structopt" version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" dependencies = [ - "clap 2.34.0", + "clap", "lazy_static", "structopt-derive", ] @@ -2693,12 +2579,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "vec_map" version = "0.8.2" diff --git a/raftify-cli/Cargo.lock b/raftify-cli/Cargo.lock index 3575ba14..4402bdca 100644 --- a/raftify-cli/Cargo.lock +++ b/raftify-cli/Cargo.lock @@ -1239,26 +1239,13 @@ dependencies = [ name = "raftify_cli" version = "0.1.0" dependencies = [ - "async-trait", - "bincode", "built", - "bytes", - "chrono", "clap", - "heed", - "heed-traits", - "jopemachine-raft", "log", - "parking_lot", - "prost", "raftify", "serde", "serde_json", "slog", - "slog-stdlog", - "thiserror", - "tokio", - "tonic", "tonic-build", ] diff --git a/raftify-cli/Cargo.toml b/raftify-cli/Cargo.toml index c75a6e9b..0c3394d4 100644 --- a/raftify-cli/Cargo.toml +++ b/raftify-cli/Cargo.toml @@ -2,27 +2,16 @@ name = "raftify_cli" version = "0.1.0" edition = "2021" +description = "Raftify CLI tool" +license = "MIT/Apache-2.0" [dependencies] -async-trait = "0.1.82" -bincode = "1.3" -bytes = "1.7.2" -heed = "0.20.5" -heed-traits = "0.20" log = { version = "0.4", features = ["std"] } -parking_lot = "0.12.3" -prost = "0.11" -raft = { version = "0.7.9", features = ["prost-codec", "default-logger"], default-features = false, package = "jopemachine-raft" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" slog = "2" -slog-stdlog = "4" -thiserror = "1.0" -tokio = { version = "1.40", features = ["full"] } -tonic = "0.9.2" built = "0.5" clap = { version = "4.5.18", features = ["derive"] } -chrono = "0.4.38" raftify = { version = "0.1.78", features = ["heed_storage", "inmemory_storage"] } [lib] diff --git a/raftify/Cargo.toml b/raftify/Cargo.toml index a54c0c89..54e24024 100644 --- a/raftify/Cargo.toml +++ b/raftify/Cargo.toml @@ -27,7 +27,6 @@ thiserror = "1.0" tokio = { version = "1.40", features = ["full"] } tonic = "0.9.2" built = "0.5" -clap = { version = "4.5.18", features = ["derive"] } chrono = "0.4.38" [features] From f2a83ea47899739d317a0ac606a8420dbb52a819 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Fri, 27 Sep 2024 17:58:38 +0900 Subject: [PATCH 13/19] WIP --- Cargo.lock | 140 ++++++++++++++++++++++++- Cargo.toml | 1 + examples/memstore/Cargo.toml | 4 +- examples/memstore/src/cli/main.rs | 5 +- examples/memstore/src/state_machine.rs | 3 - raftify-cli/Cargo.lock | 2 +- raftify-cli/Cargo.toml | 2 +- raftify-cli/src/commands/debug.rs | 4 +- 8 files changed, 147 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 481147ed..4169d19a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -262,6 +262,55 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" version = "1.0.87" @@ -525,12 +574,52 @@ dependencies = [ "ansi_term", "atty", "bitflags 1.3.2", - "strsim", + "strsim 0.8.0", "textwrap", "unicode-width", "vec_map", ] +[[package]] +name = "clap" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.77", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + [[package]] name = "color-backtrace" version = "0.6.1" @@ -541,6 +630,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + [[package]] name = "const_format" version = "0.2.33" @@ -977,6 +1072,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "heed" version = "0.20.5" @@ -1185,6 +1286,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -1361,6 +1468,7 @@ dependencies = [ "color-backtrace", "log", "raftify", + "raftify_cli", "serde", "serde_json", "tokio", @@ -1797,6 +1905,22 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "raftify_cli" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89fca418beb04ff1d175598c386db1d8bc40b20b0f2a1e645fe9d5bb0789a7d6" +dependencies = [ + "built", + "clap 4.5.18", + "log", + "raftify", + "serde", + "serde_json", + "slog", + "tonic-build", +] + [[package]] name = "rand" version = "0.8.5" @@ -2121,13 +2245,19 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "structopt" version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" dependencies = [ - "clap", + "clap 2.34.0", "lazy_static", "structopt-derive", ] @@ -2579,6 +2709,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "vec_map" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index a415dc18..d099eaed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,3 +36,4 @@ memstore-example-harness = { path = "examples/memstore" } [patch.crates-io] jopemachine-raft = { path = "./raft-rs" } +raftify = { path = "./raftify" } diff --git a/examples/memstore/Cargo.toml b/examples/memstore/Cargo.toml index 40af1cc3..8daf2057 100644 --- a/examples/memstore/Cargo.toml +++ b/examples/memstore/Cargo.toml @@ -17,8 +17,6 @@ name = "raftify-client-example" path = "src/client/main.rs" [dependencies] -raftify.workspace = true - actix-rt = "2.10" actix-web = "4.9.0" async-trait = "0.1.82" @@ -28,6 +26,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1.40", features = ["full"] } color-backtrace = "0.6.1" +raftify = { version = "0.1.78" } +raftify_cli = { version = "0.1.1" } [features] default = ["heed_storage"] diff --git a/examples/memstore/src/cli/main.rs b/examples/memstore/src/cli/main.rs index 50bf79bb..deef0b41 100644 --- a/examples/memstore/src/cli/main.rs +++ b/examples/memstore/src/cli/main.rs @@ -1,11 +1,10 @@ -// use raftify::{cli::cli_handler, Result}; use raftify::Result; +use raftify_cli::cli_handler; use memstore_example_harness::state_machine::{HashStore, LogEntry, StorageType}; #[tokio::main] async fn main() -> Result<()> { - // cli_handler::(None).await?; - + cli_handler::(None).await?; Ok(()) } diff --git a/examples/memstore/src/state_machine.rs b/examples/memstore/src/state_machine.rs index 05928b10..3f78c34c 100644 --- a/examples/memstore/src/state_machine.rs +++ b/examples/memstore/src/state_machine.rs @@ -7,9 +7,6 @@ use std::{ sync::{Arc, RwLock}, }; -#[cfg(feature = "inmemory_storage")] -pub use raftify::MemStorage as StorageType; - #[cfg(feature = "heed_storage")] pub use raftify::HeedStorage as StorageType; diff --git a/raftify-cli/Cargo.lock b/raftify-cli/Cargo.lock index 4402bdca..c682b77f 100644 --- a/raftify-cli/Cargo.lock +++ b/raftify-cli/Cargo.lock @@ -1237,7 +1237,7 @@ dependencies = [ [[package]] name = "raftify_cli" -version = "0.1.0" +version = "0.1.1" dependencies = [ "built", "clap", diff --git a/raftify-cli/Cargo.toml b/raftify-cli/Cargo.toml index 0c3394d4..9cae6bc1 100644 --- a/raftify-cli/Cargo.toml +++ b/raftify-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "raftify_cli" -version = "0.1.0" +version = "0.1.1" edition = "2021" description = "Raftify CLI tool" license = "MIT/Apache-2.0" diff --git a/raftify-cli/src/commands/debug.rs b/raftify-cli/src/commands/debug.rs index 1a9f6316..eb6de53a 100644 --- a/raftify-cli/src/commands/debug.rs +++ b/raftify-cli/src/commands/debug.rs @@ -11,10 +11,9 @@ use raftify::{ }, raft_node::utils::format_debugging_info, raft_service, Config, Result, StableStorage, + HeedStorage, StorageType, }; -use raftify::{HeedStorage, StorageType}; - pub fn debug_persisted(path: &str, logger: slog::Logger) -> Result<()> { let config = Config { log_dir: path.to_string(), @@ -91,6 +90,7 @@ pub fn debug_persitsted_all(path_str: &str, logger: s dir_entries.sort(); for name in dir_entries { + println!("*----- {name} -----*"); debug_persisted::(&format!("{}/{}", path_str, name), logger.clone())?; println!(); } From c45417fbcbf766aca766b937c8b7532f095be34c Mon Sep 17 00:00:00 2001 From: Gyubong Date: Sun, 29 Sep 2024 16:04:35 +0900 Subject: [PATCH 14/19] WIP - Reflect changes in python binding --- binding/python/Cargo.lock | 21 ++++++++++++++++-- binding/python/Cargo.toml | 5 +++-- .../{state_machine.rs => abstract_types.rs} | 8 ++++++- binding/python/src/bindings/cli.rs | 9 +++++--- binding/python/src/bindings/config.rs | 16 +++++--------- .../src/bindings/{role.rs => initial_role.rs} | 0 binding/python/src/bindings/mod.rs | 4 ++-- binding/python/src/bindings/peer.rs | 2 +- binding/python/src/bindings/peers.rs | 2 +- .../python/src/bindings/raft_bootstrapper.rs | 22 ++++++++++++++----- binding/python/src/bindings/raft_node.rs | 10 ++++----- binding/python/src/lib.rs | 6 ++--- raftify-cli/src/commands/debug.rs | 2 +- 13 files changed, 70 insertions(+), 37 deletions(-) rename binding/python/src/bindings/{state_machine.rs => abstract_types.rs} (97%) rename binding/python/src/bindings/{role.rs => initial_role.rs} (100%) diff --git a/binding/python/Cargo.lock b/binding/python/Cargo.lock index 70a41542..ec6204ec 100644 --- a/binding/python/Cargo.lock +++ b/binding/python/Cargo.lock @@ -1049,7 +1049,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", "wasi", "windows-sys 0.52.0", @@ -1490,7 +1490,7 @@ dependencies = [ [[package]] name = "raftify-py" -version = "0.1.67" +version = "0.1.78" dependencies = [ "async-trait", "bincode", @@ -1502,6 +1502,7 @@ dependencies = [ "pyo3-asyncio", "pythonize", "raftify", + "raftify_cli", "serde", "slog", "slog-async", @@ -1512,6 +1513,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "raftify_cli" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89fca418beb04ff1d175598c386db1d8bc40b20b0f2a1e645fe9d5bb0789a7d6" +dependencies = [ + "built", + "clap", + "log", + "raftify", + "serde", + "serde_json", + "slog", + "tonic-build", +] + [[package]] name = "rand" version = "0.8.5" diff --git a/binding/python/Cargo.toml b/binding/python/Cargo.toml index 215bbe0a..244ca933 100644 --- a/binding/python/Cargo.toml +++ b/binding/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "raftify-py" -version = "0.1.67" +version = "0.1.78" authors = ["Lablup Inc."] license = "Apache-2.0" repository = "https://github.com/lablup/raftify" @@ -19,7 +19,8 @@ pyo3-asyncio = { version = "0.20.0", features = ["tokio-runtime"] } pythonize = "0.20.0" tokio = { version = "1.4", features = ["full"] } async-trait = "0.1.48" -raftify = { version = "=0.1.69", default-features = false } +raftify = { version = "=0.1.78", features = ["heed_storage"] , default-features = false } +raftify_cli = { version = "=0.1.1" } slog = { version = "2.2", features = ["max_level_trace", "release_max_level_trace"] } slog-envlogger = "2.1.0" slog-term = "2.9.0" diff --git a/binding/python/src/bindings/state_machine.rs b/binding/python/src/bindings/abstract_types.rs similarity index 97% rename from binding/python/src/bindings/state_machine.rs rename to binding/python/src/bindings/abstract_types.rs index a9e63132..723aff0a 100644 --- a/binding/python/src/bindings/state_machine.rs +++ b/binding/python/src/bindings/abstract_types.rs @@ -2,7 +2,13 @@ use async_trait::async_trait; use once_cell::sync::Lazy; use pyo3::{prelude::*, types::PyBytes}; use pyo3_asyncio::TaskLocals; -use raftify::{AbstractLogEntry, AbstractStateMachine, Error, Result}; +use raftify::{ + raft::{ + prelude::{ConfState, Entry, HardState, Snapshot}, + GetEntriesContext, RaftState, Storage, + }, + AbstractLogEntry, AbstractStateMachine, Error, Result, +}; use std::{fmt, sync::Mutex}; use super::{ diff --git a/binding/python/src/bindings/cli.rs b/binding/python/src/bindings/cli.rs index 79d4601f..06bb943e 100644 --- a/binding/python/src/bindings/cli.rs +++ b/binding/python/src/bindings/cli.rs @@ -1,14 +1,17 @@ use pyo3::prelude::*; use pyo3_asyncio::tokio::future_into_py; -use raftify::cli::cli_handler; +use raftify::HeedStorage; +use raftify_cli::cli_handler; -use super::state_machine::{PyFSM, PyLogEntry}; +use super::abstract_types::{PyFSM, PyLogEntry}; // When args is None, std::env::args is automatically used. #[pyfunction] pub fn cli_main<'a>(args: Option>, py: Python<'a>) -> PyResult<&'a PyAny> { future_into_py(py, async move { - cli_handler::(args).await.unwrap(); + cli_handler::(args) + .await + .unwrap(); Ok(()) }) } diff --git a/binding/python/src/bindings/config.rs b/binding/python/src/bindings/config.rs index 1ae0ec92..dbb2cae9 100644 --- a/binding/python/src/bindings/config.rs +++ b/binding/python/src/bindings/config.rs @@ -15,10 +15,9 @@ pub struct PyConfig { pub lmdb_map_size: u64, pub cluster_id: String, pub conf_change_request_timeout: f32, + pub bootstrap_from_snapshot: bool, pub initial_peers: Option, pub snapshot_interval: Option, - pub restore_wal_from: Option, - pub restore_wal_snapshot_from: Option, } #[pymethods] @@ -36,8 +35,7 @@ impl PyConfig { conf_change_request_timeout: Option, initial_peers: Option, snapshot_interval: Option, - restore_wal_from: Option, - restore_wal_snapshot_from: Option, + bootstrap_from_snapshot: Option, ) -> Self { let cfg = Config::default(); @@ -56,8 +54,8 @@ impl PyConfig { conf_change_request_timeout.unwrap_or(cfg.conf_change_request_timeout); let initial_peers = initial_peers; let snapshot_interval = snapshot_interval; - let restore_wal_from = restore_wal_from; - let restore_wal_snapshot_from = restore_wal_snapshot_from; + let bootstrap_from_snapshot = + bootstrap_from_snapshot.unwrap_or(cfg.bootstrap_from_snapshot); Self { raft_config, @@ -71,8 +69,7 @@ impl PyConfig { conf_change_request_timeout, initial_peers, snapshot_interval, - restore_wal_from, - restore_wal_snapshot_from, + bootstrap_from_snapshot, } } } @@ -91,8 +88,7 @@ impl From for Config { conf_change_request_timeout: config.conf_change_request_timeout, initial_peers: config.initial_peers.map(|peers| peers.inner), raft_config: config.raft_config.inner, - restore_wal_from: config.restore_wal_from, - restore_wal_snapshot_from: config.restore_wal_snapshot_from, + bootstrap_from_snapshot: config.bootstrap_from_snapshot, } } } diff --git a/binding/python/src/bindings/role.rs b/binding/python/src/bindings/initial_role.rs similarity index 100% rename from binding/python/src/bindings/role.rs rename to binding/python/src/bindings/initial_role.rs diff --git a/binding/python/src/bindings/mod.rs b/binding/python/src/bindings/mod.rs index 20011073..44c3c787 100644 --- a/binding/python/src/bindings/mod.rs +++ b/binding/python/src/bindings/mod.rs @@ -1,3 +1,4 @@ +pub mod abstract_types; pub mod cli; pub mod cluster_join_ticket; pub mod confchange_request; @@ -11,7 +12,6 @@ pub mod raft_bootstrapper; pub mod raft_client; pub mod raft_node; pub mod raft_rs; -pub mod role; +pub mod initial_role; pub mod slogger; -pub mod state_machine; pub mod utils; diff --git a/binding/python/src/bindings/peer.rs b/binding/python/src/bindings/peer.rs index 5e7ad650..9f34a4e4 100644 --- a/binding/python/src/bindings/peer.rs +++ b/binding/python/src/bindings/peer.rs @@ -2,7 +2,7 @@ use pyo3::prelude::*; use pyo3_asyncio::tokio::future_into_py; use raftify::Peer; -use super::role::PyInitialRole; +use super::initial_role::PyInitialRole; #[derive(Clone)] #[pyclass(name = "Peer")] diff --git a/binding/python/src/bindings/peers.rs b/binding/python/src/bindings/peers.rs index f5bed161..a8400413 100644 --- a/binding/python/src/bindings/peers.rs +++ b/binding/python/src/bindings/peers.rs @@ -7,7 +7,7 @@ use raftify::Peers; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, hash::BuildHasherDefault}; -use super::{peer::PyPeer, role::PyInitialRole}; +use super::{peer::PyPeer, initial_role::PyInitialRole}; #[derive(Serialize, Deserialize, Clone)] #[pyclass(dict, name = "Peers")] diff --git a/binding/python/src/bindings/raft_bootstrapper.rs b/binding/python/src/bindings/raft_bootstrapper.rs index b9ec6adf..9600fc03 100644 --- a/binding/python/src/bindings/raft_bootstrapper.rs +++ b/binding/python/src/bindings/raft_bootstrapper.rs @@ -1,20 +1,20 @@ use pyo3::{exceptions::PyException, prelude::*, types::PyString}; use pyo3_asyncio::tokio::future_into_py; -use raftify::Raft; +use raftify::{HeedStorage, Raft}; use std::sync::Arc; use super::{ + abstract_types::{PyFSM, PyLogEntry}, cluster_join_ticket::PyClusterJoinTicket, config::PyConfig, logger::PyLogger, raft_node::PyRaftNode, - state_machine::{PyFSM, PyLogEntry}, }; #[derive(Clone)] #[pyclass(name = "Raft")] pub struct PyRaftFacade { - inner: Raft, + inner: Raft, } #[pymethods] @@ -30,9 +30,16 @@ impl PyRaftFacade { let fsm = PyFSM::new(fsm); let addr = addr.to_string(); + let storage = HeedStorage::create( + &config.log_dir.clone(), + &config.clone().into(), + Arc::new(PyLogger::new(logger.clone())), + ).expect("Failed to create heed storage"); + let raft = Raft::bootstrap( node_id, addr, + storage, fsm, config.into(), Arc::new(PyLogger::new(logger)), @@ -49,9 +56,12 @@ impl PyRaftFacade { py: Python<'a>, ) -> PyResult<&'a PyAny> { future_into_py(py, async move { - let ticket = Raft::::request_id(raft_addr, peer_addr.to_owned()) - .await - .unwrap(); + let ticket = Raft::::request_id( + raft_addr, + peer_addr.to_owned(), + ) + .await + .unwrap(); Ok(PyClusterJoinTicket { inner: ticket }) }) } diff --git a/binding/python/src/bindings/raft_node.rs b/binding/python/src/bindings/raft_node.rs index 5e422ea6..282ecd70 100644 --- a/binding/python/src/bindings/raft_node.rs +++ b/binding/python/src/bindings/raft_node.rs @@ -1,23 +1,23 @@ use pyo3::{prelude::*, types::PyString}; use pyo3_asyncio::tokio::future_into_py; -use raftify::RaftNode; +use raftify::{HeedStorage, RaftNode}; use super::{ + abstract_types::{PyFSM, PyLogEntry}, cluster_join_ticket::PyClusterJoinTicket, peers::PyPeers, raft_rs::eraftpb::{conf_change_v2::PyConfChangeV2, message::PyMessage}, - role::PyInitialRole, - state_machine::{PyFSM, PyLogEntry}, + initial_role::PyInitialRole, }; #[derive(Clone)] #[pyclass(name = "RaftNode")] pub struct PyRaftNode { - pub inner: RaftNode, + pub inner: RaftNode, } impl PyRaftNode { - pub fn new(inner: RaftNode) -> Self { + pub fn new(inner: RaftNode) -> Self { PyRaftNode { inner } } } diff --git a/binding/python/src/lib.rs b/binding/python/src/lib.rs index d8fe56e3..46af3a2d 100644 --- a/binding/python/src/lib.rs +++ b/binding/python/src/lib.rs @@ -20,7 +20,7 @@ fn raftify(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -37,12 +37,12 @@ fn raftify(py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(bindings::cli::cli_main, m)?)?; m.add_function(wrap_pyfunction!( - bindings::state_machine::set_log_entry_deserializer, + bindings::abstract_types::set_log_entry_deserializer, m )?)?; m.add_function(wrap_pyfunction!( - bindings::state_machine::set_fsm_deserializer, + bindings::abstract_types::set_fsm_deserializer, m )?)?; diff --git a/raftify-cli/src/commands/debug.rs b/raftify-cli/src/commands/debug.rs index eb6de53a..47a316bb 100644 --- a/raftify-cli/src/commands/debug.rs +++ b/raftify-cli/src/commands/debug.rs @@ -29,7 +29,7 @@ pub fn debug_persisted(path: &str, logger: slog::Logg }), )?, StorageType::InMemory => { - panic!("Inmemory storage does not support this feature"); + panic!("InMemory storage does not support this feature"); } _ => { panic!("Unsupported storage type"); From 26bba81e94c9111c3e8b7ba2f08219dd6c4113d4 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 30 Sep 2024 09:52:48 +0900 Subject: [PATCH 15/19] chore: Remove useless debugging statement --- raftify/src/raft_node/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index f672e4e2..30080a44 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -1400,7 +1400,6 @@ impl< if !ready.entries().is_empty() { let entries = &ready.entries()[..]; let store = self.raw_node.mut_store(); - println!("Applying entries: {:?}", entries); store.append(entries)?; } From bad8c80b386ca148b897806d7f5beded2cec60e9 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 30 Sep 2024 09:56:30 +0900 Subject: [PATCH 16/19] Merge --- binding/python/Cargo.lock | 6 +++--- binding/python/src/bindings/abstract_types.rs | 8 +------- binding/python/src/bindings/mod.rs | 2 +- binding/python/src/bindings/peers.rs | 2 +- binding/python/src/bindings/raft_bootstrapper.rs | 13 ++++++------- binding/python/src/bindings/raft_node.rs | 2 +- 6 files changed, 13 insertions(+), 20 deletions(-) diff --git a/binding/python/Cargo.lock b/binding/python/Cargo.lock index ec6204ec..94f61b7a 100644 --- a/binding/python/Cargo.lock +++ b/binding/python/Cargo.lock @@ -1049,7 +1049,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "wasi", "windows-sys 0.52.0", @@ -1462,9 +1462,9 @@ dependencies = [ [[package]] name = "raftify" -version = "0.1.69" +version = "0.1.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572cae29ab16123a80f731a4ef1680a77ed64a98da5e43ad7ed29ac67f6f5679" +checksum = "f42c008e7e6a5b6431062fac27c4b61e7a78a7e87feeffba5d44d80cfdd657d8" dependencies = [ "async-trait", "bincode", diff --git a/binding/python/src/bindings/abstract_types.rs b/binding/python/src/bindings/abstract_types.rs index 723aff0a..a9e63132 100644 --- a/binding/python/src/bindings/abstract_types.rs +++ b/binding/python/src/bindings/abstract_types.rs @@ -2,13 +2,7 @@ use async_trait::async_trait; use once_cell::sync::Lazy; use pyo3::{prelude::*, types::PyBytes}; use pyo3_asyncio::TaskLocals; -use raftify::{ - raft::{ - prelude::{ConfState, Entry, HardState, Snapshot}, - GetEntriesContext, RaftState, Storage, - }, - AbstractLogEntry, AbstractStateMachine, Error, Result, -}; +use raftify::{AbstractLogEntry, AbstractStateMachine, Error, Result}; use std::{fmt, sync::Mutex}; use super::{ diff --git a/binding/python/src/bindings/mod.rs b/binding/python/src/bindings/mod.rs index 44c3c787..18f61f01 100644 --- a/binding/python/src/bindings/mod.rs +++ b/binding/python/src/bindings/mod.rs @@ -5,6 +5,7 @@ pub mod confchange_request; pub mod config; pub mod errors; pub mod formatter; +pub mod initial_role; pub mod logger; pub mod peer; pub mod peers; @@ -12,6 +13,5 @@ pub mod raft_bootstrapper; pub mod raft_client; pub mod raft_node; pub mod raft_rs; -pub mod initial_role; pub mod slogger; pub mod utils; diff --git a/binding/python/src/bindings/peers.rs b/binding/python/src/bindings/peers.rs index a8400413..4b2c952e 100644 --- a/binding/python/src/bindings/peers.rs +++ b/binding/python/src/bindings/peers.rs @@ -7,7 +7,7 @@ use raftify::Peers; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, hash::BuildHasherDefault}; -use super::{peer::PyPeer, initial_role::PyInitialRole}; +use super::{initial_role::PyInitialRole, peer::PyPeer}; #[derive(Serialize, Deserialize, Clone)] #[pyclass(dict, name = "Peers")] diff --git a/binding/python/src/bindings/raft_bootstrapper.rs b/binding/python/src/bindings/raft_bootstrapper.rs index 9600fc03..a2a2dc7f 100644 --- a/binding/python/src/bindings/raft_bootstrapper.rs +++ b/binding/python/src/bindings/raft_bootstrapper.rs @@ -34,7 +34,8 @@ impl PyRaftFacade { &config.log_dir.clone(), &config.clone().into(), Arc::new(PyLogger::new(logger.clone())), - ).expect("Failed to create heed storage"); + ) + .expect("Failed to create heed storage"); let raft = Raft::bootstrap( node_id, @@ -56,12 +57,10 @@ impl PyRaftFacade { py: Python<'a>, ) -> PyResult<&'a PyAny> { future_into_py(py, async move { - let ticket = Raft::::request_id( - raft_addr, - peer_addr.to_owned(), - ) - .await - .unwrap(); + let ticket = + Raft::::request_id(raft_addr, peer_addr.to_owned()) + .await + .unwrap(); Ok(PyClusterJoinTicket { inner: ticket }) }) } diff --git a/binding/python/src/bindings/raft_node.rs b/binding/python/src/bindings/raft_node.rs index 282ecd70..d5bc4fdc 100644 --- a/binding/python/src/bindings/raft_node.rs +++ b/binding/python/src/bindings/raft_node.rs @@ -5,9 +5,9 @@ use raftify::{HeedStorage, RaftNode}; use super::{ abstract_types::{PyFSM, PyLogEntry}, cluster_join_ticket::PyClusterJoinTicket, + initial_role::PyInitialRole, peers::PyPeers, raft_rs::eraftpb::{conf_change_v2::PyConfChangeV2, message::PyMessage}, - initial_role::PyInitialRole, }; #[derive(Clone)] From 077d01fec1c9f791ca36d4c1a22eda05acd9cf4b Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 30 Sep 2024 11:23:00 +0900 Subject: [PATCH 17/19] WIP --- examples/memstore/dynamic-members/src/main.rs | 5 +-- examples/memstore/src/web_server_api.rs | 5 ++- examples/memstore/static-members/src/main.rs | 7 +++-- examples/src/config.rs | 3 +- raftify/src/error.rs | 3 ++ raftify/src/raft_node/mod.rs | 31 ++++++++++++++----- 6 files changed, 39 insertions(+), 15 deletions(-) diff --git a/examples/memstore/dynamic-members/src/main.rs b/examples/memstore/dynamic-members/src/main.rs index f24a8174..ce53f160 100644 --- a/examples/memstore/dynamic-members/src/main.rs +++ b/examples/memstore/dynamic-members/src/main.rs @@ -66,8 +66,6 @@ async fn main() -> std::result::Result<(), Box> { let options = Options::from_args(); let store = HashStore::new(); - let mut cfg = build_config(); - let (raft, raft_handle) = match options.peer_addr { Some(peer_addr) => { log::info!("Running in Follower mode"); @@ -76,6 +74,8 @@ async fn main() -> std::result::Result<(), Box> { .await .unwrap(); let node_id = ticket.reserved_id; + let mut cfg = build_config(node_id); + cfg.initial_peers = Some(ticket.peers.clone().into()); let storage_pth = get_storage_path(cfg.log_dir.as_str(), node_id); @@ -106,6 +106,7 @@ async fn main() -> std::result::Result<(), Box> { None => { log::info!("Bootstrap a Raft Cluster"); + let cfg: raftify::Config = build_config(1); let storage_pth = get_storage_path(cfg.log_dir.as_str(), 1); ensure_directory_exist(storage_pth.as_str())?; diff --git a/examples/memstore/src/web_server_api.rs b/examples/memstore/src/web_server_api.rs index 143004e1..c8bd7655 100644 --- a/examples/memstore/src/web_server_api.rs +++ b/examples/memstore/src/web_server_api.rs @@ -78,7 +78,10 @@ async fn snapshot(data: web::Data<(HashStore, Raft)>) -> impl Responder { .hard_state() .expect("Failed to get hard state"); - raft.1.make_snapshot(last_index, hard_state.term).await; + raft.1 + .make_snapshot(last_index, hard_state.term) + .await + .expect("Failed to make snapshot"); "OK".to_string() } } diff --git a/examples/memstore/static-members/src/main.rs b/examples/memstore/static-members/src/main.rs index 6676246d..9228d730 100644 --- a/examples/memstore/static-members/src/main.rs +++ b/examples/memstore/static-members/src/main.rs @@ -38,6 +38,7 @@ struct Options { raft_addr: String, #[structopt(long)] web_server: Option, + // TODO: Make "bootstrap_from_snapshot" option here } #[actix_rt::main] @@ -66,13 +67,13 @@ async fn main() -> std::result::Result<(), Box> { let store = HashStore::new(); let initial_peers = load_peers("cluster_config.toml").await?; - let mut cfg = build_config(); - cfg.initial_peers = Some(initial_peers.clone()); - let node_id = initial_peers .get_node_id_by_addr(options.raft_addr.clone()) .unwrap(); + let mut cfg = build_config(node_id); + cfg.initial_peers = Some(initial_peers.clone()); + let storage_pth = get_storage_path(cfg.log_dir.as_str(), node_id); ensure_directory_exist(storage_pth.as_str())?; diff --git a/examples/src/config.rs b/examples/src/config.rs index 215d4f91..851de5e5 100644 --- a/examples/src/config.rs +++ b/examples/src/config.rs @@ -1,7 +1,8 @@ use raftify::{Config, RaftConfig}; -pub fn build_config() -> Config { +pub fn build_config(node_id: u64) -> Config { let raft_config = RaftConfig { + id: node_id, election_tick: 10, heartbeat_tick: 3, ..Default::default() diff --git a/raftify/src/error.rs b/raftify/src/error.rs index 11e009cb..5d36a5d8 100644 --- a/raftify/src/error.rs +++ b/raftify/src/error.rs @@ -34,6 +34,9 @@ pub enum Error { #[error("Shut down by Ctrl+C signal")] CtrlC, + #[error("RaftNode LocalRequestMsg Receive error: {0}")] + RecvError(#[from] tokio::sync::oneshot::error::RecvError), + #[error("Encoding error")] EncodingError(String), #[error("Decoding error")] diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index 30080a44..9c78c8ef 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -414,7 +414,7 @@ impl< } } - pub async fn make_snapshot(&self, index: u64, term: u64) { + pub async fn make_snapshot(&self, index: u64, term: u64) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::MakeSnapshot { @@ -424,9 +424,9 @@ impl< }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::MakeSnapshot {} => (), + LocalResponseMsg::MakeSnapshot {} => Ok(()), _ => unreachable!(), } } @@ -516,7 +516,7 @@ impl< should_be_leader: bool, mut log_storage: LogStorage, fsm: FSM, - mut config: Config, + config: Config, raft_addr: SocketAddr, logger: Arc, tx_server: mpsc::Sender>, @@ -524,7 +524,9 @@ impl< tx_local: mpsc::Sender>, rx_local: mpsc::Receiver>, ) -> Result { - config.raft_config.id = node_id; + assert_eq!(config.raft_config.id, node_id); + println!("node_id: {:?}", node_id); + println!("config: {:?}", config.raft_config.id); config.validate()?; let mut snapshot = log_storage.snapshot(0, log_storage.last_index()?)?; @@ -559,8 +561,14 @@ impl< conf_state.set_learners(learners); } - if last_idx == 0 || config.bootstrap_from_snapshot { + if last_idx == 0 { + logger.info("Bootstrapping cluster init..."); log_storage.apply_snapshot(snapshot)?; + } else if config.bootstrap_from_snapshot { + logger.info("Bootstrapping from snapshot..."); + log_storage.apply_snapshot(snapshot)?; + } else { + logger.info("Bootstrapping from existing logs..."); } let mut raw_node = RawNode::new(&config.raft_config, log_storage.clone(), logger.clone())?; @@ -1103,8 +1111,8 @@ impl< }) .unwrap(); - #[cfg(feature = "inmemory_storage")] - todo!("Implement this for inmemory storage"); + // #[cfg(feature = "inmemory_storage")] + // todo!("Implement LocalRequestMsg::GetStorage request handler for inmemory storage"); } LocalRequestMsg::DebugNode { tx_msg } => { tx_msg @@ -1174,7 +1182,14 @@ impl< term, tx_msg, } => { + // println!("Make snapshot!! 2 2"); + // let r = self.make_snapshot(index, term).await; + // println!("Make snapshot!! 2 3 r: {:?}", r); + // if let Err(e) = r { + // return Err(e); + // } self.make_snapshot(index, term).await?; + println!("Make snapshot!! 2 3"); tx_msg.send(LocalResponseMsg::MakeSnapshot {}).unwrap(); } LocalRequestMsg::JoinCluster { tickets, tx_msg } => { From 472636dd6fde04f9ec126cf4f0751e92b93eaadc Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 30 Sep 2024 11:35:13 +0900 Subject: [PATCH 18/19] feat: Improve error handling --- examples/memstore/src/web_server_api.rs | 9 +- harness/src/utils.rs | 4 +- harness/tests/data_replication.rs | 6 +- harness/tests/leader_election.rs | 8 +- raftify/src/raft_node/mod.rs | 136 ++++++++++++------------ 5 files changed, 85 insertions(+), 78 deletions(-) diff --git a/examples/memstore/src/web_server_api.rs b/examples/memstore/src/web_server_api.rs index c8bd7655..f43a13ad 100644 --- a/examples/memstore/src/web_server_api.rs +++ b/examples/memstore/src/web_server_api.rs @@ -28,7 +28,12 @@ async fn get(data: web::Data<(HashStore, Raft)>, path: web::Path) -> impl R #[get("/leader")] async fn leader_id(data: web::Data<(HashStore, Raft)>) -> impl Responder { let raft = data.clone(); - let leader_id = raft.1.get_leader_id().await.to_string(); + let leader_id = raft + .1 + .get_leader_id() + .await + .expect("Failed to get leader id") + .to_string(); format!("{:?}", leader_id) } @@ -68,6 +73,7 @@ async fn snapshot(data: web::Data<(HashStore, Raft)>) -> impl Responder { .1 .storage() .await + .expect("Failed to get storage") .last_index() .expect("Failed to get last index"); @@ -75,6 +81,7 @@ async fn snapshot(data: web::Data<(HashStore, Raft)>) -> impl Responder { .1 .storage() .await + .expect("Failed to get storage") .hard_state() .expect("Failed to get hard state"); diff --git a/harness/src/utils.rs b/harness/src/utils.rs index 679c1f9e..912b0495 100644 --- a/harness/src/utils.rs +++ b/harness/src/utils.rs @@ -89,7 +89,7 @@ pub async fn wait_for_until_cluster_size_increase(raft: Raft, target: usize) { )); loop { - let size = raft.get_cluster_size().await; + let size = raft.get_cluster_size().await.unwrap(); if size >= target { break; } @@ -107,7 +107,7 @@ pub async fn wait_for_until_cluster_size_decrease(raft: Raft, target: usize) { )); loop { - let size = raft.get_cluster_size().await; + let size = raft.get_cluster_size().await.unwrap(); if size <= target { break; } diff --git a/harness/tests/data_replication.rs b/harness/tests/data_replication.rs index 73699cbb..cad05036 100644 --- a/harness/tests/data_replication.rs +++ b/harness/tests/data_replication.rs @@ -37,7 +37,7 @@ pub async fn test_data_replication() { // Data should be replicated to all nodes. for (_, raft) in rafts.iter_mut() { - let store = raft.state_machine().await; + let store = raft.state_machine().await.unwrap(); let store_lk = store.0.read().unwrap(); assert_eq!(store_lk.get(&1).unwrap(), "test"); } @@ -59,7 +59,7 @@ pub async fn test_data_replication() { wait_for_until_cluster_size_increase(raft_1.clone(), 4).await; let raft_4 = rafts.get(&4).unwrap(); - let store = raft_4.state_machine().await; + let store = raft_4.state_machine().await.unwrap(); let store_lk = store.0.read().unwrap(); // Data should be replicated to new joined node. @@ -80,7 +80,7 @@ pub async fn test_data_replication() { // New entry data should be replicated to all nodes including new joined node. for (_, raft) in rafts.iter() { // stop - let store = raft.state_machine().await; + let store = raft.state_machine().await.unwrap(); let store_lk = store.0.read().unwrap(); assert_eq!(store_lk.get(&2).unwrap(), "test2"); } diff --git a/harness/tests/leader_election.rs b/harness/tests/leader_election.rs index d56adec4..77dc8b4f 100644 --- a/harness/tests/leader_election.rs +++ b/harness/tests/leader_election.rs @@ -36,7 +36,7 @@ pub async fn test_leader_election_in_three_node_example() { wait_for_until_cluster_size_decrease(raft_2.clone(), 2).await; - let leader_id = raft_2.get_leader_id().await; + let leader_id = raft_2.get_leader_id().await.unwrap(); let timer = timeout(Duration::from_secs(5), async { while leader_id == 0 { @@ -85,7 +85,7 @@ pub async fn test_leader_election_in_five_node_example() { sleep(Duration::from_secs(2)).await; - let leader_id = raft_2.get_leader_id().await; + let leader_id = raft_2.get_leader_id().await.unwrap(); assert!( [2, 3, 4, 5].contains(&leader_id), @@ -106,10 +106,10 @@ pub async fn test_leader_election_in_five_node_example() { wait_for_until_cluster_size_decrease(raft_k.clone(), 3).await; sleep(Duration::from_secs(2)).await; - let leader_id = raft_k.get_leader_id().await; + let leader_id = raft_k.get_leader_id().await.unwrap(); assert!(leader_id != 0); - assert_eq!(raft_k.get_cluster_size().await, 3); + assert_eq!(raft_k.get_cluster_size().await.unwrap(), 3); sleep(Duration::from_secs(2)).await; diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index 9c78c8ef..03aa4c3d 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -106,63 +106,68 @@ impl< }) } - pub async fn is_leader(&self) -> bool { + pub async fn is_leader(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::IsLeader { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::IsLeader { is_leader } => is_leader, + LocalResponseMsg::IsLeader { is_leader } => Ok(is_leader), _ => unreachable!(), } } - pub async fn get_id(&self) -> u64 { + pub async fn get_id(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetId { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::GetId { id } => id, + LocalResponseMsg::GetId { id } => Ok(id), _ => unreachable!(), } } - pub async fn get_leader_id(&self) -> u64 { + pub async fn get_leader_id(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetLeaderId { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::GetLeaderId { leader_id } => leader_id, + LocalResponseMsg::GetLeaderId { leader_id } => Ok(leader_id), _ => unreachable!(), } } - pub async fn get_peers(&self) -> Peers { + pub async fn get_peers(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetPeers { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::GetPeers { peers } => peers, + LocalResponseMsg::GetPeers { peers } => Ok(peers), _ => unreachable!(), } } - pub async fn add_peer(&self, id: u64, addr: A, role: Option) { + pub async fn add_peer( + &self, + id: u64, + addr: A, + role: Option, + ) -> Result<()> { let addr = addr.to_socket_addrs().unwrap().next().unwrap().to_string(); let (tx, rx) = oneshot::channel(); self.tx_local @@ -174,24 +179,24 @@ impl< }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::AddPeer {} => (), + LocalResponseMsg::AddPeer {} => Ok(()), _ => unreachable!(), } } - pub async fn add_peers(&self, peers: HashMap) { + pub async fn add_peers(&self, peers: HashMap) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::AddPeers { peers, tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::AddPeers {} => (), + LocalResponseMsg::AddPeers {} => Ok(()), _ => unreachable!(), } } @@ -202,7 +207,7 @@ impl< .send(LocalRequestMsg::DebugNode { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { LocalResponseMsg::DebugNode { result_json } => Ok(result_json), @@ -210,30 +215,30 @@ impl< } } - pub async fn state_machine(&self) -> FSM { + pub async fn state_machine(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetStateMachine { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::GetStateMachine { store } => store, + LocalResponseMsg::GetStateMachine { store } => Ok(store), _ => unreachable!(), } } - pub async fn storage(&self) -> LogStorage { + pub async fn storage(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetStorage { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::GetStorage { storage } => storage, + LocalResponseMsg::GetStorage { storage } => Ok(storage), _ => unreachable!(), } } @@ -248,7 +253,7 @@ impl< .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { LocalResponseMsg::Propose { result } => match result { ResponseResult::Success => (), @@ -265,7 +270,10 @@ impl< Ok(()) } - pub async fn change_config(&self, conf_change: ConfChangeV2) -> ConfChangeResponseResult { + pub async fn change_config( + &self, + conf_change: ConfChangeV2, + ) -> Result { let (tx, rx) = oneshot::channel(); let conf_change: ConfChangeRequest = conf_change.into(); self.tx_local @@ -276,7 +284,7 @@ impl< .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { LocalResponseMsg::ConfigChange { result } => match result { ConfChangeResponseResult::WrongLeader { leader_addr, .. } => { @@ -291,48 +299,48 @@ impl< if result.result_type == raft_service::ChangeConfigResultType::ChangeConfigSuccess as i32 { - ConfChangeResponseResult::JoinSuccess { + Ok(ConfChangeResponseResult::JoinSuccess { assigned_ids: result.assigned_ids, peers: deserialize(result.peers.as_slice()).unwrap(), - } + }) } else { - ConfChangeResponseResult::Error(Error::Unknown) + Ok(ConfChangeResponseResult::Error(Error::Unknown)) } } - _ => result, + _ => Ok(result), }, _ => unreachable!(), } } - pub async fn get_cluster_size(&self) -> usize { + pub async fn get_cluster_size(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetClusterSize { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::GetClusterSize { size } => size, + LocalResponseMsg::GetClusterSize { size } => Ok(size), _ => unreachable!(), } } - pub async fn quit(&self) { + pub async fn quit(&self) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::Quit { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::Quit {} => (), + LocalResponseMsg::Quit {} => Ok(()), _ => unreachable!(), } } - pub async fn transfer_leader(&self, node_id: u64) { + pub async fn transfer_leader(&self, node_id: u64) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::TransferLeader { @@ -341,27 +349,27 @@ impl< }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::TransferLeader {} => (), + LocalResponseMsg::TransferLeader {} => Ok(()), _ => unreachable!(), } } - pub async fn campaign(&self) { + pub async fn campaign(&self) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::Campaign { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::Campaign {} => (), + LocalResponseMsg::Campaign {} => Ok(()), _ => unreachable!(), } } - pub async fn demote(&self, term: u64, leader_id: u64) { + pub async fn demote(&self, term: u64, leader_id: u64) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::Demote { @@ -371,22 +379,22 @@ impl< }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::Demote {} => (), + LocalResponseMsg::Demote {} => Ok(()), _ => unreachable!(), } } - pub async fn leave(&self) { + pub async fn leave(&self) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::Leave { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::ConfigChange { result: _result } => (), + LocalResponseMsg::ConfigChange { result: _result } => Ok(()), _ => unreachable!(), } } @@ -398,7 +406,7 @@ impl< .unwrap(); } - pub async fn send_message(&self, message: RaftMessage) { + pub async fn send_message(&self, message: RaftMessage) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::SendMessage { @@ -407,9 +415,9 @@ impl< }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::SendMessage {} => (), + LocalResponseMsg::SendMessage {} => Ok(()), _ => unreachable!(), } } @@ -431,7 +439,7 @@ impl< } } - pub async fn join_cluster(&self, tickets: Vec) { + pub async fn join_cluster(&self, tickets: Vec) -> Result<()> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::JoinCluster { @@ -440,25 +448,25 @@ impl< }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::JoinCluster {} => (), + LocalResponseMsg::JoinCluster {} => Ok(()), _ => unreachable!(), } } /// # Safety /// TODO: Write this. - pub async unsafe fn get_raw_node(&self) -> Arc>> { + pub async unsafe fn get_raw_node(&self) -> Result>>> { let (tx, rx) = oneshot::channel(); self.tx_local .send(LocalRequestMsg::GetRawNode { tx_msg: tx }) .await .unwrap(); - let resp = rx.await.unwrap(); + let resp = rx.await?; match resp { - LocalResponseMsg::GetRawNode { raw_node } => raw_node, + LocalResponseMsg::GetRawNode { raw_node } => Ok(raw_node), _ => unreachable!(), } } @@ -479,7 +487,7 @@ pub struct RaftNodeCore< FSM: AbstractStateMachine + Clone + 'static, > { pub raw_node: RawNode, - // pub log_storage: LogStorage, + // pub log_storage: LogStorage, # Since there is no particular reason to store it, we do not save the log_storage. pub fsm: FSM, pub peers: Arc>, response_seq: AtomicU64, @@ -600,7 +608,6 @@ impl< tx_self, rx_self, _phantom_log_entry_typ: PhantomData, - // log_storage, }) } @@ -1182,14 +1189,7 @@ impl< term, tx_msg, } => { - // println!("Make snapshot!! 2 2"); - // let r = self.make_snapshot(index, term).await; - // println!("Make snapshot!! 2 3 r: {:?}", r); - // if let Err(e) = r { - // return Err(e); - // } self.make_snapshot(index, term).await?; - println!("Make snapshot!! 2 3"); tx_msg.send(LocalResponseMsg::MakeSnapshot {}).unwrap(); } LocalRequestMsg::JoinCluster { tickets, tx_msg } => { From 9936b254535cf55ab209d63bb5a1e891aa56f3e0 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Mon, 30 Sep 2024 11:52:35 +0900 Subject: [PATCH 19/19] fix: dynamic bootstrapping --- examples/memstore/dynamic-members/src/main.rs | 26 ++++++++++++++----- raftify/src/raft_node/mod.rs | 2 -- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/examples/memstore/dynamic-members/src/main.rs b/examples/memstore/dynamic-members/src/main.rs index ce53f160..bae0c886 100644 --- a/examples/memstore/dynamic-members/src/main.rs +++ b/examples/memstore/dynamic-members/src/main.rs @@ -10,7 +10,7 @@ use raftify::{ }; use slog::Drain; use slog_envlogger::LogBuilder; -use std::sync::Arc; +use std::{fs, path::Path, sync::Arc}; use structopt::StructOpt; use example_harness::{ @@ -98,16 +98,30 @@ async fn main() -> std::result::Result<(), Box> { )?; let handle = tokio::spawn(raft.clone().run()); - raft.add_peers(ticket.peers.clone()).await; - raft.join_cluster(vec![ticket]).await; + raft.add_peers(ticket.peers.clone()) + .await + .expect("Failed to add peers"); + raft.join_cluster(vec![ticket]) + .await + .expect("Failed to join cluster"); (raft, handle) } None => { log::info!("Bootstrap a Raft Cluster"); - let cfg: raftify::Config = build_config(1); - let storage_pth = get_storage_path(cfg.log_dir.as_str(), 1); + // NOTE: Due to the characteristic of dynamic bootstrapping, + // it cannot be bootstrapped directly from the WAL log. + // Therefore, in this example, we delete the previous logs if they exist and then bootstrap. + let log_dir = Path::new("./logs"); + + if fs::metadata(log_dir).is_ok() { + fs::remove_dir_all(log_dir)?; + } + + let leader_node_id = 1; + let cfg = build_config(leader_node_id); + let storage_pth = get_storage_path(cfg.log_dir.as_str(), leader_node_id); ensure_directory_exist(storage_pth.as_str())?; #[cfg(feature = "inmemory_storage")] @@ -118,7 +132,7 @@ async fn main() -> std::result::Result<(), Box> { .expect("Failed to create heed storage"); let raft = Raft::bootstrap( - 1, + leader_node_id, options.raft_addr, log_storage, store.clone(), diff --git a/raftify/src/raft_node/mod.rs b/raftify/src/raft_node/mod.rs index 03aa4c3d..5462c629 100644 --- a/raftify/src/raft_node/mod.rs +++ b/raftify/src/raft_node/mod.rs @@ -533,8 +533,6 @@ impl< rx_local: mpsc::Receiver>, ) -> Result { assert_eq!(config.raft_config.id, node_id); - println!("node_id: {:?}", node_id); - println!("config: {:?}", config.raft_config.id); config.validate()?; let mut snapshot = log_storage.snapshot(0, log_storage.last_index()?)?;