From b7ee0171777368ccaed745b39d98771b94f32e1a Mon Sep 17 00:00:00 2001 From: Parth Desai Date: Tue, 16 Jan 2024 17:25:27 +0400 Subject: [PATCH] upgrade sdk to gemini-3g-2024-jan-08 --- Cargo.lock | 313 ++++++++++++++++++++++++++++++++++++++-------- Cargo.toml | 4 +- dsn/Cargo.toml | 6 +- farmer/Cargo.toml | 15 +-- farmer/src/lib.rs | 153 +++++++++++++--------- node/Cargo.toml | 46 +++---- traits/Cargo.toml | 6 +- utils/Cargo.toml | 12 +- 8 files changed, 399 insertions(+), 156 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcee9956..516e5025 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -911,6 +911,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "autotools" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77" +dependencies = [ + "cc", +] + [[package]] name = "axum" version = "0.6.20" @@ -1567,6 +1576,15 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -1919,7 +1937,7 @@ checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" [[package]] name = "cross-domain-message-gossip" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "futures", "parity-scale-codec", @@ -2057,6 +2075,12 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "cty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" + [[package]] name = "curve25519-dalek" version = "2.1.3" @@ -2506,7 +2530,7 @@ dependencies = [ [[package]] name = "domain-block-builder" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -2523,7 +2547,7 @@ dependencies = [ [[package]] name = "domain-block-preprocessor" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-trait", "domain-runtime-primitives", @@ -2549,7 +2573,7 @@ dependencies = [ [[package]] name = "domain-client-consensus-relay-chain" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-trait", "sc-consensus", @@ -2563,7 +2587,7 @@ dependencies = [ [[package]] name = "domain-client-message-relayer" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-channel", "cross-domain-message-gossip", @@ -2584,7 +2608,7 @@ dependencies = [ [[package]] name = "domain-client-operator" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "domain-block-builder", "domain-block-preprocessor", @@ -2625,7 +2649,7 @@ dependencies = [ [[package]] name = "domain-client-subnet-gossip" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "futures", "parity-scale-codec", @@ -2644,7 +2668,7 @@ dependencies = [ [[package]] name = "domain-eth-service" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "clap", "domain-runtime-primitives", @@ -2679,7 +2703,7 @@ dependencies = [ [[package]] name = "domain-pallet-executive" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-benchmarking", "frame-executive", @@ -2698,7 +2722,7 @@ dependencies = [ [[package]] name = "domain-runtime-primitives" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "parity-scale-codec", "scale-info", @@ -2714,7 +2738,7 @@ dependencies = [ [[package]] name = "domain-service" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-trait", "cross-domain-message-gossip", @@ -3085,7 +3109,7 @@ dependencies = [ [[package]] name = "evm-domain-runtime" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "domain-pallet-executive", "domain-runtime-primitives", @@ -3565,7 +3589,7 @@ source = "git+https://github.com/subspace/frontier?rev=37ee45323120b21adc1d69ae7 dependencies = [ "evm", "frame-support", - "num_enum", + "num_enum 0.6.1", "parity-scale-codec", "scale-info", "serde", @@ -4488,6 +4512,40 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hwlocality" +version = "1.0.0-alpha.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6020affad7f95b46f12607a8714aa70bd02c8df3b3abf9ef5c8cd2f7ae57a033" +dependencies = [ + "arrayvec 0.7.4", + "bitflags 2.4.1", + "derive_more", + "errno", + "hwlocality-sys", + "libc", + "num_enum 0.7.2", + "thiserror", +] + +[[package]] +name = "hwlocality-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ecc1a90eeca65f95f08a7d32132cd4d35c9ee95ed89e32d0342a4bf7b5d644" +dependencies = [ + "autotools", + "cmake", + "flate2", + "hex-literal", + "libc", + "pkg-config", + "reqwest", + "sha3", + "tar", + "windows-sys 0.52.0", +] + [[package]] name = "hyper" version = "0.14.27" @@ -5097,6 +5155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3979b5c37ece694f1f5e51e7ecc871fdb0f517ed04ee45f88d15d6d553cb9664" dependencies = [ "cc", + "cty", "libc", ] @@ -6835,7 +6894,16 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1" dependencies = [ - "num_enum_derive", + "num_enum_derive 0.6.1", +] + +[[package]] +name = "num_enum" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +dependencies = [ + "num_enum_derive 0.7.2", ] [[package]] @@ -6850,6 +6918,17 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "num_enum_derive" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "object" version = "0.30.4" @@ -6926,7 +7005,7 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "orml-vesting" version = "0.4.1-dev" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-support", "frame-system", @@ -6998,7 +7077,7 @@ dependencies = [ [[package]] name = "pallet-domain-id" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-support", "frame-system", @@ -7011,7 +7090,7 @@ dependencies = [ [[package]] name = "pallet-domains" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "domain-runtime-primitives", "frame-benchmarking", @@ -7123,7 +7202,7 @@ dependencies = [ [[package]] name = "pallet-messenger" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-benchmarking", "frame-support", @@ -7142,7 +7221,7 @@ dependencies = [ [[package]] name = "pallet-offences-subspace" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-support", "frame-system", @@ -7156,7 +7235,7 @@ dependencies = [ [[package]] name = "pallet-operator-rewards" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-support", "frame-system", @@ -7169,7 +7248,7 @@ dependencies = [ [[package]] name = "pallet-rewards" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-support", "frame-system", @@ -7181,7 +7260,7 @@ dependencies = [ [[package]] name = "pallet-runtime-configs" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-support", "frame-system", @@ -7194,7 +7273,7 @@ dependencies = [ [[package]] name = "pallet-subspace" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-benchmarking", "frame-support", @@ -7252,7 +7331,7 @@ dependencies = [ [[package]] name = "pallet-transaction-fees" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-support", "frame-system", @@ -7308,7 +7387,7 @@ dependencies = [ [[package]] name = "pallet-transporter" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "domain-runtime-primitives", "frame-benchmarking", @@ -8267,6 +8346,46 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "reqwest" +version = "0.11.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +dependencies = [ + "base64 0.21.5", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite 0.2.13", + "rustls 0.21.8", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.25.2", + "winreg", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -8846,7 +8965,7 @@ dependencies = [ [[package]] name = "sc-consensus-subspace" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-trait", "futures", @@ -8886,7 +9005,7 @@ dependencies = [ [[package]] name = "sc-consensus-subspace-rpc" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-oneshot", "futures", @@ -9203,7 +9322,7 @@ dependencies = [ [[package]] name = "sc-proof-of-time" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "atomic", "core_affinity", @@ -9423,7 +9542,7 @@ dependencies = [ [[package]] name = "sc-subspace-block-relay" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-channel", "async-trait", @@ -9448,7 +9567,7 @@ dependencies = [ [[package]] name = "sc-subspace-chain-specs" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "sc-chain-spec", "sc-service", @@ -9723,6 +9842,7 @@ dependencies = [ "derive_builder 0.12.0", "derive_more", "futures", + "libmimalloc-sys", "lru 0.11.1", "parking_lot 0.12.1", "pin-project", @@ -10437,7 +10557,7 @@ dependencies = [ [[package]] name = "sp-consensus-subspace" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-trait", "log", @@ -10552,7 +10672,7 @@ dependencies = [ [[package]] name = "sp-domain-digests" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -10561,7 +10681,7 @@ dependencies = [ [[package]] name = "sp-domains" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "blake2", "domain-runtime-primitives", @@ -10592,7 +10712,7 @@ dependencies = [ [[package]] name = "sp-domains-fraud-proof" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "domain-block-preprocessor", "domain-runtime-primitives", @@ -10624,7 +10744,7 @@ dependencies = [ [[package]] name = "sp-executive" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-trait", "parity-scale-codec", @@ -10716,7 +10836,7 @@ dependencies = [ [[package]] name = "sp-messenger" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "frame-support", "hash-db 0.16.0", @@ -10746,7 +10866,7 @@ dependencies = [ [[package]] name = "sp-objects" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "sp-api", "sp-std", @@ -11209,7 +11329,7 @@ dependencies = [ [[package]] name = "subspace-archiving" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "parity-scale-codec", "rayon", @@ -11222,7 +11342,7 @@ dependencies = [ [[package]] name = "subspace-core-primitives" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "blake3", "derive_more", @@ -11245,7 +11365,7 @@ dependencies = [ [[package]] name = "subspace-erasure-coding" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "kzg", "rust-kzg-blst", @@ -11255,7 +11375,7 @@ dependencies = [ [[package]] name = "subspace-farmer" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "anyhow", "async-lock", @@ -11273,9 +11393,13 @@ dependencies = [ "fs4 0.7.0", "futures", "hex", + "hwlocality", "jsonrpsee", + "libc", + "libmimalloc-sys", "lru 0.11.1", "mimalloc", + "num_cpus", "parity-scale-codec", "parking_lot 0.12.1", "prometheus-client 0.22.0", @@ -11302,13 +11426,14 @@ dependencies = [ "tracing", "tracing-subscriber 0.3.18", "ulid", + "windows-sys 0.52.0", "zeroize", ] [[package]] name = "subspace-farmer-components" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-lock", "async-trait", @@ -11338,7 +11463,7 @@ dependencies = [ [[package]] name = "subspace-metrics" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "actix-web", "parking_lot 0.12.1", @@ -11350,7 +11475,7 @@ dependencies = [ [[package]] name = "subspace-networking" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "actix-web", "async-mutex", @@ -11389,7 +11514,7 @@ dependencies = [ [[package]] name = "subspace-proof-of-space" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "chacha20", "derive_more", @@ -11402,7 +11527,7 @@ dependencies = [ [[package]] name = "subspace-proof-of-time" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "aes 0.8.3", "subspace-core-primitives", @@ -11412,7 +11537,7 @@ dependencies = [ [[package]] name = "subspace-rpc-primitives" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "hex", "serde", @@ -11424,7 +11549,7 @@ dependencies = [ [[package]] name = "subspace-runtime" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "domain-runtime-primitives", "frame-benchmarking", @@ -11476,7 +11601,7 @@ dependencies = [ [[package]] name = "subspace-runtime-primitives" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "pallet-transaction-payment", "serde", @@ -11517,7 +11642,7 @@ dependencies = [ [[package]] name = "subspace-service" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "async-trait", "atomic", @@ -11587,7 +11712,7 @@ dependencies = [ [[package]] name = "subspace-verification" version = "0.1.0" -source = "git+https://github.com/subspace/subspace?rev=d504fed67492e5363b34308767d3281a0b9e21cf#d504fed67492e5363b34308767d3281a0b9e21cf" +source = "git+https://github.com/subspace/subspace?rev=bd435100200b3dcce6d6f50534d52e3cd039ca8e#bd435100200b3dcce6d6f50534d52e3cd039ca8e" dependencies = [ "parity-scale-codec", "schnorrkel", @@ -11757,6 +11882,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.12" @@ -13243,6 +13379,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -13273,6 +13418,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -13285,6 +13445,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -13297,6 +13463,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -13309,6 +13481,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -13321,6 +13499,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -13333,6 +13517,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -13345,6 +13535,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -13357,6 +13553,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" version = "0.5.19" @@ -13462,6 +13664,15 @@ dependencies = [ "time", ] +[[package]] +name = "xattr" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc6ab6ec1907d1a901cdbcd2bd4cb9e7d64ce5c9739cbb97d3c391acd8c7fae" +dependencies = [ + "libc", +] + [[package]] name = "xml-rs" version = "0.8.19" diff --git a/Cargo.toml b/Cargo.toml index bbf11127..ec025fb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ sdk-substrate = { path = "substrate" } sdk-utils = { path = "utils" } static_assertions = "1.1.0" -subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } # The only triple tested and confirmed as working in `jemallocator` crate is `x86_64-unknown-linux-gnu` [target.'cfg(all(target_arch = "x86_64", target_vendor = "unknown", target_os = "linux", target_env = "gnu"))'.dev-dependencies] @@ -28,7 +28,7 @@ derive_more = "0.99" fdlimit = "0.2" futures = "0.3" serde_json = "1" -subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } tempfile = "3" tokio = { version = "1.34.0", features = ["rt-multi-thread", "macros"] } tracing = "0.1" diff --git a/dsn/Cargo.toml b/dsn/Cargo.toml index 79d15464..82d7f2c4 100644 --- a/dsn/Cargo.toml +++ b/dsn/Cargo.toml @@ -14,13 +14,13 @@ hex = "0.4.3" parking_lot = "0.12" prometheus-client = "0.22.0" sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } -sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } sdk-utils = { path = "../utils" } serde = { version = "1", features = ["derive"] } sp-blockchain = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } -subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-networking = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-networking = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } tracing = "0.1" [features] diff --git a/farmer/Cargo.toml b/farmer/Cargo.toml index a94b3477..86071a61 100644 --- a/farmer/Cargo.toml +++ b/farmer/Cargo.toml @@ -13,19 +13,20 @@ derive_builder = "0.12" derive_more = "0.99" futures = "0.3" lru = "0.11.0" +libmimalloc-sys = { version = "0.1.35", features = ["extended"] } parking_lot = "0.12" pin-project = "1" rayon = "1.7.0" sdk-traits = { path = "../traits" } sdk-utils = { path = "../utils" } serde = { version = "1", features = ["derive"] } -subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-networking = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf", features = ["parallel"] } -subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-erasure-coding = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-networking = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e", features = ["parallel"] } +subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } thiserror = "1" tokio = { version = "1.34.0", features = ["fs", "rt", "tracing", "macros", "parking_lot", "rt-multi-thread", "signal"] } tokio-stream = { version = "0.1", features = ["sync", "time"] } diff --git a/farmer/src/lib.rs b/farmer/src/lib.rs index 3678fda9..34bde5f3 100644 --- a/farmer/src/lib.rs +++ b/farmer/src/lib.rs @@ -36,6 +36,7 @@ use subspace_farmer::utils::farmer_piece_getter::FarmerPieceGetter; use subspace_farmer::utils::piece_validator::SegmentCommitmentPieceValidator; use subspace_farmer::utils::readers_and_pieces::ReadersAndPieces; use subspace_farmer::{Identity, KNOWN_PEERS_CACHE_SIZE}; +use subspace_farmer::thread_pool_manager::PlottingThreadPoolManager; use subspace_farmer_components::plotting::PlottedSector; use subspace_farmer_components::sector::{sector_size, SectorMetadataChecksummed}; use subspace_networking::libp2p::kad::RecordKey; @@ -43,8 +44,9 @@ use subspace_networking::utils::multihash::ToMultihash; use subspace_networking::KnownPeersManager; use subspace_rpc_primitives::{FarmerAppInfo, SolutionResponse}; use tokio::sync::{mpsc, oneshot, watch, Mutex, Semaphore}; -use tracing::{debug, error, warn}; +use tracing::{debug, error, info, warn}; use tracing_futures::Instrument; +use subspace_farmer::utils::{all_cpu_cores, create_plotting_thread_pool_manager, thread_pool_core_indices}; /// Description of the farm #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] @@ -77,26 +79,10 @@ mod builder { use sdk_traits::Node; use sdk_utils::{ByteSize, PublicKey}; use serde::{Deserialize, Serialize}; - use tracing::warn; use super::BuildError; use crate::{FarmDescription, Farmer}; - fn available_parallelism() -> usize { - match std::thread::available_parallelism() { - Ok(parallelism) => parallelism.get(), - Err(error) => { - warn!( - %error, - "Unable to identify available parallelism, you might want to configure thread pool sizes with CLI \ - options manually" - ); - - 0 - } - } - } - #[derive( Debug, Clone, @@ -175,38 +161,33 @@ mod builder { pub max_pieces_in_sector: Option, /// Size of PER FARM thread pool used for farming (mostly for blocking /// I/O, but also for some compute-intensive operations during - /// proving), defaults to number of CPU cores available in - /// the system - #[builder(default = "available_parallelism()")] - pub farming_thread_pool_size: usize, - /// Size of PER FARM thread pool used for plotting, defaults to number - /// of CPU cores available in the system. + /// proving), defaults to number of logical CPUs + /// available on UMA system and number of logical CPUs in + /// first NUMA node on NUMA system. + #[builder(default)] + pub farming_thread_pool_size: Option, + /// Size of one thread pool used for plotting, defaults to number of logical CPUs available + /// on UMA system and number of logical CPUs available in NUMA node on NUMA system. /// - /// NOTE: The fact that this parameter is per farm doesn't mean farmer - /// will plot multiple sectors concurrently, see - /// `sector_downloading_concurrency` and - /// `sector_encoding_concurrency` options. - #[builder(default = "available_parallelism()")] - pub plotting_thread_pool_size: usize, - /// Size of PER FARM thread pool used for replotting, typically smaller - /// pool than for plotting to not affect farming as much, - /// defaults to half of the number of CPU cores available in the - /// system. + /// Number of thread pools is defined by `--sector-encoding-concurrency` option, different + /// thread pools might have different number of threads if NUMA nodes do not have the same size. /// - /// NOTE: The fact that this parameter is per farm doesn't mean farmer - /// will replot multiple sectors concurrently, see - /// `sector-downloading-concurrency` and - /// `sector-encoding-concurrency` options. - #[builder(default = "available_parallelism() / 2")] - pub replotting_thread_pool_size: usize, - /// Sector downloading concurrency - #[builder(default = "NonZeroUsize::new(2).expect(\"2 > 0\")")] - #[derivative(Default(value = "NonZeroUsize::new(2).expect(\"2 > 0\")"))] - pub sector_downloading_concurrency: NonZeroUsize, - /// Sector encoding concurrency - #[builder(default = "NonZeroUsize::new(1).expect(\"1 > 0\")")] - #[derivative(Default(value = "NonZeroUsize::new(1).expect(\"1 > 0\")"))] - pub sector_encoding_concurrency: NonZeroUsize, + /// Threads will be pinned to corresponding CPU cores at creation. + #[builder(default)] + pub plotting_thread_pool_size: Option, + /// the plotting process, defaults to `--sector-downloading-concurrency` + 1 to download future + /// sector ahead of time + #[builder(default)] + pub sector_downloading_concurrency: Option, + /// Defines how many sectors farmer will encode concurrently, defaults to 1 on UMA system and + /// number of NUMA nodes on NUMA system. It is further restricted by + /// `sector_downloading_concurrency` and setting this option higher than + /// `sector_downloading_concurrency` will have no effect. + #[builder(default)] + pub sector_encoding_concurrency: Option, + /// Threads will be pinned to corresponding CPU cores at creation. + #[builder(default)] + pub replotting_thread_pool_size: Option, } impl Builder { @@ -484,8 +465,66 @@ impl Config { }; let mut plotting_delay_senders = Vec::with_capacity(farms.len()); - let downloading_semaphore = Arc::new(Semaphore::new(sector_downloading_concurrency.get())); - let encoding_semaphore = Arc::new(Semaphore::new(sector_encoding_concurrency.get())); + + let plotting_thread_pool_core_indices = + thread_pool_core_indices(plotting_thread_pool_size, sector_encoding_concurrency); + let replotting_thread_pool_core_indices = { + let mut replotting_thread_pool_core_indices = + thread_pool_core_indices(replotting_thread_pool_size, sector_encoding_concurrency); + if replotting_thread_pool_size.is_none() { + // The default behavior is to use all CPU cores, but for replotting we just want half + replotting_thread_pool_core_indices + .iter_mut() + .for_each(|set| set.truncate(set.cpu_cores().len() / 2)); + } + replotting_thread_pool_core_indices + }; + + let downloading_semaphore = Arc::new(Semaphore::new( + sector_downloading_concurrency + .map(|sector_downloading_concurrency| sector_downloading_concurrency.get()) + .unwrap_or(plotting_thread_pool_core_indices.len() + 1), + )); + + let all_cpu_cores = all_cpu_cores(); + let plotting_thread_pool_manager = create_plotting_thread_pool_manager( + plotting_thread_pool_core_indices + .into_iter() + .zip(replotting_thread_pool_core_indices), + )?; + let farming_thread_pool_size = farming_thread_pool_size + .map(|farming_thread_pool_size| farming_thread_pool_size.get()) + .unwrap_or_else(|| { + all_cpu_cores + .first() + .expect("Not empty according to function description; qed") + .cpu_cores() + .len() + }); + + if all_cpu_cores.len() > 1 { + info!(numa_nodes = %all_cpu_cores.len(), "NUMA system detected"); + + if all_cpu_cores.len() > farms.len() { + warn!( + numa_nodes = %all_cpu_cores.len(), + farms_count = %farms.len(), + "Too few disk farms, CPU will not be utilized fully during plotting, same number of farms as NUMA \ + nodes or more is recommended" + ); + } + } + + // TODO: Remove code or environment variable once identified whether it helps or not + if std::env::var("NUMA_ALLOCATOR").is_ok() && all_cpu_cores.len() > 1 { + unsafe { + libmimalloc_sys::mi_option_set( + libmimalloc_sys::mi_option_use_numa_nodes, + all_cpu_cores.len() as std::ffi::c_long, + ); + } + } + for (disk_farm_idx, description) in farms.iter().enumerate() { let (plotting_delay_sender, plotting_delay_receiver) = @@ -503,11 +542,9 @@ impl Config { kzg: kzg.clone(), erasure_coding: erasure_coding.clone(), farming_thread_pool_size, - plotting_thread_pool_size, - replotting_thread_pool_size, plotting_delay: Some(plotting_delay_receiver), downloading_semaphore: Arc::clone(&downloading_semaphore), - encoding_semaphore: Arc::clone(&encoding_semaphore), + plotting_thread_pool_manager: plotting_thread_pool_manager.clone() }) .await?; @@ -835,11 +872,9 @@ struct FarmOptions<'a, PG, N: sdk_traits::Node> { pub erasure_coding: ErasureCoding, pub max_pieces_in_sector: u16, pub farming_thread_pool_size: usize, - pub plotting_thread_pool_size: usize, - pub replotting_thread_pool_size: usize, pub plotting_delay: Option>, pub downloading_semaphore: Arc, - pub encoding_semaphore: Arc, + pub plotting_thread_pool_manager: PlottingThreadPoolManager, } impl Farm { @@ -855,11 +890,9 @@ impl Farm { erasure_coding, max_pieces_in_sector, farming_thread_pool_size, - plotting_thread_pool_size, - replotting_thread_pool_size, plotting_delay, downloading_semaphore, - encoding_semaphore, + plotting_thread_pool_manager }: FarmOptions< '_, impl subspace_farmer_components::plotting::PieceGetter + Clone + Send + Sync + 'static, @@ -884,11 +917,9 @@ impl Farm { piece_getter, cache_percentage, downloading_semaphore, - encoding_semaphore, farm_during_initial_plotting: false, farming_thread_pool_size, - plotting_thread_pool_size, - replotting_thread_pool_size, + plotting_thread_pool_manager, plotting_delay, }; let single_disk_farm_fut = SingleDiskFarm::new::<_, _, T>(description, disk_farm_idx); diff --git a/node/Cargo.toml b/node/Cargo.toml index c39dbcc5..861b2eec 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -7,35 +7,35 @@ edition = "2021" [dependencies] anyhow = "1" backoff = "0.4" -cross-domain-message-gossip = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +cross-domain-message-gossip = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } derivative = "2.2.0" derive_builder = "0.12" derive_more = "0.99" -domain-client-message-relayer = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -domain-client-operator = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -domain-eth-service = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -domain-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -domain-service = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -evm-domain-runtime = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +domain-client-message-relayer = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +domain-client-operator = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +domain-eth-service = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +domain-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +domain-service = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +evm-domain-runtime = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } fp-evm = { version = "3.0.0-dev", git = "https://github.com/subspace/frontier", rev = "37ee45323120b21adc1d69ae7348bd0f7282eeae" } frame-system = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } futures = "0.3" hex-literal = "0.4" -pallet-rewards = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -pallet-subspace = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +pallet-rewards = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +pallet-subspace = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } parity-scale-codec = "3.6.3" parking_lot = "0.12" pin-project = "1" sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sc-consensus-slots = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } -sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +sc-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } sc-executor = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sc-network = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sc-network-sync = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sc-rpc-api = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sc-service = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } sc-storage-monitor = { version = "0.1.0", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } -sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +sc-subspace-chain-specs = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } sc-telemetry = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sc-transaction-pool-api = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sc-utils = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } @@ -47,21 +47,21 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sp-blockchain = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sp-consensus = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } -sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +sp-consensus-subspace = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } sp-core = { version = "21.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } -sp-domains = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -sp-messenger = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +sp-domains = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +sp-domains-fraud-proof = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +sp-messenger = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sp-version = { version = "22.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } -subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-networking = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-service = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-farmer-components = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-networking = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-service = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } tokio = { version = "1.34.0", features = ["fs", "rt", "tracing", "macros", "parking_lot", "rt-multi-thread", "signal"] } tokio-stream = { version = "0.1", features = ["sync", "time"] } tracing = "0.1" diff --git a/traits/Cargo.toml b/traits/Cargo.toml index 382236c8..bb2260b5 100644 --- a/traits/Cargo.toml +++ b/traits/Cargo.toml @@ -9,9 +9,9 @@ async-trait = "0.1" parking_lot = "0.12" sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } sdk-dsn = { path = "../dsn" } -subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-proof-of-space = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } [features] default = [] diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 87283374..138e8d4d 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -19,7 +19,7 @@ futures = "0.3" jsonrpsee-core = "0.16" libp2p-core = { git = "https://github.com/subspace/rust-libp2p", rev = "d6339da35589d86bae6ecb25a5121c02f2e5b90e" } parity-scale-codec = "3.6.3" -sc-consensus-subspace-rpc = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +sc-consensus-subspace-rpc = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } sc-network = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } sc-rpc = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } sc-rpc-api = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c", default-features = false } @@ -32,11 +32,11 @@ sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/polkadot-s sp-storage = { version = "13.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "c63a8b28a9fd26d42116b0dcef1f2a5cefb9cd1c" } ss58-registry = "1.33" # Unused for now. TODO: add `serde` feature to `subspace-core-primitives` in `subspace-archiver` -subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } -subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "d504fed67492e5363b34308767d3281a0b9e21cf" } +subspace-core-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-farmer = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-rpc-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-runtime = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } +subspace-runtime-primitives = { git = "https://github.com/subspace/subspace", rev = "bd435100200b3dcce6d6f50534d52e3cd039ca8e" } thiserror = "1" tokio = { version = "1.34.0", features = ["fs", "rt", "tracing", "macros", "parking_lot", "rt-multi-thread", "signal"] } tracing = "0.1"