diff --git a/.config/forest.dic b/.config/forest.dic index 0021f6a51e9a..c289a51613ec 100644 --- a/.config/forest.dic +++ b/.config/forest.dic @@ -10,10 +10,12 @@ BLAKE2b blockchain/M BLS callee +CAR/SM CBOR -CID/M +CID/SM CIDs CLI +clonable codec cron crypto @@ -22,6 +24,7 @@ daemonize DB/S DNS Enum +EOF Ethereum exa FIL @@ -40,7 +43,6 @@ Kademlia libp2p localhost mainnet -testnet MDNS mempool Merkle @@ -65,6 +67,7 @@ stdout synchronizer syscall/S TCP +testnet tipset/SM tipsetkey/S TOML @@ -72,6 +75,7 @@ trie TTY URL validator/S +varint verifier VM/SM VRF diff --git a/CHANGELOG.md b/CHANGELOG.md index 0879871786d7..7542714956d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,13 +31,18 @@ rocksdb - [#3047](https://github.com/ChainSafe/forest/pull/3047): Remove support for compiling with delegated consensus +- [#3086](https://github.com/ChainSafe/forest/pull/3085): + `forest-cli snapshot validate` no longer supports URLs. Download the snapshot + and then run the command. ### Added - [#2816](https://github.com/ChainSafe/forest/issues/2816): Support `2k` devnet. - - [#3026](https://github.com/ChainSafe/forest/pull/3026): Expose `forest-cli state diff ...` +- [#3086](https://github.com/ChainSafe/forest/pull/3085): + `forest-cli snapshot validate` is faster and uses less disk space, operating + directly on the snapshot rather than loading through a database. ### Changed diff --git a/Cargo.lock b/Cargo.lock index dfd2e0bc871c..8395ad96da70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,9 +71,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if 1.0.0", "cipher 0.4.4", @@ -101,7 +101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "209b47e8954a928e1d72e86eca7000ebb6655fe1436d33eefc2201cad027e237" dependencies = [ "aead 0.5.2", - "aes 0.8.2", + "aes 0.8.3", "cipher 0.4.4", "ctr 0.9.2", "ghash 0.5.0", @@ -198,15 +198,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anstyle-parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] @@ -288,9 +288,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8868f09ff8cea88b079da74ae569d9b8c62a23c68c746240b704ee6f7525c89c" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "asn1-rs" @@ -449,7 +449,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -466,7 +466,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -698,7 +698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" dependencies = [ "arrayref", - "arrayvec 0.7.3", + "arrayvec 0.7.4", "constant_time_eq 0.2.6", ] @@ -720,7 +720,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6637f448b9e61dfadbdcbae9a885fadee1f3eaffb1f8d3c1965d3ade8bdfd44f" dependencies = [ "arrayref", - "arrayvec 0.7.3", + "arrayvec 0.7.4", "constant_time_eq 0.2.6", ] @@ -731,7 +731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "729b71f35bd3fa1a4c86b85d32c8b9069ea7fe14f7a53cfabb65f62d4265b888" dependencies = [ "arrayref", - "arrayvec 0.7.3", + "arrayvec 0.7.4", "cc", "cfg-if 1.0.0", "constant_time_eq 0.2.6", @@ -899,9 +899,9 @@ checksum = "16acab52167770e9e6dc5c5fd3029750830ecaf1e8012575816a214846d22b7c" [[package]] name = "boa_unicode" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f7df6584b38b34a6bdb457bff1c96eadb01362d43420e61a89b301a5f3db2c1" +checksum = "53e4c2363034392bf23f3d2ad8b3ba50ec7de4126374b329370b6902c72487ca" dependencies = [ "unicode-general-category", ] @@ -1149,9 +1149,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.4" +version = "4.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80672091db20273a15cf9fdd4e47ed43b5091ec9841bf4c6145c9dfbbcae09ed" +checksum = "bba77a07e4489fb41bd90e8d4201c3eb246b3c2c9ea2ba0bddd6c1d1df87db7d" dependencies = [ "clap_builder", "clap_derive", @@ -1160,9 +1160,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.4" +version = "4.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1458a1df40e1e2afebb7ab60ce55c1fa8f431146205aa5f4887e0b111c27636" +checksum = "2c9b4a88bb4bc35d3d6f65a21b0f0bafe9c894fa00978de242c555ec28bea1c0" dependencies = [ "anstream", "anstyle", @@ -1180,7 +1180,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1790,7 +1790,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1812,7 +1812,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core 0.20.1", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1899,7 +1899,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2070,7 +2070,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2305,7 +2305,7 @@ checksum = "55a9a55d1dab3b07854648d48e366f684aefe2ac78ae28cec3bf65e3cd53d9a3" dependencies = [ "execute-command-tokens", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3151,7 +3151,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3587,7 +3587,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" dependencies = [ "libc", - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -3631,7 +3631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40" dependencies = [ "opaque-debug", - "polyval 0.6.0", + "polyval 0.6.1", ] [[package]] @@ -3688,9 +3688,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes 1.4.0", "fnv", @@ -3766,15 +3766,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.1" @@ -3907,9 +3898,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes 1.4.0", "futures-channel", @@ -4141,9 +4132,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" @@ -4263,9 +4254,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libipld" @@ -4536,7 +4527,7 @@ version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39d5ef876a2b2323d63c258e63c2f8e36f205fe5a11f0b3095d59635650790ff" dependencies = [ - "arrayvec 0.7.3", + "arrayvec 0.7.4", "asynchronous-codec", "bytes 1.4.0", "either", @@ -5562,11 +5553,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.1", "libc", ] @@ -5757,7 +5748,7 @@ dependencies = [ "libc", "redox_syscall 0.3.5", "smallvec", - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -5820,9 +5811,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", "phf_shared", @@ -5830,9 +5821,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", "rand 0.8.5", @@ -5840,22 +5831,22 @@ dependencies = [ [[package]] name = "phf_macros" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92aacdc5f16768709a569e913f7451034034178b05bdc8acda226659a3dccc66" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator", "phf_shared", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.22", ] [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] @@ -5877,7 +5868,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -6000,9 +5991,9 @@ dependencies = [ [[package]] name = "polyval" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6" +checksum = "d52cff9d1d4dee5fe6d03729099f4a310a41179e0a10dbf542039873f2e826fb" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -6115,9 +6106,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] @@ -6354,9 +6345,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ "proc-macro2", ] @@ -6844,9 +6835,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ "base64 0.21.2", ] @@ -7033,7 +7024,7 @@ checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -7050,9 +7041,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.97" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" dependencies = [ "itoa", "ryu", @@ -7076,7 +7067,7 @@ checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -7146,16 +7137,16 @@ dependencies = [ "darling 0.20.1", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] name = "serde_yaml" -version = "0.9.21" +version = "0.9.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85" dependencies = [ - "indexmap 1.9.3", + "indexmap 2.0.0", "itoa", "ryu", "serde", @@ -7367,7 +7358,7 @@ checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -7491,7 +7482,7 @@ version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f756ef2dd06efda2eb30bf6806399d493072d8469b0a724f1905dc051fea59c1" dependencies = [ - "aes 0.8.2", + "aes 0.8.3", "anyhow", "bellperson", "blake2b_simd", @@ -7687,9 +7678,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" dependencies = [ "proc-macro2", "quote", @@ -7743,9 +7734,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-lexicon" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" +checksum = "1b1c7f239eb94671427157bd93b3694320f3668d4e1eff08c7285366fd777fac" [[package]] name = "tempfile" @@ -7793,7 +7784,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -7906,11 +7897,12 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "374442f06ee49c3a28a8fc9f01a2596fed7559c6b99b31279c3261778e77d84f" dependencies = [ "autocfg", + "backtrace", "bytes 1.4.0", "libc", "mio", @@ -7942,7 +7934,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -8122,13 +8114,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -8451,9 +8443,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.3.4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" +checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" dependencies = [ "getrandom 0.2.10", ] @@ -8570,7 +8562,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", "wasm-bindgen-shared", ] @@ -8604,7 +8596,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -9197,7 +9189,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -9215,7 +9207,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -9235,9 +9227,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm 0.48.0", "windows_aarch64_msvc 0.48.0", @@ -9527,7 +9519,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] diff --git a/scripts/tests/forest_cli_check.sh b/scripts/tests/forest_cli_check.sh index 4514b56ac5fe..688f665503e6 100755 --- a/scripts/tests/forest_cli_check.sh +++ b/scripts/tests/forest_cli_check.sh @@ -37,18 +37,22 @@ popd -: validate calibnet snapshot +: validate latest calibnet snapshot pushd "$(mktemp --directory)" - : : fetch a calibnet snapshot + : : fetch a compressed calibnet snapshot "$FOREST_CLI_PATH" --chain calibnet snapshot fetch test "$(num-files-here)" -eq 1 + uncompress_me=$(find . -type f | head -1) + + : : decompress it, as validate does not support compressed snapshots + zstd --decompress --rm "$uncompress_me" validate_me=$(find . -type f | head -1) : : validating under calibnet chain should succeed - "$FOREST_CLI_PATH" --chain calibnet snapshot validate "$validate_me" --force + "$FOREST_CLI_PATH" --chain calibnet snapshot validate "$validate_me" : : validating under mainnet chain should fail - if "$FOREST_CLI_PATH" --chain mainnet snapshot validate "$validate_me" --force; then + if "$FOREST_CLI_PATH" --chain mainnet snapshot validate "$validate_me"; then exit 1 fi rm -- * diff --git a/src/car_backed_blockstore.rs b/src/car_backed_blockstore.rs new file mode 100644 index 000000000000..2cc6e7502bfc --- /dev/null +++ b/src/car_backed_blockstore.rs @@ -0,0 +1,328 @@ +// Copyright 2019-2023 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +//! It can often be time, memory, or disk prohibitive to read large snapshots into a database like [`ParityDb`](crate::db::parity_db::ParityDb). +//! +//! This module provides an implementer of [`Blockstore`] that simply wraps a [CAR file](https://ipld.io/specs/transport/car/carv1). +//! **Note that all operations on this store are blocking**. +//! +//! On creation, [`CarBackedBlockstore`] builds an in-memory index of the [`Cid`]s in the file, +//! and their offsets into that file. +//! +//! When a block is requested [`CarBackedBlockstore`] scrolls to that offset, and reads the block, on-demand. +//! +//! Writes for new data (which doesn't exist in the CAR already) are currently cached in-memory. +//! +//! Random-access performance is expected to be poor, as the OS will have to load separate parts of the file from disk, and flush it for each read. +//! However, (near) linear access should be pretty good, as file chunks will be pre-fetched. +//! See also the remarks below about block ordering. +//! +//! # CAR Layout and seeking +//! +//! CARs consist of _varint frames_, are a concatenation of the _body length_ as an [`unsigned_varint`], and the _frame body_ itself. +//! [`unsigned_varint::codec`] can be used to read frames piecewise into memory. +//! +//! ```text +//! varint frame +//! │◄───────────────────────►│ +//! │ │ +//! ├───────────┬─────────────┤ +//! │varint: │ │ +//! │body length│frame body │ +//! └───────────┼─────────────┤ +//! │ │ +//! frame body ►│◄───────────►│ +//! offset =body length +//! ``` +//! +//! The first varint frame is a _header frame_, where the frame body is a [`CarHeader`] encoded using [`ipld_dagcbor`](serde_ipld_dagcbor). +//! +//! Subsequent varint frames are _block frames_, where the frame body is a concatenation of a [`Cid`] and the _block data_ addressed by that CID. +//! +//! ```text +//! block frame ►│ +//! body offset │ +//! │ =body length +//! │◄────────────►│ +//! ┌───────────┼───┬──────────┤ +//! │body length│cid│block data│ +//! └───────────┴───┼──────────┤ +//! │◄────────►│ +//! │ =block data length +//! block data │ +//! offset ►│ +//! ``` +//! +//! ## Block ordering +//! > _... a filecoin-deterministic car-file is currently implementation-defined as containing all DAG-forming blocks in first-seen order, as a result of a depth-first DAG traversal starting from a single root._ +//! +//! - [CAR documentation](https://ipld.io/specs/transport/car/carv1/#determinism) +//! +//! # Future work +//! - [`fadvise`](https://linux.die.net/man/2/posix_fadvise)-based APIs to pre-fetch parts of the file, to improve random access performance. +//! - Use an inner [`Blockstore`] for writes. +//! - Support compressed snapshots. +//! Note that [`zstd`](https://github.com/facebook/zstd/blob/e4aeaebc201ba49fec50b087aeb15343c63712e5/doc/zstd_compression_format.md#zstandard-frames) archives are also composed of frames. +//! Snapshots typically comprise of a single frame, but that would require decompressing all preceding data, precluding random access. +//! So compressed snapshot support would compression per-frame, or maybe per block of frames. +//! - Support multiple files by concatenating them. +//! - Use safe arithmetic for all operations - a malicious frame shouldn't cause a crash. +//! - Theoretically, [`CarBackedBlockstore`] should be clonable (or even [`Sync`]) with very low overhead, so that multiple threads could perform operations concurrently. + +use ahash::AHashMap; +use cid::Cid; +use fvm_ipld_blockstore::Blockstore; +use fvm_ipld_car::CarHeader; +use parking_lot::Mutex; +use std::collections::hash_map::Entry::{Occupied, Vacant}; +use std::io::{ + self, BufReader, + ErrorKind::{InvalidData, Other, UnexpectedEof, Unsupported}, + Read, Seek, SeekFrom, +}; +use tracing::{debug, trace}; + +/// If you seek to `offset` (from the start of the file), and read `length` bytes, +/// you should get data that corresponds to a [`Cid`] (but NOT the [`Cid`] itself). +#[derive(Debug)] +struct BlockDataLocation { + offset: u64, + length: u32, +} + +/// See [module documentation](mod@self) for more. +pub struct CarBackedBlockstore { + // https://github.com/ChainSafe/forest/issues/3096 + inner: Mutex>, +} + +impl CarBackedBlockstore +where + ReaderT: Read + Seek, +{ + pub fn roots(&self) -> Vec { + self.inner.lock().roots.clone() + } + + /// To be correct: + /// - `reader` must read immutable data. e.g if it is a file, it should be [`flock`](https://linux.die.net/man/2/flock)ed. + /// [`Blockstore`] API calls may panic if this is not upheld. + /// - `reader`'s buffer should have enough room for the [`CarHeader`] and any [`Cid`]s. + #[tracing::instrument(level = "debug", skip_all)] + pub fn new(mut reader: ReaderT) -> cid::Result { + let CarHeader { roots, version } = read_header(&mut reader)?; + if version != 1 { + return Err(cid::Error::Io(io::Error::new( + Unsupported, + "file must be CARv1", + ))); + } + + // When indexing, we perform small reads of the length and CID before seeking + // A small buffer helps ~5% (n=1) + let mut buf_reader = BufReader::with_capacity(128, reader); + + // now create the index + let index = std::iter::from_fn(|| read_block_location_or_eof(&mut buf_reader).transpose()) + .collect::, _>>()?; + match index.len() { + 0 => { + return Err(cid::Error::Io(io::Error::new( + InvalidData, + "CARv1 files must not be empty", + ))) + } + num_blocks => debug!(num_blocks, "indexed CAR"), + } + + Ok(Self { + inner: Mutex::new(CarBackedBlockstoreInner { + // discarding the buffer is ok - we only seek within this now + reader: buf_reader.into_inner(), + index, + roots, + write_cache: AHashMap::new(), + }), + }) + } +} + +struct CarBackedBlockstoreInner { + reader: ReaderT, + write_cache: AHashMap>, + index: AHashMap, + roots: Vec, +} + +impl Blockstore for CarBackedBlockstore +where + ReaderT: Read + Seek, +{ + #[tracing::instrument(level = "trace", skip(self))] + fn get(&self, k: &Cid) -> anyhow::Result>> { + let CarBackedBlockstoreInner { + reader, + write_cache, + index, + .. + } = &mut *self.inner.lock(); + match (index.get(k), write_cache.entry(*k)) { + (Some(_location), Occupied(cached)) => { + trace!("evicting from write cache"); + Ok(Some(cached.remove())) + } + (Some(BlockDataLocation { offset, length }), Vacant(_)) => { + trace!("fetching from disk"); + reader.seek(SeekFrom::Start(*offset))?; + let mut data = vec![0; usize::try_from(*length).unwrap()]; + reader.read_exact(&mut data)?; + Ok(Some(data)) + } + (None, Occupied(cached)) => { + trace!("getting from write cache"); + Ok(Some(cached.get().clone())) + } + (None, Vacant(_)) => { + trace!("not found"); + Ok(None) + } + } + } + + /// # Panics + /// - If the write cache contains different data with this CID + /// - See also [`Self::new`]. + #[tracing::instrument(level = "trace", skip(self, block))] + fn put_keyed(&self, k: &Cid, block: &[u8]) -> anyhow::Result<()> { + let CarBackedBlockstoreInner { + write_cache, index, .. + } = &mut *self.inner.lock(); + match (index.get(k), write_cache.entry(*k)) { + (None, Occupied(already)) => match already.get() == block { + true => { + trace!("already in cache"); + Ok(()) + } + false => panic!("mismatched content on second write for CID {k}"), + }, + (None, Vacant(vacant)) => { + trace!(bytes = block.len(), "insert into cache"); + vacant.insert(block.to_owned()); + Ok(()) + } + (Some(_), Vacant(_)) => { + trace!("already on disk"); + Ok(()) + } + (Some(_), Occupied(_)) => { + unreachable!("we don't a CID in the write cache if it exists on disk") + } + } + } +} + +#[tracing::instrument(level = "trace", skip_all, ret, err)] +fn read_header(mut reader: impl Read) -> io::Result { + let header_len = read_u32_or_eof(&mut reader)?.ok_or(io::Error::from(UnexpectedEof))?; + let mut buffer = vec![0; usize::try_from(header_len).unwrap()]; + reader.read_exact(&mut buffer)?; + fvm_ipld_encoding::from_slice(&buffer).map_err(|e| io::Error::new(InvalidData, e)) +} + +/// Importantly, we seek _past_ the data, rather than read any in. +/// This allows us to keep indexing fast. +/// +/// [`Ok(None)`] on EOF +#[tracing::instrument(level = "trace", skip_all, ret)] +fn read_block_location_or_eof( + mut reader: (impl Read + Seek), +) -> cid::Result> { + let Some((frame_body_offset, body_length)) = next_varint_frame(&mut reader)? else { + return Ok(None) + }; + let cid = Cid::read_bytes(&mut reader)?; + // tradeoff: we perform a second syscall here instead of in Blockstore::get, + // and keep BlockDataLocation purely for the blockdata + let block_data_offset = reader.stream_position()?; + let next_frame_offset = frame_body_offset + u64::from(body_length); + let block_data_length = u32::try_from(next_frame_offset - block_data_offset).unwrap(); + reader.seek(SeekFrom::Start(next_frame_offset))?; + Ok(Some(( + cid, + BlockDataLocation { + offset: block_data_offset, + length: block_data_length, + }, + ))) +} + +fn next_varint_frame(mut reader: (impl Read + Seek)) -> io::Result> { + Ok(match read_u32_or_eof(&mut reader)? { + Some(body_length) => { + let frame_body_offset = reader.stream_position()?; + Some((frame_body_offset, body_length)) + } + None => None, + }) +} + +fn read_u32_or_eof(mut reader: impl Read) -> io::Result> { + use unsigned_varint::io::{ + read_u32, + ReadError::{Decode, Io}, + }; + + let mut byte = [0u8; 1]; // detect EOF + match reader.read(&mut byte)? { + 0 => Ok(None), + 1 => read_u32(byte.chain(reader)) + .map_err(|varint_error| match varint_error { + Io(e) => e, + Decode(e) => io::Error::new(InvalidData, e), + other => io::Error::new(Other, other), + }) + .map(Some), + _ => unreachable!(), + } +} + +#[cfg(test)] +mod tests { + use super::CarBackedBlockstore; + + use futures_util::AsyncRead; + use fvm_ipld_blockstore::{Blockstore as _, MemoryBlockstore}; + use fvm_ipld_car::{Block, CarReader}; + + #[test] + fn test() { + let car = include_bytes!("../test-snapshots/chain4.car"); + let reference = reference(futures::io::Cursor::new(car)); + let car_backed = CarBackedBlockstore::new(std::io::Cursor::new(car)).unwrap(); + + assert_eq!(car_backed.inner.lock().index.len(), 1222); + assert_eq!(car_backed.inner.lock().roots.len(), 1); + + let cids = { + let holding_lock = car_backed.inner.lock(); + holding_lock.index.keys().cloned().collect::>() + }; + + for cid in cids { + let expected = reference.get(&cid).unwrap().unwrap(); + let actual = car_backed.get(&cid).unwrap().unwrap(); + assert_eq!(expected, actual); + } + } + + fn reference(reader: impl AsyncRead + Send + Unpin) -> MemoryBlockstore { + futures::executor::block_on(async { + let blockstore = MemoryBlockstore::new(); + let mut blocks = CarReader::new(reader).await.unwrap(); + while let Some(Block { cid, data }) = blocks.next_block().await.unwrap() { + blockstore.put_keyed(&cid, &data).unwrap() + } + blockstore + }) + } +} diff --git a/src/cli/subcommands/snapshot_cmd.rs b/src/cli/subcommands/snapshot_cmd.rs index 6ab3480d29df..0e727263696b 100644 --- a/src/cli/subcommands/snapshot_cmd.rs +++ b/src/cli/subcommands/snapshot_cmd.rs @@ -1,29 +1,25 @@ // Copyright 2019-2023 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use std::{path::PathBuf, sync::Arc}; - +use super::*; use crate::blocks::{tipset_keys_json::TipsetKeysJson, Tipset, TipsetKeys}; +use crate::car_backed_blockstore::CarBackedBlockstore; use crate::chain::ChainStore; +use crate::cli::subcommands::{cli_error_and_die, handle_rpc_err}; use crate::cli_shared::snapshot::{self, TrustedVendor}; -use crate::db::db_engine::{db_root, open_proxy_db}; -use crate::genesis::{forest_load_car, read_genesis_header}; +use crate::genesis::read_genesis_header; use crate::ipld::{recurse_links_hash, CidHashSet}; use crate::networks::NetworkChain; use crate::rpc_api::{chain_api::ChainExportParams, progress_api::GetProgressType}; use crate::rpc_client::{chain_ops::*, progress_ops::get_progress}; use crate::shim::clock::ChainEpoch; use crate::utils::io::ProgressBar; -use anyhow::bail; +use anyhow::{bail, Context as _}; use chrono::Utc; use clap::Subcommand; -use dialoguer::{theme::ColorfulTheme, Confirm}; +use std::path::PathBuf; +use std::sync::Arc; use tempfile::TempDir; -use tokio_util::compat::TokioAsyncReadCompatExt; - -use super::*; - -use crate::cli::subcommands::{cli_error_and_die, handle_rpc_err}; #[derive(Debug, Subcommand)] pub enum SnapshotCommands { @@ -55,11 +51,8 @@ pub enum SnapshotCommands { /// Number of block headers to validate from the tip #[arg(long, default_value = "2000")] recent_stateroots: i64, - /// Path to snapshot file + /// Path to an uncompressed snapshot (CAR) snapshot: PathBuf, - /// Force validation and answers yes to all prompts. - #[arg(long)] - force: bool, }, } @@ -149,8 +142,7 @@ impl SnapshotCommands { Self::Validate { recent_stateroots, snapshot, - force, - } => validate(&config, recent_stateroots, snapshot, *force).await, + } => validate(&config, recent_stateroots, snapshot).await, } } } @@ -159,62 +151,36 @@ async fn validate( config: &Config, recent_stateroots: &i64, snapshot: &PathBuf, - force: bool, ) -> anyhow::Result<()> { - let confirm = force - || atty::is(atty::Stream::Stdin) - && Confirm::with_theme(&ColorfulTheme::default()) - .with_prompt(format!( - "This will result in using approximately {} MB of data. Proceed?", - std::fs::metadata(snapshot)?.len() / (1024 * 1024) - )) - .default(false) - .interact() - .unwrap_or_default(); - - if confirm { - let tmp_chain_data_path = TempDir::new()?; - let db_path = db_root( - tmp_chain_data_path - .path() - .join(config.chain.network.to_string()) - .as_path(), - ); - let db = open_proxy_db(db_path, config.db_config().clone())?; - - let genesis = read_genesis_header( - config.client.genesis_file.as_ref(), - config.chain.genesis_bytes(), - &db, - ) - .await?; - - let chain_store = Arc::new(ChainStore::new( - db, - config.chain.clone(), - &genesis, - tmp_chain_data_path.path(), - )?); - - let (cids, _n_records) = { - let reader = - crate::utils::net::reader(snapshot.as_path().display().to_string().as_str()) - .await?; - forest_load_car(chain_store.blockstore().clone(), reader.compat()).await? - }; - - let ts = chain_store.tipset_from_keys(&TipsetKeys::new(cids))?; - - validate_links_and_genesis_traversal( - &chain_store, - ts, - chain_store.blockstore(), - *recent_stateroots, - &Tipset::from(genesis), - &config.chain.network, - ) - .await?; - } + let store = Arc::new( + CarBackedBlockstore::new(std::fs::File::open(snapshot)?) + .context("couldn't read input CAR file - is it compressed?")?, + ); + let genesis = read_genesis_header( + config.client.genesis_file.as_ref(), + config.chain.genesis_bytes(), + &store, + ) + .await?; + + let chain_store = Arc::new(ChainStore::new( + store, + config.chain.clone(), + &genesis, + TempDir::new()?.path(), + )?); + + let ts = chain_store.tipset_from_keys(&TipsetKeys::new(chain_store.db.roots()))?; + + validate_links_and_genesis_traversal( + &chain_store, + ts, + chain_store.blockstore(), + *recent_stateroots, + &Tipset::from(genesis), + &config.chain.network, + ) + .await?; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 2f28e5459fa0..b760e649bed5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ cfg_if::cfg_if! { mod auth; mod beacon; mod blocks; +mod car_backed_blockstore; mod chain; mod chain_sync; mod cli;