From 2ecfb221698f86605eff1f7964a3ead05e52308c Mon Sep 17 00:00:00 2001 From: jangko Date: Tue, 19 Sep 2023 08:56:25 +0700 Subject: [PATCH 1/3] Fix unlisted exception due to recent modification in CoreDB interface --- fluffy/eth_data/history_data_json_store.nim | 2 +- fluffy/network/history/history_network.nim | 6 +++--- nimbus_verified_proxy/rpc/rpc_utils.nim | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fluffy/eth_data/history_data_json_store.nim b/fluffy/eth_data/history_data_json_store.nim index 61ef6ceae4..3c88f7304f 100644 --- a/fluffy/eth_data/history_data_json_store.nim +++ b/fluffy/eth_data/history_data_json_store.nim @@ -171,7 +171,7 @@ proc getGenesisHeader*(id: NetworkId = MainNet): BlockHeader = try: toGenesisHeader(params) - except RlpError: + except RlpError, CatchableError: raise (ref Defect)(msg: "Genesis should be valid") # Reading JSON Portal content and content keys diff --git a/fluffy/network/history/history_network.nim b/fluffy/network/history/history_network.nim index d9584c12bd..151852c857 100644 --- a/fluffy/network/history/history_network.nim +++ b/fluffy/network/history/history_network.nim @@ -188,9 +188,9 @@ proc calcRootHash(items: Transactions | PortalReceipts | Withdrawals): Hash256 = for i, item in items: try: tr.put(rlp.encode(i), item.asSeq()) - except RlpError as e: - # TODO: Investigate this RlpError as it doesn't sound like this is - # something that can actually occur. + except CatchableError as e: + # tr.put now is a generic interface to whatever underlying db + # and it can raise exception if the backend db is something like aristo raiseAssert(e.msg) return tr.rootHash diff --git a/nimbus_verified_proxy/rpc/rpc_utils.nim b/nimbus_verified_proxy/rpc/rpc_utils.nim index 2a67648837..a97f233024 100644 --- a/nimbus_verified_proxy/rpc/rpc_utils.nim +++ b/nimbus_verified_proxy/rpc/rpc_utils.nim @@ -84,7 +84,7 @@ template asEthHash(hash: BlockHash): etypes.Hash256 = proc calculateTransactionData( items: openArray[TypedTransaction]): - (etypes.Hash256, seq[TxHash], uint64) {.raises: [RlpError].} = + (etypes.Hash256, seq[TxHash], uint64) {.raises: [RlpError, CatchableError].} = ## returns tuple composed of ## - root of transactions trie ## - list of transactions hashes @@ -122,7 +122,7 @@ func blockHeaderSize( return uint64(len(rlp.encode(bh))) proc asBlockObject*( - p: ExecutionData): BlockObject {.raises: [RlpError, ValueError].} = + p: ExecutionData): BlockObject {.raises: [RlpError, ValueError, CatchableError].} = # TODO: currently we always calculate txHashes as BlockObject does not have # option of returning full transactions. It needs fixing at nim-web3 library # level From 051dd48c54e2a396612d19b09c6a7f3eae914c10 Mon Sep 17 00:00:00 2001 From: jangko Date: Sat, 9 Sep 2023 21:48:25 +0700 Subject: [PATCH 2/3] Refactor beacon skeleton --- nimbus/db/storage_types.nim | 17 +- nimbus/sync/beacon/skeleton_algo.nim | 501 ++++++++++++++ nimbus/sync/beacon/skeleton_db.nim | 215 ++++++ nimbus/sync/beacon/skeleton_desc.nim | 99 +++ nimbus/sync/beacon/skeleton_main.nim | 178 +++++ nimbus/sync/beacon/skeleton_utils.nim | 123 ++++ nimbus/sync/does-not-compile/skeleton.nim | 591 ---------------- nimbus/utils/utils.nim | 23 + tests/all_tests.nim | 3 +- tests/does-not-compile/test_skeleton.nim | 648 ------------------ tests/test_beacon/setup_env.nim | 195 ++++++ tests/test_beacon/test_1_initsync.nim | 201 ++++++ tests/test_beacon/test_2_extend.nim | 125 ++++ tests/test_beacon/test_3_sethead_genesis.nim | 109 +++ tests/test_beacon/test_4_fill_canonical.nim | 64 ++ .../test_5_canonical_past_genesis.nim | 67 ++ tests/test_beacon/test_6_abort_filling.nim | 83 +++ .../test_beacon/test_7_abort_and_backstep.nim | 63 ++ tests/test_beacon/test_8_pos_too_early.nim | 65 ++ tests/test_beacon/test_skeleton.nim | 47 ++ 20 files changed, 2168 insertions(+), 1249 deletions(-) create mode 100644 nimbus/sync/beacon/skeleton_algo.nim create mode 100644 nimbus/sync/beacon/skeleton_db.nim create mode 100644 nimbus/sync/beacon/skeleton_desc.nim create mode 100644 nimbus/sync/beacon/skeleton_main.nim create mode 100644 nimbus/sync/beacon/skeleton_utils.nim delete mode 100644 nimbus/sync/does-not-compile/skeleton.nim delete mode 100644 tests/does-not-compile/test_skeleton.nim create mode 100644 tests/test_beacon/setup_env.nim create mode 100644 tests/test_beacon/test_1_initsync.nim create mode 100644 tests/test_beacon/test_2_extend.nim create mode 100644 tests/test_beacon/test_3_sethead_genesis.nim create mode 100644 tests/test_beacon/test_4_fill_canonical.nim create mode 100644 tests/test_beacon/test_5_canonical_past_genesis.nim create mode 100644 tests/test_beacon/test_6_abort_filling.nim create mode 100644 tests/test_beacon/test_7_abort_and_backstep.nim create mode 100644 tests/test_beacon/test_8_pos_too_early.nim create mode 100644 tests/test_beacon/test_skeleton.nim diff --git a/nimbus/db/storage_types.nim b/nimbus/db/storage_types.nim index 667e401cd7..9b85406087 100644 --- a/nimbus/db/storage_types.nim +++ b/nimbus/db/storage_types.nim @@ -26,8 +26,8 @@ type finalizedHash skeletonProgress skeletonBlockHashToNumber - skeletonBlock - skeletonTransaction + skeletonHeader + skeletonBody snapSyncAccount snapSyncStorageSlot snapSyncStateRoot @@ -100,17 +100,16 @@ proc skeletonBlockHashToNumberKey*(h: Hash256): DbKey {.inline.} = result.data[1 .. 32] = h.data result.dataEndPos = uint8 32 -proc skeletonBlockKey*(u: BlockNumber): DbKey {.inline.} = - result.data[0] = byte ord(skeletonBlock) +proc skeletonHeaderKey*(u: BlockNumber): DbKey {.inline.} = + result.data[0] = byte ord(skeletonHeader) doAssert sizeof(u) <= 32 copyMem(addr result.data[1], unsafeAddr u, sizeof(u)) result.dataEndPos = uint8 sizeof(u) -proc skeletonTransactionKey*(u: BlockNumber): DbKey {.inline.} = - result.data[0] = byte ord(skeletonTransaction) - doAssert sizeof(u) <= 32 - copyMem(addr result.data[1], unsafeAddr u, sizeof(u)) - result.dataEndPos = uint8 sizeof(u) +proc skeletonBodyKey*(h: Hash256): DbKey {.inline.} = + result.data[0] = byte ord(skeletonBody) + result.data[1 .. 32] = h.data + result.dataEndPos = uint8 32 proc snapSyncAccountKey*(h: openArray[byte]): DbKey {.inline.} = doAssert(h.len == 32) diff --git a/nimbus/sync/beacon/skeleton_algo.nim b/nimbus/sync/beacon/skeleton_algo.nim new file mode 100644 index 0000000000..4bbf7e7f5d --- /dev/null +++ b/nimbus/sync/beacon/skeleton_algo.nim @@ -0,0 +1,501 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + ./skeleton_desc, + ./skeleton_utils, + ./skeleton_db + +{.push gcsafe, raises: [].} + +logScope: + topics = "skeleton" + +proc isLinked*(sk: SkeletonRef): Result[bool, string] = + ## Returns true if the skeleton chain is linked to canonical + if sk.isEmpty: + return ok(false) + + let sc = sk.last + + # if its genesis we are linked + if sc.tail == 0: + return ok(true) + + let head = sk.blockHeight + if sc.tail > head + 1: + return ok(false) + + let number = sc.tail - 1 + let maybeHeader = sk.getHeader(number).valueOr: + return err("isLinked: " & error) + + # The above sc.tail > head - 1 + # assure maybeHeader.isSome + doAssert maybeHeader.isSome + + let nextHeader = maybeHeader.get + let linked = sc.next == nextHeader.blockHash + if linked and sk.len > 1: + # Remove all other subchains as no more relevant + sk.removeAllButLast() + sk.writeProgress() + + return ok(linked) + +proc fastForwardHead(sk: SkeletonRef, last: Segment, target: uint64): Result[void, string] = + # Try fast forwarding the chain head to the number + let + head = last.head + maybeHead = sk.getHeader(head, true).valueOr: + return err(error) + + if maybeHead.isNone: + return ok() + + var + headBlock = maybeHead.get + headBlockHash = headBlock.blockHash + + for newHead in head + 1 .. target: + let maybeHead = sk.getHeader(newHead, true).valueOr: + return err(error) + + if maybeHead.isNone: + break + + let newBlock = maybeHead.get + if newBlock.parentHash != headBlockHash: + # Head can't be updated forward + break + + headBlock = newBlock + headBlockHash = headBlock.blockHash + + last.head = headBlock.u64 + debug "lastchain head fast forwarded", + `from`=head, to=last.head, tail=last.tail + ok() + +proc trySubChainsMerge*(sk: SkeletonRef): Result[bool, string] = + var + merged = false + edited = false + + # If the subchain extended into the next subchain, we need to handle + # the overlap. Since there could be many overlaps, do this in a loop. + while sk.len > 1 and sk.second.head >= sk.last.tail: + # Extract some stats from the second subchain + let sc = sk.second + + # Since we just overwrote part of the next subchain, we need to trim + # its head independent of matching or mismatching content + if sc.tail >= sk.last.tail: + # Fully overwritten, get rid of the subchain as a whole + debug "Previous subchain fully overwritten", sub=sc + sk.removeSecond() + edited = true + continue + else: + # Partially overwritten, trim the head to the overwritten size + debug "Previous subchain partially overwritten", sub=sc + sc.head = sk.last.tail - 1 + edited = true + + # If the old subchain is an extension of the new one, merge the two + # and let the skeleton syncer restart (to clean internal state) + let + maybeSecondHead = sk.getHeader(sk.second.head).valueOr: + return err(error) + secondHeadHash = maybeSecondHead.blockHash + + if maybeSecondHead.isSome and secondHeadHash == sk.last.next: + # only merge if we can integrate a big progress, as each merge leads + # to disruption of the block fetcher to start a fresh + if (sc.head - sc.tail) > sk.conf.subchainMergeMinimum: + debug "Previous subchain merged head", sub=sc + sk.last.tail = sc.tail + sk.last.next = sc.next + sk.removeSecond() + # If subchains were merged, all further available headers + # are invalid since we skipped ahead. + merged = true + else: + debug "Subchain ignored for merge", sub=sc + sk.removeSecond() + edited = true + + if edited: sk.writeProgress() + ok(merged) + +#[ + if headers.len > 0: + let first {.used.} = headers[0] + let last {.used.} = headers[^1] + let sc {.used.} = if sk.subchains.len > 0: + sk.subchains[0] + else: + SkeletonSubchain() + debug "Skeleton putBlocks start", + count = headers.len, + first = first.blockNumber, + hash = short(first.blockHash), + fork = sk.toFork(first.blockNumber), + last = last.blockNumber, + hash = short(last.blockHash), + fork = sk.toFork(last.blockNumber), + head = sc.head, + tail = sc.tail, + next = short(sc.next) +]# + +proc putBlocks*(sk: SkeletonRef, headers: openArray[BlockHeader]): + Result[StatusAndNumber, string] = + ## Writes skeleton blocks to the db by number + ## @returns number of blocks saved + var + merged = false + tailUpdated = false + + if sk.len == 0: + return err("no subchain set") + + for header in headers: + let + number = header.u64 + headerHash = header.blockHash + + if number >= sk.last.tail: + # These blocks should already be in skeleton, and might be coming in + # from previous events especially if the previous subchains merge + continue + elif number == 0: + let genesisHash = sk.genesisHash + if headerHash == genesisHash: + return err("Skeleton pubBlocks with invalid genesis block " & + "number=" & $number & + ", hash=" & headerHash.short & + ", genesisHash=" & genesisHash.short) + continue + + # Extend subchain or create new segment if necessary + if sk.last.next == headerHash: + sk.putHeader(header) + sk.pulled += 1 + sk.last.tail = number + sk.last.next = header.parentHash + tailUpdated = true + else: + # Critical error, we expect new incoming blocks to extend the canonical + # subchain which is the [0]'th + debug "Blocks don't extend canonical subchain", + sub=sk.last, + number, + hash=headerHash.short + return err("Blocks don't extend canonical subchain") + + merged = sk.trySubChainsMerge().valueOr: + return err(error) + + if tailUpdated or merged: + sk.progress.canonicalHeadReset = true + + # If its merged, we need to break as the new tail could be quite ahead + # so we need to clear out and run the reverse block fetcher again + if merged: break + + sk.writeProgress() + + # Print a progress report making the UX a bit nicer + if getTime() - sk.logged > STATUS_LOG_INTERVAL: + var left = sk.last.tail - 1 - sk.blockHeight + if sk.progress.linked: left = 0 + if left > 0: + sk.logged = getTime() + if sk.pulled == 0: + info "Beacon sync starting", left=left + else: + let sinceStarted = getTime() - sk.started + let eta = (sinceStarted div sk.pulled.int64) * left.int64 + info "Syncing beacon headers", + downloaded=sk.pulled, left=left, eta=eta + + sk.progress.linked = sk.isLinked().valueOr: + return err(error) + + var res = StatusAndNumber(number: headers.len.uint64) + # If the sync is finished, start filling the canonical chain. + if sk.progress.linked: + res.status.incl FillCanonical + + if merged: + res.status.incl SyncMerged + ok(res) + +proc backStep(sk: SkeletonRef): Result[uint64, string] = + if sk.conf.fillCanonicalBackStep <= 0: + return ok(0) + + let sc = sk.last + var + newTail = sc.tail + maybeTailHeader: Opt[BlockHeader] + + while true: + newTail = newTail + sk.conf.fillCanonicalBackStep + maybeTailHeader = sk.getHeader(newTail, true).valueOr: + return err(error) + if maybeTailHeader.isSome or newTail > sc.head: break + + if newTail > sc.head: + newTail = sc.head + maybeTailHeader = sk.getHeader(newTail, true).valueOr: + return err(error) + + if maybeTailHeader.isSome and newTail > 0: + debug "Backstepped skeleton", head=sc.head, tail=newTail + let tailHeader = maybeTailHeader.get + sk.last.tail = tailHeader.u64 + sk.last.next = tailHeader.parentHash + sk.writeProgress() + return ok(newTail) + else: + # we need a new head, emptying the subchains + sk.clear() + sk.writeProgress() + debug "Couldn't backStep subchain 0, dropping subchains for new head signal" + return ok(0) + + # unreachable? + sk.progress.linked = sk.isLinked().valueOr: + return err(error) + ok(0) + +# Inserts skeleton blocks into canonical chain and runs execution. +proc fillCanonicalChain*(sk: SkeletonRef): Result[void, string] = + if sk.filling: return ok() + sk.filling = true + + var + canonicalHead = sk.blockHeight + maybeOldHead = Opt.none BlockHeader + + let subchain = sk.last + if sk.progress.canonicalHeadReset: + # Grab previous head block in case of resettng canonical head + let oldHead = sk.canonicalHead().valueOr: + return err(error) + maybeOldHead = Opt.some oldHead + + if subchain.tail > canonicalHead + 1: + return err("Canonical head should already be on or " & + "ahead subchain tail canonicalHead=" & + $canonicalHead & ", tail=" & $subchain.tail) + + let newHead = if subchain.tail > 0: subchain.tail - 1 + else: 0 + debug "Resetting canonicalHead for fillCanonicalChain", + `from`=canonicalHead, to=newHead + + canonicalHead = newHead + sk.resetCanonicalHead(canonicalHead, oldHead.u64) + sk.progress.canonicalHeadReset = false + + let start {.used.} = canonicalHead + # This subchain is a reference to update the tail for + # the very subchain we are filling the data for + + debug "Starting canonical chain fill", + canonicalHead, subchainHead=subchain.head + + while sk.filling and canonicalHead < subchain.head: + # Get next block + let + number = canonicalHead + 1 + maybeHeader = sk.getHeader(number).valueOr: + return err(error) + + if maybeHeader.isNone: + # This shouldn't happen, but if it does because of some issues, + # we should back step and fetch again + debug "fillCanonicalChain block not found, backStepping", number + sk.backStep().isOkOr: + return err(error) + break + + # Insert into chain + let header = maybeHeader.get + let res = sk.insertBlock(header, true) + if res.isErr: + debug "fillCanonicalChain putBlock", msg=res.error + if maybeOldHead.isSome: + let oldHead = maybeOldHead.get + if oldHead.u64 >= number: + # Put original canonical head block back if reorg fails + sk.insertBlock(oldHead, true).isOkOr: + return err(error) + + let numBlocksInserted = res.valueOr: 0 + if numBlocksInserted != 1: + debug "Failed to put block from skeleton chain to canonical", + number=number, + hash=header.blockHashStr, + parentHash=header.parentHash.short + + # Lets log some parent by number and parent by hash, that may help to understand whats going on + let parent = sk.getHeader(number - 1).valueOr: + return err(error) + debug "ParentByNumber", number=parent.numberStr, hash=parent.blockHashStr + + let parentWithHash = sk.getHeader(header.parentHash).valueOr: + return err(error) + + debug "parentByHash", + number=parentWithHash.numberStr, + hash=parentWithHash.blockHashStr + + sk.backStep().isOkOr: + return err(error) + break + + canonicalHead += numBlocksInserted + sk.fillLogIndex += numBlocksInserted + + # Delete skeleton block to clean up as we go, if block is fetched and chain is linked + # it will be fetched from the chain without any issues + sk.deleteHeaderAndBody(header) + if sk.fillLogIndex >= 20: + info "Skeleton canonical chain fill status", + canonicalHead, + chainHead=sk.blockHeight, + subchainHead=subchain.head + sk.fillLogIndex = 0 + + sk.filling = false + debug "Successfully put blocks from skeleton chain to canonical", + start, `end`=canonicalHead, + skeletonHead=subchain.head + ok() + +proc processNewHead*(sk: SkeletonRef, head: BlockHeader, + force = false): Result[bool, string] = + ## processNewHead does the internal shuffling for a new head marker and either + ## accepts and integrates it into the skeleton or requests a reorg. Upon reorg, + ## the syncer will tear itself down and restart with a fresh head. It is simpler + ## to reconstruct the sync state than to mutate it. + ## @returns true if the chain was reorged + + # If the header cannot be inserted without interruption, return an error for + # the outer loop to tear down the skeleton sync and restart it + let + number = head.u64 + headHash = head.blockHash + genesisHash = sk.genesisHash + + if number == 0: + if headHash != genesisHash: + return err("Invalid genesis setHead announcement " & + "number=" & $number & + ", hash=" & headHash.short & + ", genesisHash=" & genesisHash.short + ) + # genesis announcement + return ok(false) + + + let last = if sk.isEmpty: + debug "Skeleton empty, comparing against genesis head=0 tail=0", + newHead=number + # set the lastchain to genesis for comparison in + # following conditions + segment(0, 0, zeroBlockHash) + else: + sk.last + + if last.tail > number: + # Not a noop / double head announce, abort with a reorg + if force: + debug "Skeleton setHead before tail, resetting skeleton", + tail=last.tail, head=last.head, newHead=number + last.head = number + last.tail = number + last.next = head.parentHash + else: + debug "Skeleton announcement before tail, will reset skeleton", + tail=last.tail, head=last.head, newHead=number + return ok(true) + + elif last.head >= number: + # Check if its duplicate announcement, if not trim the head and + # let the match run after this if block + let mayBeDupBlock = sk.getHeader(number).valueOr: + return err(error) + + let maybeDupHash = mayBeDupBlock.blockHash + if mayBeDupBlock.isSome and mayBeDupHash == headHash: + debug "Skeleton duplicate announcement", + tail=last.tail, head=last.head, number, hash=headHash.short + return ok(false) + else: + # Since its not a dup block, so there is reorg in the chain or at least + # in the head which we will let it get addressed after this if else block + if force: + debug "Skeleton differing announcement", + tail=last.tail, + head=last.head, + number=number, + expected=mayBeDupHash.short, + actual=headHash.short + else: + debug "Skeleton stale announcement", + tail=last.tail, + head=last.head, + number + return ok(true) + + elif last.head + 1 < number: + if force: + sk.fastForwardHead(last, number - 1).isOkOr: + return err(error) + + # If its still less than number then its gapped head + if last.head + 1 < number: + debug "Beacon chain gapped setHead", + head=last.head, newHead=number + return ok(true) + else: + debug "Beacon chain gapped announcement", + head=last.head, newHead=number + return ok(true) + + let maybeParent = sk.getHeader(number - 1).valueOr: + return err(error) + + let parentHash = maybeParent.blockHash + if maybeParent.isNone or parentHash != head.parentHash: + if force: + debug "Beacon chain forked", + ancestor=maybeParent.numberStr, + hash=maybeParent.blockHashStr, + want=head.parentHash.short + return ok(true) + + if force: + last.head = number + if sk.isEmpty: + # If there was no subchain to being with i.e. initialized from genesis + # and no reorg then push in subchains else the reorg handling will + # push the new chain + sk.push(last) + sk.progress.linked = sk.isLinked.valueOr: + return err(error) + + debug "Beacon chain extended new", last + return ok(false) diff --git a/nimbus/sync/beacon/skeleton_db.nim b/nimbus/sync/beacon/skeleton_db.nim new file mode 100644 index 0000000000..4022b936da --- /dev/null +++ b/nimbus/sync/beacon/skeleton_db.nim @@ -0,0 +1,215 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + eth/rlp, + eth/common/eth_types_rlp, + ./skeleton_desc, + ./skeleton_utils, + ../../db/storage_types, + ../../utils/utils, + ../../core/chain + +export + eth_types_rlp.blockHash + +{.push gcsafe, raises: [].} + +logScope: + topics = "skeleton" + +# ------------------------------------------------------------------------------ +# Private helpers +# ------------------------------------------------------------------------------ + +template get(sk: SkeletonRef, key: untyped): untyped = + get(sk.db.kvt, key.toOpenArray) + +template put(sk: SkeletonRef, key, val: untyped): untyped = + put(sk.db.kvt, key.toOpenArray, val) + +template del(sk: SkeletonRef, key: untyped): untyped = + del(sk.db.kvt, key.toOpenArray) + +proc append(w: var RlpWriter, s: Segment) = + w.startList(3) + w.append(s.head) + w.append(s.tail) + w.append(s.next) + +proc append(w: var RlpWriter, p: Progress) = + w.startList(3) + w.append(p.segments) + w.append(p.linked) + w.append(p.canonicalHeadReset) + +proc readImpl(rlp: var Rlp, T: type Segment): Segment {.raises: [RlpError].} = + rlp.tryEnterList() + Segment( + head: rlp.read(uint64), + tail: rlp.read(uint64), + next: rlp.read(Hash256), + ) + +proc readImpl(rlp: var Rlp, T: type Progress): Progress {.raises: [RlpError].} = + rlp.tryEnterList() + Progress( + segments: rlp.read(seq[Segment]), + linked : rlp.read(bool), + canonicalHeadReset: rlp.read(bool), + ) + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc getHeader*(sk: SkeletonRef, + number: uint64, + onlySkeleton: bool = false): Result[Opt[BlockHeader], string] = + ## Gets a block from the skeleton or canonical db by number. + try: + let rawHeader = sk.get(skeletonHeaderKey(number.toBlockNumber)) + if rawHeader.len != 0: + let output = rlp.decode(rawHeader, BlockHeader) + return ok(Opt.some output) + + if onlySkeleton: + return ok(Opt.none BlockHeader) + + # As a fallback, try to get the block from the canonical chain + # in case it is available there + var output: BlockHeader + if sk.db.getBlockHeader(number.toBlockNumber, output): + return ok(Opt.some output) + + ok(Opt.none BlockHeader) + except RlpError as ex: + err(ex.msg) + +proc getHeader*(sk: SkeletonRef, + blockHash: Hash256, + onlySkeleton: bool = false): + Result[Opt[BlockHeader], string] = + ## Gets a skeleton block from the db by hash + try: + let rawNumber = sk.get(skeletonBlockHashToNumberKey(blockHash)) + if rawNumber.len != 0: + var output: BlockHeader + let number = rlp.decode(rawNumber, BlockNumber) + if sk.db.getBlockHeader(number, output): + return ok(Opt.some output) + + if onlySkeleton: + return ok(Opt.none BlockHeader) + + # As a fallback, try to get the block from the canonical chain + # in case it is available there + var output: BlockHeader + if sk.db.getBlockHeader(blockHash, output): + return ok(Opt.some output) + + ok(Opt.none BlockHeader) + except RlpError as ex: + err(ex.msg) + +proc putHeader*(sk: SkeletonRef, header: BlockHeader) = + ## Writes a skeleton block header to the db by number + let encodedHeader = rlp.encode(header) + sk.put(skeletonHeaderKey(header.blockNumber), encodedHeader) + sk.put( + skeletonBlockHashToNumberKey(header.blockHash), + rlp.encode(header.blockNumber) + ) + +proc putBody*(sk: SkeletonRef, header: BlockHeader, body: BlockBody): Result[void, string] = + ## Writes block body to db + try: + let + encodedBody = rlp.encode(body) + bodyHash = sumHash(body) + headerHash = header.blockHash + keyHash = sumHash(headerHash, bodyHash) + sk.put(skeletonBodyKey(keyHash), encodedBody) + ok() + except CatchableError as ex: + err(ex.msg) + +proc getBody*(sk: SkeletonRef, header: BlockHeader): Result[Opt[BlockBody], string] = + ## Reads block body from db + ## sumHash is the hash of [txRoot, ommersHash, wdRoot] + try: + let + bodyHash = header.sumHash + headerHash = header.blockHash + keyHash = sumHash(headerHash, bodyHash) + rawBody = sk.get(skeletonBodyKey(keyHash)) + if rawBody.len > 0: + return ok(Opt.some rlp.decode(rawBody, BlockBody)) + ok(Opt.none BlockBody) + except RlpError as ex: + err(ex.msg) + +proc writeProgress*(sk: SkeletonRef) = + ## Writes the progress to db + for sub in sk.subchains: + debug "Writing sync progress subchains", sub + + let encodedProgress = rlp.encode(sk.progress) + sk.put(skeletonProgressKey(), encodedProgress) + +proc readProgress*(sk: SkeletonRef): Result[void, string] = + ## Reads the SkeletonProgress from db + try: + let rawProgress = sk.get(skeletonProgressKey()) + if rawProgress.len == 0: + return ok() + + sk.progress = rlp.decode(rawProgress, Progress) + ok() + except RlpError as ex: + err(ex.msg) + +proc deleteHeaderAndBody*(sk: SkeletonRef, header: BlockHeader) = + ## Deletes a skeleton block from the db by number + sk.del(skeletonHeaderKey(header.blockNumber)) + sk.del(skeletonBlockHashToNumberKey(header.blockHash)) + sk.del(skeletonBodyKey(header.sumHash)) + +proc canonicalHead*(sk: SkeletonRef): Result[BlockHeader, string] = + ## Returns Opt.some or error, never returns Opt.none + try: + ok(sk.db.getCanonicalHead()) + except CatchableError as ex: + err(ex.msg) + +proc resetCanonicalHead*(sk: SkeletonRef, newHead, oldHead: uint64) = + debug "RESET CANONICAL", newHead, oldHead + sk.chain.com.syncCurrent = newHead.toBlockNumber + +proc insertBlocks*(sk: SkeletonRef, + headers: openArray[BlockHeader], + body: openArray[BlockBody], + fromEngine: bool): Result[uint64, string] = + try: + let res = sk.chain.persistBlocks(headers, body) + if res != ValidationResult.OK: + return err("insertBlocks validation error") + ok(headers.len.uint64) + except CatchableError as ex: + err(ex.msg) + +proc insertBlock*(sk: SkeletonRef, + header: BlockHeader, + fromEngine: bool): Result[uint64, string] = + let maybeBody = sk.getBody(header).valueOr: + return err(error) + if maybeBody.isNone: + return err("insertBlock: Block body not found: " & $header.u64) + sk.insertBlocks([header], [maybeBody.get], fromEngine) diff --git a/nimbus/sync/beacon/skeleton_desc.nim b/nimbus/sync/beacon/skeleton_desc.nim new file mode 100644 index 0000000000..466e8aecfc --- /dev/null +++ b/nimbus/sync/beacon/skeleton_desc.nim @@ -0,0 +1,99 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + std/times, + chronicles, + results, + eth/common/eth_types, + ../../utils/utils, + ../../db/core_db, + ../../core/chain + +export + eth_types, + core_db, + chain, + chronicles, + results, + times + +{.push gcsafe, raises: [].} + +logScope: + topics = "skeleton" + +type + # Contiguous header chain segment that is backed by the database, + # but may not be linked to the live chain. The skeleton downloader may produce + # a new one of these every time it is restarted until the subchain grows large + # enough to connect with a previous subchain. + Segment* = ref object + head*: uint64 # Block number of the newest header in the subchain + tail*: uint64 # Block number of the oldest header in the subchain + next*: Hash256 # Block hash of the next oldest header in the subchain + + # Database entry to allow suspending and resuming a chain sync. + # As the skeleton header chain is downloaded backwards, restarts can and + # will produce temporarily disjoint subchains. There is no way to restart a + # suspended skeleton sync without prior knowledge of all prior suspension points. + Progress* = ref object + segments*: seq[Segment] + linked* : bool + canonicalHeadReset*: bool + + SkeletonConfig* = ref object + fillCanonicalBackStep*: uint64 + subchainMergeMinimum* : uint64 + + # The Skeleton chain class helps support beacon sync by accepting head blocks + # while backfill syncing the rest of the chain. + SkeletonRef* = ref object + progress*: Progress + pulled* : uint64 # Number of headers downloaded in this run + filling* : bool # Whether we are actively filling the canonical chain + started* : Time # Timestamp when the skeleton syncer was created + logged* : Time # Timestamp when progress was last logged to user + db* : CoreDBRef + chain* : ChainRef + conf* : SkeletonConfig + fillLogIndex*: uint64 + + SkeletonStatus* = enum + SkeletonOk + + # SyncReorged is a signal that the head chain of + # the current sync cycle was (partially) reorged, thus the skeleton syncer + # should abort and restart with the new state. + SyncReorged + + # ReorgDenied is returned if an attempt is made to extend the beacon chain + # with a new header, but it does not link up to the existing sync. + ReorgDenied + + # SyncMerged is a signal that the current sync cycle merged with + # a previously aborted subchain, thus the skeleton syncer + # should abort and restart with the new state. + SyncMerged + + # Request to do fillCanonicalChain + FillCanonical + + StatusAndNumber* = object + status*: set[SkeletonStatus] + number*: uint64 + + StatusAndReorg* = object + status*: set[SkeletonStatus] + reorg* : bool + + BodyRange* = object + min*: uint64 + max*: uint64 diff --git a/nimbus/sync/beacon/skeleton_main.nim b/nimbus/sync/beacon/skeleton_main.nim new file mode 100644 index 0000000000..2dc8eaf8aa --- /dev/null +++ b/nimbus/sync/beacon/skeleton_main.nim @@ -0,0 +1,178 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + ./skeleton_desc, + ./skeleton_utils, + ./skeleton_db, + ./skeleton_algo + +export + skeleton_desc, + skeleton_algo.isLinked, + skeleton_algo.putBlocks, + skeleton_algo.fillCanonicalChain + +{.push gcsafe, raises: [].} + +logScope: + topics = "skeleton" + +# ------------------------------------------------------------------------------ +# Constructors +# ------------------------------------------------------------------------------ + +proc new*(_: type SkeletonRef, chain: ChainRef): SkeletonRef = + SkeletonRef( + progress: Progress(), + pulled : 0, + filling : false, + chain : chain, + db : chain.db, + started : getTime(), + logged : getTime(), + conf : SkeletonConfig( + fillCanonicalBackStep: 100, + subchainMergeMinimum : 1000, + ), + ) + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc open*(sk: SkeletonRef): Result[void, string] = + if sk.chain.com.ttd.isNone: + return err("Cannot create skeleton as ttd not set") + sk.readProgress().isOkOr: + return err(error) + sk.started = getTime() + ok() + +proc setHead*(sk: SkeletonRef, head: BlockHeader, + force = true, init = false, + reorgthrow = false): Result[StatusAndReorg, string] = + ## Announce and integrate a new head. + ## @params head - The block being attempted as a new head + ## @params force - Flag to indicate if this is just a check of + ## worthiness or a actually new head + ## @params init - Flag this is the first time since the beacon + ## sync start to perform additional tasks + ## @params reorgthrow - Flag to indicate if we would actually like + ## to throw if there is a reorg + ## instead of just returning the boolean + ## + ## @returns True if the head (will) cause a reorg in the + ## canonical skeleton subchain + + let + number = head.u64 + + debug "New skeleton head announced", + number, + hash=head.blockHashStr, + force + + let reorg = sk.processNewHead(head, force).valueOr: + return err(error) + + if force and reorg: + # It could just be a reorg at this head with previous tail preserved + let + subchain = if sk.isEmpty: Segment(nil) + else: sk.last + maybeParent = sk.getHeader(number - 1).valueOr: + return err(error) + parentHash = maybeParent.blockHash + + if subchain.isNil or maybeParent.isNone or parentHash != head.parentHash: + let sub = segment(number, number, head.parentHash) + sk.push(sub) + debug "Created new subchain", sub + else: + # Only the head differed, tail is preserved + subchain.head = number + # Reset the filling of canonical head from tail on reorg + sk.progress.canonicalHeadReset = true + + # Put this block irrespective of the force + sk.putHeader(head) + + if init: + sk.trySubChainsMerge().isOkOr: + return err(error) + + if (force and reorg) or init: + sk.progress.linked = sk.isLinked.valueOr: + return err(error) + + if force or init: + sk.writeProgress() + + var res = StatusAndReorg(reorg: reorg) + if force and sk.progress.linked: + res.status.incl FillCanonical + + # Earlier we were throwing on reorg, essentially for the purposes for + # killing the reverse fetcher + # but it can be handled properly in the calling fn without erroring + if reorg and reorgthrow: + if force: + res.status.incl SyncReorged + else: + res.status.incl ReorgDenied + + ok(res) + +proc initSync*(sk: SkeletonRef, head: BlockHeader, + reorgthrow = false): Result[StatusAndReorg, string] = + ## Setup the skeleton to init sync with head + ## @params head - The block with which we want to init the skeleton head + ## @params reorgthrow - If we would like the function to throw instead of + ## silently return if there is reorg of the skeleton head + ## + ## @returns True if the skeleton was reorged trying to init else false + + sk.setHead(head, true, true, reorgthrow) + +func bodyRange*(sk: SkeletonRef): Result[BodyRange, string] = + ## Get range of bodies need to be downloaded by synchronizer + var canonicalHead = sk.blockHeight + let subchain = sk.last + + if sk.progress.canonicalHeadReset: + if subchain.tail > canonicalHead + 1: + return err("Canonical head should already be on or " & + "ahead subchain tail canonicalHead=" & + $canonicalHead & ", tail=" & $subchain.tail) + let newHead = if subchain.tail > 0: subchain.tail - 1 + else: 0 + canonicalHead = newHead + + ok(BodyRange( + min: canonicalHead, + max: subchain.head, + )) + +# ------------------------------------------------------------------------------ +# Getters and setters +# ------------------------------------------------------------------------------ + +func fillCanonicalBackStep*(sk: SkeletonRef): uint64 = + sk.conf.fillCanonicalBackStep + +func subchainMergeMinimum*(sk: SkeletonRef): uint64 = + sk.conf.subchainMergeMinimum + +proc `fillCanonicalBackStep=`*(sk: SkeletonRef, val: uint64) = + sk.conf.fillCanonicalBackStep = val + +proc `subchainMergeMinimum=`*(sk: SkeletonRef, val: uint64) = + sk.conf.subchainMergeMinimum = val diff --git a/nimbus/sync/beacon/skeleton_utils.nim b/nimbus/sync/beacon/skeleton_utils.nim new file mode 100644 index 0000000000..d3291f243a --- /dev/null +++ b/nimbus/sync/beacon/skeleton_utils.nim @@ -0,0 +1,123 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + ./skeleton_desc + +{.push gcsafe, raises: [].} + +logScope: + topics = "skeleton" + +const + # How often to log sync status (in ms) + STATUS_LOG_INTERVAL* = initDuration(microseconds = 8000) + zeroBlockHash* = Hash256() + +# ------------------------------------------------------------------------------ +# Misc helpers +# ------------------------------------------------------------------------------ + +func u64*(h: BlockHeader): uint64 = + h.blockNumber.truncate(uint64) + +func blockHash*(x: Opt[BlockHeader]): Hash256 = + if x.isSome: x.get.blockHash + else: zeroBlockHash + +func numberStr*(x: Opt[BlockHeader]): string = + if x.isSome: $(x.get.u64) + else: "N/A" + +func blockHashStr*(x: Opt[BlockHeader]): string = + if x.isSome: x.get.blockHash.short + else: "N/A" + +func blockHashStr*(x: BlockHeader): string = + x.blockHash.short + +# ------------------------------------------------------------------------------ +# Segment helpers +# ------------------------------------------------------------------------------ + +func segment*(head, tail: uint64, next: Hash256): Segment = + Segment(head: head, tail: tail, next: next) + +func short(s: Segment): string = + s.next.short + +func `$`*(s: Segment): string = + result = "head: " & $s.head & + ", tail: " & $s.tail & + ", next: " & s.short + +# ------------------------------------------------------------------------------ +# Progress helpers +# ------------------------------------------------------------------------------ + +proc add*(ss: Progress, s: Segment) = + ss.segments.add s + +proc add*(ss: Progress, head, tail: uint64, next: Hash256) = + ss.add Segment(head: head, tail: tail, next: next) + +# ------------------------------------------------------------------------------ +# SkeletonRef helpers +# ------------------------------------------------------------------------------ + +func isEmpty*(sk: SkeletonRef): bool = + sk.progress.segments.len == 0 + +func notEmpty*(sk: SkeletonRef): bool = + sk.progress.segments.len > 0 + +func blockHeight*(sk: SkeletonRef): uint64 = + sk.chain.com.syncCurrent.truncate(uint64) + +func genesisHash*(sk: SkeletonRef): Hash256 = + sk.chain.com.genesisHash + +func len*(sk: SkeletonRef): int = + sk.progress.segments.len + +func last*(sk: SkeletonRef): Segment = + sk.progress.segments[^1] + +func second*(sk: SkeletonRef): Segment = + sk.progress.segments[^2] + +iterator subchains*(sk: SkeletonRef): Segment = + for sub in sk.progress.segments: + yield sub + +iterator pairs*(sk: SkeletonRef): (int, Segment) = + for i, sub in sk.progress.segments: + yield (i, sub) + +proc push*(sk: SkeletonRef, s: Segment) = + sk.progress.add s + +proc push*(sk: SkeletonRef, head, tail: uint64, next: Hash256) = + sk.progress.add(head, tail, next) + +proc removeLast*(sk: SkeletonRef) = + discard sk.progress.segments.pop + +proc removeSecond*(sk: SkeletonRef) = + sk.progress.segments.delete(sk.len-2) + +proc removeAllButLast*(sk: SkeletonRef) = + let last = sk.progress.segments.pop + for sub in sk.subchains: + debug "Canonical subchain linked with main, removing junked chains", sub + sk.progress.segments = @[last] + +proc clear*(sk: SkeletonRef) = + sk.progress.segments.setLen(0) diff --git a/nimbus/sync/does-not-compile/skeleton.nim b/nimbus/sync/does-not-compile/skeleton.nim deleted file mode 100644 index 3e5dda03ff..0000000000 --- a/nimbus/sync/does-not-compile/skeleton.nim +++ /dev/null @@ -1,591 +0,0 @@ -import - std/[times], - eth/[common, rlp], - eth/trie/db, - #stew/results, - stint, chronicles, - ../db/[db_chain, storage_types], - ".."/[utils, chain_config], - ../p2p/chain - -{.push raises: [].} - -logScope: - topics = "skeleton" - -type - # Contiguous header chain segment that is backed by the database, - # but may not be linked to the live chain. The skeleton downloader may produce - # a new one of these every time it is restarted until the subchain grows large - # enough to connect with a previous subchain. - SkeletonSubchain* = object - head*: UInt256 # Block number of the newest header in the subchain - tail*: UInt256 # Block number of the oldest header in the subchain - next*: Hash256 # Block hash of the next oldest header in the subchain - - # Database entry to allow suspending and resuming a chain - # sync. As the skeleton header chain is downloaded backwards, restarts can and - # will produce temporarily disjoint subchains. There is no way to restart a - # suspended skeleton sync without prior knowledge of all prior suspension points. - SkeletonProgress = seq[SkeletonSubchain] - - # The Skeleton chain class helps support beacon sync by accepting head blocks - # while backfill syncing the rest of the chain. - SkeletonRef* = ref object - subchains: SkeletonProgress - started : Time # Timestamp when the skeleton syncer was created - logged : Time # Timestamp when progress was last logged to user - pulled : int64 # Number of headers downloaded in this run - filling : bool # Whether we are actively filling the canonical chain - chainTTD : DifficultyInt - chainDB : ChainDBRef - chain : Chain - - # config - skeletonFillCanonicalBackStep: int - skeletonSubchainMergeMinimum: int - syncTargetHeight: int - ignoreTxs: bool - - SkeletonError* = object of CatchableError - - # SyncReorged is an internal helper error to signal that the head chain of - # the current sync cycle was (partially) reorged, thus the skeleton syncer - # should abort and restart with the new state. - ErrSyncReorged* = object of SkeletonError - - # ReorgDenied is returned if an attempt is made to extend the beacon chain - # with a new header, but it does not link up to the existing sync. - ErrReorgDenied* = object of SkeletonError - - # SyncMerged is an internal helper error to signal that the current sync - # cycle merged with a previously aborted subchain, thus the skeleton syncer - # should abort and restart with the new state. - ErrSyncMerged* = object of SkeletonError - - ErrHeaderNotFound* = object of SkeletonError - -const - # How often to log sync status (in ms) - STATUS_LOG_INTERVAL = initDuration(microseconds = 8000) - -proc new*(_: type SkeletonRef, chain: Chain): SkeletonRef = - new(result) - result.chain = chain - result.chainDB = chain.db - result.started = getTime() - result.logged = getTime() - result.pulled = 0'i64 - result.filling = false - result.chainTTD = chain.db.ttd() - result.skeletonFillCanonicalBackStep = 100 - result.skeletonSubchainMergeMinimum = 1000 - #result.syncTargetHeight = ? - result.ignoreTxs = false - -template get(sk: SkeletonRef, key: untyped): untyped = - get(sk.chainDB.db, key.toOpenArray) - -template put(sk: SkeletonRef, key, val: untyped): untyped = - put(sk.chainDB.db, key.toOpenArray, val) - -template del(sk: SkeletonRef, key: untyped): untyped = - del(sk.chainDB.db, key.toOpenArray) - -template toFork(sk: SkeletonRef, number: untyped): untyped = - toFork(sk.chainDB.config, number) - -template blockHeight(sk: SkeletonRef): untyped = - sk.chainDB.currentBlock - -# Reads the SkeletonProgress from db -proc readSyncProgress(sk: SkeletonRef) {.raises: [RlpError].} = - let rawProgress = sk.get(skeletonProgressKey()) - if rawProgress.len == 0: return - sk.subchains = rlp.decode(rawProgress, SkeletonProgress) - -# Writes the SkeletonProgress to db -proc writeSyncProgress(sk: SkeletonRef) = - for x in sk.subchains: - debug "Writing sync progress subchains", - head=x.head, tail=x.tail, next=short(x.next) - - let encodedProgress = rlp.encode(sk.subchains) - sk.put(skeletonProgressKey(), encodedProgress) - -proc open*(sk: SkeletonRef){.raises: [RlpError].} = - sk.readSyncProgress() - sk.started = getTime() - -# Gets a block from the skeleton or canonical db by number. -proc getHeader*(sk: SkeletonRef, - number: BlockNumber, - output: var BlockHeader, - onlySkeleton: bool = false): bool {.raises: [RlpError].} = - let rawHeader = sk.get(skeletonBlockKey(number)) - if rawHeader.len != 0: - output = rlp.decode(rawHeader, BlockHeader) - return true - else: - if onlySkeleton: return false - # As a fallback, try to get the block from the canonical chain in case it is available there - return sk.chainDB.getBlockHeader(number, output) - -# Gets a skeleton block from the db by hash -proc getHeaderByHash*(sk: SkeletonRef, - hash: Hash256, - output: var BlockHeader): bool {.raises: [RlpError].} = - let rawNumber = sk.get(skeletonBlockHashToNumberKey(hash)) - if rawNumber.len == 0: - return false - return sk.getHeader(rlp.decode(rawNumber, BlockNumber), output) - -# Deletes a skeleton block from the db by number -proc deleteBlock(sk: SkeletonRef, header: BlockHeader) = - sk.del(skeletonBlockKey(header.blockNumber)) - sk.del(skeletonBlockHashToNumberKey(header.blockHash)) - sk.del(skeletonTransactionKey(header.blockNumber)) - -# Writes a skeeton block to the db by number -proc putHeader*(sk: SkeletonRef, header: BlockHeader) = - let encodedHeader = rlp.encode(header) - sk.put(skeletonBlockKey(header.blockNumber), encodedHeader) - sk.put( - skeletonBlockHashToNumberKey(header.blockHash), - rlp.encode(header.blockNumber) - ) - -proc putBlock(sk: SkeletonRef, header: BlockHeader, txs: openArray[Transaction]) = - let encodedHeader = rlp.encode(header) - sk.put(skeletonBlockKey(header.blockNumber), encodedHeader) - sk.put( - skeletonBlockHashToNumberKey(header.blockHash), - rlp.encode(header.blockNumber) - ) - sk.put(skeletonTransactionKey(header.blockNumber), rlp.encode(txs)) - -proc getTxs( - sk: SkeletonRef, number: BlockNumber, - output: var seq[Transaction]) {.raises: [CatchableError].} = - let rawTxs = sk.get(skeletonTransactionKey(number)) - if rawTxs.len > 0: - output = rlp.decode(rawTxs, seq[Transaction]) - else: - raise newException(SkeletonError, - "getTxs: no transactions from block number " & $number) - -# Bounds returns the current head and tail tracked by the skeleton syncer. -proc bounds*(sk: SkeletonRef): SkeletonSubchain = - sk.subchains[0] - -# Returns true if the skeleton chain is linked to canonical -proc isLinked*(sk: SkeletonRef): bool {.raises: [CatchableError].} = - if sk.subchains.len == 0: return false - let sc = sk.bounds() - - # make check for genesis if tail is 1? - let head = sk.blockHeight - if sc.tail > head + 1.toBlockNumber: - return false - - var nextHeader: BlockHeader - let number = sc.tail - 1.toBlockNumber - if sk.getHeader(number, nextHeader): - return sc.next == nextHeader.blockHash - else: - raise newException(ErrHeaderNotFound, "isLinked: No header with number=" & $number) - -proc trySubChainsMerge(sk: SkeletonRef): bool {.raises: RlpError].} = - var - merged = false - edited = false - head: BlockHeader - - let subchainMergeMinimum = sk.skeletonSubchainMergeMinimum.u256 - # If the subchain extended into the next subchain, we need to handle - # the overlap. Since there could be many overlaps, do this in a loop. - while sk.subchains.len > 1 and - sk.subchains[1].head >= sk.subchains[0].tail: - # Extract some stats from the second subchain - let sc = sk.subchains[1] - - # Since we just overwrote part of the next subchain, we need to trim - # its head independent of matching or mismatching content - if sc.tail >= sk.subchains[0].tail: - # Fully overwritten, get rid of the subchain as a whole - debug "Previous subchain fully overwritten", - head=sc.head, tail=sc.tail, next=short(sc.next) - sk.subchains.delete(1) - edited = true - continue - else: - # Partially overwritten, trim the head to the overwritten size - debug "Previous subchain partially overwritten", - head=sc.head, tail=sc.tail, next=short(sc.next) - sk.subchains[1].head = sk.subchains[0].tail - 1.toBlockNumber - edited = true - - # If the old subchain is an extension of the new one, merge the two - # and let the skeleton syncer restart (to clean internal state) - if sk.getHeader(sk.subchains[1].head, head) and - head.blockHash == sk.subchains[0].next: - # only merge is we can integrate a big progress, as each merge leads - # to disruption of the block fetcher to start a fresh - if (sc.head - sc.tail) > subchainMergeMinimum: - debug "Previous subchain merged head", - head=sc.head, tail=sc.tail, next=short(sc.next) - sk.subchains[0].tail = sc.tail - sk.subchains[0].next = sc.next - sk.subchains.delete(1) - # If subchains were merged, all further available headers - # are invalid since we skipped ahead. - merged = true - else: - debug "Subchain ignored for merge", - head=sc.head, tail=sc.tail, next=short(sc.next) - sk.subchains.delete(1) - edited = true - - if edited: sk.writeSyncProgress() - return merged - -proc backStep(sk: SkeletonRef) {.raises: [RlpError].}= - if sk.skeletonFillCanonicalBackStep <= 0: - return - - let sc = sk.bounds() - var - hasTail: bool - tailHeader: BlockHeader - newTail = sc.tail - - while true: - newTail = newTail + sk.skeletonFillCanonicalBackStep.u256 - hasTail = sk.getHeader(newTail, tailHeader, true) - if hasTail or newTail > sc.head: break - - if newTail > sc.head: - newTail = sc.head - hasTail = sk.getHeader(newTail, tailHeader, true) - - if hasTail and newTail > 0.toBlockNumber: - trace "Backstepped skeleton", head=sc.head, tail=newTail - sk.subchains[0].tail = newTail - sk.subchains[0].next = tailHeader.parentHash - sk.writeSyncProgress() - else: - # we need a new head, emptying the subchains - sk.subchains = @[] - sk.writeSyncProgress() - warn "Couldn't backStep subchain 0, dropping subchains for new head signal" - -# processNewHead does the internal shuffling for a new head marker and either -# accepts and integrates it into the skeleton or requests a reorg. Upon reorg, -# the syncer will tear itself down and restart with a fresh head. It is simpler -# to reconstruct the sync state than to mutate it. -# -# @returns true if the chain was reorged -proc processNewHead( - sk: SkeletonRef, - head: BlockHeader, - force = false): bool {.raises: [RlpError].} = - # If the header cannot be inserted without interruption, return an error for - # the outer loop to tear down the skeleton sync and restart it - let number = head.blockNumber - - if sk.subchains.len == 0: - warn "Skeleton reorged and cleaned, no current subchain", newHead=number - return true - - let lastchain = sk.subchains[0] - if lastchain.tail >= number: - # If the chain is down to a single beacon header, and it is re-announced - # once more, ignore it instead of tearing down sync for a noop. - if lastchain.head == lastchain.tail: - var header: BlockHeader - let hasHeader = sk.getHeader(number, header) - # TODO: what should we do when hasHeader == false? - if hasHeader and header.blockHash == head.blockHash: - return false - - # Not a noop / double head announce, abort with a reorg - if force: - warn "Beacon chain reorged", - tail=lastchain.tail, head=lastchain.head, newHead=number - return true - - if lastchain.head + 1.toBlockNumber < number: - if force: - warn "Beacon chain gapped", - head=lastchain.head, newHead=number - return true - - var parent: BlockHeader - let hasParent = sk.getHeader(number - 1.toBlockNumber, parent) - if hasParent and parent.blockHash != head.parentHash: - if force: - warn "Beacon chain forked", - ancestor=parent.blockNumber, hash=short(parent.blockHash), - want=short(head.parentHash) - return true - - # Update the database with the new sync stats and insert the new - # head header. We won't delete any trimmed skeleton headers since - # those will be outside the index space of the many subchains and - # the database space will be reclaimed eventually when processing - # blocks above the current head. - sk.putHeader(head) - sk.subchains[0].head = number - sk.writeSyncProgress() - return false - -# Inserts skeleton blocks into canonical chain and runs execution. -proc fillCanonicalChain*(sk: SkeletonRef) {.raises: [CatchableError].} = - if sk.filling: return - sk.filling = true - - var canonicalHead = sk.blockHeight - let start = canonicalHead - let sc = sk.bounds() - debug "Starting canonical chain fill", - canonicalHead=canonicalHead, subchainHead=sc.head - - var fillLogIndex = 0 - while sk.filling and canonicalHead < sc.head: - # Get next block - let number = canonicalHead + 1.toBlockNumber - var header: BlockHeader - let hasHeader = sk.getHeader(number, header) - if not hasHeader: - # This shouldn't happen, but if it does because of some issues, we should back step - # and fetch again - debug "fillCanonicalChain block number not found, backStepping", - number=number - sk.backStep() - break - - # Insert into chain - var body: BlockBody - if not sk.ignoreTxs: - sk.getTxs(header.blockNumber, body.transactions) - let res = sk.chain.persistBlocks([header], [body]) - if res != ValidationResult.OK: - let hardFork = sk.toFork(number) - error "Failed to put block from skeleton chain to canonical", - number=number, - fork=hardFork, - hash=short(header.blockHash) - - sk.backStep() - break - - # Delete skeleton block to clean up as we go - sk.deleteBlock(header) - canonicalHead += 1.toBlockNumber - inc fillLogIndex # num block inserted - if fillLogIndex > 50: - trace "Skeleton canonical chain fill status", - canonicalHead=canonicalHead, - chainHead=sk.blockHeight, - subchainHead=sc.head - fillLogIndex = 0 - - sk.filling = false - trace "Successfully put blocks from skeleton chain to canonical target", - start=start, stop=canonicalHead, skeletonHead=sc.head, - syncTargetHeight=sk.syncTargetHeight - -# Announce and integrate a new head. -# throws if the new head causes a reorg. -proc setHead*( - sk: SkeletonRef, head: BlockHeader, - force = false) {.raises: [CatchableError].} = - debug "New skeleton head announced", - number=head.blockNumber, - hash=short(head.blockHash), - force=force - - let reorged = sk.processNewHead(head, force) - - # If linked, fill the canonical chain. - if force and sk.isLinked(): - sk.fillCanonicalChain() - - if reorged: - if force: - raise newException(ErrSyncReorged, "setHead: sync reorg") - else: - raise newException(ErrReorgDenied, "setHead: reorg denied") - -# Attempts to get the skeleton sync into a consistent state wrt any -# past state on disk and the newly requested head to sync to. -proc initSync*( - sk: SkeletonRef, head: BlockHeader) {.raises: [CatchableError].} = - let number = head.blockNumber - - if sk.subchains.len == 0: - # Start a fresh sync with a single subchain represented by the currently sent - # chain head. - sk.subchains.add(SkeletonSubchain( - head: number, - tail: number, - next: head.parentHash - )) - debug "Created initial skeleton subchain", - head=number, tail=number - else: - # Print some continuation logs - for x in sk.subchains: - debug "Restarting skeleton subchain", - head=x.head, tail=x.tail, next=short(x.next) - - # Create a new subchain for the head (unless the last can be extended), - # trimming anything it would overwrite - let headchain = SkeletonSubchain( - head: number, - tail: number, - next: head.parentHash - ) - - while sk.subchains.len > 0: - # If the last chain is above the new head, delete altogether - let lastchain = addr sk.subchains[0] - if lastchain.tail >= headchain.tail: - debug "Dropping skeleton subchain", - head=lastchain.head, tail=lastchain.tail - sk.subchains.delete(0) # remove `lastchain` - continue - # Otherwise truncate the last chain if needed and abort trimming - if lastchain.head >= headchain.tail: - debug "Trimming skeleton subchain", - oldHead=lastchain.head, newHead=headchain.tail - 1.toBlockNumber, - tail=lastchain.tail - lastchain.head = headchain.tail - 1.toBlockNumber - break - - # If the last subchain can be extended, we're lucky. Otherwise create - # a new subchain sync task. - var extended = false - if sk.subchains.len > 0: - let lastchain = addr sk.subchains[0] - if lastchain.head == headchain.tail - 1.toBlockNumber: - var header: BlockHeader - let lasthead = sk.getHeader(lastchain.head, header) - if lasthead and header.blockHash == head.parentHash: - debug "Extended skeleton subchain with new", - head=headchain.tail, tail=lastchain.tail - lastchain.head = headchain.tail - extended = true - if not extended: - debug "Created new skeleton subchain", - head=number, tail=number - sk.subchains.insert(headchain) - - sk.putHeader(head) - sk.writeSyncProgress() - - # If the sync is finished, start filling the canonical chain. - if sk.isLinked(): - sk.fillCanonicalChain() - -# Writes skeleton blocks to the db by number -# @returns number of blocks saved -proc putBlocks*( - sk: SkeletonRef, headers: openArray[BlockHeader]): - int {.raises: [CatchableError].} = - var merged = false - - if headers.len > 0: - let first {.used.} = headers[0] - let last {.used.} = headers[^1] - let sc {.used.} = if sk.subchains.len > 0: - sk.subchains[0] - else: - SkeletonSubchain() - debug "Skeleton putBlocks start", - count = headers.len, - first = first.blockNumber, - hash = short(first.blockHash), - fork = sk.toFork(first.blockNumber), - last = last.blockNumber, - hash = short(last.blockHash), - fork = sk.toFork(last.blockNumber), - head = sc.head, - tail = sc.tail, - next = short(sc.next) - - for header in headers: - let number = header.blockNumber - if number >= sk.subchains[0].tail: - # These blocks should already be in skeleton, and might be coming in - # from previous events especially if the previous subchains merge - continue - - # Extend subchain or create new segment if necessary - if sk.subchains[0].next == header.blockHash: - sk.putHeader(header) - sk.pulled += 1'i64 - sk.subchains[0].tail -= 1.toBlockNumber - sk.subchains[0].next = header.parentHash - else: - # Critical error, we expect new incoming blocks to extend the canonical - # subchain which is the [0]'th - let fork = sk.toFork(number) - warn "Blocks don't extend canonical subchain", - head=sk.subchains[0].head, - tail=sk.subchains[0].tail, - next=short(sk.subchains[0].next), - number=number, - hash=short(header.blockHash), - fork=fork - raise newException(SkeletonError, "Blocks don't extend canonical subchain") - - merged = sk.trySubChainsMerge() - # If its merged, we need to break as the new tail could be quite ahead - # so we need to clear out and run the reverse block fetcher again - if merged: break - - sk.writeSyncProgress() - - # Print a progress report making the UX a bit nicer - if getTime() - sk.logged > STATUS_LOG_INTERVAL: - var left = sk.bounds().tail - 1.toBlockNumber - sk.blockHeight - if sk.isLinked(): left = 0.toBlockNumber - if left > 0.toBlockNumber: - sk.logged = getTime() - if sk.pulled == 0: - info "Beacon sync starting", left=left - else: - let sinceStarted = getTime() - sk.started - let eta = (sinceStarted div sk.pulled) * left.truncate(int64) - info "Syncing beacon headers", - downloaded=sk.pulled, left=left, eta=eta - - # If the sync is finished, start filling the canonical chain. - if sk.isLinked(): - sk.fillCanonicalChain() - - if merged: - raise newException(ErrSyncMerged, "putBlocks: sync merged") - - return headers.len - -proc `subchains=`*(sk: SkeletonRef, subchains: openArray[SkeletonSubchain]) = - sk.subchains = @subchains - -proc len*(sk: SkeletonRef): int = - sk.subchains.len - -iterator items*(sk: SkeletonRef): SkeletonSubChain = - for x in sk.subchains: - yield x - -iterator pairs*(sk: SkeletonRef): tuple[key: int, val: SkeletonSubChain] = - for i, x in sk.subchains: - yield (i, x) - -proc ignoreTxs*(sk: SkeletonRef): bool = - sk.ignoreTxs - -proc `ignoreTxs=`*(sk: SkeletonRef, val: bool) = - sk.ignoreTxs = val diff --git a/nimbus/utils/utils.nim b/nimbus/utils/utils.nim index 960c47f748..7b7e7b42f0 100644 --- a/nimbus/utils/utils.nim +++ b/nimbus/utils/utils.nim @@ -2,6 +2,7 @@ import std/math, eth/[rlp, common/eth_types_rlp], stew/byteutils, + nimcrypto, ../db/core_db export eth_types_rlp @@ -24,6 +25,28 @@ template calcWithdrawalsRoot*(withdrawals: openArray[Withdrawal]): Hash256 = template calcReceiptRoot*(receipts: openArray[Receipt]): Hash256 = calcRootHash(receipts) +func sumHash*(hashes: varargs[Hash256]): Hash256 = + var ctx: sha256 + ctx.init() + for hash in hashes: + ctx.update hash.data + ctx.finish result.data + ctx.clear() + +proc sumHash*(body: BlockBody): Hash256 {.gcsafe, raises: [RlpError]} = + let txRoot = calcTxRoot(body.transactions) + let ommersHash = keccakHash(rlp.encode(body.uncles)) + let wdRoot = if body.withdrawals.isSome: + calcWithdrawalsRoot(body.withdrawals.get) + else: EMPTY_ROOT_HASH + sumHash(txRoot, ommersHash, wdRoot) + +proc sumHash*(header: BlockHeader): Hash256 = + let wdRoot = if header.withdrawalsRoot.isSome: + header.withdrawalsRoot.get + else: EMPTY_ROOT_HASH + sumHash(header.txRoot, header.ommersHash, wdRoot) + func generateAddress*(address: EthAddress, nonce: AccountNonce): EthAddress = result[0..19] = keccakHash(rlp.encodeList(address, nonce)).data.toOpenArray(12, 31) diff --git a/tests/all_tests.nim b/tests/all_tests.nim index a4aec53328..f3aa16c150 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -49,4 +49,5 @@ cliBuilder: ./test_keyed_queue_rlp, ./test_txpool, ./test_merge, - ./test_eip4844 + ./test_eip4844, + ./test_beacon/test_skeleton diff --git a/tests/does-not-compile/test_skeleton.nim b/tests/does-not-compile/test_skeleton.nim deleted file mode 100644 index 2d3bc00e35..0000000000 --- a/tests/does-not-compile/test_skeleton.nim +++ /dev/null @@ -1,648 +0,0 @@ -import - unittest2, - chronicles, - stew/[results, byteutils], - eth/[common, trie/db], - ../nimbus/sync/skeleton, - ../nimbus/db/db_chain, - ../nimbus/p2p/chain, - ../nimbus/[chain_config, config, genesis, constants], - ./test_helpers, - ./test_txpool/helpers - -const - baseDir = [".", "tests"] - repoDir = [".", "customgenesis"] - genesisFile = "post-merge.json" - -type - Subchain = object - head: int - tail: int - - TestEnv = object - conf : NimbusConf - chainDB : ChainDBRef - chain : Chain - - CCModify = proc(cc: NetworkParams) - -# TODO: too bad that blockHash -# cannot be executed at compile time -let - block49 = BlockHeader( - blockNumber: 49.toBlockNumber - ) - block49B = BlockHeader( - blockNumber: 49.toBlockNumber, - extraData: @['B'.byte] - ) - block50 = BlockHeader( - blockNumber: 50.toBlockNumber, - parentHash: block49.blockHash - ) - block51 = BlockHeader( - blockNumber: 51.toBlockNumber, - parentHash: block50.blockHash - ) - -proc initEnv(ccm: CCModify = nil): TestEnv = - let - conf = makeConfig(@[ - "--custom-network:" & genesisFile.findFilePath(baseDir,repoDir).value - ]) - - if ccm.isNil.not: - ccm(conf.networkParams) - - let - chainDB = newChainDBRef( - newCoreDbRef LegacyDbMemory, - conf.pruneMode == PruneMode.Full, - conf.networkId, - conf.networkParams - ) - chain = newChain(chainDB) - - initializeEmptyDb(chainDB) - - result = TestEnv( - conf: conf, - chainDB: chainDB, - chain: chain - ) - -proc `subchains=`(sk: SkeletonRef, subchains: openArray[Subchain]) = - var sc = newSeqOfCap[SkeletonSubchain](subchains.len) - for i in 0.. 0: - skeleton.subchains = testCase.oldState - - skeleton.initSync(testCase.head) - - check skeleton.len == testCase.newState.len - for i, sc in skeleton: - check sc.head == testCase.newState[i].head.toBlockNumber - check sc.tail == testCase.newState[i].tail.toBlockNumber - -suite "sync extend": - type - TestCase = object - head : BlockHeader # New head header to announce to reorg to - extend : BlockHeader # New head header to announce to extend with - newState: seq[Subchain] # Expected sync state after the reorg - err : string # Whether extension succeeds or not - - let testCases = [ - # Initialize a sync and try to extend it with a subsequent block. - TestCase( - head: block49, - extend: block50, - newState: @[Subchain(head: 50, tail: 49)], - ), - # Initialize a sync and try to extend it with the existing head block. - TestCase( - head: block49, - extend: block49, - newState: @[Subchain(head: 49, tail: 49)], - ), - # Initialize a sync and try to extend it with a sibling block. - TestCase( - head: block49, - extend: block49B, - newState: @[Subchain(head: 49, tail: 49)], - err: "ErrReorgDenied", - ), - # Initialize a sync and try to extend it with a number-wise sequential - # header, but a hash wise non-linking one. - TestCase( - head: block49B, - extend: block50, - newState: @[Subchain(head: 49, tail: 49)], - err: "ErrReorgDenied", - ), - # Initialize a sync and try to extend it with a non-linking future block. - TestCase( - head: block49, - extend: block51, - newState: @[Subchain(head: 49, tail: 49)], - err: "ErrReorgDenied", - ), - # Initialize a sync and try to extend it with a past canonical block. - TestCase( - head: block50, - extend: block49, - newState: @[Subchain(head: 50, tail: 50)], - err: "ErrReorgDenied", - ), - # Initialize a sync and try to extend it with a past sidechain block. - TestCase( - head: block50, - extend: block49B, - newState: @[Subchain(head: 50, tail: 50)], - err: "ErrReorgDenied", - ) - ] - - for z, testCase in testCases: - test "test case #" & $z: - let env = initEnv() - let skeleton = SkeletonRef.new(env.chain) - skeleton.open() - - skeleton.initSync(testCase.head) - - try: - skeleton.setHead(testCase.extend) - check testCase.err.len == 0 - except Exception as e: - check testCase.err.len > 0 - check testCase.err == e.name - - check skeleton.len == testCase.newState.len - for i, sc in skeleton: - check sc.head == testCase.newState[i].head.toBlockNumber - check sc.tail == testCase.newState[i].tail.toBlockNumber - - -template testCond(expr: untyped) = - if not (expr): - return TestStatus.Failed - -template testCond(expr, body: untyped) = - if not (expr): - body - return TestStatus.Failed - -proc linkedToGenesis(env: TestEnv): TestStatus = - result = TestStatus.OK - env.chain.validateBlock = false - let skeleton = SkeletonRef.new(env.chain) - - let - genesis = env.chainDB.getCanonicalHead() - block1 = BlockHeader( - blockNumber: 1.toBlockNumber, parentHash: genesis.blockHash, difficulty: 100.u256 - ) - block2 = BlockHeader( - blockNumber: 2.toBlockNumber, parentHash: block1.blockHash, difficulty: 100.u256 - ) - block3 = BlockHeader( - blockNumber: 3.toBlockNumber, parentHash: block2.blockHash, difficulty: 100.u256 - ) - block4 = BlockHeader( - blockNumber: 4.toBlockNumber, parentHash: block3.blockHash, difficulty: 100.u256 - ) - block5 = BlockHeader( - blockNumber: 5.toBlockNumber, parentHash: block4.blockHash, difficulty: 100.u256 - ) - - skeleton.open() - skeleton.initSync(block4) - - skeleton.ignoreTxs = true - discard skeleton.putBlocks([block3, block2]) - testCond env.chainDB.currentBlock == 0.toBlockNumber: - error "canonical height should be at genesis" - - discard skeleton.putBlocks([block1]) - testCond env.chainDB.currentBlock == 4.toBlockNumber: - error "canonical height should update after being linked" - - skeleton.setHead(block5, false) - testCond env.chainDB.currentBlock == 4.toBlockNumber: - error "canonical height should not change when setHead is set with force=false" - - skeleton.setHead(block5, true) - testCond env.chainDB.currentBlock == 5.toBlockNumber: - error "canonical height should change when setHead is set with force=true" - - var h: BlockHeader - for header in [block1, block2, block3, block4, block5]: - var res = skeleton.getHeader(header.blockNumber, h, true) - testCond res == false: - error "skeleton block should be cleaned up after filling canonical chain", - number=header.blockNumber - - res = skeleton.getHeaderByHash(header.blockHash, h) - testCond res == false: - error "skeleton block should be cleaned up after filling canonical chain", - number=header.blockNumber - -proc linkedPastGenesis(env: TestEnv): TestStatus = - result = TestStatus.OK - env.chain.validateBlock = false - let skeleton = SkeletonRef.new(env.chain) - - skeleton.open() - let - genesis = env.chainDB.getCanonicalHead() - block1 = BlockHeader( - blockNumber: 1.toBlockNumber, parentHash: genesis.blockHash, difficulty: 100.u256 - ) - block2 = BlockHeader( - blockNumber: 2.toBlockNumber, parentHash: block1.blockHash, difficulty: 100.u256 - ) - block3 = BlockHeader( - blockNumber: 3.toBlockNumber, parentHash: block2.blockHash, difficulty: 100.u256 - ) - block4 = BlockHeader( - blockNumber: 4.toBlockNumber, parentHash: block3.blockHash, difficulty: 100.u256 - ) - block5 = BlockHeader( - blockNumber: 5.toBlockNumber, parentHash: block4.blockHash, difficulty: 100.u256 - ) - - var body: BlockBody - let vr = env.chain.persistBlocks([block1, block2], [body, body]) - testCond vr == ValidationResult.OK - - skeleton.initSync(block4) - testCond env.chainDB.currentBlock == 2.toBlockNumber: - error "canonical height should be at block 2" - - skeleton.ignoreTxs = true - discard skeleton.putBlocks([block3]) - testCond env.chainDB.currentBlock == 4.toBlockNumber: - error "canonical height should update after being linked" - - skeleton.setHead(block5, false) - testCond env.chainDB.currentBlock == 4.toBlockNumber: - error "canonical height should not change when setHead with force=false" - - skeleton.setHead(block5, true) - testCond env.chainDB.currentBlock == 5.toBlockNumber: - error "canonical height should change when setHead with force=true" - - var h: BlockHeader - for header in [block3, block4, block5]: - var res = skeleton.getHeader(header.blockNumber, h, true) - testCond res == false: - error "skeleton block should be cleaned up after filling canonical chain", - number=header.blockNumber - - res = skeleton.getHeaderByHash(header.blockHash, h) - testCond res == false: - error "skeleton block should be cleaned up after filling canonical chain", - number=header.blockNumber - -proc ccmAbortTerminalInvalid(cc: NetworkParams) = - cc.config.terminalTotalDifficulty = some(200.u256) - cc.genesis.extraData = hexToSeqByte("0x000000000000000000") - cc.genesis.difficulty = UInt256.fromHex("0x01") - -proc abortTerminalInvalid(env: TestEnv): TestStatus = - result = TestStatus.OK - env.chain.validateBlock = false - let skeleton = SkeletonRef.new(env.chain) - - let - genesisBlock = env.chainDB.getCanonicalHead() - block1 = BlockHeader( - blockNumber: 1.toBlockNumber, parentHash: genesisBlock.blockHash, difficulty: 100.u256 - ) - block2 = BlockHeader( - blockNumber: 2.toBlockNumber, parentHash: block1.blockHash, difficulty: 100.u256 - ) - block3PoW = BlockHeader( - blockNumber: 3.toBlockNumber, parentHash: block2.blockHash, difficulty: 100.u256 - ) - block3PoS = BlockHeader( - blockNumber: 3.toBlockNumber, parentHash: block2.blockHash, difficulty: 0.u256 - #{ common, hardforkByTTD: BigInt(200) } - ) - block4InvalidPoS = BlockHeader( - blockNumber: 4.toBlockNumber, parentHash: block3PoW.blockHash, difficulty: 0.u256 - #{ common, hardforkByTTD: BigInt(200) } - ) - block4PoS = BlockHeader( - blockNumber: 4.toBlockNumber, parentHash: block3PoS.blockHash, difficulty: 0.u256 - #{ common, hardforkByTTD: BigInt(200) } - ) - block5 = BlockHeader( - blockNumber: 5.toBlockNumber, parentHash: block4PoS.blockHash, difficulty: 0.u256 - #{ common, hardforkByTTD: BigInt(200) } - ) - - skeleton.ignoreTxs = true - skeleton.open() - skeleton.initSync(block4InvalidPoS) - - discard skeleton.putBlocks([block3PoW, block2]) - testCond env.chainDB.currentBlock == 0.toBlockNumber: - error "canonical height should be at genesis" - - discard skeleton.putBlocks([block1]) - testCond env.chainDB.currentBlock == 2.toBlockNumber: - error "canonical height should stop at block 2 (valid terminal block), since block 3 is invalid (past ttd)" - - try: - skeleton.setHead(block5, false) - except ErrReorgDenied: - testCond true - except: - testCond false - - testCond env.chainDB.currentBlock == 2.toBlockNumber: - error "canonical height should not change when setHead is set with force=false" - - # Put correct chain - skeleton.initSync(block4PoS) - try: - discard skeleton.putBlocks([block3PoS]) - except ErrSyncMerged: - testCond true - except: - testCond false - - testCond env.chainDB.currentBlock == 4.toBlockNumber: - error "canonical height should now be at head with correct chain" - - var header: BlockHeader - testCond env.chainDB.getBlockHeader(env.chainDB.highestBlock, header): - error "cannot get block header", number = env.chainDB.highestBlock - - testCond header.blockHash == block4PoS.blockHash: - error "canonical height should now be at head with correct chain" - - skeleton.setHead(block5, false) - testCond skeleton.bounds().head == 5.toBlockNumber: - error "should update to new height" - -proc ccmAbortAndBackstep(cc: NetworkParams) = - cc.config.terminalTotalDifficulty = some(200.u256) - cc.genesis.extraData = hexToSeqByte("0x000000000000000000") - cc.genesis.difficulty = UInt256.fromHex("0x01") - -proc abortAndBackstep(env: TestEnv): TestStatus = - result = TestStatus.OK - env.chain.validateBlock = false - let skeleton = SkeletonRef.new(env.chain) - - let - genesisBlock = env.chainDB.getCanonicalHead() - block1 = BlockHeader( - blockNumber: 1.toBlockNumber, parentHash: genesisBlock.blockHash, difficulty: 100.u256 - ) - block2 = BlockHeader( - blockNumber: 2.toBlockNumber, parentHash: block1.blockHash, difficulty: 100.u256 - ) - block3PoW = BlockHeader( - blockNumber: 3.toBlockNumber, parentHash: block2.blockHash, difficulty: 100.u256 - ) - block4InvalidPoS = BlockHeader( - blockNumber: 4.toBlockNumber, parentHash: block3PoW.blockHash, difficulty: 0.u256 - #{ common, hardforkByTTD: 200 } - ) - - skeleton.open() - skeleton.ignoreTxs = true - skeleton.initSync(block4InvalidPoS) - discard skeleton.putBlocks([block3PoW, block2]) - - testCond env.chainDB.currentBlock == 0.toBlockNumber: - error "canonical height should be at genesis" - - discard skeleton.putBlocks([block1]) - testCond env.chainDB.currentBlock == 2.toBlockNumber: - error "canonical height should stop at block 2 (valid terminal block), since block 3 is invalid (past ttd)" - - testCond skeleton.bounds().tail == 4.toBlockNumber: - error "Subchain should have been backstepped to 4" - -proc ccmAbortPOSTooEarly(cc: NetworkParams) = - cc.config.terminalTotalDifficulty = some(200.u256) - #skeletonFillCanonicalBackStep: 0, - cc.genesis.difficulty = UInt256.fromHex("0x01") - -proc abortPOSTooEarly(env: TestEnv): TestStatus = - result = TestStatus.OK - env.chain.validateBlock = false - let skeleton = SkeletonRef.new(env.chain) - - let - genesisBlock = env.chainDB.getCanonicalHead() - block1 = BlockHeader( - blockNumber: 1.toBlockNumber, parentHash: genesisBlock.blockHash, difficulty: 100.u256 - ) - block2 = BlockHeader( - blockNumber: 2.toBlockNumber, parentHash: block1.blockHash, difficulty: 100.u256 - ) - block2PoS = BlockHeader( - blockNumber: 2.toBlockNumber, parentHash: block1.blockHash, difficulty: 0.u256 - ) - block3 = BlockHeader( - blockNumber: 3.toBlockNumber, parentHash: block2.blockHash, difficulty: 0.u256 - ) - - skeleton.ignoreTxs = true - skeleton.open() - skeleton.initSync(block2PoS) - discard skeleton.putBlocks([block1]) - - testCond env.chainDB.currentBlock == 1.toBlockNumber: - error "canonical height should stop at block 1 (valid PoW block), since block 2 is invalid (invalid PoS, not past ttd)" - - # Put correct chain - skeleton.initSync(block3) - try: - discard skeleton.putBlocks([block2]) - except ErrSyncMerged: - testCond true - except: - testCond false - - testCond env.chainDB.currentBlock == 3.toBlockNumber: - error "canonical height should now be at head with correct chain" - - var header: BlockHeader - testCond env.chainDB.getBlockHeader(env.chainDB.highestBlock, header): - error "cannot get block header", number = env.chainDB.highestBlock - - testCond header.blockHash == block3.blockHash: - error "canonical height should now be at head with correct chain" - -suite "fillCanonicalChain tests": - type - TestCase = object - name: string - ccm : CCModify - run : proc(env: TestEnv): TestStatus - - const testCases = [ - TestCase( - name: "should fill the canonical chain after being linked to genesis", - run : linkedToGenesis - ), - TestCase( - name: "should fill the canonical chain after being linked to a canonical block past genesis", - run : linkedPastGenesis - ), - TestCase( - name: "should abort filling the canonical chain if the terminal block is invalid", - ccm : ccmAbortTerminalInvalid, - run : abortTerminalInvalid - ), - TestCase( - name: "should abort filling the canonical chain and backstep if the terminal block is invalid", - ccm : ccmAbortAndBackstep, - run : abortAndBackstep - ), - TestCase( - name: "should abort filling the canonical chain if a PoS block comes too early without hitting ttd", - ccm : ccmAbortPOSTooEarly, - run : abortPOSTooEarly - ) - ] - for testCase in testCases: - test testCase.name: - let env = initEnv(testCase.ccm) - check testCase.run(env) == TestStatus.OK diff --git a/tests/test_beacon/setup_env.nim b/tests/test_beacon/setup_env.nim new file mode 100644 index 0000000000..00f5a69f83 --- /dev/null +++ b/tests/test_beacon/setup_env.nim @@ -0,0 +1,195 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + stew/byteutils, + ../../nimbus/core/chain, + ../../nimbus/core/pow/difficulty, + ../../nimbus/config, + ../../nimbus/common, + ../../nimbus/sync/beacon/skeleton_desc + +const + genesisFile = "tests/customgenesis/post-merge.json" + +type + Subchain* = object + head*: uint64 + tail*: uint64 + + TestEnv* = object + conf* : NimbusConf + chain*: ChainRef + + CCModify = proc(cc: NetworkParams) + +let + block49* = BlockHeader( + blockNumber: 49.toBlockNumber + ) + block49B* = BlockHeader( + blockNumber: 49.toBlockNumber, + extraData: @['B'.byte] + ) + block50* = BlockHeader( + blockNumber: 50.toBlockNumber, + parentHash: block49.blockHash + ) + block50B* = BlockHeader( + blockNumber: 50.toBlockNumber, + parentHash: block49.blockHash, + gasLimit: 999.GasInt, + ) + block51* = BlockHeader( + blockNumber: 51.toBlockNumber, + parentHash: block50.blockHash + ) + +proc setupEnv*(extraValidation: bool = false, ccm: CCModify = nil): TestEnv = + let + conf = makeConfig(@[ + "--custom-network:" & genesisFile + ]) + + if ccm.isNil.not: + ccm(conf.networkParams) + + let + com = CommonRef.new( + newCoreDbRef LegacyDbMemory, + conf.pruneMode == PruneMode.Full, + conf.networkId, + conf.networkParams + ) + chain = newChain(com, extraValidation) + + com.initializeEmptyDb() + TestEnv( + conf : conf, + chain: chain, + ) + +func subchain*(head, tail: uint64): Subchain = + Subchain(head: head, tail: tail) + +func header*(bn: uint64, temp, parent: BlockHeader, diff: uint64): BlockHeader = + BlockHeader( + blockNumber: bn.toBlockNumber, + parentHash : parent.blockHash, + difficulty : diff.u256, + timestamp : fromUnix(parent.timestamp.toUnix + 1), + gasLimit : temp.gasLimit, + stateRoot : temp.stateRoot, + txRoot : temp.txRoot, + fee : temp.fee, + receiptRoot: temp.receiptRoot, + ommersHash : temp.ommersHash, + withdrawalsRoot: temp.withdrawalsRoot, + blobGasUsed: temp.blobGasUsed, + excessBlobGas: temp.excessBlobGas, + parentBeaconBlockRoot: temp.parentBeaconBlockRoot, + ) + +func header*(com: CommonRef, bn: uint64, temp, parent: BlockHeader): BlockHeader = + result = header(bn, temp, parent, 0) + result.difficulty = com.calcDifficulty(result.timestamp, parent) + +func header*(bn: uint64, temp, parent: BlockHeader, + diff: uint64, stateRoot: string): BlockHeader = + result = header(bn, temp, parent, diff) + result.stateRoot = Hash256(data: hextoByteArray[32](stateRoot)) + +func header*(com: CommonRef, bn: uint64, temp, parent: BlockHeader, + stateRoot: string): BlockHeader = + result = com.header(bn, temp, parent) + result.stateRoot = Hash256(data: hextoByteArray[32](stateRoot)) + +func emptyBody*(): BlockBody = + BlockBody( + transactions: @[], + uncles: @[], + withdrawals: none(seq[Withdrawal]), + ) + +template fillCanonical(skel, z, stat) = + if z.status == stat and FillCanonical in z.status: + let xx = skel.fillCanonicalChain() + check xx.isOk + if xx.isErr: + debugEcho "FillCanonicalChain: ", xx.error + break + +template initSyncT*(skel, blk: untyped, r = false) = + let x = skel.initSync(blk) + check x.isOk + if x.isErr: + debugEcho "initSync:", x.error + break + let z = x.get + check z.reorg == r + +template setHeadT*(skel, blk, frc, r) = + let x = skel.setHead(blk, frc) + check x.isOk + if x.isErr: + debugEcho "setHead:", x.error + break + let z = x.get + check z.reorg == r + +template initSyncT*(skel, blk, r, stat) = + let x = skel.initSync(blk) + check x.isOk + if x.isErr: + debugEcho "initSync:", x.error + break + let z = x.get + check z.reorg == r + check z.status == stat + fillCanonical(skel, z, stat) + +template setHeadT*(skel, blk, frc, r, stat) = + let x = skel.setHead(blk, frc) + check x.isOk + if x.isErr: + debugEcho "setHead:", x.error + break + let z = x.get + check z.reorg == r + check z.status == stat + fillCanonical(skel, z, stat) + +template putBlocksT*(skel, blocks, numBlocks, stat) = + let x = skel.putBlocks(blocks) + check x.isOk + if x.isErr: + debugEcho "putBlocks: ", x.error + break + let z = x.get + check z.number == numBlocks + check z.status == stat + fillCanonical(skel, z, stat) + +template isLinkedT*(skel, r) = + let x = skel.isLinked() + check x.isOk + if x.isErr: + debugEcho "isLinked: ", x.error + break + check x.get == r + +template getHeaderClean*(skel, headers) = + for header in headers: + var r = skel.getHeader(header.u64, true) + check r.isOk + check r.get.isNone + r = skel.getHeader(header.blockHash, true) + check r.isOk + check r.get.isNone diff --git a/tests/test_beacon/test_1_initsync.nim b/tests/test_beacon/test_1_initsync.nim new file mode 100644 index 0000000000..3c69028fd2 --- /dev/null +++ b/tests/test_beacon/test_1_initsync.nim @@ -0,0 +1,201 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + unittest2, + ./setup_env, + ../../nimbus/sync/beacon/skeleton_main, + ../../nimbus/sync/beacon/skeleton_utils, + ../../nimbus/sync/beacon/skeleton_db + +type + TestCase = object + name : string + blocks : seq[BlockHeader] # Database content (besides the genesis) + oldState: seq[Subchain] # Old sync state with various interrupted subchains + head : BlockHeader # New head header to announce to reorg to + newState: seq[Subchain] # Expected sync state after the reorg + reorg : bool + +let testCases = [ + # The sync is expected to create a single subchain with the requested head. + TestCase( + name: "Completely empty database with only the genesis set.", + head: block50, + newState: @[subchain(50, 50)], + reorg: true + ), + # This is a synthetic case, just for the sake of covering things. + TestCase( + name: "Empty database with only the genesis set with a leftover empty sync progress", + head: block50, + newState: @[subchain(50, 50)], + reorg: true + ), + # The old subchain should be left as is and a new one appended to the sync status. + TestCase( + name: "A single leftover subchain is present, older than the new head.", + oldState: @[subchain(10, 5)], + head: block50, + newState: @[ + subchain(10, 5), + subchain(50, 50), + ], + reorg: true + ), + # The old subchains should be left as is and a new one appended to the sync status. + TestCase( + name: "Multiple leftover subchains are present, older than the new head.", + oldState: @[ + subchain(10, 5), + subchain(20, 15), + ], + head: block50, + newState: @[ + subchain(10, 5), + subchain(20, 15), + subchain(50, 50), + ], + reorg: true + ), + # The newer subchain should be deleted and a fresh one created for the head. + TestCase( + name: "A single leftover subchain is present, newer than the new head.", + oldState: @[subchain(65, 60)], + head: block50, + newState: @[subchain(50, 50)], + reorg: true + ), + # The newer subchains should be deleted and a fresh one created for the head. + TestCase( + name: "Multiple leftover subchain is present, newer than the new head.", + oldState: @[ + subchain(65, 60), + subchain(75, 70), + ], + head: block50, + newState: @[subchain(50, 50)], + reorg: true + ), + # than the announced head. The head should delete the newer one, + # keeping the older one. + TestCase( + name: "Two leftover subchains are present, one fully older and one fully newer", + oldState: @[ + subchain(10, 5), + subchain(65, 60), + ], + head: block50, + newState: @[ + subchain(10, 5), + subchain(50, 50), + ], + reorg: true + ), + # than the announced head. The head should delete the newer + # ones, keeping the older ones. + TestCase( + name: "Multiple leftover subchains are present, some fully older and some fully newer", + oldState: @[ + subchain(10, 5), + subchain(20, 15), + subchain(65, 60), + subchain(75, 70), + ], + head: block50, + newState: @[ + subchain(10, 5), + subchain(20, 15), + subchain(50, 50), + ], + reorg: true + ), + # it with one more header. We expect the subchain head to be pushed forward. + TestCase( + name: "A single leftover subchain is present and the new head is extending", + blocks: @[block49], + oldState: @[subchain(49, 5)], + head: block50, + newState: @[subchain(50, 5)], + reorg: false + ), + # A single leftover subchain is present. A new head is announced that + # links into the middle of it, correctly anchoring into an existing + # header. We expect the old subchain to be truncated and extended with + # the new head. + TestCase( + name: "Duplicate announcement should not modify subchain", + blocks: @[block49, block50], + oldState: @[subchain(100, 5)], + head: block50, + newState: @[subchain(100, 5)], + reorg: false + ), + # A single leftover subchain is present. A new head is announced that + # links into the middle of it, correctly anchoring into an existing + # header. We expect the old subchain to be truncated and extended with + # the new head. + TestCase( + name: "A new alternate head is announced in the middle should truncate subchain", + blocks: @[block49, block50], + oldState: @[subchain(100, 5)], + head: block50B, + newState: @[subchain(50, 5)], + reorg: true + ), + # A single leftover subchain is present. A new head is announced that + # links into the middle of it, but does not anchor into an existing + # header. We expect the old subchain to be truncated and a new chain + # be created for the dangling head. + TestCase( + name: "The old subchain to be truncated and a new chain be created for the dangling head", + blocks: @[block49B], + oldState: @[subchain(100, 5)], + head: block50, + newState: @[ + subchain(49, 5), + subchain(50, 50), + ], + reorg: true + ), + ] + +proc test1*() = + suite "Tests various sync initializations": + # based on previous leftovers in the database + # and announced heads. + for z in testCases: + test z.name: + let env = setupEnv() + let skel = SkeletonRef.new(env.chain) + let res = skel.open() + check res.isOk + if res.isErr: + debugEcho res.error + break + + for header in z.blocks: + skel.putHeader(header) + + for x in z.oldState: + skel.push(x.head, x.tail, Hash256()) + + let r = skel.initSync(z.head).valueOr: + debugEcho "initSync: ", error + check false + break + + check r.status.card == 0 + check r.reorg == z.reorg + + check skel.len == z.newState.len + for i, sc in skel: + check sc.head == z.newState[i].head + check sc.tail == z.newState[i].tail diff --git a/tests/test_beacon/test_2_extend.nim b/tests/test_beacon/test_2_extend.nim new file mode 100644 index 0000000000..b718a83963 --- /dev/null +++ b/tests/test_beacon/test_2_extend.nim @@ -0,0 +1,125 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + unittest2, + ./setup_env, + ../../nimbus/sync/beacon/skeleton_main, + ../../nimbus/sync/beacon/skeleton_utils + +# Tests that a running skeleton sync can be extended with properly linked up +# headers but not with side chains. + +type + TestCase = object + name : string + blocks : seq[BlockHeader] # Database content (besides the genesis) + head : BlockHeader # New head header to announce to reorg to + extend : BlockHeader # New head header to announce to extend with + force : bool # To force set head not just to extend + newState: seq[Subchain] # Expected sync state after the reorg + err : Opt[SkeletonStatus] # Whether extension succeeds or not + +let testCases = [ + # Initialize a sync and try to extend it with a subsequent block. + TestCase( + name: "Initialize a sync and try to extend it with a subsequent block", + head: block49, + extend: block50, + force: true, + newState: @[subchain(50, 49)], + ), + # Initialize a sync and try to extend it with the existing head block. + TestCase( + name: "Initialize a sync and try to extend it with the existing head block", + blocks: @[block49], + head: block49, + extend: block49, + newState: @[subchain(49, 49)], + ), + # Initialize a sync and try to extend it with a sibling block. + TestCase( + name: "Initialize a sync and try to extend it with a sibling block", + head: block49, + extend: block49B, + newState: @[subchain(49, 49)], + err: Opt.some ReorgDenied, + ), + # Initialize a sync and try to extend it with a number-wise sequential + # header, but a hash wise non-linking one. + TestCase( + name: "Initialize a sync and try to extend it with a number-wise sequential alternate block", + head: block49B, + extend: block50, + newState: @[subchain(49, 49)], + err: Opt.some ReorgDenied, + ), + # Initialize a sync and try to extend it with a non-linking future block. + TestCase( + name: "Initialize a sync and try to extend it with a non-linking future block", + head: block49, + extend: block51, + newState: @[subchain(49, 49)], + err: Opt.some ReorgDenied, + ), + # Initialize a sync and try to extend it with a past canonical block. + TestCase( + name: "Initialize a sync and try to extend it with a past canonical block", + head: block50, + extend: block49, + newState: @[subchain(50, 50)], + err: Opt.some ReorgDenied, + ), + # Initialize a sync and try to extend it with a past sidechain block. + TestCase( + name: "Initialize a sync and try to extend it with a past sidechain block", + head: block50, + extend: block49B, + newState: @[subchain(50, 50)], + err: Opt.some ReorgDenied, + ), + ] + +proc test2*() = + suite "Tests that a running skeleton sync can be extended": + for z in testCases: + test z.name: + let env = setupEnv() + let skel = SkeletonRef.new(env.chain) + let res = skel.open() + check res.isOk + if res.isErr: + debugEcho res.error + check false + break + + let x = skel.initSync(z.head).valueOr: + debugEcho "initSync: ", error + check false + break + + check x.status.card == 0 + check x.reorg == true + + let r = skel.setHead(z.extend, z.force, false, true).valueOr: + debugEcho "setHead: ", error + check false + break + + if z.err.isSome: + check r.status.card == 1 + check z.err.get in r.status + else: + check r.status.card == 0 + + check skel.len == z.newState.len + for i, sc in skel: + check sc.head == z.newState[i].head + check sc.tail == z.newState[i].tail diff --git a/tests/test_beacon/test_3_sethead_genesis.nim b/tests/test_beacon/test_3_sethead_genesis.nim new file mode 100644 index 0000000000..9fcd76f427 --- /dev/null +++ b/tests/test_beacon/test_3_sethead_genesis.nim @@ -0,0 +1,109 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + unittest2, + ./setup_env, + ../../nimbus/sync/beacon/skeleton_main, + ../../nimbus/sync/beacon/skeleton_utils + +proc test3*() = + suite "should init/setHead properly from genesis": + let env = setupEnv() + let skel = SkeletonRef.new(env.chain) + + test "skel open ok": + let res = skel.open() + check res.isOk + if res.isErr: + debugEcho res.error + check false + break + + let + genesis = env.chain.com.genesisHeader + block1 = header(1, genesis, genesis, 100) + block2 = header(2, genesis, block1, 100) + block3 = header(3, genesis, block1, 100) + + test "should not reorg on genesis init": + skel.initSyncT(genesis, false) + + test "should not reorg on genesis announcement": + skel.setHeadT(genesis, false, false) + + test "should not reorg on genesis setHead": + skel.setHeadT(genesis, true, false) + + test "no subchain should have been created": + check skel.len == 0 + + test "should not allow putBlocks since no subchain set": + let r = skel.putBlocks([block1]) + check r.isErr + + test "canonical height should be at genesis": + check skel.blockHeight == 0 + + test "should not reorg on valid first block": + skel.setHeadT(block1, false, false) + + test "no subchain should have been created": + check skel.len == 0 + + test "should not reorg on valid first block": + skel.setHeadT(block1, true, false) + + test "subchain should have been created": + check skel.len == 1 + + test "head should be set to first block": + check skel.last.head == 1 + + test "subchain status should be linked": + skel.isLinkedT(true) + + test "should not reorg on valid second block": + skel.setHeadT(block2, true, false) + + test "subchain should be same": + check skel.len == 1 + + test "head should be set to first block": + check skel.last.head == 2 + + test "subchain status should stay linked": + skel.isLinkedT(true) + + test "should not extend on invalid third block": + skel.setHeadT(block3, false, true) + + # since its not a forced update so shouldn"t affect subchain status + test "subchain should be same": + check skel.len == 1 + + test "head should be set to second block": + check skel.last.head == 2 + + test "subchain status should stay linked": + skel.isLinkedT(true) + + test "should not extend on invalid third block": + skel.setHeadT(block3, true, true) + + # since its not a forced update so shouldn"t affect subchain status + test "new subchain should be created": + check skel.len == 2 + + test "head should be set to third block": + check skel.last.head == 3 + + test "subchain status should not be linked anymore": + skel.isLinkedT(false) diff --git a/tests/test_beacon/test_4_fill_canonical.nim b/tests/test_beacon/test_4_fill_canonical.nim new file mode 100644 index 0000000000..d38b7c390f --- /dev/null +++ b/tests/test_beacon/test_4_fill_canonical.nim @@ -0,0 +1,64 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + unittest2, + ./setup_env, + ../../nimbus/sync/beacon/skeleton_main, + ../../nimbus/sync/beacon/skeleton_utils, + ../../nimbus/sync/beacon/skeleton_db + +proc test4*() = + suite "should fill the canonical chain after being linked to genesis": + let env = setupEnv() + let skel = SkeletonRef.new(env.chain) + + test "skel open ok": + let res = skel.open() + check res.isOk + if res.isErr: + debugEcho res.error + check false + break + + let + genesis = env.chain.com.genesisHeader + block1 = header(1, genesis, genesis, 100) + block2 = header(2, genesis, block1, 100) + block3 = header(3, genesis, block2, 100) + block4 = header(4, genesis, block3, 100) + block5 = header(5, genesis, block4, 100) + emptyBody = emptyBody() + + test "put body": + for header in [block1, block2, block3, block4, block5]: + let res = skel.putBody(header, emptyBody) + check res.isOk + + test "canonical height should be at genesis": + skel.initSyncT(block4, true) + skel.putBlocksT([block3, block2], 2, {}) + check skel.blockHeight == 0 + + test "canonical height should update after being linked": + skel.putBlocksT([block1], 1, {FillCanonical}) + check skel.blockHeight == 4 + + test "canonical height should not change when setHead is set with force=false": + skel.setHeadT(block5, false, false, {}) + check skel.blockHeight == 4 + + test "canonical height should change when setHead is set with force=true": + skel.setHeadT(block5, true, false, {FillCanonical}) + check skel.blockHeight == 5 + + test "skel header should be cleaned up after filling canonical chain": + let headers = [block1, block2, block3, block4, block5] + skel.getHeaderClean(headers) diff --git a/tests/test_beacon/test_5_canonical_past_genesis.nim b/tests/test_beacon/test_5_canonical_past_genesis.nim new file mode 100644 index 0000000000..622e5848ef --- /dev/null +++ b/tests/test_beacon/test_5_canonical_past_genesis.nim @@ -0,0 +1,67 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + unittest2, + ./setup_env, + ../../nimbus/sync/beacon/skeleton_main, + ../../nimbus/sync/beacon/skeleton_utils, + ../../nimbus/sync/beacon/skeleton_db + +proc test5*() = + suite "should fill the canonical chain after being linked to a canonical block past genesis": + let env = setupEnv() + let skel = SkeletonRef.new(env.chain) + + test "skel open ok": + let res = skel.open() + check res.isOk + if res.isErr: + debugEcho res.error + check false + break + + let + genesis = env.chain.com.genesisHeader + block1 = header(1, genesis, genesis, 100) + block2 = header(2, genesis, block1, 100) + block3 = header(3, genesis, block2, 100) + block4 = header(4, genesis, block3, 100) + block5 = header(5, genesis, block4, 100) + emptyBody = emptyBody() + + test "put body": + for header in [block1, block2, block3, block4, block5]: + let res = skel.putBody(header, emptyBody) + check res.isOk + + test "canonical height should be at block 2": + let r = skel.insertBlocks([block1, block2], [emptyBody, emptyBody], false) + check r.isOk + check r.get == 2 + + skel.initSyncT(block4, true) + check skel.blockHeight == 2 + + test "canonical height should update after being linked": + skel.putBlocksT([block3], 1, {FillCanonical}) + check skel.blockHeight == 4 + + test "canonical height should not change when setHead with force=false": + skel.setHeadT(block5, false, false, {}) + check skel.blockHeight == 4 + + test "canonical height should change when setHead with force=true": + skel.setHeadT(block5, true, false, {FillCanonical}) + check skel.blockHeight == 5 + + test "skel header should be cleaned up after filling canonical chain": + let headers = [block3, block4, block5] + skel.getHeaderClean(headers) diff --git a/tests/test_beacon/test_6_abort_filling.nim b/tests/test_beacon/test_6_abort_filling.nim new file mode 100644 index 0000000000..7402c9d491 --- /dev/null +++ b/tests/test_beacon/test_6_abort_filling.nim @@ -0,0 +1,83 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + std/options, + stew/byteutils, + unittest2, + ./setup_env, + ../../nimbus/sync/beacon/skeleton_main, + ../../nimbus/sync/beacon/skeleton_utils, + ../../nimbus/sync/beacon/skeleton_db + +proc ccm(cc: NetworkParams) = + cc.config.terminalTotalDifficulty = some(262000.u256) + cc.genesis.extraData = hexToSeqByte("0x000000000000000000") + cc.genesis.difficulty = 1.u256 + +proc test6*() = + suite "should abort filling the canonical chain if the terminal block is invalid": + + let env = setupEnv(extraValidation = true, ccm) + let skel = SkeletonRef.new(env.chain) + + test "skel open ok": + let res = skel.open() + check res.isOk + if res.isErr: + debugEcho res.error + check false + break + + let + genesis = env.chain.com.genesisHeader + block1 = env.chain.com.header(1, genesis, genesis, + "6BD9564DD3F4028E3E56F62F1BE52EC8F893CC4FD7DB75DB6A1DC3EB2858998C") + block2 = env.chain.com.header(2, block1, block1, + "32DAA84E151F4C8C6BD4D9ADA4392488FFAFD42ACDE1E9C662B3268C911A5CCC") + block3PoW = env.chain.com.header(3, block2, block2) + block3PoS = header(3, block2, block2, 0) + block4InvalidPoS = header(4, block3PoS, block3PoW, 0) + block4PoS = header(4, block3PoS, block3PoS, 0) + block5 = header(5, block4PoS, block4PoS, 0) + emptyBody = emptyBody() + + test "put body": + for header in [block1, block2, block3PoW, block3PoS, block4InvalidPoS, block4PoS, block5]: + let res = skel.putBody(header, emptyBody) + check res.isOk + + test "canonical height should be at genesis": + skel.initSyncT(block4InvalidPoS, true) + skel.putBlocksT([block3PoW, block2], 2, {}) + check skel.blockHeight == 0 + + test "canonical height should stop at block 2": + # (valid terminal block), since block 3 is invalid (past ttd) + skel.putBlocksT([block1], 1, {FillCanonical}) + check skel.blockHeight == 2 + + test "canonical height should not change when setHead is set with force=false": + skel.setHeadT(block5, false, true, {}) + check skel.blockHeight == 2 + + test "canonical height should now be at head with correct chain": + # Put correct chain + skel.initSyncT(block4PoS, true, {}) + skel.putBlocksT([block3PoS], 1, {FillCanonical}) + check skel.blockHeight == 4 + + test "canonical height should now be at head with correct chain": + let latestHash = env.chain.currentBlock().blockHash + check latestHash == block4PoS.blockHash + + test "should update to new height": + skel.setHeadT(block5, true, false) + check skel.last.head == 5 diff --git a/tests/test_beacon/test_7_abort_and_backstep.nim b/tests/test_beacon/test_7_abort_and_backstep.nim new file mode 100644 index 0000000000..92b4062b20 --- /dev/null +++ b/tests/test_beacon/test_7_abort_and_backstep.nim @@ -0,0 +1,63 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + unittest2, + stew/byteutils, + ./setup_env, + ../../nimbus/sync/beacon/skeleton_main, + ../../nimbus/sync/beacon/skeleton_utils, + ../../nimbus/sync/beacon/skeleton_db + +proc ccm(cc: NetworkParams) = + cc.config.terminalTotalDifficulty = some(262000.u256) + cc.genesis.extraData = hexToSeqByte("0x000000000000000000") + cc.genesis.difficulty = 1.u256 + +proc test7*() = + suite "should abort filling the canonical chain and backstep if the terminal block is invalid": + let env = setupEnv(extraValidation = true, ccm) + let skel = SkeletonRef.new(env.chain) + + test "skel open ok": + let res = skel.open() + check res.isOk + if res.isErr: + debugEcho res.error + check false + break + + let + genesis = env.chain.com.genesisHeader + block1 = env.chain.com.header(1, genesis, genesis, + "6BD9564DD3F4028E3E56F62F1BE52EC8F893CC4FD7DB75DB6A1DC3EB2858998C") + block2 = env.chain.com.header(2, block1, block1, + "32DAA84E151F4C8C6BD4D9ADA4392488FFAFD42ACDE1E9C662B3268C911A5CCC") + block3PoW = env.chain.com.header(3, block2, block2) + block4InvalidPoS = header(4, block3PoW, block3PoW, 0) + emptyBody = emptyBody() + + test "put body": + for header in [block1, block2, block3PoW, block4InvalidPoS]: + let res = skel.putBody(header, emptyBody) + check res.isOk + + test "canonical height should be at genesis": + skel.initSyncT(block4InvalidPoS, true) + skel.putBlocksT([block3PoW, block2], 2, {}) + check skel.blockHeight == 0 + + test "canonical height should stop at block 2": + # (valid terminal block), since block 3 is invalid (past ttd) + skel.putBlocksT([block1], 1, {FillCanonical}) + check skel.blockHeight == 2 + + test "Subchain should have been backstepped to 4": + check skel.last.tail == 4 diff --git a/tests/test_beacon/test_8_pos_too_early.nim b/tests/test_beacon/test_8_pos_too_early.nim new file mode 100644 index 0000000000..44eda5cd1e --- /dev/null +++ b/tests/test_beacon/test_8_pos_too_early.nim @@ -0,0 +1,65 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + unittest2, + stew/byteutils, + ./setup_env, + ../../nimbus/sync/beacon/skeleton_main, + ../../nimbus/sync/beacon/skeleton_utils, + ../../nimbus/sync/beacon/skeleton_db + +proc ccm(cc: NetworkParams) = + cc.config.terminalTotalDifficulty = some(262000.u256) + cc.genesis.extraData = hexToSeqByte("0x000000000000000000") + cc.genesis.difficulty = 1.u256 + +proc test8*() = + suite "should abort filling the canonical chain if a PoS block comes too early without hitting ttd": + let env = setupEnv(extraValidation = true, ccm) + let skel = SkeletonRef.new(env.chain) + skel.fillCanonicalBackStep = 0 + + test "skel open ok": + let res = skel.open() + check res.isOk + if res.isErr: + debugEcho res.error + check false + break + + let + genesis = env.chain.com.genesisHeader + block1 = env.chain.com.header(1, genesis, genesis, + "6BD9564DD3F4028E3E56F62F1BE52EC8F893CC4FD7DB75DB6A1DC3EB2858998C") + block2 = env.chain.com.header(2, block1, block1, + "32DAA84E151F4C8C6BD4D9ADA4392488FFAFD42ACDE1E9C662B3268C911A5CCC") + block2PoS = header(2, block1, block1, 0) + block3 = header(3, block2, block2, 0) + emptyBody = emptyBody() + + test "put body": + for header in [block1, block2, block2PoS, block3]: + let res = skel.putBody(header, emptyBody) + check res.isOk + + test "canonical height should stop at block 1": + # (valid PoW block), since block 2 is invalid (invalid PoS, not past ttd) + skel.initSyncT(block2PoS, true) + skel.putBlocksT([block1], 1, {FillCanonical}) + check skel.blockHeight == 1 + + test "canonical height should now be at head with correct chain": + # Put correct chain + skel.initSyncT(block3, true) + skel.putBlocksT([block2], 1, {FillCanonical}) + check skel.blockHeight == 3 + let latestHash = env.chain.currentBlock().blockHash + check latestHash == block3.blockHash diff --git a/tests/test_beacon/test_skeleton.nim b/tests/test_beacon/test_skeleton.nim new file mode 100644 index 0000000000..b6e0b8fd36 --- /dev/null +++ b/tests/test_beacon/test_skeleton.nim @@ -0,0 +1,47 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at +# https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at +# https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed +# except according to those terms. + +import + unittest2, + test_1_initsync, + test_2_extend, + test_3_sethead_genesis, + test_4_fill_canonical, + test_5_canonical_past_genesis, + test_6_abort_filling, + test_7_abort_and_backstep, + test_8_pos_too_early, + ../../nimbus/sync/beacon/skeleton_main, + ./setup_env + +proc ccm(cc: NetworkParams) = + cc.config.terminalTotalDifficulty = none(UInt256) + cc.genesis.difficulty = 1.u256 + +proc skeletonMain*() = + test1() + test2() + test3() + test4() + test5() + test6() + test7() + test8() + + suite "skeleton open should error if ttd not set": + let env = setupEnv(extraValidation = true, ccm) + let skel = SkeletonRef.new(env.chain) + + test "skel open error": + let res = skel.open() + check res.isErr + +when isMainModule: + skeletonMain() From b9c1d36c3f2959fadefe79b328fa22e0652c029e Mon Sep 17 00:00:00 2001 From: jangko Date: Tue, 19 Sep 2023 10:28:10 +0700 Subject: [PATCH 3/3] More unlisted exception fix --- fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim | 7 +++---- nimbus/utils/utils.nim | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim b/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim index b1877fc40e..6aa704c9c3 100644 --- a/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim +++ b/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim @@ -195,9 +195,8 @@ proc calculateTransactionData( try: let tx = distinctBase(t) tr.put(rlp.encode(i), tx) - except RlpError as e: - # TODO: Investigate this RlpError as it doesn't sound like this is - # something that can actually occur. + except CatchableError as e: + # tr.put interface can raise exception raiseAssert(e.msg) return tr.rootHash() @@ -218,7 +217,7 @@ proc calculateWithdrawalsRoot( amount: distinctBase(w.amount) ) tr.put(rlp.encode(i), rlp.encode(withdrawal)) - except RlpError as e: + except CatchableError as e: raiseAssert(e.msg) return tr.rootHash() diff --git a/nimbus/utils/utils.nim b/nimbus/utils/utils.nim index 7b7e7b42f0..2dd0b36ded 100644 --- a/nimbus/utils/utils.nim +++ b/nimbus/utils/utils.nim @@ -33,7 +33,7 @@ func sumHash*(hashes: varargs[Hash256]): Hash256 = ctx.finish result.data ctx.clear() -proc sumHash*(body: BlockBody): Hash256 {.gcsafe, raises: [RlpError]} = +proc sumHash*(body: BlockBody): Hash256 {.gcsafe, raises: [CatchableError]} = let txRoot = calcTxRoot(body.transactions) let ommersHash = keccakHash(rlp.encode(body.uncles)) let wdRoot = if body.withdrawals.isSome: