builder API liveness failsafe (#4746)

* builder API liveness failsafe

* add test summary change

AllTests-mainnet.md

@@ -258,11 +258,12 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 ## Honest validator
 ```diff
 + General pubsub topics                                                                      OK
++ Liveness failsafe conditions                                                               OK
 + Mainnet attestation topics                                                                 OK
 + isNearSyncCommitteePeriod                                                                  OK
 + is_aggregator                                                                              OK
 ```
-OK: 4/4 Fail: 0/4 Skip: 0/4
+OK: 5/5 Fail: 0/5 Skip: 0/5
 ## ImportKeystores requests [Beacon Node] [Preset: mainnet]
 ```diff
 + ImportKeystores/ListKeystores/DeleteKeystores [Beacon Node] [Preset: mainnet]              OK
@@ -635,4 +636,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 352/357 Fail: 0/357 Skip: 5/357
+OK: 353/358 Fail: 0/358 Skip: 5/358

beacon_chain/spec/validator.nim

@@ -409,3 +409,55 @@ func is_aggregator*(committee_len: uint64, slot_signature: ValidatorSig): bool =
     modulo = max(1'u64, committee_len div TARGET_AGGREGATORS_PER_COMMITTEE)
   bytes_to_uint64(eth2digest(
     slot_signature.toRaw()).data.toOpenArray(0, 7)) mod modulo == 0
+
+# https://github.com/ethereum/builder-specs/pull/47
+func livenessFailsafeInEffect*(
+    block_roots: array[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest],
+    slot: Slot): bool =
+  const
+    MAX_MISSING_CONTIGUOUS = 3
+    MAX_MISSING_WINDOW = 5
+
+  static: doAssert MAX_MISSING_WINDOW > MAX_MISSING_CONTIGUOUS
+  if slot <= MAX_MISSING_CONTIGUOUS:
+    # Cannot ever trigger and allows a bit of safe arithmetic. Furthermore
+    # there's notionally always a genesis block, which pushes the earliest
+    # possible failure out an additional slot.
+    return false
+
+  # Using this slightly convoluted construction to handle wraparound better;
+  # baseIndex + faultInspectionWindow can overflow array but only exactly by
+  # the required amount. Furthermore, go back one more slot to address using
+  # that it looks ahead rather than looks back and whether a block's missing
+  # requires seeing the previous block_root.
+  let
+    faultInspectionWindow = min(distinctBase(slot) - 1, SLOTS_PER_EPOCH)
+    baseIndex = (slot + SLOTS_PER_HISTORICAL_ROOT - faultInspectionWindow) mod
+      SLOTS_PER_HISTORICAL_ROOT
+    endIndex = baseIndex + faultInspectionWindow - 1
+
+  doAssert endIndex mod SLOTS_PER_HISTORICAL_ROOT ==
+    (slot - 1) mod SLOTS_PER_HISTORICAL_ROOT
+
+  var
+    totalMissing = 0
+    streakLen = 0
+    maxStreakLen = 0
+
+  for i in baseIndex .. endIndex:
+    # This look-forward means checking slot i for being missing uses i - 1
+    if  block_roots[(i mod SLOTS_PER_HISTORICAL_ROOT).int] ==
+        block_roots[((i + 1) mod SLOTS_PER_HISTORICAL_ROOT).int]:
+      totalMissing += 1
+      if totalMissing > MAX_MISSING_WINDOW:
+        return true
+
+      streakLen += 1
+      if streakLen > maxStreakLen:
+        maxStreakLen = streakLen
+        if maxStreakLen > MAX_MISSING_CONTIGUOUS:
+          return true
+    else:
+      streakLen = 0
+
+  false

beacon_chain/validators/validator_duties.nim

@@ -681,6 +681,7 @@ proc proposeBlockMEV[
     node: BeaconNode, head: BlockRef, validator: AttachedValidator, slot: Slot,
     randao: ValidatorSig, validator_index: ValidatorIndex):
     Future[Opt[BlockRef]] {.async.} =
+  # Used by the BN's own validators, but not the REST server
   when SBBB is bellatrix_mev.SignedBlindedBeaconBlock:
     type EPH = bellatrix.ExecutionPayloadHeader
   elif SBBB is capella_mev.SignedBlindedBeaconBlock:
@@ -749,6 +750,9 @@ proc makeBlindedBeaconBlockForHeadAndSlot*[
   ## Requests a beacon node to produce a valid blinded block, which can then be
   ## signed by a validator. A blinded block is a block with only a transactions
   ## root, rather than a full transactions list.
+  ##
+  ## This function is used by the validator client, but not the beacon node for
+  ## its own validators.
   when BBB is bellatrix_mev.BlindedBeaconBlock:
     type EPH = bellatrix.ExecutionPayloadHeader
   elif BBB is capella_mev.BlindedBeaconBlock:
@@ -760,6 +764,11 @@ proc makeBlindedBeaconBlockForHeadAndSlot*[
     pubkey =
       # Relevant state for knowledge of validators
       withState(node.dag.headState):
+        if livenessFailsafeInEffect(
+            forkyState.data.block_roots.data, forkyState.data.slot):
+          # It's head block's slot which matters here, not proposal slot
+          return err("Builder API liveness failsafe in effect")
+
         if distinctBase(validator_index) >= forkyState.data.validators.lenu64:
           debug "makeBlindedBeaconBlockForHeadAndSlot: invalid validator index",
             head = shortLog(head),
@@ -819,29 +828,35 @@ proc proposeBlock(node: BeaconNode,
         res.get()
 
   if node.config.payloadBuilderEnable:
-    let newBlockMEV =
-      if slot.epoch >= node.dag.cfg.DENEB_FORK_EPOCH:
-        debugRaiseAssert $denebImplementationMissing & ": proposeBlock"
-        await proposeBlockMEV[
-            capella_mev.SignedBlindedBeaconBlock](
-          node, head, validator, slot, randao, validator_index)
-      elif slot.epoch >= node.dag.cfg.CAPELLA_FORK_EPOCH:
-        await proposeBlockMEV[
-            capella_mev.SignedBlindedBeaconBlock](
-          node, head, validator, slot, randao, validator_index)
-      else:
-        await proposeBlockMEV[
-            bellatrix_mev.SignedBlindedBeaconBlock](
-          node, head, validator, slot, randao, validator_index)
-
-    if newBlockMEV.isSome:
-      # This might be equivalent to the `head` passed in, but it signals that
-      # `submitBlindedBlock` ran, so don't do anything else. Otherwise, it is
-      # fine to try again with the local EL.
-      if newBlockMEV.get == head:
-        # Returning same block as head indicates failure to generate new block
-        beacon_block_builder_missed_without_fallback.inc()
-      return newBlockMEV.get
+    let failsafeInEffect =
+      withState(node.dag.headState):
+        # Head slot, not proposal slot, matters here
+        livenessFailsafeInEffect(
+          forkyState.data.block_roots.data, forkyState.data.slot)
+    if not failsafeInEffect:
+      let newBlockMEV =
+        if slot.epoch >= node.dag.cfg.DENEB_FORK_EPOCH:
+          debugRaiseAssert $denebImplementationMissing & ": proposeBlock"
+          await proposeBlockMEV[
+              capella_mev.SignedBlindedBeaconBlock](
+            node, head, validator, slot, randao, validator_index)
+        elif slot.epoch >= node.dag.cfg.CAPELLA_FORK_EPOCH:
+          await proposeBlockMEV[
+              capella_mev.SignedBlindedBeaconBlock](
+            node, head, validator, slot, randao, validator_index)
+        else:
+          await proposeBlockMEV[
+              bellatrix_mev.SignedBlindedBeaconBlock](
+            node, head, validator, slot, randao, validator_index)
+
+      if newBlockMEV.isSome:
+        # This might be equivalent to the `head` passed in, but it signals that
+        # `submitBlindedBlock` ran, so don't do anything else. Otherwise, it is
+        # fine to try again with the local EL.
+        if newBlockMEV.get == head:
+          # Returning same block as head indicates failure to generate new block
+          beacon_block_builder_missed_without_fallback.inc()
+        return newBlockMEV.get
 
   # TODO Compare the value of the MEV block and the execution block
   #      obtained from the EL below:

tests/test_honest_validator.nim

@@ -198,3 +198,86 @@ suite "Honest validator":
     for i in 1'u64 .. 20'u64:
       for j in (SYNC_COMMITTEE_SUBNET_COUNT + 1'u64) .. 7'u64:
         check: nearSyncCommitteePeriod((EPOCHS_PER_SYNC_COMMITTEE_PERIOD * i - j).Epoch).isNone
+
+  test "Liveness failsafe conditions":
+    var x: array[Limit SLOTS_PER_HISTORICAL_ROOT, Eth2Digest]
+    const MAX_MISSING_CONTIGUOUS = 3
+    const MAX_MISSING_WINDOW = 5
+    const FAULT_INSPECTION_WINDOW = 32
+
+    # There haven't been enough slots to trigger any of the conditions
+    for i in 0 .. MAX_MISSING_CONTIGUOUS + 1:
+      check: not livenessFailsafeInEffect(x, i.Slot)
+    # But once there are, the default all-equals array shouldn't allow it. An
+    # additional slot is gained because it's notionally not possible for some
+    # genesis block not to exist.
+    for i in MAX_MISSING_CONTIGUOUS + 2 .. FAULT_INSPECTION_WINDOW + 10:
+      check: livenessFailsafeInEffect(x, i.Slot)
+
+    for i in FAULT_INSPECTION_WINDOW * 2 ..< FAULT_INSPECTION_WINDOW * 3:
+      x[i].data[0] = i.uint8
+
+    # There haven't been enough slots to trigger any of the conditions; unlike
+    # first round this doesn't line up with genesis-adjacent slots and doesn't
+    # have that additional genesis block additional-slot-before-trigger.
+    for i in
+        FAULT_INSPECTION_WINDOW * 3 ..
+        FAULT_INSPECTION_WINDOW * 3 + MAX_MISSING_CONTIGUOUS:
+      check: not livenessFailsafeInEffect(x, i.Slot)
+    for i in
+        FAULT_INSPECTION_WINDOW * 3 + MAX_MISSING_CONTIGUOUS + 1 ..
+        FAULT_INSPECTION_WINDOW * 4:
+      check: livenessFailsafeInEffect(x, i.Slot)
+
+    # This time, add some extant blocks to extend non-liveness-failsafe conditions
+    for i in FAULT_INSPECTION_WINDOW * 4 ..< FAULT_INSPECTION_WINDOW * 5:
+      x[i].data[0] = i.uint8
+    # extend last entry to simulate missing blocks
+    for i in
+        FAULT_INSPECTION_WINDOW * 5 ..<
+        FAULT_INSPECTION_WINDOW * 5 + MAX_MISSING_CONTIGUOUS:
+      x[i].data[0] = (FAULT_INSPECTION_WINDOW * 5 - 1).uint8
+    # next real block
+    x[FAULT_INSPECTION_WINDOW * 5 + MAX_MISSING_CONTIGUOUS].data[0] = 34
+
+    for i in
+        FAULT_INSPECTION_WINDOW * 5 ..
+        FAULT_INSPECTION_WINDOW * 3 + MAX_MISSING_CONTIGUOUS * 2:
+      check: not livenessFailsafeInEffect(x, i.Slot)
+    for i in
+        FAULT_INSPECTION_WINDOW * 5 + MAX_MISSING_CONTIGUOUS * 2 + 1 ..
+        FAULT_INSPECTION_WINDOW * 6:
+      check: livenessFailsafeInEffect(x, i.Slot)
+
+    # Add some all-present blocks for a few epochs
+    for i in FAULT_INSPECTION_WINDOW * 6 ..< FAULT_INSPECTION_WINDOW * 9:
+      x[i].data[0] = i.uint8
+    static: doAssert MAX_MISSING_WINDOW > MAX_MISSING_CONTIGUOUS
+    # This satisfies contiguous-missing limit, but not total-per-window limit
+    for i in countup(
+        FAULT_INSPECTION_WINDOW * 9,
+        FAULT_INSPECTION_WINDOW * 9 + MAX_MISSING_CONTIGUOUS * 2, 2):
+      x[i].data[0] = i.uint8
+      x[i + 1].data[0] = i.uint8   # missing block
+
+    for i in
+        FAULT_INSPECTION_WINDOW * 7 ..
+        FAULT_INSPECTION_WINDOW * 9 + MAX_MISSING_WINDOW * 2 - 1:
+      # i.e. two fullly covered epochs then get into MAX_MISSING_WINDOW * 2 - 1
+      # of the every-other-block is present. Because only MAX_MISSING_WINDOW of
+      # these can exist, it's the ones at (FIW*9 base of 0): 1, 3, 5, 7, 9 that
+      # are missing. Can get up to 9 here, i.e. by 2 * MAX_MISSING_WINDOW, as a
+      # result of 50% duty cycle pattern.
+      check: not livenessFailsafeInEffect(x, i.Slot)
+    for i in
+        FAULT_INSPECTION_WINDOW * 9 + MAX_MISSING_WINDOW * 2 ..
+        FAULT_INSPECTION_WINDOW * 10:
+      check: livenessFailsafeInEffect(x, i.Slot)
+
+    # Check wraparound is sane; same mod-equivalent slots but actually near
+    # genesis don't trigger liveness failures, as they clamp the inspection
+    # window at element 0 of array rather than wrapping backwards.
+    for i in
+        SLOTS_PER_HISTORICAL_ROOT ..
+        SLOTS_PER_HISTORICAL_ROOT + FAULT_INSPECTION_WINDOW:
+      check: livenessFailsafeInEffect(x, i.Slot)