add: get_custody_columns for das_core (#6532)

* add: get_custody_columnns in das core specs with tests * apply review changes * review changes 2 * review 3
status-im · Sep 4, 2024 · 444d1dd · 444d1dd
1 parent e6ebefc
commit 444d1dd
Show file tree

Hide file tree

Showing 5 changed files with 281 additions and 1 deletion.
diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md
@@ -175,6 +175,19 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 + Tail block only in common                                                                  OK
 ```
 OK: 2/2 Fail: 0/2 Skip: 0/2
+## EF - EIP7594 - Networking [Preset: mainnet]
+```diff
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
+```
+OK: 9/9 Fail: 0/9 Skip: 0/9
 ## EF - KZG
 ```diff
 + KZG - Blob to KZG commitment - blob_to_kzg_commitment_case_invalid_blob_59d64ff6b4648fad   OK
@@ -1100,4 +1113,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 749/754 Fail: 0/754 Skip: 5/754
+OK: 758/763 Fail: 0/763 Skip: 5/763
diff --git a/beacon_chain/spec/datatypes/eip7594.nim b/beacon_chain/spec/datatypes/eip7594.nim
@@ -0,0 +1,110 @@
+# beacon_chain
+# Copyright (c) 2022-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/[sequtils],
+  "."/[altair, base, deneb], 
+  kzg4844/[kzg, kzg_abi]
+
+from std/strutils import join
+
+export base
+
+const
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/polynomial-commitments-sampling.md#cells
+  FIELD_ELEMENTS_PER_EXT_BLOB* = 2 * kzg_abi.FIELD_ELEMENTS_PER_BLOB
+  # Number of field elements in a Reed-Solomon extended blob |
+  FIELD_ELEMENTS_PER_CELL* = 64 # Number of field elements in a cell |
+  BYTES_PER_CELL* = FIELD_ELEMENTS_PER_CELL * kzg_abi.BYTES_PER_FIELD_ELEMENT
+  # The number of bytes in a cell |
+  CELLS_PER_EXT_BLOB* = FIELD_ELEMENTS_PER_EXT_BLOB div FIELD_ELEMENTS_PER_CELL
+  # The number of cells in an extended blob |
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/p2p-interface.md#preset
+  KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH* = 4
+
+type
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/polynomial-commitments-sampling.md#custom-types 
+  BLSFieldElement* = KzgBytes32
+  G2Point* = array[96, byte]
+  PolynomialCoeff* = List[BLSFieldElement, FIELD_ELEMENTS_PER_EXT_BLOB]
+  Coset* = array[FIELD_ELEMENTS_PER_CELL, BLSFieldElement]
+  CosetEvals* = array[FIELD_ELEMENTS_PER_CELL, BLSFieldElement]
+  Cell* = KzgCell
+  Cells* = KzgCells
+  CellsAndProofs* = KzgCellsAndKzgProofs
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#custom-types
+  RowIndex* = uint64
+  ColumnIndex* = uint64
+  CellIndex* = uint64
+
+const
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#data-size
+  NUMBER_OF_COLUMNS* = 128
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#networking
+  DATA_COLUMN_SIDECAR_SUBNET_COUNT* = 128
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#custody-setting
+  SAMPLES_PER_SLOT* = 8
+  CUSTODY_REQUIREMENT* = 4
+
+type
+  DataColumn* = List[KzgCell, Limit(MAX_BLOB_COMMITMENTS_PER_BLOCK)]
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#datacolumnsidecar
+  DataColumnSidecar* = object
+    index*: ColumnIndex # Index of column in extended matrix
+    column*: DataColumn
+    kzg_commitments*: KzgCommitments
+    kzg_proofs*: KzgProofs
+    signed_block_header*: SignedBeaconBlockHeader
+    kzg_commitments_inclusion_proof*:
+      array[KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH, Eth2Digest]
+
+  DataColumnSidecars* = seq[ref DataColumnSidecar]
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/p2p-interface.md#datacolumnidentifier
+  DataColumnIdentifier* = object
+    block_root*: Eth2Digest
+    index*: ColumnIndex
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#matrixentry
+  MatrixEntry* = object
+    cell*: Cell
+    kzg_proof*: KzgProof
+    column_index*: ColumnIndex
+    row_index*: RowIndex
+
+  # Not in spec, defined in order to compute custody subnets
+  CscBits* = BitArray[DATA_COLUMN_SIDECAR_SUBNET_COUNT]
+
+  CscCount* = uint8
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/p2p-interface.md#metadata
+  MetaData* = object
+    seq_number*: uint64
+    attnets*: AttnetBits
+    syncnets*: SyncnetBits
+    custody_subnet_count*: CscCount 
+
+func shortLog*(v: DataColumnSidecar): auto =
+  (
+    index: v.index,
+    kzg_commitments: v.kzg_commitments.len,
+    kzg_proofs: v.kzg_proofs.len,
+    block_header: shortLog(v.signed_block_header.message),
+  )
+
+func shortLog*(v: seq[DataColumnSidecar]): auto =
+  "[" & v.mapIt(shortLog(it)).join(", ") & "]"
+
+func shortLog*(x: seq[DataColumnIdentifier]): string =
+  "[" & x.mapIt(shortLog(it.block_root) & "/" & $it.index).join(", ") & "]"
diff --git a/beacon_chain/spec/eip7594_helpers.nim b/beacon_chain/spec/eip7594_helpers.nim
@@ -0,0 +1,105 @@
+# beacon_chain
+# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+# Uncategorized helper functions from the spec
+import
+  std/[algorithm, hashes],
+  results,
+  eth/p2p/discoveryv5/[node],
+  ./[helpers, digest],
+  ./datatypes/[eip7594]
+
+proc sortedColumnIndices*(columnsPerSubnet: ColumnIndex,
+                          subnetIds: HashSet[uint64]):
+                          seq[ColumnIndex] =
+  var res: seq[ColumnIndex] = @[]
+  for i in 0'u64 ..< columnsPerSubnet:
+    for subnetId in subnetIds:
+      let index = DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnetId
+      res.add(ColumnIndex(index))
+  res.sort
+  res
+
+proc sortedColumnIndexList*(columnsPerSubnet: ColumnIndex, 
+                            subnetIds: HashSet[uint64]): 
+                            List[ColumnIndex, NUMBER_OF_COLUMNS] =
+  var
+    res: seq[ColumnIndex]
+  for i in 0'u64 ..< columnsPerSubnet:
+    for subnetId in subnetIds:
+      let index = DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnetId
+      res.add(ColumnIndex(index))
+  res.sort()
+  let list = List[ColumnIndex, NUMBER_OF_COLUMNS].init(res)
+  list
+
+proc get_custody_column_subnets*(node_id: NodeId, 
+                                 custody_subnet_count: uint64): 
+                                 Result[HashSet[uint64], cstring] =
+
+  # Decouples the custody subnet computation part from
+  # `get_custody_columns`, in order to later use this subnet list
+  # in order to maintain subscription to specific column subnets.
+
+  if not (custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT):
+    return err("Eip7594: Custody subnet count exceeds the DATA_COLUMN_SIDECAR_SUBNET_COUNT")
+
+  var 
+    subnet_ids: HashSet[uint64]
+    current_id = node_id
+
+  while subnet_ids.lenu64 < custody_subnet_count:
+    var
+      hashed_bytes: array[8, byte]
+
+    let
+      current_id_bytes = current_id.toBytesLE()
+      hashed_current_id = eth2digest(current_id_bytes)
+
+    hashed_bytes[0..7] = hashed_current_id.data.toOpenArray(0,7)
+    let subnet_id = bytes_to_uint64(hashed_bytes) mod 
+      DATA_COLUMN_SIDECAR_SUBNET_COUNT
+
+    subnet_ids.incl(subnet_id)
+
+    if current_id == UInt256.high.NodeId:
+      # Overflow prevention
+      current_id = NodeId(StUint[256].zero)
+    current_id += NodeId(StUint[256].one)
+
+  ok(subnet_ids)
+
+# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#get_custody_columns
+proc get_custody_columns*(node_id: NodeId, 
+                          custody_subnet_count: uint64): 
+                          seq[ColumnIndex] =
+  let
+    subnet_ids = 
+      get_custody_column_subnets(node_id, custody_subnet_count).get
+  const
+    columns_per_subnet = 
+      NUMBER_OF_COLUMNS div DATA_COLUMN_SIDECAR_SUBNET_COUNT
+
+  sortedColumnIndices(ColumnIndex(columns_per_subnet), subnet_ids)
+
+
+proc get_custody_column_list*(node_id: NodeId, 
+                          custody_subnet_count: uint64): 
+                          List[ColumnIndex, NUMBER_OF_COLUMNS] =
+
+  # Not in spec in the exact format, but it is useful in sorting custody columns 
+  # before sending, data_column_sidecars_by_range requests
+  let
+    subnet_ids = 
+      get_custody_column_subnets(node_id, custody_subnet_count).get
+  const
+    columns_per_subnet = 
+      NUMBER_OF_COLUMNS div DATA_COLUMN_SIDECAR_SUBNET_COUNT
+
+  sortedColumnIndexList(ColumnIndex(columns_per_subnet), subnet_ids)
diff --git a/tests/consensus_spec/all_tests.nim b/tests/consensus_spec/all_tests.nim
@@ -15,4 +15,5 @@
 
 import
   ./test_fixture_kzg,
+  ./test_fixture_networking,
   ./test_fixture_ssz_generic_types
diff --git a/tests/consensus_spec/test_fixture_networking.nim b/tests/consensus_spec/test_fixture_networking.nim
@@ -0,0 +1,51 @@
+# beacon_chain
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+{.used.}
+
+import
+  std/[json, streams],
+  yaml,
+  kzg4844/[kzg, kzg_abi],
+  stint,
+  eth/p2p/discoveryv5/[node],
+  ../../beacon_chain/spec/eip7594_helpers,
+  ../testutil,
+  ./fixtures_utils, ./os_ops
+
+from std/sequtils import mapIt
+
+proc runGetCustodyColumns(suiteName, path: string) =
+  let relativePathComponent = path.relativeTestPathComponent()
+  test "Networking - Get Custody Columns - " & relativePathComponent:
+    type TestMetaYaml = object
+      node_id: string
+      custody_subnet_count: uint64
+      result: seq[uint64]
+    let
+      meta = block:
+        var s = openFileStream(path/"meta.yaml")
+        defer: close(s)
+        var res: TestMetaYaml
+        yaml.load(s, res)
+        res
+      node_id = UInt256.fromDecimal(meta.node_id)
+      custody_subnet_count = meta.custody_subnet_count
+      reslt = (meta.result).mapIt(it)
+
+    let columns = get_custody_columns(node_id, custody_subnet_count)
+
+    for i in 0..<columns.lenu64:
+      check columns[i] == reslt[i]
+
+suite "EF - EIP7594 - Networking" & preset():
+  const presetPath = SszTestsDir/const_preset
+  let basePath =
+    presetPath/"eip7594"/"networking"/"get_custody_columns"/"pyspec_tests"
+  for kind, path in walkDir(basePath, relative = true, checkDir = true):
+    runGetCustodyColumns(suiteName, basePath/path)