add: get_custody_columns for das_core

AllTests-mainnet.md

@@ -175,6 +175,19 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 + Tail block only in common                                                                  OK
 ```
 OK: 2/2 Fail: 0/2 Skip: 0/2
+## EF - EIP7594 - Networking [Preset: mainnet]
+```diff
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
++ Networking - Get Custody Columns - mainnet/eip7594/networking/get_custody_columns/pyspec_t OK
+```
+OK: 9/9 Fail: 0/9 Skip: 0/9
 ## EF - KZG
 ```diff
 + KZG - Blob to KZG commitment - blob_to_kzg_commitment_case_invalid_blob_59d64ff6b4648fad   OK
@@ -1100,4 +1113,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 749/754 Fail: 0/754 Skip: 5/754
+OK: 758/763 Fail: 0/763 Skip: 5/763

beacon_chain/spec/datatypes/eip7594.nim

@@ -0,0 +1,101 @@
+# beacon_chain
+# Copyright (c) 2022-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  std/[sequtils],
+  "."/[altair, base, deneb], 
+  kzg4844/[kzg, kzg_abi]
+
+from std/strutils import join
+
+export base
+
+const
+  FIELD_ELEMENTS_PER_EXT_BLOB* = 2 * kzg_abi.FIELD_ELEMENTS_PER_BLOB
+    # Number of field elements in a Reed-Solomon extended blob |
+  FIELD_ELEMENTS_PER_CELL* = 64 # Number of field elements in a cell |
+  BYTES_PER_CELL* = FIELD_ELEMENTS_PER_CELL * kzg_abi.BYTES_PER_FIELD_ELEMENT
+    # The number of bytes in a cell |
+  CELLS_PER_EXT_BLOB* = FIELD_ELEMENTS_PER_EXT_BLOB div FIELD_ELEMENTS_PER_CELL
+    # The number of cells in an extended blob |
+  KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH* = 4
+
+type
+  BLSFieldElement* = KzgBytes32
+  G2Point* = array[96, byte]
+  PolynomialCoeff* = List[BLSFieldElement, FIELD_ELEMENTS_PER_EXT_BLOB]
+  Coset* = array[FIELD_ELEMENTS_PER_CELL, BLSFieldElement]
+  CosetEvals* = array[FIELD_ELEMENTS_PER_CELL, BLSFieldElement]
+  Cell* = KzgCell
+  Cells* = KzgCells
+  CellsAndProofs* = KzgCellsAndKzgProofs
+  CellID* = uint64
+  RowIndex* = uint64
+  ColumnIndex* = uint64
+  CellIndex* = uint64
+
+const
+  NUMBER_OF_COLUMNS* = 128
+  MAX_CELLS_IN_EXTENDED_MATRIX* = MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS
+
+  DATA_COLUMN_SIDECAR_SUBNET_COUNT* = 128
+  SAMPLES_PER_SLOT* = 8
+  CUSTODY_REQUIREMENT* = 4
+  TARGET_NUMBER_OF_PEERS* = 70
+
+type
+  DataColumn* = List[KzgCell, Limit(MAX_BLOB_COMMITMENTS_PER_BLOCK)]
+  ExtendedMatrix* = List[KzgCell, Limit(MAX_CELLS_IN_EXTENDED_MATRIX)]
+
+  DataColumnSidecar* = object
+    index*: ColumnIndex # Index of column in extended matrix
+    column*: DataColumn
+    kzg_commitments*: KzgCommitments
+    kzg_proofs*: KzgProofs
+    signed_block_header*: SignedBeaconBlockHeader
+    kzg_commitments_inclusion_proof*:
+      array[KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH, Eth2Digest]
+
+  DataColumnSidecars* = seq[ref DataColumnSidecar]
+
+  DataColumnIdentifier* = object
+    block_root*: Eth2Digest
+    index*: ColumnIndex
+
+  MatrixEntry* = object
+    cell*: Cell
+    kzg_proof*: KzgProof
+    column_index*: ColumnIndex
+    row_index*: RowIndex
+
+  # Not in spec, defined in order to compute custody subnets
+  CscBits* = BitArray[DATA_COLUMN_SIDECAR_SUBNET_COUNT]
+
+  CscCount* = uint8
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.4/specs/_features/eip7594/p2p-interface.md#metadata
+  MetaData* = object
+    seq_number*: uint64
+    attnets*: AttnetBits
+    syncnets*: SyncnetBits
+    custody_subnet_count*: CscCount 
+
+func shortLog*(v: DataColumnSidecar): auto =
+  (
+    index: v.index,
+    kzg_commitments: v.kzg_commitments.len,
+    kzg_proofs: v.kzg_proofs.len,
+    block_header: shortLog(v.signed_block_header.message),
+  )
+
+func shortLog*(v: seq[DataColumnSidecar]): auto =
+  "[" & v.mapIt(shortLog(it)).join(", ") & "]"
+
+func shortLog*(x: seq[DataColumnIdentifier]): string =
+  "[" & x.mapIt(shortLog(it.block_root) & "/" & $it.index).join(", ") & "]"

beacon_chain/spec/eip7594_helpers.nim

@@ -0,0 +1,116 @@
+# beacon_chain
+# Copyright (c) 2018-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+# Uncategorized helper functions from the spec
+import
+  std/[algorithm],
+  results,
+  sequtils,
+  hashes,
+  ssz_serialization/[
+    proofs,
+    types],
+  ./[beacon_time, crypto],
+  kzg4844/[kzg, kzg_abi],
+  eth/p2p/discoveryv5/[node],
+  ./[helpers, digest],
+
+  ./datatypes/[eip7594, deneb]
+
+proc sortedColumnIndices*(columnsPerSubnet: ColumnIndex, subnetIds: HashSet[uint64]): seq[ColumnIndex] =
+  var res: seq[ColumnIndex] = @[]
+  for i in 0'u64 ..< columnsPerSubnet:
+    for subnetId in subnetIds:
+      let index = DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnetId
+      res.add(ColumnIndex(index))
+  res.sort
+  res
+
+proc sortedColumnIndexList*(columnsPerSubnet: ColumnIndex, 
+                          subnetIds: HashSet[uint64]): 
+                          List[ColumnIndex, NUMBER_OF_COLUMNS] =
+  var
+    res: seq[ColumnIndex]
+    list: List[ColumnIndex, NUMBER_OF_COLUMNS]
+  for i in 0'u64 ..< columnsPerSubnet:
+    for subnetId in subnetIds:
+      let index = DATA_COLUMN_SIDECAR_SUBNET_COUNT * i + subnetId
+      res.add(ColumnIndex(index))
+  res.sort()
+  for elem in res:
+    discard list.add(ColumnIndex(elem))
+  list
+
+proc get_custody_column_subnet*(node_id: NodeId, 
+                                custody_subnet_count: uint64): 
+                                Result[HashSet[uint64], cstring] =
+
+  # Decouples the custody subnet computation part from
+  # `get_custody_columns`, in order to later use this subnet list
+  # in order to maintain subscription to specific column subnets.
+
+  if not (custody_subnet_count <= DATA_COLUMN_SIDECAR_SUBNET_COUNT):
+    return err("Eip7594: Custody subnet count exceeds the DATA_COLUMN_SIDECAR_SUBNET_COUNT")
+
+  var 
+    subnet_ids: HashSet[uint64]
+    current_id = node_id
+
+  while subnet_ids.len < int(custody_subnet_count):
+
+    var
+      current_id_bytes: array[32, byte]
+      hashed_bytes: array[8, byte]
+
+    current_id_bytes = current_id.toBytesBE()
+    current_id_bytes.reverse()
+
+    let
+      hashed_current_id = eth2digest(current_id_bytes)
+
+    hashed_bytes[0..7] = hashed_current_id.data.toOpenArray(0,7)
+    var subnet_id = bytes_to_uint64(hashed_bytes) mod 
+      DATA_COLUMN_SIDECAR_SUBNET_COUNT
+
+    if subnet_id notin subnet_ids:
+        subnet_ids.incl(subnet_id)
+
+    if current_id == UInt256.high.NodeId:
+        # Overflow prevention
+        current_id = NodeId(StUint[256].zero)
+    current_id += NodeId(StUint[256].one)
+
+  ok(subnet_ids)
+
+# https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/_features/eip7594/das-core.md#get_custody_columns
+proc get_custody_columns*(node_id: NodeId, 
+                          custody_subnet_count: uint64): 
+                          Result[seq[ColumnIndex], cstring] =
+
+  let subnet_ids = get_custody_column_subnet(node_id, custody_subnet_count).get
+
+  # columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT
+  let columns_per_subnet = NUMBER_OF_COLUMNS div DATA_COLUMN_SIDECAR_SUBNET_COUNT
+
+  ok(sortedColumnIndices(ColumnIndex(columns_per_subnet), subnet_ids))
+
+
+proc get_custody_column_list*(node_id: NodeId, 
+                          custody_subnet_count: uint64): 
+                          Result[List[ColumnIndex, NUMBER_OF_COLUMNS], cstring] =
+
+  # Not in spec in the exact format, but it is useful in sorting custody columns 
+  # before sending, data_column_sidecars_by_range requests
+
+  let subnet_ids = get_custody_column_subnet(node_id, custody_subnet_count).get
+
+  # columns_per_subnet = NUMBER_OF_COLUMNS // DATA_COLUMN_SIDECAR_SUBNET_COUNT
+  let columns_per_subnet = NUMBER_OF_COLUMNS div DATA_COLUMN_SIDECAR_SUBNET_COUNT
+
+  ok(sortedColumnIndexList(ColumnIndex(columns_per_subnet), subnet_ids))

tests/consensus_spec/all_tests.nim

@@ -15,4 +15,5 @@
 
 import
   ./test_fixture_kzg,
+  ./test_fixture_networking,
   ./test_fixture_ssz_generic_types

tests/consensus_spec/test_fixture_networking.nim

@@ -0,0 +1,78 @@
+# beacon_chain
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+{.used.}
+
+import
+  std/[json, streams],
+  yaml,
+  kzg4844/[kzg, kzg_abi],
+  stint,
+  chronicles,
+  eth/p2p/discoveryv5/[node],
+  results,
+  stew/[byteutils],
+  ../../beacon_chain/spec/eip7594_helpers,
+  ../testutil,
+  ./fixtures_utils, ./os_ops
+
+from std/sequtils import anyIt, mapIt, toSeq
+from std/strutils import rsplit
+
+func toUInt64(s: SomeInteger): Opt[uint64] =
+  if s < 0:
+    return Opt.none uint64
+  try:
+    Opt.some uint64(s)
+  except ValueError:
+    Opt.none uint64
+
+func fromHex[N: static int](s: string): Opt[array[N, byte]] =
+  if s.len != 2*(N+1):
+    # 0x prefix
+    return Opt.none array[N, byte]
+
+  try:
+    Opt.some fromHex(array[N, byte], s)
+  except ValueError:
+    Opt.none array[N, byte]
+
+proc runGetCustodyColumns(suiteName, path: string) =
+  let relativePathComponent = path.relativeTestPathComponent()
+  test "Networking - Get Custody Columns - " & relativePathComponent:
+    type TestMetaYaml = object
+      node_id: string
+      custody_subnet_count: uint64
+      result: Option[seq[uint64]]
+    let
+      meta = block:
+        var s = openFileStream(path/"meta.yaml")
+        defer: close(s)
+        var res: TestMetaYaml
+        yaml.load(s, res)
+        res
+      node_id = UInt256.fromDecimal(meta.node_id)
+      custody_subnet_count = toUInt64(meta.custody_subnet_count)
+      reslt = (meta.result.get).mapIt(uint64(it))
+
+    if custody_subnet_count.isNone:
+      check meta.result.isNone
+    else:
+      let columns = get_custody_columns(node_id, custody_subnet_count.get)
+      if columns.isErr:
+        check meta.result.isNone
+      else:
+        for i in 0..<columns.get.lenu64:
+          check columns.get[i] == uint64(reslt[i])
+
+suite "EF - EIP7594 - Networking" & preset():
+  const presetPath = SszTestsDir/const_preset
+  let basePath =
+    presetPath/"eip7594"/"networking"/"get_custody_columns"/"pyspec_tests"
+  for kind, path in walkDir(basePath, relative = true, checkDir = true):
+    runGetCustodyColumns(suiteName, basePath/path)