Spectests (#13940)

* Update `consensus_spec_version` to `v1.5.0-alpha.1`.

* `CustodyColumns`: Fix and implement spec tests.

* Make deepsource happy.

* `^uint64(0)` => `math.MaxUint64`.

* Fix `TestLoadConfigFile` test.

beacon-chain/core/peerdas/helpers.go

@@ -2,11 +2,11 @@ package peerdas
 
 import (
 	"encoding/binary"
+	"math"
 
 	cKzg4844 "github.com/ethereum/c-kzg-4844/bindings/go"
-	"github.com/holiman/uint256"
-
 	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/holiman/uint256"
 	errors "github.com/pkg/errors"
 	fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v5/config/params"
@@ -26,7 +26,7 @@ const (
 )
 
 type (
-	extendedMatrix []cKzg4844.Cell
+	ExtendedMatrix []cKzg4844.Cell
 
 	cellCoordinate struct {
 		blobIndex uint64
@@ -35,10 +35,15 @@ type (
 )
 
 var (
+	// Custom errors
 	errCustodySubnetCountTooLarge = errors.New("custody subnet count larger than data column sidecar subnet count")
 	errCellNotFound               = errors.New("cell not found (should never happen)")
+
+	// maxUint256 is the maximum value of a uint256.
+	maxUint256 = &uint256.Int{math.MaxUint64, math.MaxUint64, math.MaxUint64, math.MaxUint64}
 )
 
+// CustodyColumns computes the columns the node should custody.
 // https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#helper-functions
 func CustodyColumns(nodeId enode.ID, custodySubnetCount uint64) (map[uint64]bool, error) {
 	dataColumnSidecarSubnetCount := params.BeaconConfig().DataColumnSidecarSubnetCount
@@ -75,28 +80,43 @@ func CustodyColumnSubnets(nodeId enode.ID, custodySubnetCount uint64) (map[uint6
 	// First, compute the subnet IDs that the node should participate in.
 	subnetIds := make(map[uint64]bool, custodySubnetCount)
 
-	for i := uint64(0); uint64(len(subnetIds)) < custodySubnetCount; i++ {
-		nodeIdUInt256, nextNodeIdUInt256 := new(uint256.Int), new(uint256.Int)
-		nodeIdUInt256.SetBytes(nodeId.Bytes())
-		nextNodeIdUInt256.Add(nodeIdUInt256, uint256.NewInt(i))
-		nextNodeIdUInt64 := nextNodeIdUInt256.Uint64()
-		nextNodeId := bytesutil.Uint64ToBytesLittleEndian(nextNodeIdUInt64)
+	// Convert the node ID to a big int.
+	nodeIdUInt256 := new(uint256.Int).SetBytes(nodeId.Bytes())
 
-		hashedNextNodeId := hash.Hash(nextNodeId)
-		subnetId := binary.LittleEndian.Uint64(hashedNextNodeId[:8]) % dataColumnSidecarSubnetCount
+	// Handle the maximum value of a uint256 case.
+	if nodeIdUInt256.Cmp(maxUint256) == 0 {
+		nodeIdUInt256 = uint256.NewInt(0)
+	}
 
-		if _, exists := subnetIds[subnetId]; !exists {
-			subnetIds[subnetId] = true
-		}
+	one := uint256.NewInt(1)
+
+	for i := uint256.NewInt(0); uint64(len(subnetIds)) < custodySubnetCount; i.Add(i, one) {
+		// Augment the node ID with the index.
+		augmentedNodeIdUInt256 := new(uint256.Int).Add(nodeIdUInt256, i)
+
+		// Convert to big endian bytes.
+		augmentedNodeIdBytesBigEndian := augmentedNodeIdUInt256.Bytes()
+
+		// Convert to little endian.
+		augmentedNodeIdBytesLittleEndian := bytesutil.ReverseByteOrder(augmentedNodeIdBytesBigEndian)
+
+		// Hash the result.
+		hashedAugmentedNodeId := hash.Hash(augmentedNodeIdBytesLittleEndian)
+
+		// Get the subnet ID.
+		subnetId := binary.LittleEndian.Uint64(hashedAugmentedNodeId[:8]) % dataColumnSidecarSubnetCount
+
+		// Add the subnet to the map.
+		subnetIds[subnetId] = true
 	}
 
 	return subnetIds, nil
 }
 
 // ComputeExtendedMatrix computes the extended matrix from the blobs.
 // https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#compute_extended_matrix
-func ComputeExtendedMatrix(blobs []cKzg4844.Blob) (extendedMatrix, error) {
-	matrix := make(extendedMatrix, 0, extendedMatrixSize)
+func ComputeExtendedMatrix(blobs []cKzg4844.Blob) (ExtendedMatrix, error) {
+	matrix := make(ExtendedMatrix, 0, extendedMatrixSize)
 
 	for i := range blobs {
 		// Chunk a non-extended blob into cells representing the corresponding extended blob.
@@ -114,8 +134,8 @@ func ComputeExtendedMatrix(blobs []cKzg4844.Blob) (extendedMatrix, error) {
 
 // RecoverMatrix recovers the extended matrix from some cells.
 // https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#recover_matrix
-func RecoverMatrix(cellFromCoordinate map[cellCoordinate]cKzg4844.Cell, blobCount uint64) (extendedMatrix, error) {
-	matrix := make(extendedMatrix, 0, extendedMatrixSize)
+func RecoverMatrix(cellFromCoordinate map[cellCoordinate]cKzg4844.Cell, blobCount uint64) (ExtendedMatrix, error) {
+	matrix := make(ExtendedMatrix, 0, extendedMatrixSize)
 
 	for blobIndex := uint64(0); blobIndex < blobCount; blobIndex++ {
 		// Filter all cells that belong to the current blob.
@@ -152,6 +172,7 @@ func RecoverMatrix(cellFromCoordinate map[cellCoordinate]cKzg4844.Cell, blobCoun
 	return matrix, nil
 }
 
+// DataColumnSidecars computes the data column sidecars from the signed block and blobs.
 // https://github.com/ethereum/consensus-specs/blob/dev/specs/_features/eip7594/das-core.md#recover_matrix
 func DataColumnSidecars(signedBlock interfaces.SignedBeaconBlock, blobs []cKzg4844.Blob) ([]*ethpb.DataColumnSidecar, error) {
 	blobsCount := len(blobs)

config/params/config.go

@@ -257,6 +257,7 @@ type BeaconChainConfig struct {
 	SamplesPerSlot                        uint64           `yaml:"SAMPLES_PER_SLOT"`                             // SamplesPerSlot refers to the humber of random samples a node queries per slot.
 	CustodyRequirement                    uint64           `yaml:"CUSTODY_REQUIREMENT"`                          // CustodyRequirement refers to the minimum amount of subnets a peer must custody and serve samples from.
 	MinEpochsForDataColumnSidecarsRequest primitives.Epoch `yaml:"MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS"` // MinEpochsForDataColumnSidecarsRequest is the minimum number of epochs the node will keep the data columns for.
+	MaxCellsInExtendMatrix                uint64           `yaml:"MAX_CELLS_IN_EXTENDED_MATRIX"`                 // MaxCellsInExtendMatrix is the maximum number of cells in the extended data matrix.
 
 	// Networking Specific Parameters
 	GossipMaxSize                   uint64          `yaml:"GOSSIP_MAX_SIZE" spec:"true"`                    // GossipMaxSize is the maximum allowed size of uncompressed gossip messages.

config/params/mainnet_config.go

@@ -297,6 +297,7 @@ var mainnetBeaconConfig = &BeaconChainConfig{
 	SamplesPerSlot:                        8,
 	CustodyRequirement:                    1,
 	MinEpochsForDataColumnSidecarsRequest: 4096,
+	MaxCellsInExtendMatrix:                768,
 
 	// Values related to networking parameters.
 	GossipMaxSize:                   10 * 1 << 20, // 10 MiB

testing/spectest/mainnet/eip7594/networking/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@prysm//tools/go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["custody_columns_test.go"],
+    data = glob(["*.yaml"]) + [
+        "@consensus_spec_tests_mainnet//:test_data",
+    ],
+    tags = ["spectest"],
+    deps = ["//testing/spectest/shared/eip7594/networking:go_default_library"],
+)

testing/spectest/mainnet/eip7594/networking/custody_columns_test.go

@@ -0,0 +1,11 @@
+package networking
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v5/testing/spectest/shared/eip7594/networking"
+)
+
+func TestMainnet_EIP7594_Networking_CustodyColumns(t *testing.T) {
+	networking.RunCustodyColumnsTest(t, "mainnet")
+}

testing/spectest/minimal/eip7594/networking/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@prysm//tools/go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["custody_columns_test.go"],
+    data = glob(["*.yaml"]) + [
+        "@consensus_spec_tests_minimal//:test_data",
+    ],
+    tags = ["spectest"],
+    deps = ["//testing/spectest/shared/eip7594/networking:go_default_library"],
+)

testing/spectest/minimal/eip7594/networking/custody_columns_test.go

@@ -0,0 +1,11 @@
+package networking
+
+import (
+	"testing"
+
+	"github.com/prysmaticlabs/prysm/v5/testing/spectest/shared/eip7594/networking"
+)
+
+func TestMainnet_EIP7594_Networking_CustodyColumns(t *testing.T) {
+	networking.RunCustodyColumnsTest(t, "minimal")
+}

testing/spectest/shared/eip7594/networking/BUILD.bazel

@@ -0,0 +1,17 @@
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    testonly = True,
+    srcs = ["custody_columns.go"],
+    importpath = "github.com/prysmaticlabs/prysm/v5/testing/spectest/shared/eip7594/networking",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//beacon-chain/core/peerdas:go_default_library",
+        "//testing/require:go_default_library",
+        "//testing/spectest/utils:go_default_library",
+        "//testing/util:go_default_library",
+        "@com_github_ethereum_go_ethereum//p2p/enode:go_default_library",
+        "@in_gopkg_yaml_v3//:go_default_library",
+    ],
+)

testing/spectest/shared/eip7594/networking/custody_columns.go

@@ -0,0 +1,64 @@
+package networking
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/peerdas"
+	"github.com/prysmaticlabs/prysm/v5/testing/require"
+	"github.com/prysmaticlabs/prysm/v5/testing/spectest/utils"
+	"github.com/prysmaticlabs/prysm/v5/testing/util"
+	"gopkg.in/yaml.v3"
+)
+
+type Config struct {
+	NodeId             *big.Int `yaml:"node_id"`
+	CustodySubnetCount uint64   `yaml:"custody_subnet_count"`
+	Expected           []uint64 `yaml:"result"`
+}
+
+// RunCustodyColumnsTest executes custody columns spec tests.
+func RunCustodyColumnsTest(t *testing.T, config string) {
+	err := utils.SetConfig(t, config)
+	require.NoError(t, err, "failed to set config")
+
+	// Retrieve the test vector folders.
+	testFolders, testsFolderPath := utils.TestFolders(t, config, "eip7594", "networking/get_custody_columns/pyspec_tests")
+	if len(testFolders) == 0 {
+		t.Fatalf("no test folders found for %s", testsFolderPath)
+	}
+
+	for _, folder := range testFolders {
+		t.Run(folder.Name(), func(t *testing.T) {
+			var (
+				config      Config
+				nodeIdBytes [32]byte
+			)
+
+			// Load the test vector.
+			file, err := util.BazelFileBytes(testsFolderPath, folder.Name(), "meta.yaml")
+			require.NoError(t, err, "failed to retrieve the `meta.yaml` YAML file")
+
+			// Unmarshal the test vector.
+			err = yaml.Unmarshal(file, &config)
+			require.NoError(t, err, "failed to unmarshal the YAML file")
+
+			// Get the node ID.
+			copy(nodeIdBytes[:], config.NodeId.Bytes())
+			nodeId := enode.ID(nodeIdBytes)
+
+			// Compute the custodied columns.
+			actual, err := peerdas.CustodyColumns(nodeId, config.CustodySubnetCount)
+			require.NoError(t, err, "failed to compute the custody columns")
+
+			// Compare the results.
+			require.Equal(t, len(config.Expected), len(actual), "expected %d custody columns, got %d", len(config.Expected), len(actual))
+
+			for _, result := range config.Expected {
+				ok := actual[result]
+				require.Equal(t, true, ok, "expected column %d to be in custody columns", result)
+			}
+		})
+	}
+}