Enable deneb ref tests + merkle proof ref test support

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/Eth2ReferenceTestCase.java

@@ -23,6 +23,7 @@
 import tech.pegasys.teku.reference.altair.rewards.RewardsTestExecutorBellatrix;
 import tech.pegasys.teku.reference.common.epoch_processing.EpochProcessingTestExecutor;
 import tech.pegasys.teku.reference.common.operations.OperationsTestExecutor;
+import tech.pegasys.teku.reference.deneb.merkle_proof.MerkleProofTests;
 import tech.pegasys.teku.reference.phase0.bls.BlsTests;
 import tech.pegasys.teku.reference.phase0.forkchoice.ForkChoiceTestExecutor;
 import tech.pegasys.teku.reference.phase0.genesis.GenesisTests;
@@ -83,17 +84,14 @@ public abstract class Eth2ReferenceTestCase {
           .putAll(TransitionTestExecutor.TRANSITION_TEST_TYPES)
           .putAll(ForkUpgradeTestExecutor.FORK_UPGRADE_TEST_TYPES)
           .putAll(RewardsTestExecutorBellatrix.REWARDS_TEST_TYPES)
+          .putAll(MerkleProofTests.MERKLE_PROOF_TEST_TYPES)
           .build();
 
   protected void runReferenceTest(final TestDefinition testDefinition) throws Throwable {
     getExecutorFor(testDefinition).runTest(testDefinition);
   }
 
   private TestExecutor getExecutorFor(final TestDefinition testDefinition) {
-    // TODO: re-enable Deneb tests once migration to inclusion proof #7654 is complete
-    if (testDefinition.getFork().equals(TestFork.DENEB)) {
-      return TestExecutor.IGNORE_TESTS;
-    }
     // Look for fork-specific tests first
     TestExecutor testExecutor =
         switch (testDefinition.getFork()) {

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/deneb/merkle_proof/MerkleProofTests.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Consensys Software Inc., 2023
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.reference.deneb.merkle_proof;
+
+import com.google.common.collect.ImmutableMap;
+import tech.pegasys.teku.reference.TestExecutor;
+
+public class MerkleProofTests {
+  public static final ImmutableMap<String, TestExecutor> MERKLE_PROOF_TEST_TYPES =
+      ImmutableMap.<String, TestExecutor>builder()
+          .put("merkle_proof/single_merkle_proof", new SingleMerkleProofTestExecutor())
+          .build();
+}

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/deneb/merkle_proof/SingleMerkleProofTestExecutor.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright Consensys Software Inc., 2023
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.reference.deneb.merkle_proof;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static tech.pegasys.teku.ethtests.finder.MerkleProofTestFinder.PROOF_DATA_FILE;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.io.IOException;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.tuweni.bytes.Bytes32;
+import tech.pegasys.teku.ethtests.finder.TestDefinition;
+import tech.pegasys.teku.infrastructure.ssz.collections.SszBytes32Vector;
+import tech.pegasys.teku.infrastructure.ssz.primitive.SszBytes32;
+import tech.pegasys.teku.infrastructure.ssz.schema.collections.SszBytes32VectorSchema;
+import tech.pegasys.teku.reference.TestDataUtils;
+import tech.pegasys.teku.reference.TestExecutor;
+import tech.pegasys.teku.spec.config.SpecConfigDeneb;
+import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody;
+import tech.pegasys.teku.spec.logic.common.helpers.Predicates;
+
+public class SingleMerkleProofTestExecutor implements TestExecutor {
+  private static final Pattern TEST_NAME_PATTERN = Pattern.compile("(.+)/(.+)");
+  private static final String OBJECT_SSZ_FILE = "object.ssz_snappy";
+
+  @Override
+  public void runTest(final TestDefinition testDefinition) throws Throwable {
+
+    final Matcher matcher = TEST_NAME_PATTERN.matcher(testDefinition.getTestName());
+    if (!matcher.find()) {
+      throw new RuntimeException(
+          "Can't extract object and proof type from " + testDefinition.getTestName());
+    }
+
+    final String objectType = matcher.group(1);
+    final String proofType = matcher.group(2);
+
+    final Data data = loadDataFile(testDefinition, Data.class);
+
+    switch (objectType) {
+      case "BeaconBlockBody":
+        runBeaconBlockBodyTest(testDefinition, proofType, data);
+        break;
+      default:
+        throw new RuntimeException("Unknown object type " + objectType);
+    }
+  }
+
+  protected <T> T loadDataFile(final TestDefinition testDefinition, final Class<T> type)
+      throws IOException {
+    String dataFile =
+        testDefinition.getTestName().endsWith(".yaml")
+            ? testDefinition.getTestName()
+            : PROOF_DATA_FILE;
+    return TestDataUtils.loadYaml(testDefinition, dataFile, type);
+  }
+
+  private static class Data {
+    @JsonProperty(value = "leaf", required = true)
+    private String leaf;
+
+    @JsonProperty(value = "leaf_index", required = true)
+    private int leafIndex;
+
+    @JsonProperty(value = "branch", required = true)
+    private List<String> branch;
+  }
+
+  void runBeaconBlockBodyTest(
+      final TestDefinition testDefinition, final String proofType, final Data data) {
+    final Predicates predicates = new Predicates(testDefinition.getSpec().getGenesisSpecConfig());
+    final BeaconBlockBody beaconBlockBody =
+        TestDataUtils.loadSsz(
+            testDefinition,
+            OBJECT_SSZ_FILE,
+            testDefinition.getSpec().getGenesisSchemaDefinitions().getBeaconBlockBodySchema());
+
+    switch (proofType) {
+      case "blob_kzg_commitment_merkle_proof":
+        assertThat(
+                predicates.isValidMerkleBranch(
+                    Bytes32.fromHexString(data.leaf),
+                    createKzgCommitmentMerkleProofBranchFromData(testDefinition, data.branch),
+                    getKzgCommitmentInclusionProofDepth(testDefinition),
+                    data.leafIndex,
+                    beaconBlockBody.hashTreeRoot()))
+            .isTrue();
+        break;
+      default:
+        throw new RuntimeException("Unknown proof type " + proofType);
+    }
+  }
+
+  private SszBytes32Vector createKzgCommitmentMerkleProofBranchFromData(
+      final TestDefinition testDefinition, final List<String> branch) {
+    final SszBytes32VectorSchema<?> kzgCommitmentInclusionProofSchema =
+        testDefinition
+            .getSpec()
+            .getGenesisSchemaDefinitions()
+            .toVersionDeneb()
+            .orElseThrow()
+            .getBlobSidecarSchema()
+            .getKzgCommitmentInclusionProofSchema();
+    return kzgCommitmentInclusionProofSchema.createFromElements(
+        branch.stream().map(Bytes32::fromHexString).map(SszBytes32::of).toList());
+  }
+
+  private int getKzgCommitmentInclusionProofDepth(final TestDefinition testDefinition) {
+    return SpecConfigDeneb.required(testDefinition.getSpec().getGenesisSpecConfig())
+        .getKzgCommitmentInclusionProofDepth();
+  }
+}

eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/MerkleProofTestFinder.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright Consensys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.ethtests.finder;
+
+import static tech.pegasys.teku.ethtests.finder.ReferenceTestFinder.unchecked;
+
+import com.google.errorprone.annotations.MustBeClosed;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+@SuppressWarnings("MustBeClosedChecker")
+public class MerkleProofTestFinder implements TestFinder {
+
+  public static final String PROOF_DATA_FILE = "proof.yaml";
+
+  @Override
+  @MustBeClosed
+  public Stream<TestDefinition> findTests(
+      final String fork, final String config, final Path testRoot) throws IOException {
+    final Path merkleProofTestDir = testRoot.resolve("merkle_proof");
+    if (!merkleProofTestDir.toFile().exists()) {
+      return Stream.empty();
+    }
+    return Files.list(merkleProofTestDir)
+        .flatMap(unchecked(dir -> findMerkleProofTests(fork, config, testRoot, dir)));
+  }
+
+  @MustBeClosed
+  private Stream<TestDefinition> findMerkleProofTests(
+      final String fork, final String config, final Path testRoot, final Path testCategoryDir)
+      throws IOException {
+    final String testType = testRoot.relativize(testCategoryDir).toString();
+    return Files.walk(testCategoryDir)
+        .filter(path -> path.resolve(PROOF_DATA_FILE).toFile().exists())
+        .map(
+            testDir -> {
+              final String testName = testCategoryDir.relativize(testDir).toString();
+              return new TestDefinition(
+                  fork, config, testType, testName, testRoot.relativize(testDir));
+            });
+  }
+}

eth-tests/src/main/java/tech/pegasys/teku/ethtests/finder/ReferenceTestFinder.java

@@ -58,7 +58,8 @@ private static Stream<TestDefinition> findTestTypes(final Path specDirectory) th
                       new SszTestFinder("ssz_generic"),
                       new SszTestFinder("ssz_static"),
                       new ShufflingTestFinder(),
-                      new PyspecTestFinder())
+                      new PyspecTestFinder(),
+                      new MerkleProofTestFinder())
                   .flatMap(unchecked(finder -> finder.findTests(fork, spec, testsPath)));
             });
   }