Merge branch 'master' into check-network-on-start

CHANGELOG.md

@@ -15,9 +15,11 @@
 - Implemented [PostAggregateAndProofsV2](https://ethereum.github.io/beacon-APIs/?urls.primaryName=dev#/Validator/publishAggregateAndProofsV2) (adding support for Electra)
 - Added support for [Ephemery Testnet](https://github.com/ephemery.dev) `--network=ephemery`
 - Updated bootnodes for Holesky network
-- Added new `--p2p-flood-publish-enabled` parameter to control whenever flood publishing behaviour is enabled (applies to all subnets). Previous teku versions always had this behaviour enabled. Default is `true`.
-- Added a fix for [CVE-2024-7254](https://avd.aquasec.com/nvd/2024/cve-2024-7254/)
+- Disabled flood publish behaviour on all p2p subnets. `--Xp2p-flood-publish-enabled` experimental parameter can be used to re-enable it, restoring previous behaviour.
+- Add a fix for [CVE-2024-7254](https://avd.aquasec.com/nvd/2024/cve-2024-7254/)
 - Updated LUKSO configuration with Deneb fork scheduled for epoch 123075 (November 20, 2024, 16:20:00 UTC)
+- Support for `IDONTWANT` libp2p protocol messages
+- `/eth/v1/node/peers` endpoint now populates `enr` field of the peer whenever is possible
 
 ### Bug Fixes
  - Removed a warning from logs about non blinded blocks being requested (#8562)

data/beaconrestapi/src/integration-test/resources/tech/pegasys/teku/beaconrestapi/beacon/schema/Peer.json

@@ -10,7 +10,7 @@
     },
     "enr" : {
       "type" : "string",
-      "description" : "Ethereum node record. Not currently populated. [Read more](https://eips.ethereum.org/EIPS/eip-778)",
+      "description" : "Ethereum node record. [Read more](https://eips.ethereum.org/EIPS/eip-778)",
       "example" : "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"
     },
     "last_seen_p2p_address" : {

data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeerById.java

@@ -26,17 +26,17 @@
 import java.util.function.Function;
 import tech.pegasys.teku.api.DataProvider;
 import tech.pegasys.teku.api.NetworkDataProvider;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.EndpointMetadata;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiEndpoint;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiRequest;
-import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
 
 public class GetPeerById extends RestApiEndpoint {
   public static final String ROUTE = "/eth/v1/node/peers/{peer_id}";
 
-  private static final SerializableTypeDefinition<Eth2Peer> PEERS_BY_ID_RESPONSE_TYPE =
-      SerializableTypeDefinition.object(Eth2Peer.class)
+  private static final SerializableTypeDefinition<Eth2PeerWithEnr> PEERS_BY_ID_RESPONSE_TYPE =
+      SerializableTypeDefinition.object(Eth2PeerWithEnr.class)
           .name("GetPeerResponse")
           .withField("data", PEER_DATA_TYPE, Function.identity())
           .build();
@@ -64,7 +64,8 @@ public GetPeerById(final DataProvider provider) {
   @Override
   public void handleRequest(final RestApiRequest request) throws JsonProcessingException {
     request.header(Header.CACHE_CONTROL, CACHE_NONE);
-    Optional<Eth2Peer> peer = network.getEth2PeerById(request.getPathParameter(PEER_ID_PARAMETER));
+    final Optional<Eth2PeerWithEnr> peer =
+        network.getEth2PeerById(request.getPathParameter(PEER_ID_PARAMETER));
     if (peer.isEmpty()) {
       request.respondError(SC_NOT_FOUND, "Peer not found");
     } else {

data/beaconrestapi/src/main/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeers.java

@@ -23,18 +23,17 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.function.Function;
 import tech.pegasys.teku.api.DataProvider;
 import tech.pegasys.teku.api.NetworkDataProvider;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.api.response.v1.node.Direction;
 import tech.pegasys.teku.api.response.v1.node.State;
 import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition;
 import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.EndpointMetadata;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiEndpoint;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.RestApiRequest;
-import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
 
 public class GetPeers extends RestApiEndpoint {
   public static final String ROUTE = "/eth/v1/node/peers";
@@ -45,41 +44,42 @@ public class GetPeers extends RestApiEndpoint {
   private static final DeserializableTypeDefinition<Direction> DIRECTION_TYPE =
       DeserializableTypeDefinition.enumOf(Direction.class);
 
-  static final SerializableTypeDefinition<Eth2Peer> PEER_DATA_TYPE =
-      SerializableTypeDefinition.object(Eth2Peer.class)
+  static final SerializableTypeDefinition<Eth2PeerWithEnr> PEER_DATA_TYPE =
+      SerializableTypeDefinition.object(Eth2PeerWithEnr.class)
           .name("Peer")
           .withField(
               "peer_id",
               string(
                   "Cryptographic hash of a peer’s public key. "
                       + "'[Read more](https://docs.libp2p.io/concepts/peer-id/)",
                   "QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N"),
-              eth2Peer -> eth2Peer.getId().toBase58())
+              eth2Peer -> eth2Peer.peer().getId().toBase58())
           .withOptionalField(
               "enr",
               string(
-                  "Ethereum node record. Not currently populated. "
-                      + "[Read more](https://eips.ethereum.org/EIPS/eip-778)",
+                  "Ethereum node record. " + "[Read more](https://eips.ethereum.org/EIPS/eip-778)",
                   "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrk"
                       + "Tfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYp"
                       + "Ma2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"),
-              eth2Peer -> Optional.empty())
+              Eth2PeerWithEnr::enr)
           .withField(
               "last_seen_p2p_address",
               string(
                   "Multiaddr used in last peer connection. "
                       + "[Read more](https://docs.libp2p.io/reference/glossary/#multiaddr)",
                   "/ip4/7.7.7.7/tcp/4242/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N"),
-              eth2Peer -> eth2Peer.getAddress().toExternalForm())
+              eth2Peer -> eth2Peer.peer().getAddress().toExternalForm())
           .withField(
               "state",
               STATE_TYPE,
-              eth2Peer -> eth2Peer.isConnected() ? State.connected : State.disconnected)
+              eth2Peer -> eth2Peer.peer().isConnected() ? State.connected : State.disconnected)
           .withField(
               "direction",
               DIRECTION_TYPE,
               eth2Peer ->
-                  eth2Peer.connectionInitiatedLocally() ? Direction.outbound : Direction.inbound)
+                  eth2Peer.peer().connectionInitiatedLocally()
+                      ? Direction.outbound
+                      : Direction.inbound)
           .build();
 
   private static final SerializableTypeDefinition<Integer> PEERS_META_TYPE =
@@ -119,19 +119,19 @@ public GetPeers(final DataProvider provider) {
 
   @Override
   public void handleRequest(final RestApiRequest request) throws JsonProcessingException {
-    request.respondOk(new PeersData(network.getEth2Peers()), NO_CACHE);
+    request.respondOk(new PeersData(network.getEth2PeersWithEnr()), NO_CACHE);
   }
 
   static class PeersData {
-    private final List<Eth2Peer> peers;
+    private final List<Eth2PeerWithEnr> peers;
     private final Integer count;
 
-    PeersData(final List<Eth2Peer> peers) {
+    PeersData(final List<Eth2PeerWithEnr> peers) {
       this.peers = peers;
       this.count = peers.size();
     }
 
-    public List<Eth2Peer> getPeers() {
+    public List<Eth2PeerWithEnr> getPeers() {
       return peers;
     }
 

data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeerByIdTest.java

@@ -14,6 +14,7 @@
 package tech.pegasys.teku.beaconrestapi.handlers.v1.node;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -26,17 +27,25 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.util.Optional;
+import org.apache.tuweni.units.bigints.UInt256;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.beaconrestapi.AbstractMigratedBeaconHandlerTest;
 import tech.pegasys.teku.infrastructure.http.HttpErrorResponse;
 import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
+import tech.pegasys.teku.networking.p2p.discovery.DiscoveryNetwork;
+import tech.pegasys.teku.networking.p2p.discovery.DiscoveryService;
 import tech.pegasys.teku.networking.p2p.mock.MockNodeId;
 import tech.pegasys.teku.networking.p2p.network.PeerAddress;
+import tech.pegasys.teku.networking.p2p.peer.NodeId;
 
 public class GetPeerByIdTest extends AbstractMigratedBeaconHandlerTest {
   private final MockNodeId peerId = new MockNodeId(123456);
+  private final NodeId peerNodeId = mock(NodeId.class);
   private final Eth2Peer peer = mock(Eth2Peer.class);
+  private static final String ENR_STUB = "enr:test";
+  private final Eth2PeerWithEnr peerWithEnr = new Eth2PeerWithEnr(peer, Optional.of(ENR_STUB));
 
   @BeforeEach
   void setUp() {
@@ -50,21 +59,29 @@ void setUp() {
 
   @Test
   public void shouldReturnNotFoundIfPeerNotFound() throws Exception {
-    when(network.getEth2PeerById(peerId.toBase58())).thenReturn(Optional.empty());
-
+    when(eth2P2PNetwork.parseNodeId(peerId.toBase58())).thenReturn(peerNodeId);
+    when(eth2P2PNetwork.getPeer(eq(peerNodeId))).thenReturn(Optional.empty());
     handler.handleRequest(request);
+
     assertThat(request.getResponseCode()).isEqualTo(SC_NOT_FOUND);
     assertThat(request.getResponseBody())
         .isEqualTo(new HttpErrorResponse(SC_NOT_FOUND, "Peer not found"));
   }
 
   @Test
   public void shouldReturnPeerIfFound() throws Exception {
-    when(network.getEth2PeerById(eq(peerId.toBase58()))).thenReturn(Optional.of(peer));
+    when(eth2P2PNetwork.parseNodeId(peerId.toBase58())).thenReturn(peerNodeId);
+    when(eth2P2PNetwork.getPeer(eq(peerNodeId))).thenReturn(Optional.of(peer));
+    when(peer.getDiscoveryNodeId()).thenReturn(Optional.of(UInt256.ONE));
+    final DiscoveryNetwork<?> discoveryNetwork = mock(DiscoveryNetwork.class);
+    when(eth2P2PNetwork.getDiscoveryNetwork()).thenReturn(Optional.of(discoveryNetwork));
+    final DiscoveryService discoveryService = mock(DiscoveryService.class);
+    when(discoveryNetwork.getDiscoveryService()).thenReturn(discoveryService);
+    when(discoveryService.lookupEnr(any())).thenReturn(Optional.of(ENR_STUB));
     handler.handleRequest(request);
 
     assertThat(request.getResponseCode()).isEqualTo(SC_OK);
-    assertThat(request.getResponseBody()).isEqualTo(peer);
+    assertThat(request.getResponseBody()).isEqualTo(peerWithEnr);
   }
 
   @Test
@@ -79,10 +96,11 @@ void metadata_shouldHandle500() throws JsonProcessingException {
 
   @Test
   void metadata_shouldHandle200() throws JsonProcessingException {
-    final String data = getResponseStringFromMetadata(handler, SC_OK, peer);
+    final String data = getResponseStringFromMetadata(handler, SC_OK, peerWithEnr);
     assertThat(data)
         .isEqualTo(
             "{\"data\":{\"peer_id\":\"1111111111111111111111111111177em\","
+                + "\"enr\":\"enr:test\","
                 + "\"last_seen_p2p_address\":\"1111111111111111111111111111177em\",\"state\":\"connected\",\"direction\":\"inbound\"}}");
   }
 }

data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/handlers/v1/node/GetPeersTest.java

@@ -24,9 +24,11 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.util.List;
+import java.util.Optional;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.api.NetworkDataProvider;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.beaconrestapi.AbstractMigratedBeaconHandlerTest;
 import tech.pegasys.teku.beaconrestapi.handlers.v1.node.GetPeers.PeersData;
 import tech.pegasys.teku.infrastructure.restapi.endpoints.CacheLength;
@@ -42,7 +44,10 @@ public class GetPeersTest extends AbstractMigratedBeaconHandlerTest {
   private final Eth2Peer peer2 = mock(Eth2Peer.class);
 
   private final NetworkDataProvider networkDataProvider = mock(NetworkDataProvider.class);
-  private final List<Eth2Peer> data = List.of(peer1, peer2);
+  private final List<Eth2PeerWithEnr> data =
+      List.of(
+          new Eth2PeerWithEnr(peer1, Optional.empty()),
+          new Eth2PeerWithEnr(peer2, Optional.empty()));
   private final GetPeers.PeersData peersData = new PeersData(data);
 
   @BeforeEach
@@ -62,7 +67,7 @@ void setup() {
   @Test
   public void shouldReturnListOfPeers() throws Exception {
 
-    when(networkDataProvider.getEth2Peers()).thenReturn(data);
+    when(networkDataProvider.getEth2PeersWithEnr()).thenReturn(data);
 
     handler.handleRequest(request);
     assertThat(request.getResponseCode()).isEqualTo(SC_OK);

data/provider/src/main/java/tech/pegasys/teku/api/NetworkDataProvider.java

@@ -15,6 +15,9 @@
 
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Function;
+import org.apache.tuweni.units.bigints.UInt256;
+import tech.pegasys.teku.api.peer.Eth2PeerWithEnr;
 import tech.pegasys.teku.api.response.v1.node.Direction;
 import tech.pegasys.teku.api.response.v1.node.Peer;
 import tech.pegasys.teku.api.response.v1.node.State;
@@ -23,14 +26,28 @@
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.networking.eth2.Eth2P2PNetwork;
 import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
+import tech.pegasys.teku.networking.p2p.discovery.DiscoveryNetwork;
+import tech.pegasys.teku.networking.p2p.discovery.DiscoveryService;
 import tech.pegasys.teku.networking.p2p.peer.NodeId;
 import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.metadata.MetadataMessage;
 
 public class NetworkDataProvider {
+  final Function<Optional<UInt256>, Optional<String>> enrLookupFunction;
+
   private final Eth2P2PNetwork network;
 
   public NetworkDataProvider(final Eth2P2PNetwork network) {
     this.network = network;
+    this.enrLookupFunction =
+        maybeNodeId -> {
+          if (maybeNodeId.isEmpty()) {
+            return Optional.empty();
+          }
+          final Optional<DiscoveryService> maybeDiscoveryService =
+              network.getDiscoveryNetwork().map(DiscoveryNetwork::getDiscoveryService);
+          return maybeDiscoveryService.flatMap(
+              discoveryService -> discoveryService.lookupEnr(maybeNodeId.get()));
+        };
   }
 
   /**
@@ -80,6 +97,16 @@ public List<Eth2Peer> getEth2Peers() {
     return network.streamPeers().toList();
   }
 
+  public List<Eth2PeerWithEnr> getEth2PeersWithEnr() {
+    return network
+        .streamPeers()
+        .map(
+            eth2Peer ->
+                new Eth2PeerWithEnr(
+                    eth2Peer, enrLookupFunction.apply(eth2Peer.getDiscoveryNodeId())))
+        .toList();
+  }
+
   public PeerCount getPeerCount() {
     long disconnected = 0;
     long connected = 0;
@@ -102,9 +129,14 @@ public List<Eth2Peer> getPeerScores() {
     return network.streamPeers().toList();
   }
 
-  public Optional<Eth2Peer> getEth2PeerById(final String peerId) {
+  public Optional<Eth2PeerWithEnr> getEth2PeerById(final String peerId) {
     final NodeId nodeId = network.parseNodeId(peerId);
-    return network.getPeer(nodeId);
+    return network
+        .getPeer(nodeId)
+        .map(
+            eth2Peer ->
+                new Eth2PeerWithEnr(
+                    eth2Peer, enrLookupFunction.apply(eth2Peer.getDiscoveryNodeId())));
   }
 
   private Peer toPeer(final Eth2Peer eth2Peer) {

data/provider/src/main/java/tech/pegasys/teku/api/peer/Eth2PeerWithEnr.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Consensys Software Inc., 2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.api.peer;
+
+import java.util.Optional;
+import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
+
+public record Eth2PeerWithEnr(Eth2Peer peer, Optional<String> enr) {}

gradle/versions.gradle

@@ -159,7 +159,7 @@ dependencyManagement {
 
     // discovery includes tuweni libraries under a different name so version resolution doesn't work
     // exclude them here and leave them to be included on the classpath by the version we use
-    dependency('tech.pegasys.discovery:discovery:24.6.0') {
+    dependency('tech.pegasys.discovery:discovery:24.9.1') {
       exclude 'org.apache.tuweni:bytes'
       exclude 'org.apache.tuweni:crypto'
       exclude 'org.apache.tuweni:units'

networking/eth2/src/main/java/tech/pegasys/teku/networking/eth2/Eth2P2PNetworkBuilder.java

@@ -46,8 +46,10 @@
 import tech.pegasys.teku.networking.eth2.gossip.topics.Eth2GossipTopicFilter;
 import tech.pegasys.teku.networking.eth2.gossip.topics.OperationProcessor;
 import tech.pegasys.teku.networking.eth2.gossip.topics.ProcessedAttestationSubscriptionProvider;
+import tech.pegasys.teku.networking.eth2.peers.DiscoveryNodeIdExtractor;
 import tech.pegasys.teku.networking.eth2.peers.Eth2PeerManager;
 import tech.pegasys.teku.networking.eth2.peers.Eth2PeerSelectionStrategy;
+import tech.pegasys.teku.networking.eth2.peers.LibP2PDiscoveryNodeIdExtractor;
 import tech.pegasys.teku.networking.eth2.rpc.beaconchain.methods.StatusMessageFactory;
 import tech.pegasys.teku.networking.eth2.rpc.core.encodings.RpcEncoding;
 import tech.pegasys.teku.networking.p2p.connection.PeerPools;
@@ -139,6 +141,8 @@ public Eth2P2PNetwork build() {
     // Setup eth2 handlers
     final SubnetSubscriptionService attestationSubnetService = new SubnetSubscriptionService();
     final SubnetSubscriptionService syncCommitteeSubnetService = new SubnetSubscriptionService();
+
+    final DiscoveryNodeIdExtractor discoveryNodeIdExtractor = new LibP2PDiscoveryNodeIdExtractor();
     final RpcEncoding rpcEncoding =
         RpcEncoding.createSszSnappyEncoding(spec.getNetworkingConfig().getMaxChunkSize());
     if (statusMessageFactory == null) {
@@ -161,7 +165,8 @@ public Eth2P2PNetwork build() {
             config.getPeerRateLimit(),
             config.getPeerRequestLimit(),
             spec,
-            kzg);
+            kzg,
+            discoveryNodeIdExtractor);
     final Collection<RpcMethod<?, ?, ?>> eth2RpcMethods =
         eth2PeerManager.getBeaconChainMethods().all();
     rpcMethods.addAll(eth2RpcMethods);